├── frontend ├── src │ ├── App.css │ ├── vite-env.d.ts │ ├── constants │ │ ├── routes.ts │ │ ├── dashboard.tsx │ │ └── agentInfo.tsx │ ├── lib │ │ ├── utils.ts │ │ └── types.ts │ ├── assets │ │ └── TheagenticBench │ │ │ ├── theagentic-bench-favicon.svg │ │ │ └── theagentic-bench-logo.svg │ ├── main.tsx │ ├── components │ │ ├── navbar │ │ │ ├── SidebarItem.tsx │ │ │ ├── Header.tsx │ │ │ ├── Sidebar.tsx │ │ │ └── ChatHistoryItem.tsx │ │ ├── customUI │ │ │ ├── ErrorAlert.tsx │ │ │ ├── ModelSelect.tsx │ │ │ ├── Accordion.tsx │ │ │ ├── TerminalBlock.tsx │ │ │ ├── SidebarMenu.tsx │ │ │ ├── Loading.tsx │ │ │ ├── DeleteAlert.tsx │ │ │ ├── CodeBlock.tsx │ │ │ └── PromptInput.tsx │ │ ├── playground │ │ │ ├── AgentInfo.tsx │ │ │ └── ModelSelect.tsx │ │ └── ui │ │ │ ├── alert.tsx │ │ │ ├── scroll-area.tsx │ │ │ ├── button.tsx │ │ │ ├── card.tsx │ │ │ ├── accordion.tsx │ │ │ ├── alert-dialog.tsx │ │ │ ├── select.tsx │ │ │ └── dropdown-menu.tsx │ ├── pages │ │ ├── Layout.tsx │ │ └── Playground.tsx │ ├── App.tsx │ └── index.css ├── .dockerignore ├── postcss.config.js ├── Dockerfile ├── tsconfig.json ├── .gitignore ├── components.json ├── tsconfig.node.json ├── vite.config.ts ├── eslint.config.js ├── tsconfig.app.json ├── index.html ├── package.json ├── README.md └── tailwind.config.js ├── agentic_bench ├── __init__.py ├── utils │ ├── __init__.py │ ├── message_handler.py │ ├── initializers │ │ ├── __init__.py │ │ ├── text_extractor.py │ │ ├── rag_constants.py │ │ └── graph_initializer.py │ ├── executors │ │ ├── __init__.py │ │ └── executor_utils │ │ │ ├── __init__.py │ │ │ ├── extract_command_line_args.py │ │ │ ├── _base.py │ │ │ ├── _common.py │ │ │ └── _func_with_reqs.py │ ├── stream_response_format.py │ ├── types.py │ ├── calculate_md5_hash_of_file.py │ ├── oai_client.py │ ├── models.py │ ├── markdown_browser │ │ ├── __init__.py │ │ └── abstract_markdown_browser.py │ ├── convert_messages.py │ ├── cancellation_token.py │ ├── get_openai_format_json_messages_from_pydantic_message_response.py │ ├── image.py │ └── prompts.py ├── Dockerfile ├── main.py ├── requirements.txt ├── README.md ├── .gitignore ├── agents │ ├── web_surfer.py │ └── rag_agent.py └── ledger.py ├── assets ├── orchestrator1.png ├── orchestrator2.png ├── orchestrator3.png ├── orchestrator4.png ├── orchestrator5.png ├── orchestrator6.png ├── orchestrator7.png ├── ta_bench_diagram.png └── ta_bench_logo.svg ├── .gitmodules ├── .gitignore ├── docker-compose.yaml ├── .env.sample ├── README.md ├── LICENSE └── CustomAgentInstructions.md /frontend/src/App.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /agentic_bench/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /frontend/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /frontend/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /agentic_bench/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .cancellation_token import CancellationToken 2 | -------------------------------------------------------------------------------- /assets/orchestrator1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheAgenticAI/TheAgenticBench/HEAD/assets/orchestrator1.png -------------------------------------------------------------------------------- /assets/orchestrator2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheAgenticAI/TheAgenticBench/HEAD/assets/orchestrator2.png -------------------------------------------------------------------------------- /assets/orchestrator3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheAgenticAI/TheAgenticBench/HEAD/assets/orchestrator3.png -------------------------------------------------------------------------------- /assets/orchestrator4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheAgenticAI/TheAgenticBench/HEAD/assets/orchestrator4.png -------------------------------------------------------------------------------- /assets/orchestrator5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheAgenticAI/TheAgenticBench/HEAD/assets/orchestrator5.png -------------------------------------------------------------------------------- /assets/orchestrator6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheAgenticAI/TheAgenticBench/HEAD/assets/orchestrator6.png -------------------------------------------------------------------------------- /assets/orchestrator7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheAgenticAI/TheAgenticBench/HEAD/assets/orchestrator7.png -------------------------------------------------------------------------------- /assets/ta_bench_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheAgenticAI/TheAgenticBench/HEAD/assets/ta_bench_diagram.png -------------------------------------------------------------------------------- /frontend/postcss.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "TheAgenticBrowser"] 2 | path = TheAgenticBrowser 3 | url = https://github.com/TheAgenticAI/TheAgenticBrowser.git 4 | -------------------------------------------------------------------------------- /agentic_bench/utils/message_handler.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | class BroadcastMessage(BaseModel): 4 | message: str 5 | request_halt: bool = False 6 | -------------------------------------------------------------------------------- /frontend/src/constants/routes.ts: -------------------------------------------------------------------------------- 1 | export const PAGE_ROUTES = { 2 | home: "/playground", 3 | chat: (id?: string) => 4 | id ? `/playground/chat/${id}` : "/playground/chat/:id", 5 | }; 6 | -------------------------------------------------------------------------------- /frontend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:22-alpine 2 | 3 | WORKDIR /app 4 | 5 | COPY package.json . 6 | 7 | RUN npm install 8 | 9 | COPY . . 10 | 11 | EXPOSE 3000 12 | 13 | CMD [ "npm", "run", "dev" ] -------------------------------------------------------------------------------- /frontend/src/lib/utils.ts: -------------------------------------------------------------------------------- 1 | import { clsx, type ClassValue } from "clsx" 2 | import { twMerge } from "tailwind-merge" 3 | 4 | export function cn(...inputs: ClassValue[]) { 5 | return twMerge(clsx(inputs)) 6 | } 7 | -------------------------------------------------------------------------------- /agentic_bench/utils/initializers/__init__.py: -------------------------------------------------------------------------------- 1 | from .graph_initializer import GraphInitializer 2 | from .text_extractor import extract_text_from_directory_async 3 | 4 | __all__ = ["GraphInitializer","extract_text_from_directory_async"] 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .DS_store 3 | .env 4 | terminal_logs/* 5 | data/ 6 | result/ 7 | save_folder/ 8 | saved_path/ 9 | code_files/ 10 | benchvenv 11 | agentic_bench/graph_data_folder/* 12 | agentic_bench/Pdf_data_folder/* 13 | -------------------------------------------------------------------------------- /agentic_bench/utils/executors/__init__.py: -------------------------------------------------------------------------------- 1 | from .docker_code_executor import DockerCommandLineCodeExecutor 2 | from .local_code_executor import LocalCommandLineCodeExecutor 3 | 4 | __all__ = ["DockerCommandLineCodeExecutor", "LocalCommandLineCodeExecutor"] 5 | -------------------------------------------------------------------------------- /agentic_bench/utils/stream_response_format.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List 3 | 4 | @dataclass 5 | class StreamResponse: 6 | agent_name: str 7 | instructions: str 8 | steps: List[str] 9 | status_code: int 10 | output: str 11 | -------------------------------------------------------------------------------- /frontend/src/assets/TheagenticBench/theagentic-bench-favicon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /frontend/src/main.tsx: -------------------------------------------------------------------------------- 1 | import { StrictMode } from 'react' 2 | import { createRoot } from 'react-dom/client' 3 | import './index.css' 4 | import App from './App.tsx' 5 | 6 | createRoot(document.getElementById('root')!).render( 7 | 8 | 9 | , 10 | ) 11 | -------------------------------------------------------------------------------- /frontend/src/lib/types.ts: -------------------------------------------------------------------------------- 1 | export interface SystemMessage { 2 | agent_name: string; 3 | instructions: string; 4 | steps: string[]; 5 | output: string; 6 | status_code: number; 7 | } 8 | 9 | export interface Message { 10 | role: string; 11 | prompt?: string; 12 | data?: SystemMessage[]; 13 | } 14 | -------------------------------------------------------------------------------- /frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [], 3 | "references": [ 4 | { 5 | "path": "./tsconfig.app.json" 6 | }, 7 | { 8 | "path": "./tsconfig.node.json" 9 | } 10 | ], 11 | "compilerOptions": { 12 | "baseUrl": ".", 13 | "paths": { 14 | "@/*": ["./src/*"] 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /frontend/src/constants/dashboard.tsx: -------------------------------------------------------------------------------- 1 | export const DUMMY_PROMPTS = [ 2 | "Perform a competitor analysis of top EV automobile companies and generate a summarized report.", 3 | "Which is the most downloaded audio related dataset on Hugging face currently?", 4 | "Write a python program to check if a number is armstrong number or not.", 5 | ]; 6 | -------------------------------------------------------------------------------- /agentic_bench/utils/types.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from pydantic import BaseModel 3 | from dataclasses import dataclass 4 | 5 | @dataclass 6 | class FunctionCall: 7 | id: str 8 | # JSON args 9 | arguments: str 10 | # Function to call 11 | name: str 12 | 13 | class FunctionExecutionResult(BaseModel): 14 | content: str 15 | call_id: str 16 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | # Logs 3 | logs 4 | *.log 5 | npm-debug.log* 6 | yarn-debug.log* 7 | yarn-error.log* 8 | pnpm-debug.log* 9 | lerna-debug.log* 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /agentic_bench/utils/calculate_md5_hash_of_file.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | 3 | def calculate_md5(file_path): 4 | # Create an MD5 hash object 5 | md5_hash = hashlib.md5() 6 | 7 | # Open the file in binary mode 8 | with open(file_path, "rb") as f: 9 | # Read and update the hash in chunks to handle large files 10 | for chunk in iter(lambda: f.read(4096), b""): 11 | md5_hash.update(chunk) 12 | 13 | # Return the hexadecimal digest of the hash 14 | return md5_hash.hexdigest() -------------------------------------------------------------------------------- /agentic_bench/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim 2 | 3 | WORKDIR /app 4 | 5 | COPY requirements.txt . 6 | RUN apt-get update && apt-get install -y \ 7 | build-essential \ 8 | cmake \ 9 | g++ \ 10 | && rm -rf /var/lib/apt/lists/* 11 | 12 | RUN export PYTHONPATH=/app 13 | RUN apt-get update -y && apt-get install build-essential -y 14 | 15 | RUN pip install --no-cache-dir -r requirements.txt 16 | 17 | COPY . . 18 | 19 | EXPOSE 8081 20 | 21 | CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8081"] 22 | -------------------------------------------------------------------------------- /frontend/src/components/navbar/SidebarItem.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | interface SidebarItemProps { 4 | children: React.ReactNode; 5 | onClick?: () => void; 6 | } 7 | 8 | export const SidebarItem: React.FC = ({ 9 | children, 10 | onClick, 11 | }) => { 12 | return ( 13 |
17 | {children} 18 |
19 | ); 20 | }; 21 | -------------------------------------------------------------------------------- /agentic_bench/utils/oai_client.py: -------------------------------------------------------------------------------- 1 | from pydantic_ai.models.openai import OpenAIModel 2 | from openai import AsyncOpenAI 3 | import os 4 | from dotenv import load_dotenv 5 | 6 | load_dotenv() 7 | 8 | def get_client(): 9 | api_key = os.getenv("AGENTIC_BENCH_MODEL_API_KEY") 10 | base_url = os.getenv("AGENTIC_BENCH_MODEL_BASE_URL") 11 | 12 | client = AsyncOpenAI(api_key=api_key, 13 | base_url=base_url, 14 | max_retries=3, 15 | timeout=10000) 16 | return client 17 | -------------------------------------------------------------------------------- /frontend/components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "new-york", 4 | "rsc": false, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "tailwind.config.js", 8 | "css": "src/index.css", 9 | "baseColor": "zinc", 10 | "cssVariables": true, 11 | "prefix": "" 12 | }, 13 | "aliases": { 14 | "components": "@/components", 15 | "utils": "@/lib/utils", 16 | "ui": "@/components/ui", 17 | "lib": "@/lib", 18 | "hooks": "@/hooks" 19 | }, 20 | "iconLibrary": "lucide" 21 | } -------------------------------------------------------------------------------- /frontend/src/components/customUI/ErrorAlert.tsx: -------------------------------------------------------------------------------- 1 | import {AlertCircle} from "lucide-react"; 2 | 3 | import {Alert, AlertDescription} from "@/components/ui/alert"; 4 | 5 | export const ErrorAlert = ({errorMessage}: {errorMessage?: string}) => { 6 | return ( 7 | 8 | 12 | 13 | {errorMessage || "Something went wrong."} 14 | 15 | 16 | ); 17 | }; 18 | -------------------------------------------------------------------------------- /agentic_bench/utils/executors/executor_utils/__init__.py: -------------------------------------------------------------------------------- 1 | from ._base import CodeBlock, CodeExecutor, CodeResult 2 | from ._func_with_reqs import ( 3 | Alias, 4 | FunctionWithRequirements, 5 | FunctionWithRequirementsStr, 6 | Import, 7 | ImportFromModule, 8 | with_requirements, 9 | ) 10 | 11 | __all__ = [ 12 | "CodeBlock", 13 | "CodeExecutor", 14 | "CodeResult", 15 | "Alias", 16 | "ImportFromModule", 17 | "Import", 18 | "FunctionWithRequirements", 19 | "FunctionWithRequirementsStr", 20 | "with_requirements", 21 | ] 22 | -------------------------------------------------------------------------------- /agentic_bench/utils/models.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | from typing import Dict, Optional 3 | 4 | class FactModel(BaseModel): 5 | facts: str 6 | 7 | class PlanModel(BaseModel): 8 | plan: str 9 | 10 | class LedgerAnswer(BaseModel): 11 | """Model for individual ledger answers""" 12 | 13 | answer: bool | str 14 | explanation: Optional[str] = None 15 | 16 | class LedgerModel(BaseModel): 17 | """Main ledger state model""" 18 | 19 | is_request_satisfied: LedgerAnswer 20 | is_in_loop: LedgerAnswer 21 | is_progress_being_made: LedgerAnswer 22 | next_speaker: LedgerAnswer 23 | instruction_or_question: LedgerAnswer 24 | -------------------------------------------------------------------------------- /frontend/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", 4 | "target": "ES2022", 5 | "lib": ["ES2023"], 6 | "module": "ESNext", 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "isolatedModules": true, 13 | "moduleDetection": "force", 14 | "noEmit": true, 15 | 16 | /* Linting */ 17 | "strict": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "noFallthroughCasesInSwitch": true, 21 | "noUncheckedSideEffectImports": true 22 | }, 23 | "include": ["vite.config.ts"] 24 | } 25 | -------------------------------------------------------------------------------- /agentic_bench/utils/markdown_browser/__init__.py: -------------------------------------------------------------------------------- 1 | from .abstract_markdown_browser import AbstractMarkdownBrowser 2 | from .markdown_search import AbstractMarkdownSearch, BingMarkdownSearch 3 | 4 | # TODO: Fix mdconvert 5 | from .mdconvert import ( # type: ignore 6 | DocumentConverterResult, 7 | FileConversionException, 8 | MarkdownConverter, 9 | UnsupportedFormatException, 10 | ) 11 | from .requests_markdown_browser import RequestsMarkdownBrowser 12 | 13 | __all__ = ( 14 | "AbstractMarkdownBrowser", 15 | "RequestsMarkdownBrowser", 16 | "AbstractMarkdownSearch", 17 | "BingMarkdownSearch", 18 | "MarkdownConverter", 19 | "UnsupportedFormatException", 20 | "FileConversionException", 21 | "DocumentConverterResult", 22 | ) 23 | -------------------------------------------------------------------------------- /frontend/vite.config.ts: -------------------------------------------------------------------------------- 1 | import react from "@vitejs/plugin-react-swc"; 2 | import {config} from "dotenv"; 3 | import path from "path"; 4 | import {defineConfig} from "vite"; 5 | import svgr from "vite-plugin-svgr"; 6 | 7 | config(); 8 | 9 | // https://vite.dev/config/ 10 | export default defineConfig({ 11 | plugins: [react(), svgr()], 12 | resolve: { 13 | alias: { 14 | "@": path.resolve(__dirname, "./src"), 15 | }, 16 | }, 17 | preview: { 18 | port: 3000, 19 | strictPort: true, 20 | }, 21 | server: { 22 | port: 3000, 23 | strictPort: true, 24 | host: true, 25 | origin: "http://0.0.0.0:3000", 26 | }, 27 | define: { 28 | "process.env.MODEL_NAME": JSON.stringify( 29 | process.env.AGENTIC_BENCH_MODEL_NAME 30 | ), 31 | }, 32 | }); 33 | -------------------------------------------------------------------------------- /agentic_bench/main.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI, WebSocket 2 | from typing import Optional, List 3 | 4 | from orchestrator import SystemOrchestrator 5 | 6 | app: FastAPI = FastAPI() 7 | 8 | async def generate_response(task: str, websocket: Optional[WebSocket] = None): 9 | orchestrator: SystemOrchestrator = SystemOrchestrator() 10 | return await orchestrator.run(task, websocket) 11 | 12 | @app.get("/agent/chat") 13 | async def agent_chat(task: str) -> List: 14 | final_agent_response = await generate_response(task) 15 | return final_agent_response 16 | 17 | @app.websocket("/ws") 18 | async def websocket_endpoint(websocket: WebSocket): 19 | await websocket.accept() 20 | while True: 21 | data = await websocket.receive_text() 22 | await generate_response(data, websocket) 23 | -------------------------------------------------------------------------------- /frontend/eslint.config.js: -------------------------------------------------------------------------------- 1 | import js from '@eslint/js' 2 | import globals from 'globals' 3 | import reactHooks from 'eslint-plugin-react-hooks' 4 | import reactRefresh from 'eslint-plugin-react-refresh' 5 | import tseslint from 'typescript-eslint' 6 | 7 | export default tseslint.config( 8 | { ignores: ['dist'] }, 9 | { 10 | extends: [js.configs.recommended, ...tseslint.configs.recommended], 11 | files: ['**/*.{ts,tsx}'], 12 | languageOptions: { 13 | ecmaVersion: 2020, 14 | globals: globals.browser, 15 | }, 16 | plugins: { 17 | 'react-hooks': reactHooks, 18 | 'react-refresh': reactRefresh, 19 | }, 20 | rules: { 21 | ...reactHooks.configs.recommended.rules, 22 | 'react-refresh/only-export-components': [ 23 | 'warn', 24 | { allowConstantExport: true }, 25 | ], 26 | }, 27 | }, 28 | ) 29 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | agentic_bench: 5 | build: 6 | context: ./agentic_bench 7 | dockerfile: Dockerfile 8 | volumes: 9 | - ./agentic_bench:/app 10 | env_file: 11 | - .env 12 | restart: always 13 | network_mode: host 14 | 15 | agentic_browser: 16 | build: 17 | context: ./TheAgenticBrowser 18 | dockerfile: Dockerfile 19 | volumes: 20 | - ./TheAgenticBrowser:/app 21 | env_file: 22 | - .env 23 | restart: always 24 | network_mode: host 25 | 26 | frontend: 27 | build: 28 | context: ./frontend 29 | dockerfile: Dockerfile 30 | volumes: 31 | - ./frontend:/app 32 | - /app/node_modules 33 | env_file: 34 | - .env 35 | depends_on: 36 | - agentic_bench 37 | - agentic_browser 38 | restart: always 39 | network_mode: host -------------------------------------------------------------------------------- /frontend/src/constants/agentInfo.tsx: -------------------------------------------------------------------------------- 1 | export const DUMMY_AGENTS = [ 2 | { 3 | title: "Orchestrator", 4 | content: "Manages task assignment to agents and controls flow.", 5 | }, 6 | { 7 | title: "WebAgent", 8 | content: 9 | "Can access any web-page to extract information or perform actions.", 10 | }, 11 | { 12 | title: "FileSurferAgent", 13 | content: "Can access files in file-system and perform actions.", 14 | }, 15 | { 16 | title: "RAGAgent", 17 | content: "Agent specialized in using RAG to answer questions. Utilizes the existing data uploaded by user", 18 | }, 19 | { 20 | title: "CoderAgent", 21 | content: 22 | "Writes code that is needed to transform data from other agents to complete user's query.", 23 | }, 24 | { 25 | title: "CodeExecutorAgent", 26 | content: "Executes code from coder agent and handles errors/retries.", 27 | }, 28 | ]; 29 | -------------------------------------------------------------------------------- /frontend/src/components/customUI/ModelSelect.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Select, 3 | SelectContent, 4 | SelectItem, 5 | SelectTrigger, 6 | SelectValue, 7 | } from "@/components/ui/select"; 8 | import {useState} from "react"; 9 | 10 | export interface ModelProps { 11 | name: string; 12 | icon: React.ReactElement; 13 | type: string; 14 | } 15 | 16 | export const ModelSelect = ({models}: {models: ModelProps[]}) => { 17 | const [selectedModel, setSelectedModel] = useState(models[0].name); 18 | 19 | return ( 20 | 30 | ); 31 | }; 32 | -------------------------------------------------------------------------------- /frontend/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", 4 | "target": "ES2021", 5 | "useDefineForClassFields": true, 6 | "lib": ["ES2021", "DOM", "DOM.Iterable"], 7 | "module": "ESNext", 8 | "skipLibCheck": true, 9 | 10 | /* Bundler mode */ 11 | "moduleResolution": "bundler", 12 | "allowImportingTsExtensions": true, 13 | "isolatedModules": true, 14 | "moduleDetection": "force", 15 | "noEmit": true, 16 | "jsx": "react-jsx", 17 | 18 | /* Linting */ 19 | "strict": true, 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | "noFallthroughCasesInSwitch": true, 23 | "noUncheckedSideEffectImports": true, 24 | "baseUrl": ".", 25 | "paths": { 26 | "@/*": ["./src/*"] 27 | } 28 | }, 29 | "include": ["src"], 30 | "baseUrl": ".", 31 | "paths": { 32 | "@/*": ["./src/*"] 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /agentic_bench/utils/executors/executor_utils/extract_command_line_args.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | def extract_command_line_args(lang, filename, human_input_or_command_line_args): 4 | human_input_or_command_line_args = " ".join(human_input_or_command_line_args).strip() 5 | 6 | extension = filename.split('.')[-1] if '.' in filename else 'py' if lang.startswith('python') else lang 7 | 8 | # Define prefixes to remove 9 | prefixes = [f"{lang} {filename}", f"{lang}", f"{filename}"] 10 | for prefix in prefixes: 11 | if human_input_or_command_line_args.startswith(prefix): 12 | human_input_or_command_line_args = human_input_or_command_line_args[len(prefix):].strip() 13 | break 14 | 15 | # Split into arguments and filter out matches of *.extension 16 | args = human_input_or_command_line_args.split() 17 | args = [arg for arg in args if not re.fullmatch(rf".*\.{extension}", arg)] 18 | 19 | return args -------------------------------------------------------------------------------- /frontend/src/components/customUI/Accordion.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Accordion, 3 | AccordionContent, 4 | AccordionItem, 5 | AccordionTrigger, 6 | } from "@/components/ui/accordion"; 7 | import {BrainElectricity} from "iconoir-react"; 8 | 9 | interface AccordionItemProps { 10 | title: string; 11 | content: string; 12 | } 13 | 14 | export const AccordionGroup = ({items}: {items: AccordionItemProps[]}) => { 15 | return ( 16 | 17 | {items.map((item, idx) => ( 18 | 19 | 20 |
21 | 22 | {item.title} 23 |
24 | 25 | {item.content} 26 | 27 | ))} 28 | 29 | ); 30 | }; 31 | -------------------------------------------------------------------------------- /frontend/src/pages/Layout.tsx: -------------------------------------------------------------------------------- 1 | import {Header} from "@/components/navbar/Header"; 2 | import {AgentInfo} from "@/components/playground/AgentInfo"; 3 | import {Message} from "@/lib/types"; 4 | import {useState} from "react"; 5 | import {Chat} from "./Chat"; 6 | import Playground from "./Playground"; 7 | // import Sidebar from "../components/navbar/Sidebar"; 8 | 9 | const Layout = () => { 10 | const [messages, setMessages] = useState([]); 11 | 12 | return ( 13 |
14 | {/* 15 | for future use 16 | 17 | */} 18 |
19 |
20 | {messages && messages.length > 0 ? ( 21 | 22 | ) : ( 23 | 24 | )} 25 | 26 |
27 |
28 | ); 29 | }; 30 | 31 | export default Layout; 32 | -------------------------------------------------------------------------------- /frontend/src/App.tsx: -------------------------------------------------------------------------------- 1 | import {IconoirProvider} from "iconoir-react"; 2 | import {QueryClient, QueryClientProvider} from "react-query"; 3 | import {Navigate, Route, BrowserRouter as Router, Routes} from "react-router"; 4 | import {PAGE_ROUTES} from "./constants/routes"; 5 | import Layout from "./pages/Layout"; 6 | 7 | const queryClient = new QueryClient(); 8 | 9 | function App() { 10 | return ( 11 | 12 | 20 | 21 | 22 | } /> 23 | } /> 24 | 25 | 26 | 27 | 28 | ); 29 | } 30 | 31 | export default App; 32 | -------------------------------------------------------------------------------- /frontend/src/components/customUI/TerminalBlock.tsx: -------------------------------------------------------------------------------- 1 | import Markdown from "react-markdown"; 2 | import rehypeRaw from "rehype-raw"; 3 | import remarkBreaks from "remark-breaks"; 4 | 5 | export const TerminalBlock = ({content}: {content: string}) => { 6 | return ( 7 |
8 |
9 |
10 | Terminal 11 |
12 |
13 | 18 | {content.replaceAll("\\n", "\n")} 19 | 20 |
21 |
22 |
23 | ); 24 | }; 25 | -------------------------------------------------------------------------------- /frontend/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10 | 11 | 12 | 13 | 17 | 21 | 25 | theAgenticBench 26 | 27 | 28 |
29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /agentic_bench/utils/convert_messages.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union 2 | from utils.types import FunctionCall, FunctionExecutionResult 3 | from utils.image import Image 4 | 5 | # Convenience type 6 | UserContent = Union[str, List[Union[str, Image]]] 7 | AssistantContent = Union[str, List[FunctionCall]] 8 | FunctionExecutionContent = List[FunctionExecutionResult] 9 | SystemContent = str 10 | 11 | # Convert UserContent to a string 12 | def message_content_to_str( 13 | message_content: ( 14 | UserContent | AssistantContent | SystemContent | FunctionExecutionContent 15 | ), 16 | ) -> str: 17 | if isinstance(message_content, str): 18 | return message_content 19 | elif isinstance(message_content, List): 20 | converted: List[str] = list() 21 | for item in message_content: 22 | if isinstance(item, str): 23 | converted.append(item.rstrip()) 24 | elif isinstance(item, Image): 25 | converted.append("") 26 | else: 27 | converted.append(str(item).rstrip()) 28 | return "\n".join(converted) 29 | else: 30 | raise AssertionError("Unexpected response type.") 31 | -------------------------------------------------------------------------------- /frontend/src/components/customUI/SidebarMenu.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | DropdownMenu, 3 | DropdownMenuContent, 4 | DropdownMenuItem, 5 | DropdownMenuTrigger, 6 | } from "@/components/ui/dropdown-menu"; 7 | import {Bin, EditPencil} from "iconoir-react"; 8 | 9 | export const SidebarMenu = ({ 10 | children, 11 | handleStartEdit, 12 | setAlertOpen, 13 | }: { 14 | children: React.ReactNode; 15 | handleStartEdit: (e: React.MouseEvent) => void; 16 | setAlertOpen: React.Dispatch>; 17 | }) => { 18 | return ( 19 | 20 | 21 | {children} 22 | 23 | 24 | 28 | 29 | Rename 30 | 31 | setAlertOpen(true)} 34 | > 35 | 36 | Delete 37 | 38 | 39 | 40 | ); 41 | }; 42 | -------------------------------------------------------------------------------- /frontend/src/components/customUI/Loading.tsx: -------------------------------------------------------------------------------- 1 | import {cn} from "@/lib/utils"; 2 | interface LoadingProps { 3 | className?: string; 4 | dotClassName?: string; 5 | size?: "sm" | "md" | "lg"; 6 | } 7 | const Loading = ({className, dotClassName, size = "md"}: LoadingProps) => { 8 | const sizes = { 9 | sm: "w-1 h-1", 10 | md: "w-2 h-2", 11 | lg: "w-3 h-3", 12 | }; 13 | const containerSizes = { 14 | sm: "h-4", 15 | md: "h-6", 16 | lg: "h-8", 17 | }; 18 | return ( 19 |
27 |
35 |
43 |
51 |
52 | ); 53 | }; 54 | export default Loading; 55 | -------------------------------------------------------------------------------- /frontend/src/components/customUI/DeleteAlert.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | AlertDialog, 3 | AlertDialogAction, 4 | AlertDialogCancel, 5 | AlertDialogContent, 6 | AlertDialogDescription, 7 | AlertDialogFooter, 8 | AlertDialogHeader, 9 | AlertDialogTitle, 10 | } from "@/components/ui/alert-dialog"; 11 | import {buttonVariants} from "../ui/button"; 12 | 13 | export function DeleteAlert({ 14 | isOpen, 15 | handleDelete, 16 | setAlertOpen, 17 | }: { 18 | isOpen: boolean; 19 | handleDelete: (e: React.MouseEvent) => void; 20 | setAlertOpen: React.Dispatch>; 21 | }) { 22 | return ( 23 | 24 | 25 | 26 | Delete Chat? 27 | 28 | This chat will be deleted forever. The task history cannot be 29 | restored. 30 | 31 | 32 | 33 | Cancel 34 | 38 | Continue 39 | 40 | 41 | 42 | 43 | ); 44 | } 45 | -------------------------------------------------------------------------------- /frontend/src/components/playground/AgentInfo.tsx: -------------------------------------------------------------------------------- 1 | import {DUMMY_AGENTS} from "@/constants/agentInfo"; 2 | import {Box3dCenter} from "iconoir-react"; 3 | import {Dot} from "lucide-react"; 4 | import {AccordionGroup} from "../customUI/Accordion"; 5 | 6 | // for future versions 7 | // import {ModelSelect} from "../customUI/ModelSelect"; 8 | 9 | const { VITE_MODEL_NAME } = import.meta.env; 10 | 11 | export const AgentInfo = () => { 12 | return ( 13 |
14 |
15 |

Connected Model

16 | 17 | {/* 18 | for future versions 19 | 20 | */} 21 |
22 |
23 | 24 | {VITE_MODEL_NAME} 25 |
26 | 27 | 28 | 29 |
30 |
31 |
32 |

Agents

33 | 34 |
35 |
36 | ); 37 | }; 38 | -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | ### Agentic Bench ### 2 | AGENTIC_BENCH_MODEL_NAME= 3 | AGENTIC_BENCH_MODEL_BASE_URL= 4 | AGENTIC_BENCH_MODEL_API_KEY= 5 | AGENTIC_BENCH_SUPPORTED_EXECUTORS='["Docker", "Local"]' 6 | AGENTIC_BENCH_EXECUTOR="Local" 7 | # RAG Agent Configuration 8 | LLAMA_API_KEY= 9 | OPENAI_MODEL_NAME= 10 | OPENAI_API_KEY= 11 | 12 | ### Autoscrapper ### 13 | AGENTIC_BROWSER_TEXT_MODEL= 14 | AGENTIC_BROWSER_TEXT_API_KEY= 15 | AGENTIC_BROWSER_TEXT_BASE_URL= 16 | 17 | # Screenshot Analysis Configuration 18 | AGENTIC_BROWSER_SS_ENABLED= 19 | AGENTIC_BROWSER_SS_MODEL= 20 | AGENTIC_BROWSER_SS_API_KEY= 21 | AGENTIC_BROWSER_SS_BASE_URL= 22 | # Google Search Configuration 23 | GOOGLE_API_KEY= 24 | GOOGLE_CX= 25 | 26 | # Browser Configuration 27 | BROWSER_STORAGE_DIR= 28 | 29 | ### Logging ### 30 | LOGFIRE_TOKEN= 31 | 32 | ### Frontend Websocket Configuration ### 33 | VITE_WEBSOCKET_URL= 34 | VITE_MODEL_NAME=${AGENTIC_BENCH_MODEL_NAME} # either use existing ${AGENTIC_BENCH_MODEL_NAME} or give a text model name eg. "gpt-4o" 35 | -------------------------------------------------------------------------------- /agentic_bench/utils/cancellation_token.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from asyncio import Future 3 | from typing import Any, Callable, List 4 | 5 | class CancellationToken: 6 | """A token used to cancel pending async calls""" 7 | 8 | def __init__(self) -> None: 9 | self._cancelled: bool = False 10 | self._lock: threading.Lock = threading.Lock() 11 | self._callbacks: List[Callable[[], None]] = [] 12 | 13 | def cancel(self) -> None: 14 | """Cancel pending async calls linked to this cancellation token.""" 15 | with self._lock: 16 | if not self._cancelled: 17 | self._cancelled = True 18 | for callback in self._callbacks: 19 | callback() 20 | 21 | def is_cancelled(self) -> bool: 22 | """Check if the CancellationToken has been used""" 23 | with self._lock: 24 | return self._cancelled 25 | 26 | def add_callback(self, callback: Callable[[], None]) -> None: 27 | """Attach a callback that will be called when cancel is invoked""" 28 | with self._lock: 29 | if self._cancelled: 30 | callback() 31 | else: 32 | self._callbacks.append(callback) 33 | 34 | def link_future(self, future: Future[Any]) -> Future[Any]: 35 | """Link a pending async call to a token to allow its cancellation""" 36 | with self._lock: 37 | if self._cancelled: 38 | future.cancel() 39 | else: 40 | 41 | def _cancel() -> None: 42 | future.cancel() 43 | self._callbacks.append(_cancel) 44 | return future 45 | -------------------------------------------------------------------------------- /agentic_bench/utils/executors/executor_utils/_base.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from typing import List, Protocol, runtime_checkable 5 | 6 | from utils import CancellationToken 7 | 8 | @dataclass 9 | class CodeBlock: 10 | """A code block extracted fromm an agent message.""" 11 | 12 | code: str 13 | packages: List 14 | language: str 15 | human_input_or_command_line_args:str 16 | 17 | @dataclass 18 | class CodeResult: 19 | """Result of a code execution.""" 20 | 21 | exit_code: int 22 | output: str 23 | 24 | @runtime_checkable 25 | class CodeExecutor(Protocol): 26 | """Executes code blocks and returns the result.""" 27 | 28 | async def execute_code_blocks( 29 | self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken 30 | ) -> CodeResult: 31 | """Execute code blocks and return the result. 32 | 33 | This method should be implemented by the code executor. 34 | 35 | Args: 36 | code_blocks (List[CodeBlock]): The code blocks to execute. 37 | 38 | Returns: 39 | CodeResult: The result of the code execution. 40 | 41 | Raises: 42 | ValueError: Errors in user inputs 43 | asyncio.TimeoutError: Code execution timeouts 44 | asyncio.CancelledError: CancellationToken evoked during execution 45 | """ 46 | ... 47 | 48 | async def restart(self) -> None: 49 | """Restart the code executor. 50 | 51 | This method should be implemented by the code executor. 52 | 53 | This method is called when the agent is reset. 54 | """ 55 | ... 56 | -------------------------------------------------------------------------------- /agentic_bench/utils/markdown_browser/abstract_markdown_browser.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from typing import Union 3 | 4 | class AbstractMarkdownBrowser(ABC): 5 | """ 6 | An abstract class for a Markdown web browser. 7 | 8 | All MarkdownBrowers work by: 9 | 10 | (1) fetching a web page by URL (via requests, Selenium, Playwright, etc.) 11 | (2) converting the page's HTML or DOM to Markdown 12 | (3) operating on the Markdown 13 | 14 | Such browsers are simple, and suitable for read-only agentic use. 15 | They cannot be used to interact with complex web applications. 16 | """ 17 | 18 | @abstractmethod 19 | def __init__(self) -> None: 20 | pass 21 | 22 | @property 23 | @abstractmethod 24 | def address(self) -> str: 25 | pass 26 | 27 | @abstractmethod 28 | def set_address(self, uri_or_path: str) -> None: 29 | pass 30 | 31 | @property 32 | @abstractmethod 33 | def viewport(self) -> str: 34 | pass 35 | 36 | @property 37 | @abstractmethod 38 | def page_content(self) -> str: 39 | pass 40 | 41 | @abstractmethod 42 | def page_down(self) -> None: 43 | pass 44 | 45 | @abstractmethod 46 | def page_up(self) -> None: 47 | pass 48 | 49 | @abstractmethod 50 | def visit_page(self, path_or_uri: str) -> str: 51 | pass 52 | 53 | @abstractmethod 54 | def open_local_file(self, local_path: str) -> str: 55 | pass 56 | 57 | @abstractmethod 58 | def find_on_page(self, query: str) -> Union[str, None]: 59 | pass 60 | 61 | @abstractmethod 62 | def find_next(self) -> Union[str, None]: 63 | pass 64 | -------------------------------------------------------------------------------- /frontend/src/index.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | :root { 6 | font-family: DM Sans, serif; 7 | line-height: 1.5; 8 | font-weight: 400; 9 | 10 | color-scheme: light dark; 11 | color: "hsl(var(--foreground))"; 12 | background: "hsl(var(--background))"; 13 | 14 | box-sizing: border-box; 15 | 16 | font-synthesis: none; 17 | text-rendering: optimizeLegibility; 18 | -webkit-font-smoothing: antialiased; 19 | -moz-osx-font-smoothing: grayscale; 20 | } 21 | 22 | body { 23 | margin: 0; 24 | min-width: 320px; 25 | min-height: 100vh; 26 | } 27 | 28 | @layer base { 29 | :root { 30 | --background: 0 0% 7%; 31 | --foreground: 0 0% 96%; 32 | --card: 0 0% 100%; 33 | --card-foreground: 240 10% 3.9%; 34 | --popover: 0 0% 12%; 35 | --popover-secondary: 0 0% 16%; 36 | --popover-foreground: 0 0% 96%; 37 | --primary: 0 0% 96%; 38 | --primary-foreground: 0 0% 7%; 39 | --secondary: 0 0% 12%; 40 | --secondary-foreground: 0 0% 96%; 41 | --muted: 0 0% 9%; 42 | --muted-foreground: 0 0% 60%; 43 | --accent: 240 4.8% 95.9%; 44 | --accent-foreground: 240 5.9% 10%; 45 | --destructive: 0 20% 18%; 46 | --destructive-foreground: 0 85% 64%; 47 | --border: 0 0% 20%; 48 | --input: 0 0% 9%; 49 | --ring: 0 0% 20%; 50 | --chart-1: 12 76% 61%; 51 | --chart-2: 173 58% 39%; 52 | --chart-3: 197 37% 24%; 53 | --chart-4: 43 74% 66%; 54 | --chart-5: 27 87% 67%; 55 | --radius: 12px; 56 | --success: 132 52% 47%; 57 | --disabled-foreground: 0 0% 32%; 58 | } 59 | } 60 | 61 | @layer base { 62 | * { 63 | @apply border-border; 64 | } 65 | body { 66 | @apply bg-background text-foreground; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /agentic_bench/utils/initializers/text_extractor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncio 3 | from llama_parse import LlamaParse 4 | from typing import List 5 | 6 | 7 | async def extract_text_from_directory_async(pdf_dir: str) -> List[str]: 8 | """ 9 | Extract text asynchronously from all PDF files in the specified directory. 10 | 11 | Args: 12 | pdf_dir (str): Path to the directory containing PDF files. 13 | 14 | Returns: 15 | List[str]: A list of extracted text, one entry per PDF file. 16 | """ 17 | if not os.path.exists(pdf_dir) or not os.path.isdir(pdf_dir): 18 | raise ValueError(f"The directory '{pdf_dir}' does not exist or is not a valid directory.") 19 | 20 | input_files = [ 21 | os.path.join(pdf_dir, file_name) 22 | for file_name in os.listdir(pdf_dir) 23 | if file_name.lower().endswith('.pdf') 24 | ] 25 | 26 | if not input_files: 27 | print("No files uploaded by the user.") 28 | return [] 29 | 30 | parser = LlamaParse( 31 | api_key=os.getenv("LLAMA_API_KEY"), 32 | result_type="text", 33 | premium_mode=False 34 | ) 35 | 36 | tasks = [parser.aload_data(file) for file in input_files] 37 | results = await asyncio.gather(*tasks, return_exceptions=True) 38 | 39 | # Ensure compatibility with the original return structure 40 | extracted_texts = [] 41 | for result in results: 42 | if isinstance(result, Exception): 43 | print(f"Error processing file: {result}") 44 | elif isinstance(result, list): 45 | # Flatten the list of text objects into plain text 46 | extracted_texts.extend([doc.text for doc in result]) 47 | else: 48 | extracted_texts.append(result.text) # Single document case 49 | 50 | return extracted_texts -------------------------------------------------------------------------------- /frontend/src/components/ui/alert.tsx: -------------------------------------------------------------------------------- 1 | import {cva, type VariantProps} from "class-variance-authority"; 2 | import * as React from "react"; 3 | 4 | import {cn} from "@/lib/utils"; 5 | 6 | const alertVariants = cva( 7 | "relative w-full rounded-lg border px-4 py-3 text-sm [&>svg]:absolute [&>svg]:left-3 [&>svg]:top-3 [&>svg]:text-foreground [&>svg~*]:pl-7", 8 | { 9 | variants: { 10 | variant: { 11 | default: "bg-background text-foreground", 12 | destructive: 13 | "bg-destructive text-destructive-foreground [&>svg]:text-destructive", 14 | }, 15 | }, 16 | defaultVariants: { 17 | variant: "default", 18 | }, 19 | } 20 | ); 21 | 22 | const Alert = React.forwardRef< 23 | HTMLDivElement, 24 | React.HTMLAttributes & VariantProps 25 | >(({className, variant, ...props}, ref) => ( 26 |
32 | )); 33 | Alert.displayName = "Alert"; 34 | 35 | const AlertTitle = React.forwardRef< 36 | HTMLParagraphElement, 37 | React.HTMLAttributes 38 | >(({className, ...props}, ref) => ( 39 |
44 | )); 45 | AlertTitle.displayName = "AlertTitle"; 46 | 47 | const AlertDescription = React.forwardRef< 48 | HTMLParagraphElement, 49 | React.HTMLAttributes 50 | >(({className, ...props}, ref) => ( 51 |
56 | )); 57 | AlertDescription.displayName = "AlertDescription"; 58 | 59 | export {Alert, AlertDescription, AlertTitle}; 60 | -------------------------------------------------------------------------------- /frontend/src/components/ui/scroll-area.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area" 3 | 4 | import { cn } from "@/lib/utils" 5 | 6 | const ScrollArea = React.forwardRef< 7 | React.ElementRef, 8 | React.ComponentPropsWithoutRef 9 | >(({ className, children, ...props }, ref) => ( 10 | 15 | 16 | {children} 17 | 18 | 19 | 20 | 21 | )) 22 | ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName 23 | 24 | const ScrollBar = React.forwardRef< 25 | React.ElementRef, 26 | React.ComponentPropsWithoutRef 27 | >(({ className, orientation = "vertical", ...props }, ref) => ( 28 | 41 | 42 | 43 | )) 44 | ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName 45 | 46 | export { ScrollArea, ScrollBar } 47 | -------------------------------------------------------------------------------- /frontend/src/components/customUI/CodeBlock.tsx: -------------------------------------------------------------------------------- 1 | import Markdown from "react-markdown"; 2 | import {Prism as SyntaxHighlighter} from "react-syntax-highlighter"; 3 | import {tomorrow} from "react-syntax-highlighter/dist/esm/styles/prism"; 4 | import rehypeRaw from "rehype-raw"; 5 | import remarkBreaks from "remark-breaks"; 6 | 7 | export const CodeBlock = ({content}: {content: string}) => { 8 | return ( 9 |
10 |
11 |
12 | example.py 13 |
14 |
15 | 30 | {String(children).replace(/\n$/, "")} 31 | 32 | ) : ( 33 | 34 | {children} 35 | 36 | ); 37 | }, 38 | }} 39 | /> 40 |
41 |
42 |
43 | ); 44 | }; 45 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agentic-bench", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite --host", 8 | "build": "tsc -b && vite build", 9 | "lint": "eslint .", 10 | "preview": "vite preview" 11 | }, 12 | "dependencies": { 13 | "@radix-ui/react-accordion": "^1.2.2", 14 | "@radix-ui/react-alert-dialog": "^1.1.4", 15 | "@radix-ui/react-dropdown-menu": "^2.1.4", 16 | "@radix-ui/react-scroll-area": "^1.2.2", 17 | "@radix-ui/react-select": "^2.1.4", 18 | "@radix-ui/react-slot": "^1.1.1", 19 | "class-variance-authority": "^0.7.1", 20 | "clsx": "^2.1.1", 21 | "dotenv": "^16.4.7", 22 | "iconoir-react": "^7.10.1", 23 | "lucide-react": "^0.469.0", 24 | "react": "^18.3.1", 25 | "react-dom": "^18.3.1", 26 | "react-markdown": "^9.0.1", 27 | "react-query": "^3.39.3", 28 | "react-router": "^7.1.1", 29 | "react-syntax-highlighter": "^15.6.1", 30 | "react-use-websocket": "^4.11.1", 31 | "rehype-raw": "^7.0.0", 32 | "remark-breaks": "^4.0.0", 33 | "remark-gfm": "^4.0.0", 34 | "tailwind-merge": "^2.6.0", 35 | "tailwindcss-animate": "^1.0.7", 36 | "vite-plugin-svgr": "^4.3.0" 37 | }, 38 | "devDependencies": { 39 | "@eslint/js": "^9.17.0", 40 | "@types/node": "^22.10.2", 41 | "@types/react": "^18.3.18", 42 | "@types/react-dom": "^18.3.5", 43 | "@types/react-syntax-highlighter": "^15.5.13", 44 | "@vitejs/plugin-react-swc": "^3.5.0", 45 | "autoprefixer": "^10.4.20", 46 | "eslint": "^9.17.0", 47 | "eslint-plugin-react-hooks": "^5.0.0", 48 | "eslint-plugin-react-refresh": "^0.4.16", 49 | "globals": "^15.14.0", 50 | "postcss": "^8.4.49", 51 | "tailwindcss": "^3.4.17", 52 | "typescript": "~5.6.2", 53 | "typescript-eslint": "^8.18.2", 54 | "vite": "^6.0.5" 55 | }, 56 | "optionalDependencies": { 57 | "@rollup/rollup-linux-x64-gnu": "^4.30.1" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /frontend/src/components/playground/ModelSelect.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Select, 3 | SelectContent, 4 | SelectGroup, 5 | SelectItem, 6 | SelectTrigger, 7 | SelectValue, 8 | } from "@/components/ui/select"; 9 | 10 | interface Model { 11 | value: string; 12 | label: string; 13 | icon: React.ReactNode; 14 | } 15 | 16 | interface ModelSelectProps { 17 | models: Model[]; 18 | value?: string; 19 | onValueChange?: (value: string) => void; 20 | } 21 | 22 | const ModelSelect: React.FC = ({ 23 | models, 24 | value, 25 | onValueChange, 26 | }) => { 27 | const selectedModel = (models as Model[]).find( 28 | (model) => model.value === value 29 | ); 30 | 31 | return ( 32 |
33 |

Model

34 | 62 |
63 | ); 64 | }; 65 | 66 | export default ModelSelect; 67 | -------------------------------------------------------------------------------- /frontend/src/components/navbar/Header.tsx: -------------------------------------------------------------------------------- 1 | import {Message} from "@/lib/types"; 2 | import { 3 | ArrowUpRight, 4 | ChatBubbleEmptySolid, 5 | Github, 6 | OpenBook, 7 | } from "iconoir-react"; 8 | import {Button} from "../ui/button"; 9 | import {SidebarItem} from "./SidebarItem"; 10 | 11 | export const Header = ({ 12 | setMessages, 13 | }: { 14 | setMessages: React.Dispatch>; 15 | }) => { 16 | return ( 17 |
18 | {"TheAgenticBench"} 28 |
29 | 40 | 42 | window.open( 43 | "https://github.com/TheAgenticAI/agentic-one-fe", 44 | "_blank" 45 | ) 46 | } 47 | > 48 | 49 |

Documentation

50 | 51 | 52 | 54 | window.open( 55 | "https://github.com/TheAgenticAI/agentic-one-fe", 56 | "_blank" 57 | ) 58 | } 59 | > 60 | 61 |

Github

62 | 63 | 64 |
65 |
66 | ); 67 | }; 68 | -------------------------------------------------------------------------------- /frontend/src/components/ui/button.tsx: -------------------------------------------------------------------------------- 1 | import {Slot} from "@radix-ui/react-slot"; 2 | import {cva, type VariantProps} from "class-variance-authority"; 3 | import * as React from "react"; 4 | 5 | import {cn} from "@/lib/utils"; 6 | 7 | const buttonVariants = cva( 8 | "inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0", 9 | { 10 | variants: { 11 | variant: { 12 | default: 13 | "bg-primary text-primary-foreground shadow hover:bg-primary/90", 14 | destructive: 15 | "bg-destructive text-destructive-foreground shadow-sm hover:bg-destructive/90", 16 | outline: 17 | "border border-border bg-background shadow-sm hover:bg-background", 18 | secondary: 19 | "bg-secondary text-secondary-foreground shadow-sm hover:bg-secondary/80", 20 | ghost: "hover:bg-accent hover:text-accent-foreground", 21 | link: "text-primary underline-offset-4 hover:underline", 22 | }, 23 | size: { 24 | default: "h-9 px-4 py-2", 25 | sm: "h-8 rounded-md px-3 text-xs", 26 | lg: "h-10 rounded-md px-8", 27 | icon: "h-9 w-9", 28 | }, 29 | }, 30 | defaultVariants: { 31 | variant: "default", 32 | size: "default", 33 | }, 34 | } 35 | ); 36 | 37 | export interface ButtonProps 38 | extends React.ButtonHTMLAttributes, 39 | VariantProps { 40 | asChild?: boolean; 41 | } 42 | 43 | const Button = React.forwardRef( 44 | ({className, variant, size, asChild = false, ...props}, ref) => { 45 | const Comp = asChild ? Slot : "button"; 46 | return ( 47 | 52 | ); 53 | } 54 | ); 55 | Button.displayName = "Button"; 56 | 57 | export {Button, buttonVariants}; 58 | -------------------------------------------------------------------------------- /frontend/src/components/ui/card.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | 3 | import { cn } from "@/lib/utils" 4 | 5 | const Card = React.forwardRef< 6 | HTMLDivElement, 7 | React.HTMLAttributes 8 | >(({ className, ...props }, ref) => ( 9 |
17 | )) 18 | Card.displayName = "Card" 19 | 20 | const CardHeader = React.forwardRef< 21 | HTMLDivElement, 22 | React.HTMLAttributes 23 | >(({ className, ...props }, ref) => ( 24 |
29 | )) 30 | CardHeader.displayName = "CardHeader" 31 | 32 | const CardTitle = React.forwardRef< 33 | HTMLDivElement, 34 | React.HTMLAttributes 35 | >(({ className, ...props }, ref) => ( 36 |
41 | )) 42 | CardTitle.displayName = "CardTitle" 43 | 44 | const CardDescription = React.forwardRef< 45 | HTMLDivElement, 46 | React.HTMLAttributes 47 | >(({ className, ...props }, ref) => ( 48 |
53 | )) 54 | CardDescription.displayName = "CardDescription" 55 | 56 | const CardContent = React.forwardRef< 57 | HTMLDivElement, 58 | React.HTMLAttributes 59 | >(({ className, ...props }, ref) => ( 60 |
61 | )) 62 | CardContent.displayName = "CardContent" 63 | 64 | const CardFooter = React.forwardRef< 65 | HTMLDivElement, 66 | React.HTMLAttributes 67 | >(({ className, ...props }, ref) => ( 68 |
73 | )) 74 | CardFooter.displayName = "CardFooter" 75 | 76 | export { Card, CardHeader, CardFooter, CardTitle, CardDescription, CardContent } 77 | -------------------------------------------------------------------------------- /frontend/src/components/navbar/Sidebar.tsx: -------------------------------------------------------------------------------- 1 | import {Button} from "@/components/ui/button"; 2 | import {PAGE_ROUTES} from "@/constants/routes"; 3 | import { 4 | ArrowUpRight, 5 | ChatBubbleEmptySolid, 6 | Github, 7 | OpenBook, 8 | } from "iconoir-react"; 9 | import {useNavigate} from "react-router"; 10 | import {SidebarItem} from "./SidebarItem"; 11 | 12 | const {VITE_PRODUCT_LOGO} = import.meta.env; 13 | 14 | const Sidebar = () => { 15 | const nav = useNavigate(); 16 | 17 | return ( 18 |
19 | {/* since logo was not given: till that time */} 20 |
21 | {"TheAgentic 32 |
33 | 43 |
44 |
45 |

History

46 |
47 |
48 | {}}> 49 | 50 |

Documentation

51 | 52 | 53 | {}}> 54 | 55 |

Github

56 | 57 | 58 |
59 |
60 |
61 | ); 62 | }; 63 | 64 | export default Sidebar; 65 | -------------------------------------------------------------------------------- /frontend/src/components/ui/accordion.tsx: -------------------------------------------------------------------------------- 1 | import * as AccordionPrimitive from "@radix-ui/react-accordion"; 2 | import * as React from "react"; 3 | 4 | import {cn} from "@/lib/utils"; 5 | import {ChevronDown} from "lucide-react"; 6 | 7 | const Accordion = AccordionPrimitive.Root; 8 | 9 | const AccordionItem = React.forwardRef< 10 | React.ElementRef, 11 | React.ComponentPropsWithoutRef 12 | >(({className, ...props}, ref) => ( 13 | 18 | )); 19 | AccordionItem.displayName = "AccordionItem"; 20 | 21 | const AccordionTrigger = React.forwardRef< 22 | React.ElementRef, 23 | React.ComponentPropsWithoutRef 24 | >(({className, children, ...props}, ref) => ( 25 | 26 | svg]:rotate-180", 30 | className 31 | )} 32 | {...props} 33 | > 34 | {children} 35 | 36 | 37 | 38 | )); 39 | AccordionTrigger.displayName = AccordionPrimitive.Trigger.displayName; 40 | 41 | const AccordionContent = React.forwardRef< 42 | React.ElementRef, 43 | React.ComponentPropsWithoutRef 44 | >(({className, children, ...props}, ref) => ( 45 | 50 |
{children}
51 | 52 | )); 53 | AccordionContent.displayName = AccordionPrimitive.Content.displayName; 54 | 55 | export {Accordion, AccordionContent, AccordionItem, AccordionTrigger}; 56 | -------------------------------------------------------------------------------- /agentic_bench/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohappyeyeballs==2.4.4 2 | aiohttp==3.11.11 3 | aiosignal==1.3.2 4 | annotated-types==0.7.0 5 | anthropic==0.42.0 6 | anyio==4.7.0 7 | asyncio-atexit==1.0.1 8 | attrs==24.3.0 9 | beautifulsoup4==4.12.3 10 | cachetools==5.5.0 11 | certifi==2024.12.14 12 | cffi==1.17.1 13 | charset-normalizer==3.4.1 14 | click==8.1.8 15 | cobble==0.1.4 16 | colorama==0.4.6 17 | cryptography==44.0.0 18 | Deprecated==1.2.15 19 | distro==1.9.0 20 | docker==7.1.0 21 | eval_type_backport==0.2.2 22 | executing==2.1.0 23 | fastapi==0.115.6 24 | frozenlist==1.5.0 25 | google-auth==2.37.0 26 | googleapis-common-protos==1.66.0 27 | griffe==1.5.4 28 | groq==0.13.1 29 | h11==0.14.0 30 | httpcore==1.0.7 31 | httpx==0.27.2 32 | idna==3.10 33 | importlib_metadata==8.5.0 34 | jiter==0.8.2 35 | jsonpath-python==1.0.6 36 | logfire==2.11.0 37 | logfire-api==2.11.0 38 | lxml==5.3.0 39 | mammoth==1.8.0 40 | markdown-it-py==3.0.0 41 | markdownify==0.14.1 42 | mdurl==0.1.2 43 | mistralai==1.2.5 44 | multidict==6.1.0 45 | mypy-extensions==1.0.0 46 | numpy==2.2.1 47 | openai==1.58.1 48 | opentelemetry-api==1.29.0 49 | opentelemetry-exporter-otlp-proto-common==1.29.0 50 | opentelemetry-exporter-otlp-proto-http==1.29.0 51 | opentelemetry-instrumentation==0.50b0 52 | opentelemetry-proto==1.29.0 53 | opentelemetry-sdk==1.29.0 54 | opentelemetry-semantic-conventions==0.50b0 55 | packaging==24.2 56 | pandas==2.2.3 57 | pathvalidate==3.2.1 58 | pdfminer==20191125 59 | pdfminer.six==20240706 60 | pillow==11.0.0 61 | propcache==0.2.1 62 | protobuf==5.29.2 63 | puremagic==1.28 64 | pyasn1==0.6.1 65 | pyasn1_modules==0.4.1 66 | pycparser==2.22 67 | pycryptodome==3.21.0 68 | pydantic==2.10.4 69 | pydantic-ai==0.0.17 70 | pydantic-ai-slim==0.0.17 71 | pydantic_core==2.27.2 72 | Pygments==2.18.0 73 | python-dateutil==2.9.0.post0 74 | python-dotenv==1.0.1 75 | python-pptx==1.0.2 76 | pytz==2024.2 77 | requests==2.32.3 78 | rich==13.9.4 79 | rsa==4.9 80 | six==1.17.0 81 | sniffio==1.3.1 82 | soupsieve==2.6 83 | starlette==0.41.3 84 | tqdm==4.67.1 85 | typing-inspect==0.9.0 86 | typing_extensions==4.12.2 87 | tzdata==2024.2 88 | urllib3==2.3.0 89 | uvicorn==0.34.0 90 | websockets==14.1 91 | wrapt==1.17.0 92 | XlsxWriter==3.2.0 93 | yarl==1.18.3 94 | zipp==3.21.0 95 | fast-graphrag==0.0.4 96 | llama_parse==0.5.19 -------------------------------------------------------------------------------- /frontend/src/components/customUI/PromptInput.tsx: -------------------------------------------------------------------------------- 1 | import {Button} from "@/components/ui/button"; 2 | import {SendSolid} from "iconoir-react"; 3 | import {useEffect, useRef} from "react"; 4 | 5 | export const PromptInput = ({ 6 | task, 7 | setTask, 8 | sendTask, 9 | disable, 10 | }: { 11 | task: string; 12 | setTask: (task: string) => void; 13 | sendTask?: () => void; 14 | disable?: boolean; 15 | }) => { 16 | const textareaRef = useRef(null); 17 | 18 | // Update height and multiline state 19 | useEffect(() => { 20 | const textarea = textareaRef.current; 21 | if (textarea) { 22 | // Set a small height initially to properly measure single line 23 | textarea.style.height = "24px"; 24 | const scrollHeight = textarea.scrollHeight; 25 | textarea.style.height = `${scrollHeight + 2}px`; 26 | } 27 | }, [task]); 28 | 29 | const handleKeyDown = (e: React.KeyboardEvent) => { 30 | if (e.key === "Enter" && !e.shiftKey) { 31 | if (sendTask) { 32 | e.preventDefault(); 33 | sendTask(); 34 | } 35 | } 36 | }; 37 | 38 | const handleInput = (event: React.ChangeEvent) => { 39 | setTask(event.target.value); 40 | }; 41 | 42 | return ( 43 | <> 44 |