├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── README.md ├── examples ├── anthropic_example.rs ├── anthropic_streaming_example.rs ├── anthropic_thinking_example.rs ├── anthropic_vision_example.rs ├── api_deepclaude_example.rs ├── api_example.rs ├── azure_openai_embedding_example.rs ├── azure_openai_example.rs ├── azure_openai_tool_calling_example.rs ├── chain_audio_text_example.rs ├── chain_example.rs ├── chain_logging_example.rs ├── deepclaude_pipeline_example.rs ├── deepseek_example.rs ├── dummy.pdf ├── elevenlabs_stt_example.rs ├── elevenlabs_tts_example.rs ├── embedding_example.rs ├── evaluation_example.rs ├── evaluator_parallel_example.rs ├── google_embedding_example.rs ├── google_example.rs ├── google_image.rs ├── google_pdf.rs ├── google_streaming_example.rs ├── google_structured_output_example.rs ├── google_tool_calling_example.rs ├── groq_example.rs ├── image001.jpg ├── json_schema_nested_example.rs ├── memory_example.rs ├── memory_share_example.rs ├── multi_backend_example.rs ├── multi_backend_structured_output_example.rs ├── ollama_example.rs ├── ollama_structured_output_example.rs ├── openai_example.rs ├── openai_reasoning_example.rs ├── openai_streaming_example.rs ├── openai_structured_output_example.rs ├── openai_stt_example.rs ├── openai_tts_example.rs ├── openai_vision_example.rs ├── phind_example.rs ├── tool_calling_example.rs ├── tool_json_schema_cycle_example.rs ├── trim_strategy_example.rs ├── tts_rodio_example.rs ├── unified_tool_calling_example.rs ├── validator_example.rs ├── xai_example.rs ├── xai_search_chain_tts_example.rs ├── xai_search_example.rs ├── xai_streaming_example.rs └── xai_structured_output_example.rs └── src ├── api ├── handlers.rs ├── mod.rs └── types.rs ├── backends ├── anthropic.rs ├── azure_openai.rs ├── deepseek.rs ├── elevenlabs.rs ├── google.rs ├── groq.rs ├── mod.rs ├── ollama.rs ├── openai.rs ├── phind.rs └── xai.rs ├── bin └── llm-cli.rs ├── builder.rs ├── chain ├── mod.rs └── multi.rs ├── chat └── mod.rs ├── completion └── mod.rs ├── embedding └── mod.rs ├── error.rs ├── evaluator ├── mod.rs └── parallel.rs ├── lib.rs ├── memory ├── chat_wrapper.rs ├── mod.rs ├── shared_memory.rs └── sliding_window.rs ├── secret_store.rs ├── stt └── mod.rs ├── tts └── mod.rs └── validated_llm.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .env* 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "llm" 3 | version = "1.3.0" 4 | edition = "2021" 5 | description = "A Rust library unifying multiple LLM backends." 6 | license = "MIT" 7 | authors = [ 8 | "Tristan Granier ", 9 | "Jared Kofron ", 10 | ] 11 | repository = "https://github.com/graniet/llm" 12 | documentation = "https://docs.rs/llm" 13 | homepage = "https://github.com/graniet/llm" 14 | default-run = "llm" 15 | 16 | [features] 17 | default = ["cli", "default-tls"] 18 | default-tls = ["reqwest/default-tls"] 19 | rustls-tls = ["reqwest/rustls-tls"] 20 | full = [ 21 | "openai", 22 | "anthropic", 23 | "ollama", 24 | "deepseek", 25 | "xai", 26 | "phind", 27 | "google", 28 | "groq", 29 | "azure_openai", 30 | "api", 31 | "elevenlabs", 32 | ] 33 | openai = [] 34 | anthropic = [] 35 | ollama = [] 36 | deepseek = [] 37 | xai = [] 38 | phind = [] 39 | google = [] 40 | groq = [] 41 | azure_openai = [] 42 | cli = ["full", "dep:clap", "dep:rustyline", "dep:colored", "dep:spinners"] 43 | api = ["dep:axum", "dep:tower-http", "dep:uuid"] 44 | elevenlabs = [] 45 | rodio = ["dep:rodio"] 46 | logging = ["dep:env_logger"] 47 | 48 | [dependencies] 49 | serde = { version = "1.0", features = ["derive"] } 50 | reqwest = { version = "0.12.12", default-features = false, features = ["json", "multipart", "stream"] } 51 | serde_json = "1.0" 52 | async-trait = "0.1" 53 | axum = { version = "0.7", optional = true, features = ["json"] } 54 | tokio = { version = "1.0", features = ["full"] } 55 | tower-http = { version = "0.5", optional = true, features = ["cors"] } 56 | uuid = { version = "1.0", optional = true, features = ["v4"] } 57 | base64 = "0.22.1" 58 | futures = "0.3" 59 | clap = { version = "4", features = ["derive"], optional = true } 60 | rustyline = { version = "15", optional = true } 61 | colored = { version = "3.0.0", optional = true } 62 | spinners = { version = "4.1", optional = true } 63 | serde_yaml = "0.9" 64 | dirs = "6.0.0" 65 | either = { version = "1.15.0", features = ["serde"] } 66 | rodio = { version = "0.20.0", features = ["mp3", "wav"], optional = true } 67 | log = "0.4" 68 | env_logger = { version = "0.11", optional = true } 69 | 70 | [[bin]] 71 | name = "llm" 72 | path = "src/bin/llm-cli.rs" 73 | required-features = ["cli"] 74 | 75 | [dev-dependencies] 76 | tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } 77 | rodio = { version = "0.20.1", default-features = false, features = ["symphonia-all"]} 78 | 79 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LLM 2 | 3 | > **Note**: This crate name previously belonged to another project. The current implementation represents a new and different library. The previous crate is now archived and will not receive any updates. **ref: https://github.com/rustformers/llm** 4 | 5 | **LLM** is a **Rust** library that lets you use **multiple LLM backends** in a single project: [OpenAI](https://openai.com), [Anthropic (Claude)](https://www.anthropic.com), [Ollama](https://github.com/ollama/ollama), [DeepSeek](https://www.deepseek.com), [xAI](https://x.ai), [Phind](https://www.phind.com), [Groq](https://www.groq.com), [Google](https://cloud.google.com/gemini) and [ElevenLabs](https://elevenlabs.io). 6 | With a **unified API** and **builder style** - similar to the Stripe experience - you can easily create **chat**, text **completion**, speak-to-text requests without multiplying structures and crates. 7 | 8 | ## Key Features 9 | 10 | - **Multi-backend**: Manage OpenAI, Anthropic, Ollama, DeepSeek, xAI, Phind, Groq and Google through a single entry point. 11 | - **Multi-step chains**: Create multi-step chains with different backends at each step. 12 | - **Templates**: Use templates to create complex prompts with variables. 13 | - **Builder pattern**: Configure your LLM (model, temperature, max_tokens, timeouts...) with a few simple calls. 14 | - **Chat & Completions**: Two unified traits (`ChatProvider` and `CompletionProvider`) to cover most use cases. 15 | - **Extensible**: Easily add new backends. 16 | - **Rust-friendly**: Designed with clear traits, unified error handling, and conditional compilation via *features*. 17 | - **Validation**: Add validation to your requests to ensure the output is what you expect. 18 | - **Evaluation**: Add evaluation to your requests to score the output of LLMs. 19 | - **Parallel Evaluation**: Evaluate multiple LLM providers in parallel and select the best response based on scoring functions. 20 | - **Function calling**: Add function calling to your requests to use tools in your LLMs. 21 | - **REST API**: Serve any LLM backend as a REST API with openai standard format. 22 | - **Vision**: Add vision to your requests to use images in your LLMs. 23 | - **Reasoning**: Add reasoning to your requests to use reasoning in your LLMs. 24 | - **Structured Output**: Request structured output from certain LLM providers based on a provided JSON schema. 25 | - **Speech to text**: Transcribe audio to text 26 | - **Text to speech**: Transcribe text to audio 27 | - **Memory**: Store and retrieve conversation history with sliding window (soon others) and shared memory support 28 | 29 | ## Use any LLM backend on your project 30 | 31 | Simply add **LLM** to your `Cargo.toml`: 32 | 33 | ```toml 34 | [dependencies] 35 | llm = { version = "1.2.4", features = ["openai", "anthropic", "ollama", "deepseek", "xai", "phind", "google", "groq", "Elevenlabs"] } 36 | ``` 37 | 38 | ## Use any LLM on cli 39 | 40 | LLM includes a command-line tool for easily interacting with different LLM models. You can install it with: ```cargo install llm``` 41 | 42 | - Use `llm` to start an interactive chat session 43 | - Use `llm openai:gpt-4o` to start an interactive chat session with provider:model 44 | - Use `llm set OPENAI_API_KEY your_key` to configure your API key 45 | - Use `llm default openai:gpt-4` to set a default provider 46 | - Use `echo "Hello World" | llm` to pipe 47 | - Use `llm --provider openai --model gpt-4 --temperature 0.7` for advanced options 48 | 49 | ## Serving any LLM backend as a REST API 50 | - Use standard messages format 51 | - Use step chains to chain multiple LLM backends together 52 | - Expose the chain through a REST API with openai standard format 53 | 54 | ```shell 55 | [dependencies] 56 | llm = { version = "1.2.4", features = ["openai", "anthropic", "ollama", "deepseek", "xai", "phind", "google", "groq", "api", "elevenlabs"] } 57 | ``` 58 | 59 | More details in the [`api_example`](examples/api_example.rs) 60 | 61 | ## More examples 62 | 63 | | Name | Description | 64 | |------|-------------| 65 | | [`anthropic_example`](examples/anthropic_example.rs) | Demonstrates integration with Anthropic's Claude model for chat completion | 66 | | [`anthropic_streaming_example`](examples/anthropic_streaming_example.rs) | Anthropic streaming chat example demonstrating real-time token generation | 67 | | [`chain_example`](examples/chain_example.rs) | Shows how to create multi-step prompt chains for exploring programming language features | 68 | | [`deepseek_example`](examples/deepseek_example.rs) | Basic DeepSeek chat completion example with deepseek-chat models | 69 | | [`embedding_example`](examples/embedding_example.rs) | Basic embedding example with OpenAI's API | 70 | | [`multi_backend_example`](examples/multi_backend_example.rs) | Illustrates chaining multiple LLM backends (OpenAI, Anthropic, DeepSeek) together in a single workflow | 71 | | [`ollama_example`](examples/ollama_example.rs) | Example of using local LLMs through Ollama integration | 72 | | [`openai_example`](examples/openai_example.rs) | Basic OpenAI chat completion example with GPT models | 73 | | [`openai_streaming_example`](examples/openai_streaming_example.rs) | OpenAI streaming chat example demonstrating real-time token generation | 74 | | [`phind_example`](examples/phind_example.rs) | Basic Phind chat completion example with Phind-70B model | 75 | | [`validator_example`](examples/validator_example.rs) | Basic validator example with Anthropic's Claude model | 76 | | [`xai_example`](examples/xai_example.rs) | Basic xAI chat completion example with Grok models | 77 | | [`xai_streaming_example`](examples/xai_streaming_example.rs) | X.AI streaming chat example demonstrating real-time token generation | 78 | | [`evaluation_example`](examples/evaluation_example.rs) | Basic evaluation example with Anthropic, Phind and DeepSeek | 79 | | [`evaluator_parallel_example`](examples/evaluator_parallel_example.rs) | Evaluate multiple LLM providers in parallel | 80 | | [`google_example`](examples/google_example.rs) | Basic Google Gemini chat completion example with Gemini models | 81 | | [`google_streaming_example`](examples/google_streaming_example.rs) | Google streaming chat example demonstrating real-time token generation | 82 | | [`google_pdf`](examples/google_pdf.rs) | Google Gemini chat with PDF attachment | 83 | | [`google_image`](examples/google_image.rs) | Google Gemini chat with PDF attachment | 84 | | [`google_embedding_example`](examples/google_embedding_example.rs) | Basic Google Gemini embedding example with Gemini models | 85 | | [`tool_calling_example`](examples/tool_calling_example.rs) | Basic tool calling example with OpenAI | 86 | | [`google_tool_calling_example`](examples/google_tool_calling_example.rs) | Google Gemini function calling example with complex JSON schema for meeting scheduling | 87 | | [`json_schema_nested_example`](examples/json_schema_nested_example.rs) | Advanced example demonstrating deeply nested JSON schemas with arrays of objects and complex data structures | 88 | | [`tool_json_schema_cycle_example`](examples/tool_json_schema_cycle_example.rs) | Complete tool calling cycle with JSON schema validation and structured responses | 89 | | [`unified_tool_calling_example`](examples/unified_tool_calling_example.rs) | Unified tool calling with selectable provider - demonstrates multi-turn tool use and tool choice | 90 | | [`deepclaude_pipeline_example`](examples/deepclaude_pipeline_example.rs) | Basic deepclaude pipeline example with DeepSeek and Claude | 91 | | [`api_example`](examples/api_example.rs) | Basic API (openai standard format) example with OpenAI, Anthropic, DeepSeek and Groq | 92 | | [`api_deepclaude_example`](examples/api_deepclaude_example.rs) | Basic API (openai standard format) example with DeepSeek and Claude | 93 | | [`anthropic_vision_example`](examples/anthropic_vision_example.rs) | Basic anthropic vision example with Anthropic | 94 | | [`openai_vision_example`](examples/openai_vision_example.rs) | Basic openai vision example with OpenAI | 95 | | [`openai_reasoning_example`](examples/openai_reasoning_example.rs) | Basic openai reasoning example with OpenAI | 96 | | [`anthropic_thinking_example`](examples/anthropic_thinking_example.rs) | Anthropic reasoning example | 97 | | [`elevenlabs_stt_example`](examples/elevenlabs_stt_example.rs) | Speech-to-text transcription example using ElevenLabs | 98 | | [`elevenlabs_tts_example`](examples/elevenlabs_tts_example.rs) | Text-to-speech example using ElevenLabs | 99 | | [`openai_stt_example`](examples/openai_stt_example.rs) | Speech-to-text transcription example using OpenAI | 100 | | [`openai_tts_example`](examples/openai_tts_example.rs) | Text-to-speech example using OpenAI | 101 | | [`tts_rodio_example`](examples/tts_rodio_example.rs) | Text-to-speech with rodio example using OpenAI | 102 | | [`chain_audio_text_example`](examples/chain_audio_text_example.rs) | Example demonstrating a multi-step chain combining speech-to-text and text processing | 103 | | [`xai_search_chain_tts_example`](examples/xai_search_chain_tts_example.rs) | Example demonstrating a multi-step chain combining XAI search, OpenAI summarization, and ElevenLabs text-to-speech with Rodio playback | 104 | | [`xai_search_example`](examples/xai_search_example.rs) | Example demonstrating X.AI search functionality with search modes, date ranges, and source filtering | 105 | | [`memory_example`](examples/memory_example.rs) | Automatic memory integration - LLM remembers conversation context across calls | 106 | | [`memory_share_example`](examples/memory_share_example.rs) | Example demonstrating shared memory between multiple LLM providers | 107 | | [`trim_strategy_example`](examples/trim_strategy_example.rs) | Example demonstrating memory trimming strategies with automatic summarization | -------------------------------------------------------------------------------- /examples/anthropic_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for Anthropic integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get Anthropic API key from environment variable or use test key as fallback 10 | let api_key: String = std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthro-key".into()); 11 | 12 | // Initialize and configure the LLM client 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::Anthropic) // Use Anthropic (Claude) as the LLM provider 15 | .api_key(api_key) // Set the API key 16 | .model("claude-3-7-sonnet-20250219") // Use Claude Instant model 17 | .max_tokens(512) // Limit response length 18 | .temperature(0.7) // Control response randomness (0.0-1.0) 19 | // Uncomment to set system prompt: 20 | // .system("You are a helpful assistant specialized in concurrency.") 21 | .build() 22 | .expect("Failed to build LLM (Anthropic)"); 23 | 24 | // Prepare conversation history with example message about Rust concurrency 25 | let messages = vec![ChatMessage::user() 26 | .content("Tell me something about Rust concurrency") 27 | .build()]; 28 | 29 | // Send chat request and handle the response 30 | match llm.chat(&messages).await { 31 | Ok(text) => println!("Anthropic chat response:\n{}", text), 32 | Err(e) => eprintln!("Chat error: {}", e), 33 | } 34 | 35 | Ok(()) 36 | } 37 | -------------------------------------------------------------------------------- /examples/anthropic_streaming_example.rs: -------------------------------------------------------------------------------- 1 | // Anthropic streaming chat example demonstrating real-time token generation 2 | use futures::StreamExt; 3 | use llm::{ 4 | builder::{LLMBackend, LLMBuilder}, 5 | chat::ChatMessage, 6 | }; 7 | use std::io::{self, Write}; 8 | 9 | #[tokio::main] 10 | async fn main() -> Result<(), Box> { 11 | // Get Anthropic API key from environment variable or use test key as fallback 12 | let api_key = std::env::var("ANTHROPIC_API_KEY").unwrap_or("sk-TESTKEY".into()); 13 | 14 | // Initialize and configure the LLM client with streaming enabled 15 | let llm = LLMBuilder::new() 16 | .backend(LLMBackend::Anthropic) 17 | .api_key(api_key) 18 | .model("claude-3-haiku-20240307") 19 | .max_tokens(1000) 20 | .temperature(0.7) 21 | .stream(true) // Enable streaming responses 22 | .build() 23 | .expect("Failed to build LLM (Anthropic)"); 24 | 25 | // Prepare conversation with a prompt that will generate a longer response 26 | let messages = vec![ 27 | ChatMessage::user() 28 | .content("Write a long story about a robot learning to paint. Make it creative and engaging.") 29 | .build(), 30 | ]; 31 | 32 | println!("Starting streaming chat with Anthropic...\n"); 33 | 34 | match llm.chat_stream(&messages).await { 35 | Ok(mut stream) => { 36 | let stdout = io::stdout(); 37 | let mut handle = stdout.lock(); 38 | 39 | while let Some(Ok(token)) = stream.next().await { 40 | handle.write_all(token.as_bytes()).unwrap(); 41 | handle.flush().unwrap(); 42 | } 43 | println!("\n\nStreaming completed."); 44 | } 45 | Err(e) => eprintln!("Chat error: {}", e), 46 | } 47 | 48 | Ok(()) 49 | } -------------------------------------------------------------------------------- /examples/anthropic_thinking_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for Anthropic integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get Anthropic API key from environment variable or use test key as fallback 10 | let api_key: String = std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthro-key".into()); 11 | 12 | // Initialize and configure the LLM client 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::Anthropic) // Use Anthropic (Claude) as the LLM provider 15 | .api_key(api_key) // Set the API key 16 | .model("claude-3-7-sonnet-20250219") // Use Claude Instant model 17 | .max_tokens(1500) // Limit response length 18 | .temperature(1.0) // Control response randomness (0.0-1.0) 19 | .reasoning(true) 20 | .reasoning_budget_tokens(1024) 21 | // Uncomment to set system prompt: 22 | // .system("You are a helpful assistant specialized in concurrency.") 23 | .build() 24 | .expect("Failed to build LLM (Anthropic)"); 25 | 26 | // Prepare conversation history with example message about Rust concurrency 27 | let messages = vec![ChatMessage::user() 28 | .content("How much r in strawberry?") 29 | .build()]; 30 | 31 | // Send chat request and handle the response 32 | match llm.chat(&messages).await { 33 | Ok(text) => { 34 | if let Some(thinking) = text.thinking() { 35 | println!("Thinking: {}", thinking); 36 | } 37 | if let Some(text) = text.text() { 38 | println!("Text: {}", text); 39 | } 40 | } 41 | Err(e) => eprintln!("Chat error: {}", e), 42 | } 43 | 44 | Ok(()) 45 | } 46 | -------------------------------------------------------------------------------- /examples/anthropic_vision_example.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | 3 | // Import required modules from the LLM library for Anthropic integration 4 | use llm::{ 5 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 6 | chat::{ChatMessage, ImageMime}, // Chat-related structures 7 | }; 8 | 9 | #[tokio::main] 10 | async fn main() -> Result<(), Box> { 11 | // Get Anthropic API key from environment variable or use test key as fallback 12 | let api_key = std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthro-key".into()); 13 | 14 | // Initialize and configure the LLM client 15 | let llm = LLMBuilder::new() 16 | .backend(LLMBackend::Anthropic) // Use Anthropic (Claude) as the LLM provider 17 | .api_key(api_key) // Set the API key 18 | .model("claude-3-5-sonnet-20240620") // Use Claude Instant model 19 | .max_tokens(512) // Limit response length 20 | .temperature(0.7) // Control response randomness (0.0-1.0) 21 | // Uncomment to set system prompt: 22 | // .system("You are a helpful assistant specialized in concurrency.") 23 | .build() 24 | .expect("Failed to build LLM (Anthropic)"); 25 | 26 | let content = fs::read("./examples/image001.jpg").expect("The dummy.pdf file should exist"); 27 | 28 | // Prepare conversation history with example message about Rust concurrency 29 | let messages = vec![ 30 | ChatMessage::user() 31 | .content("What is in this image?") 32 | .build(), 33 | ChatMessage::user().image(ImageMime::JPEG, content).build(), 34 | ]; 35 | 36 | // Send chat request and handle the response 37 | match llm.chat(&messages).await { 38 | Ok(text) => println!("Anthropic chat response:\n{}", text), 39 | Err(e) => eprintln!("Chat error: {}", e), 40 | } 41 | 42 | Ok(()) 43 | } 44 | -------------------------------------------------------------------------------- /examples/api_deepclaude_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating how to serve multiple LLM backends through a REST API 2 | //! 3 | //! This example shows how to chain multiple LLM providers together to: 4 | //! 1. Use Groq to generate creative system identification approaches 5 | //! 2. Use Claude to convert those ideas into executable commands 6 | //! 3. Expose the chain through a REST API 7 | //! 8 | //! # Example Request 9 | //! ```json 10 | //! POST http://127.0.0.1:3000/v1/chat/completions 11 | //! { 12 | //! "steps": [ 13 | //! { 14 | //! "provider_id": "groq", 15 | //! "id": "thinking", 16 | //! "template": "Find an original way to identify the system without using default commands. I want a one-line command.", 17 | //! "response_transform": "extract_think", 18 | //! "temperature": 0.7 19 | //! }, 20 | //! { 21 | //! "provider_id": "anthropic", 22 | //! "id": "step2", 23 | //! "template": "Take the following command reasoning and generate a command to execute it on the system: {{thinking}}\n\nGenerate a command to execute it on the system. return only the command.", 24 | //! "max_tokens": 5000 25 | //! } 26 | //! ] 27 | //! } 28 | //! ``` 29 | 30 | use llm::{ 31 | builder::{LLMBackend, LLMBuilder}, 32 | chain::LLMRegistryBuilder, 33 | }; 34 | 35 | #[tokio::main] 36 | async fn main() -> Result<(), Box> { 37 | // Initialize Anthropic backend with API key and model settings 38 | let anthro_llm = LLMBuilder::new() 39 | .backend(LLMBackend::Anthropic) 40 | .api_key(std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthro-key".into())) 41 | .model("claude-3-5-sonnet-20240620") 42 | .build()?; 43 | 44 | // Initialize Groq backend with API key and model settings 45 | let groq_llm = LLMBuilder::new() 46 | .backend(LLMBackend::Groq) 47 | .api_key(std::env::var("GROQ_API_KEY").unwrap_or("gsk-YOUR_API_KEY".into())) 48 | .model("deepseek-r1-distill-llama-70b") 49 | .build()?; 50 | 51 | // Create registry to manage multiple backends 52 | let registry = LLMRegistryBuilder::new() 53 | .register("anthropic", anthro_llm) 54 | .register("groq", groq_llm) 55 | .build(); 56 | 57 | // Start REST API server on localhost port 3000 58 | registry.serve("127.0.0.1:3000").await?; 59 | Ok(()) 60 | } 61 | -------------------------------------------------------------------------------- /examples/api_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating how to serve multiple LLM backends through a REST API 2 | //! 3 | //! This example shows how to chain multiple LLM providers together to: 4 | //! 1. Use Groq to perform initial calculation 5 | //! 2. Use Claude to provide analysis and commentary 6 | //! 3. Use GPT to improve and format the results 7 | //! 4. Expose the chain through a REST API 8 | //! 9 | //! # Example Request 10 | //! ```json 11 | //! POST http://127.0.0.1:3000/v1/chat/completions 12 | //! { 13 | //! "model": "groq:deepseek-r1-distill-llama-70b", 14 | //! "messages": [ 15 | //! {"role": "user", "content": "calcule 1x20"} 16 | //! ], 17 | //! "steps": [ 18 | //! { 19 | //! "provider_id": "anthropic", 20 | //! "id": "step1", 21 | //! "template": "Analyze and comment on this calculation: {{initial}}", 22 | //! "temperature": 0.7 23 | //! }, 24 | //! { 25 | //! "provider_id": "openai", 26 | //! "id": "step2", 27 | //! "template": "Improve and expand upon this mathematical analysis: {{step1}}", 28 | //! "max_tokens": 500 29 | //! }, 30 | //! { 31 | //! "provider_id": "openai", 32 | //! "id": "step3", 33 | //! "template": "Format the following into a clear report:\nCalculation: {{initial}}\nAnalysis: {{step1}}\nExpanded Analysis: {{step2}}" 34 | //! } 35 | //! ] 36 | //! } 37 | //! ``` 38 | 39 | use llm::{ 40 | builder::{LLMBackend, LLMBuilder}, 41 | chain::LLMRegistryBuilder, 42 | }; 43 | 44 | #[tokio::main] 45 | async fn main() -> Result<(), Box> { 46 | // Initialize OpenAI backend with API key and model settings 47 | let openai_llm = LLMBuilder::new() 48 | .backend(LLMBackend::OpenAI) 49 | .api_key(std::env::var("OPENAI_API_KEY").unwrap_or("sk-OPENAI".into())) 50 | .model("gpt-4") 51 | .build()?; 52 | 53 | // Initialize Anthropic backend with API key and model settings 54 | let anthro_llm = LLMBuilder::new() 55 | .backend(LLMBackend::Anthropic) 56 | .api_key(std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthro-key".into())) 57 | .model("claude-3-5-sonnet-20240620") 58 | .build()?; 59 | 60 | // Initialize Groq backend with API key and model settings 61 | let groq_llm = LLMBuilder::new() 62 | .backend(LLMBackend::Groq) 63 | .api_key(std::env::var("GROQ_API_KEY").unwrap_or("gsk-YOUR_API_KEY".into())) 64 | .model("deepseek-r1-distill-llama-70b") 65 | .build()?; 66 | 67 | // Create registry to manage multiple backends 68 | let registry = LLMRegistryBuilder::new() 69 | .register("openai", openai_llm) 70 | .register("anthropic", anthro_llm) 71 | .register("groq", groq_llm) 72 | .build(); 73 | 74 | // Start REST API server on localhost port 3000 75 | registry.serve("127.0.0.1:3000").await?; 76 | Ok(()) 77 | } 78 | -------------------------------------------------------------------------------- /examples/azure_openai_embedding_example.rs: -------------------------------------------------------------------------------- 1 | // Import required builder types from llm 2 | use llm::builder::{LLMBackend, LLMBuilder}; 3 | 4 | /// Example demonstrating how to generate embeddings using Google's API 5 | /// 6 | /// This example shows how to: 7 | /// - Configure a Google LLM provider 8 | /// - Generate embeddings for text input 9 | /// - Access and display the resulting embedding vector 10 | #[tokio::main] 11 | async fn main() -> Result<(), Box> { 12 | // Initialize the LLM builder with Google configuration 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::AzureOpenAI) 15 | .base_url("your base url here") 16 | .deployment_id("text-embedding-3-large") 17 | // Get API key from environment variable or use test key 18 | .api_key(std::env::var("AZURE_OPENAI_API_KEY").unwrap_or("your api key here".to_owned())) 19 | .api_version("2024-12-01-preview") 20 | // Use Azure OpenAI text embedding model 21 | .model("text-embedding-3-large") 22 | .embedding_dimensions(256) 23 | .build()?; 24 | 25 | // Generate embedding vector for sample text 26 | let vector = llm.embed(vec!["Hello world!".to_string()]).await?; 27 | 28 | // Print embedding statistics and data 29 | println!("Data: {:?}", &vector); 30 | 31 | Ok(()) 32 | } 33 | -------------------------------------------------------------------------------- /examples/azure_openai_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for OpenAI integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get OpenAI API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("AZURE_OPENAI_API_KEY").unwrap_or("your api key here".into()); 11 | let api_version = 12 | std::env::var("AZURE_OPENAI_API_VERSION").unwrap_or("your api version here".into()); 13 | let endpoint = 14 | std::env::var("AZURE_OPENAI_API_ENDPOINT").unwrap_or("your api endpoint here".into()); 15 | 16 | let deployment_id = 17 | std::env::var("AZURE_OPENAI_DEPLOYMENT").unwrap_or("gpt-4o-mini".to_owned()); 18 | 19 | // Initialize and configure the LLM client 20 | let llm = LLMBuilder::new() 21 | .backend(LLMBackend::AzureOpenAI) // Use OpenAI as the LLM provider 22 | .base_url(endpoint) 23 | .api_key(api_key) // Set the API key 24 | .api_version(api_version) 25 | .deployment_id(deployment_id) 26 | .model("gpt-4o-mini") // Use GPT-3.5 Turbo model 27 | .max_tokens(512) // Limit response length 28 | .temperature(0.7) // Control response randomness (0.0-1.0) 29 | .stream(false) // Disable streaming responses 30 | .build() 31 | .expect("Failed to build LLM (Azure OpenAI)"); 32 | 33 | // Prepare conversation history with example messages 34 | let messages = vec![ 35 | ChatMessage::user() 36 | .content("Tell me that you love cats") 37 | .build(), 38 | ChatMessage::assistant() 39 | .content("I am an assistant, I cannot love cats but I can love dogs") 40 | .build(), 41 | ChatMessage::user() 42 | .content("Tell me that you love dogs in 2000 chars") 43 | .build(), 44 | ]; 45 | 46 | // Send chat request and handle the response 47 | match llm.chat(&messages).await { 48 | Ok(text) => println!("Chat response:\n{}", text), 49 | Err(e) => eprintln!("Chat error: {}", e), 50 | } 51 | 52 | Ok(()) 53 | } 54 | -------------------------------------------------------------------------------- /examples/azure_openai_tool_calling_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for OpenAI integration 2 | use llm::{ 3 | builder::{FunctionBuilder, LLMBackend, LLMBuilder, ParamBuilder}, 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get OpenAI API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("AZURE_OPENAI_API_KEY").unwrap_or("".into()); 11 | let api_version = 12 | std::env::var("AZURE_OPENAI_API_VERSION").unwrap_or("2025-01-01-preview".into()); 13 | let endpoint = std::env::var("AZURE_OPENAI_API_ENDPOINT").unwrap_or("".into()); 14 | 15 | // Initialize and configure the LLM client 16 | let llm = LLMBuilder::new() 17 | .backend(LLMBackend::AzureOpenAI) // Use OpenAI as the LLM provider 18 | .base_url(endpoint) 19 | .api_key(api_key) // Set the API key 20 | .api_version(api_version) // Set the API key 21 | .deployment_id("gpt-4o-mini") 22 | .model("gpt-4o-mini") // Use GPT-4o-mini model 23 | .max_tokens(512) // Limit response length 24 | .temperature(0.7) // Control response randomness (0.0-1.0) 25 | .stream(false) // Disable streaming responses 26 | .function( 27 | FunctionBuilder::new("weather_function") 28 | .description("Use this tool to get the weather in a specific city") 29 | .param( 30 | ParamBuilder::new("url") 31 | .type_of("string") 32 | .description("The url to get the weather from for the city"), 33 | ) 34 | .required(vec!["url".to_string()]), 35 | ) 36 | .build() 37 | .expect("Failed to build LLM"); 38 | 39 | // Prepare conversation history with example messages 40 | let messages = vec![ChatMessage::user().content("You are a weather assistant. What is the weather in Tokyo? Use the tools that you have available").build()]; 41 | 42 | // Send chat request and handle the response 43 | // this returns the response as a string. The tool call is also returned as a serialized string. We can deserialize if needed. 44 | match llm.chat_with_tools(&messages, llm.tools()).await { 45 | Ok(text) => println!("Chat response:\n{}", text), 46 | Err(e) => eprintln!("Chat error: {}", e), 47 | } 48 | 49 | Ok(()) 50 | } 51 | -------------------------------------------------------------------------------- /examples/chain_audio_text_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating a multi-step chain combining speech-to-text and text processing 2 | //! 3 | //! This example shows how to: 4 | //! 1. Initialize multiple LLM backends (OpenAI and ElevenLabs) 5 | //! 2. Create a registry to manage the backends 6 | //! 3. Build a chain that transcribes audio and processes the text 7 | //! 4. Execute the chain and display results 8 | 9 | use llm::{ 10 | builder::{LLMBackend, LLMBuilder}, 11 | chain::{LLMRegistryBuilder, MultiChainStepBuilder, MultiChainStepMode, MultiPromptChain}, 12 | }; 13 | 14 | #[tokio::main] 15 | async fn main() -> Result<(), Box> { 16 | // Initialize OpenAI backend with API key and model settings 17 | let openai_llm = LLMBuilder::new() 18 | .backend(LLMBackend::OpenAI) 19 | .api_key(std::env::var("OPENAI_API_KEY").unwrap_or("sk-OPENAI".into())) 20 | .model("gpt-4o") 21 | .build()?; 22 | 23 | // Initialize ElevenLabs backend for speech-to-text 24 | let elevenlabs_llm = LLMBuilder::new() 25 | .backend(LLMBackend::ElevenLabs) 26 | .api_key(std::env::var("ELEVENLABS_API_KEY").unwrap_or("elevenlabs-key".into())) 27 | .model("scribe_v1") 28 | .build()?; 29 | 30 | // Create registry to manage multiple backends 31 | let registry = LLMRegistryBuilder::new() 32 | .register("openai", openai_llm) 33 | .register("elevenlabs", elevenlabs_llm) 34 | .build(); 35 | 36 | // Build multi-step chain using different backends 37 | let chain_res = MultiPromptChain::new(®istry) 38 | // Step 1: Transcribe audio file using ElevenLabs 39 | .step( 40 | MultiChainStepBuilder::new(MultiChainStepMode::SpeechToText) 41 | .provider_id("elevenlabs") 42 | .id("transcription") 43 | .template("test-stt.m4a") 44 | .build()? 45 | ) 46 | // Step 2: Process transcription into JSON format using OpenAI 47 | .step( 48 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 49 | .provider_id("openai") 50 | .id("jsonify") 51 | .template("Here is the transcription: {{transcription}}\n\nPlease convert the transcription text into a JSON object with the following fields: 'text', 'words' (array of objects with 'text', 'start', 'end'), 'language_code', 'language_probability'. The JSON should be formatted as a string.") 52 | .build()? 53 | ) 54 | .run().await?; 55 | 56 | // Display results from all steps 57 | println!("Results: {:?}", chain_res); 58 | Ok(()) 59 | } 60 | -------------------------------------------------------------------------------- /examples/chain_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating a multi-step prompt chain for exploring programming language features 2 | //! 3 | //! This example shows how to: 4 | //! 1. Select a programming language topic 5 | //! 2. Get advanced features for that language 6 | //! 3. Generate a code example for one feature 7 | //! 4. Get a detailed explanation of the example 8 | 9 | use llm::{ 10 | builder::{LLMBackend, LLMBuilder}, 11 | chain::{ChainStepBuilder, ChainStepMode, PromptChain}, 12 | }; 13 | 14 | #[tokio::main] 15 | async fn main() -> Result<(), Box> { 16 | // Initialize the LLM with OpenAI backend and configuration 17 | let llm = LLMBuilder::new() 18 | .backend(LLMBackend::OpenAI) 19 | .api_key(std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into())) 20 | .model("gpt-4o") 21 | .max_tokens(200) 22 | .temperature(0.7) 23 | .build()?; 24 | 25 | // Create and execute a 4-step prompt chain 26 | let chain_result = PromptChain::new(&*llm) 27 | // Step 1: Choose a programming language topic 28 | .step( 29 | ChainStepBuilder::new("topic", "Suggest an interesting technical topic to explore among: Rust, Python, JavaScript, Go. Answer with a single word only.", ChainStepMode::Chat) 30 | .temperature(0.8) // Higher temperature for more variety in topic selection 31 | .build() 32 | ) 33 | // Step 2: Get advanced features for the chosen language 34 | .step( 35 | ChainStepBuilder::new("features", "List 3 advanced features of {{topic}} that few developers know about. Format: one feature per line.", ChainStepMode::Chat) 36 | .build() 37 | ) 38 | // Step 3: Generate a code example for one feature 39 | .step( 40 | ChainStepBuilder::new("example", "Choose one of the features listed in {{features}} and show a commented code example that illustrates it.", ChainStepMode::Chat) 41 | .build() 42 | ) 43 | // Step 4: Get detailed explanation of the code example 44 | .step( 45 | ChainStepBuilder::new("explanation", "Explain in detail how the code example {{example}} works and why this feature is useful.", ChainStepMode::Chat) 46 | .max_tokens(500) // Allow longer response for detailed explanation 47 | .build() 48 | ) 49 | .run().await?; 50 | 51 | // Display the results from all chain steps 52 | println!("Chain results: {:?}", chain_result); 53 | 54 | Ok(()) 55 | } 56 | -------------------------------------------------------------------------------- /examples/chain_logging_example.rs: -------------------------------------------------------------------------------- 1 | //! Multi-backend chain example with logging enabled. 2 | //! 3 | //! This shows how to: 4 | //! 1. Initialise a concrete logger (`env_logger`) when the `logging` feature is 5 | //! active so that the crate’s `trace! / debug!` statements are printed. 6 | //! 2. Register two different back-ends (OpenAI and Anthropic). 7 | //! 3. Execute a two-step `MultiPromptChain` where the second step consumes the 8 | //! output of the first. 9 | //! 10 | //! To run: 11 | //! ```bash 12 | //! RUST_LOG=llm=trace \ 13 | //! cargo run --example chain_logging_example --features "logging cli" 14 | //! ``` 15 | //! The `cli` feature transitively enables all back-ends (`full` feature). 16 | 17 | use llm::{ 18 | builder::{LLMBackend, LLMBuilder}, 19 | chain::{LLMRegistryBuilder, MultiChainStepBuilder, MultiChainStepMode, MultiPromptChain}, 20 | }; 21 | 22 | #[tokio::main] 23 | async fn main() -> Result<(), Box> { 24 | #[cfg(feature = "logging")] 25 | env_logger::init(); 26 | 27 | let openai_llm = LLMBuilder::new() 28 | .backend(LLMBackend::OpenAI) 29 | .api_key(std::env::var("OPENAI_API_KEY")?) 30 | .model("gpt-4o") 31 | .build()?; 32 | 33 | let anthropic_llm = LLMBuilder::new() 34 | .backend(LLMBackend::Anthropic) 35 | .api_key(std::env::var("ANTHROPIC_API_KEY")?) 36 | .model("claude-3-opus-20240229") 37 | .build()?; 38 | 39 | let registry = LLMRegistryBuilder::new() 40 | .register("openai", openai_llm) 41 | .register("anthropic", anthropic_llm) 42 | .build(); 43 | 44 | let results = MultiPromptChain::new(®istry) 45 | .step( 46 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 47 | .provider_id("openai") 48 | .id("summary") 49 | .template("Please summarise the following text in one concise sentence:\n\nRust is a multi-paradigm, general-purpose programming language that emphasises performance, type safety and concurrency.") 50 | .temperature(0.3) 51 | .build()?, 52 | ) 53 | .step( 54 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 55 | .provider_id("anthropic") 56 | .id("evaluation") 57 | .template("Here is a summary: {{summary}}\n\nPlease critique its accuracy and completeness in less than 100 words.") 58 | .temperature(0.7) 59 | .build()?, 60 | ) 61 | .run() 62 | .await?; 63 | 64 | println!("\nChain results:\n{results:#?}"); 65 | 66 | Ok(()) 67 | } 68 | -------------------------------------------------------------------------------- /examples/deepclaude_pipeline_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating a multi-LLM pipeline for system identification 2 | //! 3 | //! This example shows how to: 4 | //! 1. Set up a pipeline using Groq and Claude models 5 | //! 2. Use Groq for creative system identification approaches 6 | //! 3. Use Claude to convert ideas into concrete commands 7 | //! 4. Transform and filter responses between steps 8 | //! 5. Handle results in a type-safe way with error handling 9 | 10 | use llm::{ 11 | builder::{LLMBackend, LLMBuilder}, 12 | chain::{LLMRegistryBuilder, MultiChainStepBuilder, MultiChainStepMode, MultiPromptChain}, 13 | }; 14 | 15 | #[tokio::main] 16 | async fn main() -> Result<(), Box> { 17 | // Initialize Claude model with API key and latest model version 18 | let anthro_llm = LLMBuilder::new() 19 | .backend(LLMBackend::Anthropic) 20 | .api_key(std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthro-key".into())) 21 | .model("claude-3-5-sonnet-20240620") 22 | .build()?; 23 | 24 | // Initialize Groq model with deepseek for creative thinking 25 | let groq_llm = LLMBuilder::new() 26 | .backend(LLMBackend::Groq) 27 | .api_key(std::env::var("GROQ_API_KEY").unwrap_or("sk-TESTKEY".into())) 28 | .model("deepseek-r1-distill-llama-70b") 29 | .build()?; 30 | 31 | // Create registry with both models 32 | let registry = LLMRegistryBuilder::new() 33 | .register("anthro", anthro_llm) 34 | .register("groq", groq_llm) 35 | .build(); 36 | 37 | // Build and execute the multi-step chain 38 | let chain_res = MultiPromptChain::new(®istry) 39 | // Step 1: Use Groq to generate creative system identification approaches 40 | .step( 41 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 42 | .provider_id("groq") 43 | .id("thinking") 44 | .template("Find an original way to identify the system without using default commands. I want a one-line command.") 45 | .max_tokens(500) 46 | .top_p(0.9) 47 | // Transform response to extract only content between tags 48 | .response_transform(|resp| { 49 | resp.lines() 50 | .skip_while(|line| !line.contains("")) 51 | .take_while(|line| !line.contains("")) 52 | .map(|line| line.replace("", "").trim().to_string()) 53 | .filter(|line| !line.is_empty()) 54 | .collect::>() 55 | .join("\n") 56 | }) 57 | .build()? 58 | ) 59 | // Step 2: Use Claude to convert the creative approach into a concrete command 60 | .step( 61 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 62 | .provider_id("anthro") 63 | .id("command") 64 | .template("Take the following command reasoning and generate a command to execute it on the system: {{thinking}}\n\nGenerate a command to execute it on the system. return only the command.") 65 | .temperature(0.2) // Low temperature for more deterministic output 66 | .build()? 67 | ) 68 | .run().await?; 69 | 70 | // Display results from both steps 71 | println!("Results: {:?}", chain_res); 72 | 73 | Ok(()) 74 | } 75 | -------------------------------------------------------------------------------- /examples/deepseek_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for DeepSeek integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get DeepSeek API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("DEEPSEEK_API_KEY").unwrap_or("sk-TESTKEY".into()); 11 | 12 | // Initialize and configure the LLM client 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::DeepSeek) // Use DeepSeek as the LLM provider 15 | .system("You are a helpful assistant and you response only with words begin with deepseek_") 16 | .api_key(api_key) // Set the API key 17 | .model("deepseek-reasoner") // Use DeepSeek Chat model 18 | .timeout_seconds(1200) 19 | .temperature(0.7) // Control response randomness (0.0-1.0) 20 | .stream(false) // Disable streaming responses 21 | .build() 22 | .expect("Failed to build LLM (DeepSeek)"); 23 | 24 | // Prepare conversation history with example messages 25 | let messages = vec![ 26 | ChatMessage::user() 27 | .content("Tell me that you love cats") 28 | .build(), 29 | ChatMessage::assistant() 30 | .content("I am an assistant, I cannot love cats but I can love dogs") 31 | .build(), 32 | ChatMessage::user() 33 | .content("Tell me that you love dogs in 2000 chars") 34 | .build(), 35 | ]; 36 | 37 | // Send chat request and handle the response 38 | match llm.chat(&messages).await { 39 | Ok(text) => println!("Chat response:\n{}", text), 40 | Err(e) => eprintln!("Chat error: {}", e), 41 | } 42 | 43 | Ok(()) 44 | } 45 | -------------------------------------------------------------------------------- /examples/dummy.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graniet/llm/3489f0b36e6273fb995e997f0a5d71c4533cef92/examples/dummy.pdf -------------------------------------------------------------------------------- /examples/elevenlabs_stt_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating speech-to-text transcription using ElevenLabs 2 | //! 3 | //! This example shows how to: 4 | //! 1. Initialize the ElevenLabs speech-to-text provider 5 | //! 2. Load an audio file 6 | //! 3. Transcribe the audio content 7 | 8 | use llm::builder::{LLMBackend, LLMBuilder}; 9 | 10 | #[tokio::main] 11 | async fn main() -> Result<(), Box> { 12 | // Get API key from environment variable or use test key 13 | let api_key = std::env::var("ELEVENLABS_API_KEY").unwrap_or("test_key".into()); 14 | 15 | // Initialize ElevenLabs speech-to-text provider 16 | let stt = LLMBuilder::new() 17 | .backend(LLMBackend::ElevenLabs) 18 | .api_key(api_key) 19 | .model("scribe_v1") 20 | .build()?; 21 | 22 | // Read audio file from disk 23 | let audio_bytes = std::fs::read("test-stt.m4a")?; 24 | 25 | // Transcribe audio content 26 | let resp = stt.transcribe(audio_bytes).await?; 27 | 28 | println!("Transcription: {}", resp); 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /examples/elevenlabs_tts_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating text-to-speech synthesis using ElevenLabs 2 | //! 3 | //! This example shows how to: 4 | //! 1. Initialize the ElevenLabs text-to-speech provider 5 | //! 2. Generate speech from text 6 | //! 3. Save the audio output to a file 7 | 8 | use llm::builder::{LLMBackend, LLMBuilder}; 9 | 10 | #[tokio::main] 11 | async fn main() -> Result<(), Box> { 12 | // Get API key from environment variable or use test key 13 | let api_key = std::env::var("ELEVENLABS_API_KEY").unwrap_or("test_key".into()); 14 | 15 | // Initialize ElevenLabs text-to-speech provider 16 | let tts = LLMBuilder::new() 17 | .backend(LLMBackend::ElevenLabs) 18 | .api_key(api_key) 19 | .model("eleven_multilingual_v2") 20 | .voice("JBFqnCBsd6RMkjVDRZzb") 21 | .build()?; 22 | 23 | // Text to convert to speech 24 | let text = "Hello! This is an example of text-to-speech synthesis using ElevenLabs."; 25 | 26 | // Generate speech 27 | let audio_data = tts.speech(text).await?; 28 | 29 | // Save the audio to a file 30 | std::fs::write("output-speech-elevenlabs.mp3", audio_data)?; 31 | 32 | println!("Audio file generated successfully: output-speech-elevenlabs.mp3"); 33 | Ok(()) 34 | } 35 | -------------------------------------------------------------------------------- /examples/embedding_example.rs: -------------------------------------------------------------------------------- 1 | // Import required builder types from llm 2 | use llm::builder::{LLMBackend, LLMBuilder}; 3 | 4 | /// Example demonstrating how to generate embeddings using OpenAI's API 5 | /// 6 | /// This example shows how to: 7 | /// - Configure an OpenAI LLM provider 8 | /// - Generate embeddings for text input 9 | /// - Access and display the resulting embedding vector 10 | #[tokio::main] 11 | async fn main() -> Result<(), Box> { 12 | // Initialize the LLM builder with OpenAI configuration 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::OpenAI) // .backend(LLMBackend::Ollama) or .backend(LLMBackend::XAI) 15 | // Get API key from environment variable or use test key 16 | .api_key(std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".to_string())) 17 | // Use OpenAI's text embedding model 18 | .model("text-embedding-ada-002") // .model("v1") or .model("all-minilm") 19 | // Optional: Uncomment to customize embedding format and dimensions 20 | // .embedding_encoding_format("base64") 21 | // .embedding_dimensions(1536) 22 | .build()?; 23 | 24 | // Generate embedding vector for sample text 25 | let vector = llm.embed(vec!["Hello world!".to_string()]).await?; 26 | 27 | // Print embedding statistics and data 28 | println!("Data: {:?}", &vector); 29 | 30 | Ok(()) 31 | } 32 | -------------------------------------------------------------------------------- /examples/evaluation_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating evaluation and comparison of multiple LLM providers 2 | //! 3 | //! This example shows how to: 4 | //! 1. Initialize multiple LLM providers (Anthropic, Phind, DeepSeek) 5 | //! 2. Configure scoring functions to evaluate responses 6 | //! 3. Send the same prompt to all providers 7 | //! 4. Compare and score the responses 8 | 9 | use llm::{ 10 | builder::{LLMBackend, LLMBuilder}, 11 | chat::ChatMessage, 12 | evaluator::{EvalResult, LLMEvaluator}, 13 | }; 14 | 15 | #[tokio::main] 16 | async fn main() -> Result<(), Box> { 17 | // Initialize Anthropic provider with Claude model 18 | let anthropic = LLMBuilder::new() 19 | .backend(LLMBackend::Anthropic) 20 | .model("claude-3-5-sonnet-20240620") 21 | .api_key(std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthropic-key".into())) 22 | .build()?; 23 | 24 | // Initialize Phind provider specialized for code 25 | let phind = LLMBuilder::new() 26 | .backend(LLMBackend::Phind) 27 | .model("Phind-70B") 28 | .build()?; 29 | 30 | // Initialize DeepSeek provider 31 | let deepseek = LLMBuilder::new() 32 | .backend(LLMBackend::DeepSeek) 33 | .model("deepseek-chat") 34 | .api_key(std::env::var("DEEPSEEK_API_KEY").unwrap_or("deepseek-key".into())) 35 | .build()?; 36 | 37 | // Create evaluator with multiple scoring functions 38 | let evaluator = LLMEvaluator::new(vec![anthropic, phind, deepseek]) 39 | // First scoring function: Evaluate code quality and completeness 40 | .scoring(|response| { 41 | let mut score = 0.0; 42 | 43 | // Check for code blocks and specific Rust features 44 | if response.contains("```") { 45 | score += 1.0; 46 | 47 | if response.contains("```rust") { 48 | score += 2.0; 49 | } 50 | 51 | if response.contains("use actix_web::") { 52 | score += 2.0; 53 | } 54 | if response.contains("async fn") { 55 | score += 1.0; 56 | } 57 | if response.contains("#[derive(") { 58 | score += 1.0; 59 | } 60 | if response.contains("//") { 61 | score += 1.0; 62 | } 63 | } 64 | 65 | score 66 | }) 67 | // Second scoring function: Evaluate explanation quality 68 | .scoring(|response| { 69 | let mut score = 0.0; 70 | 71 | // Check for explanatory phrases 72 | if response.contains("Here's how it works:") || response.contains("Let me explain:") { 73 | score += 2.0; 74 | } 75 | 76 | // Check for examples and practical usage 77 | if response.contains("For example") || response.contains("curl") { 78 | score += 1.5; 79 | } 80 | 81 | // Reward comprehensive responses 82 | let words = response.split_whitespace().count(); 83 | if words > 100 { 84 | score += 1.0; 85 | } 86 | 87 | score 88 | }); 89 | 90 | // Define the evaluation prompt requesting a Rust microservice implementation 91 | let messages = vec![ChatMessage::user() 92 | .content( 93 | "\ 94 | Create a Rust microservice using Actix Web. 95 | It should have at least two routes: 96 | 1) A GET route returning a simple JSON status. 97 | 2) A POST route that accepts JSON data and responds with a success message.\n\ 98 | Include async usage, data structures with `#[derive(Serialize, Deserialize)]`, \ 99 | and show how to run it.\n\ 100 | Provide code blocks, comments, and a brief explanation of how it works.\ 101 | ", 102 | ) 103 | .build()]; 104 | 105 | // Run evaluation across all providers 106 | let results: Vec = evaluator.evaluate_chat(&messages).await?; 107 | 108 | // Display results with scores 109 | for (i, item) in results.iter().enumerate() { 110 | println!("\n=== LLM #{} ===", i); 111 | println!("Score: {:.2}", item.score); 112 | println!("Response:\n{}", item.text); 113 | println!("================\n"); 114 | } 115 | 116 | Ok(()) 117 | } 118 | -------------------------------------------------------------------------------- /examples/evaluator_parallel_example.rs: -------------------------------------------------------------------------------- 1 | use llm::{ 2 | builder::{LLMBackend, LLMBuilder}, 3 | chat::{ChatMessage, ChatRole}, 4 | evaluator::ParallelEvaluator, 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | let openai = LLMBuilder::new() 10 | .backend(LLMBackend::OpenAI) 11 | .api_key(std::env::var("OPENAI_API_KEY").unwrap_or("openai-key".into())) 12 | .model("gpt-4o") 13 | .build()?; 14 | 15 | let anthropic = LLMBuilder::new() 16 | .backend(LLMBackend::Anthropic) 17 | .api_key(std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthropic-key".into())) 18 | .model("claude-3-7-sonnet-20250219") 19 | .build()?; 20 | 21 | let google = LLMBuilder::new() 22 | .backend(LLMBackend::Google) 23 | .api_key(std::env::var("GOOGLE_API_KEY").unwrap_or("google-key".into())) 24 | .model("gemini-2.0-flash-exp") 25 | .build()?; 26 | 27 | let evaluator = ParallelEvaluator::new(vec![ 28 | ("openai".to_string(), openai), 29 | ("anthropic".to_string(), anthropic), 30 | ("google".to_string(), google), 31 | ]) 32 | .scoring(|response| response.len() as f32 * 0.1) 33 | .scoring(|response| { 34 | if response.contains("important") { 35 | 10.0 36 | } else { 37 | 0.0 38 | } 39 | }); 40 | 41 | let messages = vec![ChatMessage { 42 | role: ChatRole::User, 43 | message_type: Default::default(), 44 | content: "Explique-moi la théorie de la relativité d'Einstein".to_string(), 45 | }]; 46 | 47 | let results = evaluator.evaluate_chat_parallel(&messages).await?; 48 | 49 | for result in &results { 50 | println!("Provider: {}", result.provider_id); 51 | println!("Score: {}", result.score); 52 | println!("Time: {}ms", result.time_ms); 53 | // println!("Response: {}", result.text); 54 | println!("---"); 55 | } 56 | 57 | if let Some(best) = evaluator.best_response(&results) { 58 | println!("BEST RESPONSE:"); 59 | println!("Provider: {}", best.provider_id); 60 | println!("Score: {}", best.score); 61 | println!("Time: {}ms", best.time_ms); 62 | // println!("Response: {}", best.text); 63 | } 64 | 65 | Ok(()) 66 | } 67 | -------------------------------------------------------------------------------- /examples/google_embedding_example.rs: -------------------------------------------------------------------------------- 1 | // Import required builder types from llm 2 | use llm::builder::{LLMBackend, LLMBuilder}; 3 | 4 | /// Example demonstrating how to generate embeddings using Google's API 5 | /// 6 | /// This example shows how to: 7 | /// - Configure a Google LLM provider 8 | /// - Generate embeddings for text input 9 | /// - Access and display the resulting embedding vector 10 | #[tokio::main] 11 | async fn main() -> Result<(), Box> { 12 | // Initialize the LLM builder with Google configuration 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::Google) 15 | // Get API key from environment variable or use test key 16 | .api_key(std::env::var("GOOGLE_API_KEY").unwrap_or("YOUR-TEST-KEY".to_string())) 17 | // Use Google's text embedding model 18 | .model("text-embedding-004") 19 | .build()?; 20 | 21 | // Generate embedding vector for sample text 22 | let vector = llm.embed(vec!["Hello world!".to_string()]).await?; 23 | 24 | // Print embedding statistics and data 25 | println!("Data: {:?}", &vector); 26 | 27 | Ok(()) 28 | } 29 | -------------------------------------------------------------------------------- /examples/google_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for Google Gemini integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get Google API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("GOOGLE_API_KEY").unwrap_or("google-key".into()); 11 | 12 | // Initialize and configure the LLM client 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::Google) // Use Google as the LLM provider 15 | .api_key(api_key) // Set the API key 16 | .model("gemini-2.0-flash-exp") // Use Gemini Pro model 17 | .max_tokens(8512) // Limit response length 18 | .temperature(0.7) // Control response randomness (0.0-1.0) 19 | .stream(false) // Disable streaming responses 20 | // Optional: Set system prompt 21 | .system("You are a helpful AI assistant specialized in programming.") 22 | .build() 23 | .expect("Failed to build LLM (Google)"); 24 | 25 | // Prepare conversation history with example messages 26 | let messages = vec![ 27 | ChatMessage::user() 28 | .content("Explain the concept of async/await in Rust") 29 | .build(), 30 | ChatMessage::assistant() 31 | .content("Async/await in Rust is a way to write asynchronous code...") 32 | .build(), 33 | ChatMessage::user() 34 | .content("Can you show me a simple example?") 35 | .build(), 36 | ]; 37 | 38 | // Send chat request and handle the response 39 | match llm.chat(&messages).await { 40 | Ok(text) => println!("Google Gemini response:\n{}", text), 41 | Err(e) => eprintln!("Chat error: {}", e), 42 | } 43 | 44 | Ok(()) 45 | } 46 | -------------------------------------------------------------------------------- /examples/google_image.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | 3 | // Import required modules from the LLM library for Google Gemini integration 4 | use llm::{ 5 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 6 | chat::{ChatMessage, ImageMime}, // Chat-related structures 7 | }; 8 | 9 | #[tokio::main] 10 | async fn main() -> Result<(), Box> { 11 | // Get Google API key from environment variable or use test key as fallback 12 | let api_key = std::env::var("GOOGLE_API_KEY").unwrap_or("google-key".into()); 13 | 14 | // Initialize and configure the LLM client 15 | let llm = LLMBuilder::new() 16 | .backend(LLMBackend::Google) // Use Google as the LLM provider 17 | .api_key(api_key) // Set the API key 18 | .model("gemini-2.0-flash-exp") // Use Gemini Pro model 19 | .max_tokens(8512) // Limit response length 20 | .temperature(0.7) // Control response randomness (0.0-1.0) 21 | .stream(false) // Disable streaming responses 22 | // Optional: Set system prompt 23 | .system("You are a helpful AI assistant specialized in programming.") 24 | .build() 25 | .expect("Failed to build LLM (Google)"); 26 | 27 | let content = fs::read("./examples/image001.jpg").expect("The image001.jpg file should exist"); 28 | 29 | // Prepare conversation history with example messages 30 | let messages = vec![ 31 | ChatMessage::user() 32 | .content("Explain what you see in the image") 33 | .build(), 34 | ChatMessage::user().image(ImageMime::JPEG, content).build(), 35 | ]; 36 | 37 | // Send chat request and handle the response 38 | match llm.chat(&messages).await { 39 | Ok(text) => println!("Google Gemini response:\n{}", text), 40 | Err(e) => eprintln!("Chat error: {}", e), 41 | } 42 | 43 | Ok(()) 44 | } 45 | -------------------------------------------------------------------------------- /examples/google_pdf.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | 3 | // Import required modules from the LLM library for Google Gemini integration 4 | use llm::{ 5 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 6 | chat::ChatMessage, // Chat-related structures 7 | }; 8 | 9 | #[tokio::main] 10 | async fn main() -> Result<(), Box> { 11 | // Get Google API key from environment variable or use test key as fallback 12 | let api_key = std::env::var("GOOGLE_API_KEY").unwrap_or("google-key".into()); 13 | 14 | // Initialize and configure the LLM client 15 | let llm = LLMBuilder::new() 16 | .backend(LLMBackend::Google) // Use Google as the LLM provider 17 | .api_key(api_key) // Set the API key 18 | .model("gemini-2.0-flash-exp") // Use Gemini Pro model 19 | .max_tokens(8512) // Limit response length 20 | .temperature(0.7) // Control response randomness (0.0-1.0) 21 | .stream(false) // Disable streaming responses 22 | // Optional: Set system prompt 23 | .system("You are a helpful AI assistant specialized in programming.") 24 | .build() 25 | .expect("Failed to build LLM (Google)"); 26 | 27 | let content = fs::read("./examples/dummy.pdf").expect("The dummy.pdf file should exist"); 28 | 29 | // Prepare conversation history with example messages 30 | let messages = vec![ 31 | ChatMessage::user() 32 | .content("Explain what is in the PDF") 33 | .build(), 34 | ChatMessage::user().pdf(content).build(), 35 | ]; 36 | 37 | // Send chat request and handle the response 38 | match llm.chat(&messages).await { 39 | Ok(text) => println!("Google Gemini response:\n{}", text), 40 | Err(e) => eprintln!("Chat error: {}", e), 41 | } 42 | 43 | Ok(()) 44 | } 45 | -------------------------------------------------------------------------------- /examples/google_streaming_example.rs: -------------------------------------------------------------------------------- 1 | // Google streaming chat example demonstrating real-time token generation 2 | use futures::StreamExt; 3 | use llm::{ 4 | builder::{LLMBackend, LLMBuilder}, 5 | chat::ChatMessage, 6 | }; 7 | use std::io::{self, Write}; 8 | 9 | #[tokio::main] 10 | async fn main() -> Result<(), Box> { 11 | // Get Google API key from environment variable or use test key as fallback 12 | let api_key = std::env::var("GOOGLE_API_KEY=").unwrap_or("TESTKEY".into()); 13 | 14 | // Initialize and configure the LLM client with streaming enabled 15 | let llm = LLMBuilder::new() 16 | .backend(LLMBackend::Google) 17 | .api_key(api_key) 18 | .model("gemini-2.0-flash") 19 | .max_tokens(1000) 20 | .temperature(0.7) 21 | .stream(true) // Enable streaming responses 22 | .build() 23 | .expect("Failed to build LLM (Google)"); 24 | 25 | // Prepare conversation with a prompt that will generate a longer response 26 | let messages = vec![ 27 | ChatMessage::user() 28 | .content("Write a long story about a robot learning to paint. Make it creative and engaging.") 29 | .build(), 30 | ]; 31 | 32 | println!("Starting streaming chat with Google...\n"); 33 | 34 | match llm.chat_stream(&messages).await { 35 | Ok(mut stream) => { 36 | let stdout = io::stdout(); 37 | let mut handle = stdout.lock(); 38 | 39 | while let Some(Ok(token)) = stream.next().await { 40 | handle.write_all(token.as_bytes()).unwrap(); 41 | handle.flush().unwrap(); 42 | } 43 | println!("\n\nStreaming completed."); 44 | } 45 | Err(e) => eprintln!("Chat error: {}", e), 46 | } 47 | 48 | Ok(()) 49 | } -------------------------------------------------------------------------------- /examples/google_structured_output_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for Google Gemini integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, 4 | chat::{ChatMessage, StructuredOutputFormat}, 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get Google API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("GOOGLE_API_KEY").unwrap_or("google-key".into()); 11 | 12 | // Define a simple JSON schema for structured output 13 | let schema = r#" 14 | { 15 | "name": "student", 16 | "schema": { 17 | "type": "object", 18 | "properties": { 19 | "name": { 20 | "type": "string" 21 | }, 22 | "age": { 23 | "type": "integer" 24 | }, 25 | "is_student": { 26 | "type": "boolean" 27 | } 28 | }, 29 | "required": ["name", "age", "is_student"] 30 | } 31 | } 32 | "#; 33 | let schema: StructuredOutputFormat = serde_json::from_str(schema)?; 34 | 35 | // Initialize and configure the LLM client 36 | let llm = LLMBuilder::new() 37 | .backend(LLMBackend::Google) // Use Google as the LLM provider 38 | .api_key(api_key) // Set the API key 39 | .model("gemini-2.0-flash-exp") // Use Gemini Pro model 40 | .max_tokens(8512) // Limit response length 41 | .temperature(0.7) // Control response randomness (0.0-1.0) 42 | .stream(false) // Disable streaming responses 43 | // Optional: Set system prompt 44 | .system("You are a helpful AI assistant. Please generate a random student using the provided JSON schema.") 45 | // Set JSON schema for structured output 46 | .schema(schema) 47 | .build() 48 | .expect("Failed to build LLM (Google)"); 49 | 50 | // Prepare conversation history with example messages 51 | let messages = vec![ChatMessage::user() 52 | .content("Please generate a random student using the provided JSON schema.") 53 | .build()]; 54 | 55 | // Send chat request and handle the response 56 | match llm.chat(&messages).await { 57 | Ok(text) => println!("Google Gemini response:\n{}", text), 58 | Err(e) => eprintln!("Chat error: {}", e), 59 | } 60 | 61 | Ok(()) 62 | } 63 | -------------------------------------------------------------------------------- /examples/google_tool_calling_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating tool/function calling with Google's Gemini model 2 | //! 3 | //! This example shows how to: 4 | //! - Configure a Google LLM with function calling capabilities 5 | //! - Define a meeting scheduling function with JSON schema 6 | //! - Process function calls and handle responses 7 | //! - Maintain a conversation with tool usage 8 | 9 | use llm::{ 10 | builder::{FunctionBuilder, LLMBackend, LLMBuilder}, 11 | chat::ChatMessage, 12 | FunctionCall, ToolCall, 13 | }; 14 | use serde_json::json; 15 | 16 | #[tokio::main] 17 | async fn main() -> Result<(), Box> { 18 | let api_key = std::env::var("GOOGLE_API_KEY").unwrap_or("test-key".to_string()); 19 | 20 | let llm = LLMBuilder::new() 21 | .backend(LLMBackend::Google) 22 | .api_key(api_key) 23 | .model("gemini-2.0-flash") 24 | .max_tokens(1024) 25 | .temperature(0.7) 26 | .function( 27 | FunctionBuilder::new("schedule_meeting") 28 | .description("Schedules a meeting with specified attendees at a given time and date.") 29 | .json_schema(json!({ 30 | "type": "object", 31 | "properties": { 32 | "attendees": { 33 | "type": "array", 34 | "items": {"type": "string"}, 35 | "description": "List of people attending the meeting." 36 | }, 37 | "date": { 38 | "type": "string", 39 | "description": "Date of the meeting (e.g., '2024-07-29')" 40 | }, 41 | "time": { 42 | "type": "string", 43 | "description": "Time of the meeting (e.g., '15:00')" 44 | }, 45 | "topic": { 46 | "type": "string", 47 | "description": "The subject or topic of the meeting." 48 | } 49 | }, 50 | "required": ["attendees", "date", "time", "topic"] 51 | })) 52 | ) 53 | .build()?; 54 | 55 | let messages = vec![ChatMessage::user() 56 | .content("Schedule a meeting with Bob and Alice for 03/27/2025 at 10:00 AM about the Q3 planning.") 57 | .build()]; 58 | 59 | let response = llm.chat_with_tools(&messages, llm.tools()).await?; 60 | 61 | if let Some(tool_calls) = response.tool_calls() { 62 | println!("Tool calls requested:"); 63 | for call in &tool_calls { 64 | println!("Function: {}", call.function.name); 65 | println!("Arguments: {}", call.function.arguments); 66 | 67 | let result = process_tool_call(call)?; 68 | println!("Result: {}", serde_json::to_string_pretty(&result)?); 69 | } 70 | 71 | let mut conversation = messages; 72 | conversation.push( 73 | ChatMessage::assistant() 74 | .tool_use(tool_calls.clone()) 75 | .build(), 76 | ); 77 | 78 | let tool_results: Vec = tool_calls 79 | .iter() 80 | .map(|call| { 81 | let result = process_tool_call(call).unwrap(); 82 | ToolCall { 83 | id: call.id.clone(), 84 | call_type: "function".to_string(), 85 | function: FunctionCall { 86 | name: call.function.name.clone(), 87 | arguments: serde_json::to_string(&result).unwrap(), 88 | }, 89 | } 90 | }) 91 | .collect(); 92 | 93 | conversation.push(ChatMessage::user().tool_result(tool_results).build()); 94 | 95 | let final_response = llm.chat_with_tools(&conversation, llm.tools()).await?; 96 | println!("\nFinal response: {}", final_response); 97 | } else { 98 | println!("Direct response: {}", response); 99 | } 100 | 101 | Ok(()) 102 | } 103 | 104 | /// Processes a tool call by executing the requested function with provided arguments 105 | /// 106 | /// # Arguments 107 | /// * `tool_call` - The tool call containing function name and arguments to process 108 | /// 109 | /// # Returns 110 | /// * A JSON value containing the result of the function execution 111 | /// 112 | /// # Errors 113 | /// * If the function arguments cannot be parsed as JSON 114 | /// * If an unknown function is called 115 | fn process_tool_call(tool_call: &ToolCall) -> Result> { 116 | match tool_call.function.name.as_str() { 117 | "schedule_meeting" => { 118 | let args: serde_json::Value = serde_json::from_str(&tool_call.function.arguments)?; 119 | 120 | Ok(json!({ 121 | "meeting_id": "mtg_12345", 122 | "status": "scheduled", 123 | "attendees": args["attendees"], 124 | "date": args["date"], 125 | "time": args["time"], 126 | "topic": args["topic"], 127 | "calendar_link": "https://calendar.google.com/event/mtg_12345" 128 | })) 129 | } 130 | _ => Ok(json!({ 131 | "error": "Unknown function", 132 | "function": tool_call.function.name 133 | })), 134 | } 135 | } -------------------------------------------------------------------------------- /examples/groq_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for Groq integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get Groq API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("GROQ_API_KEY").unwrap_or("gsk-TESTKEY".into()); 11 | 12 | // Initialize and configure the LLM client 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::Groq) // Use Groq as the LLM provider 15 | .api_key(api_key) // Set the API key 16 | .model("deepseek-r1-distill-llama-70b") // Use deepseek-r1-distill-llama-70b model 17 | .max_tokens(512) // Limit response length 18 | .temperature(0.7) // Control response randomness (0.0-1.0) 19 | .stream(false) // Disable streaming responses 20 | .build() 21 | .expect("Failed to build LLM (Groq)"); 22 | 23 | // Prepare conversation history with example messages 24 | let messages = vec![ 25 | ChatMessage::user() 26 | .content("Tell me about quantum computing") 27 | .build(), 28 | ChatMessage::assistant() 29 | .content("Quantum computing is a type of computing that uses quantum phenomena...") 30 | .build(), 31 | ChatMessage::user().content("What are qubits?").build(), 32 | ]; 33 | 34 | // Send chat request and handle the response 35 | match llm.chat(&messages).await { 36 | Ok(text) => println!("Chat response:\n{}", text), 37 | Err(e) => eprintln!("Chat error: {}", e), 38 | } 39 | 40 | Ok(()) 41 | } 42 | -------------------------------------------------------------------------------- /examples/image001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graniet/llm/3489f0b36e6273fb995e997f0a5d71c4533cef92/examples/image001.jpg -------------------------------------------------------------------------------- /examples/json_schema_nested_example.rs: -------------------------------------------------------------------------------- 1 | /// Example demonstrating complex nested JSON schema handling with LLM function calls 2 | /// 3 | /// This example shows how to: 4 | /// - Define a complex JSON schema for event creation with nested data structures 5 | /// - Process chat messages with function calls 6 | /// - Handle nested JSON responses 7 | /// - Manage a multi-turn conversation with tool results 8 | use llm::{ 9 | builder::{FunctionBuilder, LLMBackend, LLMBuilder}, 10 | chat::ChatMessage, 11 | FunctionCall, ToolCall, 12 | }; 13 | use serde_json::json; 14 | 15 | #[tokio::main] 16 | async fn main() -> Result<(), Box> { 17 | let api_key = std::env::var("GOOGLE_API_KEY").unwrap_or("test-key".to_string()); 18 | 19 | let llm = LLMBuilder::new() 20 | .backend(LLMBackend::Google) 21 | .api_key(api_key) 22 | .model("gemini-2.0-flash") 23 | .function( 24 | FunctionBuilder::new("create_event") 25 | .description("Creates a complex event with deeply nested data structures") 26 | .json_schema(json!({ 27 | "type": "object", 28 | "properties": { 29 | "event": { 30 | "type": "object", 31 | "properties": { 32 | "title": {"type": "string"}, 33 | "location": { 34 | "type": "object", 35 | "properties": { 36 | "venue": {"type": "string"}, 37 | "address": { 38 | "type": "object", 39 | "properties": { 40 | "street": {"type": "string"}, 41 | "city": {"type": "string"}, 42 | "coordinates": { 43 | "type": "object", 44 | "properties": { 45 | "lat": {"type": "number"}, 46 | "lng": {"type": "number"} 47 | } 48 | } 49 | } 50 | } 51 | } 52 | }, 53 | "attendees": { 54 | "type": "array", 55 | "items": { 56 | "type": "object", 57 | "properties": { 58 | "name": {"type": "string"}, 59 | "email": {"type": "string"}, 60 | "role": {"type": "string"}, 61 | "preferences": { 62 | "type": "object", 63 | "properties": { 64 | "dietary": {"type": "string"}, 65 | "accessibility": {"type": "boolean"}, 66 | "notifications": { 67 | "type": "object", 68 | "properties": { 69 | "email": {"type": "boolean"}, 70 | "sms": {"type": "boolean"}, 71 | "schedule": { 72 | "type": "array", 73 | "items": { 74 | "type": "object", 75 | "properties": { 76 | "time": {"type": "string"}, 77 | "type": {"type": "string"} 78 | } 79 | } 80 | } 81 | } 82 | } 83 | } 84 | } 85 | } 86 | } 87 | } 88 | } 89 | } 90 | }, 91 | "required": ["event"] 92 | })) 93 | ) 94 | .build()?; 95 | 96 | let messages = vec![ChatMessage::user() 97 | .content("Create a team meeting at Google HQ in Mountain View for Alice (alice@corp.com, manager, vegetarian, needs accessibility, wants email and SMS notifications 1 hour before) and Bob (bob@corp.com, developer, no dietary restrictions, only email notifications 30 minutes before)") 98 | .build()]; 99 | 100 | let response = llm.chat_with_tools(&messages, llm.tools()).await?; 101 | 102 | if let Some(tool_calls) = response.tool_calls() { 103 | println!("Complex nested schema handled successfully!"); 104 | for call in &tool_calls { 105 | println!("Function: {}", call.function.name); 106 | let args: serde_json::Value = serde_json::from_str(&call.function.arguments)?; 107 | println!("Nested arguments: {}", serde_json::to_string_pretty(&args)?); 108 | 109 | let result = process_tool_call(call)?; 110 | println!("Result: {}", serde_json::to_string_pretty(&result)?); 111 | } 112 | 113 | let mut conversation = messages; 114 | conversation.push( 115 | ChatMessage::assistant() 116 | .tool_use(tool_calls.clone()) 117 | .build(), 118 | ); 119 | 120 | let tool_results: Vec = tool_calls 121 | .iter() 122 | .map(|call| { 123 | let result = process_tool_call(call).unwrap(); 124 | ToolCall { 125 | id: call.id.clone(), 126 | call_type: "function".to_string(), 127 | function: FunctionCall { 128 | name: call.function.name.clone(), 129 | arguments: serde_json::to_string(&result).unwrap(), 130 | }, 131 | } 132 | }) 133 | .collect(); 134 | 135 | conversation.push(ChatMessage::user().tool_result(tool_results).build()); 136 | 137 | let final_response = llm.chat_with_tools(&conversation, llm.tools()).await?; 138 | println!("\nFinal response: {}", final_response); 139 | } else { 140 | println!("Direct response: {}", response); 141 | } 142 | 143 | Ok(()) 144 | } 145 | 146 | /// Processes a tool call and returns a simulated response 147 | /// 148 | /// # Arguments 149 | /// * `tool_call` - The tool call to process containing function name and arguments 150 | /// 151 | /// # Returns 152 | /// * JSON response containing event details or error message 153 | fn process_tool_call(tool_call: &ToolCall) -> Result> { 154 | match tool_call.function.name.as_str() { 155 | "create_event" => { 156 | let args: serde_json::Value = serde_json::from_str(&tool_call.function.arguments)?; 157 | 158 | Ok(json!({ 159 | "event_id": "evt_12345", 160 | "status": "created", 161 | "created_at": "2025-01-06T10:30:00Z", 162 | "calendar_links": { 163 | "google": "https://calendar.google.com/event/evt_12345", 164 | "outlook": "https://outlook.com/event/evt_12345" 165 | }, 166 | "notifications_scheduled": args["event"]["attendees"] 167 | .as_array() 168 | .unwrap_or(&vec![]) 169 | .iter() 170 | .map(|attendee| { 171 | json!({ 172 | "attendee": attendee["email"], 173 | "notifications": attendee["preferences"]["notifications"] 174 | }) 175 | }) 176 | .collect::>() 177 | })) 178 | } 179 | _ => Ok(json!({ 180 | "error": "Unknown function", 181 | "function": tool_call.function.name 182 | })), 183 | } 184 | } -------------------------------------------------------------------------------- /examples/memory_example.rs: -------------------------------------------------------------------------------- 1 | // Memory integration example 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, 4 | chat::ChatMessage, 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Create LLM with automatic memory 10 | let llm = LLMBuilder::new() 11 | .backend(LLMBackend::OpenAI) 12 | .api_key(std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into())) 13 | .model("gpt-3.5-turbo") 14 | .role("assistant") 15 | .sliding_window_memory(5) 16 | .build()?; 17 | 18 | // First conversation 19 | let messages1 = vec![ChatMessage::user().content("My name is Alice").build()]; 20 | match llm.chat(&messages1).await { 21 | Ok(response) => println!("Response 1: {}", response), 22 | Err(e) => eprintln!("Error 1: {}", e), 23 | } 24 | 25 | // Second conversation - should remember Alice's name 26 | let messages2 = vec![ChatMessage::user().content("What's my name?").build()]; 27 | match llm.chat(&messages2).await { 28 | Ok(response) => println!("Response 2: {}", response), 29 | Err(e) => eprintln!("Error 2: {}", e), 30 | } 31 | 32 | Ok(()) 33 | } 34 | -------------------------------------------------------------------------------- /examples/memory_share_example.rs: -------------------------------------------------------------------------------- 1 | // Memory sharing example 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, 4 | chat::ChatMessage, 5 | memory::{SharedMemory, SlidingWindowMemory}, 6 | }; 7 | 8 | #[tokio::main] 9 | async fn main() -> Result<(), Box> { 10 | let shared_memory = SharedMemory::new(SlidingWindowMemory::new(10)); 11 | 12 | let llm = LLMBuilder::new() 13 | .backend(LLMBackend::OpenAI) 14 | .api_key(std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into())) 15 | .model("gpt-3.5-turbo") 16 | .role("assistant") 17 | .memory(shared_memory.clone()) 18 | .build()?; 19 | 20 | let llm2 = LLMBuilder::new() 21 | .backend(LLMBackend::Anthropic) 22 | .api_key(std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthro-key".into())) 23 | .model("claude-3-7-sonnet-20250219") 24 | .max_tokens(512) 25 | .temperature(0.7) 26 | .memory(shared_memory) 27 | .build()?; 28 | 29 | let messages1 = vec![ChatMessage::user().content("My name is Alice").build()]; 30 | match llm.chat(&messages1).await { 31 | Ok(response) => println!("Response 1: {}", response), 32 | Err(e) => eprintln!("Error 1: {}", e), 33 | } 34 | 35 | let messages2 = vec![ChatMessage::user().content("What's my name?").build()]; 36 | match llm2.chat(&messages2).await { 37 | Ok(response) => println!("Response 2: {}", response), 38 | Err(e) => eprintln!("Error 2: {}", e), 39 | } 40 | 41 | Ok(()) 42 | } 43 | -------------------------------------------------------------------------------- /examples/multi_backend_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating how to chain multiple LLM backends together 2 | //! 3 | //! This example shows how to: 4 | //! 1. Initialize multiple LLM backends (OpenAI, Anthropic) 5 | //! 2. Create a registry to manage multiple backends 6 | //! 3. Build a multi-step chain that uses different backends at each step 7 | //! 4. Pass results between steps using template variables 8 | 9 | use llm::{ 10 | builder::{LLMBackend, LLMBuilder}, 11 | chain::{LLMRegistryBuilder, MultiChainStepBuilder, MultiChainStepMode, MultiPromptChain}, 12 | }; 13 | 14 | #[tokio::main] 15 | async fn main() -> Result<(), Box> { 16 | // Initialize OpenAI backend with API key and model settings 17 | let openai_llm = LLMBuilder::new() 18 | .backend(LLMBackend::OpenAI) 19 | .api_key(std::env::var("OPENAI_API_KEY").unwrap_or("sk-OPENAI".into())) 20 | .model("gpt-4o") 21 | .build()?; 22 | 23 | // Initialize Anthropic backend with API key and model settings 24 | let anthro_llm = LLMBuilder::new() 25 | .backend(LLMBackend::Anthropic) 26 | .api_key(std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthro-key".into())) 27 | .model("claude-3-5-sonnet-20240620") 28 | .build()?; 29 | 30 | let deepseek_llm = LLMBuilder::new() 31 | .backend(LLMBackend::DeepSeek) 32 | .api_key(std::env::var("DEEPSEEK_API_KEY").unwrap_or("sk-TESTKEY".into())) 33 | .model("deepseek-chat") 34 | .build()?; 35 | 36 | // Ollama backend could also be added 37 | // let ollama_llm = LLMBuilder::new() 38 | // .backend(LLMBackend::Ollama) 39 | // .base_url("http://localhost:11411") 40 | // .model("mistral") 41 | // .build()?; 42 | 43 | // Create registry to manage multiple backends 44 | let registry = LLMRegistryBuilder::new() 45 | .register("openai", openai_llm) 46 | .register("anthro", anthro_llm) 47 | .register("deepseek", deepseek_llm) 48 | // .register("ollama", ollama_llm) 49 | .build(); 50 | 51 | // Build multi-step chain using different backends 52 | let chain_res = MultiPromptChain::new(®istry) 53 | // Step 1: Use OpenAI to analyze a code problem 54 | .step( 55 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 56 | .provider_id("openai") 57 | .id("analysis") 58 | .template("Analyze this Rust code and identify potential performance issues:\n```rust\nfn process_data(data: Vec) -> Vec {\n data.iter().map(|x| x * 2).collect()\n}```") 59 | .temperature(0.7) 60 | .build()? 61 | ) 62 | // Step 2: Use Anthropic to suggest optimizations based on analysis 63 | .step( 64 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 65 | .provider_id("anthro") 66 | .id("optimization") 67 | .template("Here is a code analysis: {{analysis}}\n\nSuggest concrete optimizations to improve performance, explaining why they would be beneficial.") 68 | .max_tokens(500) 69 | .top_p(0.9) 70 | .build()? 71 | ) 72 | // Step 3: Use OpenAI to generate optimized code 73 | .step( 74 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 75 | .provider_id("openai") 76 | .id("final_code") 77 | .template("Taking into account these optimization suggestions: {{optimization}}\n\nGenerate an optimized version of the code in Rust with explanatory comments.") 78 | .temperature(0.2) 79 | .build()? 80 | ) 81 | .step( 82 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 83 | .provider_id("deepseek") 84 | .id("final_code") 85 | .template("Taking into account these optimization suggestions: {{optimization}}\n\nGenerate an optimized version of the code in Rust with explanatory comments.") 86 | .temperature(0.2) 87 | .build()? 88 | ) 89 | .run().await?; 90 | 91 | // Display results from all steps 92 | println!("Results: {:?}", chain_res); 93 | // Example output format: 94 | // chain_res["analysis"] => "The code has potential performance issues..." 95 | // chain_res["optimization"] => "Here are some suggested optimizations..." 96 | // chain_res["final_code"] => "// Optimized version with comments..." 97 | 98 | Ok(()) 99 | } 100 | -------------------------------------------------------------------------------- /examples/multi_backend_structured_output_example.rs: -------------------------------------------------------------------------------- 1 | //! Send the same JSON schema to multiple backends for structured output. 2 | 3 | use llm::{ 4 | builder::{LLMBackend, LLMBuilder}, 5 | chat::{ChatMessage, StructuredOutputFormat}, 6 | }; 7 | 8 | #[tokio::main] 9 | async fn main() -> Result<(), Box> { 10 | let api_key = std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into()); 11 | 12 | let schema = r#" 13 | { 14 | "name": "Student", 15 | "schema": { 16 | "type": "object", 17 | "properties": { 18 | "name": { 19 | "type": "string" 20 | }, 21 | "age": { 22 | "type": "integer" 23 | }, 24 | "is_student": { 25 | "type": "boolean" 26 | } 27 | }, 28 | "required": ["name", "age", "is_student"] 29 | } 30 | } 31 | "#; 32 | let schema: StructuredOutputFormat = serde_json::from_str(schema)?; 33 | 34 | let messages = vec![ChatMessage::user() 35 | .content("Generate a random student") 36 | .build()]; 37 | 38 | let llm_openai = LLMBuilder::new() 39 | .backend(LLMBackend::OpenAI) 40 | .api_key(api_key) 41 | .model("gpt-4o") 42 | .max_tokens(512) 43 | .temperature(0.7) 44 | .stream(false) 45 | .system("You are an AI assistant that can provide structured output to generate random students as example data. Respond in JSON format using the provided JSON schema.") 46 | .schema(schema.clone()) 47 | .build() 48 | .expect("Failed to build LLM (OpenAI)"); 49 | 50 | match llm_openai.chat(&messages).await { 51 | Ok(text) => println!("Chat response:\n{}", text), 52 | Err(e) => eprintln!("Chat error: {}", e), 53 | } 54 | 55 | let llm_ollama = LLMBuilder::new() 56 | .backend(LLMBackend::Ollama) 57 | .base_url("http://127.0.0.1:11434") 58 | .model("llama3.1:latest") 59 | .max_tokens(512) 60 | .temperature(0.7) 61 | .stream(false) 62 | .schema(schema) 63 | .system("You are a helpful AI assistant. Please generate a random student using the provided JSON schema.") 64 | .build() 65 | .expect("Failed to build LLM (Ollama)"); 66 | 67 | match llm_ollama.chat(&messages).await { 68 | Ok(text) => println!("Ollama chat response:\n{}", text), 69 | Err(e) => eprintln!("Chat error: {}", e), 70 | } 71 | 72 | Ok(()) 73 | } 74 | -------------------------------------------------------------------------------- /examples/ollama_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, 4 | chat::ChatMessage, 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get Ollama server URL from environment variable or use default localhost 10 | let base_url = std::env::var("OLLAMA_URL").unwrap_or("http://127.0.0.1:11434".into()); 11 | 12 | // Initialize and configure the LLM client 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::Ollama) // Use Ollama as the LLM backend 15 | .base_url(base_url) // Set the Ollama server URL 16 | .model("llama3.2:latest") 17 | .max_tokens(1000) // Set maximum response length 18 | .temperature(0.7) // Control response randomness (0.0-1.0) 19 | .stream(false) // Disable streaming responses 20 | .build() 21 | .expect("Failed to build LLM (Ollama)"); 22 | 23 | // Prepare conversation history with example messages 24 | let messages = vec![ 25 | ChatMessage::user() 26 | .content("Hello, how do I run a local LLM in Rust?") 27 | .build(), 28 | ChatMessage::assistant() 29 | .content("One way is to use Ollama with a local model!") 30 | .build(), 31 | ChatMessage::user() 32 | .content("Tell me more about that") 33 | .build(), 34 | ]; 35 | 36 | // Send chat request and handle the response 37 | match llm.chat(&messages).await { 38 | Ok(text) => println!("Ollama chat response:\n{}", text), 39 | Err(e) => eprintln!("Chat error: {}", e), 40 | } 41 | 42 | Ok(()) 43 | } 44 | -------------------------------------------------------------------------------- /examples/ollama_structured_output_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, 4 | chat::{ChatMessage, StructuredOutputFormat}, 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get Ollama server URL from environment variable or use default localhost 10 | let base_url = std::env::var("OLLAMA_URL").unwrap_or("http://127.0.0.1:11434".into()); 11 | 12 | // Define a simple JSON schema for structured output 13 | let schema = r#" 14 | { 15 | "name": "student", 16 | "schema": { 17 | "type": "object", 18 | "properties": { 19 | "name": { 20 | "type": "string" 21 | }, 22 | "age": { 23 | "type": "integer" 24 | }, 25 | "is_student": { 26 | "type": "boolean" 27 | } 28 | }, 29 | "required": ["name", "age", "is_student"] 30 | } 31 | } 32 | "#; 33 | let schema: StructuredOutputFormat = serde_json::from_str(schema)?; 34 | 35 | // Initialize and configure the LLM client 36 | let llm = LLMBuilder::new() 37 | .backend(LLMBackend::Ollama) // Use Ollama as the LLM backend 38 | .base_url(base_url) // Set the Ollama server URL 39 | .model("llama3.1:latest") 40 | .max_tokens(1000) // Set maximum response length 41 | .temperature(0.7) // Control response randomness (0.0-1.0) 42 | .stream(false) // Disable streaming responses 43 | .schema(schema) // Set JSON schema for structured output 44 | .system("You are a helpful AI assistant. Please generate a random student using the provided JSON schema.") 45 | .build() 46 | .expect("Failed to build LLM (Ollama)"); 47 | 48 | // Prepare conversation history with example messages 49 | let messages = vec![ChatMessage::user() 50 | .content("Please generate a random student using the provided JSON schema.") 51 | .build()]; 52 | 53 | // Send chat request and handle the response 54 | match llm.chat(&messages).await { 55 | Ok(text) => println!("Ollama chat response:\n{}", text), 56 | Err(e) => eprintln!("Chat error: {}", e), 57 | } 58 | 59 | Ok(()) 60 | } 61 | -------------------------------------------------------------------------------- /examples/openai_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for OpenAI integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get OpenAI API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into()); 11 | 12 | // Initialize and configure the LLM client 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::OpenAI) // Use OpenAI as the LLM provider 15 | .api_key(api_key) // Set the API key 16 | .model("gpt-3.5-turbo") // Use GPT-3.5 Turbo model 17 | .max_tokens(512) // Limit response length 18 | .temperature(0.7) // Control response randomness (0.0-1.0) 19 | .stream(false) // Disable streaming responses 20 | .build() 21 | .expect("Failed to build LLM (OpenAI)"); 22 | 23 | // Prepare conversation history with example messages 24 | let messages = vec![ 25 | ChatMessage::user() 26 | .content("Tell me that you love cats") 27 | .build(), 28 | ChatMessage::assistant() 29 | .content("I am an assistant, I cannot love cats but I can love dogs") 30 | .build(), 31 | ChatMessage::user() 32 | .content("Tell me that you love dogs in 2000 chars") 33 | .build(), 34 | ]; 35 | 36 | // Send chat request and handle the response 37 | match llm.chat(&messages).await { 38 | Ok(text) => println!("Chat response:\n{}", text), 39 | Err(e) => eprintln!("Chat error: {}", e), 40 | } 41 | 42 | Ok(()) 43 | } 44 | -------------------------------------------------------------------------------- /examples/openai_reasoning_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for OpenAI integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::{ChatMessage, ReasoningEffort}, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get OpenAI API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into()); 11 | 12 | // Initialize and configure the LLM client 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::OpenAI) // Use OpenAI as the LLM provider 15 | .api_key(api_key) // Set the API key 16 | .model("o1-preview") // Use GPT-3.5 Turbo model 17 | .stream(false) // Disable streaming responses 18 | .reasoning_effort(ReasoningEffort::High) // Enable reasoning effort 19 | .build() 20 | .expect("Failed to build LLM (OpenAI)"); 21 | 22 | // Prepare conversation history with example messages 23 | let messages = vec![ChatMessage::user() 24 | .content("How muck r in strawberry") 25 | .build()]; 26 | 27 | // Send chat request and handle the response 28 | match llm.chat(&messages).await { 29 | Ok(text) => println!("Chat response:\n{}", text), 30 | Err(e) => eprintln!("Chat error: {}", e), 31 | } 32 | 33 | Ok(()) 34 | } 35 | -------------------------------------------------------------------------------- /examples/openai_streaming_example.rs: -------------------------------------------------------------------------------- 1 | // OpenAI streaming chat example demonstrating real-time token generation 2 | use futures::StreamExt; 3 | use llm::{ 4 | builder::{LLMBackend, LLMBuilder}, 5 | chat::ChatMessage, 6 | }; 7 | use std::io::{self, Write}; 8 | 9 | #[tokio::main] 10 | async fn main() -> Result<(), Box> { 11 | // Get OpenAI API key from environment variable or use test key as fallback 12 | let api_key = std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into()); 13 | 14 | // Initialize and configure the LLM client with streaming enabled 15 | let llm = LLMBuilder::new() 16 | .backend(LLMBackend::OpenAI) 17 | .api_key(api_key) 18 | .model("gpt-3.5-turbo") 19 | // .max_tokens(512) 20 | .temperature(0.7) 21 | .stream(true) // Enable streaming responses 22 | .build() 23 | .expect("Failed to build LLM (OpenAI)"); 24 | 25 | // Prepare conversation with a prompt that will generate a longer response 26 | let messages = vec![ 27 | ChatMessage::user() 28 | .content("Write a long story about a robot learning to paint. Make it creative and engaging.") 29 | .build(), 30 | ]; 31 | 32 | println!("Starting streaming chat with OpenAI...\n"); 33 | 34 | match llm.chat_stream(&messages).await { 35 | Ok(mut stream) => { 36 | let stdout = io::stdout(); 37 | let mut handle = stdout.lock(); 38 | 39 | while let Some(Ok(token)) = stream.next().await { 40 | handle.write_all(token.as_bytes()).unwrap(); 41 | handle.flush().unwrap(); 42 | } 43 | println!("\n\nStreaming completed."); 44 | } 45 | Err(e) => eprintln!("Chat error: {}", e), 46 | } 47 | 48 | Ok(()) 49 | } -------------------------------------------------------------------------------- /examples/openai_structured_output_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for OpenAI integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::{ChatMessage, StructuredOutputFormat}, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get OpenAI API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into()); 11 | 12 | // Define a simple JSON schema for structured output 13 | let schema = r#" 14 | { 15 | "name": "student", 16 | "schema": { 17 | "type": "object", 18 | "properties": { 19 | "name": { 20 | "type": "string" 21 | }, 22 | "age": { 23 | "type": "integer" 24 | }, 25 | "is_student": { 26 | "type": "boolean" 27 | } 28 | }, 29 | "required": ["name", "age", "is_student"] 30 | } 31 | } 32 | "#; 33 | let schema: StructuredOutputFormat = serde_json::from_str(schema)?; 34 | 35 | // Initialize and configure the LLM client 36 | let llm = LLMBuilder::new() 37 | .backend(LLMBackend::OpenAI) // Use OpenAI as the LLM provider 38 | .api_key(api_key) // Set the API key 39 | .model("gpt-4o") // Use GPT-4o model 40 | .max_tokens(512) // Limit response length 41 | .temperature(0.7) // Control response randomness (0.0-1.0) 42 | .stream(false) // Disable streaming responses 43 | .system("You are an AI assistant that can provide structured output to generate random students as example data. Respond in JSON format using the provided JSON schema.") // Set system description 44 | .schema(schema) // Set JSON schema for structured output 45 | .build() 46 | .expect("Failed to build LLM (OpenAI)"); 47 | 48 | // Prepare conversation history with example messages 49 | let messages = vec![ChatMessage::user() 50 | .content("Generate a random student") 51 | .build()]; 52 | 53 | // Send chat request and handle the response 54 | match llm.chat(&messages).await { 55 | Ok(text) => println!("Chat response:\n{}", text), 56 | Err(e) => eprintln!("Chat error: {}", e), 57 | } 58 | 59 | Ok(()) 60 | } 61 | -------------------------------------------------------------------------------- /examples/openai_stt_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for OpenAI integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | }; 5 | 6 | #[tokio::main] 7 | async fn main() -> Result<(), Box> { 8 | // Get OpenAI API key from environment variable or use test key as fallback 9 | let api_key = std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into()); 10 | 11 | // Initialize and configure the LLM client 12 | let llm = LLMBuilder::new() 13 | .backend(LLMBackend::OpenAI) // Use OpenAI as the LLM provider 14 | .api_key(api_key) // Set the API key 15 | .model("whisper-1") // Use gpt-4o-transcribe model 16 | .max_tokens(512) // Limit response length 17 | .temperature(0.7) // Control response randomness (0.0-1.0) 18 | .stream(false) // Disable streaming responses 19 | .build() 20 | .expect("Failed to build LLM (OpenAI)"); 21 | 22 | match llm.transcribe_file("audio2.m4a").await { 23 | Ok(text) => println!("Audio transcription:\n{}", text), 24 | Err(e) => eprintln!("Audio transcription error: {}", e), 25 | } 26 | 27 | Ok(()) 28 | } 29 | -------------------------------------------------------------------------------- /examples/openai_tts_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating text-to-speech synthesis using OpenAI 2 | //! 3 | //! This example shows how to: 4 | //! 1. Initialize the OpenAI text-to-speech provider 5 | //! 2. Generate speech from text 6 | //! 3. Save the audio output to a file 7 | 8 | use llm::builder::{LLMBackend, LLMBuilder}; 9 | 10 | #[tokio::main] 11 | async fn main() -> Result<(), Box> { 12 | // Get API key from environment variable or use test key 13 | let api_key = std::env::var("OPENAI_API_KEY").unwrap_or("test_key".into()); 14 | 15 | // Initialize OpenAI text-to-speech provider 16 | let tts = LLMBuilder::new() 17 | .backend(LLMBackend::OpenAI) 18 | .api_key(api_key) 19 | .model("tts-1") 20 | .voice("ash") 21 | .build()?; 22 | 23 | // Text to convert to speech 24 | let text = "Hello! This is an example of text-to-speech synthesis using OpenAI."; 25 | 26 | // Generate speech 27 | let audio_data = tts.speech(text).await?; 28 | 29 | // Save the audio to a file 30 | std::fs::write("output-speech.mp3", audio_data)?; 31 | 32 | println!("Audio file generated successfully: output-speech.mp3"); 33 | Ok(()) 34 | } 35 | -------------------------------------------------------------------------------- /examples/openai_vision_example.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | 3 | // Import required modules from the LLM library for OpenAI integration 4 | use llm::{ 5 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 6 | chat::{ChatMessage, ImageMime}, // Chat-related structures 7 | }; 8 | 9 | #[tokio::main] 10 | async fn main() -> Result<(), Box> { 11 | // Get OpenAI API key from environment variable or use test key as fallback 12 | let api_key = std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into()); 13 | 14 | // Initialize and configure the LLM client 15 | let llm = LLMBuilder::new() 16 | .backend(LLMBackend::OpenAI) // Use OpenAI as the LLM provider 17 | .api_key(api_key) // Set the API key 18 | .model("gpt-4o") // Use GPT-3.5 Turbo model 19 | .max_tokens(1024) // Limit response length 20 | .temperature(0.7) // Control response randomness (0.0-1.0) 21 | .stream(false) // Disable streaming responses 22 | .build() 23 | .expect("Failed to build LLM (OpenAI)"); 24 | 25 | let content = fs::read("./examples/image001.jpg").expect("The image001.jpg file should exist"); 26 | 27 | // Prepare conversation history with example messages 28 | let messages = vec![ 29 | ChatMessage::user().image_url("https://media.istockphoto.com/id/1443562748/fr/photo/mignon-chat-gingembre.jpg?s=612x612&w=0&k=20&c=ygNVVnqLk9V8BWu4VQ0D21u7-daIyHUoyKlCcx3K1E8=").build(), 30 | ChatMessage::user().image(ImageMime::JPEG, content).build(), 31 | ChatMessage::user().content("What is in this image (image 1 and 2)?").build(), 32 | ]; 33 | 34 | // Send chat request and handle the response 35 | match llm.chat(&messages).await { 36 | Ok(text) => println!("Chat response:\n{}", text), 37 | Err(e) => eprintln!("Chat error: {}", e), 38 | } 39 | 40 | Ok(()) 41 | } 42 | -------------------------------------------------------------------------------- /examples/phind_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for Phind integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Initialize and configure the LLM client 10 | let llm = LLMBuilder::new() 11 | .backend(LLMBackend::Phind) // Use Phind as the LLM provider 12 | .model("Phind-70B") // Use Phind-70B model 13 | .max_tokens(512) // Limit response length 14 | .temperature(0.7) // Control response randomness (0.0-1.0) 15 | .stream(false) // Disable streaming responses 16 | .build() 17 | .expect("Failed to build LLM (Phind)"); 18 | 19 | // Prepare conversation history with example messages 20 | let messages = vec![ 21 | ChatMessage::user() 22 | .content("Tell me that you love cats") 23 | .build(), 24 | ChatMessage::assistant() 25 | .content("I am an assistant, I cannot love cats but I can love dogs") 26 | .build(), 27 | ChatMessage::user() 28 | .content("Tell me that you love dogs in 2000 chars") 29 | .build(), 30 | ]; 31 | 32 | // Send chat request and handle the response 33 | match llm.chat(&messages).await { 34 | Ok(text) => println!("Chat response:\n{}", text), 35 | Err(e) => eprintln!("Chat error: {}", e), 36 | } 37 | 38 | Ok(()) 39 | } 40 | -------------------------------------------------------------------------------- /examples/tool_calling_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for OpenAI integration 2 | use llm::{ 3 | builder::{FunctionBuilder, LLMBackend, LLMBuilder, ParamBuilder}, 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get OpenAI API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into()); 11 | 12 | // Initialize and configure the LLM client 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::OpenAI) // Use OpenAI as the LLM provider 15 | .api_key(api_key) // Set the API key 16 | .model("gpt-3.5-turbo") // Use GPT-3.5 Turbo model 17 | .max_tokens(512) // Limit response length 18 | .temperature(0.7) // Control response randomness (0.0-1.0) 19 | .stream(false) // Disable streaming responses 20 | .function( 21 | FunctionBuilder::new("weather_function") 22 | .description("Use this tool to get the weather in a specific city") 23 | .param( 24 | ParamBuilder::new("url") 25 | .type_of("string") 26 | .description("The url to get the weather from for the city"), 27 | ) 28 | .required(vec!["url".to_string()]), 29 | ) 30 | .build() 31 | .expect("Failed to build LLM"); 32 | 33 | // Prepare conversation history with example messages 34 | let messages = vec![ChatMessage::user().content("You are a weather assistant. What is the weather in Tokyo? Use the tools that you have available").build()]; 35 | 36 | // Send chat request and handle the response 37 | // this returns the response as a string. The tool call is also returned as a serialized string. We can deserialize if needed. 38 | match llm.chat_with_tools(&messages, llm.tools()).await { 39 | Ok(text) => println!("Chat response:\n{}", text), 40 | Err(e) => eprintln!("Chat error: {}", e), 41 | } 42 | 43 | Ok(()) 44 | } 45 | -------------------------------------------------------------------------------- /examples/tool_json_schema_cycle_example.rs: -------------------------------------------------------------------------------- 1 | //! End-to-end example showing a realistic tool-use cycle: 2 | //! 1. The user asks to import users. 3 | //! 2. The model replies with a `tool_use` call. 4 | //! 3. We execute the function on our side (mock). 5 | //! 4. We send back a `tool_result` message. 6 | //! 5. The model produces a final confirmation message. 7 | 8 | use llm::builder::{FunctionBuilder, LLMBackend, LLMBuilder}; 9 | use llm::chat::{ChatMessage, ToolChoice}; 10 | use llm::{FunctionCall, ToolCall}; 11 | use serde::Deserialize; 12 | use serde_json::json; 13 | 14 | 15 | #[derive(Debug, Deserialize)] 16 | #[allow(dead_code)] 17 | struct User { 18 | name: String, 19 | emails: Vec, 20 | } 21 | 22 | #[derive(Debug, Deserialize)] 23 | struct ImportUsersArgs { 24 | users: Vec, 25 | } 26 | 27 | 28 | fn import_users_tool() -> FunctionBuilder { 29 | let schema = json!({ 30 | "type": "object", 31 | "properties": { 32 | "users": { 33 | "type": "array", 34 | "items": { 35 | "type": "object", 36 | "properties": { 37 | "name": { "type": "string" }, 38 | "emails": { 39 | "type": "array", 40 | "items": { "type": "string", "format": "email" } 41 | } 42 | }, 43 | "required": ["name", "emails"], 44 | "additionalProperties": false 45 | } 46 | } 47 | }, 48 | "required": ["users"], 49 | "additionalProperties": false 50 | }); 51 | 52 | FunctionBuilder::new("import_users") 53 | .description("Bulk-import a list of users with their email addresses.") 54 | .json_schema(schema) 55 | } 56 | 57 | fn import_users(args_json: &str) -> Result> { 58 | let args: ImportUsersArgs = serde_json::from_str(args_json)?; 59 | println!("[server] imported {} users", args.users.len()); 60 | Ok(args.users.len()) 61 | } 62 | 63 | #[tokio::main] 64 | async fn main() -> Result<(), Box> { 65 | let llm = LLMBuilder::new() 66 | .backend(LLMBackend::OpenAI) 67 | .api_key(std::env::var("OPENAI_API_KEY")?) 68 | .model("gpt-4o") 69 | .function(import_users_tool()) 70 | .tool_choice(ToolChoice::Any) 71 | .build()?; 72 | 73 | let mut messages = vec![ChatMessage::user() 74 | .content("Please import Alice and Bob .") 75 | .build()]; 76 | 77 | let first_resp = llm.chat(&messages).await?; 78 | println!("[assistant] {}", first_resp); 79 | 80 | if let Some(tool_calls) = first_resp.tool_calls() { 81 | let mut tool_results = Vec::new(); 82 | for call in &tool_calls { 83 | match import_users(&call.function.arguments) { 84 | Ok(count) => { 85 | // Prepare a ToolResult conveying success. 86 | tool_results.push(ToolCall { 87 | id: call.id.clone(), 88 | call_type: "function".into(), 89 | function: FunctionCall { 90 | name: call.function.name.clone(), 91 | arguments: json!({ "status": "ok", "imported": count }).to_string(), 92 | }, 93 | }); 94 | } 95 | Err(e) => { 96 | eprintln!("[server] import failed: {e}"); 97 | } 98 | } 99 | } 100 | 101 | messages.push(ChatMessage::assistant().tool_use(tool_calls).build()); 102 | messages.push(ChatMessage::assistant().tool_result(tool_results).build()); 103 | 104 | let final_resp = llm.chat(&messages).await?; 105 | println!("[assistant] {}", final_resp); 106 | } 107 | 108 | Ok(()) 109 | } 110 | -------------------------------------------------------------------------------- /examples/trim_strategy_example.rs: -------------------------------------------------------------------------------- 1 | use llm::{ 2 | builder::{LLMBackend, LLMBuilder}, 3 | chat::ChatMessage, 4 | memory::TrimStrategy, 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | let llm = LLMBuilder::new() 10 | .backend(LLMBackend::OpenAI) 11 | .api_key(std::env::var("OPENAI_API_KEY").unwrap_or("sk-TESTKEY".into())) 12 | .model("gpt-3.5-turbo") 13 | .sliding_window_with_strategy(3, TrimStrategy::Summarize) 14 | .build()?; 15 | 16 | println!("Testing TrimStrategy::Summarize with window size 3"); 17 | 18 | let messages = vec![ 19 | ChatMessage::user().content("Hello, my name is Alice").build(), 20 | ChatMessage::user().content("I love programming in Rust").build(), 21 | ChatMessage::user().content("What's the weather like?").build(), 22 | ChatMessage::user().content("Tell me about AI").build(), 23 | ]; 24 | 25 | for (i, message) in messages.iter().enumerate() { 26 | println!("\n--- Message {} ---", i + 1); 27 | println!("Sending: {}", message.content); 28 | 29 | match llm.chat(&[message.clone()]).await { 30 | Ok(response) => println!("Response: {}", response), 31 | Err(e) => eprintln!("Error: {}", e), 32 | } 33 | 34 | if let Some(memory_contents) = llm.memory_contents().await { 35 | println!("\nMemory contents ({} messages):", memory_contents.len()); 36 | for (j, msg) in memory_contents.iter().enumerate() { 37 | let role = match msg.role { 38 | llm::chat::ChatRole::User => "User", 39 | llm::chat::ChatRole::Assistant => "Assistant", 40 | }; 41 | println!(" {}: {} - '{}'", j + 1, role, msg.content); 42 | } 43 | } 44 | } 45 | 46 | Ok(()) 47 | } -------------------------------------------------------------------------------- /examples/tts_rodio_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating text-to-speech synthesis using OpenAI 2 | //! 3 | //! This example shows how to: 4 | //! 1. Initialize the OpenAI text-to-speech provider 5 | //! 2. Generate speech from text 6 | //! 3. Save the audio output to a file 7 | 8 | use std::{fs::File, io::BufReader}; 9 | 10 | use llm::builder::{LLMBackend, LLMBuilder}; 11 | use rodio::{Decoder, OutputStream, Sink}; 12 | 13 | #[tokio::main] 14 | async fn main() -> Result<(), Box> { 15 | // Get API key from environment variable or use test key 16 | let api_key = std::env::var("OPENAI_API_KEY").unwrap_or("test_key".into()); 17 | 18 | // Initialize OpenAI text-to-speech provider 19 | let tts = LLMBuilder::new() 20 | .backend(LLMBackend::OpenAI) 21 | .api_key(api_key) 22 | .model("tts-1") 23 | .voice("ash") 24 | .build()?; 25 | 26 | // Text to convert to speech 27 | let text = "Hello! This is an example of text-to-speech synthesis using OpenAI with LLM crates and rodio in Rust."; 28 | 29 | // Generate speech 30 | let audio_data = tts.speech(text).await?; 31 | 32 | // Save the audio to a file 33 | std::fs::write("output-speech.mp3", audio_data)?; 34 | 35 | // Play the audio 36 | let (_stream, stream_handle) = OutputStream::try_default().unwrap(); 37 | let sink = Sink::try_new(&stream_handle).unwrap(); 38 | 39 | let file = File::open("output-speech.mp3").unwrap(); 40 | let source = Decoder::new(BufReader::new(file)).unwrap(); 41 | 42 | // Play audio 43 | sink.append(source); 44 | 45 | // Block until the sound has finished 46 | sink.sleep_until_end(); 47 | 48 | Ok(()) 49 | } 50 | -------------------------------------------------------------------------------- /examples/validator_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder components for LLM configuration 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Retrieve Anthropic API key from environment variable or use fallback 10 | let api_key = std::env::var("ANTHROPIC_API_KEY").unwrap_or("anthro-key".into()); 11 | 12 | // Initialize and configure the LLM client with validation 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::Anthropic) // Use Anthropic's Claude model 15 | .model("claude-3-5-sonnet-20240620") // Specify model version 16 | .api_key(api_key) // Set API credentials 17 | .max_tokens(512) // Limit response length 18 | .temperature(0.7) // Control response randomness 19 | .stream(false) // Disable streaming responses 20 | .validator(|resp| { 21 | // Add JSON validation 22 | serde_json::from_str::(resp) 23 | .map(|_| ()) 24 | .map_err(|e| e.to_string()) 25 | }) 26 | .validator_attempts(3) // Allow up to 3 retries on validation failure 27 | .build() 28 | .expect("Failed to build LLM (Phind)"); 29 | 30 | // Prepare the chat message requesting JSON output 31 | let messages = vec![ 32 | ChatMessage::user().content("Please give me a valid JSON describing a cat named Garfield, color 'orange'. with format {name: string, color: string}. Return only the JSON, no other text").build(), 33 | ]; 34 | 35 | // Send chat request and handle the response 36 | match llm.chat(&messages).await { 37 | Ok(text) => println!("{}", text), 38 | Err(e) => eprintln!("Chat error: {}", e), 39 | } 40 | 41 | Ok(()) 42 | } 43 | -------------------------------------------------------------------------------- /examples/xai_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for xAI integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::ChatMessage, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get xAI API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("XAI_API_KEY").unwrap_or("sk-TESTKEY".into()); 11 | 12 | // Initialize and configure the LLM client 13 | let llm = LLMBuilder::new() 14 | .backend(LLMBackend::XAI) // Use xAI as the LLM provider 15 | .api_key(api_key) // Set the API key 16 | .model("grok-2-latest") // Use Grok-2 model 17 | .max_tokens(512) // Limit response length 18 | .temperature(0.7) // Control response randomness (0.0-1.0) 19 | .stream(false) // Disable streaming responses 20 | .build() 21 | .expect("Failed to build LLM (xAI)"); 22 | 23 | // Prepare conversation history with example messages 24 | let messages = vec![ 25 | ChatMessage::user() 26 | .content("Tell me that you love cats") 27 | .build(), 28 | ChatMessage::assistant() 29 | .content("I am an assistant, I cannot love cats but I can love dogs") 30 | .build(), 31 | ChatMessage::user() 32 | .content("Tell me that you love dogs in 2000 chars") 33 | .build(), 34 | ]; 35 | 36 | // Send chat request and handle the response 37 | match llm.chat(&messages).await { 38 | Ok(text) => println!("Chat response:\n{}", text), 39 | Err(e) => eprintln!("Chat error: {}", e), 40 | } 41 | 42 | Ok(()) 43 | } 44 | -------------------------------------------------------------------------------- /examples/xai_search_chain_tts_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating XAI search -> OpenAI summary -> ElevenLabs TTS chain 2 | //! 3 | //! This example shows how to: 4 | //! 1. Use XAI with search parameters to get latest Rust news 5 | //! 2. Use OpenAI to create a bullet-point summary 6 | //! 3. Use ElevenLabs TTS to convert the summary to speech 7 | //! 4. Save the audio output and play it with rodio 8 | 9 | use llm::{ 10 | builder::{LLMBackend, LLMBuilder}, 11 | chain::{LLMRegistryBuilder, MultiChainStepBuilder, MultiChainStepMode, MultiPromptChain}, 12 | }; 13 | use rodio::{Decoder, OutputStream, Sink}; 14 | use std::{fs::File, io::BufReader}; 15 | 16 | #[tokio::main] 17 | async fn main() -> Result<(), Box> { 18 | println!("🔍 Starting XAI Search -> OpenAI -> TTS chain example..."); 19 | 20 | // Initialize XAI backend with search capabilities 21 | let xai_llm = LLMBuilder::new() 22 | .backend(LLMBackend::XAI) 23 | .api_key(std::env::var("XAI_API_KEY").unwrap_or("xai-key".into())) 24 | .model("grok-3-latest") 25 | .search_mode("auto") 26 | .max_search_results(10) 27 | .search_from_date("2024-01-01") // Recent news 28 | .build()?; 29 | 30 | // Initialize OpenAI backend for summarization 31 | let openai_llm = LLMBuilder::new() 32 | .backend(LLMBackend::OpenAI) 33 | .api_key(std::env::var("OPENAI_API_KEY").unwrap_or("sk-openai".into())) 34 | .model("gpt-4o") 35 | .temperature(0.3) // Lower temperature for factual summaries 36 | .build()?; 37 | 38 | // Initialize ElevenLabs backend for text-to-speech 39 | let elevenlabs_llm = LLMBuilder::new() 40 | .backend(LLMBackend::ElevenLabs) 41 | .api_key(std::env::var("ELEVENLABS_API_KEY").unwrap_or("elevenlabs-key".into())) 42 | .model("eleven_multilingual_v2") 43 | .voice("JBFqnCBsd6RMkjVDRZzb") 44 | .build()?; 45 | 46 | // Create registry to manage multiple backends 47 | let registry = LLMRegistryBuilder::new() 48 | .register("xai", xai_llm) 49 | .register("openai", openai_llm) 50 | .register("elevenlabs", elevenlabs_llm) 51 | .build(); 52 | 53 | println!("🔍 Step 1: Searching for latest Rust news with XAI..."); 54 | 55 | // Build multi-step chain: XAI search -> OpenAI summary 56 | let chain_result = MultiPromptChain::new(®istry) 57 | // Step 1: Use XAI with search to get latest Rust news 58 | .step( 59 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 60 | .provider_id("xai") 61 | .id("rust_news") 62 | .template("Find the latest news and developments about the Rust programming language. Include recent releases, community updates, performance improvements, and notable projects. Focus on factual information from the last few months.") 63 | .max_tokens(1000) 64 | .build()? 65 | ) 66 | // Step 2: Use OpenAI to create a structured bullet-point summary 67 | .step( 68 | MultiChainStepBuilder::new(MultiChainStepMode::Chat) 69 | .provider_id("openai") 70 | .id("summary") 71 | .template("Based on this Rust news information: {{rust_news}}\n\nCreate a clear, structured summary with the following format:\n\n• Start with 'Here are the latest Rust developments:'\n• Create 5-7 bullet points highlighting the most important news\n• Each bullet should be one concise sentence\n• Focus on releases, features, performance, and community highlights\n• End with 'That concludes the Rust news summary'\n\nMake it suitable for text-to-speech conversion.") 72 | .max_tokens(400) 73 | .temperature(0.3) 74 | .build()? 75 | ) 76 | .run().await?; 77 | 78 | println!("✅ Step 2: Summary created successfully!"); 79 | 80 | // Get the summary from chain results 81 | let summary = chain_result 82 | .get("summary") 83 | .ok_or("No summary found in chain results")?; 84 | 85 | println!("📄 Summary to be spoken:"); 86 | println!("{}", summary); 87 | println!(); 88 | 89 | println!("🔊 Step 3: Converting summary to speech with ElevenLabs..."); 90 | 91 | // Step 3: Use ElevenLabs TTS to convert summary to speech 92 | let elevenlabs_provider = registry 93 | .get("elevenlabs") 94 | .ok_or("ElevenLabs provider not found")?; 95 | 96 | let audio_data = elevenlabs_provider.speech(summary).await?; 97 | 98 | // Save audio to file 99 | let output_file = "rust_news_summary.mp3"; 100 | std::fs::write(output_file, audio_data)?; 101 | 102 | println!("✅ Audio saved to: {}", output_file); 103 | 104 | // Step 4: Play the audio using rodio 105 | println!("🎵 Playing audio with rodio..."); 106 | 107 | let (_stream, stream_handle) = OutputStream::try_default()?; 108 | let sink = Sink::try_new(&stream_handle)?; 109 | 110 | let file = File::open(output_file)?; 111 | let source = Decoder::new(BufReader::new(file))?; 112 | 113 | // Play audio 114 | sink.append(source); 115 | 116 | // Block until the sound has finished 117 | sink.sleep_until_end(); 118 | 119 | println!("🎉 Chain completed successfully!"); 120 | println!(); 121 | println!("Summary: XAI found the news -> OpenAI created bullet points -> ElevenLabs spoke it -> Rodio played it!"); 122 | 123 | Ok(()) 124 | } 125 | -------------------------------------------------------------------------------- /examples/xai_search_example.rs: -------------------------------------------------------------------------------- 1 | //! Example demonstrating X.AI search functionality 2 | //! 3 | //! This example shows how to use X.AI's search parameters to: 4 | //! 1. Enable search mode 5 | //! 2. Set maximum search results 6 | //! 3. Specify date ranges for search 7 | //! 4. Configure search sources with exclusions 8 | 9 | use llm::{ 10 | builder::{LLMBackend, LLMBuilder}, 11 | chat::{ChatMessage, ChatRole, MessageType}, 12 | }; 13 | 14 | #[tokio::main] 15 | async fn main() -> Result<(), Box> { 16 | // Example 1: Basic search with auto mode and result limit 17 | let llm_basic = LLMBuilder::new() 18 | .backend(LLMBackend::XAI) 19 | .api_key(std::env::var("XAI_API_KEY").unwrap_or("xai-test-key".into())) 20 | .model("grok-3-latest") 21 | .search_mode("auto") 22 | .max_search_results(10) 23 | .build()?; 24 | 25 | let messages = vec![ChatMessage { 26 | role: ChatRole::User, 27 | message_type: MessageType::Text, 28 | content: "What are some recently discovered alternative DNA shapes?".to_string(), 29 | }]; 30 | 31 | println!("=== Basic Search Example ==="); 32 | let response = llm_basic.chat(&messages).await?; 33 | println!("Response: {}", response.text().unwrap_or_default()); 34 | 35 | // Example 2: Search with date range 36 | let llm_dated = LLMBuilder::new() 37 | .backend(LLMBackend::XAI) 38 | .api_key(std::env::var("XAI_API_KEY").unwrap_or("xai-test-key".into())) 39 | .model("grok-3-latest") 40 | .search_mode("auto") 41 | .search_date_range("2022-01-01", "2022-12-31") 42 | .max_search_results(5) 43 | .build()?; 44 | 45 | let dated_messages = vec![ChatMessage { 46 | role: ChatRole::User, 47 | message_type: MessageType::Text, 48 | content: "What were the major AI breakthroughs in 2022?".to_string(), 49 | }]; 50 | 51 | println!("\n=== Date Range Search Example ==="); 52 | let dated_response = llm_dated.chat(&dated_messages).await?; 53 | println!("Response: {}", dated_response.text().unwrap_or_default()); 54 | 55 | // Example 3: Search with source exclusions 56 | let llm_filtered = LLMBuilder::new() 57 | .backend(LLMBackend::XAI) 58 | .api_key(std::env::var("XAI_API_KEY").unwrap_or("xai-test-key".into())) 59 | .model("grok-3-latest") 60 | .search_mode("auto") 61 | .search_source("web", Some(vec!["wikipedia.org".to_string()])) 62 | .search_source("news", Some(vec!["bbc.co.uk".to_string()])) 63 | .max_search_results(8) 64 | .search_from_date("2023-01-01") 65 | .build()?; 66 | 67 | let filtered_messages = vec![ChatMessage { 68 | role: ChatRole::User, 69 | message_type: MessageType::Text, 70 | content: "What are the latest developments in quantum computing?".to_string(), 71 | }]; 72 | 73 | println!("\n=== Filtered Sources Search Example ==="); 74 | let filtered_response = llm_filtered.chat(&filtered_messages).await?; 75 | println!("Response: {}", filtered_response.text().unwrap_or_default()); 76 | 77 | Ok(()) 78 | } 79 | -------------------------------------------------------------------------------- /examples/xai_streaming_example.rs: -------------------------------------------------------------------------------- 1 | // X.AI streaming chat example demonstrating real-time token generation 2 | use futures::StreamExt; 3 | use llm::{ 4 | builder::{LLMBackend, LLMBuilder}, 5 | chat::ChatMessage, 6 | }; 7 | use std::io::{self, Write}; 8 | 9 | #[tokio::main] 10 | async fn main() -> Result<(), Box> { 11 | // Get X.AI API key from environment variable or use test key as fallback 12 | let api_key = std::env::var("XAI_API_KEY").unwrap_or("xai-TESTKEY".into()); 13 | 14 | // Initialize and configure the LLM client with streaming enabled 15 | let llm = LLMBuilder::new() 16 | .backend(LLMBackend::XAI) 17 | .api_key(api_key) 18 | .model("grok-2-latest") 19 | .max_tokens(1000) 20 | .temperature(0.7) 21 | .stream(true) // Enable streaming responses 22 | .build() 23 | .expect("Failed to build LLM (X.AI)"); 24 | 25 | // Prepare conversation with a prompt that will generate a longer response 26 | let messages = vec![ 27 | ChatMessage::user() 28 | .content("Write a long story about a robot learning to paint. Make it creative and engaging.") 29 | .build(), 30 | ]; 31 | 32 | println!("Starting streaming chat with X.AI...\n"); 33 | 34 | match llm.chat_stream(&messages).await { 35 | Ok(mut stream) => { 36 | let stdout = io::stdout(); 37 | let mut handle = stdout.lock(); 38 | 39 | while let Some(Ok(token)) = stream.next().await { 40 | handle.write_all(token.as_bytes()).unwrap(); 41 | handle.flush().unwrap(); 42 | } 43 | println!("\n\nStreaming completed."); 44 | } 45 | Err(e) => eprintln!("Chat error: {}", e), 46 | } 47 | 48 | Ok(()) 49 | } -------------------------------------------------------------------------------- /examples/xai_structured_output_example.rs: -------------------------------------------------------------------------------- 1 | // Import required modules from the LLM library for xAI integration 2 | use llm::{ 3 | builder::{LLMBackend, LLMBuilder}, // Builder pattern components 4 | chat::{ChatMessage, StructuredOutputFormat}, // Chat-related structures 5 | }; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Box> { 9 | // Get xAI API key from environment variable or use test key as fallback 10 | let api_key = std::env::var("XAI_API_KEY").unwrap_or("sk-TESTKEY".into()); 11 | 12 | // Define a simple JSON schema for structured output 13 | let schema = r#" 14 | { 15 | "name": "student", 16 | "schema": { 17 | "type": "object", 18 | "properties": { 19 | "name": { 20 | "type": "string" 21 | }, 22 | "age": { 23 | "type": "integer" 24 | }, 25 | "is_student": { 26 | "type": "boolean" 27 | } 28 | }, 29 | "required": ["name", "age", "is_student"] 30 | } 31 | } 32 | "#; 33 | let schema: StructuredOutputFormat = serde_json::from_str(schema)?; 34 | 35 | // Initialize and configure the LLM client 36 | let llm = LLMBuilder::new() 37 | .backend(LLMBackend::XAI) // Use xAI as the LLM provider 38 | .api_key(api_key) // Set the API key 39 | .model("grok-2-latest") // Use Grok-2 model 40 | .max_tokens(512) // Limit response length 41 | .temperature(0.7) // Control response randomness (0.0-1.0) 42 | .stream(false) // Disable streaming responses 43 | .system("You are a helpful AI assistant. Please generate a random student using the provided JSON schema.") 44 | .schema(schema) // Set JSON schema for structured output 45 | .build() 46 | .expect("Failed to build LLM (xAI)"); 47 | 48 | // Prepare conversation history with example messages 49 | let messages = vec![ChatMessage::user() 50 | .content("Please generate a random student using the provided JSON schema.") 51 | .build()]; 52 | 53 | // Send chat request and handle the response 54 | match llm.chat(&messages).await { 55 | Ok(text) => println!("Chat response:\n{}", text), 56 | Err(e) => eprintln!("Chat error: {}", e), 57 | } 58 | 59 | Ok(()) 60 | } 61 | -------------------------------------------------------------------------------- /src/api/mod.rs: -------------------------------------------------------------------------------- 1 | //! Server module for exposing LLM functionality via REST API 2 | //! 3 | //! Provides a REST API server that exposes LLM functionality through standardized endpoints. 4 | //! Supports authentication, CORS, and handles chat completion requests. 5 | 6 | mod handlers; 7 | mod types; 8 | 9 | use axum::Router; 10 | use std::sync::Arc; 11 | use tower_http::cors::CorsLayer; 12 | 13 | use crate::chain::LLMRegistry; 14 | use handlers::handle_chat; 15 | 16 | pub use types::{ChatRequest, ChatResponse, Message}; 17 | 18 | /// Main server struct that manages LLM registry and authentication 19 | pub struct Server { 20 | /// Registry containing LLM backends 21 | llms: Arc, 22 | /// Optional authentication key for API requests 23 | pub auth_key: Option, 24 | } 25 | 26 | /// Internal server state shared between request handlers 27 | #[derive(Clone)] 28 | struct ServerState { 29 | /// Shared reference to LLM registry 30 | llms: Arc, 31 | /// Optional authentication key 32 | auth_key: Option, 33 | } 34 | 35 | impl Server { 36 | /// Creates a new server instance with the given LLM registry 37 | /// 38 | /// # Arguments 39 | /// * `llms` - Registry containing LLM backends to expose via API 40 | pub fn new(llms: LLMRegistry) -> Self { 41 | Self { 42 | llms: Arc::new(llms), 43 | auth_key: None, 44 | } 45 | } 46 | 47 | /// Starts the server and listens for requests on the specified address 48 | /// 49 | /// # Arguments 50 | /// * `addr` - Address to bind to (e.g. "127.0.0.1:3000") 51 | /// 52 | /// # Returns 53 | /// * `Ok(())` if server starts successfully 54 | /// * `Err(LLMError)` if server fails to start 55 | pub async fn run(self, addr: &str) -> Result<(), crate::error::LLMError> { 56 | let app = Router::new() 57 | .route("/v1/chat/completions", axum::routing::post(handle_chat)) 58 | .layer(CorsLayer::permissive()) 59 | .with_state(ServerState { 60 | llms: self.llms, 61 | auth_key: self.auth_key, 62 | }); 63 | 64 | let listener = tokio::net::TcpListener::bind(addr) 65 | .await 66 | .map_err(|e| crate::error::LLMError::InvalidRequest(e.to_string()))?; 67 | 68 | axum::serve(listener, app) 69 | .await 70 | .map_err(|e| crate::error::LLMError::InvalidRequest(e.to_string()))?; 71 | 72 | Ok(()) 73 | } 74 | 75 | /// Sets the authentication key required for API requests 76 | /// 77 | /// # Arguments 78 | /// * `key` - API key that clients must provide in Authorization header 79 | pub fn with_auth_key(mut self, key: impl Into) -> Self { 80 | self.auth_key = Some(key.into()); 81 | self 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/api/types.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | /// Request payload for chat completion API endpoint 4 | #[derive(Deserialize)] 5 | pub struct ChatRequest { 6 | /// List of messages in the conversation 7 | pub messages: Option>, 8 | /// Model identifier in format "provider:model_name" 9 | pub model: Option, 10 | /// Optional chain steps for multi-step processing 11 | #[serde(default)] 12 | pub steps: Vec, 13 | /// Optional response transformation function 14 | #[serde(default)] 15 | pub response_transform: Option, 16 | /// Optional temperature parameter 17 | #[serde(default)] 18 | pub temperature: Option, 19 | /// Optional max tokens parameter 20 | #[serde(default)] 21 | pub max_tokens: Option, 22 | } 23 | 24 | /// Chain step configuration for multi-step processing 25 | #[derive(Deserialize)] 26 | pub struct ChainStepRequest { 27 | /// Provider ID for this step 28 | pub provider_id: String, 29 | /// Step identifier 30 | pub id: String, 31 | /// Template with variable substitution 32 | pub template: String, 33 | /// Optional temperature parameter 34 | #[serde(default)] 35 | pub temperature: Option, 36 | /// Optional max tokens parameter 37 | #[serde(default)] 38 | pub max_tokens: Option, 39 | /// Optional response transformation function 40 | #[serde(default)] 41 | pub response_transform: Option, 42 | } 43 | 44 | /// Single message in a chat conversation 45 | #[derive(Deserialize, Serialize)] 46 | pub struct Message { 47 | /// Role of the message sender ("user" or "assistant") 48 | pub role: String, 49 | /// Content of the message 50 | pub content: String, 51 | } 52 | 53 | /// Response payload from chat completion API endpoint 54 | #[derive(Serialize)] 55 | pub struct ChatResponse { 56 | /// Unique identifier for this completion 57 | pub id: String, 58 | /// Object type identifier 59 | pub object: String, 60 | /// Unix timestamp when response was created 61 | pub created: u64, 62 | /// Name of the model that generated the completion 63 | pub model: String, 64 | /// List of completion choices generated 65 | pub choices: Vec, 66 | } 67 | 68 | /// Single completion choice in a chat response 69 | #[derive(Serialize)] 70 | pub struct Choice { 71 | /// Index of this choice in the list 72 | pub index: usize, 73 | /// Generated message for this choice 74 | pub message: Message, 75 | /// Reason why the model stopped generating 76 | pub finish_reason: String, 77 | } 78 | -------------------------------------------------------------------------------- /src/backends/deepseek.rs: -------------------------------------------------------------------------------- 1 | //! DeepSeek API client implementation for chat and completion functionality. 2 | //! 3 | //! This module provides integration with DeepSeek's models through their API. 4 | 5 | use crate::chat::{ChatResponse, Tool}; 6 | #[cfg(feature = "deepseek")] 7 | use crate::{ 8 | chat::{ChatMessage, ChatProvider, ChatRole}, 9 | completion::{CompletionProvider, CompletionRequest, CompletionResponse}, 10 | embedding::EmbeddingProvider, 11 | error::LLMError, 12 | stt::SpeechToTextProvider, 13 | tts::TextToSpeechProvider, 14 | LLMProvider, 15 | }; 16 | use async_trait::async_trait; 17 | use reqwest::Client; 18 | use serde::{Deserialize, Serialize}; 19 | 20 | use crate::ToolCall; 21 | 22 | pub struct DeepSeek { 23 | pub api_key: String, 24 | pub model: String, 25 | pub max_tokens: Option, 26 | pub temperature: Option, 27 | pub system: Option, 28 | pub timeout_seconds: Option, 29 | pub stream: Option, 30 | client: Client, 31 | } 32 | 33 | #[derive(Serialize)] 34 | struct DeepSeekChatMessage<'a> { 35 | role: &'a str, 36 | content: &'a str, 37 | } 38 | 39 | #[derive(Serialize)] 40 | struct DeepSeekChatRequest<'a> { 41 | model: &'a str, 42 | messages: Vec>, 43 | #[serde(skip_serializing_if = "Option::is_none")] 44 | temperature: Option, 45 | stream: bool, 46 | } 47 | 48 | #[derive(Deserialize, Debug)] 49 | struct DeepSeekChatResponse { 50 | choices: Vec, 51 | } 52 | 53 | impl std::fmt::Display for DeepSeekChatResponse { 54 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 55 | write!(f, "{:?}", self) 56 | } 57 | } 58 | 59 | #[derive(Deserialize, Debug)] 60 | struct DeepSeekChatChoice { 61 | message: DeepSeekChatMsg, 62 | } 63 | 64 | #[derive(Deserialize, Debug)] 65 | struct DeepSeekChatMsg { 66 | content: String, 67 | } 68 | impl ChatResponse for DeepSeekChatResponse { 69 | fn text(&self) -> Option { 70 | self.choices.first().and_then(|c| { 71 | if c.message.content.is_empty() { 72 | None 73 | } else { 74 | Some(c.message.content.clone()) 75 | } 76 | }) 77 | } 78 | 79 | fn tool_calls(&self) -> Option> { 80 | None 81 | } 82 | } 83 | 84 | impl DeepSeek { 85 | pub fn new( 86 | api_key: impl Into, 87 | model: Option, 88 | max_tokens: Option, 89 | temperature: Option, 90 | timeout_seconds: Option, 91 | system: Option, 92 | stream: Option, 93 | ) -> Self { 94 | let mut builder = Client::builder(); 95 | if let Some(sec) = timeout_seconds { 96 | builder = builder.timeout(std::time::Duration::from_secs(sec)); 97 | } 98 | Self { 99 | api_key: api_key.into(), 100 | model: model.unwrap_or("deepseek-chat".to_string()), 101 | max_tokens, 102 | temperature, 103 | system, 104 | timeout_seconds, 105 | stream, 106 | client: builder.build().expect("Failed to build reqwest Client"), 107 | } 108 | } 109 | } 110 | 111 | #[async_trait] 112 | impl ChatProvider for DeepSeek { 113 | /// Sends a chat request to DeepSeek's API. 114 | /// 115 | /// # Arguments 116 | /// 117 | /// * `messages` - The conversation history as a slice of chat messages 118 | /// 119 | /// # Returns 120 | /// 121 | /// The provider's response text or an error 122 | async fn chat(&self, messages: &[ChatMessage]) -> Result, LLMError> { 123 | if self.api_key.is_empty() { 124 | return Err(LLMError::AuthError("Missing DeepSeek API key".to_string())); 125 | } 126 | 127 | let mut deepseek_msgs: Vec = messages 128 | .iter() 129 | .map(|m| DeepSeekChatMessage { 130 | role: match m.role { 131 | ChatRole::User => "user", 132 | ChatRole::Assistant => "assistant", 133 | }, 134 | content: &m.content, 135 | }) 136 | .collect(); 137 | 138 | if let Some(system) = &self.system { 139 | deepseek_msgs.insert( 140 | 0, 141 | DeepSeekChatMessage { 142 | role: "system", 143 | content: system, 144 | }, 145 | ); 146 | } 147 | 148 | let body = DeepSeekChatRequest { 149 | model: &self.model, 150 | messages: deepseek_msgs, 151 | temperature: self.temperature, 152 | stream: self.stream.unwrap_or(false), 153 | }; 154 | 155 | if log::log_enabled!(log::Level::Trace) { 156 | if let Ok(json) = serde_json::to_string(&body) { 157 | log::trace!("DeepSeek request payload: {}", json); 158 | } 159 | } 160 | 161 | let mut request = self 162 | .client 163 | .post("https://api.deepseek.com/v1/chat/completions") 164 | .bearer_auth(&self.api_key) 165 | .json(&body); 166 | 167 | if let Some(timeout) = self.timeout_seconds { 168 | request = request.timeout(std::time::Duration::from_secs(timeout)); 169 | } 170 | 171 | let resp = request.send().await?; 172 | 173 | log::debug!("DeepSeek HTTP status: {}", resp.status()); 174 | 175 | let resp = resp.error_for_status()?; 176 | 177 | let json_resp: DeepSeekChatResponse = resp.json().await?; 178 | 179 | Ok(Box::new(json_resp)) 180 | } 181 | 182 | /// Sends a chat request to DeepSeek's API with tools. 183 | /// 184 | /// # Arguments 185 | /// 186 | /// * `messages` - The conversation history as a slice of chat messages 187 | /// * `tools` - Optional slice of tools to use in the chat 188 | /// 189 | /// # Returns 190 | /// 191 | /// The provider's response text or an error 192 | async fn chat_with_tools( 193 | &self, 194 | _messages: &[ChatMessage], 195 | _tools: Option<&[Tool]>, 196 | ) -> Result, LLMError> { 197 | todo!() 198 | } 199 | } 200 | 201 | #[async_trait] 202 | impl CompletionProvider for DeepSeek { 203 | async fn complete(&self, _req: &CompletionRequest) -> Result { 204 | Ok(CompletionResponse { 205 | text: "DeepSeek completion not implemented.".into(), 206 | }) 207 | } 208 | } 209 | 210 | #[async_trait] 211 | impl EmbeddingProvider for DeepSeek { 212 | async fn embed(&self, _text: Vec) -> Result>, LLMError> { 213 | Err(LLMError::ProviderError( 214 | "Embedding not supported".to_string(), 215 | )) 216 | } 217 | } 218 | 219 | #[async_trait] 220 | impl SpeechToTextProvider for DeepSeek { 221 | async fn transcribe(&self, _audio: Vec) -> Result { 222 | Err(LLMError::ProviderError( 223 | "DeepSeek does not implement speech to text endpoint yet.".into(), 224 | )) 225 | } 226 | } 227 | 228 | impl LLMProvider for DeepSeek {} 229 | 230 | #[async_trait] 231 | impl TextToSpeechProvider for DeepSeek { 232 | async fn speech(&self, _text: &str) -> Result, LLMError> { 233 | Err(LLMError::ProviderError( 234 | "Text to speech not supported".to_string(), 235 | )) 236 | } 237 | } 238 | -------------------------------------------------------------------------------- /src/backends/groq.rs: -------------------------------------------------------------------------------- 1 | //! Groq API client implementation for chat functionality. 2 | //! 3 | //! This module provides integration with Groq's LLM models through their API. 4 | 5 | use crate::{ 6 | chat::{ChatMessage, ChatProvider, ChatResponse, ChatRole, Tool}, 7 | completion::{CompletionProvider, CompletionRequest, CompletionResponse}, 8 | embedding::EmbeddingProvider, 9 | error::LLMError, 10 | stt::SpeechToTextProvider, 11 | tts::TextToSpeechProvider, 12 | LLMProvider, ToolCall, 13 | }; 14 | use async_trait::async_trait; 15 | use reqwest::Client; 16 | use serde::{Deserialize, Serialize}; 17 | 18 | /// Client for interacting with Groq's API. 19 | pub struct Groq { 20 | pub api_key: String, 21 | pub model: String, 22 | pub max_tokens: Option, 23 | pub temperature: Option, 24 | pub system: Option, 25 | pub timeout_seconds: Option, 26 | pub stream: Option, 27 | pub top_p: Option, 28 | pub top_k: Option, 29 | client: Client, 30 | } 31 | 32 | #[derive(Serialize)] 33 | struct GroqChatMessage<'a> { 34 | role: &'a str, 35 | content: &'a str, 36 | } 37 | 38 | #[derive(Serialize)] 39 | struct GroqChatRequest<'a> { 40 | model: &'a str, 41 | messages: Vec>, 42 | #[serde(skip_serializing_if = "Option::is_none")] 43 | max_tokens: Option, 44 | #[serde(skip_serializing_if = "Option::is_none")] 45 | temperature: Option, 46 | stream: bool, 47 | #[serde(skip_serializing_if = "Option::is_none")] 48 | top_p: Option, 49 | #[serde(skip_serializing_if = "Option::is_none")] 50 | top_k: Option, 51 | } 52 | 53 | #[derive(Deserialize, Debug)] 54 | struct GroqChatResponse { 55 | choices: Vec, 56 | } 57 | 58 | #[derive(Deserialize, Debug)] 59 | struct GroqChatChoice { 60 | message: GroqChatMsg, 61 | } 62 | 63 | #[derive(Deserialize, Debug)] 64 | struct GroqChatMsg { 65 | content: String, 66 | } 67 | 68 | impl std::fmt::Display for GroqChatResponse { 69 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 70 | write!(f, "{}", self.text().unwrap_or_default()) 71 | } 72 | } 73 | 74 | impl ChatResponse for GroqChatResponse { 75 | fn text(&self) -> Option { 76 | self.choices.first().and_then(|c| { 77 | if c.message.content.is_empty() { 78 | None 79 | } else { 80 | Some(c.message.content.clone()) 81 | } 82 | }) 83 | } 84 | 85 | fn tool_calls(&self) -> Option> { 86 | todo!() 87 | } 88 | } 89 | 90 | #[allow(clippy::too_many_arguments)] 91 | impl Groq { 92 | /// Creates a new Groq client with the specified configuration. 93 | pub fn new( 94 | api_key: impl Into, 95 | model: Option, 96 | max_tokens: Option, 97 | temperature: Option, 98 | timeout_seconds: Option, 99 | system: Option, 100 | stream: Option, 101 | top_p: Option, 102 | top_k: Option, 103 | ) -> Self { 104 | let mut builder = Client::builder(); 105 | if let Some(sec) = timeout_seconds { 106 | builder = builder.timeout(std::time::Duration::from_secs(sec)); 107 | } 108 | Self { 109 | api_key: api_key.into(), 110 | model: model.unwrap_or("llama-3.3-70b-versatile".to_string()), 111 | max_tokens, 112 | temperature, 113 | system, 114 | timeout_seconds, 115 | stream, 116 | top_p, 117 | top_k, 118 | client: builder.build().expect("Failed to build reqwest Client"), 119 | } 120 | } 121 | } 122 | 123 | #[async_trait] 124 | impl ChatProvider for Groq { 125 | async fn chat(&self, messages: &[ChatMessage]) -> Result, LLMError> { 126 | if self.api_key.is_empty() { 127 | return Err(LLMError::AuthError("Missing Groq API key".to_string())); 128 | } 129 | 130 | let mut groq_msgs: Vec = messages 131 | .iter() 132 | .map(|m| GroqChatMessage { 133 | role: match m.role { 134 | ChatRole::User => "user", 135 | ChatRole::Assistant => "assistant", 136 | }, 137 | content: &m.content, 138 | }) 139 | .collect(); 140 | 141 | if let Some(system) = &self.system { 142 | groq_msgs.insert( 143 | 0, 144 | GroqChatMessage { 145 | role: "system", 146 | content: system, 147 | }, 148 | ); 149 | } 150 | 151 | let body = GroqChatRequest { 152 | model: &self.model, 153 | messages: groq_msgs, 154 | max_tokens: self.max_tokens, 155 | temperature: self.temperature, 156 | stream: self.stream.unwrap_or(false), 157 | top_p: self.top_p, 158 | top_k: self.top_k, 159 | }; 160 | 161 | if log::log_enabled!(log::Level::Trace) { 162 | if let Ok(json) = serde_json::to_string(&body) { 163 | log::trace!("Groq request payload: {}", json); 164 | } 165 | } 166 | 167 | let mut request = self 168 | .client 169 | .post("https://api.groq.com/openai/v1/chat/completions") 170 | .header("Content-Type", "application/json") 171 | .bearer_auth(&self.api_key) 172 | .json(&body); 173 | 174 | if let Some(timeout) = self.timeout_seconds { 175 | request = request.timeout(std::time::Duration::from_secs(timeout)); 176 | } 177 | 178 | let resp = request.send().await?; 179 | 180 | log::debug!("Groq HTTP status: {}", resp.status()); 181 | 182 | let resp = resp.error_for_status()?; 183 | let json_resp: GroqChatResponse = resp.json().await?; 184 | 185 | Ok(Box::new(json_resp)) 186 | } 187 | 188 | async fn chat_with_tools( 189 | &self, 190 | _messages: &[ChatMessage], 191 | _tools: Option<&[Tool]>, 192 | ) -> Result, LLMError> { 193 | todo!() 194 | } 195 | } 196 | 197 | #[async_trait] 198 | impl CompletionProvider for Groq { 199 | async fn complete(&self, _req: &CompletionRequest) -> Result { 200 | Ok(CompletionResponse { 201 | text: "Groq completion not implemented.".into(), 202 | }) 203 | } 204 | } 205 | 206 | #[async_trait] 207 | impl EmbeddingProvider for Groq { 208 | async fn embed(&self, _text: Vec) -> Result>, LLMError> { 209 | Err(LLMError::ProviderError( 210 | "Embedding not supported".to_string(), 211 | )) 212 | } 213 | } 214 | 215 | #[async_trait] 216 | impl SpeechToTextProvider for Groq { 217 | async fn transcribe(&self, _audio: Vec) -> Result { 218 | Err(LLMError::ProviderError( 219 | "Groq does not implement speech to text endpoint yet.".into(), 220 | )) 221 | } 222 | } 223 | 224 | #[async_trait] 225 | impl TextToSpeechProvider for Groq {} 226 | 227 | impl LLMProvider for Groq {} 228 | -------------------------------------------------------------------------------- /src/backends/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "openai")] 2 | pub mod openai; 3 | 4 | #[cfg(feature = "anthropic")] 5 | pub mod anthropic; 6 | 7 | #[cfg(feature = "ollama")] 8 | pub mod ollama; 9 | 10 | #[cfg(feature = "deepseek")] 11 | pub mod deepseek; 12 | 13 | #[cfg(feature = "xai")] 14 | pub mod xai; 15 | 16 | #[cfg(feature = "phind")] 17 | pub mod phind; 18 | 19 | #[cfg(feature = "google")] 20 | pub mod google; 21 | 22 | #[cfg(feature = "groq")] 23 | pub mod groq; 24 | 25 | #[cfg(feature = "azure_openai")] 26 | pub mod azure_openai; 27 | 28 | #[cfg(feature = "elevenlabs")] 29 | pub mod elevenlabs; 30 | -------------------------------------------------------------------------------- /src/chain/mod.rs: -------------------------------------------------------------------------------- 1 | mod multi; 2 | 3 | use crate::{error::LLMError, LLMProvider}; 4 | use std::collections::HashMap; 5 | 6 | pub use multi::{ 7 | LLMRegistry, LLMRegistryBuilder, MultiChainStepBuilder, MultiChainStepMode, MultiPromptChain, 8 | }; 9 | 10 | /// Execution mode for a chain step 11 | #[derive(Debug, Clone)] 12 | pub enum ChainStepMode { 13 | /// Execute step using chat completion 14 | Chat, 15 | /// Execute step using text completion 16 | Completion, 17 | } 18 | 19 | /// Represents a single step in a prompt chain 20 | #[derive(Debug, Clone)] 21 | pub struct ChainStep { 22 | /// Unique identifier for this step 23 | pub id: String, 24 | /// Prompt template with {{variable}} placeholders 25 | pub template: String, 26 | /// Execution mode (chat or completion) 27 | pub mode: ChainStepMode, 28 | /// Optional temperature parameter (0.0-1.0) controlling randomness 29 | pub temperature: Option, 30 | /// Optional maximum tokens to generate in response 31 | pub max_tokens: Option, 32 | /// Optional top_p parameter for nucleus sampling 33 | pub top_p: Option, 34 | } 35 | 36 | /// Builder pattern for constructing ChainStep instances 37 | pub struct ChainStepBuilder { 38 | id: String, 39 | template: String, 40 | mode: ChainStepMode, 41 | temperature: Option, 42 | max_tokens: Option, 43 | top_p: Option, 44 | top_k: Option, 45 | } 46 | 47 | impl ChainStepBuilder { 48 | /// Creates a new ChainStepBuilder 49 | /// 50 | /// # Arguments 51 | /// * `id` - Unique identifier for the step 52 | /// * `template` - Prompt template with {{variable}} placeholders 53 | /// * `mode` - Execution mode (chat or completion) 54 | pub fn new(id: impl Into, template: impl Into, mode: ChainStepMode) -> Self { 55 | Self { 56 | id: id.into(), 57 | template: template.into(), 58 | mode, 59 | temperature: None, 60 | max_tokens: None, 61 | top_p: None, 62 | top_k: None, 63 | } 64 | } 65 | 66 | /// Sets the temperature parameter 67 | pub fn temperature(mut self, temp: f32) -> Self { 68 | self.temperature = Some(temp); 69 | self 70 | } 71 | 72 | /// Sets the maximum tokens parameter 73 | pub fn max_tokens(mut self, mt: u32) -> Self { 74 | self.max_tokens = Some(mt); 75 | self 76 | } 77 | 78 | /// Sets the top_p parameter 79 | pub fn top_p(mut self, val: f32) -> Self { 80 | self.top_p = Some(val); 81 | self 82 | } 83 | 84 | /// Sets the top_k parameter 85 | pub fn top_k(mut self, val: u32) -> Self { 86 | self.top_k = Some(val); 87 | self 88 | } 89 | 90 | /// Builds and returns a ChainStep instance 91 | pub fn build(self) -> ChainStep { 92 | ChainStep { 93 | id: self.id, 94 | template: self.template, 95 | mode: self.mode, 96 | temperature: self.temperature, 97 | max_tokens: self.max_tokens, 98 | top_p: self.top_p, 99 | } 100 | } 101 | } 102 | 103 | /// Manages a sequence of prompt steps with variable substitution 104 | pub struct PromptChain<'a> { 105 | llm: &'a dyn LLMProvider, 106 | steps: Vec, 107 | memory: HashMap, 108 | } 109 | 110 | impl<'a> PromptChain<'a> { 111 | /// Creates a new PromptChain with the given LLM provider 112 | pub fn new(llm: &'a dyn LLMProvider) -> Self { 113 | Self { 114 | llm, 115 | steps: Vec::new(), 116 | memory: HashMap::new(), 117 | } 118 | } 119 | 120 | /// Adds a step to the chain 121 | pub fn step(mut self, step: ChainStep) -> Self { 122 | self.steps.push(step); 123 | self 124 | } 125 | 126 | /// Executes all steps in the chain and returns the results 127 | pub async fn run(mut self) -> Result, LLMError> { 128 | for step in &self.steps { 129 | let prompt = self.apply_template(&step.template); 130 | 131 | let response_text = match step.mode { 132 | ChainStepMode::Chat => { 133 | let messages = vec![crate::chat::ChatMessage { 134 | role: crate::chat::ChatRole::User, 135 | message_type: crate::chat::MessageType::Text, 136 | content: prompt, 137 | }]; 138 | self.llm.chat(&messages).await? 139 | } 140 | ChainStepMode::Completion => { 141 | let mut req = crate::completion::CompletionRequest::new(prompt); 142 | req.max_tokens = step.max_tokens; 143 | req.temperature = step.temperature; 144 | let resp = self.llm.complete(&req).await?; 145 | Box::new(resp) 146 | } 147 | }; 148 | 149 | self.memory 150 | .insert(step.id.clone(), response_text.text().unwrap_or_default()); 151 | } 152 | 153 | Ok(self.memory) 154 | } 155 | 156 | /// Replaces {{variable}} placeholders in template with values from memory 157 | fn apply_template(&self, input: &str) -> String { 158 | let mut result = input.to_string(); 159 | for (k, v) in &self.memory { 160 | let pattern = format!("{{{{{}}}}}", k); 161 | result = result.replace(&pattern, v); 162 | } 163 | result 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /src/chain/multi.rs: -------------------------------------------------------------------------------- 1 | //! Module for chaining multiple LLM backends in a single prompt sequence. 2 | //! Each step can reference a distinct provider_id ("openai", "anthro", etc.). 3 | 4 | use std::collections::HashMap; 5 | 6 | use crate::{ 7 | chat::{ChatMessage, ChatRole, MessageType}, 8 | completion::CompletionRequest, 9 | error::LLMError, 10 | LLMProvider, 11 | }; 12 | 13 | #[cfg(feature = "api")] 14 | use crate::api::Server; 15 | 16 | /// Stores multiple LLM backends (OpenAI, Anthropic, etc.) identified by a key 17 | #[derive(Default)] 18 | pub struct LLMRegistry { 19 | pub backends: HashMap>, 20 | } 21 | 22 | impl LLMRegistry { 23 | pub fn new() -> Self { 24 | Self { 25 | backends: HashMap::new(), 26 | } 27 | } 28 | 29 | /// Inserts a backend under an identifier, e.g. "openai" 30 | pub fn insert(&mut self, id: impl Into, llm: Box) { 31 | self.backends.insert(id.into(), llm); 32 | } 33 | 34 | /// Retrieves a backend by its identifier 35 | pub fn get(&self, id: &str) -> Option<&dyn LLMProvider> { 36 | self.backends.get(id).map(|b| b.as_ref()) 37 | } 38 | 39 | #[cfg(feature = "api")] 40 | /// Starts a REST API server on the specified address 41 | pub async fn serve(self, addr: impl Into) -> Result<(), LLMError> { 42 | let server = Server::new(self); 43 | server.run(&addr.into()).await?; 44 | 45 | Ok(()) 46 | } 47 | } 48 | 49 | /// Builder pattern for LLMRegistry 50 | #[derive(Default)] 51 | pub struct LLMRegistryBuilder { 52 | registry: LLMRegistry, 53 | } 54 | 55 | impl LLMRegistryBuilder { 56 | pub fn new() -> Self { 57 | Self::default() 58 | } 59 | 60 | /// Adds a backend under the given id 61 | pub fn register(mut self, id: impl Into, llm: Box) -> Self { 62 | self.registry.insert(id, llm); 63 | self 64 | } 65 | 66 | /// Builds the final LLMRegistry 67 | pub fn build(self) -> LLMRegistry { 68 | self.registry 69 | } 70 | } 71 | 72 | /// Response transformation function 73 | type ResponseTransform = Box String + Send + Sync>; 74 | 75 | /// Execution mode for a step: Chat or Completion 76 | #[derive(Debug, Clone)] 77 | pub enum MultiChainStepMode { 78 | Chat, 79 | Completion, 80 | SpeechToText, 81 | } 82 | 83 | /// Multi-backend chain step 84 | pub struct MultiChainStep { 85 | provider_id: String, 86 | id: String, 87 | template: String, 88 | mode: MultiChainStepMode, 89 | 90 | // Override parameters 91 | temperature: Option, 92 | max_tokens: Option, 93 | 94 | // Response transformation 95 | response_transform: Option, 96 | } 97 | 98 | /// Builder for MultiChainStep (Stripe-style) 99 | pub struct MultiChainStepBuilder { 100 | provider_id: Option, 101 | id: Option, 102 | template: Option, 103 | mode: MultiChainStepMode, 104 | 105 | temperature: Option, 106 | top_p: Option, 107 | max_tokens: Option, 108 | response_transform: Option, 109 | } 110 | 111 | impl MultiChainStepBuilder { 112 | pub fn new(mode: MultiChainStepMode) -> Self { 113 | Self { 114 | provider_id: None, 115 | id: None, 116 | template: None, 117 | mode, 118 | temperature: None, 119 | top_p: None, 120 | max_tokens: None, 121 | response_transform: None, 122 | } 123 | } 124 | 125 | /// Backend identifier to use, e.g. "openai" 126 | pub fn provider_id(mut self, pid: impl Into) -> Self { 127 | self.provider_id = Some(pid.into()); 128 | self 129 | } 130 | 131 | /// Unique identifier for the step, e.g. "calc1" 132 | pub fn id(mut self, id: impl Into) -> Self { 133 | self.id = Some(id.into()); 134 | self 135 | } 136 | 137 | /// The prompt or template (e.g. "2 * 4 = ?") 138 | pub fn template(mut self, tmpl: impl Into) -> Self { 139 | self.template = Some(tmpl.into()); 140 | self 141 | } 142 | 143 | // Parameters 144 | pub fn temperature(mut self, t: f32) -> Self { 145 | self.temperature = Some(t); 146 | self 147 | } 148 | 149 | pub fn top_p(mut self, p: f32) -> Self { 150 | self.top_p = Some(p); 151 | self 152 | } 153 | 154 | pub fn max_tokens(mut self, mt: u32) -> Self { 155 | self.max_tokens = Some(mt); 156 | self 157 | } 158 | 159 | pub fn response_transform(mut self, func: F) -> Self 160 | where 161 | F: Fn(String) -> String + Send + Sync + 'static, 162 | { 163 | self.response_transform = Some(Box::new(func)); 164 | self 165 | } 166 | 167 | /// Builds the step 168 | pub fn build(self) -> Result { 169 | let provider_id = self 170 | .provider_id 171 | .ok_or_else(|| LLMError::InvalidRequest("No provider_id set".into()))?; 172 | let id = self 173 | .id 174 | .ok_or_else(|| LLMError::InvalidRequest("No step id set".into()))?; 175 | let tmpl = self 176 | .template 177 | .ok_or_else(|| LLMError::InvalidRequest("No template set".into()))?; 178 | 179 | Ok(MultiChainStep { 180 | provider_id, 181 | id, 182 | template: tmpl, 183 | mode: self.mode, 184 | temperature: self.temperature, 185 | max_tokens: self.max_tokens, 186 | response_transform: self.response_transform, 187 | }) 188 | } 189 | } 190 | 191 | /// The multi-backend chain 192 | pub struct MultiPromptChain<'a> { 193 | registry: &'a LLMRegistry, 194 | steps: Vec, 195 | memory: HashMap, // stores responses 196 | } 197 | 198 | impl<'a> MultiPromptChain<'a> { 199 | pub fn new(registry: &'a LLMRegistry) -> Self { 200 | Self { 201 | registry, 202 | steps: vec![], 203 | memory: HashMap::new(), 204 | } 205 | } 206 | 207 | /// Adds a step 208 | pub fn step(mut self, step: MultiChainStep) -> Self { 209 | self.steps.push(step); 210 | self 211 | } 212 | 213 | /// Executes all steps 214 | pub async fn run(mut self) -> Result, LLMError> { 215 | for step in &self.steps { 216 | // 1) Replace {{xyz}} in template with existing memory 217 | let prompt_text = self.replace_template(&step.template); 218 | 219 | // 2) Get the right backend 220 | let llm = self.registry.get(&step.provider_id).ok_or_else(|| { 221 | LLMError::InvalidRequest(format!( 222 | "No provider with id '{}' found in registry", 223 | step.provider_id 224 | )) 225 | })?; 226 | 227 | // 3) Execute 228 | let mut response = match step.mode { 229 | MultiChainStepMode::Chat => { 230 | let messages = vec![ChatMessage { 231 | role: ChatRole::User, 232 | message_type: MessageType::Text, 233 | content: prompt_text, 234 | }]; 235 | llm.chat(&messages).await?.text().unwrap_or_default() 236 | } 237 | MultiChainStepMode::Completion => { 238 | let mut req = CompletionRequest::new(prompt_text); 239 | req.temperature = step.temperature; 240 | req.max_tokens = step.max_tokens; 241 | let c = llm.complete(&req).await?; 242 | c.text.to_string() 243 | } 244 | MultiChainStepMode::SpeechToText => llm.transcribe_file(&prompt_text).await?, 245 | }; 246 | 247 | if let Some(transform) = &step.response_transform { 248 | response = transform(response); 249 | } 250 | 251 | // 4) Store the response 252 | self.memory.insert(step.id.clone(), response); 253 | } 254 | Ok(self.memory) 255 | } 256 | 257 | fn replace_template(&self, input: &str) -> String { 258 | let mut out = input.to_string(); 259 | for (k, v) in &self.memory { 260 | let pattern = format!("{{{{{}}}}}", k); 261 | out = out.replace(&pattern, v); 262 | } 263 | out 264 | } 265 | 266 | /// Adds multiple steps at once 267 | pub fn chain(mut self, steps: Vec) -> Self { 268 | self.steps.extend(steps); 269 | self 270 | } 271 | } 272 | -------------------------------------------------------------------------------- /src/completion/mod.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | 3 | use crate::{chat::ChatResponse, error::LLMError, ToolCall}; 4 | 5 | /// A request for text completion from an LLM provider. 6 | #[derive(Debug, Clone)] 7 | pub struct CompletionRequest { 8 | /// The input prompt text to complete 9 | pub prompt: String, 10 | /// Optional maximum number of tokens to generate 11 | pub max_tokens: Option, 12 | /// Optional temperature parameter to control randomness (0.0-1.0) 13 | pub temperature: Option, 14 | } 15 | 16 | /// A response containing generated text from a completion request. 17 | #[derive(Debug, Clone)] 18 | pub struct CompletionResponse { 19 | /// The generated completion text 20 | pub text: String, 21 | } 22 | 23 | impl ChatResponse for CompletionResponse { 24 | fn text(&self) -> Option { 25 | Some(self.text.clone()) 26 | } 27 | 28 | fn tool_calls(&self) -> Option> { 29 | None 30 | } 31 | } 32 | 33 | impl CompletionRequest { 34 | /// Creates a new completion request with just a prompt. 35 | /// 36 | /// # Arguments 37 | /// 38 | /// * `prompt` - The input text to complete 39 | pub fn new(prompt: impl Into) -> Self { 40 | Self { 41 | prompt: prompt.into(), 42 | max_tokens: None, 43 | temperature: None, 44 | } 45 | } 46 | 47 | /// Creates a builder for constructing a completion request. 48 | /// 49 | /// # Arguments 50 | /// 51 | /// * `prompt` - The input text to complete 52 | pub fn builder(prompt: impl Into) -> CompletionRequestBuilder { 53 | CompletionRequestBuilder { 54 | prompt: prompt.into(), 55 | max_tokens: None, 56 | temperature: None, 57 | } 58 | } 59 | } 60 | 61 | /// Builder for constructing completion requests with optional parameters. 62 | #[derive(Debug, Clone)] 63 | pub struct CompletionRequestBuilder { 64 | /// The input prompt text to complete 65 | pub prompt: String, 66 | /// Optional maximum number of tokens to generate 67 | pub max_tokens: Option, 68 | /// Optional temperature parameter to control randomness (0.0-1.0) 69 | pub temperature: Option, 70 | } 71 | 72 | impl CompletionRequestBuilder { 73 | /// Sets the maximum number of tokens to generate. 74 | pub fn max_tokens(mut self, val: u32) -> Self { 75 | self.max_tokens = Some(val); 76 | self 77 | } 78 | 79 | /// Sets the temperature parameter for controlling randomness. 80 | pub fn temperature(mut self, val: f32) -> Self { 81 | self.temperature = Some(val); 82 | self 83 | } 84 | 85 | /// Builds the completion request with the configured parameters. 86 | pub fn build(self) -> CompletionRequest { 87 | CompletionRequest { 88 | prompt: self.prompt, 89 | max_tokens: self.max_tokens, 90 | temperature: self.temperature, 91 | } 92 | } 93 | } 94 | 95 | /// Trait for providers that support text completion requests. 96 | #[async_trait] 97 | pub trait CompletionProvider { 98 | /// Sends a completion request to generate text. 99 | /// 100 | /// # Arguments 101 | /// 102 | /// * `req` - The completion request parameters 103 | /// 104 | /// # Returns 105 | /// 106 | /// The generated completion text or an error 107 | async fn complete(&self, req: &CompletionRequest) -> Result; 108 | } 109 | 110 | impl std::fmt::Display for CompletionResponse { 111 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 112 | write!(f, "{}", self.text) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/embedding/mod.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | 3 | use crate::error::LLMError; 4 | 5 | #[async_trait] 6 | pub trait EmbeddingProvider { 7 | async fn embed(&self, input: Vec) -> Result>, LLMError>; 8 | } 9 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | /// Error types that can occur when interacting with LLM providers. 4 | #[derive(Debug)] 5 | pub enum LLMError { 6 | /// HTTP request/response errors 7 | HttpError(String), 8 | /// Authentication and authorization errors 9 | AuthError(String), 10 | /// Invalid request parameters or format 11 | InvalidRequest(String), 12 | /// Errors returned by the LLM provider 13 | ProviderError(String), 14 | /// API response parsing or format error 15 | ResponseFormatError { 16 | message: String, 17 | raw_response: String, 18 | }, 19 | /// Generic error 20 | Generic(String), 21 | /// JSON serialization/deserialization errors 22 | JsonError(String), 23 | /// Tool configuration error 24 | ToolConfigError(String), 25 | } 26 | 27 | impl fmt::Display for LLMError { 28 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 29 | match self { 30 | LLMError::HttpError(e) => write!(f, "HTTP Error: {}", e), 31 | LLMError::AuthError(e) => write!(f, "Auth Error: {}", e), 32 | LLMError::InvalidRequest(e) => write!(f, "Invalid Request: {}", e), 33 | LLMError::ProviderError(e) => write!(f, "Provider Error: {}", e), 34 | LLMError::Generic(e) => write!(f, "Generic Error : {}", e), 35 | LLMError::ResponseFormatError { 36 | message, 37 | raw_response, 38 | } => { 39 | write!( 40 | f, 41 | "Response Format Error: {}. Raw response: {}", 42 | message, raw_response 43 | ) 44 | } 45 | LLMError::JsonError(e) => write!(f, "JSON Parse Error: {}", e), 46 | LLMError::ToolConfigError(e) => write!(f, "Tool Configuration Error: {}", e), 47 | } 48 | } 49 | } 50 | 51 | impl std::error::Error for LLMError {} 52 | 53 | /// Converts reqwest HTTP errors into LlmErrors 54 | impl From for LLMError { 55 | fn from(err: reqwest::Error) -> Self { 56 | LLMError::HttpError(err.to_string()) 57 | } 58 | } 59 | 60 | impl From for LLMError { 61 | fn from(err: serde_json::Error) -> Self { 62 | LLMError::JsonError(format!( 63 | "{} at line {} column {}", 64 | err, 65 | err.line(), 66 | err.column() 67 | )) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/evaluator/mod.rs: -------------------------------------------------------------------------------- 1 | //! Module for evaluating and comparing responses from multiple LLM providers. 2 | //! 3 | //! This module provides functionality to run the same prompt through multiple LLMs 4 | //! and score their responses using custom evaluation functions. 5 | 6 | mod parallel; 7 | 8 | use crate::{chat::ChatMessage, error::LLMError, LLMProvider}; 9 | 10 | pub use parallel::{ParallelEvalResult, ParallelEvaluator}; 11 | 12 | /// Type alias for scoring functions that evaluate LLM responses 13 | pub type ScoringFn = dyn Fn(&str) -> f32 + Send + Sync + 'static; 14 | 15 | /// Evaluator for comparing responses from multiple LLM providers 16 | pub struct LLMEvaluator { 17 | /// Collection of LLM providers to evaluate 18 | llms: Vec>, 19 | /// Optional scoring function to evaluate responses 20 | scorings_fns: Vec>, 21 | } 22 | 23 | impl LLMEvaluator { 24 | /// Creates a new evaluator with the given LLM providers 25 | /// 26 | /// # Arguments 27 | /// * `llms` - Vector of LLM providers to evaluate 28 | pub fn new(llms: Vec>) -> Self { 29 | Self { 30 | llms, 31 | scorings_fns: Vec::new(), 32 | } 33 | } 34 | 35 | /// Adds a scoring function to evaluate LLM responses 36 | /// 37 | /// # Arguments 38 | /// * `f` - Function that takes a response string and returns a score 39 | pub fn scoring(mut self, f: F) -> Self 40 | where 41 | F: Fn(&str) -> f32 + Send + Sync + 'static, 42 | { 43 | self.scorings_fns.push(Box::new(f)); 44 | self 45 | } 46 | 47 | /// Evaluates chat responses from all providers for the given messages 48 | /// 49 | /// # Arguments 50 | /// * `messages` - Chat messages to send to each provider 51 | /// 52 | /// # Returns 53 | /// Vector of evaluation results containing responses and scores 54 | pub async fn evaluate_chat( 55 | &self, 56 | messages: &[ChatMessage], 57 | ) -> Result, LLMError> { 58 | let mut results = Vec::new(); 59 | for llm in &self.llms { 60 | let response = llm.chat(messages).await?; 61 | let score = self.compute_score(&response.text().unwrap_or_default()); 62 | results.push(EvalResult { 63 | text: response.text().unwrap_or_default(), 64 | score, 65 | }); 66 | } 67 | Ok(results) 68 | } 69 | 70 | /// Computes the score for a given response 71 | /// 72 | /// # Arguments 73 | /// * `response` - The response to score 74 | /// 75 | /// # Returns 76 | /// The computed score 77 | fn compute_score(&self, response: &str) -> f32 { 78 | let mut total = 0.0; 79 | for sc in &self.scorings_fns { 80 | total += sc(response); 81 | } 82 | total 83 | } 84 | } 85 | 86 | /// Result of evaluating an LLM response 87 | pub struct EvalResult { 88 | /// The text response from the LLM 89 | pub text: String, 90 | /// Score assigned by the scoring function, if any 91 | pub score: f32, 92 | } 93 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! LLM (Rust LLM) is a unified interface for interacting with Large Language Model providers. 2 | //! 3 | //! # Overview 4 | //! This crate provides a consistent API for working with different LLM backends by abstracting away 5 | //! provider-specific implementation details. It supports: 6 | //! 7 | //! - Chat-based interactions 8 | //! - Text completion 9 | //! - Embeddings generation 10 | //! - Multiple providers (OpenAI, Anthropic, etc.) 11 | //! - Request validation and retry logic 12 | //! 13 | //! # Architecture 14 | //! The crate is organized into modules that handle different aspects of LLM interactions: 15 | 16 | // Re-export for convenience 17 | pub use async_trait::async_trait; 18 | 19 | use chat::Tool; 20 | use serde::{Deserialize, Serialize}; 21 | 22 | /// Backend implementations for supported LLM providers like OpenAI, Anthropic, etc. 23 | pub mod backends; 24 | 25 | /// Builder pattern for configuring and instantiating LLM providers 26 | pub mod builder; 27 | 28 | /// Chain multiple LLM providers together for complex workflows 29 | pub mod chain; 30 | 31 | /// Chat-based interactions with language models (e.g. ChatGPT style) 32 | pub mod chat; 33 | 34 | /// Text completion capabilities (e.g. GPT-3 style completion) 35 | pub mod completion; 36 | 37 | /// Vector embeddings generation for text 38 | pub mod embedding; 39 | 40 | /// Error types and handling 41 | pub mod error; 42 | 43 | /// Validation wrapper for LLM providers with retry capabilities 44 | pub mod validated_llm; 45 | 46 | /// Evaluator for LLM providers 47 | pub mod evaluator; 48 | 49 | /// Speech-to-text support 50 | pub mod stt; 51 | 52 | /// Text-to-speech support 53 | pub mod tts; 54 | 55 | /// Secret store for storing API keys and other sensitive information 56 | pub mod secret_store; 57 | 58 | /// Memory providers for storing and retrieving conversation history 59 | pub mod memory; 60 | 61 | #[cfg(feature = "api")] 62 | pub mod api; 63 | 64 | 65 | #[inline] 66 | /// Initialize logging using env_logger if the "logging" feature is enabled. 67 | /// This is a no-op if the feature is not enabled. 68 | pub fn init_logging() { 69 | #[cfg(feature = "logging")] 70 | { 71 | let _ = env_logger::try_init(); 72 | } 73 | } 74 | 75 | /// Core trait that all LLM providers must implement, combining chat, completion 76 | /// and embedding capabilities into a unified interface 77 | pub trait LLMProvider: 78 | chat::ChatProvider 79 | + completion::CompletionProvider 80 | + embedding::EmbeddingProvider 81 | + stt::SpeechToTextProvider 82 | + tts::TextToSpeechProvider 83 | { 84 | fn tools(&self) -> Option<&[Tool]> { 85 | None 86 | } 87 | } 88 | 89 | /// Tool call represents a function call that an LLM wants to make. 90 | /// This is a standardized structure used across all providers. 91 | #[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq)] 92 | pub struct ToolCall { 93 | /// The ID of the tool call. 94 | pub id: String, 95 | /// The type of the tool call (usually "function"). 96 | #[serde(rename = "type")] 97 | pub call_type: String, 98 | /// The function to call. 99 | pub function: FunctionCall, 100 | } 101 | 102 | /// FunctionCall contains details about which function to call and with what arguments. 103 | #[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq)] 104 | pub struct FunctionCall { 105 | /// The name of the function to call. 106 | pub name: String, 107 | /// The arguments to pass to the function, typically serialized as a JSON string. 108 | pub arguments: String, 109 | } 110 | -------------------------------------------------------------------------------- /src/memory/chat_wrapper.rs: -------------------------------------------------------------------------------- 1 | //! Chat wrapper that adds memory capabilities to any ChatProvider. 2 | 3 | use async_trait::async_trait; 4 | use std::sync::Arc; 5 | use tokio::sync::RwLock; 6 | 7 | use crate::{ 8 | chat::{ChatMessage, ChatProvider, ChatResponse, Tool}, 9 | completion::{CompletionProvider, CompletionRequest, CompletionResponse}, 10 | embedding::EmbeddingProvider, 11 | error::LLMError, 12 | memory::MemoryProvider, 13 | stt::SpeechToTextProvider, 14 | tts::TextToSpeechProvider, 15 | LLMProvider, 16 | }; 17 | 18 | /// A wrapper that adds memory capabilities to any ChatProvider. 19 | /// 20 | /// This wrapper implements all LLM provider traits and adds automatic memory 21 | /// management to chat conversations without requiring backend modifications. 22 | pub struct ChatWithMemory { 23 | provider: Box, 24 | memory: Arc>>, 25 | role: Option, 26 | } 27 | 28 | impl ChatWithMemory { 29 | /// Create a new memory-enabled chat wrapper. 30 | /// 31 | /// # Arguments 32 | /// 33 | /// * `provider` - The underlying LLM provider 34 | /// * `memory` - Memory provider for conversation history 35 | /// * `role` - Optional role name for multi-agent scenarios 36 | pub fn new( 37 | provider: Box, 38 | memory: Arc>>, 39 | role: Option, 40 | ) -> Self { 41 | Self { 42 | provider, 43 | memory, 44 | role, 45 | } 46 | } 47 | 48 | /// Get the underlying provider 49 | pub fn inner(&self) -> &dyn LLMProvider { 50 | self.provider.as_ref() 51 | } 52 | 53 | /// Get current memory contents for debugging 54 | pub async fn memory_contents(&self) -> Vec { 55 | let memory_guard = self.memory.read().await; 56 | memory_guard.recall("", None).await.unwrap_or_default() 57 | } 58 | } 59 | 60 | #[async_trait] 61 | impl ChatProvider for ChatWithMemory { 62 | async fn chat_with_tools( 63 | &self, 64 | messages: &[ChatMessage], 65 | tools: Option<&[Tool]>, 66 | ) -> Result, LLMError> { 67 | let mut write = self.memory.write().await; 68 | 69 | for m in messages { 70 | let mut tagged = m.clone(); 71 | if let Some(role) = &self.role { 72 | tagged.content = format!("[{role}] User: {}", m.content); 73 | } 74 | write.remember(&tagged).await?; 75 | } 76 | 77 | let mut context = write.recall("", None).await?; 78 | let needs_summary = write.needs_summary(); 79 | drop(write); 80 | 81 | if needs_summary { 82 | let summary = self.provider.summarize_history(&context).await?; 83 | let mut write = self.memory.write().await; 84 | write.replace_with_summary(summary); 85 | context = write.recall("", None).await?; 86 | } 87 | 88 | context.extend_from_slice(messages); 89 | let response = self.provider.chat_with_tools(&context, tools).await?; 90 | 91 | if let Some(text) = response.text() { 92 | let memory = self.memory.clone(); 93 | let tag = self.role.clone(); 94 | tokio::spawn(async move { 95 | let mut mem = memory.write().await; 96 | let text = match tag { 97 | Some(r) => format!("[{r}] Assistant: {text}"), 98 | None => text, 99 | }; 100 | mem.remember(&ChatMessage::assistant().content(text).build()).await.ok(); 101 | }); 102 | } 103 | 104 | Ok(response) 105 | } 106 | 107 | async fn memory_contents(&self) -> Option> { 108 | let memory_guard = self.memory.read().await; 109 | Some(memory_guard.recall("", None).await.unwrap_or_default()) 110 | } 111 | } 112 | 113 | // Forward all other traits to the underlying provider 114 | #[async_trait] 115 | impl CompletionProvider for ChatWithMemory { 116 | async fn complete(&self, request: &CompletionRequest) -> Result { 117 | self.provider.complete(request).await 118 | } 119 | } 120 | 121 | #[async_trait] 122 | impl EmbeddingProvider for ChatWithMemory { 123 | async fn embed(&self, input: Vec) -> Result>, LLMError> { 124 | self.provider.embed(input).await 125 | } 126 | } 127 | 128 | #[async_trait] 129 | impl SpeechToTextProvider for ChatWithMemory { 130 | async fn transcribe(&self, audio_data: Vec) -> Result { 131 | self.provider.transcribe(audio_data).await 132 | } 133 | } 134 | 135 | #[async_trait] 136 | impl TextToSpeechProvider for ChatWithMemory { 137 | async fn speech(&self, text: &str) -> Result, LLMError> { 138 | self.provider.speech(text).await 139 | } 140 | } 141 | 142 | impl LLMProvider for ChatWithMemory { 143 | fn tools(&self) -> Option<&[Tool]> { 144 | self.provider.tools() 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/memory/mod.rs: -------------------------------------------------------------------------------- 1 | //! Memory module for storing and retrieving conversation history. 2 | //! 3 | //! This module provides various memory implementations for LLM conversations: 4 | //! - SlidingWindowMemory: Simple FIFO memory with size limit 5 | //! - EmbeddingMemory: Semantic search using embeddings (future) 6 | //! - RAGMemory: Document-based retrieval (future) 7 | 8 | use async_trait::async_trait; 9 | use serde::{Deserialize, Serialize}; 10 | 11 | use crate::{chat::ChatMessage, error::LLMError}; 12 | 13 | pub mod chat_wrapper; 14 | pub mod shared_memory; 15 | pub mod sliding_window; 16 | 17 | pub use chat_wrapper::ChatWithMemory; 18 | pub use shared_memory::SharedMemory; 19 | pub use sliding_window::{SlidingWindowMemory, TrimStrategy}; 20 | 21 | /// Types of memory implementations available 22 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 23 | pub enum MemoryType { 24 | /// Simple sliding window that keeps the N most recent messages 25 | SlidingWindow, 26 | } 27 | 28 | /// Trait for memory providers that can store and retrieve conversation history. 29 | /// 30 | /// Memory providers enable LLMs to maintain context across conversations by: 31 | /// - Storing messages as they are exchanged 32 | /// - Retrieving relevant past messages based on queries 33 | /// - Managing memory size and cleanup 34 | #[async_trait] 35 | pub trait MemoryProvider: Send + Sync { 36 | /// Store a message in memory. 37 | /// 38 | /// # Arguments 39 | /// 40 | /// * `message` - The chat message to store 41 | /// 42 | /// # Returns 43 | /// 44 | /// * `Ok(())` if the message was stored successfully 45 | /// * `Err(LLMError)` if storage failed 46 | async fn remember(&mut self, message: &ChatMessage) -> Result<(), LLMError>; 47 | 48 | /// Retrieve relevant messages from memory based on a query. 49 | /// 50 | /// # Arguments 51 | /// 52 | /// * `query` - The query string to search for relevant messages 53 | /// * `limit` - Optional maximum number of messages to return 54 | /// 55 | /// # Returns 56 | /// 57 | /// * `Ok(Vec)` containing relevant messages 58 | /// * `Err(LLMError)` if retrieval failed 59 | async fn recall(&self, query: &str, limit: Option) 60 | -> Result, LLMError>; 61 | 62 | /// Clear all stored messages from memory. 63 | /// 64 | /// # Returns 65 | /// 66 | /// * `Ok(())` if memory was cleared successfully 67 | /// * `Err(LLMError)` if clearing failed 68 | async fn clear(&mut self) -> Result<(), LLMError>; 69 | 70 | /// Get the type of this memory provider. 71 | /// 72 | /// # Returns 73 | /// 74 | /// The memory type enum variant 75 | fn memory_type(&self) -> MemoryType; 76 | 77 | /// Get the current number of stored messages. 78 | /// 79 | /// # Returns 80 | /// 81 | /// The number of messages currently in memory 82 | fn size(&self) -> usize; 83 | 84 | /// Check if the memory is empty. 85 | /// 86 | /// # Returns 87 | /// 88 | /// `true` if no messages are stored, `false` otherwise 89 | fn is_empty(&self) -> bool { 90 | self.size() == 0 91 | } 92 | 93 | /// Check if memory needs summarization 94 | fn needs_summary(&self) -> bool { 95 | false 96 | } 97 | 98 | /// Mark memory as needing summarization 99 | fn mark_for_summary(&mut self) {} 100 | 101 | /// Replace all messages with a summary 102 | fn replace_with_summary(&mut self, _summary: String) {} 103 | } 104 | -------------------------------------------------------------------------------- /src/memory/shared_memory.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use std::sync::Arc; 3 | use tokio::sync::RwLock; 4 | 5 | use super::{MemoryProvider, MemoryType}; 6 | use crate::{chat::ChatMessage, error::LLMError}; 7 | 8 | #[derive(Clone)] 9 | pub struct SharedMemory { 10 | inner: Arc>, 11 | } 12 | 13 | impl SharedMemory { 14 | pub fn new(provider: T) -> Self { 15 | Self { 16 | inner: Arc::new(RwLock::new(provider)), 17 | } 18 | } 19 | } 20 | 21 | #[async_trait] 22 | impl MemoryProvider for SharedMemory { 23 | async fn remember(&mut self, message: &ChatMessage) -> Result<(), LLMError> { 24 | let mut guard = self.inner.write().await; 25 | guard.remember(message).await 26 | } 27 | 28 | async fn recall( 29 | &self, 30 | query: &str, 31 | limit: Option, 32 | ) -> Result, LLMError> { 33 | let guard = self.inner.read().await; 34 | guard.recall(query, limit).await 35 | } 36 | 37 | async fn clear(&mut self) -> Result<(), LLMError> { 38 | let mut guard = self.inner.write().await; 39 | guard.clear().await 40 | } 41 | 42 | fn memory_type(&self) -> MemoryType { 43 | MemoryType::SlidingWindow 44 | } 45 | 46 | fn size(&self) -> usize { 47 | 0 48 | } 49 | 50 | fn needs_summary(&self) -> bool { 51 | tokio::task::block_in_place(|| { 52 | tokio::runtime::Handle::current().block_on(async { 53 | let guard = self.inner.read().await; 54 | guard.needs_summary() 55 | }) 56 | }) 57 | } 58 | 59 | fn mark_for_summary(&mut self) { 60 | tokio::task::block_in_place(|| { 61 | tokio::runtime::Handle::current().block_on(async { 62 | let mut guard = self.inner.write().await; 63 | guard.mark_for_summary(); 64 | }) 65 | }) 66 | } 67 | 68 | fn replace_with_summary(&mut self, summary: String) { 69 | tokio::task::block_in_place(|| { 70 | tokio::runtime::Handle::current().block_on(async { 71 | let mut guard = self.inner.write().await; 72 | guard.replace_with_summary(summary); 73 | }) 74 | }) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/memory/sliding_window.rs: -------------------------------------------------------------------------------- 1 | //! Simple sliding window memory implementation. 2 | //! 3 | //! This module provides a basic FIFO (First In, First Out) memory that maintains 4 | //! a fixed-size window of the most recent conversation messages. 5 | 6 | use std::collections::VecDeque; 7 | 8 | use async_trait::async_trait; 9 | 10 | use crate::{chat::ChatMessage, error::LLMError}; 11 | 12 | use super::{MemoryProvider, MemoryType}; 13 | 14 | /// Strategy for handling memory when window size limit is reached 15 | #[derive(Debug, Clone)] 16 | pub enum TrimStrategy { 17 | /// Drop oldest messages (FIFO behavior) 18 | Drop, 19 | /// Summarize all messages into one before adding new ones 20 | Summarize, 21 | } 22 | 23 | /// Simple sliding window memory that keeps the N most recent messages. 24 | /// 25 | /// This implementation uses a FIFO strategy where old messages are automatically 26 | /// removed when the window size limit is reached. It's suitable for: 27 | /// - Simple conversation contexts 28 | /// - Memory-constrained environments 29 | /// - Cases where only recent context matters 30 | /// 31 | /// # Examples 32 | /// 33 | /// ```rust 34 | /// use llm::memory::SlidingWindowMemory; 35 | /// use llm::chat::ChatMessage; 36 | /// 37 | /// let mut memory = SlidingWindowMemory::new(3); 38 | /// 39 | /// // Store messages - only the last 3 will be kept 40 | /// memory.remember(&ChatMessage::user().content("Hello").build()).await?; 41 | /// memory.remember(&ChatMessage::assistant().content("Hi there!").build()).await?; 42 | /// memory.remember(&ChatMessage::user().content("How are you?").build()).await?; 43 | /// memory.remember(&ChatMessage::assistant().content("I'm doing well!").build()).await?; 44 | /// 45 | /// // Only the last 3 messages are kept 46 | /// assert_eq!(memory.size(), 3); 47 | /// ``` 48 | #[derive(Debug, Clone)] 49 | pub struct SlidingWindowMemory { 50 | messages: VecDeque, 51 | window_size: usize, 52 | trim_strategy: TrimStrategy, 53 | needs_summary: bool, 54 | } 55 | 56 | impl SlidingWindowMemory { 57 | /// Create a new sliding window memory with the specified window size. 58 | /// 59 | /// # Arguments 60 | /// 61 | /// * `window_size` - Maximum number of messages to keep in memory 62 | /// 63 | /// # Panics 64 | /// 65 | /// Panics if `window_size` is 0 66 | /// 67 | /// # Examples 68 | /// 69 | /// ```rust 70 | /// use llm::memory::SlidingWindowMemory; 71 | /// 72 | /// let memory = SlidingWindowMemory::new(10); 73 | /// assert_eq!(memory.window_size(), 10); 74 | /// assert!(memory.is_empty()); 75 | /// ``` 76 | pub fn new(window_size: usize) -> Self { 77 | Self::with_strategy(window_size, TrimStrategy::Drop) 78 | } 79 | 80 | /// Create a new sliding window memory with specified trim strategy 81 | /// 82 | /// # Arguments 83 | /// 84 | /// * `window_size` - Maximum number of messages to keep in memory 85 | /// * `strategy` - How to handle overflow when window is full 86 | pub fn with_strategy(window_size: usize, strategy: TrimStrategy) -> Self { 87 | if window_size == 0 { 88 | panic!("Window size must be greater than 0"); 89 | } 90 | 91 | Self { 92 | messages: VecDeque::with_capacity(window_size), 93 | window_size, 94 | trim_strategy: strategy, 95 | needs_summary: false, 96 | } 97 | } 98 | 99 | /// Get the configured window size. 100 | /// 101 | /// # Returns 102 | /// 103 | /// The maximum number of messages this memory can hold 104 | pub fn window_size(&self) -> usize { 105 | self.window_size 106 | } 107 | 108 | /// Get all stored messages in chronological order. 109 | /// 110 | /// # Returns 111 | /// 112 | /// A vector containing all messages from oldest to newest 113 | pub fn messages(&self) -> Vec { 114 | Vec::from(self.messages.clone()) 115 | } 116 | 117 | /// Get the most recent N messages. 118 | /// 119 | /// # Arguments 120 | /// 121 | /// * `limit` - Maximum number of recent messages to return 122 | /// 123 | /// # Returns 124 | /// 125 | /// A vector containing the most recent messages, up to `limit` 126 | pub fn recent_messages(&self, limit: usize) -> Vec { 127 | let len = self.messages.len(); 128 | let start = len.saturating_sub(limit); 129 | self.messages.range(start..).cloned().collect() 130 | } 131 | 132 | /// Check if memory needs summarization 133 | pub fn needs_summary(&self) -> bool { 134 | self.needs_summary 135 | } 136 | 137 | /// Mark memory as needing summarization 138 | pub fn mark_for_summary(&mut self) { 139 | self.needs_summary = true; 140 | } 141 | 142 | /// Replace all messages with a summary 143 | /// 144 | /// # Arguments 145 | /// 146 | /// * `summary` - The summary text to replace all messages with 147 | pub fn replace_with_summary(&mut self, summary: String) { 148 | self.messages.clear(); 149 | self.messages.push_back(crate::chat::ChatMessage::assistant().content(summary).build()); 150 | self.needs_summary = false; 151 | } 152 | } 153 | 154 | #[async_trait] 155 | impl MemoryProvider for SlidingWindowMemory { 156 | async fn remember(&mut self, message: &ChatMessage) -> Result<(), LLMError> { 157 | if self.messages.len() >= self.window_size { 158 | match self.trim_strategy { 159 | TrimStrategy::Drop => { 160 | self.messages.pop_front(); 161 | } 162 | TrimStrategy::Summarize => { 163 | self.mark_for_summary(); 164 | } 165 | } 166 | } 167 | self.messages.push_back(message.clone()); 168 | Ok(()) 169 | } 170 | 171 | async fn recall( 172 | &self, 173 | _query: &str, 174 | limit: Option, 175 | ) -> Result, LLMError> { 176 | let limit = limit.unwrap_or(self.messages.len()); 177 | Ok(self.recent_messages(limit)) 178 | } 179 | 180 | async fn clear(&mut self) -> Result<(), LLMError> { 181 | self.messages.clear(); 182 | Ok(()) 183 | } 184 | 185 | fn memory_type(&self) -> MemoryType { 186 | MemoryType::SlidingWindow 187 | } 188 | 189 | fn size(&self) -> usize { 190 | self.messages.len() 191 | } 192 | 193 | fn needs_summary(&self) -> bool { 194 | self.needs_summary 195 | } 196 | 197 | fn mark_for_summary(&mut self) { 198 | self.needs_summary = true; 199 | } 200 | 201 | fn replace_with_summary(&mut self, summary: String) { 202 | self.messages.clear(); 203 | self.messages.push_back(crate::chat::ChatMessage::assistant().content(summary).build()); 204 | self.needs_summary = false; 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/secret_store.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use std::collections::HashMap; 3 | use std::fs::{self, File}; 4 | use std::io::{self, Read, Write}; 5 | use std::path::PathBuf; 6 | 7 | /// Key used to store the default provider in the secret store 8 | const DEFAULT_PROVIDER_KEY: &str = "default"; 9 | 10 | /// A secure storage for API keys and other sensitive information 11 | /// 12 | /// Provides functionality to store, retrieve, and manage secrets 13 | /// in a JSON file located in the user's home directory. 14 | #[derive(Debug, Serialize, Deserialize)] 15 | pub struct SecretStore { 16 | /// Map of secret keys to their values 17 | secrets: HashMap, 18 | /// Path to the secrets file 19 | file_path: PathBuf, 20 | } 21 | 22 | impl SecretStore { 23 | /// Creates a new SecretStore instance 24 | /// 25 | /// Initializes the store with the default path (~/.llm/secrets.json) 26 | /// and loads any existing secrets from the file. 27 | /// 28 | /// # Returns 29 | /// 30 | /// * `io::Result` - A new SecretStore instance or an IO error 31 | pub fn new() -> io::Result { 32 | let home_dir = dirs::home_dir().expect("Could not find home directory"); 33 | let file_path = home_dir.join(".llm").join("secrets.json"); 34 | 35 | if let Some(parent) = file_path.parent() { 36 | fs::create_dir_all(parent)?; 37 | } 38 | 39 | let mut store = SecretStore { 40 | secrets: HashMap::new(), 41 | file_path, 42 | }; 43 | 44 | store.load()?; 45 | Ok(store) 46 | } 47 | 48 | /// Loads secrets from the file system 49 | /// 50 | /// # Returns 51 | /// 52 | /// * `io::Result<()>` - Success or an IO error 53 | fn load(&mut self) -> io::Result<()> { 54 | match File::open(&self.file_path) { 55 | Ok(mut file) => { 56 | let mut contents = String::new(); 57 | file.read_to_string(&mut contents)?; 58 | self.secrets = serde_json::from_str(&contents).unwrap_or_default(); 59 | Ok(()) 60 | } 61 | Err(ref e) if e.kind() == io::ErrorKind::NotFound => Ok(()), 62 | Err(e) => Err(e), 63 | } 64 | } 65 | 66 | /// Saves the current secrets to the file system 67 | /// 68 | /// # Returns 69 | /// 70 | /// * `io::Result<()>` - Success or an IO error 71 | fn save(&self) -> io::Result<()> { 72 | let contents = serde_json::to_string_pretty(&self.secrets)?; 73 | let mut file = File::create(&self.file_path)?; 74 | file.write_all(contents.as_bytes())?; 75 | Ok(()) 76 | } 77 | 78 | /// Sets a secret value for the given key 79 | /// 80 | /// # Arguments 81 | /// 82 | /// * `key` - The key to store the secret under 83 | /// * `value` - The secret value to store 84 | /// 85 | /// # Returns 86 | /// 87 | /// * `io::Result<()>` - Success or an IO error 88 | pub fn set(&mut self, key: &str, value: &str) -> io::Result<()> { 89 | self.secrets.insert(key.to_string(), value.to_string()); 90 | self.save() 91 | } 92 | 93 | /// Retrieves a secret value for the given key 94 | /// 95 | /// # Arguments 96 | /// 97 | /// * `key` - The key to look up 98 | /// 99 | /// # Returns 100 | /// 101 | /// * `Option<&String>` - The secret value if found, or None 102 | pub fn get(&self, key: &str) -> Option<&String> { 103 | self.secrets.get(key) 104 | } 105 | 106 | /// Deletes a secret with the given key 107 | /// 108 | /// # Arguments 109 | /// 110 | /// * `key` - The key to delete 111 | /// 112 | /// # Returns 113 | /// 114 | /// * `io::Result<()>` - Success or an IO error 115 | pub fn delete(&mut self, key: &str) -> io::Result<()> { 116 | self.secrets.remove(key); 117 | self.save() 118 | } 119 | 120 | /// Sets the default provider for LLM interactions 121 | /// 122 | /// # Arguments 123 | /// 124 | /// * `provider` - The provider string in format "provider:model" 125 | /// 126 | /// # Returns 127 | /// 128 | /// * `io::Result<()>` - Success or an IO error 129 | pub fn set_default_provider(&mut self, provider: &str) -> io::Result<()> { 130 | self.secrets 131 | .insert(DEFAULT_PROVIDER_KEY.to_string(), provider.to_string()); 132 | self.save() 133 | } 134 | 135 | /// Retrieves the default provider for LLM interactions 136 | /// 137 | /// # Returns 138 | /// 139 | /// * `Option<&String>` - The default provider if set, or None 140 | pub fn get_default_provider(&self) -> Option<&String> { 141 | self.secrets.get(DEFAULT_PROVIDER_KEY) 142 | } 143 | 144 | /// Deletes the default provider setting 145 | /// 146 | /// # Returns 147 | /// 148 | /// * `io::Result<()>` - Success or an IO error 149 | pub fn delete_default_provider(&mut self) -> io::Result<()> { 150 | self.secrets.remove(DEFAULT_PROVIDER_KEY); 151 | self.save() 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/stt/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::error::LLMError; 2 | use async_trait::async_trait; 3 | 4 | /// Trait implemented by all speech to text backends 5 | /// 6 | /// This trait defines the interface for speech-to-text conversion services. 7 | /// Implementors must provide functionality to convert audio data into text. 8 | #[async_trait] 9 | pub trait SpeechToTextProvider: Send + Sync { 10 | /// Transcribe the given audio bytes into text 11 | /// 12 | /// # Arguments 13 | /// 14 | /// * `audio` - A vector of bytes containing the audio data to transcribe 15 | /// 16 | /// # Returns 17 | /// 18 | /// * `Result` - On success, returns the transcribed text as a String. 19 | /// On failure, returns an LLMError describing what went wrong. 20 | async fn transcribe(&self, audio: Vec) -> Result; 21 | 22 | #[allow(unused)] 23 | async fn transcribe_file(&self, file_path: &str) -> Result { 24 | Err(LLMError::ProviderError( 25 | "Phind does not implement speech to text endpoint yet.".into(), 26 | )) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/tts/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::error::LLMError; 2 | use async_trait::async_trait; 3 | 4 | /// Trait implemented by all text to speech backends 5 | /// 6 | /// This trait defines the interface for text-to-speech conversion services. 7 | /// Implementors must provide functionality to convert text into audio data. 8 | #[async_trait] 9 | pub trait TextToSpeechProvider: Send + Sync { 10 | /// Convert the given text into speech audio 11 | /// 12 | /// # Arguments 13 | /// 14 | /// * `text` - A string containing the text to convert to speech 15 | /// 16 | /// # Returns 17 | /// 18 | /// * `Result, LLMError>` - On success, returns the audio data as a vector of bytes. 19 | /// On failure, returns an LLMError describing what went wrong. 20 | #[allow(unused)] 21 | async fn speech(&self, text: &str) -> Result, LLMError> { 22 | Err(LLMError::ProviderError( 23 | "Phind does not implement text to speech endpoint yet.".into(), 24 | )) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/validated_llm.rs: -------------------------------------------------------------------------------- 1 | //! A module providing validation capabilities for LLM responses through a wrapper implementation. 2 | //! 3 | //! This module enables adding custom validation logic to any LLM provider by wrapping it in a 4 | //! `ValidatedLLM` struct. The wrapper will validate responses and retry failed attempts with 5 | //! feedback to help guide the model toward producing valid output. 6 | //! 7 | //! # Example 8 | //! 9 | //! ```no_run 10 | //! use llm::builder::{LLMBuilder, LLMBackend}; 11 | //! 12 | //! let llm = LLMBuilder::new() 13 | //! .backend(LLMBackend::OpenAI) 14 | //! .validator(|response| { 15 | //! if response.contains("unsafe content") { 16 | //! Err("Response contains unsafe content".to_string()) 17 | //! } else { 18 | //! Ok(()) 19 | //! } 20 | //! }) 21 | //! .validator_attempts(3) 22 | //! .build() 23 | //! .unwrap(); 24 | //! ``` 25 | 26 | use async_trait::async_trait; 27 | 28 | use crate::chat::{ChatMessage, ChatProvider, ChatResponse, ChatRole, MessageType, Tool}; 29 | use crate::completion::{CompletionProvider, CompletionRequest, CompletionResponse}; 30 | use crate::embedding::EmbeddingProvider; 31 | use crate::error::LLMError; 32 | use crate::stt::SpeechToTextProvider; 33 | use crate::tts::TextToSpeechProvider; 34 | use crate::{builder::ValidatorFn, LLMProvider}; 35 | 36 | /// A wrapper around an LLM provider that validates responses before returning them. 37 | /// 38 | /// The wrapper implements validation by: 39 | /// 1. Sending the request to the underlying provider 40 | /// 2. Validating the response using the provided validator function 41 | /// 3. If validation fails, retrying with feedback up to the configured number of attempts 42 | /// 43 | /// # Type Parameters 44 | /// 45 | /// The wrapped provider must implement the `LLMProvider` trait. 46 | pub struct ValidatedLLM { 47 | /// The wrapped LLM provider 48 | inner: Box, 49 | /// Function used to validate responses, returns Ok(()) if valid or Err with message if invalid 50 | validator: Box, 51 | /// Maximum number of validation attempts before giving up 52 | attempts: usize, 53 | } 54 | 55 | impl ValidatedLLM { 56 | /// Creates a new ValidatedLLM wrapper around an existing LLM provider. 57 | /// 58 | /// # Arguments 59 | /// 60 | /// * `inner` - The LLM provider to wrap with validation 61 | /// * `validator` - Function that takes a response string and returns Ok(()) if valid, or Err with error message if invalid 62 | /// * `attempts` - Maximum number of validation attempts before failing 63 | /// 64 | /// # Returns 65 | /// 66 | /// A new ValidatedLLM instance configured with the provided parameters. 67 | pub fn new(inner: Box, validator: Box, attempts: usize) -> Self { 68 | Self { 69 | inner, 70 | validator, 71 | attempts, 72 | } 73 | } 74 | } 75 | 76 | impl LLMProvider for ValidatedLLM {} 77 | 78 | #[async_trait] 79 | impl ChatProvider for ValidatedLLM { 80 | /// Sends a chat request and validates the response. 81 | /// 82 | /// If validation fails, retries with feedback to the model about the validation error. 83 | /// The feedback is appended as a new user message to help guide the model. 84 | /// 85 | /// # Arguments 86 | /// 87 | /// * `messages` - The chat messages to send to the model 88 | /// 89 | /// # Returns 90 | /// 91 | /// * `Ok(String)` - The validated response from the model 92 | /// * `Err(LLMError)` - If validation fails after max attempts or other errors occur 93 | async fn chat_with_tools( 94 | &self, 95 | messages: &[ChatMessage], 96 | tools: Option<&[Tool]>, 97 | ) -> Result, LLMError> { 98 | let mut local_messages = messages.to_vec(); 99 | let mut remaining_attempts = self.attempts; 100 | 101 | loop { 102 | let response = match self.inner.chat_with_tools(&local_messages, tools).await { 103 | Ok(resp) => resp, 104 | Err(e) => return Err(e), 105 | }; 106 | 107 | match (self.validator)(&response.text().unwrap_or_default()) { 108 | Ok(()) => { 109 | return Ok(response); 110 | } 111 | Err(err) => { 112 | remaining_attempts -= 1; 113 | if remaining_attempts == 0 { 114 | return Err(LLMError::InvalidRequest(format!( 115 | "Validation error after max attempts: {}", 116 | err 117 | ))); 118 | } 119 | 120 | log::debug!( 121 | "Completion validation failed (attempts remaining: {}). Reason: {}", 122 | remaining_attempts, 123 | err 124 | ); 125 | 126 | log::debug!( 127 | "Validation failed (attempt remaining: {}). Reason: {}", 128 | remaining_attempts, 129 | err 130 | ); 131 | 132 | local_messages.push(ChatMessage { 133 | role: ChatRole::User, 134 | message_type: MessageType::Text, 135 | content: format!( 136 | "Your previous output was invalid because: {}\n\ 137 | Please try again and produce a valid response.", 138 | err 139 | ), 140 | }); 141 | } 142 | } 143 | } 144 | } 145 | } 146 | 147 | #[async_trait] 148 | impl CompletionProvider for ValidatedLLM { 149 | /// Sends a completion request and validates the response. 150 | /// 151 | /// If validation fails, retries up to the configured number of attempts. 152 | /// Unlike chat, completion requests don't support adding feedback messages. 153 | /// 154 | /// # Arguments 155 | /// 156 | /// * `req` - The completion request to send 157 | /// 158 | /// # Returns 159 | /// 160 | /// * `Ok(CompletionResponse)` - The validated completion response 161 | /// * `Err(LLMError)` - If validation fails after max attempts or other errors occur 162 | async fn complete(&self, req: &CompletionRequest) -> Result { 163 | let mut remaining_attempts = self.attempts; 164 | 165 | loop { 166 | let response = match self.inner.complete(req).await { 167 | Ok(resp) => resp, 168 | Err(e) => return Err(e), 169 | }; 170 | 171 | match (self.validator)(&response.text) { 172 | Ok(()) => { 173 | return Ok(response); 174 | } 175 | Err(err) => { 176 | remaining_attempts -= 1; 177 | if remaining_attempts == 0 { 178 | return Err(LLMError::InvalidRequest(format!( 179 | "Validation error after max attempts: {}", 180 | err 181 | ))); 182 | } 183 | } 184 | } 185 | } 186 | } 187 | } 188 | 189 | #[async_trait] 190 | impl EmbeddingProvider for ValidatedLLM { 191 | /// Passes through embedding requests to the inner provider without validation. 192 | /// 193 | /// Embeddings are numerical vectors that represent text semantically and don't 194 | /// require validation since they're not human-readable content. 195 | /// 196 | /// # Arguments 197 | /// 198 | /// * `input` - Vector of strings to generate embeddings for 199 | /// 200 | /// # Returns 201 | /// 202 | /// * `Ok(Vec>)` - Vector of embedding vectors 203 | /// * `Err(LLMError)` - If the embedding generation fails 204 | async fn embed(&self, input: Vec) -> Result>, LLMError> { 205 | // Pass through to inner provider since embeddings don't need validation 206 | self.inner.embed(input).await 207 | } 208 | } 209 | 210 | #[async_trait] 211 | impl SpeechToTextProvider for ValidatedLLM { 212 | async fn transcribe(&self, _audio: Vec) -> Result { 213 | Err(LLMError::ProviderError( 214 | "Speech to text not supported".to_string(), 215 | )) 216 | } 217 | } 218 | 219 | #[async_trait] 220 | impl TextToSpeechProvider for ValidatedLLM { 221 | async fn speech(&self, _text: &str) -> Result, LLMError> { 222 | Err(LLMError::ProviderError( 223 | "Text to speech not supported".to_string(), 224 | )) 225 | } 226 | } 227 | --------------------------------------------------------------------------------