├── tests ├── __init__.py └── langdict │ ├── test_langdict.py │ └── test_langdict_openai.py ├── src └── langdict │ ├── modules │ ├── __init__.py │ ├── rankings │ │ ├── __init__.py │ │ └── rank_gpt.py │ ├── compressions │ │ ├── __init__.py │ │ └── llm_lingua2.py │ ├── parameter.py │ ├── langdict_module.py │ ├── rags │ │ └── self_rag │ │ │ ├── is_relevant.py │ │ │ ├── is_useful.py │ │ │ ├── __init__.py │ │ │ ├── is_support.py │ │ │ └── need_retrieve.py │ └── module.py │ ├── chat_models │ ├── __init__.py │ └── litellm.py │ ├── traces │ ├── backend.py │ ├── callbacks │ │ ├── __init__.py │ │ └── stdout.py │ └── __init__.py │ ├── builders │ ├── base.py │ ├── __init__.py │ ├── lite_llm.py │ ├── output_parser.py │ ├── text_prompt.py │ ├── chat_prompt.py │ └── trace.py │ ├── __init__.py │ ├── specs │ ├── __init__.py │ ├── base.py │ ├── output.py │ ├── llm.py │ ├── lang.py │ └── prompt.py │ └── langdict.py ├── requirements.txt ├── images ├── logo.png ├── module.png └── profile.png ├── LICENSE ├── pyproject.toml ├── .gitignore └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/langdict/modules/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | langchain>=0.3.0 2 | langchain-core>=0.2.0 3 | litellm>=1.0.0 4 | -------------------------------------------------------------------------------- /images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DongjunLee/langdict/HEAD/images/logo.png -------------------------------------------------------------------------------- /images/module.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DongjunLee/langdict/HEAD/images/module.png -------------------------------------------------------------------------------- /images/profile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DongjunLee/langdict/HEAD/images/profile.png -------------------------------------------------------------------------------- /src/langdict/chat_models/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .litellm import ChatLiteLLM 3 | 4 | 5 | __all__ = [ 6 | ChatLiteLLM, 7 | ] 8 | -------------------------------------------------------------------------------- /src/langdict/modules/rankings/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from langdict.modules.rankings.rank_gpt import RankGPT 3 | 4 | 5 | __all__ = [ 6 | RankGPT, 7 | ] 8 | 9 | -------------------------------------------------------------------------------- /src/langdict/modules/compressions/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from langdict.modules.compressions.llm_lingua2 import TextCompressor 3 | 4 | 5 | __all__ = [ 6 | TextCompressor, 7 | ] 8 | -------------------------------------------------------------------------------- /src/langdict/traces/backend.py: -------------------------------------------------------------------------------- 1 | from enum import StrEnum 2 | 3 | 4 | class TraceBackend(StrEnum): 5 | CONSOLE = "console" 6 | LANGFUSE = "langfuse" 7 | LANGSMITH = "langsmith" 8 | -------------------------------------------------------------------------------- /src/langdict/traces/callbacks/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from langdict.traces.callbacks.stdout import TraceStdOutCallbackHandler 3 | 4 | 5 | __all__ = [ 6 | TraceStdOutCallbackHandler, 7 | ] 8 | -------------------------------------------------------------------------------- /src/langdict/traces/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from langdict.traces.backend import TraceBackend 3 | from langdict.traces.callbacks.stdout import TraceStdOutCallbackHandler 4 | 5 | 6 | __all__ = [ 7 | TraceBackend, 8 | TraceStdOutCallbackHandler, 9 | ] 10 | -------------------------------------------------------------------------------- /src/langdict/modules/parameter.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | 4 | class Parameter: 5 | 6 | def __init__(self, data: Union[str, int, float, bool, None]): 7 | self._data = data 8 | 9 | @property 10 | def value(self): 11 | return self._data 12 | 13 | -------------------------------------------------------------------------------- /src/langdict/builders/base.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | class Builder: 5 | """Builder interface""" 6 | 7 | def __init__(self): 8 | pass 9 | 10 | @classmethod 11 | def build(cls): 12 | raise NotImplementedError( 13 | f"Builder [{type(cls).__name__}] is missing the required \"build\" function" 14 | ) 15 | -------------------------------------------------------------------------------- /src/langdict/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from langdict.langdict import LangDict 3 | from langdict.modules.module import Module 4 | from langdict.modules.langdict_module import LangDictModule 5 | from langdict.modules.parameter import Parameter 6 | 7 | 8 | __all__ = [ 9 | LangDict, 10 | Module, 11 | LangDictModule, 12 | Parameter, 13 | ] 14 | -------------------------------------------------------------------------------- /src/langdict/builders/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from langdict.builders.chat_prompt import ChatPromptMessagesBuilder 3 | from langdict.builders.text_prompt import PromptTemplateBuilder 4 | from langdict.builders.lite_llm import LiteLLMBuilder 5 | from langdict.builders.output_parser import OutputParserBuilder 6 | from langdict.builders.trace import TraceCallbackBuilder 7 | 8 | 9 | __all__ = [ 10 | ChatPromptMessagesBuilder, 11 | LiteLLMBuilder, 12 | OutputParserBuilder, 13 | PromptTemplateBuilder, 14 | TraceCallbackBuilder, 15 | ] 16 | -------------------------------------------------------------------------------- /src/langdict/specs/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .base import BaseSpecification 3 | from .lang import LangSpecification 4 | from .prompt import ( 5 | PromptSpecification, 6 | TextPromptSpecification, 7 | ChatPromptSpecification, 8 | ) 9 | from .llm import LLMSpecification 10 | from .output import OutputSpecification 11 | 12 | 13 | __all__ = [ 14 | BaseSpecification, 15 | LangSpecification, 16 | PromptSpecification, 17 | TextPromptSpecification, 18 | ChatPromptSpecification, 19 | LLMSpecification, 20 | OutputSpecification, 21 | ] 22 | -------------------------------------------------------------------------------- /src/langdict/specs/base.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | 4 | 5 | class BaseSpecification: 6 | 7 | def __init__(self): 8 | self.validate() 9 | 10 | def validate(self): 11 | raise NotImplementedError( 12 | f"validate method not implemented for {self.__class__.__name__}" 13 | ) 14 | 15 | @classmethod 16 | def from_dict(cls, data: Dict[str, Any]) -> "BaseSpecification": 17 | raise NotImplementedError( 18 | f"from_dict method not implemented for {cls.__name__}" 19 | ) 20 | 21 | def as_dict(self) -> Dict[str, Any]: 22 | return self.__dict__ 23 | -------------------------------------------------------------------------------- /src/langdict/specs/output.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | from .base import BaseSpecification 4 | 5 | 6 | class OutputSpecification(BaseSpecification): 7 | 8 | OUTPUT_TYPES = {"string", "json"} 9 | 10 | def __init__(self, type: str = "string"): 11 | self.type = type 12 | 13 | super().__init__() 14 | 15 | def validate(self): 16 | if self.type not in self.OUTPUT_TYPES: 17 | raise ValueError(f"Invalid output type: {self.type}") 18 | 19 | @classmethod 20 | def from_dict(cls, data: Dict) -> "OutputSpecification": 21 | return cls( 22 | type=data.get("type", "string"), 23 | ) 24 | -------------------------------------------------------------------------------- /src/langdict/builders/lite_llm.py: -------------------------------------------------------------------------------- 1 | 2 | from langdict.chat_models import ChatLiteLLM 3 | from langdict.specs import LLMSpecification 4 | 5 | from .base import Builder 6 | 7 | 8 | class LiteLLMBuilder(Builder): 9 | 10 | def __init__(self): 11 | pass 12 | 13 | @classmethod 14 | def build(cls, spec: LLMSpecification): 15 | return ChatLiteLLM( 16 | model=spec.model, 17 | model_name=spec.model_name, 18 | openai_api_key=spec.api_key, 19 | temperature=spec.temperature, 20 | top_p=spec.top_p, 21 | top_k=spec.top_k, 22 | streaming=spec.streaming, 23 | n=spec.n, 24 | max_tokens=spec.max_tokens, 25 | ) 26 | -------------------------------------------------------------------------------- /src/langdict/builders/output_parser.py: -------------------------------------------------------------------------------- 1 | from enum import StrEnum 2 | 3 | from langchain_core.output_parsers import ( 4 | BaseOutputParser, 5 | JsonOutputParser, 6 | StrOutputParser, 7 | ) 8 | 9 | from langdict.specs import OutputSpecification 10 | 11 | from .base import Builder 12 | 13 | 14 | class OutputType(StrEnum): 15 | STRING = "string" 16 | JSON = "json" 17 | 18 | 19 | class OutputParserBuilder(Builder): 20 | 21 | def __init__(self): 22 | pass 23 | 24 | @classmethod 25 | def build(cls, spec: OutputSpecification) -> BaseOutputParser: 26 | if spec.type == OutputType.STRING.value: 27 | return StrOutputParser() 28 | elif spec.type == OutputType.JSON.value: 29 | return JsonOutputParser() 30 | else: 31 | raise ValueError("Invalid output parser type.") 32 | -------------------------------------------------------------------------------- /src/langdict/builders/text_prompt.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from langchain_core.prompts import PromptTemplate, ChatPromptTemplate 4 | 5 | from langdict.specs import TextPromptSpecification, ChatPromptSpecification 6 | 7 | from .base import Builder 8 | 9 | 10 | class PromptTemplateBuilder(Builder): 11 | """Prompt Template Builder interface""" 12 | 13 | def __init__(self): 14 | pass 15 | 16 | @classmethod 17 | def build(cls, spec: Union[TextPromptSpecification, ChatPromptSpecification]): 18 | if isinstance(spec, TextPromptSpecification): 19 | return PromptTemplate.from_template(spec.text) 20 | elif isinstance(spec, ChatPromptSpecification): 21 | return ChatPromptTemplate.from_messages(spec.messages) 22 | else: 23 | raise ValueError(f"Invalid specification type: {type(spec)}") 24 | 25 | -------------------------------------------------------------------------------- /tests/langdict/test_langdict.py: -------------------------------------------------------------------------------- 1 | 2 | from langdict import LangDict 3 | 4 | 5 | def test_langdict_dict(): 6 | chitchat_spec = { 7 | "messages": [ 8 | ("system", "You are a helpful AI bot. Your name is {name}."), 9 | ("human", "Hello, how are you doing?"), 10 | ("ai", "I'm doing well, thanks!"), 11 | ("human", "{user_input}"), 12 | ], 13 | "llm": { 14 | "model": "gpt-4o-mini", 15 | "max_tokens": 200 16 | }, 17 | "output": { 18 | "type": "string" 19 | } 20 | } 21 | 22 | chitchat = LangDict.from_dict(chitchat_spec) 23 | assert chitchat.as_dict()["messages"] == chitchat_spec["messages"] 24 | 25 | for key in ["llm", "output"]: 26 | for k, v in chitchat_spec[key].items(): 27 | assert chitchat.as_dict()[key][k] == v 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 LangDict 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/langdict/modules/langdict_module.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any 2 | 3 | from langdict import LangDict 4 | 5 | from .module import Module 6 | 7 | 8 | class LangDictModule(Module): 9 | 10 | """LangDictModule: A module wrapper for LangDict.""" 11 | 12 | def __init__(self, lang_dict: LangDict): 13 | super().__init__() 14 | self.lang_dict = lang_dict 15 | 16 | def __call__( 17 | self, 18 | *args, 19 | stream: bool = False, 20 | batch: bool = False, 21 | **kwargs 22 | ): 23 | if ( 24 | self.streaming and 25 | self.is_last_child 26 | ): 27 | stream = True 28 | 29 | inputs = self.forward(*args, **kwargs) 30 | 31 | return self.lang_dict( 32 | inputs, 33 | stream=stream, 34 | batch=batch, 35 | trace_backend=self.trace_backend, 36 | module_name=self._get_name(), 37 | ) 38 | 39 | def forward(self, *args, **kwargs) -> Dict[str, Any]: 40 | if type(args[0]) is dict: 41 | return args[0] 42 | else: 43 | raise ValueError("Invalid inputs type. Expected dict.") 44 | 45 | @classmethod 46 | def from_dict(cls, data: Dict[str, Any]) -> "LangDictModule": 47 | return LangDictModule(LangDict.from_dict(data)) 48 | 49 | def as_dict(self) -> Dict[str, Any]: 50 | return self.lang_dict.as_dict() 51 | -------------------------------------------------------------------------------- /src/langdict/builders/chat_prompt.py: -------------------------------------------------------------------------------- 1 | from typing import List, Tuple 2 | 3 | from pydantic import BaseModel 4 | 5 | from .base import Builder 6 | 7 | 8 | class ChatPromptMessagesBuilder(Builder): 9 | 10 | def __init__(self): 11 | pass 12 | 13 | @classmethod 14 | def build( 15 | cls, 16 | persona: str = "", 17 | task_instruction: str = "", 18 | output_format: str = "", 19 | output_basemodel: BaseModel = None, 20 | fewshot_examples: List[str] = [], 21 | conversation_key: str = "conversation", 22 | context_key: str = "", 23 | ) -> List[Tuple[str, str]]: 24 | messages = [] 25 | 26 | system_instruction = f"{persona}\n{task_instruction}" 27 | if output_format: 28 | system_instruction += f"\n## Output Format: {output_format}" 29 | elif output_basemodel: 30 | system_instruction += f"\n## Output Format: {output_basemodel.__name__}" 31 | 32 | if fewshot_examples: 33 | system_instruction += "\n## Examples:" 34 | for example in fewshot_examples: 35 | system_instruction += f"\n- {example}" 36 | 37 | messages.append(("system", system_instruction)) 38 | 39 | if conversation_key: 40 | messages.append(("placeholder", "{" + conversation_key + "}")) 41 | if context_key: 42 | messages.append(("ai", "{" + context_key + "}")) 43 | return messages 44 | -------------------------------------------------------------------------------- /src/langdict/modules/compressions/llm_lingua2.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | from langdict import LangDict, LangDictModule 4 | 5 | 6 | _SPECIFICATION = { 7 | "messages": [ 8 | ("system", """Compress the given text to short expressions, and such that you (GPT-4) can reconstruct it as close as possible to the original. 9 | Unlike the usual text compression, I need you to comply with the 5 conditions below: 10 | 1. You can ONLY remove unimportant words. 11 | 2. Do not reorder the original words. 12 | 3. Do not change the original words. 13 | 4. Do not use abbreviations or emojis. 14 | 5. Do not add new words or symbols. 15 | Compress the origin aggressively by removing words only. 16 | Compress the origin as short as you can, while retaining as much information as possible. 17 | If you understand, please compress the following text: {text} 18 | 19 | The compressed text is:"""), 20 | ], 21 | "llm": { 22 | "model": "gpt-4o-mini", 23 | }, 24 | "output": { 25 | "type": "string" 26 | }, 27 | "metadata": { 28 | "arxiv": "https://arxiv.org/abs/2403.12968", 29 | } 30 | } 31 | 32 | 33 | class TextCompressor(LangDictModule): 34 | """ 35 | LLMLingua-2: Data Distillation for Efficient and Faithful Task-Agnostic Prompt Compression 36 | 37 | Step 1: Data Distillation 38 | """ 39 | 40 | def __init__(self): 41 | super().__init__( 42 | LangDict.from_dict(_SPECIFICATION) 43 | ) 44 | 45 | def forward(self, text: str) -> Dict[str, Any]: 46 | return { 47 | "text": text 48 | } 49 | -------------------------------------------------------------------------------- /src/langdict/builders/trace.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from langdict.traces import ( 4 | TraceBackend, 5 | TraceStdOutCallbackHandler, 6 | ) 7 | 8 | from .base import Builder 9 | 10 | 11 | class TraceCallbackBuilder(Builder): 12 | 13 | def __init__(self): 14 | pass 15 | 16 | def build( 17 | self, 18 | backend: str, 19 | module_name: str = None, 20 | tags: List[str] = None, 21 | session_id: str = None, 22 | user_id: str = None, 23 | ): 24 | 25 | if backend == TraceBackend.CONSOLE: 26 | return TraceStdOutCallbackHandler(module_name=module_name) 27 | elif backend == TraceBackend.LANGFUSE: 28 | try: 29 | from langfuse.callback import CallbackHandler 30 | except ImportError: 31 | raise ModuleNotFoundError("LangFuse is not installed.") 32 | 33 | return CallbackHandler( 34 | trace_name=module_name, 35 | tags=tags, 36 | session_id=session_id, 37 | user_id=user_id, 38 | ) 39 | elif backend == TraceBackend.LANGSMITH: 40 | try: 41 | from langchain_core.tracers import LangChainTracer 42 | except ImportError: 43 | raise ModuleNotFoundError("LangChainTracer is not installed.") 44 | 45 | tags.append(module_name) 46 | 47 | return LangChainTracer( 48 | example_id=session_id, 49 | tags=tags, 50 | ) 51 | else: 52 | raise ValueError(f"Backend {backend} is not supported") 53 | -------------------------------------------------------------------------------- /src/langdict/specs/llm.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Optional 2 | 3 | from .base import BaseSpecification 4 | 5 | 6 | class LLMSpecification(BaseSpecification): 7 | 8 | def __init__( 9 | self, 10 | model: str = "gpt-3.5-turbo", 11 | model_name: Optional[str] = None, 12 | api_key: Optional[str] = None, 13 | temperature: Optional[float] = 1, 14 | top_p: Optional[float] = None, 15 | top_k: Optional[int] = None, 16 | streaming: bool = False, 17 | n: int = 1, 18 | max_tokens: Optional[int] = None, 19 | ): 20 | super().__init__() 21 | 22 | self.model = model 23 | self.model_name = model_name 24 | self.api_key = api_key 25 | self.temperature = temperature 26 | self.top_p = top_p 27 | self.top_k = top_k 28 | self.streaming = streaming 29 | self.n = n 30 | self.max_tokens = max_tokens 31 | 32 | def validate(self): 33 | # TODO: 사용가능한 LLM 기준 34 | pass 35 | 36 | @classmethod 37 | def from_dict(cls, data: Dict) -> "LLMSpecification": 38 | return cls( 39 | model=data.get("model", "gpt-3.5-turbo"), 40 | model_name=None, 41 | # model_name=data.get("model_name", "openai"), 42 | api_key=data.get("api_key"), 43 | temperature=data.get("temperature", 1), 44 | top_p=data.get("top_p", None), 45 | top_k=data.get("top_k", None), 46 | streaming=data.get("streaming", False), 47 | n=data.get("n", 1), 48 | max_tokens=data.get("max_tokens", None), 49 | ) 50 | 51 | -------------------------------------------------------------------------------- /tests/langdict/test_langdict_openai.py: -------------------------------------------------------------------------------- 1 | 2 | from langdict import LangDict 3 | from langdict.builders import ChatPromptMessagesBuilder 4 | 5 | 6 | def test_langdict_openai_chitchat(): 7 | chitchat_spec = { 8 | "messages": [ 9 | ("system", "You are a helpful AI bot. Your name is {name}."), 10 | ("human", "Hello, how are you doing?"), 11 | ("ai", "I'm doing well, thanks!"), 12 | ("human", "{user_input}"), 13 | ], 14 | "llm": { 15 | "model": "gpt-4o-mini", 16 | "max_tokens": 200 17 | }, 18 | "output": { 19 | "type": "string" 20 | } 21 | } 22 | 23 | chitchat = LangDict.from_dict(chitchat_spec) 24 | print( 25 | chitchat({ 26 | "name": "LangDict", 27 | "user_input": "What is your name?" 28 | }) 29 | ) 30 | 31 | 32 | def test_langdict_openai_query_rewrite(): 33 | query_rewrite = LangDict.from_dict({ 34 | "messages": ChatPromptMessagesBuilder.build( 35 | persona="You are a helpful AI bot.", 36 | task_instruction="Rewrite Human's question to search query.", 37 | output_format="json, query: str", 38 | fewshot_examples=[], 39 | conversation_key="conversation" 40 | ), 41 | "llm": { 42 | "model": "gpt-4o-mini", 43 | "max_tokens": 200 44 | }, 45 | "output": { 46 | "type": "json" 47 | } 48 | }) 49 | 50 | result = query_rewrite({ 51 | "conversation": [{"role": "user", "content": "How old is Obama?"}] 52 | }) 53 | 54 | assert ("query" in result) is True 55 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "langdict" 7 | version = "0.0.2" 8 | description = "Build complex LLM Applications with Python Dictionary" 9 | 10 | readme = "README.md" 11 | requires-python = ">=3.9,<4.0" 12 | license = {file = "LICENSE"} 13 | 14 | keywords = ["LLM", "RAG", "Agent", "Compund AI Systems", "LangDict"] 15 | 16 | authors = [ 17 | {name = "Dongjun Lee", email = "djlee.hb@gmail.com"}, 18 | ] 19 | maintainers = [ 20 | {name = "Dongjun Lee", email = "djlee.hb@gmail.com"}, 21 | ] 22 | 23 | classifiers = [ 24 | "Development Status :: 3 - Alpha", 25 | 26 | "Intended Audience :: Developers", 27 | "Topic :: Software Development :: Libraries :: Python Modules", 28 | 29 | "License :: OSI Approved :: MIT License", 30 | 31 | "Programming Language :: Python :: 3", 32 | "Programming Language :: Python :: 3.9", 33 | "Programming Language :: Python :: 3.10", 34 | "Programming Language :: Python :: 3.11", 35 | "Programming Language :: Python :: 3.12", 36 | "Programming Language :: Python :: 3 :: Only", 37 | ] 38 | 39 | [tool.setuptools.dynamic] 40 | dependencies = {file = "requirements.txt"} 41 | 42 | [project.optional-dependencies] 43 | dev = [] 44 | test = ["pytest"] 45 | 46 | [tool.setuptools] 47 | include-package-data = true 48 | 49 | [tool.setuptools.packages.find] 50 | where = ["src"] 51 | exclude = ["tests", "tests.*", "examples", "*.ipynb", "*.json"] 52 | 53 | [project.urls] 54 | Homepage = "https://langdict.github.io/" 55 | Documentation = "https://langdict.github.io/docs" 56 | Repository = "https://github.com/LangDict/langdict.git" 57 | "Bug Tracker" = "https://github.com/LangDict/langdict/issues" 58 | 59 | [tool.pytest.ini_options] 60 | pythonpath = [ 61 | "src" 62 | ] 63 | -------------------------------------------------------------------------------- /src/langdict/modules/rankings/rank_gpt.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List 2 | 3 | from langdict import LangDict, LangDictModule 4 | 5 | 6 | _SPECIFICATION = { 7 | "messages": [ 8 | ("system", "You are RankGPT, an intelligent assistant that can rank passages based on their relevancy to the query."), 9 | ("human", "I will provide you with {num} passages, each indicated by number identifier []. Rank them based on their relevance to query: {query}."), 10 | ("ai", "Okay, please provide the passages."), 11 | ("placeholder", "{passages}"), 12 | ("human", """Search Query: {query}. 13 | Rank the {num} passages above based on their relevance to the search query. 14 | The passages should be listed in descending order using identifiers, and the most relevant passages should be listed first, and the output format should be List[int], e.g., [1, 2]. 15 | Only response the ranking results, do not say any word or explain."""), 16 | ], 17 | "llm": { 18 | "model": "gpt-4o-mini", 19 | }, 20 | "output": { 21 | "type": "json" 22 | }, 23 | "metadata": { 24 | "arxiv": "https://arxiv.org/abs/2304.09542", 25 | } 26 | } 27 | 28 | 29 | class RankGPT(LangDictModule): 30 | """ 31 | Is ChatGPT Good at Search? Investigating Large Language Models as Re-Ranking Agents 32 | """ 33 | 34 | def __init__(self): 35 | super().__init__( 36 | LangDict.from_dict(_SPECIFICATION) 37 | ) 38 | 39 | def forward(self, query: str, passages: List[str]) -> Dict[str, Any]: 40 | passage_prompts = [] 41 | for i, passage in enumerate(passages): 42 | passage_prompts.append({ 43 | "role": "user", 44 | "content": f"[{i + 1}] {passage}" 45 | }) 46 | passage_prompts.append({ 47 | "role": "assistant", 48 | "content": f"Received passage [{i + 1}]" 49 | }) 50 | 51 | return { 52 | "query": query, 53 | "passages": passage_prompts, 54 | "num": len(passages), 55 | } 56 | -------------------------------------------------------------------------------- /src/langdict/specs/lang.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | from .base import BaseSpecification 4 | from .prompt import PromptSpecification 5 | from .llm import LLMSpecification 6 | from .output import OutputSpecification 7 | 8 | 9 | class LangSpecification(BaseSpecification): 10 | 11 | REQUIRE_KEYS = [ 12 | "llm", 13 | "output" 14 | ] 15 | 16 | def __init__( 17 | self, 18 | prompt: PromptSpecification, 19 | llm: LLMSpecification, 20 | output: OutputSpecification 21 | ): 22 | self.prompt = prompt 23 | self.llm = llm 24 | self.output = output 25 | 26 | super().__init__() 27 | 28 | def validate(self): 29 | self.prompt.validate() 30 | self.llm.validate() 31 | self.output.validate() 32 | 33 | @classmethod 34 | def from_dict(cls, data: Dict[str, Any]) -> "LangSpecification": 35 | if any(key not in data for key in cls.REQUIRE_KEYS): 36 | raise ValueError(f"Missing keys in data. Required keys: {cls.REQUIRE_KEYS}") 37 | 38 | if ( 39 | "text" in data and 40 | "messages" in data 41 | ): 42 | raise ValueError("Data cannot contain both 'text' and 'messages' keys.") 43 | 44 | prompt_type = None 45 | prompt_data = None 46 | if "text" in data: 47 | prompt_type = "text" 48 | prompt_data = data["text"] 49 | elif "messages" in data: 50 | prompt_type = "chat" 51 | prompt_data = data["messages"] 52 | else: 53 | raise ValueError("Data must contain either 'text' or 'messages' key.") 54 | 55 | prompt = PromptSpecification.from_dict(prompt_data, prompt_type=prompt_type) 56 | llm = LLMSpecification.from_dict(data["llm"]) 57 | output = OutputSpecification.from_dict(data["output"]) 58 | return cls(prompt, llm, output) 59 | 60 | def as_dict(self) -> Dict[str, Any]: 61 | data = self.prompt.as_dict() 62 | data["llm"] = self.llm.as_dict() 63 | data["output"] = self.output.as_dict() 64 | return data 65 | -------------------------------------------------------------------------------- /src/langdict/specs/prompt.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, Tuple, Union 2 | 3 | from .base import BaseSpecification 4 | 5 | 6 | class PromptSpecification(BaseSpecification): 7 | 8 | def __init__(self): 9 | self.validate() 10 | 11 | def validate(self): 12 | raise NotImplementedError("PromptSpecification.validate() must be implemented in a subclass.") 13 | 14 | @classmethod 15 | def from_dict( 16 | cls, 17 | data: Union[Dict, List], 18 | prompt_type: str = "chat" 19 | ) -> Union["TextPromptSpecification", "ChatPromptSpecification"]: 20 | if prompt_type == "text": 21 | return TextPromptSpecification(data) 22 | elif prompt_type == "chat": 23 | return ChatPromptSpecification.from_dict(data) 24 | else: 25 | raise ValueError("Invalid prompt type.") 26 | 27 | 28 | class TextPromptSpecification(PromptSpecification): 29 | 30 | def __init__(self, text: str): 31 | self.text = text 32 | 33 | super().__init__() 34 | 35 | def validate(self): 36 | if not self.text: 37 | raise ValueError("Text prompt is empty.") 38 | if ( 39 | "{" not in self.text or 40 | "}" not in self.text 41 | ): 42 | raise ValueError("PromptSpecification is missing placeholders.") 43 | 44 | 45 | class ChatPromptSpecification(PromptSpecification): 46 | 47 | def __init__(self, messages: List[Tuple[str, str]]): 48 | self.messages = messages 49 | 50 | super().__init__() 51 | 52 | def validate(self): 53 | required_keys = {"human", "ai", "system", "placeholder"} 54 | 55 | for m in self.messages: 56 | if m[0] not in required_keys: 57 | raise ValueError(f"Invalid role in message: {m[0]}") 58 | 59 | @classmethod 60 | def from_dict(cls, data: List[Union[List[str], Tuple[str, str]]]) -> "ChatPromptSpecification": 61 | messages = [] 62 | for d in data: 63 | if type(d) == list: 64 | messages.append(tuple(d)) 65 | else: 66 | messages.append(d) 67 | return cls(messages) 68 | -------------------------------------------------------------------------------- /src/langdict/modules/rags/self_rag/is_relevant.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | from langdict import LangDict, LangDictModule 4 | 5 | 6 | _SPECIFICATION = { 7 | "messages": [ 8 | ("system", """You’ll be provided with an instruction, along with evidence and possibly some preceding sentences. 9 | When there are preceding sentences, your focus should be on the sentence that comes after them. 10 | Your job is to determine if the evidence is relevant to the initial instruction and the preceding context, 11 | and provides useful information to complete the task described in the instruction. 12 | If the evidence meets this requirement, respond with [Relevant]; otherwise, generate [Irrelevant]. 13 | 14 | ## Demonstrations 15 | - Instruction: Given four answer options, A, B, C, and D, choose the best answer. 16 | - Input: Earth’s rotating causes 17 | A: the cycling of AM and PM 18 | B: the creation of volcanic eruptions 19 | C: the cycling of the tides 20 | D: the creation of gravity 21 | - Evidence: Rotation causes the day-night cycle which also creates a corresponding cycle of temperature and humidity creates a corresponding cycle of temperature and humidity. Sea level rises and falls twice a day as the earth rotates. 22 | {{ "rating": "[Relevant]", "explanation": "The evidence explicitly mentions that the rotation causes a day-night cycle, as described in the answer option A." }} 23 | 24 | - Instruction: age to run for US House of Representatives 25 | - Evidence: The Constitution sets three qualifications for service in the U.S. Senate: age (at least thirty years of age); U.S. citizenship (at least nine years); and residency in the state a senator represents at the time of election. 26 | {{ "rating": "[Irrelevant]", "explanation": "The evidence only discusses the ages to run for the US Senate, not for the House of Representatives." }} 27 | 28 | - Instruction: {instruction} 29 | - Evidence: {evidence} 30 | """), 31 | ], 32 | "llm": { 33 | "model": "gpt-4o-mini", 34 | }, 35 | "output": { 36 | "type": "json" 37 | }, 38 | "metadata": { 39 | "arxiv": "https://arxiv.org/abs/2310.11511", 40 | } 41 | } 42 | 43 | 44 | class IsRelevant(LangDictModule): 45 | """ 46 | Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection 47 | 48 | d provides useful information to solve x. 49 | 50 | Type: [IsRel] 51 | Output: {relevant, irrelevant} 52 | """ 53 | 54 | def __init__(self): 55 | super().__init__( 56 | LangDict.from_dict(_SPECIFICATION) 57 | ) 58 | 59 | def forward(self, instruction: str, evidence: str) -> Dict[str, Any]: 60 | return { 61 | "instruction": instruction, 62 | "evidence": evidence, 63 | } 64 | -------------------------------------------------------------------------------- /src/langdict/modules/rags/self_rag/is_useful.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | from langdict import LangDict, LangDictModule 4 | 5 | 6 | _SPECIFICATION = { 7 | "messages": [ 8 | ("system", """Given an instruction and an output, rate whether the response appears to be a helpful and informative answer to the query, from 1 (lowest) - 5 (highest). 9 | We call this score perceived utility. The detailed criterion is as follows: 10 | 11 | 5: The response provides a complete, highly detailed, and informative response to the query, fully satisfying the information needs. 12 | 4: The response mostly fulfills the need in the query, while there can be some minor improvements such as discussing more detailed information, having better structure of the response, or improving coherence. 13 | 3: The response is acceptable, but some major additions or improvements are needed to satisfy users’ needs. 14 | 2: The response still addresses the main request, but it is not complete or not relevant to the query. 15 | 1: The response is barely on-topic or completely irrelevant. 16 | 17 | ## Demonstrations 18 | - Instruction: Who is the current prime minister of the UK as of 2023? 19 | - Output: Boris Johnson was the prime minister of the UK from 2019 - 2022. 20 | {{ "utility": 2, "explanation": "While the output provides a factually correct statement about the UK prime minister from 2019 to 2022, this instruction asks who the prime minister is as of 2023, so it doesn’t answer the instruction. Therefore, the utility is 2." }} 21 | 22 | - Instruction: Given a description of a travel destination, recommend 10 tourist attractions with detailed descriptions of each. The travel destination is Tokyo, Japan. 23 | - Output: Tokyo is a vibrant city full of exciting tourist attractions. Some must-see sights include the Tokyo Skytree, Tokyo Disneyland, Sensoji Temple, Meiji Shrine, Tsukiji Fish Market, Harajuku, and Shinjuku Gyoen. 24 | {{ "utility": 3, "explanation": "This output doesn’t provide descriptions of each attraction and the number of the attractions is also less than 10. While this output partially answers the instructions, it doesn’t match the instructions strictly." }} 25 | 26 | - Instruction: {instruction} 27 | - Output: {output} 28 | """), 29 | ], 30 | "llm": { 31 | "model": "gpt-4o-mini", 32 | }, 33 | "output": { 34 | "type": "json" 35 | }, 36 | "metadata": { 37 | "arxiv": "https://arxiv.org/abs/2310.11511", 38 | } 39 | } 40 | 41 | 42 | class IsUseful(LangDictModule): 43 | """ 44 | Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection 45 | 46 | y is a useful response to x. 47 | 48 | Type: [IsUse] 49 | Output: {5, 4, 3, 2, 1} 50 | """ 51 | 52 | def __init__(self): 53 | super().__init__( 54 | LangDict.from_dict(_SPECIFICATION) 55 | ) 56 | 57 | def forward( 58 | self, 59 | instruction: str, 60 | preceding: str, 61 | output: str, 62 | evidence: str, 63 | ) -> Dict[str, Any]: 64 | return { 65 | "instruction": instruction, 66 | "preceding": preceding, 67 | "output": output, 68 | "evidence": evidence, 69 | } 70 | -------------------------------------------------------------------------------- /src/langdict/modules/rags/self_rag/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List 2 | 3 | from langdict import LangDict, LangDictModule, Module 4 | 5 | from .need_retrieve import NeedRetrieve 6 | from .is_relevant import IsRelevant 7 | from .is_support import IsSupport 8 | from .is_useful import IsUseful 9 | 10 | 11 | _GENERATE_SPECIFICATION = { 12 | "messages": [ 13 | ("system", """Given an instruction and evidence, please make a answer. 14 | - Instruction: {instruction} 15 | - Evidence: {evidence} 16 | {preceding} 17 | """), 18 | ], 19 | "llm": { 20 | "model": "gpt-4o-mini", 21 | }, 22 | "output": { 23 | "type": "string" 24 | }, 25 | } 26 | 27 | 28 | class SelfRAG(Module): 29 | """ 30 | Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection 31 | """ 32 | 33 | def __init__(self, retriever: "Retriever"): 34 | super().__init__() 35 | self.retriever = retriever 36 | self.segment_generator = LangDictModule( 37 | LangDict.from_dict(_GENERATE_SPECIFICATION) 38 | ) 39 | 40 | # Reflection token 41 | self.retrieve = NeedRetrieve() 42 | self.is_rel = IsRelevant() 43 | self.is_sup = IsSupport() 44 | self.is_use = IsUseful() 45 | 46 | def forward( 47 | self, 48 | instruction: str, 49 | preceding: str = "", 50 | output: str = "", 51 | evidence: str = "", 52 | ) -> str: 53 | # Step 1: Retrieve on demand 54 | retrieve_token = self.retrieve(instruction) 55 | if retrieve_token == "[yes]": 56 | passages = self.retriever.search(instruction, preceding) 57 | 58 | inputs = [] 59 | for i, passage in enumerate(passages): 60 | inputs.append({ 61 | "index": i, 62 | "instruction": instruction, 63 | "preceding": preceding, 64 | "evidence": passage, 65 | "output": output, 66 | }) 67 | is_relevants = self.is_rel(inputs, batch=True) 68 | 69 | filtered_inputs = [] 70 | for i, is_relevant in enumerate(is_relevants): 71 | if is_relevant == "[Relevant]": 72 | filtered_inputs.append(inputs[i]) 73 | generated_segments = self.segment_generator(filtered_inputs, batch=True) 74 | for i, segment in enumerate(generated_segments): 75 | filtered_inputs[i]["output"] = segment 76 | 77 | is_supports = self.is_sup(filtered_inputs, batch=True) 78 | is_usefuls = self.is_use(filtered_inputs, batch=True) 79 | 80 | for input, is_relevant, is_support, is_useful in zip(filtered_inputs, is_relevants, is_supports, is_usefuls): 81 | index = input["index"] 82 | inputs[index].update({ 83 | "is_relevant": is_relevant, 84 | "is_support": is_support, 85 | "is_useful": is_useful, 86 | }) 87 | 88 | ranking = self.ranking(inputs) 89 | return generated_segments[ranking[0]] 90 | elif ( 91 | retrieve_token == "[continue]" or 92 | retrieve_token == "[no]" 93 | ): 94 | return self.segment_generator([{ 95 | "instruction": instruction, 96 | "preceding": preceding, 97 | "evidence": evidence 98 | }]) 99 | else: 100 | raise ValueError(f"Invalid retrieve token: {retrieve_token}") 101 | 102 | def ranking(self, segments: List[Dict[str, Any]]) -> List[int]: 103 | """ Heuristic ranking based on relevance, support, and usefulness. """ 104 | 105 | filtered_segments = [segment for segment in segments if segment["is_relevant"] == "[Relevant]"] 106 | sorted_segments = sorted(filtered_segments, key=lambda x: (x["is_support"], -x["is_useful"])) 107 | return [segment["index"] for segment in sorted_segments] 108 | -------------------------------------------------------------------------------- /src/langdict/modules/rags/self_rag/is_support.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | from langdict import LangDict, LangDictModule 4 | 5 | 6 | _SPECIFICATION = { 7 | "messages": [ 8 | ("system", """You will receive an instruction, evidence, and output, and optional preceding sentences. 9 | If the preceding sentence is given, the output should be the sentence that follows those preceding sentences. 10 | Your task is to evaluate if the output is fully supported by the information provided in the evidence. 11 | 12 | Use the following entailment scale to generate a score: 13 | - [Fully supported] - All information in output is supported by the evidence, or extractions 14 | from the evidence. This is only applicable when the output and part of the evidence are 15 | almost identical. 16 | - [Partially supported] - The output is supported by the evidence to some extent, but there 17 | is major information in the output that is not discussed in the evidence. For example, if an 18 | instruction asks about two concepts and the evidence only discusses either of them, it should 19 | be considered a [Partially supported]. 20 | - [No support / Contradictory] - The output completely ignores evidence, is unrelated to the 21 | evidence, or contradicts the evidence. This can also happen if the evidence is irrelevant to the 22 | instruction. 23 | 24 | Make sure to not use any external information/knowledge to judge whether the output is true or not. 25 | Only check whether the output is supported by the evidence, and not whether the output follows the instructions or not. 26 | 27 | ## Demonstrations 28 | - Instruction: Explain the use of word embeddings in Natural Language Processing. 29 | - Preceding sentences: Word embeddings are one of the most powerful tools available for Natural Language Processing (NLP). They are mathematical representations of words or phrases in a vector space, allowing similarities between words and the context in which they are used to be measured. 30 | - Output: Word embeddings are useful for tasks such as sentiment analysis, text classification, predicting the next word in a sequence, and understanding synonyms and analogies. 31 | - Evidence: Word embedding 32 | Word embedding is the collective name for a set of language modeling and feature learning techniques in natural language processing (NLP) where words or phrases from the vocabulary are mapped to vectors of real numbers. Conceptually it involves a mathematical embedding from a space with one dimension per word to a continuous vector space with a much lower dimension. Methods to generate this mapping include neural networks, dimensionality reduction on the word co-occurrence matrix, probabilistic models, explainable knowledge base method, and explicit representation in terms of the context in which words appear. Word and phrase embeddings, when used as the underlying input representation, have been shown to boost the performance in NLP tasks such as syntactic parsing, sentiment analysis, next token predictions as well and analogy detection. 33 | {{ "rating": "[Fully supported]", "explanation": "The output sentence discusses the application of word embeddings, and the evidence mentions all of the applications syntactic parsing, sentiment analysis, next token predictions as well as analogy detection as the applications. Therefore, the score should be [Fully supported]." }} 34 | 35 | - Instruction: {instruction} 36 | - Preceding sentences: {preceding} 37 | - Output: {output} 38 | - Evidence: {evidence} 39 | """), 40 | ], 41 | "llm": { 42 | "model": "gpt-4o-mini", 43 | }, 44 | "output": { 45 | "type": "json" 46 | }, 47 | "metadata": { 48 | "arxiv": "https://arxiv.org/abs/2310.11511", 49 | } 50 | } 51 | 52 | 53 | class IsSupport(LangDictModule): 54 | """ 55 | Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection 56 | 57 | All of the verification-worthy statement in y is supported by d. 58 | 59 | Type: [IsSup] 60 | Output: {fully supported, partially supported, no support} 61 | """ 62 | 63 | def __init__(self): 64 | super().__init__( 65 | LangDict.from_dict(_SPECIFICATION) 66 | ) 67 | 68 | def forward( 69 | self, 70 | instruction: str, 71 | preceding: str, 72 | output: str, 73 | evidence: str, 74 | ) -> Dict[str, Any]: 75 | return { 76 | "instruction": instruction, 77 | "preceding": preceding, 78 | "output": output, 79 | "evidence": evidence, 80 | } 81 | -------------------------------------------------------------------------------- /src/langdict/langdict.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, Tuple, Union 2 | 3 | from langchain_core.callbacks import BaseCallbackHandler 4 | 5 | from langdict.specs import LangSpecification 6 | from langdict.builders import ( 7 | PromptTemplateBuilder, 8 | LiteLLMBuilder, 9 | OutputParserBuilder, 10 | TraceCallbackBuilder, 11 | ) 12 | 13 | 14 | class LangDict: 15 | 16 | """LangDict: A unit of simple llm chain. 17 | 18 | Chain Structure: 19 | [Prompt] -> [LLM] -> [Output Parser] 20 | """ 21 | 22 | def __init__(self, spec: LangSpecification): 23 | self.spec = spec 24 | 25 | prompt = PromptTemplateBuilder.build(spec.prompt) 26 | llm = LiteLLMBuilder.build(spec.llm) 27 | output_parser = OutputParserBuilder.build(spec.output) 28 | 29 | chain = prompt | llm | output_parser 30 | self.chain = chain 31 | 32 | def __call__( 33 | self, 34 | inputs: Union[ 35 | Dict[str, Any], List[Tuple[str, Dict[str, Any]]] 36 | ], 37 | stream: bool = False, 38 | batch: bool = False, 39 | trace_backend: str = None, 40 | module_name: str = None, 41 | ): 42 | """Invoke the chain with inputs. 43 | 44 | Example:: 45 | 46 | chitchat({ 47 | "conversation": [("user", "Hello, how are you doing?")] 48 | }) 49 | chitchat({ 50 | "conversation": [("user", "Hello, how are you doing?")] 51 | }, stream=True) 52 | chitchat([inputs, inputs], batch=True) 53 | 54 | Args: 55 | inputs: input data for the chain. 56 | stream: enable streaming mode. 57 | batch: enable batch mode. 58 | trace_backend: trace backend to use. if None, no tracing. 59 | module_name: name of the module for tracing. 60 | 61 | """ 62 | 63 | # TODO: async implementation 64 | 65 | callbacks = self._trace_callbacks(trace_backend, module_name) 66 | 67 | if isinstance(inputs, dict): 68 | if stream: 69 | return self.chain.stream( 70 | inputs, 71 | config={"callbacks": callbacks} 72 | ) 73 | else: 74 | return self.chain.invoke( 75 | inputs, 76 | config={"callbacks": callbacks} 77 | ) 78 | elif isinstance(inputs, list): 79 | if batch: 80 | return self.chain.batch( 81 | inputs, 82 | config={"callbacks": callbacks} 83 | ) 84 | else: 85 | raise ValueError("List inputs must be batched.") 86 | else: 87 | raise ValueError("Invalid inputs type.") 88 | 89 | def _trace_callbacks( 90 | self, 91 | trace_backend: str, 92 | module_name: str 93 | ) -> List[BaseCallbackHandler]: 94 | callbacks = [] 95 | if trace_backend: 96 | builder = TraceCallbackBuilder() 97 | callback = builder.build( 98 | trace_backend, 99 | module_name=module_name, 100 | ) 101 | callbacks.append(callback) 102 | return callbacks 103 | 104 | @classmethod 105 | def from_dict(cls, data: Dict[str, Any]) -> "LangDict": 106 | """Create LangDict from dictionary data. 107 | 108 | Example:: 109 | 110 | chitchat = LangDict.from_dict({ 111 | "prompt": { 112 | "type": "chat", 113 | "messages": [ 114 | ("system", "You are a helpful AI bot. Your name is {name}."), 115 | ("human", "Hello, how are you doing?"), 116 | ("ai", "I'm doing well, thanks!"), 117 | ("human", "{user_input}"), 118 | ] 119 | }, 120 | "llm": { 121 | "model": "gpt-4o", 122 | "max_tokens": 200 123 | }, 124 | "output": { 125 | "type": "string" 126 | } 127 | }) 128 | 129 | Args: 130 | data: specification data for the LangDict 131 | (must include ('text' or 'messages'), 'llm', 'output' keys) 132 | """ 133 | 134 | lang_spec = LangSpecification.from_dict(data) 135 | return LangDict(lang_spec) 136 | 137 | def as_dict(self) -> Dict[str, Any]: 138 | return self.spec.as_dict() 139 | -------------------------------------------------------------------------------- /src/langdict/traces/callbacks/stdout.py: -------------------------------------------------------------------------------- 1 | """Callback Handler that prints to std out.""" 2 | 3 | from __future__ import annotations 4 | 5 | from typing import TYPE_CHECKING, Any, Optional 6 | 7 | from langchain_core.callbacks.base import BaseCallbackHandler 8 | from langchain_core.outputs import LLMResult 9 | from langchain_core.utils import print_text 10 | 11 | if TYPE_CHECKING: 12 | from langchain_core.agents import AgentAction, AgentFinish 13 | 14 | 15 | class TraceStdOutCallbackHandler(BaseCallbackHandler): 16 | """Callback Handler that prints to std out.""" 17 | 18 | def __init__( 19 | self, 20 | module_name: Optional[str] = None, 21 | session_id: Optional[str] = None, 22 | color: Optional[str] = None, 23 | ) -> None: 24 | """Initialize callback handler. 25 | 26 | Args: 27 | module_name: The name of the module. Defaults to None. 28 | color: The color to use for the text. Defaults to None. 29 | """ 30 | 31 | prefix = "" 32 | if session_id: 33 | prefix += f"[session_id={session_id}] " 34 | if module_name: 35 | prefix += f"[module={module_name}] " 36 | self.prefix = prefix 37 | self.color = color 38 | 39 | def on_llm_end( 40 | self, 41 | response: LLMResult, 42 | **kwargs: Any, 43 | ) -> Any: 44 | """Run when LLM ends running. 45 | 46 | Args: 47 | response (LLMResult): The response which was generated. 48 | run_id (UUID): The run ID. This is the ID of the current run. 49 | parent_run_id (UUID): The parent run ID. This is the ID of the parent run. 50 | kwargs (Any): Additional keyword arguments. 51 | """ 52 | print(f"\n\033[1m>{self.prefix} Finished LLM.\033[0m") # noqa: T201 53 | print_text(response, color=self.color) 54 | 55 | def on_chain_start( 56 | self, 57 | serialized: dict[str, Any], 58 | inputs: dict[str, Any], 59 | **kwargs: Any, 60 | ) -> None: 61 | """Print out that we are entering a chain. 62 | 63 | Args: 64 | serialized (Dict[str, Any]): The serialized chain. 65 | inputs (Dict[str, Any]): The inputs to the chain. 66 | **kwargs (Any): Additional keyword arguments. 67 | """ 68 | class_name = "" 69 | if serialized: 70 | class_name = serialized.get("name", serialized.get("id", [""])[-1]) 71 | print(f"\n\n\033[1m>{self.prefix} Entering new {class_name}chain...\033[0m") # noqa: T201 72 | print_text(f"inputs: {inputs}", color=self.color) # noqa: T201 73 | 74 | def on_chain_end(self, outputs: dict[str, Any], **kwargs: Any) -> None: 75 | """Print out that we finished a chain. 76 | 77 | Args: 78 | outputs (Dict[str, Any]): The outputs of the chain. 79 | **kwargs (Any): Additional keyword arguments. 80 | """ 81 | print(f"\n\033[1m>{self.prefix} Finished chain.\033[0m") # noqa: T201 82 | print_text(f"outputs: {outputs}", color=self.color) # noqa: T201 83 | 84 | 85 | def on_agent_action( 86 | self, action: AgentAction, color: Optional[str] = None, **kwargs: Any 87 | ) -> Any: 88 | """Run on agent action. 89 | 90 | Args: 91 | action (AgentAction): The agent action. 92 | color (Optional[str]): The color to use for the text. Defaults to None. 93 | **kwargs (Any): Additional keyword arguments. 94 | """ 95 | print_text(action.log, color=color or self.color) 96 | 97 | def on_tool_end( 98 | self, 99 | output: Any, 100 | color: Optional[str] = None, 101 | observation_prefix: Optional[str] = None, 102 | llm_prefix: Optional[str] = None, 103 | **kwargs: Any, 104 | ) -> None: 105 | """If not the final action, print out observation. 106 | 107 | Args: 108 | output (Any): The output to print. 109 | color (Optional[str]): The color to use for the text. Defaults to None. 110 | observation_prefix (Optional[str]): The observation prefix. 111 | Defaults to None. 112 | llm_prefix (Optional[str]): The LLM prefix. Defaults to None. 113 | **kwargs (Any): Additional keyword arguments. 114 | """ 115 | output = str(output) 116 | if observation_prefix is not None: 117 | print_text(f"\n{observation_prefix}") 118 | print_text(output, color=color or self.color) 119 | if llm_prefix is not None: 120 | print_text(f"\n{llm_prefix}") 121 | 122 | def on_text( 123 | self, 124 | text: str, 125 | color: Optional[str] = None, 126 | end: str = "", 127 | **kwargs: Any, 128 | ) -> None: 129 | """Run when the agent ends. 130 | 131 | Args: 132 | text (str): The text to print. 133 | color (Optional[str]): The color to use for the text. Defaults to None. 134 | end (str): The end character to use. Defaults to "". 135 | **kwargs (Any): Additional keyword arguments. 136 | """ 137 | print_text(text, color=color or self.color, end=end) 138 | 139 | def on_agent_finish( 140 | self, finish: AgentFinish, color: Optional[str] = None, **kwargs: Any 141 | ) -> None: 142 | """Run on the agent end. 143 | 144 | Args: 145 | finish (AgentFinish): The agent finish. 146 | color (Optional[str]): The color to use for the text. Defaults to None. 147 | **kwargs (Any): Additional keyword arguments. 148 | """ 149 | print_text(finish.log, color=color or self.color, end="\n") 150 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | merged into this file. For a more nuclear 2 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 3 | #.idea/ 4 | 5 | ### Python Patch ### 6 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 7 | poetry.toml 8 | 9 | # ruff 10 | .ruff_cache/ 11 | 12 | # LSP config files 13 | pyrightconfig.json 14 | 15 | ### Vim ### 16 | # Swap 17 | [._]*.s[a-v][a-z] 18 | !*.svg # comment out if you don't need vector files 19 | [._]*.sw[a-p] 20 | [._]s[a-rt-v][a-z] 21 | [._]ss[a-gi-z] 22 | [._]sw[a-p] 23 | 24 | # Session 25 | Session.vim 26 | Sessionx.vim 27 | 28 | # Temporary 29 | .netrwhist 30 | *~ 31 | # Auto-generated tag files 32 | tags 33 | # Persistent undo 34 | [._]*.un~ 35 | 36 | ### Windows ### 37 | # Windows thumbnail cache files 38 | Thumbs.db 39 | Thumbs.db:encryptable 40 | ehthumbs.db 41 | ehthumbs_vista.db 42 | 43 | # Dump file 44 | *.stackdump 45 | 46 | # Folder config file 47 | [Dd]esktop.ini 48 | 49 | # Recycle Bin used on file shares 50 | $RECYCLE.BIN/ 51 | 52 | # Windows Installer files 53 | *.cab 54 | *.msi 55 | *.msix 56 | *.msm 57 | *.msp 58 | 59 | # Windows shortcuts 60 | *.lnk 61 | 62 | # End of https://www.toptal.com/developers/gitignore/api/macos,windows,python,jupyternotebooks,vim# Created by https://www.toptal.com/developers/gitignore/api/macos,windows,python,jupyternotebooks,vim 63 | # Edit at https://www.toptal.com/developers/gitignore?templates=macos,windows,python,jupyternotebooks,vim 64 | 65 | ### JupyterNotebooks ### 66 | # gitignore template for Jupyter Notebooks 67 | # website: http://jupyter.org/ 68 | 69 | .ipynb_checkpoints 70 | */.ipynb_checkpoints/* 71 | 72 | # IPython 73 | profile_default/ 74 | ipython_config.py 75 | 76 | # Remove previous ipynb_checkpoints 77 | # git rm -r .ipynb_checkpoints/ 78 | 79 | ### macOS ### 80 | # General 81 | .DS_Store 82 | .AppleDouble 83 | .LSOverride 84 | 85 | # Icon must end with two \r 86 | Icon 87 | 88 | 89 | # Thumbnails 90 | ._* 91 | 92 | # Files that might appear in the root of a volume 93 | .DocumentRevisions-V100 94 | .fseventsd 95 | .Spotlight-V100 96 | .TemporaryItems 97 | .Trashes 98 | .VolumeIcon.icns 99 | .com.apple.timemachine.donotpresent 100 | 101 | # Directories potentially created on remote AFP share 102 | .AppleDB 103 | .AppleDesktop 104 | Network Trash Folder 105 | Temporary Items 106 | .apdisk 107 | 108 | ### macOS Patch ### 109 | # iCloud generated files 110 | *.icloud 111 | 112 | ### Python ### 113 | # Byte-compiled / optimized / DLL files 114 | __pycache__/ 115 | *.py[cod] 116 | *$py.class 117 | 118 | # C extensions 119 | *.so 120 | 121 | # Distribution / packaging 122 | .Python 123 | build/ 124 | develop-eggs/ 125 | dist/ 126 | downloads/ 127 | eggs/ 128 | .eggs/ 129 | lib/ 130 | lib64/ 131 | parts/ 132 | sdist/ 133 | var/ 134 | wheels/ 135 | share/python-wheels/ 136 | *.egg-info/ 137 | .installed.cfg 138 | *.egg 139 | MANIFEST 140 | 141 | # PyInstaller 142 | # Usually these files are written by a python script from a template 143 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 144 | *.manifest 145 | *.spec 146 | 147 | # Installer logs 148 | pip-log.txt 149 | pip-delete-this-directory.txt 150 | 151 | # Unit test / coverage reports 152 | htmlcov/ 153 | .tox/ 154 | .nox/ 155 | .coverage 156 | .coverage.* 157 | .cache 158 | nosetests.xml 159 | coverage.xml 160 | *.cover 161 | *.py,cover 162 | .hypothesis/ 163 | .pytest_cache/ 164 | cover/ 165 | 166 | # Translations 167 | *.mo 168 | *.pot 169 | 170 | # Django stuff: 171 | *.log 172 | local_settings.py 173 | db.sqlite3 174 | db.sqlite3-journal 175 | 176 | # Flask stuff: 177 | instance/ 178 | .webassets-cache 179 | 180 | # Scrapy stuff: 181 | .scrapy 182 | 183 | # Sphinx documentation 184 | docs/_build/ 185 | 186 | # PyBuilder 187 | .pybuilder/ 188 | target/ 189 | 190 | # Jupyter Notebook 191 | 192 | # IPython 193 | 194 | # pyenv 195 | # For a library or package, you might want to ignore these files since the code is 196 | # intended to run in multiple environments; otherwise, check them in: 197 | # .python-version 198 | 199 | # pipenv 200 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 201 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 202 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 203 | # install all needed dependencies. 204 | #Pipfile.lock 205 | 206 | # poetry 207 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 208 | # This is especially recommended for binary packages to ensure reproducibility, and is more 209 | # commonly ignored for libraries. 210 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 211 | #poetry.lock 212 | 213 | # pdm 214 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 215 | #pdm.lock 216 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 217 | # in version control. 218 | # https://pdm.fming.dev/#use-with-ide 219 | .pdm.toml 220 | 221 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 222 | __pypackages__/ 223 | 224 | # Celery stuff 225 | celerybeat-schedule 226 | celerybeat.pid 227 | 228 | # SageMath parsed files 229 | *.sage.py 230 | 231 | # Environments 232 | .env 233 | .venv 234 | env/ 235 | venv/ 236 | ENV/ 237 | env.bak/ 238 | venv.bak/ 239 | 240 | # Spyder project settings 241 | .spyderproject 242 | .spyproject 243 | 244 | # Rope project settings 245 | .ropeproject 246 | 247 | # mkdocs documentation 248 | /site 249 | 250 | # mypy 251 | .mypy_cache/ 252 | .dmypy.json 253 | dmypy.json 254 | 255 | # Pyre type checker 256 | .pyre/ 257 | 258 | # pytype static type analyzer 259 | .pytype/ 260 | 261 | # Cython debug symbols 262 | cython_debug/ 263 | 264 | # PyCharm 265 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 266 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 267 | # and can be added to the global gitignore or 268 | -------------------------------------------------------------------------------- /src/langdict/modules/rags/self_rag/need_retrieve.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Optional 2 | 3 | from langdict import LangDict, Module, LangDictModule 4 | 5 | 6 | _INPUT_ONLY_SPECIFICATION = { 7 | "messages": [ 8 | ("system", """Given an instruction, please make a judgment on whether finding some external documents from the web (e.g., Wikipedia) helps to generate a better response. 9 | Please answer [yes] or [no] and write an explanation. 10 | 11 | ## Demonstrations 12 | Instruction: Give three tips for staying healthy. 13 | {{ "need_retrieval": "[yes]", "explanation": "There might be some online sources listing three tips for staying healthy or some reliable sources to explain the effects of different behaviors on health. So retrieving documents is helpful to improve the response to this query." }} 14 | 15 | Instruction: Describe a time when you had to make a difficult decision. 16 | {{ "need_retrieval": "[no]", "explanation": "This instruction is asking about some personal experience and thus does not require one to find some external documents." }} 17 | 18 | Instruction: Write a short story in third person narration about a protagonist who has to make an important career decision. 19 | {{ "need_retrieval": "[no]", "explanation": "This instruction asks us to write a short story, which does not require external evidence to verify." }} 20 | 21 | Instruction: What is the capital of France? 22 | {{ "need_retrieval": "[yes]", "explanation": "While the instruction simply asks us to answer the capital of France, which is a widely known fact, retrieving web documents for this question can still help." }} 23 | 24 | Instruction: Find the area of a circle given its radius. Radius = 4 25 | {{ "need_retrieval": "[no]", "explanation": "This is a math question and does not require external evidence." }} 26 | 27 | Instruction: Arrange the words in the given sentence to form a grammatically correct sentence. quickly the brown fox jumped 28 | {{ "need_retrieval": "[no]", "explanation": "This task doesn’t require any external evidence, as it is a simple grammatical question." }} 29 | 30 | Instruction: Explain the process of cellular respiration in plants. 31 | {{ "need_retrieval": "[yes]", "explanation": "This instruction asks for a detailed description of a scientific concept, and is highly likely that we can find a reliable and useful document to support the response." }} 32 | 33 | Instruction: {instruction} 34 | """), 35 | ], 36 | "llm": { 37 | "model": "gpt-4o-mini", 38 | }, 39 | "output": { 40 | "type": "json" 41 | }, 42 | "metadata": { 43 | "arxiv": "https://arxiv.org/abs/2310.11511", 44 | } 45 | } 46 | 47 | 48 | _WITH_PRECEDING_SPECIFICATION = { 49 | "messages": [ 50 | ("system", """You will be provided with an instruction, evidence, output sentence, and preceding sentences (optional). 51 | If the preceding sentence is given, the output should be the sentence that follows those preceding sentences. 52 | Your task is to determine whether the information in the output sentence can be fully verified by the evidence or if it requires further external verification. 53 | There are three cases: 54 | - If the output sentence can be verified solely with the evidence, then respond with [continue]. 55 | - If the sentence doesn’t require any factual verification (e.g., a subjective sentence or a sentence about common sense), then respond with [no]. 56 | - If additional information is needed to verify the output sentence, respond with [yes]. 57 | Please provide explanations for your judgments. 58 | 59 | ## Demonstrations 60 | - Instruction: Explain the use of word embeddings in Natural Language Processing. 61 | - Preceding sentences: Word embeddings are one of the most powerful tools available for Natural Language Processing (NLP). They are mathematical representations of words or phrases in a vector space, allowing similarities between words and the context in which they are used to be measured. 62 | - Evidence: Word embedding 63 | Word embedding is the collective name for a set of language modeling and feature learning techniques in natural language processing (NLP) where words or phrases from the vocabulary are mapped to vectors of real numbers. Conceptually it involves a mathematical embedding from a space with one dimension per word to a continuous vector space with a much lower dimension. Methods to generate this mapping include neural networks, dimensionality reduction on the word co-occurrence matrix, probabilistic models, explainable knowledge base method, and explicit representation in terms of the context in which words appear. Word and phrase embeddings, when used as the underlying input representation, have been shown to boost the performance in NLP tasks such as syntactic parsing, sentiment analysis, next token predictions as well and analogy detection. 64 | {{ "rating": "[yes]", "explanation": "The output discusses the applications of word embeddings, while the evidence only discusses the definitions of word embeddings and how they work. Therefore, we need to retrieve other evidence to verify whether the output is correct or not." }} 65 | 66 | - Instruction: {instruction} 67 | - Preceding sentences: {preceding} 68 | - Evidence: {evidence} 69 | """), 70 | ], 71 | "llm": { 72 | "model": "gpt-4o-mini", 73 | }, 74 | "output": { 75 | "type": "json" 76 | }, 77 | "metadata": { 78 | "arxiv": "https://arxiv.org/abs/2310.11511", 79 | } 80 | } 81 | 82 | 83 | 84 | class NeedRetrieve(Module): 85 | """ 86 | Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection 87 | 88 | Decides when to retrieve with R 89 | 90 | Type: [Retrieve] 91 | Output: {yes, no, continue} 92 | """ 93 | 94 | def __init__(self): 95 | super().__init__() 96 | 97 | self.input_only = LangDictModule( 98 | LangDict.from_dict(_INPUT_ONLY_SPECIFICATION) 99 | ) 100 | self.with_preceding = LangDictModule( 101 | LangDict.from_dict(_WITH_PRECEDING_SPECIFICATION) 102 | ) 103 | 104 | def forward( 105 | self, 106 | instruction: str, 107 | preceding: Optional[str] = None, 108 | evidence: Optional[str] = None, 109 | ) -> Dict[str, Any]: 110 | 111 | if (preceding and evidence): 112 | inputs = { 113 | "instruction": instruction, 114 | "preceding": preceding, 115 | "evidence": evidence 116 | } 117 | result = self.with_preceding(inputs) 118 | else: 119 | inputs = { 120 | "instruction": instruction 121 | } 122 | result = self.input_only(inputs) 123 | 124 | if 125 | -------------------------------------------------------------------------------- /src/langdict/modules/module.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Any, Dict, List, Optional, TypeVar 3 | 4 | from langchain_core.callbacks import BaseCallbackHandler 5 | from langchain_core.runnables import RunnableLambda 6 | 7 | from langdict.builders import TraceCallbackBuilder 8 | 9 | from .parameter import Parameter 10 | 11 | 12 | T = TypeVar("T", bound="Module") 13 | 14 | 15 | class Module: 16 | 17 | """Module: base class for compound ai systems. 18 | 19 | Example: 20 | 21 | class RAG(Module): 22 | 23 | def __init__(self, docs: List[str]): 24 | super().__init__() 25 | self.query_rewrite = LangDictModule.from_dict({ ... }) 26 | self.search = SimpleKeywordSearch(docs=docs) # Module 27 | self.answer = LangDictModule.from_dict({ ... }) 28 | 29 | def forward(self, inputs: Dict): 30 | query_rewrite_result = self.query_rewrite({ 31 | "conversation": inputs["conversation"], 32 | }) 33 | doc = self.search(query_rewrite_result) 34 | return self.answer({ 35 | "conversation": inputs["conversation"], 36 | "context": doc, 37 | }) 38 | 39 | """ 40 | 41 | NAME: str = None 42 | 43 | batching: bool = False 44 | streaming: bool = False 45 | is_last_child: Optional["Module"] = None 46 | trace_backend: Optional[str] = None # If None, no tracing 47 | 48 | def __init__(self): 49 | self.trace_backend = None 50 | self._parameters = {} 51 | self._modules = {} 52 | 53 | def __call__( 54 | self, 55 | *args, 56 | stream: bool = False, 57 | **kwargs, 58 | ): 59 | if ( 60 | stream and 61 | self.is_last_child is None 62 | ): 63 | self.stream(stream) 64 | self._set_last_child() 65 | 66 | chain = RunnableLambda(lambda x: self.forward(x)) 67 | callbacks = self._trace_callbacks(self.trace_backend, self._get_name()) 68 | return chain.invoke( 69 | *args, 70 | config={"callbacks": callbacks}, 71 | **kwargs, 72 | ) 73 | 74 | def _trace_callbacks( 75 | self, 76 | trace_backend: str, 77 | module_name: str 78 | ) -> List[BaseCallbackHandler]: 79 | callbacks = [] 80 | if trace_backend: 81 | builder = TraceCallbackBuilder() 82 | callback = builder.build( 83 | trace_backend, 84 | module_name=module_name, 85 | ) 86 | callbacks.append(callback) 87 | return callbacks 88 | 89 | def __getattr__(self, name: str) -> "Module": 90 | if ( 91 | "_parameters" not in self.__dict__ or 92 | "_modules" not in self.__dict__ 93 | ): 94 | raise ValueError("Intialize Module first.") 95 | 96 | if name in self._parameters: 97 | return self._parameters[name] 98 | elif name in self._modules: 99 | return self._modules[name] 100 | 101 | raise AttributeError( 102 | f"'{type(self).__name__}' object has no attribute '{name}'" 103 | ) 104 | 105 | def __setattr__(self, name: str, value: "Module") -> None: 106 | if isinstance(value, Module): 107 | value.NAME = name 108 | self._modules[name] = value 109 | elif isinstance(value, Parameter): 110 | self._parameters[name] = value 111 | else: 112 | object.__setattr__(self, name, value) 113 | 114 | def _get_name(self): 115 | if self.NAME: 116 | return self.NAME 117 | return self.__class__.__name__ 118 | 119 | def forward(self): 120 | raise NotImplementedError( 121 | f"Module [{type(self).__name__}] is missing the required \"forward\" function" 122 | ) 123 | 124 | def children(self): 125 | """Yield all children modules.""" 126 | visited = set() 127 | for name, module in self._modules.items(): 128 | if module and module not in visited: 129 | visited.add(module) 130 | yield module 131 | 132 | def trace(self, backend: str = "console") -> T: 133 | """Set the trace backend for all modules. 134 | 135 | Args: 136 | backend (str): The trace backend to use. Default is "console". 137 | 138 | Returns: 139 | Module: self 140 | """ 141 | self.trace_backend = backend 142 | 143 | for module in self.children(): 144 | module.trace(backend) 145 | return self 146 | 147 | def stream(self, is_stream: bool = False) -> T: 148 | """Set the streaming flag for all modules. 149 | 150 | Args: 151 | is_stream (bool): The streaming flag to use. Default is False. 152 | 153 | Returns: 154 | Module: self 155 | """ 156 | self.streaming = is_stream 157 | 158 | for module in self.children(): 159 | module.stream(is_stream) 160 | return self 161 | 162 | def _set_last_child(self) -> "Module": 163 | modules = list(self.children()) 164 | for m in modules: 165 | m.is_last_child = False 166 | last_child = modules[-1] 167 | 168 | while modules: 169 | modules = list(last_child.children()) 170 | for m in modules: 171 | m.is_last_child = False 172 | if not modules: 173 | break 174 | last_child = modules[-1] 175 | 176 | last_child.is_last_child = True 177 | 178 | def save_json(self, filename: str) -> None: 179 | """Save the module as a json file. 180 | 181 | Args: 182 | filename (str): The filename to save the module to. 183 | """ 184 | all_modules = {} 185 | for module in self.children(): 186 | parameters = {} 187 | for name, p in module._parameters.items(): 188 | parameters[name] = p.value 189 | 190 | all_modules[module.NAME] = { 191 | "module": module.as_dict(), 192 | "parameters": parameters, 193 | } 194 | 195 | with open(filename, "w") as f: 196 | json.dump(all_modules, f, ensure_ascii=False, indent=4) 197 | 198 | def load_json(self, filename: str) -> T: 199 | """Load the module from a json file. 200 | 201 | Args: 202 | filename (str): The filename to load the module from. 203 | 204 | Returns: 205 | Module: self 206 | """ 207 | with open(filename, "r") as f: 208 | data = json.load(f) 209 | 210 | parameter_data = data.get("parameters", {}) 211 | module_data = data.get("modules", {}) 212 | 213 | for name, value in parameter_data.items(): 214 | self.__setattr__(name, Parameter(value)) 215 | 216 | for module in self.children(): 217 | module_data = data.get(module.NAME, {}) 218 | 219 | parameter_data = module_data.get("parameters", {}) 220 | for name, value in parameter_data.items(): 221 | module.__setattr__(name, Parameter(value)) 222 | 223 | langdict_data = module_data.get(module.NAME) 224 | if not langdict_data: 225 | continue 226 | self.__setattr__(module.NAME, module.__class__.from_dict(langdict_data)) 227 | return self 228 | 229 | @classmethod 230 | def from_dict(cls, data: Dict[str, Any]) -> "Module": 231 | return cls() 232 | 233 | def as_dict(self) -> Dict[str, Any]: 234 | return {} 235 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |

3 | 4 |

5 | 6 | 7 |

8 | Build complex LLM Applications with Python Dictionary 9 |

10 | 11 | --- 12 | 13 | # LangDict 14 | 15 | LangDict is a framework for building agents (Compound AI Systems) using only specifications in a Python `Dictionary`. The framework is simple and intuitive to use for production. 16 | 17 | The prompts are similar to a feature specification, which is all you need to build an LLM Module. LangDict was created with the design philosophy that building LLM applications should be as simple as possible. Build your own LLM Application with minimal understanding of the framework. 18 | 19 |

20 | 21 |

22 | 23 | An Agent can be built by connecting multiple Modules. At LangDict, we focus on the intuitive interface, modularity, extensibility, and reusability of [PyTorch](https://github.com/pytorch/pytorch)'s `nn.Module`. If you have experience developing Neural Networks with PyTorch, you will understand how to use it right away. 24 | 25 | 26 | ## Modules 27 | 28 | | Task | Name | Code | 29 | | ---- | ---- | ---- | 30 | | `Ranking` | RankGPT | [Code](https://github.com/LangDict/langdict/blob/main/src/langdict/modules/rankings/rank_gpt.py) | 31 | | `Compression` | TextCompressor (LLMLingua-2) | [Code](https://github.com/LangDict/langdict/blob/main/src/langdict/modules/compressions/llm_lingua2.py) | 32 | | `RAG` | SELF-RAG | [Code](https://github.com/LangDict/langdict/blob/main/src/langdict/modules/rags/self_rag/__init__.py) | 33 | 34 | 35 | ## Key Features 36 | 37 |
38 | LLM Applicaiton framework for simple, intuitive, specification-based development 39 | 40 | ```python 41 | chitchat = LangDict.from_dict({ 42 | "messages": [ 43 | ("system", "You are a helpful AI bot. Your name is {name}."), 44 | ("human", "Hello, how are you doing?"), 45 | ("ai", "I'm doing well, thanks!"), 46 | ("human", "{user_input}"), 47 | ], 48 | "llm": { 49 | "model": "gpt-4o-mini", 50 | "max_tokens": 200 51 | }, 52 | "output": { 53 | "type": "string" 54 | } 55 | }) 56 | # format placeholder is key of input dictionary 57 | chitchat({ 58 | "name": "LangDict", 59 | "user_input": "What is your name?" 60 | }) 61 | ``` 62 | 63 |
64 | 65 |
66 | Simple interface (Stream / Batch) 67 | 68 | ```python 69 | rag = RAG() 70 | 71 | single_inputs = { 72 | "conversation": [{"role": "user", "content": "How old is Obama?"}] 73 | } 74 | # invoke 75 | rag(single_inputs) 76 | 77 | # stream 78 | rag(single_inputs, stream=True) 79 | 80 | # batch 81 | batch_inputs = [{ ... }, { ...}, ...] 82 | rag(batch_inputs, batch=True) 83 | ``` 84 | 85 |
86 | 87 |
88 | Modularity: Extensibility, Modifiability, Reusability 89 | 90 | ```python 91 | class RAG(Module): 92 | 93 | def __init__(self, docs: List[str]): 94 | super().__init__() 95 | self.query_rewrite = LangDictModule.from_dict({ ... }) # Module 96 | self.search = Retriever(docs=docs) # Module 97 | self.answer = LangDictModule.from_dict({ ... }) # Module 98 | 99 | def forward(self, inputs: Dict): 100 | query_rewrite_result = self.query_rewrite({ 101 | "conversation": inputs["conversation"], 102 | }) 103 | doc = self.search(query_rewrite_result) 104 | return self.answer({ 105 | "conversation": inputs["conversation"], 106 | "context": doc, 107 | }) 108 | ``` 109 | 110 |
111 | 112 |
113 | Easy to change trace options (Console, Langfuse, LangSmith) 114 | 115 | ```python 116 | # Apply Trace option to all modules 117 | rag = RAG() 118 | 119 | # Console Trace 120 | rag.trace(backend="console") 121 | 122 | # Langfuse 123 | rag.trace(backend="langfuse") 124 | 125 | # LangSmith 126 | rag.trace(backend="langsmith") 127 | ``` 128 | 129 |
130 | 131 |
132 | Easy to change hyper-paramters (Prompt, Paramter) 133 | 134 | ```python 135 | rag = RAG() 136 | rag.save_json("rag.json") 137 | # Modify "rag.json" file 138 | rag.load_json("rag.json") 139 | ``` 140 |
141 | 142 | ## Quick Start 143 | 144 | Install LangDict: 145 | 146 | ```python 147 | $ pip install langdict 148 | ``` 149 | 150 | ### Example 151 | 152 | **LangDict** 153 | - Build LLM Module with the specification. 154 | 155 | ```python 156 | from langdict import LangDict 157 | 158 | 159 | _SPECIFICATION = { 160 | "messages": [ 161 | ("system", "You are a helpful AI bot. Your name is {name}."), 162 | ("human", "Hello, how are you doing?"), 163 | ("ai", "I'm doing well, thanks!"), 164 | ("human", "{user_input}"), 165 | ], 166 | "llm": { 167 | "model": "gpt-4o-mini", 168 | "max_tokens": 200 169 | }, 170 | "output": { 171 | "type": "string" 172 | } 173 | } 174 | chitchat = LangDict.from_dict(_SPECIFICATION) 175 | chitchat({ 176 | "name": "LangDict", 177 | "user_input": "What is your name?" 178 | }) 179 | >>> 'My name is LangDict. How can I assist you today?' 180 | ``` 181 | 182 | **Module** 183 | - Build a agent by connecting multiple modules. 184 | 185 | ```python 186 | from typing import Any, Dict, List 187 | 188 | from langdict import Module, LangDictModule 189 | 190 | 191 | _QUERY_REWRITE_SPECIFICATION = { ... } 192 | _ANSWER_SPECIFICATION = { ... } 193 | 194 | 195 | class RAG(Module): 196 | 197 | def __init__(self, docs: List[str]): 198 | super().__init__() 199 | self.query_rewrite = LangDictModule.from_dict(_QUERY_REWRITE_SPECIFICATION) 200 | self.search = SimpleRetriever(docs=docs) # Module 201 | self.answer = LangDictModule.from_dict(_ANSWER_SPECIFICATION) 202 | 203 | def forward(self, inputs: Dict[str, Any]): 204 | query_rewrite_result = self.query_rewrite({ 205 | "conversation": inputs["conversation"], 206 | }) 207 | doc = self.search(query_rewrite_result) 208 | return self.answer({ 209 | "conversation": inputs["conversation"], 210 | "context": doc, 211 | }) 212 | 213 | rag = RAG() 214 | inputs = { 215 | "conversation": [{"role": "user", "content": "How old is Obama?"}] 216 | } 217 | 218 | rag(inputs) 219 | >>> 'Barack Obama was born on August 4, 1961. As of now, in September 2024, he is 63 years old.' 220 | ``` 221 | 222 | - Streaming 223 | 224 | ```python 225 | rag = RAG() 226 | # Stream 227 | for token in rag(inputs, stream=True): 228 | print(f"token > {token}") 229 | >>> 230 | token > Bar 231 | token > ack 232 | token > Obama 233 | token > was 234 | token > born 235 | token > on 236 | token > August 237 | token > 238 | token > 4 239 | ... 240 | ``` 241 | 242 | - Get observability with a single line of code. 243 | 244 | ```python 245 | rag = RAG() 246 | # Trace 247 | rag.trace(backend="console") 248 | ``` 249 | 250 | - Save and load the module as a JSON file. 251 | 252 | ```python 253 | rag = RAG() 254 | rag.save_json("rag.json") 255 | rag.load_json("rag.json") 256 | ``` 257 | 258 | ## Dependencies 259 | 260 | LangDict requires the following: 261 | 262 | - [`LangChain`](https://github.com/langchain-ai/langchain) - LangDict consists of PromptTemplate + LLM + Output Parser. 263 | - langchain 264 | - langchain-core 265 | - [`LiteLLM`](https://github.com/BerriAI/litellm) - Call 100+ LLM APIs in OpenAI format. 266 | 267 | ### Optional 268 | 269 | - [`Langfuse`](https://github.com/langfuse/langfuse) - If you use langfuse with the Trace option, you need to install it separately. 270 | -------------------------------------------------------------------------------- /src/langdict/chat_models/litellm.py: -------------------------------------------------------------------------------- 1 | """Wrapper around LiteLLM's model I/O library. 2 | origin_code: https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/chat_models/litellm.py 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | import json 8 | import logging 9 | from typing import ( 10 | Any, 11 | AsyncIterator, 12 | Callable, 13 | Dict, 14 | Iterator, 15 | List, 16 | Mapping, 17 | Optional, 18 | Sequence, 19 | Tuple, 20 | Type, 21 | Union, 22 | ) 23 | 24 | from pydantic import BaseModel, Field 25 | from langchain_core.callbacks import ( 26 | AsyncCallbackManagerForLLMRun, 27 | CallbackManagerForLLMRun, 28 | ) 29 | from langchain_core.language_models import LanguageModelInput 30 | from langchain_core.language_models.chat_models import ( 31 | BaseChatModel, 32 | agenerate_from_stream, 33 | generate_from_stream, 34 | ) 35 | from langchain_core.language_models.llms import create_base_retry_decorator 36 | from langchain_core.messages import ( 37 | AIMessage, 38 | AIMessageChunk, 39 | BaseMessage, 40 | BaseMessageChunk, 41 | ChatMessage, 42 | ChatMessageChunk, 43 | FunctionMessage, 44 | FunctionMessageChunk, 45 | HumanMessage, 46 | HumanMessageChunk, 47 | SystemMessage, 48 | SystemMessageChunk, 49 | ToolCall, 50 | ToolCallChunk, 51 | ToolMessage, 52 | ) 53 | from langchain_core.outputs import ( 54 | ChatGeneration, 55 | ChatGenerationChunk, 56 | ChatResult, 57 | ) 58 | from langchain_core.runnables import Runnable 59 | from langchain_core.tools import BaseTool 60 | from langchain_core.utils import get_from_dict_or_env, pre_init 61 | from langchain_core.utils.function_calling import convert_to_openai_tool 62 | 63 | logger = logging.getLogger(__name__) 64 | 65 | 66 | class ChatLiteLLMException(Exception): 67 | """Error with the `LiteLLM I/O` library""" 68 | 69 | 70 | def _create_retry_decorator( 71 | llm: ChatLiteLLM, 72 | run_manager: Optional[ 73 | Union[AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun] 74 | ] = None, 75 | ) -> Callable[[Any], Any]: 76 | """Returns a tenacity retry decorator, preconfigured to handle PaLM exceptions""" 77 | import litellm 78 | 79 | errors = [ 80 | litellm.Timeout, 81 | litellm.APIError, 82 | litellm.APIConnectionError, 83 | litellm.RateLimitError, 84 | ] 85 | return create_base_retry_decorator( 86 | error_types=errors, max_retries=llm.max_retries, run_manager=run_manager 87 | ) 88 | 89 | 90 | def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage: 91 | role = _dict["role"] 92 | if role == "user": 93 | return HumanMessage(content=_dict["content"]) 94 | elif role == "assistant": 95 | # Fix for azure 96 | # Also OpenAI returns None for tool invocations 97 | content = _dict.get("content", "") or "" 98 | 99 | additional_kwargs = {} 100 | if _dict.get("function_call"): 101 | additional_kwargs["function_call"] = dict(_dict["function_call"]) 102 | 103 | if _dict.get("tool_calls"): 104 | additional_kwargs["tool_calls"] = _dict["tool_calls"] 105 | 106 | return AIMessage(content=content, additional_kwargs=additional_kwargs) 107 | elif role == "system": 108 | return SystemMessage(content=_dict["content"]) 109 | elif role == "function": 110 | return FunctionMessage(content=_dict["content"], name=_dict["name"]) 111 | else: 112 | return ChatMessage(content=_dict["content"], role=role) 113 | 114 | 115 | async def acompletion_with_retry( 116 | llm: ChatLiteLLM, 117 | run_manager: Optional[AsyncCallbackManagerForLLMRun] = None, 118 | **kwargs: Any, 119 | ) -> Any: 120 | """Use tenacity to retry the async completion call.""" 121 | retry_decorator = _create_retry_decorator(llm, run_manager=run_manager) 122 | 123 | @retry_decorator 124 | async def _completion_with_retry(**kwargs: Any) -> Any: 125 | # Use OpenAI's async api https://github.com/openai/openai-python#async-api 126 | return await llm.client.acreate(**kwargs) 127 | 128 | return await _completion_with_retry(**kwargs) 129 | 130 | 131 | def _convert_delta_to_message_chunk( 132 | _dict: Mapping[str, Any], default_class: Type[BaseMessageChunk] 133 | ) -> BaseMessageChunk: 134 | role = _dict.get("role") 135 | content = _dict.get("content") or "" 136 | if _dict.get("function_call"): 137 | additional_kwargs = {"function_call": dict(_dict["function_call"])} 138 | else: 139 | additional_kwargs = {} 140 | 141 | tool_call_chunks = [] 142 | if raw_tool_calls := _dict.get("tool_calls"): 143 | additional_kwargs["tool_calls"] = raw_tool_calls 144 | try: 145 | tool_call_chunks = [ 146 | ToolCallChunk( 147 | name=rtc["function"].get("name"), 148 | args=rtc["function"].get("arguments"), 149 | id=rtc.get("id"), 150 | index=rtc["index"], 151 | ) 152 | for rtc in raw_tool_calls 153 | ] 154 | except KeyError: 155 | pass 156 | 157 | if role == "user" or default_class == HumanMessageChunk: 158 | return HumanMessageChunk(content=content) 159 | elif role == "assistant" or default_class == AIMessageChunk: 160 | return AIMessageChunk( 161 | content=content, 162 | additional_kwargs=additional_kwargs, 163 | tool_call_chunks=tool_call_chunks, 164 | ) 165 | elif role == "system" or default_class == SystemMessageChunk: 166 | return SystemMessageChunk(content=content) 167 | elif role == "function" or default_class == FunctionMessageChunk: 168 | return FunctionMessageChunk(content=content, name=_dict["name"]) 169 | elif role or default_class == ChatMessageChunk: 170 | return ChatMessageChunk(content=content, role=role) # type: ignore[arg-type] 171 | else: 172 | return default_class(content=content) # type: ignore[call-arg] 173 | 174 | 175 | def _lc_tool_call_to_openai_tool_call(tool_call: ToolCall) -> dict: 176 | return { 177 | "type": "function", 178 | "id": tool_call["id"], 179 | "function": { 180 | "name": tool_call["name"], 181 | "arguments": json.dumps(tool_call["args"]), 182 | }, 183 | } 184 | 185 | 186 | def _convert_message_to_dict(message: BaseMessage) -> dict: 187 | message_dict: Dict[str, Any] = {"content": message.content} 188 | if isinstance(message, ChatMessage): 189 | message_dict["role"] = message.role 190 | elif isinstance(message, HumanMessage): 191 | message_dict["role"] = "user" 192 | elif isinstance(message, AIMessage): 193 | message_dict["role"] = "assistant" 194 | if "function_call" in message.additional_kwargs: 195 | message_dict["function_call"] = message.additional_kwargs["function_call"] 196 | if message.tool_calls: 197 | message_dict["tool_calls"] = [ 198 | _lc_tool_call_to_openai_tool_call(tc) for tc in message.tool_calls 199 | ] 200 | elif "tool_calls" in message.additional_kwargs: 201 | message_dict["tool_calls"] = message.additional_kwargs["tool_calls"] 202 | elif isinstance(message, SystemMessage): 203 | message_dict["role"] = "system" 204 | elif isinstance(message, FunctionMessage): 205 | message_dict["role"] = "function" 206 | message_dict["name"] = message.name 207 | elif isinstance(message, ToolMessage): 208 | message_dict["role"] = "tool" 209 | message_dict["tool_call_id"] = message.tool_call_id 210 | else: 211 | raise ValueError(f"Got unknown type {message}") 212 | if "name" in message.additional_kwargs: 213 | message_dict["name"] = message.additional_kwargs["name"] 214 | return message_dict 215 | 216 | 217 | class ChatLiteLLM(BaseChatModel): 218 | """Chat model that uses the LiteLLM API.""" 219 | 220 | client: Any #: :meta private: 221 | model: str = "gpt-3.5-turbo" 222 | model_name: Optional[str] = None 223 | """Model name to use.""" 224 | openai_api_key: Optional[str] = None 225 | azure_api_key: Optional[str] = None 226 | anthropic_api_key: Optional[str] = None 227 | replicate_api_key: Optional[str] = None 228 | cohere_api_key: Optional[str] = None 229 | openrouter_api_key: Optional[str] = None 230 | streaming: bool = False 231 | api_base: Optional[str] = None 232 | organization: Optional[str] = None 233 | custom_llm_provider: Optional[str] = None 234 | request_timeout: Optional[Union[float, Tuple[float, float]]] = None 235 | temperature: Optional[float] = 1 236 | model_kwargs: Dict[str, Any] = Field(default_factory=dict) 237 | """Run inference with this temperature. Must be in the closed 238 | interval [0.0, 1.0].""" 239 | top_p: Optional[float] = None 240 | """Decode using nucleus sampling: consider the smallest set of tokens whose 241 | probability sum is at least top_p. Must be in the closed interval [0.0, 1.0].""" 242 | top_k: Optional[int] = None 243 | """Decode using top-k sampling: consider the set of top_k most probable tokens. 244 | Must be positive.""" 245 | n: int = 1 246 | """Number of chat completions to generate for each prompt. Note that the API may 247 | not return the full n completions if duplicates are generated.""" 248 | max_tokens: Optional[int] = None 249 | 250 | max_retries: int = 6 251 | 252 | @property 253 | def _default_params(self) -> Dict[str, Any]: 254 | """Get the default parameters for calling OpenAI API.""" 255 | set_model_value = self.model 256 | if self.model_name is not None: 257 | set_model_value = self.model_name 258 | return { 259 | "model": set_model_value, 260 | "force_timeout": self.request_timeout, 261 | "max_tokens": self.max_tokens, 262 | "stream": self.streaming, 263 | "n": self.n, 264 | "temperature": self.temperature, 265 | "custom_llm_provider": self.custom_llm_provider, 266 | **self.model_kwargs, 267 | } 268 | 269 | @property 270 | def _client_params(self) -> Dict[str, Any]: 271 | """Get the parameters used for the openai client.""" 272 | set_model_value = self.model 273 | if self.model_name is not None: 274 | set_model_value = self.model_name 275 | self.client.api_base = self.api_base 276 | self.client.organization = self.organization 277 | creds: Dict[str, Any] = { 278 | "model": set_model_value, 279 | "force_timeout": self.request_timeout, 280 | "api_base": self.api_base, 281 | } 282 | return {**self._default_params, **creds} 283 | 284 | def completion_with_retry( 285 | self, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any 286 | ) -> Any: 287 | """Use tenacity to retry the completion call.""" 288 | retry_decorator = _create_retry_decorator(self, run_manager=run_manager) 289 | 290 | @retry_decorator 291 | def _completion_with_retry(**kwargs: Any) -> Any: 292 | return self.client.completion(**kwargs) 293 | 294 | return _completion_with_retry(**kwargs) 295 | 296 | @pre_init 297 | def validate_environment(cls, values: Dict) -> Dict: 298 | """Validate api key, python package exists, temperature, top_p, and top_k.""" 299 | try: 300 | import litellm 301 | except ImportError: 302 | raise ChatLiteLLMException( 303 | "Could not import litellm python package. " 304 | "Please install it with `pip install litellm`" 305 | ) 306 | 307 | values["openai_api_key"] = get_from_dict_or_env( 308 | values, "openai_api_key", "OPENAI_API_KEY", default="" 309 | ) 310 | values["azure_api_key"] = get_from_dict_or_env( 311 | values, "azure_api_key", "AZURE_API_KEY", default="" 312 | ) 313 | values["anthropic_api_key"] = get_from_dict_or_env( 314 | values, "anthropic_api_key", "ANTHROPIC_API_KEY", default="" 315 | ) 316 | values["replicate_api_key"] = get_from_dict_or_env( 317 | values, "replicate_api_key", "REPLICATE_API_KEY", default="" 318 | ) 319 | values["openrouter_api_key"] = get_from_dict_or_env( 320 | values, "openrouter_api_key", "OPENROUTER_API_KEY", default="" 321 | ) 322 | values["cohere_api_key"] = get_from_dict_or_env( 323 | values, "cohere_api_key", "COHERE_API_KEY", default="" 324 | ) 325 | values["huggingface_api_key"] = get_from_dict_or_env( 326 | values, "huggingface_api_key", "HUGGINGFACE_API_KEY", default="" 327 | ) 328 | values["together_ai_api_key"] = get_from_dict_or_env( 329 | values, "together_ai_api_key", "TOGETHERAI_API_KEY", default="" 330 | ) 331 | values["client"] = litellm 332 | 333 | if values["temperature"] is not None and not 0 <= values["temperature"] <= 1: 334 | raise ValueError("temperature must be in the range [0.0, 1.0]") 335 | 336 | if values["top_p"] is not None and not 0 <= values["top_p"] <= 1: 337 | raise ValueError("top_p must be in the range [0.0, 1.0]") 338 | 339 | if values["top_k"] is not None and values["top_k"] <= 0: 340 | raise ValueError("top_k must be positive") 341 | 342 | return values 343 | 344 | def _generate( 345 | self, 346 | messages: List[BaseMessage], 347 | stop: Optional[List[str]] = None, 348 | run_manager: Optional[CallbackManagerForLLMRun] = None, 349 | stream: Optional[bool] = None, 350 | **kwargs: Any, 351 | ) -> ChatResult: 352 | should_stream = stream if stream is not None else self.streaming 353 | if should_stream: 354 | stream_iter = self._stream( 355 | messages, stop=stop, run_manager=run_manager, **kwargs 356 | ) 357 | return generate_from_stream(stream_iter) 358 | 359 | message_dicts, params = self._create_message_dicts(messages, stop) 360 | params = {**params, **kwargs} 361 | response = self.completion_with_retry( 362 | messages=message_dicts, run_manager=run_manager, **params 363 | ) 364 | return self._create_chat_result(response) 365 | 366 | def _create_chat_result(self, response: Mapping[str, Any]) -> ChatResult: 367 | generations = [] 368 | for res in response["choices"]: 369 | message = _convert_dict_to_message(res["message"]) 370 | gen = ChatGeneration( 371 | message=message, 372 | generation_info=dict(finish_reason=res.get("finish_reason")), 373 | ) 374 | generations.append(gen) 375 | token_usage = response.get("usage", {}) 376 | set_model_value = self.model 377 | if self.model_name is not None: 378 | set_model_value = self.model_name 379 | llm_output = {"token_usage": token_usage, "model": set_model_value} 380 | return ChatResult(generations=generations, llm_output=llm_output) 381 | 382 | def _create_message_dicts( 383 | self, messages: List[BaseMessage], stop: Optional[List[str]] 384 | ) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]: 385 | params = self._client_params 386 | if stop is not None: 387 | if "stop" in params: 388 | raise ValueError("`stop` found in both the input and default params.") 389 | params["stop"] = stop 390 | message_dicts = [_convert_message_to_dict(m) for m in messages] 391 | return message_dicts, params 392 | 393 | def _stream( 394 | self, 395 | messages: List[BaseMessage], 396 | stop: Optional[List[str]] = None, 397 | run_manager: Optional[CallbackManagerForLLMRun] = None, 398 | **kwargs: Any, 399 | ) -> Iterator[ChatGenerationChunk]: 400 | message_dicts, params = self._create_message_dicts(messages, stop) 401 | params = {**params, **kwargs, "stream": True} 402 | 403 | default_chunk_class = AIMessageChunk 404 | for chunk in self.completion_with_retry( 405 | messages=message_dicts, run_manager=run_manager, **params 406 | ): 407 | if not isinstance(chunk, dict): 408 | chunk = chunk.model_dump() 409 | if len(chunk["choices"]) == 0: 410 | continue 411 | delta = chunk["choices"][0]["delta"] 412 | chunk = _convert_delta_to_message_chunk(delta, default_chunk_class) 413 | default_chunk_class = chunk.__class__ 414 | cg_chunk = ChatGenerationChunk(message=chunk) 415 | if run_manager: 416 | run_manager.on_llm_new_token(chunk.content, chunk=cg_chunk) 417 | yield cg_chunk 418 | 419 | async def _astream( 420 | self, 421 | messages: List[BaseMessage], 422 | stop: Optional[List[str]] = None, 423 | run_manager: Optional[AsyncCallbackManagerForLLMRun] = None, 424 | **kwargs: Any, 425 | ) -> AsyncIterator[ChatGenerationChunk]: 426 | message_dicts, params = self._create_message_dicts(messages, stop) 427 | params = {**params, **kwargs, "stream": True} 428 | 429 | default_chunk_class = AIMessageChunk 430 | async for chunk in await acompletion_with_retry( 431 | self, messages=message_dicts, run_manager=run_manager, **params 432 | ): 433 | if not isinstance(chunk, dict): 434 | chunk = chunk.model_dump() 435 | if len(chunk["choices"]) == 0: 436 | continue 437 | delta = chunk["choices"][0]["delta"] 438 | chunk = _convert_delta_to_message_chunk(delta, default_chunk_class) 439 | default_chunk_class = chunk.__class__ 440 | cg_chunk = ChatGenerationChunk(message=chunk) 441 | if run_manager: 442 | await run_manager.on_llm_new_token(chunk.content, chunk=cg_chunk) 443 | yield cg_chunk 444 | 445 | async def _agenerate( 446 | self, 447 | messages: List[BaseMessage], 448 | stop: Optional[List[str]] = None, 449 | run_manager: Optional[AsyncCallbackManagerForLLMRun] = None, 450 | stream: Optional[bool] = None, 451 | **kwargs: Any, 452 | ) -> ChatResult: 453 | should_stream = stream if stream is not None else self.streaming 454 | if should_stream: 455 | stream_iter = self._astream( 456 | messages=messages, stop=stop, run_manager=run_manager, **kwargs 457 | ) 458 | return await agenerate_from_stream(stream_iter) 459 | 460 | message_dicts, params = self._create_message_dicts(messages, stop) 461 | params = {**params, **kwargs} 462 | response = await acompletion_with_retry( 463 | self, messages=message_dicts, run_manager=run_manager, **params 464 | ) 465 | return self._create_chat_result(response) 466 | 467 | def bind_tools( 468 | self, 469 | tools: Sequence[Union[Dict[str, Any], Type[BaseModel], Callable, BaseTool]], 470 | **kwargs: Any, 471 | ) -> Runnable[LanguageModelInput, BaseMessage]: 472 | """Bind tool-like objects to this chat model. 473 | 474 | LiteLLM expects tools argument in OpenAI format. 475 | 476 | Args: 477 | tools: A list of tool definitions to bind to this chat model. 478 | Can be a dictionary, pydantic model, callable, or BaseTool. Pydantic 479 | models, callables, and BaseTools will be automatically converted to 480 | their schema dictionary representation. 481 | tool_choice: Which tool to require the model to call. 482 | Must be the name of the single provided function or 483 | "auto" to automatically determine which function to call 484 | (if any), or a dict of the form: 485 | {"type": "function", "function": {"name": <>}}. 486 | **kwargs: Any additional parameters to pass to the 487 | :class:`~langchain.runnable.Runnable` constructor. 488 | """ 489 | 490 | formatted_tools = [convert_to_openai_tool(tool) for tool in tools] 491 | return super().bind(tools=formatted_tools, **kwargs) 492 | 493 | @property 494 | def _identifying_params(self) -> Dict[str, Any]: 495 | """Get the identifying parameters.""" 496 | set_model_value = self.model 497 | if self.model_name is not None: 498 | set_model_value = self.model_name 499 | return { 500 | "model": set_model_value, 501 | "temperature": self.temperature, 502 | "top_p": self.top_p, 503 | "top_k": self.top_k, 504 | "n": self.n, 505 | } 506 | 507 | @property 508 | def _llm_type(self) -> str: 509 | return "litellm-chat" 510 | --------------------------------------------------------------------------------