├── .gitignore ├── LICENSE ├── README.md ├── README_zh.md ├── assets ├── logo.png ├── meta0423.gif └── overview.png ├── bmtools ├── __init__.py ├── agent │ ├── BabyagiTools.py │ ├── apitool.py │ ├── autogpt │ │ ├── __init__.py │ │ ├── agent.py │ │ ├── prompt.py │ │ └── prompt_generator.py │ ├── autogptmulti │ │ ├── __init__.py │ │ ├── agent.py │ │ ├── prompt.py │ │ └── prompt_generator.py │ ├── executor.py │ ├── singletool.py │ ├── tools_controller.py │ └── translator.py ├── knowledge │ ├── __init__.py │ └── knowledge_extraction.py ├── models │ ├── __init__.py │ ├── cpmbee_model.py │ ├── customllm.py │ ├── llama_model.py │ ├── lora_model.py │ ├── opt_model.py │ ├── readme.txt │ └── t5_model.py ├── tools │ ├── __init__.py │ ├── airbnb │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── arxiv │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── bing_search │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test_bing.py │ ├── chemical │ │ ├── __init__.py │ │ └── prop │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── readme.md │ │ │ └── test_prop.py │ ├── code_interpreter │ │ ├── __init__.py │ │ ├── api.py │ │ └── readme.md │ ├── database │ │ ├── __init__.py │ │ ├── api.py │ │ ├── config.ini │ │ ├── data │ │ │ ├── script.py │ │ │ └── tpch10x │ │ │ │ ├── text2res_multi_table.json │ │ │ │ ├── text2res_origin.json │ │ │ │ └── text2res_single_table.json │ │ ├── readme.md │ │ ├── test.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── data.py │ │ │ ├── database.py │ │ │ ├── db_parser.py │ │ │ ├── db_utils.py │ │ │ └── llm.py │ ├── db_diag │ │ ├── __init__.py │ │ ├── anomaly_detection.py │ │ ├── api.py │ │ ├── config.ini │ │ ├── database_monitoring_metrics │ │ ├── example_generate.py │ │ ├── prometheus.py │ │ ├── readme.md │ │ ├── root_causes_dbmind.jsonl │ │ ├── test.py │ │ └── utils │ │ │ ├── data.py │ │ │ ├── database.py │ │ │ ├── db_parser.py │ │ │ ├── db_utils.py │ │ │ └── llm.py │ ├── file_operation │ │ ├── README.md │ │ ├── __init__.py │ │ ├── api.py │ │ └── test.py │ ├── film │ │ ├── __init__.py │ │ └── douban │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── readme.md │ │ │ ├── test.py │ │ │ └── test_douban.py │ ├── google_places │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── google_scholar │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── google_serper │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── gradio_tools │ │ ├── __init__.py │ │ ├── api.py │ │ ├── florida-drivers-license.jpeg │ │ ├── readme.md │ │ ├── test.py │ │ └── tools │ │ │ ├── __init__.py │ │ │ ├── bark.py │ │ │ ├── clip_interrogator.py │ │ │ ├── document_qa.py │ │ │ ├── gradio_tool.py │ │ │ ├── image_captioning.py │ │ │ ├── image_to_music.py │ │ │ ├── prompt_generator.py │ │ │ ├── sam_with_clip.py │ │ │ ├── stable_diffusion.py │ │ │ ├── text_to_video.py │ │ │ └── whisper.py │ ├── hugging_tools │ │ ├── __init__.py │ │ ├── api.py │ │ ├── files │ │ │ ├── boat.png │ │ │ ├── doc.jpg │ │ │ └── test.flac │ │ ├── readme.md │ │ ├── sources │ │ │ ├── audio_models.txt │ │ │ ├── cv_models.txt │ │ │ ├── docs.json │ │ │ └── nlp_models.txt │ │ └── test.py │ ├── image_generation │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── job_search │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── kg │ │ ├── __init__.py │ │ └── wikidata │ │ │ ├── __init__.py │ │ │ ├── ai-plugin.json │ │ │ ├── api.py │ │ │ ├── examples.json │ │ │ ├── property.csv │ │ │ ├── readme.md │ │ │ ├── test.py │ │ │ └── utils.py │ ├── map │ │ ├── __init__.py │ │ ├── baidu_map │ │ │ ├── baidu_api.py │ │ │ ├── readme.md │ │ │ └── test_baidu.py │ │ └── bing_map │ │ │ ├── api.py │ │ │ ├── readme .md │ │ │ └── test.py │ ├── meta_analysis │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── office │ │ ├── __init__.py │ │ └── ppt │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── readme.md │ │ │ └── templates │ │ │ ├── flat.pptx │ │ │ ├── green.pptx │ │ │ ├── orange.pptx │ │ │ ├── tech.pptx │ │ │ └── wooden.pptx │ ├── python │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── registry.py │ ├── retriever.py │ ├── sceneXplain │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── serve.py │ ├── shell │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── stock │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── tool.py │ ├── translation │ │ ├── __init__.py │ │ ├── baidu │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ └── test.py │ │ ├── nllb │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ └── test.py │ │ └── readme.md │ ├── travel │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── tutorial │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── walmart │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── weather │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── wikipedia │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ ├── wolframalpha │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py │ └── zillow │ │ ├── __init__.py │ │ ├── api.py │ │ ├── readme.md │ │ └── test.py └── utils │ ├── __init__.py │ └── logging.py ├── host_local_tools.py ├── multi_test.py ├── requirements.txt ├── secret_keys.sh ├── setup.py ├── test.py ├── test_multi.py └── web_demo.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled python files 2 | *.pyc 3 | 4 | # Compiled C extensions 5 | *.so 6 | 7 | # Distribution / packaging 8 | dist/ 9 | build/ 10 | *.egg-info/ 11 | 12 | # IDE / editor files 13 | .idea/ 14 | *.swp 15 | *~ 16 | 17 | # Virtual environment 18 | venv/ 19 | env/ 20 | 21 | __pycache__/ 22 | .vscode/ 23 | .DS_Store 24 | 25 | cache/ 26 | 27 | secret_keys_mine.sh 28 | my_secret_keys.sh 29 | 30 | data/ 31 | -------------------------------------------------------------------------------- /assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/assets/logo.png -------------------------------------------------------------------------------- /assets/meta0423.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/assets/meta0423.gif -------------------------------------------------------------------------------- /assets/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/assets/overview.png -------------------------------------------------------------------------------- /bmtools/__init__.py: -------------------------------------------------------------------------------- 1 | from .tools.serve import ToolServer 2 | from .utils.logging import get_logger 3 | -------------------------------------------------------------------------------- /bmtools/agent/autogpt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/agent/autogpt/__init__.py -------------------------------------------------------------------------------- /bmtools/agent/autogpt/prompt.py: -------------------------------------------------------------------------------- 1 | import time 2 | from typing import Any, Callable, List 3 | 4 | from pydantic import BaseModel 5 | 6 | from .prompt_generator import get_prompt 7 | from langchain.prompts.chat import ( 8 | BaseChatPromptTemplate, 9 | ) 10 | from langchain.schema import BaseMessage, HumanMessage, SystemMessage 11 | from langchain.tools.base import BaseTool 12 | from langchain.vectorstores.base import VectorStoreRetriever 13 | 14 | 15 | class AutoGPTPrompt(BaseChatPromptTemplate, BaseModel): 16 | ai_name: str 17 | ai_role: str 18 | tools: List[BaseTool] 19 | token_counter: Callable[[str], int] 20 | send_token_limit: int = 4196 21 | 22 | def construct_full_prompt(self, goals: List[str]) -> str: 23 | prompt_start = """Your decisions must always be made independently 24 | without seeking user assistance. Play to your strengths 25 | as an LLM and pursue simple strategies with no legal complications. 26 | If you have completed all your tasks, 27 | make sure to use the "finish" command.""" 28 | 29 | # Construct full prompt 30 | full_prompt = ( 31 | f"You are {self.ai_name}, {self.ai_role}\n{prompt_start}\n\nGOALS:\n\n" 32 | ) 33 | for i, goal in enumerate(goals): 34 | full_prompt += f"{i+1}. {goal}\n" 35 | 36 | full_prompt += f"\n\n{get_prompt(self.tools)}" 37 | return full_prompt 38 | 39 | def format_messages(self, **kwargs: Any) -> List[BaseMessage]: 40 | base_prompt = SystemMessage(content=self.construct_full_prompt(kwargs["goals"])) 41 | time_prompt = SystemMessage( 42 | content=f"The current time and date is {time.strftime('%c')}" 43 | ) 44 | used_tokens = self.token_counter(base_prompt.content) + self.token_counter( 45 | time_prompt.content 46 | ) 47 | memory: VectorStoreRetriever = kwargs["memory"] 48 | previous_messages = kwargs["messages"] 49 | relevant_docs = memory.get_relevant_documents(str(previous_messages[-10:])) 50 | relevant_memory = [d.page_content for d in relevant_docs] 51 | relevant_memory_tokens = sum( 52 | [self.token_counter(doc) for doc in relevant_memory] 53 | ) 54 | while used_tokens + relevant_memory_tokens > 2500: 55 | relevant_memory = relevant_memory[:-1] 56 | relevant_memory_tokens = sum( 57 | [self.token_counter(doc) for doc in relevant_memory] 58 | ) 59 | content_format = ( 60 | f"This reminds you of these events " 61 | f"from your past:\n{relevant_memory}\n\n" 62 | ) 63 | memory_message = SystemMessage(content=content_format) 64 | used_tokens += len(memory_message.content) 65 | historical_messages: List[BaseMessage] = [] 66 | for message in previous_messages[-10:][::-1]: 67 | message_tokens = self.token_counter(message.content) 68 | if used_tokens + message_tokens > self.send_token_limit - 1000: 69 | break 70 | historical_messages = [message] + historical_messages 71 | input_message = HumanMessage(content=kwargs["user_input"]) 72 | messages: List[BaseMessage] = [base_prompt, time_prompt, memory_message] 73 | messages += historical_messages 74 | messages.append(input_message) 75 | return messages 76 | -------------------------------------------------------------------------------- /bmtools/agent/autogptmulti/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/agent/autogptmulti/__init__.py -------------------------------------------------------------------------------- /bmtools/agent/autogptmulti/prompt.py: -------------------------------------------------------------------------------- 1 | import time 2 | from typing import Any, Callable, List 3 | 4 | from pydantic import BaseModel 5 | 6 | from .prompt_generator import get_prompt 7 | from langchain.prompts.chat import ( 8 | BaseChatPromptTemplate, 9 | ) 10 | from langchain.schema import BaseMessage, HumanMessage, SystemMessage 11 | from langchain.tools.base import BaseTool 12 | from langchain.vectorstores.base import VectorStoreRetriever 13 | 14 | 15 | class AutoGPTPrompt(BaseChatPromptTemplate, BaseModel): 16 | ai_name: str 17 | ai_role: str 18 | tools: List[BaseTool] 19 | token_counter: Callable[[str], int] 20 | send_token_limit: int = 4196 21 | 22 | def construct_full_prompt(self, goals: List[str]) -> str: 23 | prompt_start = """Your decisions must always be made independently 24 | without seeking user assistance. Play to your strengths 25 | as an LLM and pursue simple strategies with no legal complications. 26 | Once you have completed your goal or have found that it can not be finished with current commands, 27 | make sure to use the "finish" command immediately.""" 28 | 29 | # Construct full prompt 30 | full_prompt = ( 31 | f"You are {self.ai_name}, {self.ai_role}\n{prompt_start}\n\nGOALS:\n\n" 32 | ) 33 | if isinstance(goals, list): 34 | for i, goal in enumerate(goals): 35 | full_prompt += f"{i+1}. {goal}\n" 36 | else: 37 | full_prompt += f"{goals}\n" 38 | full_prompt += f"\n\n{get_prompt(self.tools)}" 39 | return full_prompt 40 | 41 | def format_messages(self, **kwargs: Any) -> List[BaseMessage]: 42 | base_prompt = SystemMessage(content=self.construct_full_prompt(kwargs["goals"])) 43 | time_prompt = SystemMessage( 44 | content=f"The current time and date is {time.strftime('%c')}" 45 | ) 46 | used_tokens = self.token_counter(base_prompt.content) + self.token_counter( 47 | time_prompt.content 48 | ) 49 | memory: VectorStoreRetriever = kwargs["memory"] 50 | previous_messages = kwargs["messages"] 51 | 52 | content_format = ( 53 | f"This reminds you of these events " 54 | f"you have already used, and NEVER conduct repeated or unrelated commands:\n" 55 | ) 56 | memory_message = SystemMessage(content=content_format) 57 | used_tokens += len(memory_message.content) 58 | historical_messages: List[BaseMessage] = [] 59 | for message in previous_messages[-10:][::-1]: 60 | message_tokens = self.token_counter(message.content) 61 | if used_tokens + message_tokens > self.send_token_limit - 1000: 62 | break 63 | historical_messages = [message] + historical_messages 64 | input_message = HumanMessage(content=kwargs["user_input"]) 65 | messages: List[BaseMessage] = [base_prompt, time_prompt, memory_message] 66 | messages += historical_messages 67 | messages.append(input_message) 68 | return messages 69 | -------------------------------------------------------------------------------- /bmtools/knowledge/__init__.py: -------------------------------------------------------------------------------- 1 | from .knowledge_extraction import KnowledgeExtraction -------------------------------------------------------------------------------- /bmtools/knowledge/knowledge_extraction.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import requests 4 | import numpy as np 5 | import paramiko 6 | 7 | from nltk.stem import WordNetLemmatizer 8 | from nltk.corpus import wordnet, stopwords 9 | from nltk.tokenize import word_tokenize 10 | import nltk 11 | 12 | from bmtools.tools.database.utils.db_parser import get_conf 13 | from bmtools.tools.database.utils.database import DBArgs, Database 14 | from bmtools.models.customllm import CustomLLM 15 | from bmtools.tools.db_diag.anomaly_detection import detect_anomalies 16 | from bmtools.tools.db_diag.anomaly_detection import prometheus 17 | 18 | from bmtools.tools.db_diag.example_generate import bm25 19 | 20 | # match with external knowledge for in-context learning 21 | 22 | class KnowledgeExtraction(): 23 | 24 | def __init__(self, file_path, topk=3, keyword_matching_func=bm25): 25 | 26 | # select an attribute in the jsons to embed 27 | self.names = {"matched_attr": "cause_name"} 28 | self.cause_name = self.names["matched_attr"] 29 | 30 | nltk.download('stopwords') 31 | nltk.download('punkt') 32 | nltk.download('averaged_perceptron_tagger') 33 | nltk.download('wordnet') 34 | self.wnl = WordNetLemmatizer() 35 | self.keyword_matching_func = keyword_matching_func 36 | 37 | self.topk = topk 38 | 39 | self.corpus, self.preprocessed_corpus, self.matched_attr, self.stop_words = self.knowledge_load(file_path) 40 | 41 | def knowledge_load(self, file_path): 42 | 43 | # file_path = "/bmtools/tools/db_diag/root_causes_dbmind.jsonl" 44 | with open(str(os.getcwd()) + file_path, 'r') as f: 45 | data = json.load(f) 46 | self.corpus = [example["desc"] for example in data] 47 | self.matched_attr = [example[self.names["matched_attr"]] for example in data] 48 | self.stop_words = set(stopwords.words('english')) 49 | 50 | self.preprocessed_corpus = [] 51 | for c in self.corpus: 52 | word_tokens = word_tokenize(c) 53 | self.preprocessed_corpus.append([self.wnl.lemmatize(w,pos='n') for w in word_tokens if not w in self.stop_words]) # remove useless words and standardize words 54 | 55 | return self.corpus, self.preprocessed_corpus, self.matched_attr, self.stop_words 56 | 57 | def match(self, detailed_metrics): 58 | 59 | metrics_str = [] 60 | for metrics in detailed_metrics.keys(): 61 | metrics = metrics.replace("_"," ") 62 | word_tokens = word_tokenize(metrics) 63 | metrics_str.extend([self.wnl.lemmatize(w,pos='n') for w in word_tokens if not w in self.stop_words]) 64 | metrics_str = list(set(metrics_str)) 65 | 66 | best_index = self.keyword_matching_func(self.topk, metrics_str, self.preprocessed_corpus) 67 | best_docs = [self.corpus[b] for b in best_index] 68 | best_names = [self.matched_attr[b] for b in best_index] 69 | docs_str = "" 70 | print("Best docs: ", best_docs) 71 | for i, docs in enumerate(best_docs): 72 | docs_str = docs_str + "{}: ".format(best_names[i]) + docs + "\n\n" 73 | print("docs_str: ", docs_str) 74 | 75 | return docs_str 76 | 77 | 78 | if __name__ == "__main__": 79 | matcher = KnowledgeExtraction("/root_causes_dbmind.jsonl") 80 | print(matcher.match({"memory_resource_contention":123, "node_scrape_collector_duration_seconds": 1293})) 81 | -------------------------------------------------------------------------------- /bmtools/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .customllm import CustomLLM -------------------------------------------------------------------------------- /bmtools/models/cpmbee_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from langchain.llms.base import LLM 4 | from typing import Optional, List, Mapping, Any 5 | import torch 6 | from cpm_live.generation.bee import CPMBeeBeamSearch 7 | from cpm_live.models import CPMBeeTorch, CPMBeeConfig 8 | from cpm_live.tokenizers import CPMBeeTokenizer 9 | 10 | 11 | class CpmBeeLLM(LLM): 12 | 13 | model_name : str = "" 14 | config: CPMBeeConfig = None 15 | tokenizer: CPMBeeTokenizer = None 16 | model: CPMBeeTorch = None 17 | 18 | def __init__(self, config_path: str, ckpt_path: str, device: str="cuda") -> None: 19 | super().__init__() 20 | self.model_name = ckpt_path 21 | self.config = CPMBeeConfig.from_json_file(config_path) 22 | self.tokenizer = CPMBeeTokenizer() 23 | self.model = CPMBeeTorch(config=self.config) 24 | 25 | self.model.load_state_dict(torch.load(ckpt_path)) 26 | if device == "cuda": 27 | self.model.cuda() 28 | 29 | @property 30 | def _llm_type(self) -> str: 31 | return self.model_name 32 | 33 | def _call(self, prompt, stop: Optional[List[str]] = None) -> str: 34 | # use beam search 35 | beam_search = CPMBeeBeamSearch( 36 | model=self.model, 37 | tokenizer=self.tokenizer, 38 | ) 39 | inference_results = beam_search.generate([{"source":prompt, "":""}], max_length=512, repetition_penalty=1.2, beam_size=1) 40 | output = inference_results[0][""] 41 | return output 42 | 43 | @property 44 | def _identifying_params(self) -> Mapping[str, Any]: 45 | """Get the identifying parameters.""" 46 | return {"model_name": self.model_name} 47 | 48 | if __name__ == "__main__": 49 | llm = CpmBeeLLM(config_path="path/to/cpm-bee/config.json", ckpt_path="path/to/cpm-bee/checkpoint/") 50 | print(llm("You are an task creation AI that uses the result of an execution agent to create new tasks with the following objective: What's the weather in Shanghai today? Should I bring an umbrella?, The last completed task has the result: According to the weather report, it is sunny in Shanghai today and there is no precipitation, so you do not need to bring an umbrella.. This result was based on this task description: Make a todo list about this objective: What's the weather in Shanghai today? Should I bring an umbrella?. These are incomplete tasks: . Based on the result, create new tasks to be completed by the AI system that do not overlap with incomplete tasks. Do not generate repetitive tasks (e.g., tasks that have already been completed). If there is not futher task needed to complete the objective, only return NO TASK. Now return the tasks as an array.")) 51 | 52 | -------------------------------------------------------------------------------- /bmtools/models/customllm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from langchain.llms.base import LLM 4 | from typing import Optional, List, Mapping, Any 5 | 6 | class CustomLLM(LLM): 7 | 8 | n: int = 0 9 | 10 | @property 11 | def _llm_type(self) -> str: 12 | return "custom" 13 | 14 | def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str: 15 | output = "your customized response" 16 | 17 | return output 18 | 19 | @property 20 | def _identifying_params(self) -> Mapping[str, Any]: 21 | """Get the identifying parameters.""" 22 | return {"n": self.n} 23 | 24 | if __name__ == "__main__": 25 | llm = CustomLLM() 26 | print(llm("You are an task creation AI that uses the result of an execution agent to create new tasks with the following objective: What's the weather in Shanghai today? Should I bring an umbrella?, The last completed task has the result: According to the weather report, it is sunny in Shanghai today and there is no precipitation, so you do not need to bring an umbrella.. This result was based on this task description: Make a todo list about this objective: What's the weather in Shanghai today? Should I bring an umbrella?. These are incomplete tasks: . Based on the result, create new tasks to be completed by the AI system that do not overlap with incomplete tasks. Do not generate repetitive tasks (e.g., tasks that have already been completed). If there is not futher task needed to complete the objective, only return NO TASK. Now return the tasks as an array.")) -------------------------------------------------------------------------------- /bmtools/models/llama_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from langchain.llms.base import LLM 4 | from typing import Optional, List, Mapping, Any 5 | from transformers import AutoTokenizer, AutoModelForCausalLM 6 | 7 | class LlamaModel(LLM): 8 | 9 | model_name: str = "" 10 | tokenizer: AutoTokenizer = None 11 | model: AutoModelForCausalLM = None 12 | use_gpu: bool = True 13 | 14 | def __init__(self, model_name_or_path: str, device: str="cuda", cpu_offloading: bool=False) -> None: 15 | super().__init__() 16 | self.model_name = model_name_or_path 17 | self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=False) 18 | self.model = AutoModelForCausalLM.from_pretrained( 19 | model_name_or_path, low_cpu_mem_usage=True 20 | ) 21 | if self.tokenizer.pad_token_id == None: 22 | self.tokenizer.add_special_tokens({"bos_token": "", "eos_token": "", "pad_token": ""}) 23 | self.model.resize_token_embeddings(len(self.tokenizer)) 24 | self.use_gpu = (True if device == "cuda" else False) 25 | if (device == "cuda" and not cpu_offloading) or device == "mps": 26 | self.model.to(device) 27 | 28 | @property 29 | def _llm_type(self) -> str: 30 | return self.model_name 31 | 32 | def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str: 33 | inputs = self.tokenizer( 34 | prompt, 35 | padding=True, 36 | max_length=self.tokenizer.model_max_length, 37 | truncation=True, 38 | return_tensors="pt" 39 | ) 40 | inputs_len = inputs["input_ids"].shape[1] 41 | generated_outputs = self.model.generate( 42 | input_ids=(inputs["input_ids"].cuda() if self.use_gpu else inputs["input_ids"]), 43 | attention_mask=(inputs["attention_mask"].cuda() if self.use_gpu else inputs["attention_mask"]), 44 | max_new_tokens=512, 45 | eos_token_id=self.tokenizer.eos_token_id, 46 | bos_token_id=self.tokenizer.bos_token_id, 47 | pad_token_id=self.tokenizer.pad_token_id, 48 | ) 49 | decoded_output = self.tokenizer.batch_decode( 50 | generated_outputs[..., inputs_len:], skip_special_tokens=True) 51 | output = decoded_output[0] 52 | return output 53 | 54 | @property 55 | def _identifying_params(self) -> Mapping[str, Any]: 56 | """Get the identifying parameters.""" 57 | return {"model_name": self.model_name} 58 | 59 | if __name__ == "__main__": 60 | # can accept all huggingface LlamaModel family 61 | llm = LlamaModel("decapoda-research/llama-7b-hf") 62 | print(llm("You are an task creation AI that uses the result of an execution agent to create new tasks with the following objective: What's the weather in Shanghai today? Should I bring an umbrella?, The last completed task has the result: According to the weather report, it is sunny in Shanghai today and there is no precipitation, so you do not need to bring an umbrella.. This result was based on this task description: Make a todo list about this objective: What's the weather in Shanghai today? Should I bring an umbrella?. These are incomplete tasks: . Based on the result, create new tasks to be completed by the AI system that do not overlap with incomplete tasks. Do not generate repetitive tasks (e.g., tasks that have already been completed). If there is not futher task needed to complete the objective, only return NO TASK. Now return the tasks as an array.")) -------------------------------------------------------------------------------- /bmtools/models/lora_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from langchain.llms.base import LLM 4 | from typing import Optional, List, Mapping, Any 5 | from transformers import AutoTokenizer, AutoModelForCausalLM 6 | from peft import PeftModel 7 | 8 | class LoraModel(LLM): 9 | 10 | model_name: str = "" 11 | tokenizer: AutoTokenizer = None 12 | model: PeftModel = None 13 | use_gpu: bool = True 14 | 15 | def __init__(self, base_name_or_path: str, model_name_or_path: str, device: str="cuda", cpu_offloading: bool=False, load_8bit: bool=False) -> None: 16 | super().__init__() 17 | self.model_name = model_name_or_path 18 | self.tokenizer = AutoTokenizer.from_pretrained(base_name_or_path, use_fast=False) 19 | model = AutoModelForCausalLM.from_pretrained( 20 | base_name_or_path, 21 | load_in_8bit=load_8bit, 22 | device_map="auto" 23 | ) 24 | self.model = PeftModel.from_pretrained( 25 | model, 26 | model_name_or_path 27 | ) 28 | if self.tokenizer.pad_token_id == None: 29 | self.tokenizer.add_special_tokens({"bos_token": "", "eos_token": "", "pad_token": ""}) 30 | self.model.resize_token_embeddings(len(self.tokenizer)) 31 | self.use_gpu = (True if device == "cuda" else False) 32 | if (device == "cuda" and not cpu_offloading) or device == "mps": 33 | self.model.to(device) 34 | 35 | @property 36 | def _llm_type(self) -> str: 37 | return self.model_name 38 | 39 | def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str: 40 | inputs = self.tokenizer( 41 | prompt, 42 | padding=True, 43 | max_length=self.tokenizer.model_max_length, 44 | truncation=True, 45 | return_tensors="pt" 46 | ) 47 | inputs_len = inputs["input_ids"].shape[1] 48 | generated_outputs = self.model.generate( 49 | input_ids=(inputs["input_ids"].cuda() if self.use_gpu else inputs["input_ids"]), 50 | attention_mask=(inputs["attention_mask"].cuda() if self.use_gpu else inputs["attention_mask"]), 51 | max_new_tokens=512, 52 | eos_token_id=self.tokenizer.eos_token_id, 53 | bos_token_id=self.tokenizer.bos_token_id, 54 | pad_token_id=self.tokenizer.pad_token_id, 55 | ) 56 | decoded_output = self.tokenizer.batch_decode( 57 | generated_outputs[..., inputs_len:], skip_special_tokens=True) 58 | output = decoded_output[0] 59 | return output 60 | 61 | @property 62 | def _identifying_params(self) -> Mapping[str, Any]: 63 | """Get the identifying parameters.""" 64 | return {"model_name": self.model_name} 65 | 66 | if __name__ == "__main__": 67 | llm = LoraModel(base_name_or_path="huggyllama/llama-7b", model_name_or_path="pooruss-lsh/tool-llama7b-single-tool-lora") 68 | print(llm("You are an task creation AI that uses the result of an execution agent to create new tasks with the following objective: What's the weather in Shanghai today? Should I bring an umbrella?, The last completed task has the result: According to the weather report, it is sunny in Shanghai today and there is no precipitation, so you do not need to bring an umbrella.. This result was based on this task description: Make a todo list about this objective: What's the weather in Shanghai today? Should I bring an umbrella?. These are incomplete tasks: . Based on the result, create new tasks to be completed by the AI system that do not overlap with incomplete tasks. Do not generate repetitive tasks (e.g., tasks that have already been completed). If there is not futher task needed to complete the objective, only return NO TASK. Now return the tasks as an array.")) -------------------------------------------------------------------------------- /bmtools/models/opt_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from langchain.llms.base import LLM 4 | from typing import Optional, List, Mapping, Any 5 | from transformers import AutoTokenizer, OPTForCausalLM 6 | 7 | 8 | class OPTModel(LLM): 9 | model_name: str = "" 10 | tokenizer: AutoTokenizer = None 11 | model: OPTForCausalLM = None 12 | 13 | def __init__(self, huggingface_model_name: str) -> None: 14 | super().__init__() 15 | self.model_name = huggingface_model_name 16 | self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) 17 | self.model = OPTForCausalLM.from_pretrained(self.model_name) 18 | 19 | @property 20 | def _llm_type(self) -> str: 21 | return self.model_name 22 | 23 | def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str: 24 | 25 | inputs = self.tokenizer( 26 | prompt, 27 | padding=True, 28 | max_length=512, # 512 by default，tokenizer.model_max_length=1000000000000000019884624838656 29 | truncation=True, 30 | return_tensors="pt" 31 | ) 32 | 33 | inputs_len = inputs["input_ids"].shape[1] 34 | 35 | generated_outputs = self.model.generate( 36 | inputs['input_ids'], 37 | max_new_tokens=512, 38 | ) 39 | decoded_output = self.tokenizer.batch_decode( 40 | generated_outputs[..., inputs_len:], skip_special_tokens=True, clean_up_tokenization_spaces=False) 41 | 42 | output = decoded_output[0] 43 | return output 44 | 45 | @property 46 | def _identifying_params(self) -> Mapping[str, Any]: 47 | """Get the identifying parameters.""" 48 | return {"model_name": self.model_name} 49 | 50 | if __name__ == "__main__": 51 | llm = OPTModel("facebook/opt-350m") 52 | # print(llm("Hey, are you consciours? Can you talk to me?")) 53 | print(llm("You are an task creation AI that uses the result of an execution agent to create new tasks with the following objective: What's the weather in Shanghai today? Should I bring an umbrella?, The last completed task has the result: According to the weather report, it is sunny in Shanghai today and there is no precipitation, so you do not need to bring an umbrella.. This result was based on this task description: Make a todo list about this objective: What's the weather in Shanghai today? Should I bring an umbrella?. These are incomplete tasks: . Based on the result, create new tasks to be completed by the AI system that do not overlap with incomplete tasks. Do not generate repetitive tasks (e.g., tasks that have already been completed). If there is not futher task needed to complete the objective, only return NO TASK. Now return the tasks as an array.")) 54 | 55 | -------------------------------------------------------------------------------- /bmtools/models/readme.txt: -------------------------------------------------------------------------------- 1 | customllm.py：通用模板文件 2 | llama_model.py：LlaMA model系列处理文件 3 | opt_model.py：OPT model系列处理文件 4 | t5_model.py：T5 model系列处理文件 -------------------------------------------------------------------------------- /bmtools/models/t5_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from langchain.llms.base import LLM 4 | from typing import Optional, List, Mapping, Any 5 | from transformers import T5Tokenizer, T5ForConditionalGeneration 6 | 7 | 8 | class T5Model(LLM): 9 | model_name: str = "" 10 | tokenizer: T5Tokenizer = None 11 | model: T5ForConditionalGeneration = None 12 | 13 | def __init__(self, huggingface_model_name: str) -> None: 14 | super().__init__() 15 | self.model_name = huggingface_model_name 16 | self.tokenizer = T5Tokenizer.from_pretrained(self.model_name) 17 | self.model = T5ForConditionalGeneration.from_pretrained(self.model_name) 18 | 19 | @property 20 | def _llm_type(self) -> str: 21 | return self.model_name 22 | 23 | def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str: 24 | 25 | inputs = self.tokenizer( 26 | prompt, 27 | padding=True, 28 | max_length=self.tokenizer.model_max_length, 29 | truncation=True, 30 | return_tensors="pt" 31 | ) 32 | 33 | # inputs_len = inputs["input_ids"].shape[1] 34 | 35 | generated_outputs = self.model.generate( 36 | inputs["input_ids"], 37 | max_new_tokens=512, 38 | ) 39 | decoded_output = self.tokenizer.batch_decode( 40 | generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False) 41 | 42 | output = decoded_output[0] 43 | return output 44 | 45 | 46 | @property 47 | def _identifying_params(self) -> Mapping[str, Any]: 48 | """Get the identifying parameters.""" 49 | return {"model_name": self.model_name} 50 | 51 | if __name__ == "__main__": 52 | llm = T5Model("t5-small") 53 | # print(llm("translate English to German: The house is wonderful.")) 54 | print(llm("You are an task creation AI that uses the result of an execution agent to create new tasks with the following objective: What's the weather in Shanghai today? Should I bring an umbrella?, The last completed task has the result: According to the weather report, it is sunny in Shanghai today and there is no precipitation, so you do not need to bring an umbrella.. This result was based on this task description: Make a todo list about this objective: What's the weather in Shanghai today? Should I bring an umbrella?. These are incomplete tasks: . Based on the result, create new tasks to be completed by the AI system that do not overlap with incomplete tasks. Do not generate repetitive tasks (e.g., tasks that have already been completed). If there is not futher task needed to complete the objective, only return NO TASK. Now return the tasks as an array.")) 55 | -------------------------------------------------------------------------------- /bmtools/tools/__init__.py: -------------------------------------------------------------------------------- 1 | from . import chemical 2 | from . import film 3 | from . import kg 4 | from . import stock 5 | from . import weather 6 | from . import wikipedia 7 | from . import wolframalpha 8 | from . import office 9 | from . import bing_search 10 | from . import map 11 | from . import translation 12 | from . import tutorial 13 | from . import file_operation 14 | from . import meta_analysis 15 | from . import database 16 | from . import db_diag 17 | from . import code_interpreter 18 | from . import hugging_tools 19 | from . import arxiv 20 | from . import zillow 21 | from . import google_scholar 22 | from . import google_places 23 | from . import google_serper 24 | from . import python 25 | from . import sceneXplain 26 | from . import shell 27 | from . import image_generation 28 | from . import airbnb 29 | from . import job_search 30 | from . import gradio_tools 31 | from . import travel 32 | from . import walmart 33 | 34 | 35 | from .tool import Tool 36 | from .registry import register 37 | from .serve import ToolServer 38 | -------------------------------------------------------------------------------- /bmtools/tools/airbnb/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("airbnb") 5 | def airbnb(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/airbnb/readme.md: -------------------------------------------------------------------------------- 1 | # Airbnb Service 2 | 3 | Contributor: [Kunlun Zhu](https://github.com/Kunlun-Zhu) 4 | 5 | You can get your RAIPID key here: https://rapidapi.com/hub 6 | 7 | You ought to subscribe 'Airbnb API' in your account to use this tool 8 | 9 | # Short-term Rental and Housing Information Tool 10 | 11 | This tool, named `Short-term Rental and Housing Information`, is designed to interact with the Airbnb API to search for properties, get property details, check availability, get property reviews, and retrieve the checkout price. The tool operates by making HTTP requests to the Airbnb API and formatting the responses into an easily usable form. 12 | 13 | ## Main Functionality 14 | 15 | 1. **Search for Properties**: This functionality allows you to search for properties based on a variety of parameters like the number of adults, children, and infants, property type, amenities, check-in and check-out dates, and many more. This is done using the `search_property` function. 16 | 17 | 2. **Search Property by Coordinates**: This function allows you to search for properties in a specific geographic area defined by the northeast and southwest coordinates of the area. This is done using the `search_property_by_coordinates` function. 18 | 19 | 3. **Search for Destination**: The `search_destination` function helps to perform a destination search given a query and optionally a country. It returns positions 'ID' information. 20 | 21 | 4. **Get Property Details**: The `get_property_details` function is used to retrieve detailed information about a specific property. This includes the number of rooms, amenities, location, and other relevant information. 22 | 23 | 5. **Check Property Availability**: This function, `check_availability`, allows you to check if a property is available for booking. 24 | 25 | 6. **Get Property Reviews**: You can use the `get_property_reviews` function to retrieve reviews of a property. 26 | 27 | 7. **Get Property Checkout Price**: The `get_property_checkout_price` function is used to get the checkout cost of a property given its ID and check-in date. 28 | 29 | This tool provides a simple and effective way to interact with the Airbnb API, making it easier for developers to incorporate Airbnb data into their applications. -------------------------------------------------------------------------------- /bmtools/tools/airbnb/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'Airbnb', "http://127.0.0.1:8079/tools/airbnb/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("List some houses to rent in Santa Monica, CA.") -------------------------------------------------------------------------------- /bmtools/tools/arxiv/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("arxiv") 5 | def arxiv(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/arxiv/api.py: -------------------------------------------------------------------------------- 1 | from ..tool import Tool 2 | from typing import Any 3 | import arxiv 4 | 5 | 6 | def build_tool(config) -> Tool: 7 | tool = Tool( 8 | "Arxiv", 9 | "Look up for information from scientific articles on arxiv.org", 10 | name_for_model="Arxiv", 11 | description_for_model=( 12 | "Search information from Arxiv.org " 13 | "Useful for when you need to answer questions about Physics, Mathematics, " 14 | "Computer Science, Quantitative Biology, Quantitative Finance, Statistics, " 15 | "Electrical Engineering, and Economics " 16 | "from scientific articles on arxiv.org. " 17 | "Input should be a search query." 18 | ), 19 | logo_url="https://your-app-url.com/.well-known/logo.png", 20 | contact_email="hello@contact.com", 21 | legal_info_url="hello@legal.com" 22 | ) 23 | 24 | arxiv_exceptions: Any # :meta private: 25 | top_k_results: int = 3 26 | ARXIV_MAX_QUERY_LENGTH = 300 27 | doc_content_chars_max: int = 4000 28 | 29 | @tool.get("/get_arxiv_article_information") 30 | def get_arxiv_article_information(query : str): 31 | '''Run Arxiv search and get the article meta information. 32 | ''' 33 | param = { 34 | "q": query 35 | } 36 | try: 37 | results = arxiv.Search( # type: ignore 38 | query[: ARXIV_MAX_QUERY_LENGTH], max_results = top_k_results 39 | ).results() 40 | except arxiv_exceptions as ex: 41 | return f"Arxiv exception: {ex}" 42 | docs = [ 43 | f"Published: {result.updated.date()}\nTitle: {result.title}\n" 44 | f"Authors: {', '.join(a.name for a in result.authors)}\n" 45 | f"Summary: {result.summary}" 46 | for result in results 47 | ] 48 | if docs: 49 | return "\n\n".join(docs)[: doc_content_chars_max] 50 | else: 51 | return "No good Arxiv Result was found" 52 | 53 | return tool 54 | -------------------------------------------------------------------------------- /bmtools/tools/arxiv/readme.md: -------------------------------------------------------------------------------- 1 | # Arxiv Queries 2 | 3 | Contributor: [Sihan Zhao](https://github.com/Sarah816) 4 | 5 | ## Tool Description 6 | This Python-based tool offers a streamlined way to look up scientific articles on Arxiv.org. Named "Arxiv", this tool is particularly helpful when you need to answer questions about Physics, Mathematics, Computer Science, Quantitative Biology, Quantitative Finance, Statistics, Electrical Engineering, and Economics based on scientific articles from Arxiv.org. 7 | 8 | ### Tool Specifications 9 | 10 | - **Name**: Arxiv 11 | - **Purpose**: Look up for information from scientific articles on arxiv.org 12 | - **Logo**: ![Arxiv Logo](https://your-app-url.com/.well-known/logo.png) 13 | - **Contact Email**: hello@contact.com 14 | - **Legal Information**: [Legal Information](hello@legal.com) 15 | 16 | ### Core Functionality 17 | 18 | 1. `get_arxiv_article_information` 19 | 20 | This method takes a search query and returns meta-information about the Arxiv articles that match this query. The method uses an API to search articles on Arxiv.org and returns details like the date of publication, title of the article, names of the authors, and the summary of the article. 21 | 22 | The method follows these steps: 23 | 24 | - It takes a query as a string input. 25 | - The query is passed to the Arxiv Search API. 26 | - The method fetches the top three results. 27 | - For each result, it collects information about the publication date, title, authors, and summary. 28 | - It returns this information as a string. 29 | 30 | If the search operation encounters an error, the method returns a message describing the Arxiv exception. If no suitable articles are found on Arxiv.org that match the query, it returns a message stating that no good Arxiv result was found. 31 | 32 | ### Constants 33 | 34 | - **ARXIV_MAX_QUERY_LENGTH**: Maximum length of a query that can be passed to the Arxiv Search API. It's set to 300. 35 | - **doc_content_chars_max**: Maximum characters of the Arxiv results to be returned. It's set to 4000. 36 | - **top_k_results**: The maximum number of Arxiv Search results to be returned. It's set to 3. 37 | 38 | Please note that the parameters can be optional and have their own default values. You should consult the method's documentation to understand the default behavior and the specific role of each parameter. -------------------------------------------------------------------------------- /bmtools/tools/arxiv/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'arxiv', "http://127.0.0.1:8079/tools/arxiv/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("List some papers written by Timo Schick") -------------------------------------------------------------------------------- /bmtools/tools/bing_search/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("bing_search") 4 | def bing_search(): 5 | from .api import build_tool 6 | return build_tool -------------------------------------------------------------------------------- /bmtools/tools/bing_search/readme.md: -------------------------------------------------------------------------------- 1 | # Bing search tool 2 | 3 | Contributor [ChengQian](https://github.com/qiancheng0) -------------------------------------------------------------------------------- /bmtools/tools/bing_search/test_bing.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from .api import build_tool, BingAPI 3 | from typing import Tuple 4 | 5 | BING_TEST_SEARCH = { 6 | "webPages": { 7 | "value": [ 8 | { 9 | "url": "a", 10 | "name": "test a", 11 | "snippet": "page a" 12 | }, 13 | { 14 | "url": "b", 15 | "name": "test b", 16 | "snippet": "page b" 17 | }, 18 | { 19 | "url": "c", 20 | "name": "test c", 21 | "snippet": "page c" 22 | } 23 | ] 24 | } 25 | } 26 | 27 | class MockBingAPI(BingAPI): 28 | def __init__(self): 29 | pass 30 | 31 | def search(self, key_words : str, max_retry : int = 3): 32 | return BING_TEST_SEARCH 33 | 34 | def load_page(self, url : str, max_retry : int = 3) -> Tuple[bool, str]: 35 | if url == "a": 36 | return True, "This is page a" 37 | elif url == "b": 38 | return True, "This is page b" 39 | elif url == "c": 40 | return True, "This is page c" 41 | else: 42 | return False, "Timeout for loading this page, Please try to load another one or search again." 43 | 44 | 45 | app = build_tool({"debug": True, "bing_api": MockBingAPI()}) 46 | client = TestClient(app) 47 | 48 | def test_bing(): 49 | # test search top 3 50 | response = client.get("/search_top3", params={"key_words": "test"}) 51 | 52 | output = "" 53 | for idx, item in enumerate(BING_TEST_SEARCH["webPages"]["value"]): 54 | output += "page: " + str(idx+1) + "\n" 55 | output += "title: " + item['name'] + "\n" 56 | output += "summary: " + item['snippet'] + "\n" 57 | assert response.status_code == 200 58 | assert response.json() == output 59 | 60 | # test load page 61 | response = client.get("/load_page_index", params={"idx": "1"}) 62 | assert response.status_code == 200 63 | assert response.json() == "This is page a" 64 | 65 | response = client.get("/load_page_index", params={"idx": "2"}) 66 | assert response.status_code == 200 67 | assert response.json() == "This is page b" 68 | -------------------------------------------------------------------------------- /bmtools/tools/chemical/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("chemical-prop") 4 | def chemical_prop(): 5 | from .prop import build_tool 6 | return build_tool 7 | -------------------------------------------------------------------------------- /bmtools/tools/chemical/prop/__init__.py: -------------------------------------------------------------------------------- 1 | from .api import build_tool -------------------------------------------------------------------------------- /bmtools/tools/chemical/prop/readme.md: -------------------------------------------------------------------------------- 1 | # Chemical Properties 2 | 3 | Contributor: [Zheni Zeng](https://github.com/Ellenzzn) 4 | 5 | ## Tool Description 6 | The tool, "Chemical Property Plugin," provides the ability to lookup a chemical's properties by querying a chemical knowledge base. The tool accepts the input in a JSON format, like {'input': 'some input'} and guides you to ask questions and search step by step. 7 | 8 | ### Tool Specifications 9 | 10 | - **Name**: Chemical Property Plugin 11 | - **Purpose**: Plugin for looking up a chemical's property using a chemical knowledge base 12 | - **Logo**: ![Chemical Property Plugin Logo](https://your-app-url.com/.well-known/logo.png) 13 | - **Contact Email**: hello@contact.com 14 | - **Legal Information**: [Legal Information](hello@legal.com) 15 | 16 | ### Core Functionality 17 | 18 | 1. `get_name` 19 | 20 | This method accepts a Compound ID (CID) and returns the top 3 synonyms for the queried compound. 21 | 22 | 2. `get_allname` 23 | 24 | This method accepts a Compound ID (CID) and returns all the possible synonyms for the queried compound. Be aware that the number of returned names can be large, so use this function with caution. 25 | 26 | 3. `get_id_by_struct` 27 | 28 | This method accepts a SMILES formula and returns the ID of the queried compound. This method should be used only if the SMILES formula is provided or retrieved in the previous step. 29 | 30 | 4. `get_id` 31 | 32 | This method accepts a compound name and returns the ID of the queried compound. If the name cannot be precisely matched, it will return the possible names. 33 | 34 | 5. `get_prop` 35 | 36 | This method accepts a Compound ID (CID) and returns the properties of the queried compound. 37 | 38 | The tool is made possible through the use of the ChemicalPropAPI, which interacts with a chemical knowledge base. -------------------------------------------------------------------------------- /bmtools/tools/chemical/prop/test_prop.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from .api import build_tool, ChemicalPropAPI 3 | from typing import Tuple, Optional, List 4 | 5 | 6 | class ChemicalPropMock(ChemicalPropAPI): 7 | def __init__(self) -> None: 8 | self._endpoint = "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/" 9 | 10 | def get_name_by_cid(self, cid : str, top_k : Optional[int] = None) -> List[str]: 11 | ans = ["A", "B", "C", "D", "E"] 12 | if top_k is None: 13 | top_k = len(ans) 14 | return ans[:top_k] 15 | 16 | def get_cid_by_struct(self, smiles : str) -> List[str]: 17 | return ["123"] 18 | 19 | def get_cid_by_name(self, name : str, name_type : Optional[str] = None) -> List[str]: 20 | return ["123"] 21 | 22 | def get_prop_by_cid(self, cid : str) -> str: 23 | return { 24 | "works": "well" 25 | } 26 | 27 | 28 | app = build_tool({"debug": True, "chemical_prop_api": ChemicalPropMock()}) 29 | client = TestClient(app) 30 | 31 | def test_get_name(): 32 | response = client.get("/get_name", params={"cid": 123}) 33 | assert response.status_code == 200 34 | assert response.json() == {"names": ["A", "B", "C"]} 35 | 36 | def test_get_all_names(): 37 | response = client.get("/get_allname", params={"cid": 123}) 38 | assert response.status_code == 200 39 | assert response.json() == {"names": ["A", "B", "C", "D", "E"]} 40 | 41 | def test_get_id_by_struct(): 42 | response = client.get("/get_id_by_struct", params={"smiles": "C1=CC=CC=C1"}) 43 | assert response.status_code == 200 44 | assert response.json() == { 45 | "state": "matched", 46 | "content": "123" 47 | } 48 | 49 | def test_get_id(): 50 | response = client.get("/get_id", params={"name": "benzene"}) 51 | assert response.status_code == 200 52 | assert response.json() == { 53 | "state": "precise", 54 | "content": "123", 55 | } 56 | 57 | def test_get_prop(): 58 | response = client.get("/get_prop", params={"cid": "123"}) 59 | assert response.status_code == 200 60 | assert response.json() == { 61 | "works": "well" 62 | } 63 | -------------------------------------------------------------------------------- /bmtools/tools/code_interpreter/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("code_interpreter") 5 | def code_interpreter(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/code_interpreter/api.py: -------------------------------------------------------------------------------- 1 | from ..tool import Tool 2 | 3 | class CodeInterpreter: 4 | def __init__(self, timeout=300): 5 | self.globals = {} 6 | self.locals = {} 7 | self.timeout = timeout 8 | 9 | def execute_code(self, code): 10 | try: 11 | # Wrap the code in an eval() call to return the result 12 | wrapped_code = f"__result__ = eval({repr(code)}, globals(), locals())" 13 | exec(wrapped_code, self.globals, self.locals) 14 | return self.locals.get('__result__', None) 15 | except Exception as e: 16 | try: 17 | # If eval fails, attempt to exec the code without returning a result 18 | exec(code, self.globals, self.locals) 19 | return "Code executed successfully." 20 | except Exception as e: 21 | return f"Error: {str(e)}" 22 | 23 | def reset_session(self): 24 | self.globals = {} 25 | self.locals = {} 26 | 27 | def build_tool(config) -> Tool: 28 | tool = Tool( 29 | "Python Code Interpreter Tool", 30 | "Execute Python Codes", 31 | name_for_model="code_interpreter", 32 | description_for_model="Plugin for executing python codes", 33 | logo_url=None, 34 | contact_email=None, 35 | legal_info_url=None 36 | ) 37 | 38 | # Usage example 39 | interpreter = CodeInterpreter() 40 | 41 | @tool.get("/execute_code") 42 | def execute_python_code(code: str): 43 | '''execute Python expressions with Python Interpreter, can be used as a simple calculator e.g., "(123 + 234) / 23 * 19" 44 | ''' 45 | return interpreter.execute_code(code) 46 | 47 | return tool -------------------------------------------------------------------------------- /bmtools/tools/code_interpreter/readme.md: -------------------------------------------------------------------------------- 1 | ## Tool Description 2 | The "Python Code Interpreter Tool" is a handy tool designed to execute Python code. It can be used as a simple Python interpreter or a calculator to compute Python expressions. 3 | 4 | ### Tool Specifications 5 | 6 | - **Name**: Python Code Interpreter Tool 7 | - **Purpose**: Execute Python codes 8 | - **Name for Model**: code_interpreter 9 | - **Model Description**: Plugin for executing python codes 10 | 11 | ### Core Functionality 12 | 13 | 1. `execute_code` 14 | 15 | This method accepts a Python code string as input and executes it using a Python interpreter. It can also be used as a simple calculator. For example, sending a request with code `(123 + 234) / 23 * 19` will return the result of this expression. 16 | 17 | The tool makes use of the `CodeInterpreter` class which is capable of running Python code. -------------------------------------------------------------------------------- /bmtools/tools/database/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("database") 4 | def register_database_tool(): 5 | from .api import build_database_tool 6 | return build_database_tool -------------------------------------------------------------------------------- /bmtools/tools/database/config.ini: -------------------------------------------------------------------------------- 1 | [postgresql] 2 | host = 3 | port = 4 | user = 5 | password = 6 | dbname = 7 | 8 | [mysql] 9 | host = 10 | port = 11 | user = 12 | password = -------------------------------------------------------------------------------- /bmtools/tools/database/data/script.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import json 4 | from difflib import ndiff 5 | import psycopg2 6 | import time 7 | 8 | ''' 9 | prepare the test samples 10 | ''' 11 | def execute_sql(sql): 12 | conn = psycopg2.connect(database='tpch10x', 13 | user='xxx', 14 | password='xxx', 15 | host='xxx', 16 | port=xxx) 17 | 18 | cur = conn.cursor() 19 | cur.execute(sql) 20 | # res = cur.fetchall()[0][0][0] 21 | res = cur.fetchall() 22 | 23 | conn.commit() 24 | cur.close() 25 | conn.close() 26 | 27 | return len(res) 28 | 29 | # Load the JSON file as a dictionary 30 | data = {} 31 | with open('text2res_single_table.json', 'r') as f: 32 | data = json.load(f) 33 | 34 | # Select only the diverse SQL statements 35 | # Find SQL statements with an edit distance of less than 10 36 | selected_sql = [] 37 | for sql1 in data: 38 | 39 | if 'sql' in sql1: 40 | sql1 = sql1['sql'] 41 | print("==========sql", sql1) 42 | start_time = time.time() 43 | res_cnt = execute_sql(sql1) 44 | end_time = time.time() 45 | elapsed_time = end_time - start_time 46 | 47 | print(res_cnt, elapsed_time) 48 | 49 | selected_sql.append({f"sql": sql1, 'res_cnt': res_cnt, 'execution_time': elapsed_time}) 50 | 51 | 52 | # Write the dictionary to a JSON file 53 | with open("text2res_single_table2.json", "w") as f: 54 | json.dump(selected_sql, f) 55 | 56 | 57 | 58 | ''' 59 | add text descriptions for queries 60 | ''' 61 | if __name__ == "__main__": 62 | 63 | llm = LLM() # add the def of your llm 64 | 65 | with open('./tpch10x/text2res_single_table2.json', 'r') as json_file: 66 | json_data = json.load(json_file) 67 | 68 | new_json_data = [] 69 | for i,item in enumerate(json_data): 70 | sql = item['sql'] 71 | print("========= ", i, sql) 72 | prompt = "Please convert the following sql query into one natural language sentence: \n" + sql + "\n Note. 1) Do not mention any other information other than the natural language sentence; 2) Must use the origin table and column names in the sql query." 73 | text = llm(prompt) 74 | item['text'] = text 75 | new_json_data.append(item) 76 | #print(llm("Describe Shanghai in 200 words.")) 77 | 78 | with open("text2res_single_table3.json", "w") as f: 79 | json.dump(new_json_data, f) 80 | 81 | 82 | 83 | ''' 84 | calculate total execution time 85 | ''' 86 | 87 | with open('text2res_origin.json', 'r') as json_file: 88 | json_data = json.load(json_file) 89 | 90 | total_time = 0 91 | 92 | for i,item in enumerate(json_data): 93 | print(item['execution_time']) 94 | total_time = total_time + float(item['execution_time']) 95 | 96 | print(total_time) 97 | -------------------------------------------------------------------------------- /bmtools/tools/database/readme.md: -------------------------------------------------------------------------------- 1 | # Database Tool 2 | 3 | Contributor: [Xuanhe Zhou](https://github.com/zhouxh19) 4 | 5 | ### API Functions 6 | 7 | - *get_database_schema*: obtain the information of target tables 8 | - *select_database_data*: fetch the query results from a database instance 9 | - *rewrite_sql*: transform a sql query into an semantic-equivalent but execution-efficient sql 10 | 11 | ### Dataset 12 | 13 | - Text2SQL Dataset 14 | 15 | - *./data/tpch10x/text2res_multi_table.json*: relativley complex database queries (2-6 tables) 16 | - *./data/tpch10x/text2res_single_table.json*: basic database queries 17 | 18 | - SQL Optimization Dataset 19 | 20 | - Samples for *[sql rewrite](https://github.com/TsinghuaDatabaseGroup/lmdb/tree/main/query_rewrite/data)* 21 | 22 | - Samples for *[index tuning](https://github.com/TsinghuaDatabaseGroup/lmdb/tree/main/index_tuning/data)* 23 | 24 | ### Setup 25 | 26 | 1. Follow the steps in [main readme](https://github.com/OpenBMB/BMTools/blob/main/README.md) 27 | 28 | 29 | 2. Rename config.ini.template into my_config.ini 30 | 31 | 3. Configure the adopted LLM model in the 84th line of ../../agent/singletool.py, e.g., 32 | 33 | ```bash 34 | self.llm = OpenAI(model_name="gpt-3.5-turbo", temperature=0.0, openai_api_key=key) 35 | ``` 36 | 37 | 4. Modify database settings in my_config.ini, e.g., 38 | 39 | ```bash 40 | [{db_system}] 41 | host = 127.0.0.1 42 | port = 5432 43 | user = postgres 44 | password = postgres 45 | dbname = postgres 46 | ``` 47 | 48 | And rename *config.ini* into *my_config.ini*. 49 | 50 | Note. {db_system} must match with that in ./api.py 51 | 52 | 4. Modify and run the test.py script to test the tool 53 | -------------------------------------------------------------------------------- /bmtools/tools/database/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/tools/database/utils/__init__.py -------------------------------------------------------------------------------- /bmtools/tools/database/utils/data.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | 4 | def subsample_data(data, subsample_size): 5 | """ 6 | Subsample data. Data is in the form of a tuple of lists. 7 | """ 8 | inputs, outputs = data 9 | assert len(inputs) == len(outputs) 10 | indices = random.sample(range(len(inputs)), subsample_size) 11 | inputs = [inputs[i] for i in indices] 12 | outputs = [outputs[i] for i in indices] 13 | return inputs, outputs 14 | 15 | 16 | def create_split(data, split_size): 17 | """ 18 | Split data into two parts. Data is in the form of a tuple of lists. 19 | """ 20 | inputs, outputs = data 21 | assert len(inputs) == len(outputs) 22 | indices = random.sample(range(len(inputs)), split_size) 23 | inputs1 = [inputs[i] for i in indices] 24 | outputs1 = [outputs[i] for i in indices] 25 | inputs2 = [inputs[i] for i in range(len(inputs)) if i not in indices] 26 | outputs2 = [outputs[i] for i in range(len(inputs)) if i not in indices] 27 | return (inputs1, outputs1), (inputs2, outputs2) 28 | -------------------------------------------------------------------------------- /bmtools/tools/database/utils/db_parser.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import configparser 3 | import logging 4 | 5 | def get_conf(conf_file, server_name): 6 | conf = configparser.ConfigParser() 7 | conf.read(conf_file) 8 | sql_server = conf[server_name] 9 | return sql_server 10 | 11 | 12 | def get_parser(): 13 | parser = argparse.ArgumentParser( 14 | description="Instruction Induction.") 15 | 16 | parser.add_argument("--db_conf", type=str, 17 | default = '../database/configs/config.ini') 18 | 19 | """ 20 | parser.add_argument("--train_data", type=str, 21 | default="./data/raw/train/rules.json") 22 | parser.add_argument("--eval_data", type=str, 23 | default="./data/raw/execute/zhenzhi.json") 24 | 25 | parser.add_argument("--data_save", type=str, 26 | default="./result/{}/data/") 27 | 28 | parser.add_argument("--seed", type=int, default=42) 29 | parser.add_argument("--runlog", type=str, 30 | default="./result/{}/exp_runtime.log") 31 | parser.add_argument("--logdir", type=str, 32 | default="./result/{}/logdir/") 33 | parser.add_argument("--model_save", type=str, 34 | default="./result/{}/model/") 35 | 36 | parser.add_argument("--gen_sample", type=int, default=20) 37 | parser.add_argument("--gen_demo", type=int, default=16) 38 | parser.add_argument("--gen_prompt_per_sample", type=int, default=5) 39 | parser.add_argument("--gen_model", type=str, default="text-davinci-003") 40 | parser.add_argument("--gen_max_tokens", type=int, default=200) 41 | 42 | parser.add_argument("--eval_sample", type=int, default=20) 43 | parser.add_argument("--eval_model", type=str, default="text-davinci-003") 44 | parser.add_argument("--eval_max_tokens", type=int, default=1000) 45 | 46 | parser.add_argument("--storage_budget", type=int, default=500) # limit storage space of built indexes 47 | """ 48 | 49 | 50 | return parser 51 | 52 | 53 | def set_logger(log_file): 54 | logger = logging.getLogger() 55 | logger.setLevel(logging.DEBUG) 56 | formatter = logging.Formatter( 57 | '%(asctime)s - %(name)s - %(levelname)s: - %(message)s', 58 | datefmt='%Y-%m-%d %H:%M:%S') 59 | 60 | # log to file 61 | fh = logging.FileHandler(log_file) 62 | fh.setLevel(logging.DEBUG) 63 | fh.setFormatter(formatter) 64 | 65 | # log to console 66 | ch = logging.StreamHandler() 67 | ch.setLevel(logging.DEBUG) 68 | ch.setFormatter(formatter) 69 | 70 | logger.addHandler(ch) 71 | logger.addHandler(fh) -------------------------------------------------------------------------------- /bmtools/tools/database/utils/db_utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sqlparse 3 | 4 | def remove_create_table(sql): 5 | return re.sub(r'(create|CREATE)\s+(table|TABLE).+?\(.+?\)\s*;','',sql, flags=re.DOTALL) 6 | 7 | def remove_create_index(sql): 8 | return re.sub(r'(create|CREATE)\s+(index|INDEX).+?\(.+?\)\s*;','',sql, flags=re.DOTALL) 9 | 10 | def remove_table(sql): 11 | return re.sub(r'(table|TABLE).+?\(.+?\)\s*;','',sql, flags=re.DOTALL) 12 | 13 | def clean_sql(sql): 14 | tmp = [] 15 | for token in sql.flatten(): 16 | if not token.is_whitespace and not token.ttype is sqlparse.tokens.Comment.Single: 17 | tmp.append(token) 18 | return strip_par(' '.join(str(t) for t in tmp)) 19 | 20 | def strip_par(s): 21 | for op in ['(',')',',','>','=','<','>=','<=','!=','<>','.',';']: 22 | s = s.replace(' {}'.format(op), op).replace('{} '.format(op), op) 23 | return s 24 | 25 | def preprocess_execute_sql(sql): 26 | sql = remove_create_table(sql) 27 | sql = remove_create_index(sql) 28 | parsed = sqlparse.parse(sql) 29 | if len(parsed) == 0: 30 | return [0, ''] 31 | sql = clean_sql(parsed[0]) 32 | if not sql: 33 | return [0, ''] 34 | if sql[-1] != ';': 35 | sql += ';' 36 | return [1, sql] -------------------------------------------------------------------------------- /bmtools/tools/db_diag/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("db_diag") 4 | def register_db_diag_tool(): 5 | from .api import build_db_diag_tool 6 | return build_db_diag_tool -------------------------------------------------------------------------------- /bmtools/tools/db_diag/anomaly_detection.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import requests 3 | import json 4 | 5 | 6 | def prometheus(url, params): 7 | res = requests.get(url='http://8.131.229.55:9090/' + url, params=params) 8 | # print(json.dumps(res.json())) 9 | 10 | #return json.dumps(res.json()) 11 | return res.json() 12 | 13 | 14 | def detect_anomalies(data, significance_level=0.05): 15 | 16 | # assume the workload is steadily running 17 | 18 | """ 19 | Detects anomalies in the given data using the KS test algorithm. 20 | 21 | Args: 22 | data (numpy.ndarray): 1-D array of data values. 23 | significance_level (float): Level of significance for the KS test (default: 0.05). 24 | 25 | Returns: 26 | numpy.ndarray: Boolean array indicating anomalies (True) and non-anomalies (False). 27 | """ 28 | 29 | """ 30 | sorted_data = np.sort(data) 31 | n = len(sorted_data) 32 | 33 | # Calculate the expected CDF assuming a normal distribution 34 | expected_cdf = np.arange(1, n + 1) / n 35 | 36 | # Calculate the empirical CDF 37 | empirical_cdf = np.searchsorted(sorted_data, sorted_data, side='right') / n 38 | 39 | # Calculate the maximum absolute difference between the expected and empirical CDFs 40 | ks_statistic = np.max(np.abs(empirical_cdf - expected_cdf)) 41 | 42 | # Calculate the critical value based on the significance level and sample size 43 | critical_value = np.sqrt(-0.5 * np.log(significance_level / 2) / n) 44 | 45 | # Compare the KS statistic with the critical value 46 | anomalies = np.where(ks_statistic > critical_value, True, False) 47 | """ 48 | 49 | # Calculate the mean and standard deviation of the data 50 | anomalies = False 51 | 52 | mean = np.mean(data) 53 | max_value = np.max(data) 54 | 55 | print("mean: ", mean) 56 | print("max_value: ", max_value) 57 | 58 | if max_value > mean: 59 | anomalies = True 60 | 61 | return anomalies -------------------------------------------------------------------------------- /bmtools/tools/db_diag/config.ini: -------------------------------------------------------------------------------- 1 | [postgresql] 2 | host = 3 | port = 4 | user = 5 | password = 6 | dbname = 7 | 8 | [mysql] 9 | host = 10 | port = 11 | user = 12 | password = 13 | 14 | [benchserver] 15 | server_address = 16 | username = 17 | password = 18 | remote_directory = -------------------------------------------------------------------------------- /bmtools/tools/db_diag/database_monitoring_metrics: -------------------------------------------------------------------------------- 1 | ['go_gc_duration_seconds', 'go_gc_duration_seconds_count', 'go_gc_duration_seconds_sum', 'go_goroutines', 'go_memstats_alloc_bytes', 'go_memstats_alloc_bytes_total', 'go_memstats_frees_total', 'go_memstats_gc_cpu_fraction', 'go_memstats_heap_alloc_bytes', 'go_memstats_heap_idle_bytes', 'go_memstats_heap_inuse_bytes', 'go_memstats_heap_objects', 'go_memstats_heap_released_bytes', 'go_memstats_heap_sys_bytes', 'go_memstats_last_gc_time_seconds', 'go_memstats_mallocs_total', 'go_memstats_mspan_inuse_bytes', 'go_memstats_stack_inuse_bytes', 'go_memstats_stack_sys_bytes', 'node_context_switches_total', 'node_cpu_seconds_total', 'node_disk_io_now', 'node_disk_io_time_seconds_total', 'node_disk_io_time_weighted_seconds_total', 'node_disk_read_bytes_total', 'node_disk_read_time_seconds_total', 'node_disk_reads_completed_total', 'node_disk_reads_merged_total', 'node_disk_write_time_seconds_total', 'node_disk_writes_completed_total', 'node_disk_writes_merged_total', 'node_disk_written_bytes_total', 'node_entropy_available_bits', 'node_filefd_allocated', 'node_filesystem_avail_bytes', 'node_filesystem_files_free', 'node_filesystem_free_bytes', 'node_forks_total', 'node_intr_total', 'node_load1', 'node_load15', 'node_load5', 'node_memory_Active_anon_bytes', 'node_memory_Active_bytes', 'node_memory_Active_file_bytes', 'node_memory_AnonHugePages_bytes', 'node_memory_AnonPages_bytes', 'node_memory_Cached_bytes', 'node_memory_Committed_AS_bytes', 'node_memory_Dirty_bytes', 'node_memory_Inactive_anon_bytes', 'node_memory_Inactive_bytes', 'node_memory_Inactive_file_bytes', 'node_memory_KReclaimable_bytes', 'node_memory_KernelStack_bytes', 'node_memory_Mapped_bytes', 'node_memory_MemAvailable_bytes', 'node_memory_MemFree_bytes', 'node_memory_PageTables_bytes', 'node_memory_SReclaimable_bytes', 'node_memory_SUnreclaim_bytes', 'node_memory_Shmem_bytes', 'node_memory_Slab_bytes', 'node_netstat_Icmp_InMsgs', 'node_netstat_Icmp_OutMsgs', 'node_netstat_Ip6_InOctets', 'node_netstat_Ip6_OutOctets', 'node_netstat_IpExt_InOctets', 'node_netstat_IpExt_OutOctets', 'node_netstat_TcpExt_TCPSynRetrans', 'node_netstat_Tcp_ActiveOpens', 'node_netstat_Tcp_CurrEstab', 'node_netstat_Tcp_InSegs', 'node_netstat_Tcp_OutRsts', 'node_netstat_Tcp_OutSegs', 'node_netstat_Tcp_PassiveOpens', 'node_netstat_Tcp_RetransSegs', 'node_netstat_Udp6_InDatagrams', 'node_netstat_Udp6_InErrors', 'node_netstat_Udp6_OutDatagrams', 'node_netstat_Udp6_RcvbufErrors', 'node_netstat_Udp_InDatagrams', 'node_netstat_Udp_OutDatagrams', 'node_network_receive_bytes_total', 'node_network_receive_packets_total', 'node_network_transmit_bytes_total', 'node_network_transmit_packets_total', 'node_procs_blocked', 'node_procs_running', 'node_schedstat_running_seconds_total', 'node_schedstat_timeslices_total', 'node_schedstat_waiting_seconds_total', 'node_scrape_collector_duration_seconds', 'node_sockstat_TCP6_inuse', 'node_sockstat_TCP_alloc', 'node_sockstat_TCP_inuse', 'node_sockstat_TCP_mem', 'node_sockstat_TCP_mem_bytes', 'node_sockstat_TCP_orphan', 'node_sockstat_TCP_tw', 'node_sockstat_UDP_inuse', 'node_sockstat_UDP_mem', 'node_sockstat_UDP_mem_bytes', 'node_sockstat_sockets_used', 'node_softnet_processed_total', 'node_time_seconds', 'node_timex_estimated_error_seconds', 'node_timex_frequency_adjustment_ratio', 'node_timex_maxerror_seconds', 'node_timex_tick_seconds', 'node_vmstat_pgfault', 'node_vmstat_pgmajfault', 'node_vmstat_pgpgin', 'node_vmstat_pgpgout', 'node_xfs_block_mapping_extent_list_deletions_total', 'node_xfs_block_mapping_extent_list_insertions_total', 'node_xfs_block_mapping_extent_list_lookups_total', 'node_xfs_block_mapping_reads_total', 'node_xfs_block_mapping_unmaps_total', 'node_xfs_block_mapping_writes_total', 'node_xfs_directory_operation_create_total', 'node_xfs_directory_operation_getdents_total', 'node_xfs_directory_operation_lookup_total', 'node_xfs_directory_operation_remove_total', 'node_xfs_extent_allocation_blocks_allocated_total', 'node_xfs_extent_allocation_blocks_freed_total', 'node_xfs_extent_allocation_extents_allocated_total', 'node_xfs_extent_allocation_extents_freed_total', 'node_xfs_inode_operation_attempts_total', 'node_xfs_inode_operation_attribute_changes_total', 'node_xfs_inode_operation_found_total', 'node_xfs_inode_operation_missed_total', 'node_xfs_inode_operation_reclaims_total', 'node_xfs_read_calls_total', 'node_xfs_vnode_active_total', 'node_xfs_vnode_reclaim_total', 'node_xfs_vnode_release_total', 'node_xfs_vnode_remove_total', 'node_xfs_write_calls_total', 'process_cpu_seconds_total', 'process_resident_memory_bytes', 'process_start_time_seconds', 'promhttp_metric_handler_requests_total', 'scrape_duration_seconds', 'scrape_samples_post_metric_relabeling', 'scrape_samples_scraped', 'up'] -------------------------------------------------------------------------------- /bmtools/tools/db_diag/prometheus.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import datetime 4 | import numpy as np 5 | 6 | def prometheus(url, params): 7 | output = requests.get(url='http://8.131.229.55:9090/' + url, params=params) 8 | output = output.json() 9 | print(output) 10 | #output = json.dumps(res.json()) 11 | output = output["data"]["result"][0]["values"] 12 | output = np.array([float(value) for _, value in output]) 13 | print(output) 14 | print(type(output)) 15 | 16 | if __name__ == '__main__': 17 | #prometheus('api/v1/query_range', {'query': '100 - (avg(irate(node_cpu_seconds_total{instance=~"123.56.63.105:9100",mode="idle"}[1m])) * 100)', 'start': '1684412385', 'end': '1684412485', 'step': '3'}) 18 | start_timestamp_str = "2023-05-19 22:21:30" 19 | dt = datetime.datetime.strptime(start_timestamp_str, "%Y-%m-%d %H:%M:%S") 20 | timestamp = dt.timestamp() 21 | start_time = timestamp 22 | 23 | end_timestamp_str = "2023-05-19 22:23:30" 24 | dt = datetime.datetime.strptime(end_timestamp_str, "%Y-%m-%d %H:%M:%S") 25 | timestamp = dt.timestamp() 26 | end_time = timestamp 27 | 28 | prometheus('api/v1/query_range', {'query': "node_memory_MemTotal_bytes{instance=~\"123.56.63.105:9100\"} - (node_memory_Cached_bytes{instance=~\"123.56.63.105:9100\"} + node_memory_Buffers_bytes{instance=~\"123.56.63.105:9100\"} + node_memory_MemFree_bytes{instance=~\"123.56.63.105:9100\"})", 'start': start_time, 'end': end_time, 'step': '3'}) -------------------------------------------------------------------------------- /bmtools/tools/db_diag/readme.md: -------------------------------------------------------------------------------- 1 | # db_diag Tool 2 | 3 | Contributor: [Xuanhe Zhou](https://github.com/zhouxh19) 4 | 5 | ### API Functions 6 | 7 | - *obtain_start_and_end_time_of_anomaly*: fetch the time period of an anomaly 8 | - *whether_is_abnormal_metric*: examine whether the values of the input metric appear to be abnormal. //todo: add classic anomaly detection algorithms 9 | - *xxx_diagnosis_agent*: diagnose the root causes of the abnormal metrics in specific region (e.g., memory/cpu problems) 10 | 11 | 12 | ### Setup 13 | 14 | 1. Follow the steps in [main readme](https://github.com/OpenBMB/BMTools/blob/main/README.md) 15 | 16 | 2. Configure the adopted LLM model in the 84th line of ../../agent/singletool.py, e.g., 17 | 18 | ```bash 19 | self.llm = OpenAI(model_name="gpt-3.5-turbo", temperature=0.0, openai_api_key=key) 20 | ``` 21 | 22 | 3. Modify the settings in *config.ini*, and rename *config.ini* into *my_config.ini* 23 | 24 | 4. Modify and run the test.py script to test the tool -------------------------------------------------------------------------------- /bmtools/tools/db_diag/test.py: -------------------------------------------------------------------------------- 1 | 2 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 3 | import datetime 4 | 5 | tool_name, tool_url = 'db_diag', "http://127.0.0.1:8079/tools/db_diag/" 6 | tool_name, tool_config = load_single_tools(tool_name, tool_url) 7 | print(tool_name, tool_config) 8 | stqa = STQuestionAnswerer() 9 | 10 | # langchain 11 | agent = stqa.load_tools(tool_name, tool_config, prompt_type="react-with-tool-description") # langchain: react-with-tool-description autogpt: autogpt 12 | 13 | # database on 123.56.63.105 14 | 15 | ''' 16 | start_timestamp_str = "2023-05-19 22:21:30" 17 | dt = datetime.datetime.strptime(start_timestamp_str, "%Y-%m-%d %H:%M:%S") 18 | timestamp = dt.timestamp() 19 | start_time = timestamp 20 | 21 | end_timestamp_str = "2023-05-19 22:23:30" 22 | dt = datetime.datetime.strptime(end_timestamp_str, "%Y-%m-%d %H:%M:%S") 23 | timestamp = dt.timestamp() 24 | end_time = timestamp 25 | 26 | print(" ===== time period: ", start_time, end_time) 27 | ''' 28 | 29 | #text = "The database performance is bad during {} to {}.".format(start_timestamp_str, end_timestamp_str) # trigger database diagnosis 30 | 31 | text = "Here is a database performance problem. Please help me to diagnose the causes and give some optimization suggestions." 32 | 33 | agent(""" {} 34 | 35 | First, obtain_start_and_end_time_of_anomaly and memorize the start and end time of the anomaly. 36 | 37 | Second, you need to diagnose the causes of the anomaly from the following two aspects: 38 | 39 | - call the whether_is_abnormal_metric API and examine whether CPU usage is high (or abnormal). Next, if the CPU usage is high (or abnormal), cpu_diagnosis_agent and obtain the diagnosis results. 40 | 41 | - call the whether_is_abnormal_metric API and examine whether memory usage is high (or abnormal). Next, if the memory usage is high (or abnormal), memory_diagnosis_agent and obtain the diagnosis results. 42 | 43 | Third, you need to match each cause with potential solutions cached in the vector database. 44 | 45 | Finally, list the above diagnosed causes and their matched solutions in easy-to-understand format using bullet points. 46 | 47 | ================================ 48 | A Demonstration example: 49 | 50 | Thought: I need to check whether the CPU usage is high or abnormal during the given time period. 51 | 52 | Action: whether_is_abnormal_metric 53 | 54 | Action Input: {{"start_time": xxxx, "end_time": xxxx, "metric_name": "cpu_usage"}} 55 | 56 | Note. 1) The first action must be obtain_start_and_end_time_of_anomaly; 57 | 2) Do not use any image in the output; 58 | 3) Give some optimization suggestions based on the diagnosis results. 59 | """.format(text)) 60 | 61 | ''' 62 | 1) Action can only be one of the following API names: obtain_start_and_end_time_of_anomaly, whether_is_abnormal_metric, obtain_values_of_metrics, cpu_diagnosis_agent, memory_diagnosis_agent. Any other content in Action is unacceptable; 63 | ''' -------------------------------------------------------------------------------- /bmtools/tools/db_diag/utils/data.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | 4 | def subsample_data(data, subsample_size): 5 | """ 6 | Subsample data. Data is in the form of a tuple of lists. 7 | """ 8 | inputs, outputs = data 9 | assert len(inputs) == len(outputs) 10 | indices = random.sample(range(len(inputs)), subsample_size) 11 | inputs = [inputs[i] for i in indices] 12 | outputs = [outputs[i] for i in indices] 13 | return inputs, outputs 14 | 15 | 16 | def create_split(data, split_size): 17 | """ 18 | Split data into two parts. Data is in the form of a tuple of lists. 19 | """ 20 | inputs, outputs = data 21 | assert len(inputs) == len(outputs) 22 | indices = random.sample(range(len(inputs)), split_size) 23 | inputs1 = [inputs[i] for i in indices] 24 | outputs1 = [outputs[i] for i in indices] 25 | inputs2 = [inputs[i] for i in range(len(inputs)) if i not in indices] 26 | outputs2 = [outputs[i] for i in range(len(inputs)) if i not in indices] 27 | return (inputs1, outputs1), (inputs2, outputs2) 28 | -------------------------------------------------------------------------------- /bmtools/tools/db_diag/utils/db_parser.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import configparser 3 | import logging 4 | 5 | def get_conf(conf_file, server_name): 6 | conf = configparser.ConfigParser() 7 | conf.read(conf_file) 8 | sql_server = conf[server_name] 9 | return sql_server 10 | 11 | 12 | def get_parser(): 13 | parser = argparse.ArgumentParser( 14 | description="Instruction Induction.") 15 | 16 | parser.add_argument("--db_conf", type=str, 17 | default = '../database/configs/config.ini') 18 | 19 | """ 20 | parser.add_argument("--train_data", type=str, 21 | default="./data/raw/train/rules.json") 22 | parser.add_argument("--eval_data", type=str, 23 | default="./data/raw/execute/zhenzhi.json") 24 | 25 | parser.add_argument("--data_save", type=str, 26 | default="./result/{}/data/") 27 | 28 | parser.add_argument("--seed", type=int, default=42) 29 | parser.add_argument("--runlog", type=str, 30 | default="./result/{}/exp_runtime.log") 31 | parser.add_argument("--logdir", type=str, 32 | default="./result/{}/logdir/") 33 | parser.add_argument("--model_save", type=str, 34 | default="./result/{}/model/") 35 | 36 | parser.add_argument("--gen_sample", type=int, default=20) 37 | parser.add_argument("--gen_demo", type=int, default=16) 38 | parser.add_argument("--gen_prompt_per_sample", type=int, default=5) 39 | parser.add_argument("--gen_model", type=str, default="text-davinci-003") 40 | parser.add_argument("--gen_max_tokens", type=int, default=200) 41 | 42 | parser.add_argument("--eval_sample", type=int, default=20) 43 | parser.add_argument("--eval_model", type=str, default="text-davinci-003") 44 | parser.add_argument("--eval_max_tokens", type=int, default=1000) 45 | 46 | parser.add_argument("--storage_budget", type=int, default=500) # limit storage space of built indexes 47 | """ 48 | 49 | 50 | return parser 51 | 52 | 53 | def set_logger(log_file): 54 | logger = logging.getLogger() 55 | logger.setLevel(logging.DEBUG) 56 | formatter = logging.Formatter( 57 | '%(asctime)s - %(name)s - %(levelname)s: - %(message)s', 58 | datefmt='%Y-%m-%d %H:%M:%S') 59 | 60 | # log to file 61 | fh = logging.FileHandler(log_file) 62 | fh.setLevel(logging.DEBUG) 63 | fh.setFormatter(formatter) 64 | 65 | # log to console 66 | ch = logging.StreamHandler() 67 | ch.setLevel(logging.DEBUG) 68 | ch.setFormatter(formatter) 69 | 70 | logger.addHandler(ch) 71 | logger.addHandler(fh) -------------------------------------------------------------------------------- /bmtools/tools/db_diag/utils/db_utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sqlparse 3 | 4 | def remove_create_table(sql): 5 | return re.sub(r'(create|CREATE)\s+(table|TABLE).+?\(.+?\)\s*;','',sql, flags=re.DOTALL) 6 | 7 | def remove_create_index(sql): 8 | return re.sub(r'(create|CREATE)\s+(index|INDEX).+?\(.+?\)\s*;','',sql, flags=re.DOTALL) 9 | 10 | def remove_table(sql): 11 | return re.sub(r'(table|TABLE).+?\(.+?\)\s*;','',sql, flags=re.DOTALL) 12 | 13 | def clean_sql(sql): 14 | tmp = [] 15 | for token in sql.flatten(): 16 | if not token.is_whitespace and not token.ttype is sqlparse.tokens.Comment.Single: 17 | tmp.append(token) 18 | return strip_par(' '.join(str(t) for t in tmp)) 19 | 20 | def strip_par(s): 21 | for op in ['(',')',',','>','=','<','>=','<=','!=','<>','.',';']: 22 | s = s.replace(' {}'.format(op), op).replace('{} '.format(op), op) 23 | return s 24 | 25 | def preprocess_execute_sql(sql): 26 | sql = remove_create_table(sql) 27 | sql = remove_create_index(sql) 28 | parsed = sqlparse.parse(sql) 29 | if len(parsed) == 0: 30 | return [0, ''] 31 | sql = clean_sql(parsed[0]) 32 | if not sql: 33 | return [0, ''] 34 | if sql[-1] != ';': 35 | sql += ';' 36 | return [1, sql] -------------------------------------------------------------------------------- /bmtools/tools/file_operation/README.md: -------------------------------------------------------------------------------- 1 | # File operation tool 2 | 3 | Contributor: [Yujia Qin](https://github.com/thuqinyj16) -------------------------------------------------------------------------------- /bmtools/tools/file_operation/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("file_operation") 5 | def file_operation(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/file_operation/api.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from ..tool import Tool 3 | 4 | 5 | def build_tool(config) -> Tool: 6 | tool = Tool( 7 | "File Operation Tool", 8 | "Write / read file to / from disk", 9 | name_for_model="file_operation", 10 | description_for_model="Plugin for operating files", 11 | logo_url=None, 12 | contact_email=None, 13 | legal_info_url=None 14 | ) 15 | 16 | @tool.get("/write_file") 17 | def write_file(file_path: str, text: str) -> str: 18 | '''write file to disk 19 | ''' 20 | write_path = ( 21 | Path(file_path) 22 | ) 23 | try: 24 | write_path.parent.mkdir(exist_ok=True, parents=False) 25 | with write_path.open("w", encoding="utf-8") as f: 26 | f.write(text) 27 | return f"File written successfully to {file_path}." 28 | except Exception as e: 29 | return "Error: " + str(e) 30 | 31 | @tool.get("/read_file") 32 | def read_file(file_path: str) -> str: 33 | '''read file from disk 34 | ''' 35 | read_path = ( 36 | Path(file_path) 37 | ) 38 | try: 39 | with read_path.open("r", encoding="utf-8") as f: 40 | content = f.read() 41 | return content 42 | except Exception as e: 43 | return "Error: " + str(e) 44 | 45 | return tool 46 | -------------------------------------------------------------------------------- /bmtools/tools/file_operation/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'file_operation', "http://127.0.0.1:8079/tools/file_operation/" 4 | tool_name, tool_config = load_single_tools(tool_name, tool_url) 5 | print(tool_name, tool_config) 6 | stqa = STQuestionAnswerer() 7 | 8 | agent = stqa.load_tools(tool_name, tool_config) 9 | agent("write hello world to test.txt") -------------------------------------------------------------------------------- /bmtools/tools/film/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("douban-film") 4 | def douban_film(): 5 | from .douban import build_tool 6 | return build_tool 7 | -------------------------------------------------------------------------------- /bmtools/tools/film/douban/__init__.py: -------------------------------------------------------------------------------- 1 | from .api import build_tool 2 | -------------------------------------------------------------------------------- /bmtools/tools/film/douban/readme.md: -------------------------------------------------------------------------------- 1 | # Douban Film Search 2 | 3 | Contributor: [Jing Yi](https://github.com/yijing16) 4 | 5 | ## Tool Description 6 | The "Film Search Plugin" is a robust tool that allows you to access up-to-date film information, filter through this information, and retrieve detailed descriptions of specific films. It utilizes a fictional API called "DoubanAPI" to pull information on films. 7 | 8 | ### Tool Specifications 9 | 10 | - **Name**: Film Search Plugin 11 | - **Purpose**: Search for up-to-date film information. 12 | - **Name for Model**: Film Search 13 | - **Model Description**: Plugin for search for up-to-date film information. 14 | - **Logo URL**: [logo](https://your-app-url.com/.well-known/logo.png) 15 | - **Contact Email**: hello@contact.com 16 | - **Legal Information URL**: hello@legal.com 17 | 18 | ### Core Functionality 19 | 20 | 1. `coming_out_filter` 21 | 22 | This method accepts a string argument that contains details about the desired region, category, number of films, and a flag indicating if sorting is needed based on the number of people looking forward to the movie. It filters through the upcoming films based on these details and provides the information. 23 | 24 | 2. `now_playing_out_filter` 25 | 26 | This method accepts a string argument that contains details about the desired region, the number of films, and a flag indicating if sorting is needed based on the film score. It filters through the currently playing films based on these details and provides the information. 27 | 28 | 3. `print_detail` 29 | 30 | This method accepts a string argument that contains the name of a specific film. It provides detailed information about the film, including the genre, director, actors, and a brief synopsis of the film's plot. 31 | 32 | All methods use the `DoubanAPI` to retrieve and filter information on films. 33 | 34 | **Note**: This tool's functionality is hypothetical and relies on the existence and proper functioning of a fictional API, DoubanAPI, which is not included in the provided code. In a real-world application, replace DoubanAPI with a functional API that can retrieve film data. -------------------------------------------------------------------------------- /bmtools/tools/film/douban/test.py: -------------------------------------------------------------------------------- 1 | 2 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 3 | 4 | tool_name, tool_url = 'douban', "http://127.0.0.1:8079/tools/douban-film/" 5 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 6 | # tools_name, tools_config = load_single_tools() 7 | print(tools_name, tools_config) 8 | 9 | qa = STQuestionAnswerer() 10 | 11 | agent = qa.load_tools(tools_name, tools_config) 12 | 13 | agent("有哪些即将上映的中国喜剧电影？哪些是大家最想看的前5部？") 14 | agent("想去电影院看一些国产电影，有评分高的吗？输出3部") 15 | agent("帮我介绍下《深海》这部电影") 16 | -------------------------------------------------------------------------------- /bmtools/tools/film/douban/test_douban.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from .api import build_tool, DoubanAPI, ComingMovieInfo, PlayingMovieInfo 3 | from typing import List 4 | 5 | 6 | class DoubanMock(DoubanAPI): 7 | def __init__(self) -> None: 8 | pass 9 | 10 | def get_coming(self) -> List[ComingMovieInfo]: 11 | return [ 12 | ComingMovieInfo(date="2020-12-12", title="test1", cate="test1", region="test1", wantWatchPeopleNum="1", link="test1"), 13 | ComingMovieInfo(date="2021-12-12", title="test2", cate="test2", region="test2", wantWatchPeopleNum="2", link="test2"), 14 | ComingMovieInfo(date="2022-12-12", title="test3", cate="test3", region="test3", wantWatchPeopleNum="3", link="test3"), 15 | ] 16 | 17 | def get_now_playing(self) -> List[PlayingMovieInfo]: 18 | return [ 19 | PlayingMovieInfo(title="test1", score="1.1", region="test1", director="test1", actors="test1", link="test1"), 20 | PlayingMovieInfo(title="test2", score="2.2", region="test2", director="test2", actors="test2", link="test2"), 21 | PlayingMovieInfo(title="test3", score="3.3", region="test3", director="test3", actors="test3", link="test3"), 22 | ] 23 | 24 | def get_movie_detail(self, url : str) -> str: 25 | return url 26 | 27 | 28 | app = build_tool({"debug": True, "douban_api": DoubanMock()}) 29 | client = TestClient(app) 30 | 31 | def test_get_coming(): 32 | response = client.get("/coming_out_filter", params={ 33 | "args": "全部, 全部, 2, True" 34 | }) 35 | assert response.status_code == 200 36 | assert response.json() == { 37 | "date": { 38 | "1": "2021-12-12", 39 | "0": "2022-12-12", 40 | }, 41 | "title": { 42 | "1": "test2", 43 | "0": "test3", 44 | }, 45 | "cate": { 46 | "1": "test2", 47 | "0": "test3", 48 | }, 49 | "region": { 50 | "1": "test2", 51 | "0": "test3", 52 | }, 53 | "wantWatchPeopleNum": { 54 | "1": "2人", 55 | "0": "3人", 56 | }, 57 | } 58 | 59 | def test_get_playing(): 60 | response = client.get("/now_playing_out_filter", params={ 61 | "args": "全部, 3, False" 62 | }) 63 | assert response.status_code == 200 64 | assert response.json() == { 65 | "title": { 66 | "0": "test1", 67 | "1": "test2", 68 | "2": "test3", 69 | }, 70 | "score": { 71 | "0": "1.1", 72 | "1": "2.2", 73 | "2": "3.3", 74 | }, 75 | "region": { 76 | "0": "test1", 77 | "1": "test2", 78 | "2": "test3", 79 | }, 80 | "director": { 81 | "0": "test1", 82 | "1": "test2", 83 | "2": "test3", 84 | }, 85 | "actors": { 86 | "0": "test1", 87 | "1": "test2", 88 | "2": "test3", 89 | }, 90 | } 91 | 92 | def test_detail(): 93 | response = client.get("/print_detail", params={ 94 | "args": "test1" 95 | }) 96 | assert response.status_code == 200 97 | assert response.json() == "test1test1" 98 | -------------------------------------------------------------------------------- /bmtools/tools/google_places/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("google_places") 4 | def google_places(): 5 | from .api import build_tool 6 | return build_tool -------------------------------------------------------------------------------- /bmtools/tools/google_places/api.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | from ..tool import Tool 4 | import os 5 | 6 | from typing import Any, Dict, List, Optional 7 | 8 | import googlemaps 9 | 10 | class GooglePlacesAPIWrapper: 11 | def __init__(self, subscription_key) -> None: 12 | self.gplaces_api_key: str = subscription_key 13 | self.google_map_client = googlemaps.Client(self.gplaces_api_key) 14 | self.top_k_results: Optional[int] = None 15 | 16 | def run(self, query: str) -> str: 17 | """Run Places search and get k number of places that exists that match.""" 18 | search_results = self.google_map_client.places(query)["results"] 19 | num_to_return = len(search_results) 20 | 21 | places = [] 22 | 23 | if num_to_return == 0: 24 | return "Google Places did not find any places that match the description" 25 | 26 | num_to_return = ( 27 | num_to_return 28 | if self.top_k_results is None 29 | else min(num_to_return, self.top_k_results) 30 | ) 31 | 32 | for i in range(num_to_return): 33 | result = search_results[i] 34 | details = self.fetch_place_details(result["place_id"]) 35 | 36 | if details is not None: 37 | places.append(details) 38 | 39 | return "\n".join([f"{i+1}. {item}" for i, item in enumerate(places)]) 40 | 41 | def fetch_place_details(self, place_id: str) -> Optional[str]: 42 | try: 43 | place_details = self.google_map_client.place(place_id) 44 | formatted_details = self.format_place_details(place_details) 45 | return formatted_details 46 | except Exception as e: 47 | logging.error(f"An Error occurred while fetching place details: {e}") 48 | return None 49 | 50 | def format_place_details(self, place_details: Dict[str, Any]) -> Optional[str]: 51 | try: 52 | name = place_details.get("result", {}).get("name", "Unkown") 53 | address = place_details.get("result", {}).get( 54 | "formatted_address", "Unknown" 55 | ) 56 | phone_number = place_details.get("result", {}).get( 57 | "formatted_phone_number", "Unknown" 58 | ) 59 | website = place_details.get("result", {}).get("website", "Unknown") 60 | 61 | formatted_details = ( 62 | f"{name}\nAddress: {address}\n" 63 | f"Phone: {phone_number}\nWebsite: {website}\n\n" 64 | ) 65 | return formatted_details 66 | except Exception as e: 67 | logging.error(f"An error occurred while formatting place details: {e}") 68 | return None 69 | 70 | 71 | def build_tool(config) -> Tool: 72 | tool = Tool( 73 | "google_places", 74 | "Look up for information about places and locations", 75 | name_for_model="google_places", 76 | description_for_model=( 77 | "A tool that query the Google Places API. " 78 | "Useful for when you need to validate or " 79 | "discover addressed from ambiguous text. " 80 | "Input should be a search query." 81 | ), 82 | logo_url="https://your-app-url.com/.well-known/logo.png", 83 | contact_email="hello@contact.com", 84 | legal_info_url="hello@legal.com" 85 | ) 86 | api_wrapper = GooglePlacesAPIWrapper(config["subscription_key"]) 87 | 88 | @tool.get("/search_places") 89 | def search_places(query : str): 90 | """Run Places search.""" 91 | return str(api_wrapper.run(query)) 92 | 93 | return tool -------------------------------------------------------------------------------- /bmtools/tools/google_places/readme.md: -------------------------------------------------------------------------------- 1 | # Google Places Queries 2 | 3 | Contributor: [Sihan Zhao](https://github.com/Sarah816) 4 | 5 | ## Tool Description 6 | The "Google Places" tool allows you to fetch information about places and locations by querying the Google Places API. This tool can be especially useful when you need to validate or discover addresses from ambiguous text. 7 | 8 | ### Tool Specifications 9 | 10 | - **Name**: Google Places 11 | - **Purpose**: Look up for information about places and locations 12 | - **Name for Model**: google_places 13 | - **Model Description**: A tool that query the Google Places API. Useful for when you need to validate or discover addressed from ambiguous text. Input should be a search query. 14 | - **Contact Email**: hello@contact.com 15 | - **Legal Information URL**: hello@legal.com 16 | 17 | ### Core Functionality 18 | 19 | 1. `search_places` 20 | 21 | This method accepts a string argument that contains a search query. It queries the Google Places API with the given input and returns information about the corresponding places and locations. 22 | 23 | The tool leverages a fictional wrapper class called `GooglePlacesAPIWrapper` to interact with the Google Places API and execute the required functionalities. 24 | 25 | **note**: The GooglePlacesAPIWrapper is a placeholder here and in a real-world scenario, this should be replaced with a properly implemented class that can fetch data from the Google Places API. The Google Places API itself requires an API key for access, which is not provided in this code. -------------------------------------------------------------------------------- /bmtools/tools/google_places/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'google_places', "http://127.0.0.1:8079/tools/google_places/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("Where is Tsinghua University?") 12 | agent("List a few locations of KFC in Beijing.") -------------------------------------------------------------------------------- /bmtools/tools/google_scholar/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("google_scholar") 5 | def google_scholar(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/google_scholar/readme.md: -------------------------------------------------------------------------------- 1 | # Google Scholar Service 2 | 3 | Contributor: [Kunlun Zhu](https://github.com/Kunlun-Zhu) 4 | 5 | Before you use this tool: pip install google-serp-api 6 | 7 | You can get your SerpAPI key here: https://serpapi.com/ -------------------------------------------------------------------------------- /bmtools/tools/google_scholar/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'Google_Scholar', "http://127.0.0.1:8079/tools/google_scholar/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("Search the profile of Jeffery Hinton?") -------------------------------------------------------------------------------- /bmtools/tools/google_serper/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("google_serper") 4 | def google_serper(): 5 | from .api import build_tool 6 | return build_tool -------------------------------------------------------------------------------- /bmtools/tools/google_serper/readme.md: -------------------------------------------------------------------------------- 1 | # Google Serper Queries 2 | 3 | Contributor: [Sihan Zhao](https://github.com/Sarah816) 4 | 5 | ## Tool Description 6 | The "google_serper" tool allows you to fetch information using the Serper.dev Google Search API. This is a low-cost Google Search API, highly useful when you need to answer questions about current events. The input should be a search query and the output is a JSON object of the query results. 7 | 8 | ### Tool Specification 9 | 10 | - **Name**: google_serper 11 | - **Purpose**: Fetch information using the Serper.dev Google Search API 12 | - **Model Name**: google_serper 13 | - **Model Description**: A tool for querying the Serper.dev Google Search API 14 | - **Logo URL**: [logo](https://your-app-url.com/.well-known/logo.png) 15 | - **Contact Email**: hello@contact.com 16 | - **Legal Information URL**: hello@legal.com 17 | 18 | ### Core Functionality 19 | 20 | 1. `search_general` 21 | 22 | This method runs the query through GoogleSearch and parses the result. The result is a JSON object of the query results. 23 | 24 | This tool uses a fictional `GoogleSerperAPIWrapper` class to interact with the Google Search API and perform the desired functionality. The actual implementation might need an API wrapper that can interact with the Google Search API. 25 | 26 | It's important to note that although the Google Search API wrapper used in this example is fictional, in reality, you would need to find an actual API that can perform Google searches and provide search results. As of my knowledge cutoff in September 2021, Google does not publicly provide its Search API, so you might need to explore alternative methods to retrieve this information while ensuring compliance with Google's terms of service. -------------------------------------------------------------------------------- /bmtools/tools/google_serper/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'google_serper', "http://127.0.0.1:8079/tools/google_serper/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("Where was the current US President born?") 12 | agent("List 5 well-known Chinese restaurants in New York and their location.") -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("gradio_tools") 4 | def gradio(): 5 | from .api import build_tool 6 | return build_tool -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/api.py: -------------------------------------------------------------------------------- 1 | from xml.dom.pulldom import SAX2DOM 2 | from ..tool import Tool 3 | from gradio_tools.tools import BarkTextToSpeechTool,StableDiffusionTool,DocQueryDocumentAnsweringTool,ImageCaptioningTool,StableDiffusionPromptGeneratorTool,TextToVideoTool,ImageToMusicTool,WhisperAudioTranscriptionTool,ClipInterrogatorTool 4 | from gradio_client.client import Job 5 | from gradio_client.utils import QueueError 6 | import time 7 | 8 | def build_tool(config) -> Tool: 9 | tool = Tool( 10 | "GradioTool", 11 | "Gradio_tools Converter", 12 | name_for_model="gradio_tool", 13 | description_for_model="Python library for converting Gradio apps into tools that can be leveraged by a large language model (LLM)-based agent to complete its task.", 14 | logo_url="https://your-app-url.com/.well-known/logo.png", 15 | contact_email="hello@contact.com", 16 | legal_info_url="hello@legal.com" 17 | ) 18 | @tool.get("/get_bark") 19 | def bark(input : str)-> str: 20 | '''Converting text into sounds that sound like a human read it. 21 | ''' 22 | bk = BarkTextToSpeechTool() 23 | return bk.run(input) 24 | @tool.get("/get_qa") 25 | def qa(input : str)-> str: 26 | '''Answering questions from the image of the document. 27 | ''' 28 | qa = DocQueryDocumentAnsweringTool() 29 | return qa.run(input) 30 | @tool.get("/get_imagecaption") 31 | def imagecaption(input : str)-> str: 32 | '''Creating a caption for an image. 33 | ''' 34 | ic = ImageCaptioningTool() 35 | return ic.run(input) 36 | @tool.get("/get_promptgenerator") 37 | def promptgenerator(input : str)-> str: 38 | '''Generating a prompt for stable diffusion and other image and video generators based on text input. 39 | ''' 40 | pg = StableDiffusionPromptGeneratorTool() 41 | return pg.run(input) 42 | @tool.get("/get_stablediffusion") 43 | def stablediffusion(input : str)-> str: 44 | '''generate images based on text input. 45 | ''' 46 | sd = StableDiffusionTool() 47 | return sd.run(input) 48 | @tool.get("/get_texttovideo") 49 | def texttovideo(input : str)-> str: 50 | '''Creating videos from text. 51 | ''' 52 | tv = TextToVideoTool() 53 | return tv.run(input) 54 | @tool.get("/get_imgtomsc") 55 | def imgtomsc(input : str)-> str: 56 | '''Creating music from images. 57 | ''' 58 | im = ImageToMusicTool() 59 | return im.run(input) 60 | @tool.get("/get_audiotrans") 61 | def imgtomsc(input : str)-> str: 62 | '''Transcribing an audio file track into text transcript. 63 | ''' 64 | at = WhisperAudioTranscriptionTool() 65 | return at.run(input) 66 | @tool.get("/get_imgprompt") 67 | def imgprompt(input : str)-> str: 68 | '''Creating a prompt for StableDiffusion that matches the input image. 69 | ''' 70 | ci = ClipInterrogatorTool() 71 | return ci.run(input) 72 | return tool -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/florida-drivers-license.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/tools/gradio_tools/florida-drivers-license.jpeg -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/readme.md: -------------------------------------------------------------------------------- 1 | # Gradio Tool 2 | `gradio_tools` provides a collection of pre-built tools that can be used immediately. These tools include: 3 | 1. get_stablediffusion - Generate an image from a given prompt using the open source stable diffusion demo hosted on [HuggingFace spaces](https://huggingface.co/spaces/stabilityai/stable-diffusion) 4 | 2. get_imagecaption - Caption an image by providing a filepath based on Niels Rogge's [HuggingFace Space](https://huggingface.co/spaces/nielsr/comparing-captioning-models) 5 | 3. get_imgtomsc - Create an audio clip that matches the style of a given image file based on Sylvain Filoni's [HuggingFace Space](https://huggingface.co/spaces/fffiloni/img-to-music) 6 | 4. get_promptgenerator - Use this tool to improve a prompt for stable diffusion and other image generators based on this [HuggingFace Space](https://huggingface.co/spaces/microsoft/Promptist) 7 | 5. get_texttovideo - A tool for creating short videos from text. Based on this [HuggingFace Space](https://huggingface.co/spaces/damo-vilab/modelscope-text-to-video-synthesis) 8 | 6. get_audiotrans - A tool for transcribing audio with Whisper. Based on this [HuggingFace Space](https://huggingface.co/spaces/abidlabs/whisper) 9 | 7. get_imgprompt - A tool for reverse engineering a prompt from a source image. Based on this [HuggingFace Space](https://huggingface.co/spaces/pharma/CLIP-Interrogator) 10 | 8. get_qa - A tool for answering questions about a document from the from the image of the document. Based on this [HuggingFace Space](https://huggingface.co/spaces/abidlabs/docquery) 11 | 9. get_bark - A tool for text-to-speech. Based on this [HuggingFace Space](https://huggingface.co/spaces/suno/bark) 12 | 13 | Contributor: [Junxi Yan](https://github.com/yanjx2021) 14 | 15 | ### Acknowledgments 16 | - [Gradio Tools](https://github.com/freddyaboulton/gradio-tools) -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | import pathlib 3 | # Langchain 4 | tool_name, tool_url = 'gradio_tools', "http://127.0.0.1:8079/tools/gradio_tools/" 5 | 6 | tool_name, tool_config = load_single_tools(tool_name, tool_url) 7 | 8 | print(tool_name, tool_config) 9 | stqa = STQuestionAnswerer() 10 | 11 | agent = stqa.load_tools(tool_name, tool_config) 12 | 13 | #test ToVideoTool 14 | agent("Create a video for a panda eating bamboo on a rock.") 15 | 16 | #test BarkTextToSpeechTool, StableDiffusionTool, StableDiffusionPromptGeneratorTool 17 | # agent("Please create a jingle for a spanish company called 'Chipi Chups' that makes lollipops. " 18 | # "The jingle should be catchy and playful and meant to appeal to all ages.") 19 | 20 | #test DocQueryDocumentAnsweringTool,ImageCaptioningTool 21 | IMG_PATH = pathlib.Path(__file__).parent / "florida-drivers-license.jpeg" 22 | agent(f"Captioning the picture in {IMG_PATH}?The Action in langchain should follow the format:'Action:API's name'.For example,'Action:get_imagecaption.'") 23 | # agent(f"Create music from picture in {IMG_PATH}?The Action in langchain should follow the format:'Action:API's name'.For example,'Action:get_imgtomsc.'") 24 | # agent(f"What is the date of birth the driver in {IMG_PATH}?The Action in langchain should follow the format:'Action:API's name'.For example,'Action:get_qa.'") -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/__init__.py: -------------------------------------------------------------------------------- 1 | from gradio_tools.tools.bark import BarkTextToSpeechTool 2 | from gradio_tools.tools.clip_interrogator import ClipInterrogatorTool 3 | from gradio_tools.tools.document_qa import DocQueryDocumentAnsweringTool 4 | from gradio_tools.tools.gradio_tool import GradioTool 5 | from gradio_tools.tools.image_captioning import ImageCaptioningTool 6 | from gradio_tools.tools.image_to_music import ImageToMusicTool 7 | from gradio_tools.tools.prompt_generator import \ 8 | StableDiffusionPromptGeneratorTool 9 | from gradio_tools.tools.stable_diffusion import StableDiffusionTool 10 | from gradio_tools.tools.text_to_video import TextToVideoTool 11 | from gradio_tools.tools.whisper import WhisperAudioTranscriptionTool 12 | from gradio_tools.tools.sam_with_clip import SAMImageSegmentationTool 13 | 14 | __all__ = [ 15 | "GradioTool", 16 | "StableDiffusionTool", 17 | "ClipInterrogatorTool", 18 | "ImageCaptioningTool", 19 | "ImageToMusicTool", 20 | "WhisperAudioTranscriptionTool", 21 | "StableDiffusionPromptGeneratorTool", 22 | "TextToVideoTool", 23 | "DocQueryDocumentAnsweringTool", 24 | "BarkTextToSpeechTool", 25 | "SAMImageSegmentationTool" 26 | ] 27 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/bark.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | from gradio_client.client import Job 6 | 7 | from gradio_tools.tools.gradio_tool import GradioTool 8 | 9 | if TYPE_CHECKING: 10 | import gradio as gr 11 | 12 | SUPPORTED_LANGS = [ 13 | ("English", "en"), 14 | ("German", "de"), 15 | ("Spanish", "es"), 16 | ("French", "fr"), 17 | ("Hindi", "hi"), 18 | ("Italian", "it"), 19 | ("Japanese", "ja"), 20 | ("Korean", "ko"), 21 | ("Polish", "pl"), 22 | ("Portuguese", "pt"), 23 | ("Russian", "ru"), 24 | ("Turkish", "tr"), 25 | ("Chinese", "zh"), 26 | ] 27 | 28 | SUPPORTED_LANGS = {lang: code for lang, code in SUPPORTED_LANGS} 29 | VOICES = ["Unconditional", "Announcer"] 30 | SUPPORTED_SPEAKERS = VOICES + [p for p in SUPPORTED_LANGS] 31 | 32 | NON_SPEECH_TOKENS = [ 33 | "[laughter]", 34 | "[laughs]", 35 | "[sighs]", 36 | "[music]", 37 | "[gasps]", 38 | "[clears throat]", 39 | "'♪' for song lyrics. Put ♪ on either side of the the text", 40 | "'…' for hesitations", 41 | ] 42 | 43 | 44 | class BarkTextToSpeechTool(GradioTool): 45 | """Tool for calling bark text-to-speech llm.""" 46 | 47 | def __init__( 48 | self, 49 | name="BarkTextToSpeech", 50 | description=( 51 | "A tool for text-to-speech. Use this tool to convert text " 52 | "into sounds that sound like a human read it. Input will be a two strings separated by a |: " 53 | "the first will be the text to read. The second will be the desired speaking language. " 54 | f"It MUST be one of the following choices {','.join(SUPPORTED_SPEAKERS)}. " 55 | f"Additionally, you can include the following non speech tokens: {NON_SPEECH_TOKENS}" 56 | "The output will the text transcript of that file." 57 | ), 58 | src="suno/bark", 59 | hf_token=None, 60 | duplicate=False, 61 | ) -> None: 62 | super().__init__(name, description, src, hf_token, duplicate) 63 | 64 | def create_job(self, query: str) -> Job: 65 | try: 66 | text, speaker = ( 67 | query[: query.rindex("|")], 68 | query[(query.rindex("|") + 1) :].strip(), 69 | ) 70 | except ValueError: 71 | text, speaker = query, "Unconditional" 72 | if speaker in VOICES: 73 | pass 74 | elif speaker in SUPPORTED_LANGS: 75 | speaker = f"Speaker 0 ({SUPPORTED_LANGS[speaker]})" 76 | else: 77 | speaker = "Unconditional" 78 | return self.client.submit(text, speaker, fn_index=3) 79 | 80 | def postprocess(self, output: str) -> str: 81 | return output 82 | 83 | def _block_input(self, gr) -> "gr.components.Component": 84 | return gr.Textbox() 85 | 86 | def _block_output(self, gr) -> "gr.components.Component": 87 | return gr.Audio() 88 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/clip_interrogator.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from gradio_client.client import Job 4 | 5 | from gradio_tools.tools.gradio_tool import GradioTool 6 | 7 | if TYPE_CHECKING: 8 | import gradio as gr 9 | 10 | 11 | class ClipInterrogatorTool(GradioTool): 12 | def __init__( 13 | self, 14 | name="ClipInterrogator", 15 | description=( 16 | "A tool for reverse engineering a prompt from a source image. " 17 | "Use this tool to create a prompt for StableDiffusion that matches the " 18 | "input image. The imput is a path to an image. The output is a text string." 19 | ), 20 | src="pharma/CLIP-Interrogator", 21 | hf_token=None, 22 | duplicate=True, 23 | ) -> None: 24 | super().__init__(name, description, src, hf_token, duplicate) 25 | 26 | def create_job(self, query: str) -> Job: 27 | return self.client.submit( 28 | query, "ViT-L (best for Stable Diffusion 1.*)", "best", fn_index=3 29 | ) 30 | 31 | def postprocess(self, output: str) -> str: 32 | return output 33 | 34 | def _block_input(self, gr) -> "gr.components.Component": 35 | return gr.Image() 36 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/document_qa.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from gradio_client.client import Job 4 | 5 | from gradio_tools.tools.gradio_tool import GradioTool 6 | 7 | if TYPE_CHECKING: 8 | import gradio as gr 9 | 10 | 11 | class DocQueryDocumentAnsweringTool(GradioTool): 12 | def __init__( 13 | self, 14 | name="DocQuery", 15 | description=( 16 | "A tool for answering questions about a document from the from the image of the document. Input will be a two strings separated by a comma: the first will be the path or URL to an image of a document. The second will be your question about the document." 17 | "The output will the text answer to your question." 18 | ), 19 | src="abidlabs/docquery", 20 | hf_token=None, 21 | duplicate=True, 22 | ) -> None: 23 | super().__init__(name, description, src, hf_token, duplicate) 24 | 25 | def create_job(self, query: str) -> Job: 26 | img, question = query.split(",") 27 | return self.client.submit(img.strip(), question.strip(), api_name="/predict") 28 | 29 | def postprocess(self, output: str) -> str: 30 | return output 31 | 32 | def _block_input(self, gr) -> "gr.components.Component": 33 | return gr.Image() 34 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/gradio_tool.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import time 4 | from abc import abstractmethod 5 | from typing import Any, Tuple, Union 6 | 7 | import gradio_client as grc 8 | import huggingface_hub 9 | from gradio_client.client import Job 10 | from gradio_client.utils import QueueError 11 | 12 | try: 13 | import langchain as lc 14 | 15 | LANGCHAIN_INSTALLED = True 16 | except (ModuleNotFoundError, ImportError): 17 | LANGCHAIN_INSTALLED = False 18 | 19 | 20 | class GradioTool: 21 | def __init__( 22 | self, 23 | name: str, 24 | description: str, 25 | src: str, 26 | hf_token: str | None = None, 27 | duplicate: bool = True, 28 | ) -> None: 29 | self.name = name 30 | self.description = description 31 | if hf_token and self._is_space(src) and duplicate: 32 | self.client = grc.Client.duplicate(from_id=src, hf_token=hf_token) 33 | self.src = self.client.space_id 34 | else: 35 | self.src = src 36 | self.client = grc.Client(self.src, hf_token=hf_token) 37 | self.description += ( 38 | "If the output is the string QUEUE_FULL you " 39 | "should try this action again and never stop until you " 40 | "don't see this message." 41 | ) 42 | self._block = None 43 | 44 | @staticmethod 45 | def _is_space(src: str) -> bool: 46 | try: 47 | huggingface_hub.get_space_runtime(src) 48 | return True 49 | except huggingface_hub.hf_api.RepositoryNotFoundError: 50 | return False 51 | 52 | @abstractmethod 53 | def create_job(self, query: str) -> Job: 54 | pass 55 | 56 | @abstractmethod 57 | def postprocess(self, output: Union[Tuple[Any], Any]) -> str: 58 | pass 59 | 60 | def run(self, query: str): 61 | job = self.create_job(query) 62 | while not job.done(): 63 | status = job.status() 64 | print(f"\nJob Status: {str(status.code)} eta: {status.eta}") 65 | time.sleep(30) 66 | try: 67 | output = self.postprocess(job.result()) 68 | except QueueError: 69 | output = "QUEUE_FULL" 70 | return output 71 | 72 | # Optional gradio functionalities 73 | def _block_input(self, gr) -> "gr.components.Component": 74 | return gr.Textbox() 75 | 76 | def _block_output(self, gr) -> "gr.components.Component": 77 | return gr.Textbox() 78 | 79 | def block_input(self) -> "gr.components.Component": 80 | try: 81 | import gradio as gr 82 | 83 | GRADIO_INSTALLED = True 84 | except (ModuleNotFoundError, ImportError): 85 | GRADIO_INSTALLED = False 86 | if not GRADIO_INSTALLED: 87 | raise ModuleNotFoundError("gradio must be installed to call block_input") 88 | else: 89 | return self._block_input(gr) 90 | 91 | def block_output(self) -> "gr.components.Component": 92 | try: 93 | import gradio as gr 94 | 95 | GRADIO_INSTALLED = True 96 | except (ModuleNotFoundError, ImportError): 97 | GRADIO_INSTALLED = False 98 | if not GRADIO_INSTALLED: 99 | raise ModuleNotFoundError("gradio must be installed to call block_output") 100 | else: 101 | return self._block_output(gr) 102 | 103 | def block(self): 104 | """Get the gradio Blocks of this tool for visualization.""" 105 | try: 106 | import gradio as gr 107 | except (ModuleNotFoundError, ImportError): 108 | raise ModuleNotFoundError("gradio must be installed to call block") 109 | if not self._block: 110 | self._block = gr.load(name=self.src, src="spaces") 111 | return self._block 112 | 113 | # Optional langchain functionalities 114 | @property 115 | def langchain(self) -> "langchain.agents.Tool": # type: ignore 116 | if not LANGCHAIN_INSTALLED: 117 | raise ModuleNotFoundError( 118 | "langchain must be installed to access langchain tool" 119 | ) 120 | 121 | return lc.agents.Tool( # type: ignore 122 | name=self.name, func=self.run, description=self.description 123 | ) 124 | 125 | def __repr__(self) -> str: 126 | return f"GradioTool(name={self.name}, src={self.src})" 127 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/image_captioning.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | from gradio_client.client import Job 6 | 7 | from gradio_tools.tools.gradio_tool import GradioTool 8 | 9 | if TYPE_CHECKING: 10 | import gradio as gr 11 | 12 | 13 | class ImageCaptioningTool(GradioTool): 14 | """Tool for captioning images.""" 15 | 16 | def __init__( 17 | self, 18 | name="ImageCaptioner", 19 | description=( 20 | "An image captioner. Use this to create a caption for an image. " 21 | "Input will be a path to an image file. " 22 | "The output will be a caption of that image." 23 | ), 24 | src="taesiri/BLIP-2", 25 | hf_token=None, 26 | duplicate=True, 27 | ) -> None: 28 | super().__init__(name, description, src, hf_token, duplicate) 29 | 30 | def create_job(self, query: str) -> Job: 31 | return self.client.submit(query.strip("'"), "Beam Search", fn_index=0) 32 | 33 | def postprocess(self, output: str) -> str: 34 | return output # type: ignore 35 | 36 | def _block_input(self, gr) -> "gr.components.Component": 37 | return gr.Image() 38 | 39 | def _block_output(self, gr) -> "gr.components.Component": 40 | return gr.Textbox() 41 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/image_to_music.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Any, Tuple, Union 2 | 3 | from gradio_client.client import Job 4 | 5 | from gradio_tools.tools.gradio_tool import GradioTool 6 | 7 | if TYPE_CHECKING: 8 | import gradio as gr 9 | 10 | 11 | class ImageToMusicTool(GradioTool): 12 | def __init__( 13 | self, 14 | name="ImagetoMusic", 15 | description=( 16 | "A tool for creating music from images. Use this tool to create a musical " 17 | "track from an image. Input will be a path to an image file. " 18 | "The output will be an audio file generated from that image." 19 | ), 20 | src="fffiloni/img-to-music", 21 | hf_token=None, 22 | duplicate=False, 23 | ) -> None: 24 | super().__init__(name, description, src, hf_token, duplicate) 25 | 26 | def create_job(self, query: str) -> Job: 27 | return self.client.submit( 28 | query.strip("'"), 15, "medium", "loop", None, fn_index=0 29 | ) 30 | 31 | def postprocess(self, output: Union[Tuple[Any], Any]) -> str: 32 | return output[1] # type: ignore 33 | 34 | def _block_input(self, gr) -> "gr.components.Component": 35 | return gr.Image() 36 | 37 | def _block_output(self, gr) -> "gr.components.Component": 38 | return gr.Audio() 39 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/prompt_generator.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from gradio_client.client import Job 4 | 5 | from gradio_tools.tools.gradio_tool import GradioTool 6 | 7 | 8 | class StableDiffusionPromptGeneratorTool(GradioTool): 9 | def __init__( 10 | self, 11 | name="StableDiffusionPromptGenerator", 12 | description=( 13 | "Use this tool to improve a prompt for stable diffusion and other image and video generators. " 14 | "This tool will refine your prompt to include key words and phrases that make " 15 | "stable diffusion and other art generation algorithms perform better. The input is a prompt text string " 16 | "and the output is a prompt text string" 17 | ), 18 | src="microsoft/Promptist", 19 | hf_token=None, 20 | duplicate=False, 21 | ) -> None: 22 | super().__init__(name, description, src, hf_token, duplicate) 23 | 24 | def create_job(self, query: str) -> Job: 25 | return self.client.submit(query, api_name="/predict") 26 | 27 | def postprocess(self, output: str) -> str: 28 | return output 29 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/sam_with_clip.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | from gradio_client.client import Job 6 | 7 | from gradio_tools.tools.gradio_tool import GradioTool 8 | 9 | if TYPE_CHECKING: 10 | import gradio as gr 11 | 12 | 13 | 14 | class SAMImageSegmentationTool(GradioTool): 15 | """Tool for segmenting images based on natural language queries.""" 16 | 17 | def __init__( 18 | self, 19 | name="SAMImageSegmentation", 20 | description=( 21 | "A tool for identifying objects in images. " 22 | "Input will be a five strings separated by a |: " 23 | "the first will be the full path or URL to an image file. " 24 | "The second will be the string query describing the objects to identify in the image. " 25 | "The query string should be as detailed as possible. " 26 | "The third will be the predicted_iou_threshold, if not specified by the user set it to 0.9. " 27 | "The fourth will be the stability_score_threshold, if not specified by the user set it to 0.8. " 28 | "The fifth is the clip_threshold, if not specified by the user set it to 0.85. " 29 | "The output will the a path with an image file with the identified objects overlayed in the image." 30 | ), 31 | src="curt-park/segment-anything-with-clip", 32 | hf_token=None, 33 | duplicate=False, 34 | ) -> None: 35 | super().__init__(name, description, src, hf_token, duplicate) 36 | 37 | def create_job(self, query: str) -> Job: 38 | try: 39 | image, query, predicted_iou_threshold, stability_score_threshold, clip_threshold = query.split("|") 40 | except ValueError as e: 41 | raise ValueError("Not enough arguments passed to the SAMImageSegmentationTool! " 42 | "Expected 5 (image, query, predicted_iou_threshold, stability_score_threshold, clip_threshold)") from e 43 | return self.client.submit(float(predicted_iou_threshold), 44 | float(stability_score_threshold), 45 | float(clip_threshold), 46 | image, query.strip(), api_name="/predict") 47 | 48 | def postprocess(self, output: str) -> str: 49 | return output 50 | 51 | def _block_input(self, gr) -> "gr.components.Component": 52 | return gr.Textbox() 53 | 54 | def _block_output(self, gr) -> "gr.components.Component": 55 | return gr.Audio() 56 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/stable_diffusion.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | from typing import TYPE_CHECKING, Any, Tuple 5 | 6 | from gradio_client.client import Job 7 | 8 | from gradio_tools.tools.gradio_tool import GradioTool 9 | 10 | if TYPE_CHECKING: 11 | import gradio as gr 12 | 13 | 14 | class StableDiffusionTool(GradioTool): 15 | """Tool for calling stable diffusion from llm""" 16 | 17 | def __init__( 18 | self, 19 | name="StableDiffusion", 20 | description=( 21 | "An image generator. Use this to generate images based on " 22 | "text input. Input should be a description of what the image should " 23 | "look like. The output will be a path to an image file." 24 | ), 25 | src="gradio-client-demos/stable-diffusion", 26 | hf_token=None, 27 | duplicate=False, 28 | ) -> None: 29 | super().__init__(name, description, src, hf_token, duplicate) 30 | 31 | def create_job(self, query: str) -> Job: 32 | return self.client.submit(query, "", 9, fn_index=1) 33 | 34 | def postprocess(self, output: Tuple[Any] | Any) -> str: 35 | assert isinstance(output, str) 36 | return [ 37 | os.path.join(output, i) 38 | for i in os.listdir(output) 39 | if not i.endswith("json") 40 | ][0] 41 | 42 | def _block_input(self, gr) -> "gr.components.Component": 43 | return gr.Textbox() 44 | 45 | def _block_output(self, gr) -> "gr.components.Component": 46 | return gr.Image() 47 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/text_to_video.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from gradio_client.client import Job 4 | 5 | from gradio_tools.tools.gradio_tool import GradioTool 6 | 7 | if TYPE_CHECKING: 8 | import gradio as gr 9 | 10 | 11 | class TextToVideoTool(GradioTool): 12 | def __init__( 13 | self, 14 | name="TextToVideo", 15 | description=( 16 | "A tool for creating videos from text." 17 | "Use this tool to create videos from text prompts. " 18 | "Input will be a text prompt describing a video scene. " 19 | "The output will be a path to a video file." 20 | ), 21 | src="damo-vilab/modelscope-text-to-video-synthesis", 22 | hf_token=None, 23 | duplicate=False, 24 | ) -> None: 25 | super().__init__(name, description, src, hf_token, duplicate) 26 | 27 | def create_job(self, query: str) -> Job: 28 | return self.client.submit(query, -1, 16, 25, fn_index=1) 29 | 30 | def postprocess(self, output: str) -> str: 31 | return output 32 | 33 | def _block_output(self, gr) -> "gr.components.Component": 34 | return gr.Video() 35 | -------------------------------------------------------------------------------- /bmtools/tools/gradio_tools/tools/whisper.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | from gradio_client.client import Job 6 | 7 | from gradio_tools.tools.gradio_tool import GradioTool 8 | 9 | if TYPE_CHECKING: 10 | import gradio as gr 11 | 12 | 13 | class WhisperAudioTranscriptionTool(GradioTool): 14 | def __init__( 15 | self, 16 | name="WhisperAudioTranscription", 17 | description=( 18 | "A tool for transcribing audio. Use this tool to transcribe an audio file. " 19 | "track from an image. Input will be a path to an audio file. " 20 | "The output will the text transcript of that file." 21 | ), 22 | src="abidlabs/whisper", 23 | hf_token=None, 24 | duplicate=False, 25 | ) -> None: 26 | super().__init__(name, description, src, hf_token, duplicate) 27 | 28 | def create_job(self, query: str) -> Job: 29 | return self.client.submit(query, api_name="/predict") 30 | 31 | def postprocess(self, output: str) -> str: 32 | return output 33 | 34 | def _block_input(self, gr) -> "gr.components.Component": 35 | return gr.Audio() 36 | -------------------------------------------------------------------------------- /bmtools/tools/hugging_tools/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("hugging_tools") 4 | def hugging_tools(): 5 | from .api import build_tool 6 | return build_tool 7 | -------------------------------------------------------------------------------- /bmtools/tools/hugging_tools/files/boat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/tools/hugging_tools/files/boat.png -------------------------------------------------------------------------------- /bmtools/tools/hugging_tools/files/doc.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/tools/hugging_tools/files/doc.jpg -------------------------------------------------------------------------------- /bmtools/tools/hugging_tools/files/test.flac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/tools/hugging_tools/files/test.flac -------------------------------------------------------------------------------- /bmtools/tools/hugging_tools/readme.md: -------------------------------------------------------------------------------- 1 | # Tutorial Service 2 | 3 | Contributor: [Shizuo Tian](https://github.com/BlitherBoom812) -------------------------------------------------------------------------------- /bmtools/tools/hugging_tools/sources/audio_models.txt: -------------------------------------------------------------------------------- 1 | tts: ['speechbrain/tts-tacotron2-ljspeech', 'facebook/fastspeech2-en-ljspeech', 'espnet/kan-bayashi_ljspeech_joint_finetune_conformer_fastspeech2_hifigan', 'facebook/fastspeech2-en-200_speaker-cv4', 'Voicemod/fastspeech2-en-200_speaker-cv4', 'espnet/english_male_ryanspeech_fastspeech2', 'Voicemod/fastspeech2-en-male1', 'espnet/kan-bayashi_ljspeech_vits', 'Voicemod/fastspeech2-en-ljspeech', 'espnet/kan-bayashi_ljspeech_tacotron2', 'mio/tokiwa_midori', 'mio/amadeus', 'mio/Artoria'] 2 | automatic-speech-recognition:['openai/whisper-large-v2', 'openai/whisper-base', 'jonatasgrosman/wav2vec2-large-xlsr-53-english', 'openai/whisper-medium', 'facebook/wav2vec2-base-960h', 'facebook/wav2vec2-large-960h-lv60-self', 'facebook/s2t-small-librispeech-asr', 'openai/whisper-tiny', 'openai/whisper-tiny.en'] 3 | audio-to-audio:['JorisCos/DCCRNet_Libri1Mix_enhsingle_16k', 'speechbrain/sepformer-wham', 'facebook/xm_transformer_unity_hk-en', 'facebook/xm_transformer_sm_all-en'] 4 | audio-classification: ['ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition', 'harshit345/xlsr-wav2vec-speech-emotion-recognition', 'speechbrain/lang-id-voxlingua107-ecapa', 'anton-l/sew-mid-100k-ft-keyword-spotting', 'anton-l/wav2vec2-base-lang-id', 'anton-l/wav2vec2-random-tiny-classifier', 'superb/wav2vec2-base-superb-er', 'superb/hubert-large-superb-er', 'superb/wav2vec2-base-superb-sid', 'speechbrain/spkrec-xvect-voxceleb', 'MIT/ast-finetuned-audioset-10-10-0.4593', 'audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim', 'TalTechNLP/voxlingua107-epaca-tdnn', 'speechbrain/lang-id-commonlanguage_ecapa', 'speechbrain/urbansound8k_ecapa', 'sahita/language-identification', 'bookbot/wav2vec2-adult-child-cls', 'anton-l/wav2vec2-base-ft-keyword-spotting', 'hackathon-pln-es/wav2vec2-base-finetuned-sentiment-mesd', 'superb/wav2vec2-base-superb-ks', 'superb/hubert-base-superb-er', 'hackathon-pln-es/wav2vec2-base-finetuned-sentiment-classification-MESD'] -------------------------------------------------------------------------------- /bmtools/tools/hugging_tools/sources/cv_models.txt: -------------------------------------------------------------------------------- 1 | visual-question-answering:['dandelin/vilt-b32-finetuned-vqa', 'azwierzc/vilt-b32-finetuned-vqa-pl', 'Bingsu/temp_vilt_vqa', 'tufa15nik/vilt-finetuned-vqasi'] 2 | document-question-answering:['impira/layoutlm-document-qa', 'tiennvcs/layoutlmv2-base-uncased-finetuned-infovqa', 'pardeepSF/layoutlm-vqa', 'jinhybr/OCR-DocVQA-Donut', 'impira/layoutlm-invoices', 'faisalraza/layoutlm-invoices', 'xhyi/layoutlmv3_docvqa_t11c5000', 'tiennvcs/layoutlmv2-large-uncased-finetuned-infovqa', 'tiennvcs/layoutlmv2-large-uncased-finetuned-vi-infovqa', 'tiennvcs/layoutlmv2-base-uncased-finetuned-vi-infovqa', 'naver-clova-ix/donut-base-finetuned-docvqa', 'tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa', 'MariaK/layoutlmv2-base-uncased_finetuned_docvqa_v2'] 3 | image-to-image: ['lambdalabs/sd-image-variations-diffusers'] 4 | text-to-image:['prompthero/openjourney-v4', 'hakurei/waifu-diffusion', 'gsdf/Counterfeit-V2.0', 'runwayml/stable-diffusion-v1-5', 'wavymulder/Analog-Diffusion', 'andite/pastel-mix', 'andite/anything-v4.0', 'gsdf/Counterfeit-V2.5', 'CompVis/stable-diffusion-v1-4', 'prompthero/openjourney', 'stabilityai/stable-diffusion-2', 'stabilityai/stable-diffusion-2-1', 'nitrosocke/Ghibli-Diffusion', 'nitrosocke/Arcane-Diffusion', 'nitrosocke/mo-di-diffusion', 'DGSpitzer/Cyberpunk-Anime-Diffusion', 'dreamlike-art/dreamlike-diffusion-1.0', 'riffusion/riffusion-model-v1', 'Linaqruf/anything-v3.0', 'Envvi/Inkpunk-Diffusion', 'hassanblend/hassanblend1.4', 'nitrosocke/redshift-diffusion'] 5 | image-segmentation:['nvidia/segformer-b5-finetuned-ade-640-640', 'nvidia/segformer-b1-finetuned-cityscapes-1024-1024', 'facebook/detr-resnet-50-panoptic', 'facebook/maskformer-swin-base-coco', 'nvidia/segformer-b0-finetuned-ade-512-512'] 6 | object-detection:['facebook/detr-resnet-50', 'hustvl/yolos-tiny'] 7 | image-classification: ['google/vit-base-patch16-224', 'facebook/deit-base-distilled-patch16-224', 'microsoft/dit-base-finetuned-rvlcdip'] 8 | image-to-text: ['nlpconnect/vit-gpt2-image-captioning', 'ydshieh/vit-gpt2-coco-en', 'microsoft/trocr-small-handwritten', 'Salesforce/blip-image-captioning-large', 'microsoft/trocr-small-printed', 'microsoft/trocr-large-printed', 'kha-white/manga-ocr-base', 'microsoft/trocr-base-handwritten', 'microsoft/trocr-base-printed', 'microsoft/trocr-large-handwritten', 'naver-clova-ix/donut-base'] -------------------------------------------------------------------------------- /bmtools/tools/hugging_tools/test.py: -------------------------------------------------------------------------------- 1 | 2 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 3 | 4 | tool_name, tool_url = 'hugging_tools', "http://127.0.0.1:8079/tools/hugging_tools/" 5 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 6 | print(tools_name, tools_config) 7 | 8 | qa = STQuestionAnswerer() 9 | print(tools_config) 10 | agent = qa.load_tools(tools_name, tools_config) 11 | 12 | # # NLP task 13 | # # Translation 14 | # answer = agent("Please translate this into English: Bonjour!") 15 | # print(answer) 16 | 17 | # # Sentence similarity 18 | # answer = agent("Which of these two sentences has a similar meaning to the other? Sentence 1: The cat sat on the mat. Sentence 2: The feline rested on the floor.") 19 | # print(answer) 20 | 21 | # # Question answering 22 | # answer = agent("What is the capital of France?") 23 | # print(answer) 24 | 25 | # # Text classification 26 | # answer = agent("Which category does this text belong to? This book is a thrilling mystery novel.") 27 | # print(answer) 28 | 29 | # # Token classification 30 | # answer = agent("Which words in this sentence are key words? The quick brown fox jumped over the lazy dog.") 31 | # print(answer) 32 | 33 | # # Text2Text generation 34 | # answer = agent("Please generate a paragraph about the benefits of exercise using the following prompt: 'Regular exercise can help improve'") 35 | # print(answer) 36 | 37 | # # Summarization 38 | # answer = agent("""Please provide a summary of the article with the following passage: 39 | 40 | # China, officially known as the People's Republic of China, is one of the largest countries in the world. It is located in East Asia, bordered by 14 countries and the East China Sea, Yellow Sea, South China Sea, and Taiwan Strait. With a population of over 1.4 billion people, China is the most populous country in the world. 41 | 42 | # """) 43 | # print(answer) 44 | 45 | # # Conversational 46 | # answer = agent("Can you generate a response to the following prompt in a conversational manner? 'What do you think about the weather today?'") 47 | # print(answer) 48 | 49 | # # Text generation 50 | # answer = agent("Please generate a paragraph of text about artificial intelligence using the following prompt: 'Artificial intelligence is'") 51 | # print(answer) 52 | 53 | # # Audio task 54 | # # automatic speech recognition 55 | # answer = agent("What does the man say in the audio `test.flac`?") 56 | # print(answer) 57 | 58 | # # audio to audio 59 | # answer = agent("Enhance this speech `test.flac` more clearly.") 60 | # print(answer) 61 | 62 | # # text to speech & audio to audio 63 | # answer = agent("Generate a audio where a man says: hello, world!, and then enchance the generated audio.") 64 | # print(answer) 65 | 66 | # # audio classification 67 | # answer = agent("classify this audio: `test.flac`") 68 | # print(answer) 69 | 70 | # # CV task 71 | # # text-to-image 72 | # answer = agent("Please generate a picture with prompt: a cat.") 73 | # print(answer) 74 | 75 | # # image-to-text 76 | # answer = agent("What does the picture `boat.png` shows?") 77 | # print(answer) 78 | 79 | # # image-segmentation 80 | # answer = agent("Please divide the `boat.png` into proper segments using appropriate models.") 81 | # print(answer) 82 | 83 | # # object-detection 84 | # answer = agent("Detect the objects in the picture `boat.png`.") 85 | # print(answer) 86 | 87 | # # image-classification 88 | # answer = agent("Classify the picture `boat.png`.") 89 | # print(answer) 90 | 91 | # # visual-question-answering 92 | # answer = agent("Answer the question according to the photo `boat.png`: what is it?") 93 | # print(answer) 94 | 95 | # # document-question-answering 96 | # answer = agent("Answer the question based on the content of the document screenshot `doc.jpg`: What is the topic of this document?(hint: use the document-question-answering)") 97 | # print(answer) -------------------------------------------------------------------------------- /bmtools/tools/image_generation/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("image_generation") 4 | def image_generation(): 5 | from .api import build_tool 6 | return build_tool -------------------------------------------------------------------------------- /bmtools/tools/image_generation/readme.md: -------------------------------------------------------------------------------- 1 | # Image Generation 2 | 3 | Contributor: [Sihan Zhao](https://github.com/Sarah816) 4 | 5 | # Image Generator Tool 6 | 7 | This Markdown document provides an introduction to the `Image Generator` function implemented in the given code. The function is designed to generate an image based on a text description using the Steamship API. 8 | 9 | ## Function Summary 10 | 11 | The `Image Generator` function is a tool that generates an image based on a text description. It utilizes the Steamship API to perform the image generation process. 12 | 13 | ## Usage 14 | 15 | To use the `Image Generator` function, follow the steps below: 16 | 17 | 1. Set up the required configuration parameters. 18 | 2. Ensure that the `STEAMSHIP_API_KEY` environment variable is set. 19 | 3. Call the `generate_image` endpoint with a text query to generate an image. 20 | 4. The function returns the UUID (Universally Unique Identifier) of the generated image. 21 | 22 | ## Configuration 23 | 24 | The `Image Generator` tool accepts the following configuration parameters: 25 | 26 | - **Model Name**: Specifies the model to use for image generation. The default model is DALL-E. 27 | - **Size**: Specifies the desired size of the generated image. The default size is 512x512. 28 | - **Return URLs**: Determines whether to return public URLs for the generated images. By default, this option is disabled. 29 | - **Steamship API Key**: The API key required to authenticate and use the Steamship API. It should be set as the `STEAMSHIP_API_KEY` environment variable. 30 | 31 | ## Endpoint 32 | 33 | The `Image Generator` tool provides the following endpoint: 34 | 35 | ### `/generate_image` 36 | 37 | - **Method**: GET 38 | - **Parameters**: 39 | - `query`: The detailed text-2-image prompt describing the image to be generated. 40 | - **Returns**: 41 | - The UUID of the generated image. 42 | 43 | ## Error Handling 44 | 45 | If the `STEAMSHIP_API_KEY` is not provided or is empty, a `RuntimeError` is raised, indicating that the API key should be obtained from the Steamship website and added to the environment variables. 46 | 47 | If the tool is unable to generate an image, a `RuntimeError` is raised. 48 | -------------------------------------------------------------------------------- /bmtools/tools/image_generation/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'image_generation', "http://127.0.0.1:8079/tools/image_generation/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("How would you visualize a girl dancing ballet?") -------------------------------------------------------------------------------- /bmtools/tools/job_search/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("job_search") 5 | def job_search(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/job_search/readme.md: -------------------------------------------------------------------------------- 1 | # Job Search 2 | 3 | Contributor: [Kunlun Zhu](https://github.com/Kunlun-Zhu) 4 | 5 | You can get your RAIPID key here: https://rapidapi.com/hub 6 | 7 | You ought to subscribe 'Zillow API' in your account to use this tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/job_search/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'JobSearch', "http://127.0.0.1:8079/tools/job_search/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("List some jobs about python development in Santa Monica, CA.") -------------------------------------------------------------------------------- /bmtools/tools/kg/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("wikidata") 5 | def wikidata(): 6 | from .wikidata import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/kg/wikidata/__init__.py: -------------------------------------------------------------------------------- 1 | from .api import build_tool 2 | -------------------------------------------------------------------------------- /bmtools/tools/kg/wikidata/ai-plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema_version": "v1", 3 | "name_for_model": "Search in Wikidata", 4 | "name_for_human": "Search in Wikidata", 5 | "description_for_model": "Plugin for answering factual questions in wikidata.", 6 | "description_for_human": "answering factual questions in wikidata.", 7 | "auth": { 8 | "type": "user_http", 9 | "authorization_type": "bearer" 10 | }, 11 | "api": { 12 | "type": "local", 13 | "url": "https://your-app-url.com/.well-known/openapi.yaml", 14 | "has_user_authentication": false 15 | }, 16 | "author_github": "", 17 | "logo_url": "https://your-app-url.com/.well-known/logo.png", 18 | "contact_email": "hello@contact.com", 19 | "legal_info_url": "hello@legal.com" 20 | } 21 | -------------------------------------------------------------------------------- /bmtools/tools/kg/wikidata/examples.json: -------------------------------------------------------------------------------- 1 | { 2 | "examples": [ 3 | { 4 | "role": "user", 5 | "content": "\"question\": \"What is the OSM tag or key of family medicine?\",\"background\": \" {\"OSM TAG\": \"P1282\",\"family madicine\": \"Q3505712\"}\" \"query\": \"SELECT ?osm WHERE {\n wd:Q3505712 wdt:P1282 ?osm.\n}\"" 6 | }, 7 | { 8 | "role": "user", 9 | "content": "\"question\": \"Who was nominated for Academy Award for Best Supporting Actor for A History of Violence?\",\"background\": \" {\"A History of Violence\": \"Q300439\", \"nominated for\": \"P1411\", \"Academy Award for Best Supporting Actor\": \"Q106291\", \"cast member\": \"P161\"}\" \"query\": \"SELECT ?actor WHERE{\n wd:Q300439 wdt:P161 ?actor.\n ?actor wdt:P1411 wd:Q106291.\n}\"" 10 | }, 11 | { 12 | "role": "user", 13 | "content": "\"question\": \"Which person who weighs less than 75 kilograms has the sport number 99?\",\"background\": \" {\"sports number\": \"P1618\", \"mass\": \"P2067\"}\" \"query\": \"SELECT ?person ?personLabel WHERE{\n ?person wdt:P2067 ?weight.\n ?person wdt:P1618 \"99\".\n FILTER(?weight < 75).\n SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE], en\". }\n}\"" 14 | }, 15 | { 16 | "role": "user", 17 | "content": "\"question\": \"How many Alabama counties have the captital of Huntsville and have the licence plate code of 39?\",\"background\": \" {\"Alabama\": \"Q173\", \"Huntsville\": \"Q79860\", \"contains the administrative territorial entity\": \"P150\", \"licence plate code\": \"P395\", \"capital of\": \"P1376\"}\" \"query\": \"SELECT ?county_sentence\nWHERE {\n wd:Q173 pq:P150 ?county_sentence.\n ?county_sentence pq:P395 \"39\".\n wd:Q79860 pq:P1376 ?county_sentence.\n}\n}\"" 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /bmtools/tools/kg/wikidata/readme.md: -------------------------------------------------------------------------------- 1 | # Wikidata Query 2 | 3 | Contributor: [Yuzhang Zhu](https://github.com/Zhu-Yuzhang) 4 | 5 | # Search in Wikidata Tool 6 | 7 | This Markdown document provides an introduction to the `Search in Wikidata` function implemented in the given code. The function allows users to search for factual information in Wikidata by querying entities, relations, and performing SPARQL queries. 8 | 9 | ## Function Summary 10 | 11 | The `Search in Wikidata` function is a tool that enables users to search for factual information in Wikidata. It provides functionality for finding entities, retrieving entity details, searching for relations, and performing SPARQL queries. 12 | 13 | ## Usage 14 | 15 | To use the `Search in Wikidata` function, follow the steps below: 16 | 17 | 1. Call the desired endpoint with appropriate parameters to perform the specific search or query operation. 18 | 2. The function returns a markdown-formatted table containing the results. 19 | 20 | ## Endpoints 21 | 22 | The `Search in Wikidata` tool provides the following endpoints: 23 | 24 | ### `/find_entity` 25 | 26 | - **Method**: GET 27 | - **Parameters**: 28 | - `input`: The input entity to find relations and properties for. 29 | 30 | ### `/find_entity_by_tail` 31 | 32 | - **Method**: GET 33 | - **Parameters**: 34 | - `input`: The input tail to find head entities and relations for. 35 | 36 | ### `/get_entity_id` 37 | 38 | - **Method**: GET 39 | - **Parameters**: 40 | - `input`: The surface form to search for entities. 41 | 42 | ### `/get_relation_id` 43 | 44 | - **Method**: GET 45 | - **Parameters**: 46 | - `input`: The surface form to search for relations. 47 | 48 | ### `/search_by_code` 49 | 50 | - **Method**: GET 51 | - **Parameters**: 52 | - `query`: The SPARQL query to perform. 53 | 54 | ## Error Handling 55 | 56 | If any invalid options or exceptions occur during the execution of the function, appropriate error messages are printed. 57 | -------------------------------------------------------------------------------- /bmtools/tools/kg/wikidata/test.py: -------------------------------------------------------------------------------- 1 | 2 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 3 | 4 | tool_name, tool_url = 'wikidata', "http://127.0.0.1:8079/tools/wikidata/" 5 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 6 | print(tools_name, tools_config) 7 | 8 | qa = STQuestionAnswerer() 9 | 10 | agent = qa.load_tools(tools_name, tools_config) 11 | 12 | agent("") -------------------------------------------------------------------------------- /bmtools/tools/map/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("bing_map") 5 | def bing_map(): 6 | from .bing_map.api import build_tool 7 | return build_tool 8 | 9 | @register("baidu_map") 10 | def baidu_map(): 11 | from .baidu_map.baidu_api import build_tool 12 | return build_tool 13 | -------------------------------------------------------------------------------- /bmtools/tools/map/baidu_map/readme.md: -------------------------------------------------------------------------------- 1 | # Map Service 2 | 3 | Contributor: [Kunlun Zhu](https://github.com/Kunlun-Zhu) 4 | 5 | To Get Baidu API Key [BAIDU MAP API](https://lbsyun.baidu.com/apiconsole/key?application=key#/home) 6 | After You Get Baidu API Key, chose 'SN校验方式' and obtain your SK from the setting page 7 | 8 | # Map Info Tool 9 | 10 | This Markdown document provides an introduction to the `Map Info` function implemented in the given code. The function allows users to look up map information in China using the Baidu Map API. It provides functionality for retrieving location coordinates, addresses by coordinates, nearby places, routes between locations, and distances between locations. 11 | 12 | ## Function Summary 13 | 14 | The `Map Info` function is a tool that enables users to look up map information in China. It utilizes the Baidu Map API to provide various features related to locations, addresses, nearby places, routes, and distances. 15 | 16 | ## Usage 17 | 18 | To use the `Map Info` function, follow the steps below: 19 | 20 | 1. Set up the required configuration parameters, including the `subscription_key` and `baidu_secret_key`. 21 | 2. Call the desired endpoint with appropriate parameters to perform the specific map-related operation. 22 | 3. The function returns the requested map information or relevant data. 23 | 24 | ## Endpoints 25 | 26 | The `Map Info` tool provides the following endpoints: 27 | 28 | ### `/get_location` 29 | 30 | - **Method**: GET 31 | - **Parameters**: 32 | - `address`: The address to retrieve location coordinates for. 33 | - **Returns**: The coordinates (latitude, longitude) of the specified address, or None if the coordinates cannot be obtained. 34 | 35 | ### `/get_address_by_coordinates` 36 | 37 | - **Method**: GET 38 | - **Parameters**: 39 | - `lat`: The latitude coordinate. 40 | - `lng`: The longitude coordinate. 41 | - **Returns**: The location name corresponding to the given coordinates, or None if the address cannot be obtained. 42 | 43 | ### `/get_nearby_places` 44 | 45 | - **Method**: GET 46 | - **Parameters**: 47 | - `location`: The location coordinates (latitude, longitude) to search nearby places for. 48 | - `radius`: The search radius in meters. 49 | - `keyword`: The keyword to filter the search results (default: '餐厅' for restaurants). 50 | - **Returns**: A list of nearby place names based on the specified location and search parameters. 51 | 52 | ### `/get_route` 53 | 54 | - **Method**: GET 55 | - **Parameters**: 56 | - `origin`: The origin address of the route. 57 | - `destination`: The destination address of the route. 58 | - **Returns**: A list of route descriptions for the specified origin and destination, or None if the route cannot be obtained. 59 | 60 | ### `/get_distance` 61 | 62 | - **Method**: GET 63 | - **Parameters**: 64 | - `origin`: The origin address. 65 | - `destination`: The destination address. 66 | - **Returns**: The distance in meters between the specified origin and destination. 67 | 68 | ## Error Handling 69 | 70 | If any invalid options or exceptions occur during the execution of the function, appropriate error messages will be returned. 71 | -------------------------------------------------------------------------------- /bmtools/tools/map/bing_map/api.py: -------------------------------------------------------------------------------- 1 | 2 | import requests 3 | import os 4 | import json 5 | from ...tool import Tool 6 | 7 | def build_tool(config) -> Tool: 8 | tool = Tool( 9 | "Map Info form bing map api", 10 | "Look up map information", 11 | name_for_model="BingMap", 12 | description_for_model="Plugin for look up map information", 13 | logo_url="https://your-app-url.com/.well-known/logo.png", 14 | contact_email="hello@contact.com", 15 | legal_info_url="hello@legal.com" 16 | ) 17 | 18 | KEY = config["subscription_key"] 19 | BASE_URL = 'http://dev.virtualearth.net/REST/V1/' 20 | 21 | @tool.get("/get_distance") 22 | def get_distance(start:str, end:str): 23 | """Get the distance between two locations in miles""" 24 | # Request URL 25 | url = BASE_URL + "Routes/Driving?o=json&wp.0=" + start + "&wp.1=" + end + "&key=" + KEY 26 | # GET request 27 | r = requests.get(url) 28 | data = json.loads(r.text) 29 | # Extract route information 30 | route = data["resourceSets"][0]["resources"][0] 31 | # Extract distance in miles 32 | distance = route["travelDistance"] 33 | return distance 34 | 35 | @tool.get("/get_route") 36 | def get_route(start:str, end:str): 37 | """Get the route between two locations in miles""" 38 | # Request URL 39 | url = BASE_URL + "Routes/Driving?o=json&wp.0=" + start + "&wp.1=" + end + "&key=" + KEY 40 | # GET request 41 | r = requests.get(url) 42 | data = json.loads(r.text) 43 | # Extract route information 44 | route = data["resourceSets"][0]["resources"][0] 45 | itinerary = route["routeLegs"][0]["itineraryItems"] 46 | # Extract route text information 47 | route_text = [] 48 | for item in itinerary: 49 | if "instruction" in item: 50 | route_text.append(item["instruction"]["text"]) 51 | return route_text 52 | 53 | @tool.get("/get_coordinates") 54 | def get_coordinates(location:str): 55 | """Get the coordinates of a location""" 56 | url = BASE_URL + "Locations" 57 | params = { 58 | "query": location, 59 | "key": KEY 60 | } 61 | response = requests.get(url, params=params) 62 | json_data = response.json() 63 | coordinates = json_data["resourceSets"][0]["resources"][0]["point"]["coordinates"] 64 | return coordinates 65 | 66 | @tool.get("/search_nearby") 67 | def search_nearyby(search_term:str="restaurant", latitude:float = 0.0, longitude:float = 0.0, places:str='unknown', radius:int = 5000): # radius in meters 68 | """Search for places nearby a location, within a given radius, and return the results into a list""" 69 | url = BASE_URL + "LocalSearch" 70 | if places != 'unknown': 71 | latitude = get_coordinates(places)[0] 72 | longitude = get_coordinates(places)[1] 73 | # Build the request query string 74 | params = { 75 | "query": search_term, 76 | "userLocation": f"{latitude},{longitude}", 77 | "radius": radius, 78 | "key": KEY 79 | } 80 | 81 | # Make the request 82 | response = requests.get(url, params=params) 83 | 84 | # Parse the response 85 | response_data = json.loads(response.content) 86 | 87 | # Get the results 88 | results = response_data["resourceSets"][0]["resources"] 89 | addresses = [] 90 | for result in results: 91 | name = result["name"] 92 | address = result["Address"]["formattedAddress"] 93 | addresses.append(f"{name}: {address}") 94 | return addresses 95 | 96 | return tool 97 | -------------------------------------------------------------------------------- /bmtools/tools/map/bing_map/readme .md: -------------------------------------------------------------------------------- 1 | # Map Service 2 | 3 | Contributor: [Kunlun Zhu](https://github.com/Kunlun-Zhu) 4 | 5 | # Map Info from Bing Map API 6 | 7 | This Markdown document provides an introduction to the `Map Info` function implemented in the given code. The function allows users to look up map information using the Bing Map API. It provides functionality for retrieving distances between locations, routes between locations, coordinates of a location, and searching for nearby places. 8 | 9 | ## Function Summary 10 | 11 | The `Map Info` function is a tool that enables users to look up map information using the Bing Map API. It provides features such as retrieving distances between locations, routes between locations, coordinates of a location, and searching for nearby places. 12 | 13 | ## Usage 14 | 15 | To use the `Map Info` function, follow the steps below: 16 | 17 | 1. Set up the required configuration parameters, including the `subscription_key`. 18 | 2. Call the desired endpoint with appropriate parameters to perform the specific map-related operation. 19 | 3. The function returns the requested map information or relevant data. 20 | 21 | ## Endpoints 22 | 23 | The `Map Info` tool provides the following endpoints: 24 | 25 | ### `/get_distance` 26 | 27 | - **Method**: GET 28 | - **Parameters**: 29 | - `start`: The starting location address or coordinates. 30 | - `end`: The destination location address or coordinates. 31 | - **Returns**: The distance in miles between the specified start and end locations. 32 | 33 | ### `/get_route` 34 | 35 | - **Method**: GET 36 | - **Parameters**: 37 | - `start`: The starting location address or coordinates. 38 | - `end`: The destination location address or coordinates. 39 | - **Returns**: A list of route instructions for the specified start and end locations. 40 | 41 | ### `/get_coordinates` 42 | 43 | - **Method**: GET 44 | - **Parameters**: 45 | - `location`: The location to retrieve coordinates for. 46 | - **Returns**: The coordinates (latitude, longitude) of the specified location. 47 | 48 | ### `/search_nearby` 49 | 50 | - **Method**: GET 51 | - **Parameters**: 52 | - `search_term`: The keyword to search for nearby places (default: "restaurant"). 53 | - `latitude`: The latitude coordinate of the location (default: 0.0). 54 | - `longitude`: The longitude coordinate of the location (default: 0.0). 55 | - `places`: The location to search nearby places for. If specified, latitude and longitude will be automatically retrieved (default: 'unknown'). 56 | - `radius`: The search radius in meters (default: 5000). 57 | - **Returns**: A list of nearby places based on the specified parameters. 58 | 59 | ## Error Handling 60 | 61 | If any invalid options or exceptions occur during the execution of the function, appropriate error messages will be returned. 62 | -------------------------------------------------------------------------------- /bmtools/tools/map/bing_map/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'Map', "http://127.0.0.1:8079/tools/map/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("What's the driving distance from Beijing to Shanghai?") -------------------------------------------------------------------------------- /bmtools/tools/meta_analysis/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("meta_analysis") 4 | def meta_analysis(): 5 | from .api import build_tool 6 | return build_tool 7 | 8 | -------------------------------------------------------------------------------- /bmtools/tools/meta_analysis/readme.md: -------------------------------------------------------------------------------- 1 | # Meta Analysis 2 | 3 | Contributor: [Zheni Zeng](https://github.com/Ellenzzn) 4 | 5 | 6 | 7 | A tool for searching and analyzing literatures 8 | 9 | -------------------------------------------------------------------------------- /bmtools/tools/meta_analysis/test.py: -------------------------------------------------------------------------------- 1 | 2 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 3 | 4 | tool_name, tool_url = 'meta_analysis', "http://127.0.0.1:8079/tools/meta_analysis/" 5 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 6 | print(tools_name, tools_config) 7 | 8 | qa = STQuestionAnswerer() 9 | 10 | agent = qa.load_tools(tools_name, tools_config) 11 | 12 | agent("How's the treatment for COVID-19 of Lianhua Qingwen? Answer this question by analyzing literatures that meet this criteria: trials are conducted for comparing Lianhua Qingwen with other conventional treatments for Coronavirus disease. Patients are adults with COVID-19 diagnosis.") 13 | agent("Recommend some literatures about trials of treating COVID-19 with Lianhua Qingwen for me. Print their title/abstract. ") 14 | agent.run(['Help me analyze trials of Lianhua Qingwen for treating COVID-19, and Paxlovid for treating COVID-19 separately, and then compare them.']) 15 | -------------------------------------------------------------------------------- /bmtools/tools/office/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("office-ppt") 4 | def office_ppt(): 5 | from .ppt import build_tool 6 | return build_tool 7 | -------------------------------------------------------------------------------- /bmtools/tools/office/ppt/__init__.py: -------------------------------------------------------------------------------- 1 | from .api import build_tool -------------------------------------------------------------------------------- /bmtools/tools/office/ppt/readme.md: -------------------------------------------------------------------------------- 1 | # Office slides 2 | 3 | Contributor: [Bokai Xu](https://github.com/bokesyo) -------------------------------------------------------------------------------- /bmtools/tools/office/ppt/templates/flat.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/tools/office/ppt/templates/flat.pptx -------------------------------------------------------------------------------- /bmtools/tools/office/ppt/templates/green.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/tools/office/ppt/templates/green.pptx -------------------------------------------------------------------------------- /bmtools/tools/office/ppt/templates/orange.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/tools/office/ppt/templates/orange.pptx -------------------------------------------------------------------------------- /bmtools/tools/office/ppt/templates/tech.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/tools/office/ppt/templates/tech.pptx -------------------------------------------------------------------------------- /bmtools/tools/office/ppt/templates/wooden.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/tools/office/ppt/templates/wooden.pptx -------------------------------------------------------------------------------- /bmtools/tools/python/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("python") 4 | def python(): 5 | from .api import build_tool 6 | return build_tool 7 | -------------------------------------------------------------------------------- /bmtools/tools/python/api.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | from ..tool import Tool 4 | import os 5 | 6 | import sys 7 | from io import StringIO 8 | from typing import Dict, Optional 9 | 10 | 11 | class PythonREPL: 12 | """Simulates a standalone Python REPL.""" 13 | def __init__(self) -> None: 14 | self.globals: Optional[Dict] = globals() 15 | self.locals: Optional[Dict] = None 16 | 17 | def run(self, command: str) -> str: 18 | """Run command with own globals/locals and returns anything printed.""" 19 | old_stdout = sys.stdout 20 | sys.stdout = mystdout = StringIO() 21 | try: 22 | exec(command, self.globals, self.locals) 23 | sys.stdout = old_stdout 24 | output = mystdout.getvalue() 25 | except Exception as e: 26 | sys.stdout = old_stdout 27 | output = repr(e) 28 | print(output) 29 | return output 30 | 31 | def build_tool(config) -> Tool: 32 | tool = Tool( 33 | "Python REPL", 34 | "Run python code", 35 | name_for_model="Python REPL", 36 | description_for_model=( 37 | "A Python shell. Use this to execute python commands. " 38 | "Input should be a valid python command. " 39 | "If you want to see the output of a value, you should print it out " 40 | "with `print(...)`." 41 | ), 42 | logo_url="https://your-app-url.com/.well-known/logo.png", 43 | contact_email="hello@contact.com", 44 | legal_info_url="hello@legal.com" 45 | ) 46 | 47 | python_repl = PythonREPL() 48 | sanitize_input: bool = True 49 | 50 | 51 | @tool.get("/run_python") 52 | def run_python(query : str): 53 | '''Run python code in a REPL. 54 | ''' 55 | if sanitize_input: 56 | query = query.strip().strip("```") 57 | return python_repl.run(query) 58 | 59 | 60 | return tool 61 | -------------------------------------------------------------------------------- /bmtools/tools/python/readme.md: -------------------------------------------------------------------------------- 1 | # Python commands 2 | 3 | Contributor: [Sihan Zhao](https://github.com/Sarah816) -------------------------------------------------------------------------------- /bmtools/tools/python/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'python', "http://127.0.0.1:8079/tools/python/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("What is the 10th fibonacci number?") -------------------------------------------------------------------------------- /bmtools/tools/registry.py: -------------------------------------------------------------------------------- 1 | from .tool import Tool 2 | from typing import Dict, Callable, Any, List 3 | 4 | ToolBuilder = Callable[[Any], Tool] 5 | FuncToolBuilder = Callable[[], ToolBuilder] 6 | 7 | 8 | class ToolsRegistry: 9 | def __init__(self) -> None: 10 | self.tools : Dict[str, FuncToolBuilder] = {} 11 | 12 | def register(self, tool_name : str, tool : FuncToolBuilder): 13 | print(f"will register {tool_name}") 14 | self.tools[tool_name] = tool 15 | 16 | def build(self, tool_name, config) -> Tool: 17 | ret = self.tools[tool_name]()(config) 18 | if isinstance(ret, Tool): 19 | return ret 20 | raise ValueError("Tool builder {} did not return a Tool instance".format(tool_name)) 21 | 22 | def list_tools(self) -> List[str]: 23 | return list(self.tools.keys()) 24 | 25 | tools_registry = ToolsRegistry() 26 | 27 | def register(tool_name): 28 | def decorator(tool : FuncToolBuilder): 29 | tools_registry.register(tool_name, tool) 30 | return tool 31 | return decorator 32 | 33 | def build_tool(tool_name : str, config : Any) -> Tool: 34 | print(f"will build {tool_name}") 35 | return tools_registry.build(tool_name, config) 36 | 37 | def list_tools() -> List[str]: 38 | return tools_registry.list_tools() 39 | -------------------------------------------------------------------------------- /bmtools/tools/retriever.py: -------------------------------------------------------------------------------- 1 | from langchain.embeddings import OpenAIEmbeddings 2 | from typing import List, Dict 3 | from queue import PriorityQueue 4 | import os 5 | 6 | class Retriever: 7 | def __init__(self, 8 | openai_api_key: str = None, 9 | model: str = "text-embedding-ada-002"): 10 | if openai_api_key is None: 11 | openai_api_key = os.environ.get("OPENAI_API_KEY") 12 | self.embed = OpenAIEmbeddings(openai_api_key=openai_api_key, model=model) 13 | self.documents = dict() 14 | 15 | def add_tool(self, tool_name: str, api_info: Dict) -> None: 16 | if tool_name in self.documents: 17 | return 18 | document = api_info["name_for_model"] + ". " + api_info["description_for_model"] 19 | document_embedding = self.embed.embed_documents([document]) 20 | self.documents[tool_name] = { 21 | "document": document, 22 | "embedding": document_embedding[0] 23 | } 24 | 25 | def query(self, query: str, topk: int = 3) -> List[str]: 26 | query_embedding = self.embed.embed_query(query) 27 | 28 | queue = PriorityQueue() 29 | for tool_name, tool_info in self.documents.items(): 30 | tool_embedding = tool_info["embedding"] 31 | tool_sim = self.similarity(query_embedding, tool_embedding) 32 | queue.put([-tool_sim, tool_name]) 33 | 34 | result = [] 35 | for i in range(min(topk, len(queue.queue))): 36 | tool = queue.get() 37 | result.append(tool[1]) 38 | 39 | return result 40 | 41 | def similarity(self, query: List[float], document: List[float]) -> float: 42 | return sum([i * j for i, j in zip(query, document)]) 43 | 44 | -------------------------------------------------------------------------------- /bmtools/tools/sceneXplain/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("sceneXplain") 4 | def sceneXplain(): 5 | from .api import build_tool 6 | return build_tool -------------------------------------------------------------------------------- /bmtools/tools/sceneXplain/api.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | from ..tool import Tool 4 | import os 5 | 6 | 7 | def build_tool(config) -> Tool: 8 | tool = Tool( 9 | "Image Explainer", 10 | "Tool that adds the capability to explain images.", 11 | name_for_model="Image Explainer", 12 | description_for_model=( 13 | "An Image Captioning Tool: Use this tool to generate a detailed caption " 14 | "for an image. The input can be an image file of any format, and " 15 | "the output will be a text description that covers every detail of the image." 16 | ), 17 | logo_url="https://scenex.jina.ai/SceneX%20-%20Light.svg", 18 | contact_email="hello@contact.com", 19 | legal_info_url="hello@legal.com" 20 | ) 21 | 22 | scenex_api_key = config["subscription_key"] 23 | scenex_api_url: str = ( 24 | "https://us-central1-causal-diffusion.cloudfunctions.net/describe" 25 | ) 26 | 27 | 28 | @tool.get("/describe_image") 29 | def describe_image(image : str): 30 | '''Get the text description of an image. 31 | ''' 32 | headers = { 33 | "x-api-key": f"token {scenex_api_key}", 34 | "content-type": "application/json", 35 | } 36 | payload = { 37 | "data": [ 38 | { 39 | "image": image, 40 | "algorithm": "Ember", 41 | "languages": ["en"], 42 | } 43 | ] 44 | } 45 | response = requests.post(scenex_api_url, headers=headers, json=payload) 46 | response.raise_for_status() 47 | result = response.json().get("result", []) 48 | img = result[0] if result else {} 49 | description = img.get("text", "") 50 | if not description: 51 | return "No description found." 52 | 53 | return description 54 | 55 | return tool 56 | -------------------------------------------------------------------------------- /bmtools/tools/sceneXplain/readme.md: -------------------------------------------------------------------------------- 1 | # Image Description Queries 2 | 3 | Contributor: [Sihan Zhao](https://github.com/Sarah816) -------------------------------------------------------------------------------- /bmtools/tools/sceneXplain/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'sceneXplain', "http://127.0.0.1:8079/tools/sceneXplain/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("What is in this image https://storage.googleapis.com/causal-diffusion.appspot.com/imagePrompts%2F0rw369i5h9t%2Foriginal.png?") 12 | agent("Describe Van Gogh's painting The Starry Night.") -------------------------------------------------------------------------------- /bmtools/tools/serve.py: -------------------------------------------------------------------------------- 1 | import fastapi 2 | import uvicorn 3 | from .registry import build_tool, list_tools 4 | from .retriever import Retriever 5 | from typing import List, Dict 6 | from pydantic import BaseModel 7 | 8 | class RetrieveRequest(BaseModel): 9 | query: str 10 | topk: int = 3 11 | 12 | def _bind_tool_server(t : "ToolServer"): 13 | """ Add property API to ToolServer. 14 | t.api is a FastAPI object 15 | """ 16 | 17 | @t.api.get("/") 18 | def health(): 19 | return { 20 | "status": "ok", 21 | } 22 | 23 | @t.api.get("/list") 24 | def get_tools_list(): 25 | return { 26 | "tools": t.list_tools(), 27 | } 28 | 29 | @t.api.get("/loaded") 30 | def get_loaded_tools(): 31 | return { 32 | "tools": list(t.loaded_tools), 33 | } 34 | 35 | @t.api.get("/.well-known/ai-plugin.json", include_in_schema=False) 36 | def get_api_info(): 37 | return { 38 | "schema_version": "v1", 39 | "name_for_human": "BMTools", 40 | "name_for_model": "BMTools", 41 | "description_for_human": "tools to big models", 42 | "description_for_model": "tools to big models", 43 | "auth": { 44 | "type": "none", 45 | }, 46 | "api": { 47 | "type": "openapi", 48 | "url": "/openapi.json", 49 | "is_user_authenticated": False, 50 | }, 51 | "logo_url": None, 52 | "contact_email": "", 53 | "legal_info_url": "", 54 | } 55 | 56 | @t.api.post("/retrieve") 57 | def retrieve(request: RetrieveRequest): 58 | tool_list = t.retrieve(request.query, request.topk) 59 | return { 60 | "tools": tool_list 61 | } 62 | 63 | class ToolServer: 64 | """ This class host your own API backend. 65 | """ 66 | def __init__(self) -> None: 67 | # define the root API server 68 | self.api = fastapi.FastAPI( 69 | title="BMTools", 70 | description="Tools for bigmodels", 71 | ) 72 | self.loaded_tools = dict() 73 | self.retriever = Retriever() 74 | _bind_tool_server(self) 75 | 76 | def load_tool(self, name: str, config: Dict = {}): 77 | if self.is_loaded(name): 78 | raise ValueError(f"Tool {name} is already loaded") 79 | try: 80 | tool = build_tool(name, config) 81 | except BaseException as e: 82 | print(f"Cannot load tool {name}: {repr(e)}") 83 | return 84 | self.loaded_tools[name] = tool.api_info 85 | self.retriever.add_tool(name, tool.api_info) 86 | 87 | # mount sub API server to the root API server, thus can mount all urls of sub API server to /tools/{name} route 88 | self.api.mount(f"/tools/{name}", tool, name) 89 | return 90 | 91 | def is_loaded(self, name : str): 92 | return name in self.loaded_tools 93 | 94 | def serve(self, host : str = "0.0.0.0", port : int = 8079): 95 | uvicorn.run(self.api, host=host, port=port) 96 | 97 | def list_tools(self) -> List[str]: 98 | return list_tools() 99 | 100 | def retrieve(self, query: str, topk: int = 3) -> List[str]: 101 | return self.retriever.query(query, topk) -------------------------------------------------------------------------------- /bmtools/tools/shell/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("shell") 4 | def shell(): 5 | from .api import build_tool 6 | return build_tool -------------------------------------------------------------------------------- /bmtools/tools/shell/readme.md: -------------------------------------------------------------------------------- 1 | # Shell Commands 2 | 3 | Contributor: [Sihan Zhao](https://github.com/Sarah816) 4 | 5 | # Terminal Tool 6 | 7 | This tool allows you to run shell commands on a machine. 8 | 9 | ## Setup 10 | 11 | The tool is initialized with the following parameters: 12 | 13 | - **name_for_model**: "Terminal" 14 | - **description_for_model**: "Run shell commands on this machine." 15 | - **logo_url**: "https://your-app-url.com/.well-known/logo.png" 16 | - **contact_email**: "hello@contact.com" 17 | - **legal_info_url**: "hello@legal.com" 18 | 19 | ## Endpoint 20 | 21 | The tool provides the following endpoint: 22 | 23 | - **/shell_run**: Run commands and return final output. The input should be a command string. 24 | 25 | ## Function Description 26 | 27 | - **shell_run(commands: str)**: This function runs commands and returns the final output. The commands should be a string. The function returns the output of the executed commands. -------------------------------------------------------------------------------- /bmtools/tools/shell/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'shell', "http://127.0.0.1:8079/tools/shell/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("print 'Hello World!'") -------------------------------------------------------------------------------- /bmtools/tools/stock/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("stock") 5 | def stock(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/stock/api.py: -------------------------------------------------------------------------------- 1 | 2 | import requests 3 | import json 4 | from datetime import date, datetime, timedelta 5 | import os 6 | from ..tool import Tool 7 | 8 | def build_tool(config) -> Tool: 9 | tool = Tool( 10 | "Stock Info", 11 | "Look up stock information", 12 | name_for_model="Stock", 13 | description_for_model="Plugin for look up stock information", 14 | logo_url="https://your-app-url.com/.well-known/logo.png", 15 | contact_email="hello@contact.com", 16 | legal_info_url="hello@legal.com" 17 | ) 18 | 19 | functions = ['TIME_SERIES_INTRADAY', 'TIME_SERIES_INTRADAY_EXTENDED','TIME_SERIES_DAILY', 'TIME_SERIES_DAILY_ADJUSTED'] 20 | types = ['open', 'close', 'high', 'low'] 21 | 22 | KEY = config["subscription_key"] 23 | BASE_URL = 'https://www.alphavantage.co/query?' 24 | 25 | def get_json_data(function, symbol, interval = '5min', adjusted='true', outputsize='compact', datatype='json'): 26 | url = BASE_URL + 'function=' + function + '&symbol=' + symbol + '&apikey=' + KEY 27 | r = requests.get(url) 28 | data = json.loads(r.text) 29 | return data 30 | 31 | @tool.get("/get_today_date") 32 | def get_today_date(): 33 | '''Get today's date 34 | ''' 35 | today = date.today() 36 | return today.strftime("%Y-%m-%d") 37 | 38 | @tool.get('/add_date') 39 | def add_date(date : str, days : int): 40 | '''Add days to a date. Date should be pass as 'yyyy-mm-dd'. 41 | ''' 42 | date = datetime.strptime(date, "%Y-%m-%d") 43 | new_date = date + timedelta(days=days) 44 | return new_date.strftime("%Y-%m-%d") 45 | 46 | @tool.get('/get_daily_prices') 47 | def get_daily_prices(symbol : str, date : str = ''): 48 | '''Get the stock price of an entity in the stock market. Date should be pass as 'yyyy-mm-dd'. 49 | ''' 50 | if "," in symbol: 51 | symbol, date = symbol.split(",") 52 | if date.strip() == "": 53 | return "Please specify a date and try again. You can you get_today_date to up-to-date time information." 54 | data = get_json_data('TIME_SERIES_DAILY_ADJUSTED', symbol) 55 | #print(data.keys()) 56 | time_series = data["Time Series (Daily)"] 57 | final_time = '' 58 | print(time_series) 59 | # 查找最接近查询日期的数据 60 | for timestamp, daily_data in time_series.items(): 61 | print(timestamp) 62 | if timestamp == date: 63 | open_price = daily_data["1. open"] 64 | high_price = daily_data["2. high"] 65 | low_price = daily_data["3. low"] 66 | close_price = daily_data["4. close"] 67 | volume = daily_data["6. volume"] 68 | break 69 | elif timestamp < date: 70 | final_time = timestamp 71 | open_price = time_series[timestamp]["1. open"] 72 | high_price = time_series[timestamp]["2. high"] 73 | low_price = time_series[timestamp]["3. low"] 74 | close_price = time_series[timestamp]["4. close"] 75 | volume = time_series[timestamp]["6. volume"] 76 | break 77 | return {'open':open_price, 'close':close_price, 'high':high_price, 'low':low_price, 'symbol':symbol, 'date':final_time, 'volume':volume} 78 | 79 | 80 | @tool.get('/get_open_info') 81 | def get_open_info(region : str = 'United States'): 82 | '''get information about if the market in the region is open 83 | ''' 84 | url = 'https://www.alphavantage.co/query?function=MARKET_STATUS&apikey=' + KEY 85 | r = requests.get(url) 86 | data = json.loads(r.text) 87 | for item in data['markets']: 88 | if item['region'] == region: 89 | return item['current_status'] 90 | return ' not found' 91 | 92 | @tool.get('/get_exchange_rate') 93 | def get_exchange_rate(from_currency : str = 'USD', to_currency : str = 'BTC'): 94 | '''This API returns the realtime exchange rate for a pair of digital currency (e.g., Bitcoin) and physical currency (e.g., USD). 95 | ''' 96 | url = 'https://www.alphavantage.co/query?function=CURRENCY_EXCHANGE_RATE&from_currency='+ from_currency + '&to_currency='+ to_currency + '&apikey=' + KEY 97 | r = requests.get(url) 98 | data = json.loads(r.text) 99 | try: 100 | rate = data['Realtime Currency Exchange Rate']['5. Exchange Rate'] 101 | return rate 102 | except: 103 | return 'error' 104 | 105 | return tool 106 | -------------------------------------------------------------------------------- /bmtools/tools/stock/readme.md: -------------------------------------------------------------------------------- 1 | # Stock Market Service 2 | 3 | Contributor: [Kunlun Zhu](https://github.com/Kunlun-Zhu) 4 | 5 | You can get your free Alpha Vantage key here: https://www.alphavantage.co/support/#api-key 6 | 7 | # Stock Info Tool 8 | 9 | This tool allows you to look up stock information. 10 | 11 | ## Setup 12 | 13 | The tool is initialized with the following parameters: 14 | 15 | - **name_for_model**: "Stock Info" 16 | - **description_for_model**: "Plugin for look up stock information" 17 | - **logo_url**: "https://your-app-url.com/.well-known/logo.png" 18 | - **contact_email**: "hello@contact.com" 19 | - **legal_info_url**: "hello@legal.com" 20 | 21 | ## Endpoint 22 | 23 | The tool provides the following endpoints: 24 | 25 | - **/get_today_date**: Get today's date. 26 | - **/add_date**: Add days to a date. Date should be pass as 'yyyy-mm-dd'. 27 | - **/get_daily_prices**: Get the stock price of an entity in the stock market. Date should be pass as 'yyyy-mm-dd'. 28 | - **/get_open_info**: Get information about if the market in the region is open. 29 | - **/get_exchange_rate**: This API returns the realtime exchange rate for a pair of digital currency (e.g., Bitcoin) and physical currency (e.g., USD). 30 | 31 | ## Function Descriptions 32 | 33 | - **get_today_date()**: This function returns today's date. 34 | - **add_date(date: str, days: int)**: This function adds a certain number of days to a date. The date should be passed as 'yyyy-mm-dd'. 35 | - **get_daily_prices(symbol: str, date: str = '')**: This function gets the stock price of an entity in the stock market. The date should be passed as 'yyyy-mm-dd'. 36 | - **get_open_info(region: str = 'United States')**: This function gets information about if the market in the region is open. 37 | - **get_exchange_rate(from_currency: str = 'USD', to_currency: str = 'BTC')**: This function returns the realtime exchange rate for a pair of digital currency (e.g., Bitcoin) and physical currency (e.g., USD). -------------------------------------------------------------------------------- /bmtools/tools/stock/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'Stock', "http://127.0.0.1:8079/tools/stock/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("What's the close price of Apple stock at 2022/02/21?") -------------------------------------------------------------------------------- /bmtools/tools/tool.py: -------------------------------------------------------------------------------- 1 | import fastapi 2 | from typing import Optional 3 | import copy 4 | 5 | class Tool(fastapi.FastAPI): 6 | r""" Tool is inherited from FastAPI class, thus: 7 | - It can act as a server 8 | - It has get method, you can use Tool.get method to bind a function to an url 9 | - It can be easily mounted to another server 10 | - It has a list of sub-routes, each route is a function 11 | 12 | Args: 13 | - tool_name (str): The name of the tool. 14 | - description (str): The description of the tool. 15 | - name_for_human (str, optional): The name of the tool for humans. Defaults to None. 16 | - name_for_model (str, optional): The name of the tool for models. Defaults to None. 17 | - description_for_human (str, optional): The description of the tool for humans. Defaults to None. 18 | - description_for_model (str, optional): The description of the tool for models. Defaults to None. 19 | - logo_url (str, optional): The URL of the tool's logo. Defaults to None. 20 | - author_github (str, optional): The GitHub URL of the author. Defaults to None. 21 | - contact_email (str, optional): The contact email for the tool. Defaults to "". 22 | - legal_info_url (str, optional): The URL for legal information. Defaults to "". 23 | - version (str, optional): The version of the tool. Defaults to "0.1.0". 24 | """ 25 | 26 | def __init__( 27 | self, 28 | tool_name : str, 29 | description : str, 30 | name_for_human : Optional[str] = None, 31 | name_for_model : Optional[str] = None, 32 | description_for_human : Optional[str] = None, 33 | description_for_model : Optional[str] = None, 34 | logo_url : Optional[str] = None, 35 | author_github : Optional[str] = None, 36 | contact_email : str = "", 37 | legal_info_url : str = "", 38 | version : str = "0.1.0", 39 | ): 40 | """ 41 | Diagram: 42 | Root API server (ToolServer object) 43 | │ 44 | ├───── "./weather": Tool object 45 | │ ├── "./get_weather_today": function_get_weather(location: str) -> str 46 | │ ├── "./get_weather_forecast": function_get_weather_forcast(location: str, day_offset: int) -> str 47 | │ └── "...more routes" 48 | ├───── "./wikidata": Tool object 49 | │ ├── "... more routes" 50 | └───── "... more routes" 51 | """ 52 | super().__init__( 53 | title=tool_name, 54 | description=description, 55 | version=version, 56 | ) 57 | 58 | if name_for_human is None: 59 | name_for_human = tool_name 60 | if name_for_model is None: 61 | name_for_model = name_for_human 62 | if description_for_human is None: 63 | description_for_human = description 64 | if description_for_model is None: 65 | description_for_model = description_for_human 66 | 67 | self.api_info = { 68 | "schema_version": "v1", 69 | "name_for_human": name_for_human, 70 | "name_for_model": name_for_model, 71 | "description_for_human": description_for_human, 72 | "description_for_model": description_for_model, 73 | "auth": { 74 | "type": "none", 75 | }, 76 | "api": { 77 | "type": "openapi", 78 | "url": "/openapi.json", 79 | "is_user_authenticated": False, 80 | }, 81 | "author_github": author_github, 82 | "logo_url": logo_url, 83 | "contact_email": contact_email, 84 | "legal_info_url": legal_info_url, 85 | } 86 | 87 | @self.get("/.well-known/ai-plugin.json", include_in_schema=False) 88 | def get_api_info(request : fastapi.Request): 89 | openapi_path = str(request.url).replace("/.well-known/ai-plugin.json", "/openapi.json") 90 | info = copy.deepcopy(self.api_info) 91 | info["api"]["url"] = str(openapi_path) 92 | return info 93 | -------------------------------------------------------------------------------- /bmtools/tools/translation/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("nllb-translation") 5 | def translator(): 6 | from .nllb import build_tool 7 | return build_tool 8 | 9 | @register("baidu-translation") 10 | def translator(): 11 | from .baidu import build_tool 12 | return build_tool 13 | 14 | -------------------------------------------------------------------------------- /bmtools/tools/translation/baidu/__init__.py: -------------------------------------------------------------------------------- 1 | from .api import build_tool 2 | -------------------------------------------------------------------------------- /bmtools/tools/translation/baidu/api.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import requests 4 | import hashlib 5 | from ...tool import Tool 6 | 7 | 8 | def build_tool(config) -> Tool: 9 | tool = Tool( 10 | "Translator Info", 11 | "Translate a given text from one language to another.", 12 | name_for_model="Translator", 13 | description_for_model="Plugin for translating text from one language to another.", 14 | logo_url="https://your-app-url.com/.well-known/logo.png", 15 | contact_email="shihaoliang0828@gmail.com", 16 | legal_info_url="hello@legal.com" 17 | ) 18 | subscription_key = os.getenv("BAIDU_TRANSLATE_KEY", None) 19 | if subscription_key is None: 20 | raise Exception("BAIDU_TRANSLATE_KEY is not set") 21 | secret_key = os.getenv("BAIDU_SECRET_KEY", None) 22 | if secret_key is None: 23 | raise Exception("BAIDU_SECRET_KEY is not set") 24 | endpoint = 'https://fanyi-api.baidu.com/api/trans/vip/translate' 25 | fromLang = 'auto' 26 | salt = random.randint(32768,65536) 27 | header = {'Content-Type': 'application/x-www-form-urlencoded'} 28 | 29 | @tool.get("/get_translation") 30 | def get_translation(text:str, tgt_lang:str) -> str: 31 | sign = subscription_key + text + str(salt) + secret_key 32 | md = hashlib.md5() 33 | md.update(sign.encode(encoding='utf-8')) 34 | sign =md.hexdigest() 35 | data = { 36 | "appid": subscription_key, 37 | "q": text, 38 | "from": fromLang, 39 | "to": tgt_lang, 40 | "salt": salt, 41 | "sign": sign 42 | } 43 | response = requests.post(endpoint, params= data, headers= header) 44 | text = response.json() 45 | results = text['trans_result'][0]['dst'] 46 | return results 47 | 48 | return tool 49 | -------------------------------------------------------------------------------- /bmtools/tools/translation/baidu/test.py: -------------------------------------------------------------------------------- 1 | 2 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 3 | 4 | tool_name, tool_url = 'baidu-translation', "http://127.0.0.1:8079/tools/baidu-translation/" 5 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 6 | print(tools_name, tools_config) 7 | 8 | qa = STQuestionAnswerer() 9 | print(tools_config) 10 | agent = qa.load_tools(tools_name, tools_config) 11 | 12 | answer = agent("Translate this sentence into Chinese(zh): Hello, world!") 13 | print(answer) -------------------------------------------------------------------------------- /bmtools/tools/translation/nllb/__init__.py: -------------------------------------------------------------------------------- 1 | from .api import build_tool 2 | -------------------------------------------------------------------------------- /bmtools/tools/translation/nllb/api.py: -------------------------------------------------------------------------------- 1 | 2 | from transformers import AutoModelForSeq2SeqLM, AutoTokenizer 3 | from ...tool import Tool 4 | 5 | def build_tool(config) -> Tool: 6 | tool = Tool( 7 | "Translator Info", 8 | "Translate a given text from one language to another.", 9 | name_for_model="Translator", 10 | description_for_model="Plugin for translating text from one language to another.", 11 | logo_url="https://your-app-url.com/.well-known/logo.png", 12 | contact_email="shihaoliang0828@gmail.com", 13 | legal_info_url="hello@legal.com" 14 | ) 15 | 16 | BASE_MODEL = (config["model"] if "model" in config else "facebook/nllb-200-distilled-600M") 17 | SRC_LANG = (config["src_lang"] if "src_lang" in config else "eng_Latn") 18 | tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, use_auth_token=True, src_lang=SRC_LANG) 19 | model = AutoModelForSeq2SeqLM.from_pretrained(BASE_MODEL, use_auth_token=True) 20 | @tool.get("/get_translation") 21 | def get_translation(input_text:str or list, tgt_lang:str, max_length:int) -> str or list: 22 | inputs = tokenizer(input_text, return_tensors="pt", padding=True) 23 | translated_tokens = model.generate( 24 | **inputs, forced_bos_token_id=tokenizer.lang_code_to_id[tgt_lang], max_length=max_length) 25 | if isinstance(input_text, str): 26 | translations = tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0] 27 | elif isinstance(input_text, list): 28 | translations = tokenizer.batch_decode(translated_tokens, skip_special_tokens=True) 29 | return translations 30 | 31 | return tool 32 | -------------------------------------------------------------------------------- /bmtools/tools/translation/nllb/test.py: -------------------------------------------------------------------------------- 1 | 2 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 3 | 4 | tool_name, tool_url = 'nllb-translation', "http://127.0.0.1:8079/tools/nllb-translation/" 5 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 6 | print(tools_name, tools_config) 7 | 8 | qa = STQuestionAnswerer() 9 | print(tools_config) 10 | agent = qa.load_tools(tools_name, tools_config) 11 | 12 | answer = agent("Translate this sentence into Chinese(zho_Hans): Hello, world!") 13 | print(answer) -------------------------------------------------------------------------------- /bmtools/tools/translation/readme.md: -------------------------------------------------------------------------------- 1 | # Translation Queries 2 | 3 | Contributor: [Shihao Liang](https://github.com/pooruss) 4 | 5 | # NLLB as translation tool: 6 | - Language code should be included in queries in case the NLLB model cannot recognize the target language. 7 | - Default source language is English. Change the nllb model path and source language in host_local_tools.py by passing a dict like this: {"model": ${your_nllb_path}, "src_lang": "eng_Latn"} 8 | - Refer to paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) for all supported languages. 9 | 10 | ## Translator Info Tool 11 | 12 | This tool allows you to translate a given text from one language to another. 13 | 14 | ## Setup 15 | 16 | The tool is initialized with the following parameters: 17 | 18 | - **name_for_model**: "Translator Info" 19 | - **description_for_model**: "Plugin for translating text from one language to another." 20 | - **logo_url**: "https://your-app-url.com/.well-known/logo.png" 21 | - **contact_email**: "shihaoliang0828@gmail.com" 22 | - **legal_info_url**: "hello@legal.com" 23 | 24 | ## Model 25 | 26 | The tool uses a model for translation. The base model is "facebook/nllb-200-distilled-600M" and the source language is "eng_Latn" by default. You can change these parameters in the config. 27 | 28 | ## Endpoint 29 | 30 | The tool provides the following endpoint: 31 | 32 | - **/get_translation**: Translate a given text from one language to another. The input should be a text string or a list of text strings, the target language, and the maximum length of the translated text. 33 | 34 | ## Function Description 35 | 36 | - **get_translation(input_text: str or list, tgt_lang: str, max_length: int) -> str or list**: This function translates a given text from one language to another. The input_text should be a string or a list of strings, tgt_lang should be a string representing the language code, and max_length should be an integer. The function returns the translated text or a list of translated texts. 37 | 38 | 39 | # BAIDU translate api as translation tool: 40 | - Set your APP ID and Secert Key in secret_keys.sh. 41 | - Language code should also be included in queries. 42 | - Refer to [Baidu translate](https://fanyi-api.baidu.com/product/11) for all supported languages. 43 | 44 | ## Translator Info Tool with Baidu 45 | 46 | This tool allows you to translate a given text from one language to another. 47 | 48 | ## Setup 49 | 50 | The tool is initialized with the following parameters: 51 | 52 | - **name_for_model**: "Translator Info" 53 | - **description_for_model**: "Plugin for translating text from one language to another." 54 | - **logo_url**: "https://your-app-url.com/.well-known/logo.png" 55 | - **contact_email**: "shihaoliang0828@gmail.com" 56 | - **legal_info_url**: "hello@legal.com" 57 | 58 | ## API Key 59 | 60 | The tool requires an API key and a secret key from Baidu. You can sign up for a free account at https://fanyi-api.baidu.com/, create a new API key and secret key, and add them to environment variables. 61 | 62 | ## Endpoint 63 | 64 | The tool provides the following endpoint: 65 | 66 | - **/get_translation**: Translate a given text from one language to another. The input should be a text string and the target language. 67 | 68 | ## Function Description 69 | 70 | - **get_translation(text: str, tgt_lang: str) -> str**: This function translates a given text from one language to another. The text should be a string and the target language should be a string representing the language code. The function returns the translated text. -------------------------------------------------------------------------------- /bmtools/tools/travel/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("travel") 5 | def travel(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/travel/readme.md: -------------------------------------------------------------------------------- 1 | # Travel Information 2 | 3 | Contributor: [Runchu Tian](https://github.com/Rachum-thu) 4 | 5 | You can get the API keys from https:https://serpapi.com/search-api and https://developers.amadeus.com/ -------------------------------------------------------------------------------- /bmtools/tools/travel/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | from datetime import datetime, timedelta 3 | 4 | tool_name, tool_url = 'travel', "http://127.0.0.1:8079/tools/travel/" 5 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 6 | print(tools_name, tools_config) 7 | 8 | qa = STQuestionAnswerer() 9 | 10 | agent = qa.load_tools(tools_name, tools_config) 11 | 12 | month_later = str(datetime.now() + timedelta(days=30)).split()[0] 13 | agent(f"I will go to Seattle from Beijing on {month_later}. Can you make a recommendation on hotels and flight please?") 14 | agent("Please help me rent a car near Tsinghua University.") 15 | agent("Where can I visit on weekend if I am going to University of Washington this summer?") -------------------------------------------------------------------------------- /bmtools/tools/tutorial/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("tutorial") 4 | def tutorial(): 5 | from .api import build_tool 6 | return build_tool 7 | -------------------------------------------------------------------------------- /bmtools/tools/tutorial/api.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import requests 4 | import hashlib 5 | from ..tool import Tool 6 | 7 | from langchain.prompts import PromptTemplate 8 | from langchain.chains import LLMChain 9 | from langchain.llms import OpenAI 10 | 11 | 12 | def build_tool(config) -> Tool: 13 | tool = Tool( 14 | tool_name="Tutorial", 15 | description="Provide tutorial for foundation model based on a given objective.", 16 | name_for_model="Tutorial", 17 | description_for_model="Plugin for providing tutorial for a given objective.", 18 | logo_url="https://your-app-url.com/.well-known/logo.png", 19 | contact_email="xin.cong@outlook.com", 20 | legal_info_url="hello@legal.com" 21 | ) 22 | prompt = PromptTemplate.from_template( 23 | "You are a planner who is an expert at coming up with a todo list for a given objective. Come up with a todo list for this objective: {objective}" 24 | ) 25 | 26 | key = os.environ.get("OPENAI_API_KEY") 27 | llm = OpenAI(model_name="gpt-3.5-turbo", temperature=0.0, openai_api_key=key) 28 | 29 | chain = LLMChain(llm=llm, prompt=prompt) 30 | 31 | @tool.get("/tutorial") 32 | def tutorial(text: str) -> str: 33 | """ 34 | tutorial(text: str) -> str: Providing a TODO list as a toturial for the foundation model based on the given objective. 35 | """ 36 | result = chain.run(text) 37 | return result 38 | 39 | return tool 40 | -------------------------------------------------------------------------------- /bmtools/tools/tutorial/readme.md: -------------------------------------------------------------------------------- 1 | # Tutorial Service 2 | 3 | Contributor: [Xin Cong](https://github.com/congxin95) 4 | 5 | # Tutorial Tool 6 | 7 | This tool provides a tutorial for a foundation model based on a given objective. 8 | 9 | ## Setup 10 | 11 | The tool is initialized with the following parameters: 12 | 13 | - **name_for_model**: "Tutorial" 14 | - **description_for_model**: "Plugin for providing tutorial for a given objective." 15 | - **logo_url**: "https://your-app-url.com/.well-known/logo.png" 16 | - **contact_email**: "xin.cong@outlook.com" 17 | - **legal_info_url**: "hello@legal.com" 18 | 19 | ## API Key 20 | 21 | The tool requires an API key from OpenAI. You can sign up for a free account at https://www.openai.com/, create a new API key, and add it to environment variables. 22 | 23 | ## Endpoint 24 | 25 | The tool provides the following endpoint: 26 | 27 | - **/tutorial**: Provide a TODO list as a tutorial for the foundation model based on the given objective. The input should be a text string representing the objective. 28 | 29 | ## Function Description 30 | 31 | - **tutorial(text: str) -> str**: This function provides a TODO list as a tutorial for the foundation model based on the given objective. The text should be a string representing the objective. The function returns a TODO list as a tutorial. -------------------------------------------------------------------------------- /bmtools/tools/tutorial/test.py: -------------------------------------------------------------------------------- 1 | 2 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 3 | 4 | tool_name, tool_url = 'tutorial', "http://127.0.0.1:8079/tools/tutorial/" 5 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 6 | print(tools_name, tools_config) 7 | 8 | qa = STQuestionAnswerer() 9 | print(tools_config) 10 | agent = qa.load_tools(tools_name, tools_config) 11 | 12 | answer = agent("I want to cook pizza.") 13 | print(answer) -------------------------------------------------------------------------------- /bmtools/tools/walmart/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("walmart") 5 | def walmart(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/walmart/api.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import json 4 | import requests 5 | from ..tool import Tool 6 | from typing import Optional 7 | from bs4 import BeautifulSoup 8 | from serpapi import GoogleSearch 9 | from amadeus import Client, ResponseError 10 | 11 | 12 | def build_tool(config) -> Tool: 13 | tool = Tool( 14 | "walmart Info", 15 | "Query about information about retail commodity on Walmart Platform.", 16 | name_for_model="walmart", 17 | description_for_model="""This is a plugin for look up real walmart infomation. Results from this API are inaccessible for users. Please organize and re-present them.""", 18 | logo_url="https://your-app-url.com/.well-known/logo.png", 19 | contact_email="hello@contact.com", 20 | legal_info_url="hello@legal.com" 21 | ) 22 | 23 | SERPAPI_KEY = config["subscription_key"] 24 | 25 | @tool.get("/ItemQuery") 26 | def ItemQuery( 27 | item: str, 28 | option_num: Optional[int] = 3, 29 | ): 30 | """ 31 | This API gather retail information about queried items at walmart 32 | :param item: product name presented as string. 33 | :param option_num: the number of items presented for each queried item. 34 | :return: a dict walmart retail information about queried items. 35 | """ 36 | 37 | try: 38 | 39 | params = { 40 | "engine": "walmart", 41 | "query": item, 42 | "api_key": SERPAPI_KEY, 43 | } 44 | search = GoogleSearch(params) 45 | results = search.get_dict() 46 | organic_results = results["organic_results"] 47 | 48 | item_res = [] 49 | for idx in range(min(option_num, len(results["organic_results"]))): 50 | item_res.append({}) 51 | item_res[idx]["name"] = organic_results[idx]["title"] 52 | item_res[idx]["description"] = organic_results[idx]["description"] 53 | item_res[idx]["price"] = organic_results[idx]["primary_offer"]["offer_price"] 54 | item_res[idx]["price_unit"] = organic_results[idx]["price_per_unit"]["unit"] 55 | item_res[idx]["url"] = organic_results[idx]["product_page_url"] 56 | item_res[idx]["rating"] = organic_results[idx]["rating"] 57 | item_res[idx]["seller_name"] = organic_results[idx]["seller_name"] 58 | 59 | return json.dumps(item_res) 60 | 61 | except Exception: # Handle response error exceptions 62 | return {"error": "Response error."} 63 | 64 | return tool -------------------------------------------------------------------------------- /bmtools/tools/walmart/readme.md: -------------------------------------------------------------------------------- 1 | # Walmart Information 2 | 3 | Contributor: [Runchu Tian](https://github.com/Rachum-thu) 4 | 5 | You can get the API keys from https:https://serpapi.com/search-api 6 | 7 | We will modify this plugin by designing better prompts for interaction. -------------------------------------------------------------------------------- /bmtools/tools/walmart/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'walmart', "http://127.0.0.1:8079/tools/walmart/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | 6 | qa = STQuestionAnswerer() 7 | agent = qa.load_tools(tools_name, tools_config) 8 | 9 | agent("I want to have Kung Pao Chicken tonight. Please help me buy some onions on walmart.") 10 | -------------------------------------------------------------------------------- /bmtools/tools/weather/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("weather") 5 | def weather(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/weather/readme.md: -------------------------------------------------------------------------------- 1 | # Weather Queries 2 | 3 | Contributor: [Yujia Qin](https://github.com/thuqinyj16) 4 | 5 | You can get the API keys from https://www.weatherapi.com/ 6 | 7 | # Weather Info Tool 8 | 9 | This tool allows you to look up weather information. 10 | 11 | ## Setup 12 | 13 | The tool is initialized with the following parameters: 14 | 15 | - **name_for_model**: "Weather Info" 16 | - **description_for_model**: "Plugin for look up weather information" 17 | - **logo_url**: "https://cdn.weatherapi.com/v4/images/weatherapi_logo.png" 18 | - **contact_email**: "hello@contact.com" 19 | - **legal_info_url**: "hello@legal.com" 20 | 21 | ## API Key 22 | 23 | The tool requires an API key from WeatherAPI. You can sign up for a free account at https://www.weatherapi.com/, create a new API key, and add it to environment variables. 24 | 25 | ## Endpoint 26 | 27 | The tool provides the following endpoints: 28 | 29 | - **/get_weather_today**: Get today's weather. The input should be a location string. 30 | - **/forecast_weather**: Forecast weather in the upcoming days. The input should be a location string and the number of days for the forecast. 31 | 32 | ## Function Descriptions 33 | 34 | - **get_weather_today(location: str) -> str**: This function gets today's weather for a given location. The location should be a string. The function returns a string with the weather information. 35 | - **forecast_weather(location: str, days: int) -> str**: This function forecasts the weather for a given location in the upcoming days. The location should be a string and days should be an integer. The function returns a string with the weather forecast. -------------------------------------------------------------------------------- /bmtools/tools/weather/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'weather', "http://127.0.0.1:8079/tools/weather/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("What's the weather in London today?") -------------------------------------------------------------------------------- /bmtools/tools/wikipedia/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("wikipedia") 5 | def wikipedia(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/wikipedia/readme.md: -------------------------------------------------------------------------------- 1 | # Search in Wikipedia 2 | 3 | Contributor: [Yufei Huang](https://github.com/huangyf530) 4 | 5 | # Wikipedia Search Tool 6 | 7 | This tool allows you to search for entities, view content, and disambiguate entities on Wikipedia. 8 | 9 | ## Setup 10 | 11 | The tool is initialized with the following parameters: 12 | 13 | - **name_for_model**: "Wikipedia Search" 14 | - **description_for_model**: "Plugin for wikipedia" 15 | 16 | ## Endpoint 17 | 18 | The tool provides the following endpoints: 19 | 20 | - **/search**: Search for an entity on Wikipedia. 21 | - **/lookup**: Look up a keyword in the current passage. 22 | - **/disambiguation**: Disambiguate an entity name to find other entities with similar names on Wikipedia. 23 | 24 | ## Function Descriptions 25 | 26 | - **search(entity: str) -> str**: This function searches for an entity on Wikipedia and returns the first five sentences if it exists. If not, it returns some related entities to search next. The entity should be a string. 27 | - **lookup(keyword: str) -> str**: This function looks up a keyword in the current passage and returns the next several sentences containing the keyword. The keyword should be a string. 28 | - **disambiguation(entity: str) -> str**: This function disambiguates an entity name to find other entities with similar names on Wikipedia. The entity should be a string. -------------------------------------------------------------------------------- /bmtools/tools/wikipedia/test.py: -------------------------------------------------------------------------------- 1 | 2 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 3 | import requests 4 | import json 5 | 6 | # at = "{\"entity\": \"Arthur\"s Magazine\"}" 7 | # print(at[19]) 8 | # print(len(at)) 9 | # a = json.loads("{\"entity\": \"Arthur\"s Magazine\"}") 10 | # print(a) 11 | 12 | tool_name, tool_url = 'wikipedia', "http://127.0.0.1:8079/tools/wikipedia/" 13 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 14 | print(tools_name, tools_config) 15 | 16 | qa = STQuestionAnswerer() 17 | 18 | agent = qa.load_tools(tools_name, tools_config) 19 | 20 | agent("Which magazine was started first, Arthur’s Magazine or First for Women?") 21 | 22 | agent("when was the first hunger games book published?") -------------------------------------------------------------------------------- /bmtools/tools/wolframalpha/__init__.py: -------------------------------------------------------------------------------- 1 | from ..registry import register 2 | 3 | @register("wolframalpha") 4 | def wolframalpha(): 5 | from .api import build_tool 6 | return build_tool -------------------------------------------------------------------------------- /bmtools/tools/wolframalpha/api.py: -------------------------------------------------------------------------------- 1 | 2 | import requests 3 | from ..tool import Tool 4 | from typing import Any 5 | import os 6 | import xmltodict 7 | 8 | 9 | def build_tool(config) -> Tool: 10 | tool = Tool( 11 | "Wolfram", 12 | "Wolfram", 13 | name_for_model="Wolfram", 14 | name_for_human="Wolfram", 15 | description_for_model=""""Dynamic computation and curated data from WolframAlpha and Wolfram Cloud.\nOnly use the getWolframAlphaResults endpoints; all other Wolfram endpoints are deprecated.\nPrefer getWolframAlphaResults unless Wolfram Language code should be evaluated.\nTry to include images returned by getWolframAlphaResults. Queries to getWolframAlphaResults must ALWAYS have this structure: {\"input\": query}.\n", 16 | """, 17 | description_for_human="Access computation, math, curated knowledge & real-time data through Wolfram|Alpha and Wolfram Language.", 18 | logo_url="https://www.wolframcdn.com/images/icons/Wolfram.png", 19 | contact_email="hello@contact.com", 20 | legal_info_url="hello@legal.com" 21 | ) 22 | 23 | @tool.get("/getWolframAlphaResults") 24 | def getWolframAlphaResults(input:str): 25 | """Get Wolfram|Alpha results using natural query. Queries to getWolframAlphaResults must ALWAYS have this structure: {\"input\": query}. And please directly read the output json. 26 | """ 27 | URL = "https://api.wolframalpha.com/v2/query" 28 | 29 | APPID = config["subscription_key"] 30 | 31 | params = {'appid': APPID, "input": input} 32 | 33 | response = requests.get(URL, params=params) 34 | 35 | json_data = xmltodict.parse(response.text) 36 | 37 | if 'pod' not in json_data["queryresult"]: 38 | return "WolframAlpha API cannot parse the input query." 39 | rets = json_data["queryresult"]['pod'] 40 | 41 | cleaned_rets = [] 42 | blacklist = ["@scanner", "@id", "@position", "@error", "@numsubpods", "@width", "@height", "@type", "@themes","@colorinvertable", "expressiontypes"] 43 | 44 | def filter_dict(d, blacklist): 45 | if isinstance(d, dict): 46 | return {k: filter_dict(v, blacklist) for k, v in d.items() if k not in blacklist} 47 | elif isinstance(d, list): 48 | return [filter_dict(i, blacklist) for i in d] 49 | else: 50 | return d 51 | 52 | for ret in rets: 53 | ret = filter_dict(ret, blacklist=blacklist) 54 | # Do further cleaning to retain only the input and result pods 55 | if "@title" in ret: 56 | if ret["@title"] == "Input" or ret["@title"] == "Result": 57 | cleaned_rets.append(ret) 58 | 59 | return cleaned_rets 60 | 61 | return tool -------------------------------------------------------------------------------- /bmtools/tools/wolframalpha/readme.md: -------------------------------------------------------------------------------- 1 | # Query Wolframalpha Service 2 | 3 | Contributor [Shengding Hu](https://github.com/shengdinghu) 4 | 5 | You can get the API keys from https://products.wolframalpha.com/api/ 6 | 7 | # Wolfram Tool 8 | 9 | This tool provides dynamic computation and curated data from WolframAlpha and Wolfram Cloud. 10 | 11 | ## Setup 12 | 13 | The tool is initialized with the following parameters: 14 | 15 | - **name_for_model**: "Wolfram" 16 | - **description_for_model**: "Dynamic computation and curated data from WolframAlpha and Wolfram Cloud." 17 | - **logo_url**: "https://www.wolframcdn.com/images/icons/Wolfram.png" 18 | - **contact_email**: "hello@contact.com" 19 | - **legal_info_url**: "hello@legal.com" 20 | 21 | ## Endpoint 22 | 23 | The tool provides the following endpoint: 24 | 25 | - **/getWolframAlphaResults**: Get Wolfram|Alpha results using a natural query. 26 | 27 | ## Function Description 28 | 29 | - **getWolframAlphaResults(input: str) -> dict**: This function gets Wolfram|Alpha results using a natural query. The input should be a string. The function returns a dictionary containing the results. Note that queries to getWolframAlphaResults must ALWAYS have this structure: {"input": query}. Please directly read the output JSON. -------------------------------------------------------------------------------- /bmtools/tools/wolframalpha/test.py: -------------------------------------------------------------------------------- 1 | 2 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 3 | 4 | tool_name, tool_url = 'wolframalpha', "http://127.0.0.1:8079/tools/wolframalpha/" 5 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 6 | print(tools_name, tools_config) 7 | 8 | qa = STQuestionAnswerer() 9 | 10 | agent = qa.load_tools(tools_name, tools_config) 11 | 12 | agent("Calc integral of sin(x)+2x^2+3x+1 from 0 to 1") -------------------------------------------------------------------------------- /bmtools/tools/zillow/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from ..registry import register 3 | 4 | @register("zillow") 5 | def zillow(): 6 | from .api import build_tool 7 | return build_tool 8 | -------------------------------------------------------------------------------- /bmtools/tools/zillow/readme.md: -------------------------------------------------------------------------------- 1 | # Zillow Service 2 | 3 | Contributor: [Kunlun Zhu](https://github.com/Kunlun-Zhu) 4 | 5 | You can get your RAIPID key here: https://rapidapi.com/hub 6 | 7 | You ought to subscribe 'Zillow API' in your account to use this tool -------------------------------------------------------------------------------- /bmtools/tools/zillow/test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | tool_name, tool_url = 'Zillow', "http://127.0.0.1:8079/tools/zillow/" 4 | tools_name, tools_config = load_single_tools(tool_name, tool_url) 5 | print(tools_name, tools_config) 6 | 7 | qa = STQuestionAnswerer() 8 | 9 | agent = qa.load_tools(tools_name, tools_config) 10 | 11 | agent("List some houses in Santa Monica, CA.") -------------------------------------------------------------------------------- /bmtools/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenBMB/BMTools/85869fc6e26616508f5dfafa551546993ab35f52/bmtools/utils/__init__.py -------------------------------------------------------------------------------- /multi_test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.tools_controller import load_valid_tools, MTQuestionAnswerer 2 | import jsonlines 3 | # Choose the tools that you need 4 | tools_mappings = { 5 | #"klarna": "https://www.klarna.com/", 6 | #"chemical-prop": "http://127.0.0.1:8079/tools/chemical-prop/", 7 | "wolframalpha": "http://127.0.0.1:8079/tools/wolframalpha/", 8 | #"meta_analysis": "http://127.0.0.1:8079/tools/meta_analysis/", 9 | #"map": "http://127.0.0.1:8079/tools/map/", 10 | #"douban": "http://127.0.0.1:8079/tools/douban-film/", 11 | #"weather": "http://127.0.0.1:8079/tools/weather/", 12 | "office-ppt": "http://127.0.0.1:8079/tools/office-ppt/", 13 | "wikipedia": "http://127.0.0.1:8079/tools/wikipedia/", 14 | #"nllb-translation": "http://127.0.0.1:8079/tools/nllb-translation/", 15 | "file_operation": "http://127.0.0.1:8079/tools/file_operation/", 16 | "bing_search": "http://127.0.0.1:8079/tools/bing_search/", 17 | } 18 | 19 | tools = load_valid_tools(tools_mappings) 20 | qa = MTQuestionAnswerer(all_tools=tools) 21 | 22 | agent = qa.build_runner() 23 | 24 | 25 | agent(["Who's the main actress of Titanic? What did she do apart from this film? Help me make slides with this information."]) 26 | #agent(['I want to go to Berkeley for one-week vacation. Please help me recommend some tourisms, restaurants, as well as the recent weather conditions for the place.']) 27 | #agent(["How many benzene rings are there in 9H-Carbazole-3-carboxaldehyde? and what is sin(x)*exp(x)'s plot, what is it integrated from 0 to 1? "]) 28 | 29 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | SPARQLWrapper 3 | matplotlib 4 | python-pptx 5 | bs4 6 | langchain==0.0.147 7 | openai 8 | py3langid 9 | transformers 10 | sentencepiece 11 | faiss-cpu 12 | psycopg2-binary; platform_python_implementation == "CPython" 13 | psycopg2; platform_python_implementation != "CPython" 14 | pymysql 15 | arxiv 16 | googlemaps 17 | steamship 18 | IPython 19 | google-search-results 20 | iso-639 21 | amadeus 22 | gradio 23 | gradio_tools 24 | peft 25 | # torch>=1.10 26 | # bmtrain==0.1.8.post1 27 | # jieba 28 | # tqdm 29 | # tensorboard 30 | # numpy>=1.21.0 31 | # spacy 32 | # opendelta 33 | -------------------------------------------------------------------------------- /secret_keys.sh: -------------------------------------------------------------------------------- 1 | export OPENAI_API_KEY='' 2 | export WOLFRAMALPH_APP_ID='' 3 | export WEATHER_API_KEYS='' 4 | export BING_SUBSCRIPT_KEY='' 5 | export ALPHA_VANTAGE_KEY='' 6 | export BING_MAP_KEY='' 7 | export BAIDU_TRANSLATE_KEY='' 8 | export BAIDU_SECRET_KEY='' 9 | export HUGGINGFACE_API_KEY='' 10 | export BAIDU_MAP_KEY='' 11 | export RAPIDAPI_KEY='' 12 | export SERPER_API_KEY='' 13 | export GPLACES_API_KEY='' 14 | export SCENEX_API_KEY='' 15 | export STEAMSHIP_API_KEY='' 16 | export AMADEUS_ID='' 17 | export AMADEUS_KEY='' 18 | 19 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r", encoding='utf8') as fh: 4 | long_description = fh.read() 5 | 6 | with open("requirements.txt", "r") as f: 7 | requirements = f.read().splitlines() 8 | 9 | setuptools.setup( 10 | name="bmtools", 11 | version="0.1.0", 12 | author="OpenBMB", 13 | author_email="shengdinghu@gmail.com", 14 | description="API library for big models to use tools", 15 | long_description=long_description, 16 | long_description_content_type="text/markdown", 17 | url="https://github.com/OpenBMB/BMTools", 18 | packages=setuptools.find_packages(), 19 | classifiers=[ 20 | "Programming Language :: Python :: 3", 21 | "License :: OSI Approved :: MIT License", 22 | "Operating System :: OS Independent", 23 | ], 24 | python_requires=">=3.6", 25 | #install_requires=requirements 26 | ) 27 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 2 | 3 | # Langchain 4 | tool_name, tool_url = 'weather', "http://127.0.0.1:8079/tools/weather/" 5 | tool_name, tool_config = load_single_tools(tool_name, tool_url) 6 | print(tool_name, tool_config) 7 | stqa = STQuestionAnswerer() 8 | 9 | agent = stqa.load_tools(tool_name, tool_config, prompt_type="react-with-tool-description") 10 | agent("write a weather report for SF today") 11 | 12 | # BabyAGI 13 | # tool_name, tool_url = 'weather', "http://127.0.0.1:8079/tools/weather/" 14 | # tool_name, tool_config = load_single_tools(tool_name, tool_url) 15 | # print(tool_name, tool_config) 16 | # stqa = STQuestionAnswerer() 17 | 18 | # agent = stqa.load_tools(tool_name, tool_config, prompt_type="babyagi") 19 | # agent("write a weather report for SF today") 20 | 21 | # Auto-GPT 22 | # tool_name, tool_url = 'weather', "http://127.0.0.1:8079/tools/weather/" 23 | # tool_name, tool_config = load_single_tools(tool_name, tool_url) 24 | # print(tool_name, tool_config) 25 | # stqa = STQuestionAnswerer() 26 | 27 | # agent = stqa.load_tools(tool_name, tool_config, prompt_type="autogpt") 28 | # agent.run(["write a weather report for SF today"]) 29 | 30 | 31 | """ 32 | from bmtools.agent.singletool import load_single_tools, STQuestionAnswerer 33 | 34 | tool_name, tool_url = 'wikipedia', "http://127.0.0.1:8079/tools/wikipedia/" 35 | tool_name, tool_config = load_single_tools(tool_name, tool_url) 36 | print(tool_name, tool_config) 37 | stqa = STQuestionAnswerer() 38 | 39 | agent = stqa.load_tools(tool_name, tool_config, prompt_type="babyagi") 40 | # agent = stqa.load_tools(tool_name, tool_config, prompt_type="react-with-tool-description")# prompt_type="babyagi") 41 | agent("Where is Yaoming Born?") 42 | """ 43 | -------------------------------------------------------------------------------- /test_multi.py: -------------------------------------------------------------------------------- 1 | from bmtools.agent.tools_controller import load_valid_tools, MTQuestionAnswerer 2 | tools_mappings = { 3 | 'weather': "http://127.0.0.1:8079/tools/weather/", 4 | 'file_operation': "http://127.0.0.1:8079/tools/file_operation/", 5 | } 6 | 7 | tools = load_valid_tools(tools_mappings) 8 | 9 | qa = MTQuestionAnswerer(openai_api_key='', all_tools=tools) 10 | 11 | agent = qa.build_runner() 12 | 13 | agent("what is the weather in Beijing?") --------------------------------------------------------------------------------