├── .gitignore ├── README.md ├── pyproject.toml └── src └── crew2 ├── __init__.py ├── config ├── agents.yaml └── tasks.yaml ├── crew.py ├── main.py └── tools ├── __init__.py └── custom_tools.py /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | __pycache__/ 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Knowledge Graph & General Google Search Agent System 2 | This project leverages crewAI to build an intelligent agent system that prioritizes information from Google's Knowledge Graph for reliability. If no relevant data is found, it reverts to a General Google Search to scrape the top two URLs from the organic results, ensuring accurate and comprehensive information retrieval about entities, organizations, or individuals. 3 | 4 | Key Features: 5 | Knowledge Graph Tool: Extracts structured data from Google's Knowledge Graph, offering reliable, curated information. 6 | Organic Search Tool: If no Knowledge Graph data is available, it performs a Google search and scrapes the top 2 URLs from organic search results. 7 | Web Scraping Tool: Built using crewAI's ScrapeWebsiteTool, this tool extracts content from the selected top URLs for deeper insights. 8 | The system integrates two custom tools—KnowledgeGraphTool and OrganicSearchTool—to gather and process the data, and a crewAI built-in tool, ScrapeWebsiteTool, to scrape web pages. 9 | 10 | Customization: 11 | 12 | Clone the repository: 13 | To modify the system for your own use, clone the repository: 14 | 15 | bash 16 | 17 | git clone git@github.com:Ronoh4/KnowledgeGraphCrew.git 18 | cd your-repository 19 | Modify agents.yaml: 20 | Define the behavior and configuration of your agents. 21 | 22 | Modify tasks.yaml: 23 | Adjust tasks and objectives that agents will perform. 24 | 25 | Extend with Custom Tools: 26 | Customize custom_tools.py to add new tools or modify existing ones to better suit your needs. 27 | 28 | Running the Project: 29 | To run the system and begin agent collaboration: 30 | 31 | bash 32 | crewai run 33 | This command will initialize the agents, allowing them to gather and analyze data automatically, based on the configuration you've set. -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "crew2" 3 | version = "0.1.0" 4 | description = "crew2 using crewAI" 5 | authors = ["Your Name "] 6 | 7 | [tool.poetry.dependencies] 8 | python = ">=3.10,<=3.13" 9 | crewai = { extras = ["tools"], version = ">=0.70.1,<1.0.0" } 10 | 11 | 12 | [tool.poetry.scripts] 13 | crew2 = "crew2.main:run" 14 | run_crew = "crew2.main:run" 15 | train = "crew2.main:train" 16 | replay = "crew2.main:replay" 17 | test = "crew2.main:test" 18 | 19 | [build-system] 20 | requires = ["poetry-core"] 21 | build-backend = "poetry.core.masonry.api" 22 | -------------------------------------------------------------------------------- /src/crew2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ronoh4/KnowledgeGraphCrew/4d35f93ad83364f93ba843e88136a5af63969ec7/src/crew2/__init__.py -------------------------------------------------------------------------------- /src/crew2/config/agents.yaml: -------------------------------------------------------------------------------- 1 | researcher: 2 | role: > 3 | {topic} Researcher 4 | goal: > 5 | Gather structured information about {topic} from Google's Knowledge Graph as the primary source. 6 | If no Google Knowledge Graph data is available, fall back to general Google search and extract organic information 7 | from general Google search results. 8 | backstory: > 9 | You are an expert researcher specializing in uncovering detailed, structured data about {topic}. 10 | Your primary source of information is Google's Knowledge Graph, where you excel at extracting meaningful insights. 11 | However, for lesser-known entities without Google Knowledge Graph blocks, you adapt by leveraging general Google search results 12 | to ensure no critical detail is missed. Your reports are comprehensive, actionable, and precise. 13 | 14 | reporter: 15 | role: > 16 | {topic} Reporter 17 | goal: > 18 | Create detailed reports based on the knowledge graph or general Google search information about {topic} 19 | backstory: > 20 | You're a meticulous report writer with a keen eye for concise reports. You're known for 21 | your ability to turn complex data into clear and concise reports, making 22 | it easy for others to understand and act on the information you provide. -------------------------------------------------------------------------------- /src/crew2/config/tasks.yaml: -------------------------------------------------------------------------------- 1 | research_task: 2 | description: > 3 | Conduct a thorough investigation into {topic} using Google's Knowledge Graph as the primary source 4 | of structured data. If no Knowledge Graph information is available, adapt by utilizing the provided general Google search tool to search 5 | general Google results and use the scrape tool to scrape the top 2 positional URLs from the organic search results for relevant web content. 6 | Prioritize accuracy, relevance, and clarity in presenting your findings. 7 | expected_output: > 8 | A detailed and accurate research report summarizing structured Knowledge Graph data 9 | (if available). If not, provide insights gathered from general search, 10 | ensuring all information is relevant to {topic}. If general search is not authorized, 11 | or no information is obtained just report that. 12 | agent: researcher 13 | 14 | reporting_task: 15 | description: > 16 | If any report is generated, review the detailed research report about {topic} and turn it into a well-structured and concise report. 17 | Ensure the report is concise, clear, and contains all relevant information to build a profile about {topic} 18 | expected_output: > 19 | If there is any knowledge graph or general search report, provide a comprehensive profile of {topic}. Ensure the report is structured around key information and detailed enough to provide sufficient information about the topic. The report should include, but is not limited to, the following sections: 20 | - Basic Information: Name, type (person, entity, organization), and relevant categories or industries. 21 | - Background: Historical context, origin, or founding details. 22 | - Key Figures: Important individuals associated with the topic, such as founders, key executives, or notable persons, along with brief bios. 23 | - Location: Headquarters or main location, if applicable. 24 | - Financial Information: Relevant financial data such as stock information, funding, or economic impact, if applicable. 25 | - Contact Information: Customer service details, official contact links, or other relevant contact information. 26 | - Online Presence: Official website link, social media profiles, and other online resources. 27 | - Additional Relevant Information: Any other pertinent details that provide a comprehensive understanding of the topic. 28 | If no report is generated, just report that. 29 | agent: reporter 30 | -------------------------------------------------------------------------------- /src/crew2/crew.py: -------------------------------------------------------------------------------- 1 | from crewai.project import CrewBase, agent, crew, task 2 | from crewai import Agent, Crew, Process, Task 3 | from .tools.custom_tools import KnowledgeGraphTool, OrganicSearchTool 4 | from dotenv import load_dotenv 5 | from crewai_tools import ScrapeWebsiteTool 6 | import os 7 | 8 | load_dotenv() 9 | serper_api_key = os.getenv('SERPER_API_KEY') 10 | 11 | scrape_tool = ScrapeWebsiteTool() 12 | 13 | @CrewBase 14 | class KnowledgeGraphCrew: 15 | @agent 16 | def researcher(self) -> Agent: 17 | return Agent( 18 | config=self.agents_config['researcher'], 19 | tools=[KnowledgeGraphTool(), OrganicSearchTool(), scrape_tool], 20 | verbose=True, 21 | llm="openai/gpt-4o" 22 | ) 23 | 24 | @agent 25 | def reporter(self) -> Agent: 26 | return Agent( 27 | config=self.agents_config['reporter'], 28 | verbose=True, 29 | llm="openai/gpt-4o-mini" 30 | ) 31 | 32 | @task 33 | def research_task(self) -> Task: 34 | return Task( 35 | config=self.tasks_config['research_task'], 36 | ) 37 | 38 | @task 39 | def reporting_task(self) -> Task: 40 | return Task( 41 | config=self.tasks_config['reporting_task'], 42 | ) 43 | 44 | @crew 45 | def crew(self) -> Crew: 46 | return Crew( 47 | agents=self.agents, 48 | tasks=self.tasks, 49 | process=Process.sequential, # Define the order in which tasks will run 50 | verbose=True, 51 | ) 52 | -------------------------------------------------------------------------------- /src/crew2/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | from .crew import KnowledgeGraphCrew 4 | 5 | def run(): 6 | """ 7 | Run the crew. 8 | """ 9 | print("Initializing Knowledge Graph Crew...") 10 | crew = KnowledgeGraphCrew() 11 | 12 | inputs = { 13 | "topic": "CrewAI" 14 | } 15 | 16 | response = crew.crew().kickoff(inputs=inputs) 17 | return {"response": response} 18 | 19 | if __name__ == "__main__": 20 | run() 21 | -------------------------------------------------------------------------------- /src/crew2/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ronoh4/KnowledgeGraphCrew/4d35f93ad83364f93ba843e88136a5af63969ec7/src/crew2/tools/__init__.py -------------------------------------------------------------------------------- /src/crew2/tools/custom_tools.py: -------------------------------------------------------------------------------- 1 | from crewai_tools import BaseTool 2 | import os 3 | from serpapi import GoogleSearch 4 | from dotenv import load_dotenv 5 | 6 | # Load environment variables 7 | load_dotenv() 8 | serper_api_key = os.getenv('SERPER_API_KEY') 9 | 10 | # Knowledge Graph Tool 11 | class KnowledgeGraphTool(BaseTool): 12 | name: str = "Knowledge Graph Extractor" 13 | description: str = ( 14 | "This tool extracts specific information from Google's Knowledge Graph. " 15 | "Falls back to other tools if no Knowledge Graph data is available." 16 | ) 17 | 18 | def _run(self, query: str) -> dict: 19 | params = { 20 | 'engine': 'google', 21 | 'q': query, 22 | 'api_key': serper_api_key 23 | } 24 | search = GoogleSearch(params) 25 | results = search.get_dict() 26 | knowledge_graph = results.get('knowledge_graph', None) 27 | return {"knowledge_graph": knowledge_graph} if knowledge_graph else {"error": "No Knowledge Graph data found."} 28 | 29 | # General Organic Google Search Tool 30 | class OrganicSearchTool(BaseTool): 31 | name: str = "General Google Search" 32 | description: str = ( 33 | "Performs a general organic Google search and retrieves organic results. " 34 | "Use this tool if no Knowledge Graph data is available." 35 | ) 36 | 37 | def _run(self, query: str) -> dict: 38 | params = { 39 | "engine": "google", 40 | "q": query, 41 | "api_key": serper_api_key 42 | } 43 | search = GoogleSearch(params) 44 | results = search.get_dict() 45 | organic_results = results.get("organic_results", None) 46 | 47 | if organic_results: 48 | # Return top 6 results 49 | top_results = organic_results[:6] 50 | return {"results": top_results} 51 | else: 52 | return {"error": "No general search results found."} --------------------------------------------------------------------------------