├── .gitignore ├── .gitattributes ├── requirements.txt ├── .env -example ├── app.py ├── helpers.py └── ui.py /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | venv 3 | __pycache__ 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | SimplerLLM==0.2.6 2 | networkx==3.3 3 | python-dotenv==1.0.1 4 | pyvis==0.3.2 5 | Requests==2.32.3 6 | streamlit==1.36.0 7 | streamlit_agraph==0.0.45 8 | -------------------------------------------------------------------------------- /.env -example: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY = "sk-proj-XXXXX" 2 | 3 | 4 | WORDPRESS_URL = "http://data-tools.local/" 5 | WORDPRESS_USER = "admin" 6 | WORDPRESS_APP_PASSWORD = "AaM7XXXXXX" 7 | 8 | 9 | RAPIDAPI_API_KEY = "99d7XXXX" 10 | 11 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | from pyvis.network import Network 3 | from helpers import get_topic_children 4 | 5 | 6 | def generate_subtopics_graph(graph, topic, current_level, max_level): 7 | if current_level > max_level: 8 | return 9 | 10 | print(f"Level {current_level}: Generating subtopics for '{topic}'") 11 | 12 | subtopics = get_topic_children(topic) 13 | for subtopic in subtopics: 14 | print(f" Adding edge from '{topic}' to '{subtopic}'") 15 | graph.add_edge(topic, subtopic) 16 | generate_subtopics_graph(graph, subtopic, current_level + 1, max_level) 17 | 18 | 19 | 20 | def main(): 21 | main_topic_keyword = "Machine Learning" 22 | max_level = int(input("Enter the level of sub-leveling (1-10): ")) 23 | 24 | graph = nx.DiGraph() 25 | generate_subtopics_graph(graph, main_topic_keyword, 1, max_level) 26 | 27 | # Convert networkx graph to pyvis network 28 | net = Network(notebook=True) 29 | net.from_nx(graph) 30 | 31 | # Save and display the interactive graph 32 | net.show("generated_topic_graph.html") 33 | print("Interactive graph saved as 'generated_topic_graph.html'") 34 | 35 | if __name__ == "__main__": 36 | main() 37 | -------------------------------------------------------------------------------- /helpers.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from SimplerLLM.language.llm import LLM,LLMProvider 3 | from SimplerLLM.language.llm_addons import generate_pydantic_json_model 4 | from SimplerLLM.tools.rapid_api import RapidAPIClient 5 | from pydantic import BaseModel 6 | import requests 7 | from requests.auth import HTTPBasicAuth 8 | from dotenv import load_dotenv 9 | import os 10 | 11 | # Load environment variables 12 | load_dotenv() 13 | 14 | # Constants 15 | wordpress_url = os.getenv("WORDPRESS_URL", "") 16 | wordpress_user = os.getenv("WORDPRESS_USER", "") 17 | wordpress_pass = os.getenv("WORDPRESS_APP_PASSWORD", "") 18 | 19 | class SubTopics(BaseModel): 20 | sub_topics: List[str] 21 | 22 | llm_instance = LLM.create(provider=LLMProvider.OPENAI,model_name="gpt-4o") 23 | 24 | sub_topics_prompt = """as an expert in keyword and topic research specialized in {topic}, 25 | generate {count} sub topics to write about in the form of SEARCHABLE keywords 26 | for the the following parent topic: {topic}""" 27 | 28 | draft_prompt = """ 29 | I will provide you with a [TOPIC], and your task is to generate a blog post draft for that [TOPIC]. 30 | maske the draft is SEO optimized, and covers all the aspects of the [TOPIC]. 31 | The draft should me a minimum of {word_count} words. 32 | [TOPIC] = {topic} 33 | 34 | Draft: 35 | 36 | """ 37 | 38 | def generate_draft(topic :str, word_count = 500): 39 | prompt = draft_prompt.format(topic=topic, word_count = word_count) 40 | response = llm_instance.generate_response(prompt=prompt,max_tokens=4096) 41 | return response 42 | 43 | def get_topic_children(topic :str, num_results = 3): 44 | prompt = sub_topics_prompt.format(topic=topic,count = num_results) 45 | 46 | response = generate_pydantic_json_model(model_class=SubTopics, 47 | prompt=prompt,llm_instance=llm_instance, 48 | max_tokens=1024) 49 | return response.sub_topics 50 | 51 | def post_on_wordpress(topic, content): 52 | 53 | wp_endpoint = f"{wordpress_url}/wp-json/wp/v2/posts" 54 | 55 | # The content of your new blog post 56 | post = { 57 | "title": topic, 58 | "content": content, 59 | "status": "draft" # Other statuses can be 'publish', 'pending', etc. 60 | } 61 | 62 | # Make the request to create the post 63 | response = requests.post(wp_endpoint, json=post, auth=HTTPBasicAuth(wordpress_user, wordpress_pass)) 64 | 65 | if response.status_code == 201: 66 | print("Post created successfully") 67 | else: 68 | print(f"Failed to create post. Status code: {response.status_code}") 69 | print(response.json()) 70 | 71 | def get_keyword_metrics(keywords): 72 | rapid = RapidAPIClient() 73 | response = rapid.call_api(api_url=f"https://bulk-keyword-metrics.p.rapidapi.com/seo-tools/get-bulk-keyword-metrics?keywords_count=20&query={keywords}&countryCode=US") 74 | return response -------------------------------------------------------------------------------- /ui.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | from streamlit_agraph import agraph, Node, Edge, Config 3 | from helpers import get_topic_children, get_keyword_metrics,generate_draft,post_on_wordpress 4 | import time # Import the time module 5 | 6 | def fetch_keyword_metrics(keywords): 7 | try: 8 | keyword_string = ','.join(keywords[:20]) # Limit to 20 keywords 9 | response = get_keyword_metrics(keyword_string) 10 | if response['success']: 11 | return response['result'] 12 | else: 13 | st.warning(f"API Warning: {response['message']}") 14 | return [{ 'keyword': keyword, 'searchVolume': 'N/A' } for keyword in keywords] 15 | except Exception as e: 16 | return [{ 'keyword': keyword, 'searchVolume': 'N/A' } for keyword in keywords] 17 | 18 | def generate_subtopics_tree(topic, current_level, max_level, progress_log, added_nodes, keyword_data, fetch_keywords, child_topics_count, generate_post): 19 | start_time = time.time() # Start the timer 20 | 21 | if current_level > max_level: 22 | return [], [] 23 | 24 | log_entry = f"Level {current_level}: Generating subtopics for '{topic}'" 25 | progress_log.append(log_entry) 26 | 27 | subtopics = get_topic_children(topic,child_topics_count) 28 | nodes = [] 29 | edges = [] 30 | node_color = "white" 31 | if topic not in added_nodes: 32 | 33 | if fetch_keywords: 34 | main_topic_metrics = fetch_keyword_metrics([topic]) 35 | main_search_volume = main_topic_metrics[0]['searchVolume'] if main_topic_metrics else 'N/A' 36 | keyword_data.extend(main_topic_metrics) 37 | else: 38 | main_search_volume = 'N/A' 39 | node_label = f"{topic} ( Search Volume: {main_search_volume})" 40 | if main_search_volume and main_search_volume != 'N/A': 41 | if main_search_volume > 10000: 42 | node_color = "#97eb14" 43 | elif main_search_volume > 1000: 44 | node_color = "#f4ff00" 45 | elif main_search_volume > 0: 46 | node_color = "#f10e38" 47 | else: 48 | node_color = "white" 49 | nodes.append(Node(id=topic, color = node_color, label=node_label, font={'color': 'white'})) 50 | 51 | 52 | 53 | added_nodes.add(topic) 54 | 55 | if fetch_keywords: 56 | # Fetch keyword metrics only if the checkbox is checked 57 | subtopic_metrics = fetch_keyword_metrics(subtopics) 58 | keyword_data.extend(subtopic_metrics) 59 | else: 60 | subtopic_metrics = [{'keyword': subtopic, 'searchVolume': 'N/A'} for subtopic in subtopics] 61 | 62 | for subtopic, metric in zip(subtopics, subtopic_metrics): 63 | search_volume = metric['searchVolume'] 64 | node_label = f"{subtopic} (Search Volume: {search_volume})" 65 | if search_volume and search_volume != 'N/A': 66 | if search_volume > 10000: 67 | node_color = "#97eb14" 68 | elif search_volume > 1000: 69 | node_color = "#f4ff00" 70 | elif search_volume > 0: 71 | node_color = "#f10e38" 72 | else: 73 | node_color = "white" 74 | 75 | if subtopic not in added_nodes: 76 | if generate_post: 77 | draft = generate_draft(subtopic) 78 | post_on_wordpress(subtopic, draft) 79 | 80 | nodes.append(Node(id=subtopic, color = node_color, label=node_label, font={'color': 'white'})) 81 | added_nodes.add(subtopic) 82 | edges.append(Edge(source=topic, target=subtopic)) 83 | 84 | sub_nodes, sub_edges = generate_subtopics_tree(subtopic, current_level + 1, max_level, progress_log, added_nodes, keyword_data, fetch_keywords,child_topics_count, generate_post) 85 | nodes.extend(sub_nodes) 86 | edges.extend(sub_edges) 87 | 88 | end_time = time.time() # End the timer 89 | elapsed_time = end_time - start_time # Calculate the elapsed time 90 | log_entry = f"Time taken for level {current_level} with topic '{topic}': {elapsed_time:.2f} seconds" 91 | progress_log.append(log_entry) # Log the time taken 92 | 93 | return nodes, edges 94 | 95 | 96 | def main(): 97 | st.title("Intensive Topic Research with AI") 98 | 99 | # Sidebar setup 100 | with st.sidebar: 101 | st.write("Configuration") 102 | main_topic_keyword = st.text_input("Enter the main topic:", "") 103 | max_level = st.slider("Select the level of sub-leveling (1-5):", 1, 5, 1) 104 | child_topics_count = st.slider("Select the number of max child topics (3-10):", 3, 10, 3) 105 | fetch_keywords = st.checkbox("Fetch Keyword Data", False) # Checkbox for fetching keyword data 106 | generate_posts = st.checkbox("Generate Blog Post Drafts", False) 107 | 108 | 109 | Physics = st.checkbox("Physics", False) 110 | hierarchical = st.checkbox("hierarchical", False) 111 | 112 | 113 | 114 | if "nodes" not in st.session_state or "edges" not in st.session_state or "keyword_data" not in st.session_state: 115 | st.session_state.nodes = [] 116 | st.session_state.edges = [] 117 | st.session_state.progress_log = [] 118 | st.session_state.keyword_data = [] 119 | 120 | if st.button("Start"): 121 | st.session_state.progress_log = [] 122 | st.session_state.keyword_data = [] 123 | added_nodes = set() 124 | 125 | with st.spinner("Generating..."): 126 | start_time = time.time() # Start the timer for the entire operation 127 | st.session_state.nodes, st.session_state.edges = generate_subtopics_tree( 128 | main_topic_keyword, 1, max_level, st.session_state.progress_log, added_nodes, st.session_state.keyword_data, 129 | fetch_keywords=fetch_keywords,child_topics_count=child_topics_count, generate_post=generate_posts 130 | ) 131 | end_time = time.time() # End the timer for the entire operation 132 | elapsed_time = end_time - start_time # Calculate the elapsed time for the entire operation 133 | st.session_state.progress_log.append(f"Total time to generate: {elapsed_time:.2f} seconds") 134 | 135 | if st.session_state.nodes: 136 | config = Config( 137 | width=800, 138 | height=800, 139 | directed=True, 140 | physics=Physics, 141 | hierarchical=hierarchical, 142 | nodeHighlightBehavior=True, 143 | highlightColor="#F7A7A6", # Color of the highlight 144 | collapsible=True # Enable collapsible nodes 145 | ) 146 | agraph(nodes=st.session_state.nodes, edges=st.session_state.edges, config=config) 147 | 148 | with st.expander("Detailed Log"): 149 | for log_entry in st.session_state.progress_log: 150 | st.write(log_entry) 151 | 152 | with st.expander("Keyword Data"): 153 | for data in st.session_state.keyword_data: 154 | st.write(data) 155 | 156 | if __name__ == "__main__": 157 | main() 158 | --------------------------------------------------------------------------------