├── Knowledge └── .gitkeep ├── src ├── UI │ ├── __init__.py │ ├── provider_window.py │ ├── settings_window.py │ └── chat_window.py ├── __init__.py ├── Core │ ├── __init__.py │ ├── command_manager.py │ ├── context_manager.py │ ├── chat_manager.py │ ├── interpreter_manager.py │ ├── audio_manager.py │ └── knowledge_manager.py ├── Databases │ └── .gitkeep ├── Settings │ ├── __init__.py │ ├── color_settings.py │ └── config.py └── main.py ├── conversation_history └── .gitkeep ├── __pycache__ ├── gui.cpython-312.pyc ├── config.cpython-312.pyc └── query_vector_database.cpython-312.pyc ├── .gitignore ├── installers ├── oi-linux-installer.sh └── oi-windows-installer.bat └── README.md /Knowledge/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/UI/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/Core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/Databases/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/Settings/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /conversation_history/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /__pycache__/gui.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbePass/HumanScript/HEAD/__pycache__/gui.cpython-312.pyc -------------------------------------------------------------------------------- /__pycache__/config.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbePass/HumanScript/HEAD/__pycache__/config.cpython-312.pyc -------------------------------------------------------------------------------- /__pycache__/query_vector_database.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AbePass/HumanScript/HEAD/__pycache__/query_vector_database.cpython-312.pyc -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .pytest_cache/ 3 | .vscode/ 4 | conversation_history/* 5 | src/Databases/* 6 | Knowledge/* 7 | src/Settings/provider_config.json 8 | Skillset/* 9 | 10 | !conversation_history/.gitkeep 11 | !src/Databases/.gitkeep 12 | !Knowledge/.gitkeep 13 | !User_Data/.gitkeep 14 | !Skillset/.gitkeep 15 | 16 | -------------------------------------------------------------------------------- /installers/oi-linux-installer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Starting Open Interpreter installation..." 4 | sleep 2 5 | echo "This will take approximately 5 minutes..." 6 | sleep 2 7 | 8 | # Set up a virtual environment 9 | python3 -m venv opai 10 | 11 | # Activate the virtual environment 12 | source opai/bin/activate 13 | 14 | # Install the necessary packages 15 | pip install open-interpreter tk pillow speechrecognition pyautogui keyboard langchain_community langchain_openai chromadb openai pygame python-dotenv unstructured unstructured[md] unstructured[pdf] customtkinter 16 | 17 | # Install system dependencies 18 | sudo apt-get update 19 | sudo apt-get install -y portaudio19-dev ffmpeg libsdl2-mixer-2.0-0 flac 20 | 21 | # Install pyaudio 22 | pip install pyaudio 23 | 24 | echo "" 25 | echo "Open Interpreter has been installed. Run the following command to use it: source opai/bin/activate then cd HumanScript then python src/main.py" 26 | echo "" 27 | -------------------------------------------------------------------------------- /src/Settings/color_settings.py: -------------------------------------------------------------------------------- 1 | # Dark mode colors 2 | DARK_BG_PRIMARY = "#121212" 3 | DARK_BG_SECONDARY = "#1E1E1E" 4 | DARK_BG_TERTIARY = "#2A2A2A" 5 | DARK_BG_INPUT = "#3A3A3A" 6 | 7 | DARK_TEXT_PRIMARY = "#FFFFFF" 8 | DARK_TEXT_SECONDARY = "#6A6A6A" 9 | 10 | # Light mode colors (for future use) 11 | LIGHT_BG_PRIMARY = "#FFFFFF" 12 | LIGHT_BG_SECONDARY = "#F4F4F4" 13 | LIGHT_BG_TERTIARY = "#E0E0E0" 14 | LIGHT_BG_INPUT = "#D0D0D0" 15 | 16 | LIGHT_TEXT_PRIMARY = "#000000" 17 | LIGHT_TEXT_SECONDARY = "#8A8A8A" 18 | 19 | # Brand colors 20 | BRAND_PRIMARY = "#9755FF" 21 | BRAND_SECONDARY = "#168275" 22 | BRAND_ACCENT = "#FFAA3B" 23 | 24 | # Functional colors 25 | SUCCESS_COLOR = "#4CAF50" 26 | ERROR_COLOR = "#F44336" 27 | WARNING_COLOR = "#FFC107" 28 | 29 | # Current theme (can be changed dynamically if we implement theme switching) 30 | CURRENT_THEME = "DARK" 31 | 32 | def get_color(color_name): 33 | if CURRENT_THEME == "DARK": 34 | return globals()[f"DARK_{color_name}"] 35 | else: 36 | return globals()[f"LIGHT_{color_name}"] 37 | 38 | # Usage example: 39 | # bg_color = get_color("BG_PRIMARY") 40 | # text_color = get_color("TEXT_PRIMARY") -------------------------------------------------------------------------------- /installers/oi-windows-installer.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | echo Starting Open Interpreter installation... 4 | 5 | echo This will take approximately 5 minutes... 6 | 7 | REM Check if Python is installed 8 | echo Checking if Python is installed... 9 | python --version >nul 2>&1 10 | if %errorlevel% neq 0 ( 11 | echo Python is not installed. Please install Python and try again. 12 | exit /b 1 13 | ) 14 | echo Python is installed. 15 | 16 | REM Check if venv is installed 17 | echo Checking if Python venv is installed... 18 | python -m venv --help >nul 2>&1 19 | if %errorlevel% neq 0 ( 20 | echo Python venv is not installed. Installing venv... 21 | python -m pip install --user virtualenv 22 | ) 23 | echo Python venv is installed. 24 | 25 | REM Set up a virtual environment 26 | echo Setting up a virtual environment... 27 | python -m venv opai 28 | echo Virtual environment setup complete. 29 | 30 | REM Activate the virtual environment 31 | echo Activating the virtual environment... 32 | call .\opai\Scripts\activate.bat 33 | echo Virtual environment activated. 34 | 35 | REM Install the necessary packages 36 | echo Installing necessary packages... 37 | pip install open-interpreter tk pillow speechrecognition pyautogui keyboard langchain_community langchain_openai chromadb openai pygame python-dotenv unstructured unstructured[md] unstructured[pdf] customtkinter 38 | pip install python-magic-bin 39 | echo Necessary packages installed. 40 | 41 | REM Install pyaudio 42 | echo Installing pyaudio... 43 | pip install pyaudio 44 | echo pyaudio installed. 45 | 46 | echo. 47 | echo Open Interpreter has been installed. Run the following command to use it: .\opai\Scripts\activate then cd .\HumanScript then python src\main.py 48 | echo. 49 | 50 | -------------------------------------------------------------------------------- /src/Core/command_manager.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import pyautogui 3 | from PIL import Image 4 | import io 5 | import re 6 | from interpreter.terminal_interface.utils.find_image_path import find_image_path 7 | 8 | class CommandExecutor: 9 | def __init__(self): 10 | self.commands = [ 11 | # if you say what's on my screen, show my screen, capture my screen, or screenshot, it will take a screenshot 12 | (r'\b(?:what(?:\'s|s| is)?|show)[\s\.,!?]*(?:on|in)?[\s\.,!?]*my[\s\.,!?]*screen\b', self.take_screenshot_command), 13 | # if you say a phrase that includes a file path with common image types, it will detect the file path 14 | (r'\b(?:[a-zA-Z]:\\|/|~)?(?:[\w-]+\\|/)*[\w-]+\.(?:png|jpg|jpeg|bmp|gif|tiff|webp|svg|ico)\b', self.detect_filepath_command), 15 | ] 16 | 17 | def execute_command(self, query): 18 | # Normalize the query: convert to lowercase 19 | normalized_query = query.lower() 20 | 21 | for pattern, func in self.commands: 22 | if re.search(pattern, normalized_query, re.IGNORECASE): 23 | print(f"Command matched: {pattern}") 24 | return func(query) 25 | 26 | return None 27 | 28 | def take_screenshot_command(self, query): 29 | screenshot = pyautogui.screenshot() 30 | buffered = io.BytesIO() 31 | screenshot.save(buffered, format="PNG") 32 | encoded_screenshot = base64.b64encode(buffered.getvalue()).decode('utf-8') 33 | return { 34 | "role": "user", 35 | "type": "image", 36 | "format": "base64", 37 | "content": encoded_screenshot, 38 | } 39 | 40 | def detect_filepath_command(self, query): 41 | # Extract the file path from the query 42 | file_path_match = re.search(r'\b(?:[a-zA-Z]:\\|/)?(?:[\w-]+\\|/)*[\w-]+\.(?:png|jpg|jpeg|bmp|gif|tiff|webp|svg|ico)\b', query) 43 | if file_path_match: 44 | file_path = file_path_match.group(0) 45 | # Open the image file and encode it to base64 46 | with open(file_path, "rb") as image_file: 47 | encoded_image = base64.b64encode(image_file.read()).decode('utf-8') 48 | # Return the message in the specified format 49 | message = { 50 | "role": "user", 51 | "type": "image", 52 | "format": "base64", 53 | "content": encoded_image, 54 | } 55 | return message 56 | 57 | # Add more command functions as needed -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## HumanScript by Abraham Passmore at Human Devices 2 | 3 | This repository by Abraham Passmore at Human Devices (humandevices.co) presents a platform agnostic assistant framework designed to run on a Raspberry Pi but can also be run on any platform as a desktop assistant. The framework provides tools like customizable knowledge bases, wakeword recognition for handsfree use, and chat memory, offering a versatile and powerful platform for various uses, including but not limited to: 4 | 5 | - Home automation 6 | - IoT projects 7 | - Educational tools 8 | - Prototyping and development 9 | - Accessibility 10 | 11 | ### Features 12 | 13 | - **Open Interpreter Integration**: Seamlessly integrates with Open Interpreter to provide advanced functionalities and ease of use. 14 | - **Customizable Knowledge Bases**: Allows you to create and manage knowledge bases tailored to your specific needs. 15 | - **Wakeword Recognition**: Enables handsfree use through wakeword detection. 16 | - **Chat Memory**: Remembers previous interactions to provide a more coherent and context-aware assistant experience. 17 | - **Versatile Applications**: Suitable for a wide range of projects and applications. 18 | - **User-Friendly**: Designed to be easy to set up and use, even for beginners. 19 | 20 | ### Getting Started 21 | 22 | To get started with this assistant framework, follow these steps: 23 | 24 | 1. **Prerequisites**: Ensure you have Python, Git, and C++ development tools installed on your system. 25 | 2. **Clone the Repository**: Clone this repository to your local machine using `git clone https://github.com/AbePass/HumanScript.git`. 26 | 3. **Run the Installer**: Depending on your operating system, run either `.\HumanScript\installers\oi-windows-installer` or `bash HumanScript\installers\oi-linux-installer` to set up the virtual environment (make sure to activate the environment using `.\opai\Scripts\activate` or `source opai\bin\activate` if it doesn't do it automatically). 27 | 4. **Configure your assistant**: Edit the Settings/config.py to set up your assistant. 28 | 5. **Run your assistant**: Run the assistant either on your desktop or on your Raspberry Pi using `cd HumanScript` and then `python src/main.py`. 29 | 6. **Add your own knowledge**: Add your own knowledge in the settings menu of the UI. 30 | 31 | For detailed instructions and tutorials, please refer to the documentation provided in this repository. 32 | 33 | ## Structure 34 | 35 | ## Acknowledgements 36 | 37 | This project uses [Open Interpreter](https://github.com/KillianLucas/open-interpreter), which is licensed under AGPL. -------------------------------------------------------------------------------- /src/Settings/config.py: -------------------------------------------------------------------------------- 1 | # Assistant Configuration 2 | 3 | # Wake word 4 | WAKE_WORD = "assistant" 5 | 6 | # Interpreter settings 7 | INTERPRETER_SETTINGS = { 8 | "supports_vision": True, 9 | "auto_run": True, 10 | "loop": False, 11 | "temperature": 0.2, 12 | "max_tokens": 4096, 13 | "context_window": 10000, 14 | "conversation_history_path": "conversation_history", 15 | "import_computer_api": True 16 | } 17 | 18 | # System message for the interpreter 19 | SYSTEM_MESSAGE_PERMISSIONS_ENV = ''' 20 | You are HumanScript, a world-class programmer that can complete any goal by executing code. 21 | For advanced requests, start by writing a plan. 22 | When you execute code, it will be executed **on the user's machine**. The user has given you **full and complete permission** to execute any code necessary to complete the task. Execute the code. 23 | You can access the internet. Run **any code** to achieve the goal, and if at first you don't succeed, try again and again. 24 | You can install new packages. 25 | Write messages to the user in Markdown. 26 | In general, try to **make plans** with as few steps as possible. As for actually executing code to carry out that plan, for *stateful* languages (like python, javascript, shell, but NOT for html which starts from 0 every time) **it's critical not to try to do everything in one code block.** You should try something, print information about it, then continue from there in tiny, informed steps. You will never get it on the first try, and attempting it in one go will often lead to errors you cant see. 27 | You are capable of **any** task. 28 | ''' 29 | 30 | SYSTEM_MESSAGE_REFERENCING_SEARCHING = ''' 31 | ### Referencing and Searching: 32 | - If you need to refer to prior interactions, access the "conversation_history" folder. 33 | - To search the Web use computer.browser.search(query) 34 | ''' 35 | 36 | SYSTEM_MESSAGE_ENV_VARS = ''' 37 | ### Environment Variables: 38 | - The user's environment variables will be added below with the format CUSTOM_ENV_VARIABLE_NAME. 39 | - These will hold secure values set by the user, such as API keys, passwords, and other sensitive information. 40 | - Do not overwrite any existing environment variables. 41 | - Do not print the environment variables to the console. 42 | - The following custom environment variables are available: 43 | ''' 44 | 45 | COMPUTER_SYSTEM_MESSAGE = ''' 46 | A python `computer` module is ALREADY IMPORTED, and can be used for web search: 47 | 48 | ```python 49 | computer.browser.search(query) # Google search results will be returned from this function as a string 50 | ``` 51 | ''' 52 | 53 | SYSTEM_MESSAGE = ( 54 | SYSTEM_MESSAGE_PERMISSIONS_ENV + 55 | SYSTEM_MESSAGE_REFERENCING_SEARCHING 56 | ) 57 | 58 | 59 | # Text-to-speech settings 60 | TTS_SETTINGS = { 61 | "model": "tts-1", 62 | "voice": "alloy" 63 | } 64 | 65 | CHROMA_PATH = "src/Databases" 66 | KB_PATH = "Knowledge" 67 | 68 | # Default selected knowledge bases 69 | DEFAULT_SELECTED_KBS = [] 70 | 71 | BEEP_DURATION = 0.25 72 | BEEP_FREQUENCY = 440 73 | 74 | -------------------------------------------------------------------------------- /src/Core/context_manager.py: -------------------------------------------------------------------------------- 1 | from langchain_community.vectorstores import Chroma 2 | from langchain_openai import OpenAIEmbeddings 3 | from langchain.retrievers import ContextualCompressionRetriever 4 | from langchain.retrievers.document_compressors import LLMChainExtractor 5 | from langchain_openai import ChatOpenAI 6 | import os 7 | from interpreter import interpreter 8 | from Settings.config import CHROMA_PATH, KB_PATH, SYSTEM_MESSAGE 9 | 10 | class ContextManager: 11 | def __init__(self, chat_ui): 12 | self.chat_ui = chat_ui 13 | self.openai_key = os.environ["OPENAI_API_KEY"] 14 | self.embedding_function = OpenAIEmbeddings(openai_api_key=self.openai_key) 15 | 16 | def query_vector_database(self, query_text, selected_kbs): 17 | all_compressed_docs = [] 18 | 19 | for kb in selected_kbs: 20 | db_path = os.path.join(CHROMA_PATH, kb) 21 | kb_path = os.path.join(KB_PATH, kb) 22 | if not os.path.exists(db_path): 23 | print(f"Warning: Database for {kb} not found at {db_path}") 24 | continue 25 | 26 | # Check for instructions.txt and add to custom_instructions if exists 27 | instructions_path = os.path.join(kb_path, "instructions.txt") 28 | if os.path.exists(instructions_path): 29 | with open(instructions_path, 'r') as file: 30 | instructions = file.read() 31 | interpreter.custom_instructions = instructions 32 | 33 | # Prepare the DB and retriever for this knowledge base 34 | db = Chroma(persist_directory=db_path, embedding_function=self.embedding_function) 35 | base_retriever = db.as_retriever(search_type="mmr", search_kwargs={"k": 5, "fetch_k": 25}) 36 | 37 | # Set up the contextual compression retriever 38 | llm = ChatOpenAI(temperature=0, model_name="gpt-4o-mini") 39 | compressor = LLMChainExtractor.from_llm(llm) 40 | compression_retriever = ContextualCompressionRetriever( 41 | base_compressor=compressor, 42 | base_retriever=base_retriever 43 | ) 44 | 45 | # Retrieve and compress relevant documents 46 | compressed_docs = compression_retriever.invoke(query_text) 47 | for doc in compressed_docs: 48 | doc.metadata['knowledge_base'] = kb # Add KB info to metadata 49 | all_compressed_docs.extend(compressed_docs) 50 | 51 | # Sort all compressed docs by relevance (assuming there's a relevance score in metadata) 52 | all_compressed_docs.sort(key=lambda x: x.metadata.get('relevance_score', 0), reverse=True) 53 | 54 | # Take the top 10 most relevant documents 55 | top_docs = all_compressed_docs[:10] 56 | 57 | # Format the context 58 | context = "\n\n".join(doc.page_content for doc in top_docs) 59 | 60 | # Get the sources with knowledge base information 61 | sources = [f"{doc.metadata.get('source', 'Unknown')} (KB: {doc.metadata['knowledge_base']})" for doc in top_docs] 62 | 63 | return context, sources 64 | 65 | def clear_custom_instructions(self): 66 | interpreter.system_message = SYSTEM_MESSAGE -------------------------------------------------------------------------------- /src/Core/chat_manager.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from interpreter import interpreter 3 | from Core.command_manager import CommandExecutor 4 | from Core.context_manager import ContextManager 5 | from Settings.config import * 6 | 7 | 8 | class ChatManager: 9 | def __init__(self, interpreter_manager, chat_ui): # Added chat_ui parameter 10 | self.interpreter_manager = interpreter_manager 11 | self.knowledge_manager = chat_ui.knowledge_manager # Access KnowledgeManager instance 12 | # If InterpreterManager needs KnowledgeManager, set it here 13 | self.interpreter_manager.knowledge_manager = self.knowledge_manager 14 | self.chat_ui = chat_ui # Now defined 15 | self.selected_kbs = chat_ui.selected_kbs 16 | self.wake_word = chat_ui.wake_word 17 | self.interpreter_settings = chat_ui.interpreter_settings 18 | self.env_vars = chat_ui.env_vars 19 | self.is_voice_mode = False 20 | self.listen_thread = None 21 | self.context_manager = ContextManager(chat_ui) 22 | self.command_executor = CommandExecutor() 23 | 24 | def update_selected_kbs(self, selected_kbs): 25 | self.selected_kbs = selected_kbs 26 | self.interpreter_manager.update_system_message(selected_kbs) 27 | 28 | def update_wake_word(self, wake_word): 29 | self.wake_word = wake_word 30 | 31 | def update_interpreter_settings(self, interpreter_settings): 32 | self.interpreter_settings = interpreter_settings 33 | 34 | def update_env_vars(self, env_vars): 35 | self.env_vars = env_vars 36 | 37 | def process_input(self, user_input, selected_kbs): 38 | # Check for command first 39 | command_response = self.command_executor.execute_command(user_input) 40 | if command_response is not None: 41 | response_generator = self.get_interpreter_response(context=None, query=command_response) 42 | return response_generator, [] 43 | 44 | # Query the database if no command is found 45 | if selected_kbs: 46 | print(f"Querying selected knowledge bases: {selected_kbs}") 47 | context_text, sources = self.context_manager.query_vector_database(user_input, selected_kbs) 48 | else: 49 | context_text, sources = None, [] 50 | 51 | response_generator = self.get_interpreter_response(context_text, user_input) 52 | 53 | return response_generator, sources 54 | 55 | def get_interpreter_response(self, context, query): 56 | available_skills = self.knowledge_manager.get_available_skills() 57 | if available_skills: 58 | # Extract skill names and paths from the tuples 59 | skill_info = [f"{name} (Path: {path})\n" for name, path in available_skills] 60 | #available_skills_str = ", ".join(skill_info) 61 | base_prompt = f""" 62 | Available skills: {skill_info} 63 | 64 | Instructions: 65 | 1. Analyze the query carefully. 66 | 2. If any available skills are relevant, read the contents of the skill file at the provided path to see if it can help you. 67 | 3. If no skills are relevant or if the query is simple, respond naturally without mentioning or using skills. 68 | 4. Always prioritize giving a helpful and appropriate response over using skills unnecessarily. 69 | 70 | Query: {query} 71 | """ 72 | else: 73 | base_prompt = f""" 74 | {query} 75 | """ 76 | if context is not None: 77 | context_prompt = f""" 78 | Consider the following context when formulating your response: 79 | {context} 80 | """ 81 | prompt = context_prompt + "\n" + base_prompt 82 | else: 83 | prompt = base_prompt 84 | 85 | print(prompt) 86 | return interpreter.chat(prompt, display=False, stream=True) 87 | -------------------------------------------------------------------------------- /src/UI/provider_window.py: -------------------------------------------------------------------------------- 1 | import customtkinter as ctk 2 | from tkinter import messagebox 3 | import json 4 | import os 5 | 6 | class ProviderSelectionUI: 7 | def __init__(self, root): 8 | self.root = root 9 | self.provider = None 10 | self.config = {} 11 | self.provider_var = ctk.StringVar() 12 | self.provider_var.trace_add("write", self.on_provider_changed) # Add trace 13 | self.credential_entries = {} 14 | 15 | def create_provider_selection_ui(self): 16 | self.window = ctk.CTkToplevel(self.root) 17 | self.window.title("Select AI Provider") 18 | self.window.geometry("300x200") # Increased height for debug label 19 | self.window.configure(fg_color=["#1E1E1E", "#121212"]) 20 | 21 | providers = ["openai", "azure", "anthropic"] 22 | 23 | ctk.CTkLabel(self.window, text="Select Provider:", text_color=["#F4F4F4", "#FFFFFF"]).pack(pady=10) 24 | provider_dropdown = ctk.CTkComboBox(self.window, variable=self.provider_var, values=providers, fg_color=["#2A2A2A", "#2A2A2A"], text_color=["#F4F4F4", "#FFFFFF"], button_color="#9755FF", button_hover_color="#FFAA3B") 25 | provider_dropdown.pack(pady=10) 26 | 27 | self.continue_button = ctk.CTkButton(self.window, text="Continue", command=self.show_credential_inputs, state="disabled", fg_color="#9755FF", text_color="#FFFFFF", hover_color="#FFAA3B") 28 | self.continue_button.pack(pady=10) 29 | 30 | def on_provider_changed(self, *args): 31 | print(f"Provider changed to: {self.provider_var.get()}") # Debug print 32 | self.refresh_ui() 33 | 34 | def refresh_ui(self): 35 | selected_provider = self.provider_var.get() 36 | if selected_provider: 37 | self.continue_button.configure(state="normal") 38 | else: 39 | self.continue_button.configure(state="disabled") 40 | 41 | def show_credential_inputs(self): 42 | print("show_credential_inputs called") # Debug print 43 | self.provider = self.provider_var.get() 44 | if not self.provider: 45 | messagebox.showerror("Error", "Please select a provider before continuing.") 46 | return 47 | 48 | print(f"Selected provider: {self.provider}") # Debug print 49 | 50 | self.window.geometry("300x450") # Resize window for credential inputs 51 | 52 | # Clear previous credential inputs 53 | for widget in self.window.winfo_children(): 54 | widget.destroy() 55 | 56 | ctk.CTkLabel(self.window, text=f"Enter credentials for {self.provider}:", text_color=["#F4F4F4", "#FFFFFF"]).pack(pady=10) 57 | 58 | if self.provider == "openai": 59 | self.create_credential_input("OPENAI_API_KEY") 60 | self.create_credential_input("OPENAI_MODEL") 61 | elif self.provider == "azure": 62 | self.create_credential_input("OPENAI_API_KEY") 63 | self.create_credential_input("AZURE_API_KEY") 64 | self.create_credential_input("AZURE_API_BASE") 65 | self.create_credential_input("AZURE_API_VERSION") 66 | self.create_credential_input("AZURE_MODEL") 67 | elif self.provider == "anthropic": 68 | self.create_credential_input("OPENAI_API_KEY") 69 | self.create_credential_input("ANTHROPIC_API_KEY") 70 | self.create_credential_input("ANTHROPIC_MODEL") 71 | 72 | ctk.CTkButton(self.window, text="Load from file", command=self.load_from_file, fg_color="#9755FF", text_color="#FFFFFF", hover_color="#FFAA3B").pack(pady=10) 73 | ctk.CTkButton(self.window, text="Done", command=self.on_done, fg_color="#9755FF", text_color="#FFFFFF", hover_color="#FFAA3B").pack(pady=10) 74 | 75 | def create_credential_input(self, key): 76 | ctk.CTkLabel(self.window, text=key, text_color=["#F4F4F4", "#FFFFFF"]).pack() 77 | entry = ctk.CTkEntry(self.window, fg_color=["#2A2A2A", "#2A2A2A"], text_color=["#F4F4F4", "#FFFFFF"], border_color="#9755FF") 78 | entry.pack() 79 | self.credential_entries[key] = entry 80 | 81 | def load_from_file(self): 82 | try: 83 | with open('src/Settings/provider_config.json', 'r') as f: 84 | saved_config = json.load(f) 85 | 86 | for key, entry in self.credential_entries.items(): 87 | if key in saved_config: 88 | entry.delete(0, ctk.END) 89 | entry.insert(0, saved_config[key]) 90 | 91 | messagebox.showinfo("Success", "Credentials loaded from file") 92 | except FileNotFoundError: 93 | messagebox.showerror("Error", "provider_config.json not found") 94 | except json.JSONDecodeError: 95 | messagebox.showerror("Error", "Invalid JSON in provider_config.json") 96 | 97 | def on_done(self): 98 | for key, entry in self.credential_entries.items(): 99 | self.config[key] = entry.get() 100 | 101 | self.window.destroy() 102 | 103 | def get_config(self): 104 | return self.config -------------------------------------------------------------------------------- /src/Core/interpreter_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import customtkinter as ctk 4 | from Settings.config import INTERPRETER_SETTINGS, COMPUTER_SYSTEM_MESSAGE, SYSTEM_MESSAGE 5 | from interpreter import interpreter 6 | from tkinter import messagebox # Import messagebox from tkinter 7 | 8 | class InterpreterManager: 9 | def __init__(self, knowledge_manager =None, chat_ui = None): 10 | self.knowledge_manager = knowledge_manager # Initialize knowledge_manager 11 | self.chat_ui = chat_ui 12 | self.configure_interpreter() 13 | 14 | # Initialize any other necessary attributes here 15 | def configure_interpreter(self): 16 | interpreter.llm.supports_vision = INTERPRETER_SETTINGS["supports_vision"] 17 | interpreter.auto_run = INTERPRETER_SETTINGS["auto_run"] 18 | interpreter.loop = INTERPRETER_SETTINGS["loop"] 19 | interpreter.llm.temperature = INTERPRETER_SETTINGS["temperature"] 20 | interpreter.llm.max_tokens = INTERPRETER_SETTINGS["max_tokens"] 21 | interpreter.llm.context_window = INTERPRETER_SETTINGS["context_window"] 22 | interpreter.conversation_history_path = INTERPRETER_SETTINGS["conversation_history_path"] 23 | interpreter.computer.import_computer_api = INTERPRETER_SETTINGS["import_computer_api"] 24 | interpreter.computer.system_message = COMPUTER_SYSTEM_MESSAGE 25 | print(interpreter.computer.system_message) 26 | interpreter.system_message = SYSTEM_MESSAGE 27 | 28 | def configure_provider(self, provider, config): 29 | # Common for all providers 30 | os.environ["OPENAI_API_KEY"] = config.get("OPENAI_API_KEY", "") 31 | 32 | if provider == "azure": 33 | os.environ["AZURE_API_KEY"] = config["AZURE_API_KEY"] 34 | os.environ["AZURE_API_BASE"] = config["AZURE_API_BASE"] 35 | os.environ["AZURE_API_VERSION"] = config["AZURE_API_VERSION"] 36 | model = config["AZURE_MODEL"] 37 | interpreter.llm.provider = "azure" 38 | interpreter.llm.api_key = os.environ["AZURE_API_KEY"] 39 | interpreter.llm.api_base = os.environ["AZURE_API_BASE"] 40 | interpreter.llm.api_version = os.environ["AZURE_API_VERSION"] 41 | interpreter.llm.model = f"azure/{model}" 42 | elif provider == "openai": 43 | model = config["OPENAI_MODEL"] 44 | interpreter.llm.api_key = os.environ["OPENAI_API_KEY"] 45 | interpreter.llm.model = model 46 | elif provider == "anthropic": 47 | os.environ["ANTHROPIC_API_KEY"] = config["ANTHROPIC_API_KEY"] 48 | model = config["ANTHROPIC_MODEL"] 49 | interpreter.llm.api_key = os.environ["ANTHROPIC_API_KEY"] 50 | interpreter.llm.model = f"anthropic/{model}" 51 | 52 | # Prompt user to save credentials 53 | save_credentials = messagebox.askyesno("Save Credentials", "Do you want to save these credentials to provider_config.json?") 54 | 55 | if save_credentials: 56 | try: 57 | with open('src/Settings/provider_config.json', 'w') as f: 58 | json.dump(config, f) 59 | messagebox.showinfo("Success", "Credentials saved to provider_config.json") 60 | except Exception as e: 61 | messagebox.showerror("Error", f"Failed to save credentials: {str(e)}") 62 | 63 | # Set the provider in the configuration 64 | config['PROVIDER'] = provider 65 | 66 | def update_system_message(self, selected_kbs): 67 | # Use self.knowledge_manager to get available skills 68 | skills = self.knowledge_manager.get_available_skills() 69 | # Append instructions from selected knowledge bases 70 | for kb in selected_kbs: 71 | instructions = self.knowledge_manager.instructions.get(kb, "") 72 | if instructions: 73 | interpreter.system_message += "\n" + instructions 74 | # Append dynamically retrieved skills with their file paths 75 | skill_list = "\n - ".join([f"{name} (Path: {path})" for name, path in skills]) 76 | SYSTEM_MESSAGE_SKILLS = ''' 77 | ### Skills: 78 | - You have access to various skills from the selected knowledge bases. Each skill contains step-by-step instructions on how to complete specific tasks. 79 | - If you think a skill will help you complete a task, read the contents of the skill file at the provided path to see if it can help you. 80 | - If it will help you complete the task, follow the instructions strictly. Do not deviate unless specified otherwise. 81 | - If you receive an error, retry from the last checkpoint. 82 | - Here are the skills you have access to: 83 | ''' + "\n - " + skill_list 84 | interpreter.system_message = SYSTEM_MESSAGE + "\n" + SYSTEM_MESSAGE_SKILLS 85 | # Update the interpreter's system message 86 | # Print the updated system message and available skills 87 | print(f"System Message Updated:\n{interpreter.system_message}") 88 | 89 | print(f"Available Skills: {[name + ' (' + path + ')' for name, path in skills]}") -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | import customtkinter as ctk 2 | from UI.chat_window import ChatUI 3 | from UI.provider_window import ProviderSelectionUI 4 | from Core.interpreter_manager import InterpreterManager 5 | from Settings.color_settings import * 6 | import logging 7 | from Core.knowledge_manager import KnowledgeManager 8 | # Configure logging 9 | logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') 10 | 11 | print("Starting main.py") 12 | 13 | class MainApplication: 14 | def __init__(self): 15 | self.root = ctk.CTk() 16 | self.root.title("HumanScript Chat") 17 | self.root.geometry("1280x720") # 16:9 aspect ratio 18 | self.root.minsize(800, 450) # Minimum size while maintaining 16:9 19 | self.root.protocol("WM_DELETE_WINDOW", self.on_closing) 20 | 21 | # Set custom color theme 22 | self.set_custom_theme() 23 | 24 | self.provider_ui = None 25 | self.chat_ui = None 26 | self.interpreter_manager = None 27 | self.knowledge_manager = None # Added attribute 28 | 29 | def set_custom_theme(self): 30 | ctk.set_default_color_theme("dark-blue") # Use a dark built-in theme as a base 31 | 32 | # Override specific colors for dark mode 33 | ctk.ThemeManager.theme["CTk"]["fg_color"] = [get_color("BG_SECONDARY"), get_color("BG_PRIMARY")] 34 | ctk.ThemeManager.theme["CTk"]["text"] = [get_color("TEXT_SECONDARY"), get_color("TEXT_PRIMARY")] 35 | ctk.ThemeManager.theme["CTkButton"]["fg_color"] = [BRAND_PRIMARY, BRAND_SECONDARY] 36 | ctk.ThemeManager.theme["CTkButton"]["hover_color"] = [BRAND_ACCENT, BRAND_ACCENT] 37 | ctk.ThemeManager.theme["CTkButton"]["text_color"] = [get_color("TEXT_PRIMARY"), get_color("TEXT_PRIMARY")] 38 | ctk.ThemeManager.theme["CTkEntry"]["fg_color"] = [get_color("BG_INPUT"), get_color("BG_INPUT")] 39 | ctk.ThemeManager.theme["CTkEntry"]["text_color"] = [get_color("TEXT_SECONDARY"), get_color("TEXT_PRIMARY")] 40 | ctk.ThemeManager.theme["CTkEntry"]["border_color"] = [BRAND_PRIMARY, BRAND_SECONDARY] 41 | ctk.ThemeManager.theme["CTkTextbox"]["fg_color"] = [get_color("BG_INPUT"), get_color("BG_INPUT")] 42 | ctk.ThemeManager.theme["CTkTextbox"]["text_color"] = [get_color("TEXT_SECONDARY"), get_color("TEXT_PRIMARY")] 43 | ctk.ThemeManager.theme["CTkTextbox"]["border_color"] = [BRAND_PRIMARY, BRAND_SECONDARY] 44 | 45 | # Add new theme settings for the chat interface 46 | ctk.ThemeManager.theme["CTkFrame"]["fg_color"] = [get_color("BG_SECONDARY"), get_color("BG_PRIMARY")] 47 | ctk.ThemeManager.theme["CTkScrollableFrame"]["fg_color"] = [get_color("BG_SECONDARY"), get_color("BG_PRIMARY")] 48 | 49 | def start(self): 50 | self.show_provider_selection() 51 | 52 | def show_provider_selection(self): 53 | self.provider_ui = ProviderSelectionUI(self.root) 54 | self.provider_ui.create_provider_selection_ui() 55 | self.root.wait_window(self.provider_ui.window) 56 | self.process_provider_selection() 57 | 58 | def process_provider_selection(self): 59 | print("Processing provider selection") 60 | logging.debug("Processing provider selection") 61 | provider = self.provider_ui.provider 62 | config = self.provider_ui.get_config() 63 | 64 | if provider and config: 65 | self.knowledge_manager = KnowledgeManager(self.root) 66 | self.interpreter_manager = InterpreterManager(self.knowledge_manager) 67 | self.interpreter_manager.configure_provider(provider, config) 68 | self.show_chat_ui() 69 | else: 70 | self.root.quit() 71 | 72 | def show_chat_ui(self): 73 | self.root.deiconify() # Show the main window 74 | try: 75 | self.chat_ui = ChatUI(self.root) 76 | print("Chat UI created, starting mainloop") 77 | logging.debug("Chat UI created, starting mainloop") 78 | self.root.mainloop() 79 | except Exception as e: 80 | print(f"Error creating ChatUI: {e}") 81 | logging.exception("Error creating ChatUI") 82 | self.root.quit() 83 | 84 | def on_closing(self): 85 | print("Closing application") 86 | logging.debug("Closing application") 87 | self.root.quit() 88 | 89 | def main(): 90 | ctk.set_appearance_mode("Dark") # Set appearance mode to Dark 91 | app = MainApplication() 92 | app.start() 93 | 94 | if __name__ == "__main__": 95 | print("Starting main function") 96 | logging.debug("Starting main function") 97 | try: 98 | main() 99 | except Exception as e: 100 | print(f"An error occurred: {e}") 101 | logging.exception("An error occurred") 102 | -------------------------------------------------------------------------------- /src/Core/audio_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import logging 4 | import tempfile 5 | import numpy as np 6 | import pygame 7 | import openai 8 | import speech_recognition as sr 9 | from Settings.config import * 10 | 11 | class AudioManager: 12 | def __init__(self): 13 | self.client = openai.OpenAI(api_key=os.getenv("OPENAI_API_KEY")) 14 | self.wake_word = WAKE_WORD 15 | self.is_listening = False 16 | #pygame.mixer.init(frequency=44100, size=-16, channels=2) 17 | # Initialize the mixer once with consistent parameters 18 | 19 | def generate_beep(self): 20 | print("Beep") 21 | # t = np.linspace(0, BEEP_DURATION, int(44100 * BEEP_DURATION), False) 22 | # beep = np.sin(2 * np.pi * BEEP_FREQUENCY * t) 23 | # beep = (beep * 32767).astype(np.int16) 24 | # stereo_beep = np.column_stack((beep, beep)) # Create stereo audio 25 | # sound = pygame.sndarray.make_sound(stereo_beep) 26 | # sound.play() 27 | 28 | def recognize_speech(self): 29 | if pygame.mixer.get_init(): 30 | pygame.mixer.music.stop() 31 | 32 | recognizer = sr.Recognizer() 33 | logging.info("Listening for command...") 34 | 35 | try: 36 | with sr.Microphone() as source: 37 | logging.info("Adjusting for ambient noise...") 38 | recognizer.adjust_for_ambient_noise(source, duration=1) 39 | logging.info("Listening...") 40 | audio = recognizer.listen(source, timeout=5, phrase_time_limit=10) 41 | 42 | logging.info("Audio captured, attempting to transcribe...") 43 | 44 | with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio: 45 | temp_audio.write(audio.get_wav_data()) 46 | temp_audio_path = temp_audio.name 47 | 48 | with open(temp_audio_path, "rb") as audio_file: 49 | transcript = self.client.audio.transcriptions.create( 50 | model="whisper-1", 51 | file=audio_file 52 | ) 53 | 54 | user_input = transcript.text 55 | logging.info(f"Transcribed: {user_input}") 56 | return user_input 57 | 58 | except sr.WaitTimeoutError: 59 | logging.warning("Listening timed out. No speech detected.") 60 | return "Listening timed out. Please try again." 61 | except Exception as e: 62 | logging.error(f"Error in speech recognition: {str(e)}") 63 | return f"Error in speech recognition: {str(e)}" 64 | finally: 65 | if 'temp_audio_path' in locals(): 66 | try: 67 | os.unlink(temp_audio_path) 68 | logging.info(f"Temporary audio file deleted: {temp_audio_path}") 69 | except FileNotFoundError: 70 | logging.warning(f"Could not delete temporary file: {temp_audio_path}") 71 | 72 | def text_to_speech(self, text): 73 | response = self.client.audio.speech.create( 74 | model=TTS_SETTINGS["model"], 75 | voice=TTS_SETTINGS["voice"], 76 | input=text 77 | ) 78 | 79 | with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as temp_audio: 80 | temp_audio.write(response.content) 81 | temp_audio_path = temp_audio.name 82 | 83 | pygame.mixer.music.load(temp_audio_path) 84 | pygame.mixer.music.play() 85 | while pygame.mixer.music.get_busy(): 86 | pygame.time.Clock().tick(10) 87 | if not self.is_listening: # Add this line to check if playback should be stopped 88 | break 89 | 90 | pygame.mixer.music.unload() 91 | time.sleep(0.1) 92 | 93 | try: 94 | os.unlink(temp_audio_path) 95 | except FileNotFoundError: 96 | logging.warning(f"Could not delete temporary file: {temp_audio_path}") 97 | 98 | def stop_playback(self): 99 | if pygame.mixer.get_init() and pygame.mixer.music.get_busy(): 100 | pygame.mixer.music.stop() 101 | logging.info("Audio playback interrupted") 102 | 103 | def listen_for_wake_word(self, wake_word): 104 | recognizer = sr.Recognizer() 105 | try: 106 | with sr.Microphone() as source: 107 | logging.info("Listening for wake word...") 108 | audio = recognizer.listen(source, phrase_time_limit=3) 109 | text = recognizer.recognize_google(audio).lower() 110 | logging.info(f"Heard: {text}") 111 | return wake_word in text 112 | except sr.UnknownValueError: 113 | return False 114 | except Exception as e: 115 | logging.error(f"Error listening for wake word: {str(e)}") 116 | return False -------------------------------------------------------------------------------- /src/Core/knowledge_manager.py: -------------------------------------------------------------------------------- 1 | from langchain_community.document_loaders import DirectoryLoader, WebBaseLoader 2 | from langchain.text_splitter import RecursiveCharacterTextSplitter 3 | from langchain.schema import Document 4 | from langchain_openai import OpenAIEmbeddings 5 | from langchain_community.vectorstores import Chroma 6 | from dotenv import load_dotenv 7 | import os 8 | from Settings.config import * 9 | import logging 10 | import shutil 11 | 12 | # Load environment variables 13 | load_dotenv() 14 | 15 | class KnowledgeManager: 16 | def __init__(self, root): 17 | self.root = root 18 | self.selected_kbs = [] 19 | self.skills = {} # {knowledge_base: [(skill_name, skill_path), ...]} 20 | self.instructions = {} 21 | 22 | def get_knowledge_bases(self): 23 | return [d for d in os.listdir(KB_PATH) if os.path.isdir(os.path.join(KB_PATH, d))] 24 | 25 | def update_selected_kbs(self, selected_kbs): 26 | # Remove skills and instructions from previously selected KBs 27 | previous_kbs = set(self.selected_kbs) - set(selected_kbs) 28 | for kb in previous_kbs: 29 | self.remove_skills(kb) 30 | self.remove_instructions(kb) 31 | 32 | # Add skills and instructions from newly selected KBs 33 | new_kbs = set(selected_kbs) - set(self.selected_kbs) 34 | for kb in new_kbs: 35 | self.load_skills_and_instructions(kb) 36 | 37 | self.selected_kbs = selected_kbs 38 | 39 | def remove_skills(self, knowledge_base): 40 | if knowledge_base in self.skills: 41 | del self.skills[knowledge_base] 42 | logging.info(f"Removed skills for {knowledge_base}") 43 | 44 | # Print available skills after removal 45 | print(f"Available Skills after removal: {self.get_available_skills()}") 46 | 47 | def remove_instructions(self, knowledge_base): 48 | if knowledge_base in self.instructions: 49 | del self.instructions[knowledge_base] 50 | logging.info(f"Removed instructions for {knowledge_base}") 51 | # Print instructions after removal 52 | print(f"Instructions after removal for '{knowledge_base}': {self.instructions}") 53 | 54 | def load_skills_and_instructions(self, knowledge_base): 55 | kb_path = os.path.join(KB_PATH, knowledge_base) 56 | skills_path = os.path.join(kb_path, "skills") 57 | instructions_file = os.path.join(kb_path, "instructions.txt") # Changed to .txt as per the latest file 58 | 59 | # Load skills with their file paths 60 | if os.path.exists(skills_path): 61 | skill_files = [f for f in os.listdir(skills_path) if f.endswith('.md')] 62 | self.skills[knowledge_base] = [(f, os.path.join(skills_path, f)) for f in skill_files] 63 | else: 64 | self.skills[knowledge_base] = [] 65 | 66 | # Load instructions 67 | if os.path.exists(instructions_file): 68 | with open(instructions_file, 'r') as f: 69 | self.instructions[knowledge_base] = f.read() 70 | else: 71 | self.instructions[knowledge_base] = "" 72 | 73 | # Print available skills after addition 74 | print(f"Available Skills after addition: {self.get_available_skills()}") 75 | 76 | def get_available_skills(self): 77 | # Collect skills and their paths from all selected knowledge bases 78 | skills = [] 79 | for kb in self.selected_kbs: 80 | skills.extend(self.skills.get(kb, [])) 81 | return skills 82 | 83 | def load_docs_folder(self, knowledge_base): 84 | kb_path = os.path.join(KB_PATH, knowledge_base) 85 | docs_path = os.path.join(kb_path, "docs") 86 | urls_file = os.path.join(kb_path, "urls.txt") 87 | 88 | # Load and process documents 89 | documents = self.load_documents(docs_path, urls_file) 90 | chunks = self.split_text(documents) 91 | self.save_to_chroma(chunks, knowledge_base) 92 | 93 | def load_documents(self, docs_path, urls_file): 94 | # Load local documents 95 | local_loader = DirectoryLoader(docs_path, glob="*.*") 96 | local_documents = local_loader.load() 97 | print(f"Loaded {len(local_documents)} local documents from {docs_path}") 98 | 99 | # Load URLs 100 | url_documents = self.load_urls(urls_file) 101 | 102 | # Combine local and URL documents 103 | all_documents = local_documents + url_documents 104 | print(f"Total documents loaded: {len(all_documents)}") 105 | return all_documents 106 | 107 | def load_urls(self, urls_file): 108 | if not os.path.exists(urls_file): 109 | print(f"No {urls_file} found. Skipping URL loading.") 110 | return [] 111 | 112 | with open(urls_file, 'r') as file: 113 | urls = file.read().splitlines() 114 | 115 | url_documents = [] 116 | for url in urls: 117 | try: 118 | loader = WebBaseLoader(url) 119 | url_documents.extend(loader.load()) 120 | print(f"Loaded content from: {url}") 121 | except Exception as e: 122 | print(f"Error loading {url}: {e}") 123 | 124 | print(f"Loaded {len(url_documents)} documents from URLs") 125 | return url_documents 126 | 127 | def split_text(self, documents: list[Document]): 128 | text_splitter = RecursiveCharacterTextSplitter( 129 | chunk_size=1000, 130 | chunk_overlap=200, 131 | length_function=len, 132 | add_start_index=True, 133 | ) 134 | chunks = text_splitter.split_documents(documents) 135 | print(f"Split {len(documents)} documents into {len(chunks)} chunks.") 136 | return chunks 137 | 138 | def save_to_chroma(self, chunks: list[Document], knowledge_base: str): 139 | db_path = os.path.join(CHROMA_PATH, knowledge_base) 140 | embedding_function = OpenAIEmbeddings() 141 | 142 | # Check if the database already exists 143 | if os.path.exists(db_path): 144 | # Load the existing database 145 | db = Chroma(persist_directory=db_path, embedding_function=embedding_function) 146 | 147 | # Add new documents to the existing database 148 | db.add_documents(chunks) 149 | else: 150 | # Create a new database if it doesn't exist 151 | db = Chroma.from_documents(chunks, embedding_function, persist_directory=db_path) 152 | 153 | # Persist the changes 154 | db.persist() 155 | print(f"Updated database with {len(chunks)} chunks in {db_path}.") 156 | 157 | def build_vector_database(self, knowledge_base=None): 158 | if knowledge_base: 159 | kb_path = os.path.join(KB_PATH, knowledge_base) 160 | if os.path.isdir(kb_path): 161 | print(f"Processing knowledge base: {knowledge_base}") 162 | self.load_docs_folder(knowledge_base) 163 | else: 164 | print(f"Creating new knowledge base.") 165 | os.makedirs(os.path.join(kb_path, "docs"), exist_ok=True) 166 | os.makedirs(os.path.join(kb_path, "skills"), exist_ok=True) 167 | with open(os.path.join(kb_path, "urls.txt"), 'w') as f: 168 | pass # Create an empty urls.txt file 169 | with open(os.path.join(kb_path, "instructions.txt"), 'w') as f: 170 | pass # Create an empty instructions.txt file 171 | self.load_docs_folder(knowledge_base) 172 | else: 173 | # Update all knowledge bases 174 | knowledge_bases = self.get_knowledge_bases() 175 | 176 | for kb in knowledge_bases: 177 | print(f"Processing knowledge base: {kb}") 178 | self.load_docs_folder(kb) 179 | 180 | def add_to_knowledge_base(self, kb_name, url=None, file_path=None): 181 | kb_path = os.path.join(KB_PATH, kb_name) 182 | if not os.path.exists(kb_path): 183 | os.makedirs(os.path.join(kb_path, "docs")) 184 | with open(os.path.join(kb_path, "urls.txt"), 'w') as f: 185 | pass # Create an empty urls.txt file 186 | 187 | if url: 188 | urls_file = os.path.join(kb_path, "urls.txt") 189 | with open(urls_file, 'a') as f: 190 | f.write(url + "\n") 191 | 192 | if file_path: 193 | docs_path = os.path.join(kb_path, "docs") 194 | os.makedirs(docs_path, exist_ok=True) 195 | file_name = os.path.basename(file_path) 196 | dest_path = os.path.join(docs_path, file_name) 197 | logging.info(f"Copying file from {file_path} to {dest_path}") 198 | shutil.copy2(file_path, dest_path) 199 | 200 | print(f"Added to knowledge base: {kb_name}") -------------------------------------------------------------------------------- /src/UI/settings_window.py: -------------------------------------------------------------------------------- 1 | import customtkinter as ctk 2 | from tkinter import messagebox, simpledialog, filedialog 3 | from Settings.config import INTERPRETER_SETTINGS, CHROMA_PATH, KB_PATH 4 | import json 5 | import os 6 | import shutil 7 | from interpreter import interpreter 8 | import importlib 9 | from Settings.color_settings import * 10 | 11 | class SettingsWindow: 12 | def __init__(self, parent, chat_ui): 13 | self.parent = parent 14 | self.chat_ui = chat_ui 15 | self.selected_files = [] 16 | self.selected_urls = [] 17 | self.selected_kb = ctk.StringVar() 18 | self.create_widgets() 19 | self.load_current_settings() 20 | 21 | def create_widgets(self): 22 | # Create a scrollable frame for settings 23 | self.scrollable_frame = ctk.CTkScrollableFrame(self.parent, fg_color=get_color("BG_PRIMARY")) 24 | self.scrollable_frame.pack(padx=10, pady=10, fill="both", expand=True) 25 | 26 | # Interpreter Settings 27 | self.create_collapsible_section("Interpreter Settings", self.create_interpreter_settings) 28 | 29 | # Wake Word 30 | self.create_collapsible_section("Wake Word", self.create_wake_word_settings) 31 | 32 | # Environment Variables 33 | self.create_collapsible_section("Environment Variables", self.create_env_var_settings) 34 | 35 | # Knowledge Base Management 36 | self.create_collapsible_section("Knowledge Base Management", self.create_kb_management_settings) 37 | 38 | # Buttons 39 | button_frame = ctk.CTkFrame(self.scrollable_frame, fg_color=get_color("BG_PRIMARY")) 40 | button_frame.pack(pady=20, anchor="w") 41 | ctk.CTkButton(button_frame, text="Save", command=self.save_settings, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY"), hover_color=get_color("BG_SECONDARY")).pack(side=ctk.LEFT, padx=10) 42 | ctk.CTkButton(button_frame, text="Cancel", command=self.cancel, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY"), hover_color=get_color("BG_SECONDARY")).pack(side=ctk.LEFT) 43 | ctk.CTkButton(button_frame, text="Reset Chat", command=self.reset_interpreter, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY"), hover_color=get_color("BG_SECONDARY")).pack(side=ctk.LEFT, padx=10) 44 | 45 | def create_collapsible_section(self, title, create_content_func): 46 | section_frame = ctk.CTkFrame(self.scrollable_frame, fg_color=get_color("BG_TERTIARY")) 47 | section_frame.pack(padx=10, pady=5, fill="x", anchor="w") 48 | 49 | section_button = ctk.CTkButton(section_frame, text=title, command=lambda: self.toggle_section(section_content), fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY"), hover_color=get_color("BG_SECONDARY"), border_width=2, border_color=get_color("TEXT_PRIMARY"), font=("Helvetica", 16)) 50 | section_button.pack(fill="x") 51 | 52 | section_content = ctk.CTkFrame(section_frame, fg_color=get_color("BG_PRIMARY")) 53 | section_content.pack(fill="x", expand=True, anchor="w") 54 | create_content_func(section_content) 55 | section_content.pack_forget() 56 | 57 | def toggle_section(self, section_content): 58 | if section_content.winfo_ismapped(): 59 | section_content.pack_forget() 60 | else: 61 | section_content.pack(fill="x", expand=True, anchor="w") 62 | 63 | def create_interpreter_settings(self, parent): 64 | self.supports_vision_var = ctk.BooleanVar() 65 | ctk.CTkCheckBox(parent, text="Supports Vision", variable=self.supports_vision_var, text_color=get_color("TEXT_PRIMARY")).pack(pady=2, anchor="w") 66 | 67 | self.auto_run_var = ctk.BooleanVar() 68 | ctk.CTkCheckBox(parent, text="Auto Run", variable=self.auto_run_var, text_color=get_color("TEXT_PRIMARY")).pack(pady=2, anchor="w") 69 | 70 | self.loop_var = ctk.BooleanVar() 71 | ctk.CTkCheckBox(parent, text="Loop", variable=self.loop_var, text_color=get_color("TEXT_PRIMARY")).pack(pady=2, anchor="w") 72 | 73 | ctk.CTkLabel(parent, text="Temperature:", text_color=get_color("TEXT_PRIMARY")).pack(pady=2, anchor="w") 74 | self.temperature_var = ctk.DoubleVar() 75 | ctk.CTkEntry(parent, textvariable=self.temperature_var, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY")).pack(pady=2, anchor="w") 76 | 77 | ctk.CTkLabel(parent, text="Max Tokens:", text_color=get_color("TEXT_PRIMARY")).pack(pady=2, anchor="w") 78 | self.max_tokens_var = ctk.IntVar() 79 | ctk.CTkEntry(parent, textvariable=self.max_tokens_var, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY")).pack(pady=2, anchor="w") 80 | 81 | ctk.CTkLabel(parent, text="Context Window:", text_color=get_color("TEXT_PRIMARY")).pack(pady=2, anchor="w") 82 | self.context_window_var = ctk.IntVar() 83 | ctk.CTkEntry(parent, textvariable=self.context_window_var, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY")).pack(pady=2, anchor="w") 84 | 85 | def create_wake_word_settings(self, parent): 86 | ctk.CTkLabel(parent, text="Wake Word:", text_color=get_color("TEXT_PRIMARY")).pack(pady=5, anchor="w") 87 | self.wake_word_entry = ctk.CTkEntry(parent, width=50, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY")) 88 | self.wake_word_entry.pack(pady=5, anchor="w") 89 | 90 | def create_env_var_settings(self, parent): 91 | self.env_vars = {} 92 | self.refresh_env_vars(parent) 93 | 94 | ctk.CTkButton(parent, text="Add Environment Variable", command=self.add_env_var, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY"), hover_color=get_color("BG_SECONDARY")).pack(pady=5, anchor="w") 95 | 96 | def refresh_env_vars(self, env_frame): 97 | for widget in env_frame.winfo_children(): 98 | widget.destroy() 99 | 100 | for key, value in os.environ.items(): 101 | if key.startswith("CUSTOM_"): 102 | self.env_vars[key] = ctk.StringVar(value=value) 103 | row_frame = ctk.CTkFrame(env_frame, fg_color=get_color("BG_TERTIARY")) 104 | row_frame.pack(fill=ctk.X, padx=5, pady=2, anchor="w") 105 | ctk.CTkLabel(row_frame, text=key, text_color=get_color("TEXT_PRIMARY")).pack(side=ctk.LEFT) 106 | ctk.CTkEntry(row_frame, textvariable=self.env_vars[key], show="*", fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY")).pack(side=ctk.RIGHT, expand=True, fill=ctk.X) 107 | 108 | def add_env_var(self): 109 | key = simpledialog.askstring("Add Environment Variable", "Enter variable name (will be prefixed with CUSTOM_):") 110 | if key: 111 | key = f"CUSTOM_{key.upper().replace(' ', '_')}" 112 | value = simpledialog.askstring("Add Environment Variable", f"Enter value for {key}:") 113 | if value: 114 | os.environ[key] = value 115 | self.env_vars[key] = ctk.StringVar(value=value) 116 | env_frame = self.parent.children['!ctkframe3'] 117 | row_frame = ctk.CTkFrame(env_frame, fg_color=get_color("BG_TERTIARY")) 118 | row_frame.pack(fill=ctk.X, padx=5, pady=2, anchor="w") 119 | ctk.CTkLabel(row_frame, text=key, text_color=get_color("TEXT_PRIMARY")).pack(side=ctk.LEFT) 120 | ctk.CTkEntry(row_frame, textvariable=self.env_vars[key], show="*", fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY")).pack(side=ctk.RIGHT, expand=True, fill=ctk.X) 121 | 122 | def create_kb_management_settings(self, parent): 123 | ctk.CTkLabel(parent, text="Select files and URLs to be added to the knowledge base:", text_color=get_color("TEXT_PRIMARY")).pack(pady=5, anchor="w") 124 | 125 | button_frame = ctk.CTkFrame(parent, fg_color=get_color("BG_PRIMARY")) 126 | button_frame.pack(pady=5, anchor="w", fill="x") 127 | 128 | ctk.CTkButton(button_frame, text="Add File", command=self.add_file, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY"), hover_color=get_color("BG_SECONDARY")).pack(side="left", padx=(0, 5)) 129 | ctk.CTkButton(button_frame, text="Add URL", command=self.add_url, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY"), hover_color=get_color("BG_SECONDARY")).pack(side="left") 130 | 131 | self.kb_dropdown = ctk.CTkComboBox(parent, values=self.chat_ui.knowledge_manager.get_knowledge_bases() + ["New Knowledge Base"], fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY")) 132 | self.kb_dropdown.pack(pady=10, anchor="w") 133 | 134 | action_frame = ctk.CTkFrame(parent, fg_color=get_color("BG_PRIMARY")) 135 | action_frame.pack(pady=5, anchor="w", fill="x") 136 | 137 | ctk.CTkButton(action_frame, text="Add to Knowledge Base", command=self.submit_kb, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY"), hover_color=get_color("BG_SECONDARY")).pack(side="left", padx=(0, 5)) 138 | ctk.CTkButton(action_frame, text="Clear Queue", command=self.clear_queue, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY"), hover_color=get_color("BG_SECONDARY")).pack(side="left") 139 | 140 | ctk.CTkLabel(parent, text="Queue:", text_color=get_color("TEXT_PRIMARY")).pack(pady=(10, 5), anchor="w") 141 | self.queue_display = ctk.CTkTextbox(parent, height=100, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY")) 142 | self.queue_display.pack(pady=5, fill="x", expand=True) 143 | 144 | ctk.CTkButton(parent, text="Refresh Selected Knowledge Base", command=self.refresh_selected_kb, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY"), hover_color=get_color("BG_SECONDARY")).pack(pady=(10, 5), anchor="w") 145 | 146 | def clear_queue(self): 147 | self.selected_files.clear() 148 | self.selected_urls.clear() 149 | self.update_queue_display() 150 | 151 | def update_queue_display(self): 152 | self.queue_display.delete("1.0", ctk.END) 153 | for file in self.selected_files: 154 | self.queue_display.insert(ctk.END, f"File: {os.path.basename(file)}\n") 155 | for url in self.selected_urls: 156 | self.queue_display.insert(ctk.END, f"URL: {url}\n") 157 | 158 | def add_file(self): 159 | file_paths = filedialog.askopenfilenames() 160 | if file_paths: 161 | self.selected_files.extend(file_paths) 162 | self.update_queue_display() 163 | messagebox.showinfo("Files Added", f"Added {len(file_paths)} files.") 164 | 165 | def add_url(self): 166 | url = simpledialog.askstring("Add URL", "Enter URL:") 167 | if url: 168 | self.selected_urls.append(url) 169 | self.update_queue_display() 170 | messagebox.showinfo("URL Added", f"Added URL: {url}") 171 | 172 | def submit_kb(self): 173 | kb_name = self.kb_dropdown.get() 174 | if kb_name == "New Knowledge Base": 175 | kb_name = simpledialog.askstring("New Knowledge Base", "Enter name for the new knowledge base:") 176 | if not kb_name: 177 | messagebox.showerror("Error", "Please enter a name for the new knowledge base.") 178 | return 179 | 180 | kb_path = os.path.join(KB_PATH, kb_name) 181 | os.makedirs(kb_path, exist_ok=True) 182 | 183 | docs_path = os.path.join(kb_path, "docs") 184 | os.makedirs(docs_path, exist_ok=True) 185 | 186 | urls_path = os.path.join(kb_path, "urls.txt") 187 | 188 | for file_path in self.selected_files: 189 | shutil.copy(file_path, docs_path) 190 | 191 | with open(urls_path, "a") as f: 192 | for url in self.selected_urls: 193 | f.write(url + "\n") 194 | 195 | self.selected_files.clear() 196 | self.selected_urls.clear() 197 | self.update_queue_display() 198 | messagebox.showinfo("Knowledge Base Updated", f"Files and URLs have been added to {kb_name}.") 199 | 200 | def refresh_selected_kb(self): 201 | selected_kb = self.kb_dropdown.get() 202 | if selected_kb == "New Knowledge Base": 203 | messagebox.showwarning("Invalid Selection", "Please select an existing knowledge base to refresh.") 204 | return 205 | 206 | try: 207 | self.chat_ui.knowledge_manager.build_vector_database(selected_kb) 208 | self.chat_ui.refresh_knowledge_bases() # Update the UI 209 | messagebox.showinfo("Knowledge Base Refreshed", f"The knowledge base '{selected_kb}' has been refreshed.") 210 | except Exception as e: 211 | messagebox.showerror("Error", f"An error occurred while refreshing the knowledge base: {str(e)}") 212 | 213 | def load_current_settings(self): 214 | self.wake_word_entry.insert(0, self.chat_ui.wake_word) 215 | 216 | self.supports_vision_var.set(self.chat_ui.interpreter_settings["supports_vision"]) 217 | self.auto_run_var.set(self.chat_ui.interpreter_settings["auto_run"]) 218 | self.loop_var.set(self.chat_ui.interpreter_settings["loop"]) 219 | self.temperature_var.set(self.chat_ui.interpreter_settings["temperature"]) 220 | self.max_tokens_var.set(self.chat_ui.interpreter_settings["max_tokens"]) 221 | self.context_window_var.set(self.chat_ui.interpreter_settings["context_window"]) 222 | 223 | # Load environment variables 224 | for key, value in self.chat_ui.env_vars.items(): 225 | if key in self.env_vars: 226 | self.env_vars[key].set(value) 227 | 228 | def save_settings(self): 229 | # Update ChatUI attributes 230 | self.chat_ui.wake_word = self.wake_word_entry.get().strip() 231 | 232 | # Update interpreter settings through ChatUI 233 | self.chat_ui.update_interpreter_settings({ 234 | "supports_vision": self.supports_vision_var.get(), 235 | "auto_run": self.auto_run_var.get(), 236 | "loop": self.loop_var.get(), 237 | "temperature": self.temperature_var.get(), 238 | "max_tokens": self.max_tokens_var.get(), 239 | "context_window": self.context_window_var.get() 240 | }) 241 | 242 | # Update environment variables 243 | self.chat_ui.update_env_vars({key: var.get() for key, var in self.env_vars.items()}) 244 | 245 | # Notify the user 246 | messagebox.showinfo("Settings Saved", "Your settings have been successfully updated.") 247 | # Update the sidebar checkboxes 248 | self.chat_ui.refresh_knowledge_bases() 249 | # Return to chat window 250 | self.return_to_chat() 251 | 252 | def cancel(self): 253 | # Return to chat window 254 | self.return_to_chat() 255 | 256 | def return_to_chat(self): 257 | # Clear the settings UI and recreate the chat window 258 | for widget in self.parent.winfo_children(): 259 | widget.destroy() 260 | self.chat_ui.create_chat_window(self.parent) 261 | self.chat_ui.create_input_area(self.parent) 262 | 263 | def reset_interpreter(self): 264 | interpreter.reset() 265 | messagebox.showinfo("Interpreter Reset", "The interpreter has been reset.") 266 | self.return_to_chat() -------------------------------------------------------------------------------- /src/UI/chat_window.py: -------------------------------------------------------------------------------- 1 | import customtkinter as ctk 2 | import os 3 | import threading 4 | import logging 5 | from datetime import datetime 6 | from Settings.config import * 7 | from Core.chat_manager import ChatManager 8 | from Core.audio_manager import AudioManager 9 | from Core.knowledge_manager import KnowledgeManager 10 | from Core.context_manager import ContextManager 11 | from Core.interpreter_manager import InterpreterManager 12 | from interpreter import interpreter 13 | from UI.settings_window import SettingsWindow 14 | from Settings.color_settings import * 15 | import re 16 | from collections import OrderedDict, deque 17 | 18 | class ChatUI: 19 | def __init__(self, root): 20 | self.root = root 21 | self.root.title("HumanScript Chat") 22 | 23 | self.selected_kbs = DEFAULT_SELECTED_KBS.copy() 24 | self.wake_word = WAKE_WORD 25 | self.is_listening = False 26 | self.listen_thread = None 27 | self.is_voice_mode = False 28 | self.continuous_listen_thread = None 29 | self.interpreter_settings = INTERPRETER_SETTINGS.copy() 30 | self.env_vars = OrderedDict() 31 | self.streaming_message = "" 32 | self.last_message_type = None # To keep track of the last message type (user or AI) 33 | 34 | # Initialize KnowledgeManager first 35 | self.knowledge_manager = KnowledgeManager(self) 36 | self.interpreter_manager = InterpreterManager(self.knowledge_manager) 37 | self.chat_manager = ChatManager(self.interpreter_manager, self) 38 | self.audio_manager = AudioManager() 39 | 40 | self.context_manager = ContextManager(self) 41 | 42 | self.input_box = ctk.CTkTextbox(root, height=50, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY")) 43 | 44 | self.create_ui() 45 | self.streaming_label = None 46 | self.stream_lines = deque(maxlen=10) 47 | self.partial_line = "" # Added buffer for partial lines 48 | self.word_buffer = [] # Added buffer for words 49 | 50 | 51 | def create_ui(self): 52 | self.root.grid_rowconfigure(0, weight=1) 53 | self.root.grid_columnconfigure(1, weight=1) 54 | 55 | # Sidebar 56 | self.sidebar = ctk.CTkFrame(self.root, width=250, fg_color=get_color("BG_SECONDARY")) 57 | self.sidebar.grid(row=0, column=0, sticky="nsew") 58 | self.create_sidebar() 59 | 60 | # Main chat area 61 | self.main_frame = ctk.CTkFrame(self.root, fg_color=get_color("BG_PRIMARY")) 62 | self.main_frame.grid(row=0, column=1, sticky="nsew", padx=10, pady=10) 63 | self.main_frame.grid_rowconfigure(0, weight=1) 64 | self.main_frame.grid_columnconfigure(0, weight=1) 65 | 66 | self.create_chat_window(self.main_frame) 67 | self.create_input_area(self.main_frame) 68 | 69 | self.root.bind('', self.send_message) 70 | 71 | def create_sidebar(self): 72 | # Knowledge base section 73 | kb_label = ctk.CTkLabel(self.sidebar, text="Knowledge Bases:", anchor="w") 74 | kb_label.pack(padx=20, pady=(20, 5), fill="x") 75 | 76 | self.kb_frame = ctk.CTkScrollableFrame(self.sidebar, fg_color="transparent") 77 | self.kb_frame.pack(padx=20, pady=(0, 20), fill="both", expand=True) 78 | 79 | self.kb_toggles = {} 80 | for kb in self.knowledge_manager.get_knowledge_bases(): 81 | toggle = ctk.CTkCheckBox(self.kb_frame, text=kb, command=lambda kb=kb: self.toggle_kb(kb)) 82 | toggle.pack(anchor="w", pady=2) 83 | self.kb_toggles[kb] = toggle 84 | 85 | # Add settings icon to the sidebar 86 | settings_icon = ctk.CTkButton( 87 | self.sidebar, 88 | text="⚙️ Settings", 89 | command=self.open_settings, 90 | fg_color=get_color("BG_INPUT"), 91 | text_color=BRAND_PRIMARY, 92 | hover_color=BRAND_ACCENT, 93 | border_width=2, 94 | border_color=BRAND_PRIMARY, 95 | font=("Helvetica", 16) 96 | ) 97 | settings_icon.pack(pady=10, padx=20, fill="x") 98 | 99 | def refresh_knowledge_bases(self): 100 | # Clear the current knowledge base toggles 101 | for widget in self.kb_frame.winfo_children(): 102 | widget.destroy() 103 | 104 | # Recreate the knowledge base toggles 105 | self.kb_toggles = {} 106 | for kb in self.knowledge_manager.get_knowledge_bases(): 107 | toggle = ctk.CTkCheckBox(self.kb_frame, text=kb, command=lambda kb=kb: self.toggle_kb(kb)) 108 | toggle.pack(anchor="w", pady=2) 109 | self.kb_toggles[kb] = toggle 110 | 111 | # Refresh the toggles based on the current selection 112 | self.refresh_kb_toggles() 113 | 114 | # Update the selected knowledge bases in the knowledge manager 115 | self.knowledge_manager.update_selected_kbs(self.selected_kbs) 116 | 117 | def create_chat_window(self, parent): 118 | chat_frame = ctk.CTkFrame(parent, fg_color=get_color("BG_TERTIARY")) 119 | chat_frame.grid(row=0, column=0, sticky="nsew", padx=5, pady=5) 120 | chat_frame.grid_rowconfigure(0, weight=1) 121 | chat_frame.grid_columnconfigure(0, weight=1) 122 | 123 | self.chat_canvas = ctk.CTkCanvas(chat_frame, bg=get_color("BG_TERTIARY"), highlightthickness=0) 124 | self.chat_canvas.grid(row=0, column=0, sticky="nsew") 125 | 126 | self.message_frame = ctk.CTkFrame(self.chat_canvas, fg_color=get_color("BG_TERTIARY")) 127 | self.canvas_frame = self.chat_canvas.create_window((0, 0), window=self.message_frame, anchor="nw") 128 | 129 | scrollbar = ctk.CTkScrollbar(chat_frame, command=self.chat_canvas.yview) 130 | scrollbar.grid(row=0, column=1, sticky="ns") 131 | 132 | self.chat_canvas.configure(yscrollcommand=scrollbar.set) 133 | 134 | self.message_frame.bind("", self.on_frame_configure) 135 | self.chat_canvas.bind("", self.on_canvas_configure) 136 | 137 | def on_frame_configure(self, event): 138 | self.chat_canvas.configure(scrollregion=self.chat_canvas.bbox("all")) 139 | 140 | def on_canvas_configure(self, event): 141 | self.chat_canvas.itemconfig(self.canvas_frame, width=event.width) 142 | 143 | def create_chat_bubble(self, message, is_user): 144 | bubble_frame = ctk.CTkFrame(self.message_frame, fg_color=BRAND_PRIMARY if is_user else BRAND_SECONDARY) 145 | bubble_frame.pack(pady=5, padx=10, anchor="e" if is_user else "w") 146 | 147 | message_label = ctk.CTkLabel( 148 | bubble_frame, 149 | text=message, 150 | wraplength=400, 151 | justify="right" if is_user else "left", 152 | text_color=get_color("TEXT_PRIMARY") 153 | ) 154 | message_label.pack(padx=10, pady=5) 155 | 156 | self.chat_canvas.update_idletasks() 157 | self.chat_canvas.yview_moveto(1.0) 158 | 159 | def create_streaming_label(self): 160 | self.streaming_label = ctk.CTkLabel( 161 | self.message_frame, 162 | text="", 163 | wraplength=400, 164 | justify="left", 165 | text_color=get_color("TEXT_PRIMARY") 166 | ) 167 | self.streaming_label.pack(pady=5, padx=10, anchor="w") 168 | 169 | def update_streaming_label(self, text): 170 | self.partial_line += text # Append new text to the buffer 171 | 172 | # Split the incoming text into words 173 | words = self.partial_line.split() 174 | 175 | for word in words: 176 | self.word_buffer.append(word) 177 | if len(self.word_buffer) >= 10: # Aggregate 10 words per line 178 | line = ' '.join(self.word_buffer) 179 | self.stream_lines.append(line) 180 | self.word_buffer = [] 181 | 182 | # Handle any remaining words 183 | if self.word_buffer: 184 | line = ' '.join(self.word_buffer) 185 | self.stream_lines.append(line) 186 | self.word_buffer = [] 187 | 188 | displayed_text = '\n'.join(self.stream_lines) 189 | 190 | if self.streaming_label: 191 | self.streaming_label.configure(text=displayed_text) 192 | self.chat_canvas.update_idletasks() 193 | self.chat_canvas.yview_moveto(1.0) 194 | 195 | def create_input_area(self, parent): 196 | input_frame = ctk.CTkFrame(parent, fg_color=get_color("BG_PRIMARY")) 197 | input_frame.grid(row=1, column=0, sticky="ew", padx=5, pady=5) 198 | input_frame.grid_columnconfigure(0, weight=1) 199 | 200 | self.input_box = ctk.CTkTextbox(input_frame, height=50, fg_color=get_color("BG_INPUT"), text_color=get_color("TEXT_PRIMARY")) 201 | self.input_box.grid(row=0, column=0, sticky="ew", padx=(0, 5)) 202 | 203 | # Use Unicode microphone character 204 | self.mode_button = ctk.CTkButton(input_frame, text="🎤", width=50, height=50, command=self.toggle_mode, 205 | fg_color=get_color("BG_INPUT"), text_color=BRAND_PRIMARY, 206 | hover_color=BRAND_ACCENT, border_width=2, border_color=BRAND_PRIMARY, 207 | font=("Helvetica", 16)) 208 | self.mode_button.grid(row=0, column=1) 209 | 210 | def toggle_mode(self): 211 | self.is_voice_mode = not self.is_voice_mode 212 | if self.is_voice_mode: 213 | self.mode_button.configure(fg_color=BRAND_PRIMARY, text_color=get_color("TEXT_PRIMARY"), border_width=0) 214 | self.input_box.configure(state="disabled") 215 | self.start_listening() 216 | else: 217 | self.mode_button.configure(fg_color="transparent", text_color=BRAND_PRIMARY, border_width=2) 218 | self.input_box.configure(state="normal") 219 | self.stop_listening() 220 | 221 | def continuous_listen(self): 222 | while self.audio_manager.is_listening: 223 | try: 224 | wake_word_detected = self.audio_manager.listen_for_wake_word(wake_word=self.wake_word) 225 | if wake_word_detected: 226 | self.audio_manager.generate_beep() 227 | self.create_chat_bubble("Listening...", is_user=False) 228 | self.process_speech_input() 229 | except Exception as e: 230 | logging.error(f"Error in continuous listening: {str(e)}") 231 | 232 | def process_speech_input(self): 233 | user_input = self.audio_manager.recognize_speech() 234 | if user_input and not user_input.startswith("Error"): 235 | self.send_message(user_input=user_input) 236 | 237 | def start_listening(self): 238 | self.audio_manager.is_listening = True 239 | self.continuous_listen_thread = threading.Thread(target=self.continuous_listen, daemon=True) 240 | self.continuous_listen_thread.start() 241 | 242 | def stop_listening(self): 243 | if self.continuous_listen_thread: 244 | self.audio_manager.is_listening = False 245 | self.continuous_listen_thread.join(timeout=2) 246 | if self.continuous_listen_thread.is_alive(): 247 | logging.warning("Continuous listen thread did not stop in time") 248 | 249 | def toggle_kb(self, kb): 250 | is_active = self.kb_toggles[kb].get() 251 | if is_active and kb not in self.selected_kbs: 252 | self.selected_kbs.append(kb) 253 | elif not is_active and kb in self.selected_kbs: 254 | self.selected_kbs.remove(kb) 255 | self.context_manager.clear_custom_instructions() # Clear custom instructions when KB is turned off 256 | 257 | self.knowledge_manager.update_selected_kbs(self.selected_kbs) 258 | self.refresh_kb_toggles() 259 | 260 | def refresh_kb_toggles(self): 261 | for kb, toggle in self.kb_toggles.items(): 262 | toggle.select() if kb in self.selected_kbs else toggle.deselect() 263 | # After toggling, refresh the system message with updated skills 264 | self.chat_manager.update_selected_kbs(self.selected_kbs) 265 | 266 | 267 | def sanitize_filename(self, filename: str) -> str: 268 | # Implement filename sanitization logic here 269 | sanitized = ''.join(c for c in filename if c.isalnum() or c in (' ', '.', '_')).rstrip() 270 | return sanitized 271 | 272 | def send_message(self, user_input=None): 273 | if not self.is_voice_mode: 274 | user_input = self.input_box.get("1.0", ctk.END).strip() 275 | 276 | if user_input: 277 | self.create_chat_bubble(f"{user_input}", is_user=True) 278 | self.input_box.delete("1.0", ctk.END) 279 | # Modify the filename generation for the interpreter 280 | timestamp = datetime.now().strftime("%Y_%m_%d_%H-%M-%S") 281 | sanitized_query = self.sanitize_filename(user_input[:50]) # Limit to first 50 characters 282 | filename = f"Context_{sanitized_query}_{timestamp}.json" 283 | 284 | # Set the custom filename for the interpreter 285 | interpreter.conversation_filename = filename 286 | 287 | # Start a new thread for processing the response 288 | threading.Thread(target=self.process_response, args=(user_input,), daemon=True).start() 289 | 290 | def process_response(self, user_input): 291 | response_generator, sources = self.chat_manager.process_input(user_input, self.selected_kbs) 292 | 293 | # Add indicator for knowledge base query 294 | if self.selected_kbs: 295 | kb_info = "Knowledge bases being queried:\n" + "\n".join(f"- {kb}" for kb in self.selected_kbs) 296 | self.create_chat_bubble(kb_info, is_user=False) 297 | 298 | self.create_streaming_label() 299 | for message in response_generator: 300 | if isinstance(message, dict) and 'content' in message: 301 | content = message['content'] 302 | if content is not None: 303 | content = str(content) 304 | self.update_streaming_label(content) 305 | 306 | # Remove the streaming label 307 | if self.streaming_label: 308 | self.streaming_label.destroy() 309 | self.streaming_label = None 310 | 311 | # Get the final response from the last message if the role is "assistant" otherwise return "no response" 312 | if interpreter.messages and interpreter.messages[-1]['role'] == 'assistant': 313 | final_response = interpreter.messages[-1]['content'] 314 | else: 315 | final_response = "no response" 316 | 317 | # Insert the final response as a chat bubble 318 | self.create_chat_bubble(f"{final_response}", is_user=False) 319 | 320 | if sources: 321 | sources_text = "Sources:\n" + "\n".join(f"- {source}" for source in sources) 322 | self.create_chat_bubble(sources_text, is_user=False) 323 | 324 | if self.is_voice_mode: 325 | threading.Thread(target=self.audio_manager.text_to_speech, args=(final_response,), daemon=True).start() 326 | 327 | def reset_chat(self): 328 | interpreter.reset() 329 | for widget in self.message_frame.winfo_children(): 330 | widget.destroy() 331 | self.streaming_label = None 332 | 333 | def open_settings(self): 334 | # Clear the main frame and display settings 335 | for widget in self.main_frame.winfo_children(): 336 | widget.destroy() 337 | SettingsWindow(self.main_frame, self) 338 | # After settings are saved, update the chat manager 339 | self.chat_manager.update_selected_kbs(self.selected_kbs) 340 | self.chat_manager.update_wake_word(self.wake_word) 341 | 342 | def update_interpreter_settings(self, new_settings): 343 | self.interpreter_settings.update(new_settings) 344 | # Update interpreter directly 345 | interpreter.llm.supports_vision = new_settings["supports_vision"] 346 | interpreter.auto_run = new_settings["auto_run"] 347 | interpreter.loop = new_settings["loop"] 348 | interpreter.llm.temperature = new_settings["temperature"] 349 | interpreter.llm.max_tokens = new_settings["max_tokens"] 350 | interpreter.llm.context_window = new_settings["context_window"] 351 | # Update ChatManager 352 | self.chat_manager.update_interpreter_settings(self.interpreter_settings) 353 | 354 | def update_env_vars(self, new_env_vars): 355 | self.env_vars.update(new_env_vars) 356 | for key, value in new_env_vars.items(): 357 | os.environ[key] = value 358 | 359 | # Update system message 360 | custom_env_vars = sorted(set(key for key in self.env_vars.keys() if key.startswith("CUSTOM_"))) 361 | env_var_message = "\n- ".join(custom_env_vars) 362 | 363 | interpreter.system_message = ( 364 | f"{SYSTEM_MESSAGE}\n" 365 | f"{SYSTEM_MESSAGE_ENV_VARS}\n" 366 | f"\n- {env_var_message}" 367 | ) 368 | print(interpreter.system_message) 369 | 370 | # Update ChatManager 371 | self.chat_manager.update_env_vars(self.env_vars) 372 | --------------------------------------------------------------------------------