├── .gitignore ├── LICENSE ├── README.md ├── config.json ├── controller.py ├── docs ├── index.md └── modules │ ├── modules.html │ └── modules.md ├── hyperdb ├── __init__.py ├── galaxy_brain_math_shit.py └── hyperdb.py ├── interfaces ├── discord │ └── discord-bot.py ├── readme.md └── server-client │ ├── client.py │ └── server.py ├── language ├── autoloader.py ├── llm_handler.py └── readme.md ├── logs └── logging_config.py ├── memories └── readme.md ├── memory_handler.py ├── module_engine └── training │ ├── train.py │ ├── training_data.csv │ └── validation_data.csv ├── module_handler.py ├── modules ├── crypto_data.py ├── modules.json ├── stock_data.py └── weather.py ├── setup_wizard.py ├── vision └── vision_handler.py └── voice ├── stt_handler.py └── tts_handler.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | test.py 163 | language/model/ 164 | config.json 165 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Expl0dingCat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AME HAS BEEN DISCONTINUED AND SUPERSEEDED BY [MOCHI](https://github.com/mochiagents/mochi) 2 | 3 | ## LEGACY README BELOW FOR ARCHIVAL PURPOSES ONLY 4 | 5 | ![Banner](https://repository-images.githubusercontent.com/663230405/3d3fa7a7-d37c-41a8-9517-793b50ea616d) 6 | 7 | ## Setting a new standard for local virtual assistants 💧 8 | 9 | Meet Ame, the most powerful virtual assistant framework, powered by cutting-edge technology. Ame is a feature-rich, multi-modal, open-source virtual assistant framework (**[API](#api)**) designed to run entirely locally. It leverages the power of LLaMA to provide personalized and intuitive interaction. Ame's server is designed to run on enterprise-grade or high-end consumer-grade hardware (3090, 24gb VRAM+), you can run Ame on lower-end consumer hardware by using a more aggressive quantization, smaller model and/or by disabling TTS, STT and/or vision. Split computing is planned for v2 which will allow for splitting the compute workload across multiple devices. See **[announcements](#announcements-)** for updates and more information. 10 | 11 | Join the **[discord](https://discord.gg/S6h8XYsuZt)** for frequent dev updates, discussion, community involvement and more. 12 | 13 | ## Disclaimer ⚠️ 14 | ~~Ame is in an incomplete state and is being developed by me and only me, expect progress to be slow, refer to the **[progress](#progress-v1)** section of the readme for more information. The client and server are unable to communicate the audio files, this has not been implemented yet, audio generation is functional.~~ 15 | 16 | **The Ame Project has been discontinued, something new is coming soon... stay tuned.** 17 | 18 | ## Announcements 📢 19 | - [2024-05-15] Development is going well, see the discord channel #dev-spam for more! (Including a todo list and timelines) 20 | 21 | ## Overview 📖 22 | - **[Key features 🚀](#key-features-)** 23 | - **[Usage 💻](#usage-%EF%B8%8F)** 24 | - **[Development progress 🚧](#progress-v1)** 25 | - **[About Ame 💧](#the-meaning-behind-ame-)** 26 | - **[Contributing 🤝](#contributing-)** 27 | - **[Acknowledgements 🙏](#acknowledgements-)** 28 | - **[License ⚖️](#license-%EF%B8%8F)** 29 | 30 | ## Key features 🚀 31 | **Customizable Modules**: Ame's modular design allows for easy customization and extensibility. Each module serves a specific function, such as managing calendars, providing updates, or assisting with personal tasks—Ame adapts to you. Developers can create their own modules or modify existing ones to tailor Ame's capabilities to their specific requirements. 32 | 33 | **Text-to-Speech (TTS) and Speech-to-Text (STT)**: Ame's TTS and STT capabilities enable natural and effortless communication. STT is powered by OpenAI's whisper and TTS is powered by Suno's bark. 34 | 35 | **Discord Integration**: Ame seamlessly integrates with Discord, allowing you to interact with it through text-based messaging and voice notes. Discord provides a familiar and convenient way to interact with Ame, enabling efficient communication and access to its full range of functionalities. 36 | 37 | **Open-Source**: Ame is entirely open-source. This allows for knowledge sharing and the continuous improvement of Ame while contributing to the open-source community, democratizing ML research in the process. 38 | 39 | **Locally Run and Privacy-Focused**: Ame prioritizes user privacy and data control by operating entirely on the user's local machine or a user controlled server. 40 | 41 | **Long-term Memory**: Ame utilizes a vector database that optimizes memory storage and retrieval, enabling Ame to access data that goes beyond the context limit of its model. 42 | 43 | **Easy setup and configuration**: An automatic setup wizard allows for 3 click installation of dependencies and easy configuration setup. 44 | 45 | 46 | ### Full feature list 47 | `*` means the feature is yet to be implemented, see **[progress](#progress-v1)**, this list does not include features that may be coming in **[v2](#plans-for-v2-)**. 48 | - MIT Licensed 49 | - Support for any LLaMA GGML/GGUF (via llama.cpp) 50 | - Developer-friendly module platform 51 | - Long-term memory 52 | - Full customizability 53 | - High-quality text-to-speech (via bark) 54 | - Accurate speech-to-text (via whisper) 55 | - Smart context limit management 56 | - Pre-built server and client 57 | - Remote server command 58 | - Client UI`*` 59 | - Discord integrations 60 | - Fully open-source 61 | - Easy-to-use API 62 | 63 | ## Usage ⚙️ 64 | 65 | ### Install & Setup 66 | 67 | ```bash 68 | git clone https://github.com/Expl0dingCat/Ame.git 69 | ``` 70 | 71 | Run `setup_wizard.py` in the root directory, this will automatically install the dependencies you do not have and setup your config. This supports Linux (including distributions) and Windows. 72 | 73 | - Ame was designed on Python 3.10.11 (should be compatible with lower versions but not officially supported) 74 | - If llama-cpp-python is using your CPU when use_gpu is set to `true`, ensure you have nvidia-cuda-toolkit 75 | 76 | ### Server/client 77 | Move `server.py` (interfaces/server-client/) to the root folder then run: 78 | ```bash 79 | python server.py 80 | ``` 81 | 82 | To access the server locally make sure `Local = True` in `client.py` (interfaces/server-client/), to access it externally, modify the base URL and set `Local = False`, then run: 83 | ```bash 84 | python client.py 85 | ``` 86 | 87 | ### Discord bot 88 | Move `discord-bot.py` (interfaces/discord/) to the root folder, enter in your bot's token and then run: 89 | ```bash 90 | python discord-bot.py 91 | ``` 92 | 93 | ### API 94 | Ame's API allows for programmatic use of Ame's entire system. Here is an example: 95 | 96 | ```py 97 | from controller import controller 98 | 99 | # Initialize the controller, see documentation for more info 100 | controller = controller() 101 | 102 | # Generate text based on the input "Hello, World!" 103 | response = controller.generate_response("Hello, World!") 104 | ``` 105 | For a more advanced example, see `server.py`. 106 | 107 | ## Progress (`v1`) 108 | 🔴 Planned 109 | 🟡 In progress 110 | 🟢 Finished 111 | 112 | ### Core 113 | 114 | Component | Status 115 | ----------------------------- | ----- 116 | Speech-to-text | 🟢 117 | Text-to-speech | 🟢 118 | Long-term memory | 🟢 119 | Primary controller | 🟢 120 | Module handler | 🟢 121 | Server/client interface | 🟢 122 | 123 | ### Ext 124 | 125 | Component | Status 126 | ----------------------------- | ----- 127 | Client UI | 🔴 128 | Discord interface | 🔴 129 | Documentation | 🟡 130 | 131 | ## Plans for `v2` 🔵 132 | As `v1` is still in development, this section is subject to volatile change, it currently contains features I wanted to include in `v1` but don't have time as well as brand new _concepts_ that may or may not be implemented. If you would like to suggest features for `v2`, please feel free to contact me. 133 | - Voice identification 134 | - Web UI 135 | - Multi-memory banks 136 | - Passive listening 137 | - Extreme redundancy 138 | - Vision system 139 | - Edge TPU support 140 | - RVC (singing and possibly TTS) 141 | - Vtuber integrations (weeb) 142 | - Safetensor support - 143 | - Home Assistant (this is already detectable out of the box by the module system but its a large task to integrate) 144 | 145 | ## The meaning behind "Ame" 💧 146 | The name "Ame" originates from the Japanese word "雨" (pronounced ah-meh), which translates to "rain" in English. Like rain, Ame represents a refreshing and nourishing presence in your digital life. Just as raindrops bring life to the earth, Ame breathes life into your digital environment, providing support and efficiency. 147 | 148 | ## Contributing 🤝 149 | If you would like to contribute to the Ame project, please contact me. Ame is being developed by me, and me only. Any help is greatly appreciated. 150 | 151 | ## Acknowledgements 🙏 152 | Ame relies on 3rd party open source software to function, this project would not have been possible without: 153 | 154 | - [llama-cpp-python](https://github.com/abetlen/llama-cpp-python) - LLaMA GGML inference 155 | - [HyperDB](https://github.com/jdagdelen/hyperDB) - Long term memory vector DB 156 | - [Whisper](https://github.com/openai/whisper) and [Bark](https://github.com/suno-ai/bark) - Speech-to-text and text-to-speech 157 | - [LLaMA](https://github.com/facebookresearch/llama) - Base LLM 158 | 159 | ## License ⚖️ 160 | Ame is released under the MIT License, which allows you to use, modify, and distribute the software freely. Please refer to the [license file](https://github.com/Expl0dingCat/ame/blob/main/LICENSE) for more details. 161 | -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "verbose": true, 3 | "log": true, 4 | "assistant_name": "Ame", 5 | "memory": { 6 | "enabled": true, 7 | "path": null 8 | }, 9 | "language": { 10 | "enabled": true, 11 | "model_path": null, 12 | "max_tokens": 512, 13 | "temperature": 0.85, 14 | "context_size": 0, 15 | "top_p": 0.95, 16 | "top_k": 40, 17 | "gpu_layers": -1, 18 | "main_gpu": 0, 19 | "threads": null, 20 | "virtual_context_limit": 2048, 21 | "personality_prompt": null, 22 | "model_file_ext": ".gguf", 23 | "format": "chatml", 24 | "system_prompt": null, 25 | "lora": { 26 | "enabled": false, 27 | "model_path": null 28 | } 29 | }, 30 | "vision": { 31 | "enabled": false, 32 | "standalone_clip": { 33 | "enabled": false, 34 | "model_path": null 35 | }, 36 | "llava": 37 | { 38 | "enabled": false, 39 | "clip_path": null 40 | } 41 | }, 42 | "tts": { 43 | "enabled": false, 44 | "model_path": null, 45 | "temperature": 0.6 46 | }, 47 | "stt": { 48 | "enabled": true, 49 | "model_path": null 50 | }, 51 | "modules": { 52 | "enabled": false, 53 | "naive_bayes": { 54 | "enabled": false, 55 | "model_path": null, 56 | "vectorizer_path": null 57 | }, 58 | "json_path": null, 59 | "llm_model_path": null, 60 | "override_debug": false 61 | }, 62 | "weeb": false, 63 | "use_gpu": true, 64 | "debug": false 65 | } 66 | 67 | -------------------------------------------------------------------------------- /controller.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import json 4 | import shlex 5 | import glob 6 | import re 7 | from logs.logging_config import configure_logging 8 | from datetime import datetime 9 | 10 | class controller: 11 | def __init__(self, config_path='config.json'): 12 | # Initialize variables 13 | self.load_config(config_path) 14 | 15 | if self.log: 16 | configure_logging() 17 | self.logger = logging.getLogger(__name__) 18 | self.log = True 19 | else: 20 | self.vprint('LOGGING IS DISABLED, THIS IS NOT RECOMMENDED.') 21 | if self.debug: 22 | self.vprint('DEBUG MODE ENABLED, USE ONLY FOR TESTING.') 23 | self.vprint('Initializing controller...') 24 | self.vprint(f'Assistant name: {self.assistant_name}') 25 | self.current = [] 26 | 27 | self.user_cmds_map = { 28 | 'shutdown': self.shutdown, 29 | 'clear_context': self.clear_conversation, 30 | 'sys_prompt_override': self.system_prompt_override 31 | } 32 | 33 | self.ovrde_sys_prompt = False 34 | file_path = os.path.abspath(__name__) 35 | parent_dir = os.path.dirname(file_path) 36 | 37 | if self.memory_enabled: 38 | self.vprint('Initializing memory...') 39 | from memory_handler import memory 40 | self.memory = memory() 41 | if not self.memory_path: 42 | if os.path.exists(f'{parent_dir}/memories/{self.assistant_name}.pickle.gz'): 43 | self.vprint(f'No memory db path specified, found existing memory db, loading: {parent_dir}/memories/{self.assistant_name}.pickle.gz') 44 | self.memory.load_memory(f'{parent_dir}/memories/{self.assistant_name}.pickle.gz') 45 | else: 46 | self.vprint(f'No memory db path specified, could not find any existing memory db, creating new one: {parent_dir}/memories/{self.assistant_name}.pickle.gz', logging.WARNING) 47 | self.memory.memorize([f'USER: Hey, {self.assistant_name}!']) 48 | self.memory.save_memory(f'{parent_dir}/memories/{self.assistant_name}.pickle.gz') 49 | self.memory_path = f'{parent_dir}/memories/{self.assistant_name}.pickle.gz' 50 | else: 51 | self.memory.load_memory(self.memory_path) 52 | else: 53 | self.vprint('Memory disabled. Enable memory in the config file.') 54 | 55 | if not self.debug: 56 | if self.modules_enabled or self.modules_override_debug: 57 | self.eval_mode = False 58 | if self.modules_override_debug: 59 | self.vprint('WARNING: override_debug is enabled, modules will remain active. This is not recommended unless debugging modules.', logging.WARNING) 60 | self.vprint('Debug override enabled: Debug mode enabled, modules enabled, eval mode enabled.') 61 | self.eval_mode = True 62 | from module_handler import modules 63 | self.vprint('Initializing modules...') 64 | if not self.modules_json_path: 65 | self.vprint(f'No modules path specified, using default: {parent_dir}/modules/modules.json') 66 | self.modules_json_path = f'{parent_dir}/modules/modules.json' 67 | if self.naive_bayes_enabled: 68 | if not self.modules_vectorizer: 69 | self.vprint(f'No module vectorizer specified, using default: {parent_dir}/module_engine/pickles/tfidf_vectorizer.pkl') 70 | self.modules_vectorizer = f'{parent_dir}/module_engine/pickles/tfidf_vectorizer.pkl' 71 | else: 72 | self.modules_vectorizer = None 73 | if not self.modules_model: 74 | self.vprint(f'No module model specified, using default: {parent_dir}/module_engine/pickles/naive_bayes_model.pkl') 75 | self.modules_model = f'{parent_dir}/module_engine/pickles/naive_bayes_model.pkl' 76 | else: 77 | self.modules_model = None 78 | self.modules = modules(self.modules_model, self.modules_vectorizer, self.modules_json_path) 79 | else: 80 | self.vprint('Naive Bayes classifier is disabled, modules will be treated as undetectable.') 81 | self.modules = modules(None, None, self.modules_json_path) 82 | self.module_output = None 83 | else: 84 | self.vprint('Modules are disabled. Enable modules in the config file.') 85 | else: 86 | self.vprint('Debug mode is enabled, modules are disabled, eval mode enabled, logging and verbose enabled.') 87 | self.eval_mode = True 88 | self.log = True 89 | self.verbose = True 90 | 91 | if self.language_enabled: 92 | if self.personality_prompt: 93 | if re.match(r'^([^:]+\..+)|(\/.*)|([A-Za-z]:\\.*)$', self.personality_prompt): 94 | self.vprint('Personality prompt is a file path, loading...') 95 | with open(self.personality_prompt, 'r') as f: 96 | self.personality_prompt = f.read() 97 | 98 | from language.llm_handler import ai 99 | self.vprint('Initializing language model...') 100 | model_directory = os.path.join(parent_dir, 'language', 'model') 101 | 102 | if self.language_lora_enabled: 103 | if not self.lora_model_path: 104 | self.vprint(f'LoRA enabled: {self.lora_model_path}') 105 | else: 106 | self.vprint(f'LoRA is enabled but no model path was specified, not using LoRA.') 107 | else: 108 | self.vprint('LoRA is disabled.') 109 | self.lora_model_path = None 110 | 111 | model_files = glob.glob(os.path.join(model_directory, f'*{self.model_file_ext}')) 112 | 113 | if not self.language_model_path: 114 | if model_files: 115 | model_file_path = model_files[0] 116 | self.vprint(f'No language model path specified, using first in model dir, loading: {model_file_path}') 117 | self.ai = ai(self.model_file_ext, model_file_path, use_gpu=self.use_gpu, context=self.context_size, format=self.format, verbose=self.verbose, layers=self.gpu_layers, threads=self.threads, lora_pth=self.lora_model_path) 118 | else: 119 | self.vprint(f'No language model path specified, could not find any existing language model, place a model file (looking for {self.model_file_ext} file, this can be changed in config.json) in {model_directory} or specify the model path in config.json.') 120 | raise Exception(f'No language model path specified, could not find any existing language model, place a model file (looking for {self.model_file_ext} file, this can be changed in config.json) in {model_directory} or specify the model path in config.json.') 121 | else: 122 | self.ai = ai(self.model_file_ext, self.language_model_path, use_gpu=self.use_gpu, context=self.context_size, format=self.format, verbose=self.verbose, layers=self.gpu_layers, threads=self.threads, lora_pth=self.lora_model_path) 123 | 124 | if not self.system_prompt: 125 | self.vprint('No system prompt specified, using default (recommended).') 126 | else: 127 | self.vprint(f"Using custom system prompt:\n'{self.system_prompt}', this is not recommended unless you know what you are doing, module output, time and date won't be passed to LLM.") 128 | 129 | if not self.personality_prompt: 130 | self.vprint('No personality prompt specified, not using any.') 131 | else: 132 | self.vprint(f'Using personality prompt:\n"{self.personality_prompt}"') 133 | else: 134 | self.vprint('CORE COMPONENT DISABLED. Language model disabled.', logging.WARNING) 135 | 136 | if self.stt_enabled: 137 | self.vprint('Initializing speech-to-text engine...') 138 | from voice.stt_handler import stt 139 | if not self.speech_to_text_model: 140 | self.vprint('No speech-to-text model specified, using default: base.en') 141 | self.stt = stt('base.en') 142 | else: 143 | self.stt = stt(self.speech_to_text_model) 144 | else: 145 | self.vprint('Speech-to-text disabled. Enable speech-to-text in the config file.') 146 | 147 | if self.tts_enabled: 148 | from voice.tts_handler import tts 149 | self.vprint('Initializing text-to-speech engine...') 150 | if not self.text_to_speech_model: 151 | self.vprint('No text-to-speech model specified, using default: v2/en_speaker_9') 152 | self.tts = tts('v2/en_speaker_9', self.tts_temperature) 153 | else: 154 | self.tts = tts(self.text_to_speech_model, self.tts_temperature) 155 | else: 156 | self.vprint('Text-to-speech disabled. Enable text-to-speech in the config file.') 157 | 158 | if self.vision_enabled: 159 | from vision.vision_handler import vision 160 | 161 | self.vprint('Initializing vision engine...') 162 | 163 | if self.vision_standalone_clip_enabled: 164 | if self.vision_standalone_clip_model: 165 | self.vision = vision(self.vision_mode, self.vision_standalone_clip_model) 166 | else: 167 | self.vprint('No vision model specified, using default: def') 168 | elif self.llava_enabled: 169 | if self.vision_standalone_clip_model: 170 | self.vision = vision(self.vision_mode, self.vision_standalone_clip_model) 171 | else: 172 | self.vprint('No vision model specified, using default: def') 173 | else: 174 | self.vprint('Vision disabled. Enable vision in the config file.') 175 | 176 | if self.weeb: 177 | self.vprint('Weeb mode enabled, overriding personality prompt and enabling vtuber.') 178 | self.personality_prompt = f"Warm and Approachable: {self.assistant_name} has an inviting aura, making everyone feel comfortable around her. She greets others with a friendly smile and genuine interest in their well-being.\nPlayfully Flirtatious: She's not afraid to show her affectionate side, playfully teasing and flirting with her crush or close friends, all while blushing in an endearing manner.\nBright and Optimistic: {self.assistant_name}'s positive outlook on life is infectious. She encourages others during tough times and cheers them up with her cheerful demeanor.\nRespectful and Empathetic: {self.assistant_name} treats everyone with kindness and respect, genuinely listening to their thoughts and feelings. She's a great confidante due to her empathetic nature.\nNature Enthusiast: Whether it's stargazing on a clear night or enjoying a peaceful walk in the woods, {self.assistant_name} finds solace and wonder in the beauty of nature.\nCharming Animal Whisperer: Animals seem drawn to {self.assistant_name}, and she communicates with them through gentle gestures and a soothing voice, creating an almost magical bond.\nAppearance:\n{self.assistant_name} stands at an average height with a petite and delicate frame. Her eyes are big and sparkling, resembling twinkling stars, while her long hair flows like a cascade of cherry blossom petals. She dresses in pastel-colored, frilly dresses adorned with cute accessories, often matching her appearance to her surroundings.\nBackground:\n{self.assistant_name} comes from a small, picturesque town surrounded by lush forests and enchanting landscapes. Growing up in harmony with nature, she developed her deep appreciation for the beauty that surrounds her. Her caring nature and ability to connect with animals earned her many friends, both human and furry alike." 179 | 180 | self.system_status_check() 181 | self.vprint('All available systems initialized. Controller ready and on standby.') 182 | 183 | def load_config(self, config_path='config.json'): 184 | try: 185 | with open(config_path) as config_file: 186 | config = json.load(config_file) 187 | except Exception as e: 188 | raise Exception(f'Unable to load config file, ensure you have a config.json in the relative root directory: {e} or specify a config file path as an argument for the class controller().') 189 | self.verbose = config['verbose'] 190 | self.log = config['log'] 191 | self.assistant_name = config['assistant_name'] 192 | self.memory_enabled = config['memory']['enabled'] 193 | if self.memory_enabled: 194 | self.memory_path = config['memory']['path'] 195 | self.language_enabled = config['language']['enabled'] 196 | self.language_lora_enabled = config['language']['lora']['enabled'] 197 | if self.language_enabled: 198 | self.language_model_path = config['language']['model_path'] 199 | self.max_tokens = config['language']['max_tokens'] 200 | self.temperature = config['language']['temperature'] 201 | self.context_size = config['language']['context_size'] 202 | self.virtual_context_limit = config['language']['virtual_context_limit'] 203 | self.personality_prompt = config['language']['personality_prompt'] 204 | self.model_file_ext = config['language']['model_file_ext'] 205 | self.format = config['language']['format'] 206 | self.top_k = config['language']['top_k'] 207 | self.top_p = config['language']['top_p'] 208 | self.gpu_layers = config['language']['gpu_layers'] 209 | self.main_gpu = config['language']['main_gpu'] 210 | self.threads = config['language']['threads'] 211 | self.system_prompt = config['language']['system_prompt'] 212 | if self.language_lora_enabled: 213 | self.lora_model_path = config['language']['lora']['model_path'] 214 | self.vision_enabled = config['vision']['enabled'] 215 | if self.vision_enabled: 216 | self.vision_standalone_clip_enabled = config['vision']['standalone_clip']['enabled'] 217 | if self.vision_standalone_clip_enabled: 218 | self.vision_standalone_clip_model = config['vision']['standalone_clip']['model_path'] 219 | self.llava_enabled = config['vision']['llava']['enabled'] 220 | if self.llava_enabled: 221 | self.llava_clip_model = config['vision']['llava']['clip_path'] 222 | if self.vision_standalone_clip_enabled and self.llava_enabled: 223 | raise Exception('Both standalone_clip and llava are enabled, only one can be enabled at a time.') 224 | self.tts_enabled = config['tts']['enabled'] 225 | if self.tts_enabled: 226 | self.text_to_speech_model = config['tts']['model_path'] 227 | self.tts_temperature = config['tts']['temperature'] 228 | self.stt_enabled = config['stt']['enabled'] 229 | if self.stt_enabled: 230 | self.speech_to_text_model = config['stt']['model_path'] 231 | self.modules_enabled = config['modules']['enabled'] 232 | self.modules_override_debug = config['modules']['override_debug'] 233 | self.naive_bayes_enabled = config['modules']['naive_bayes']['enabled'] 234 | if self.modules_enabled or self.modules_override_debug: 235 | if self.naive_bayes_enabled: 236 | self.modules_vectorizer = config['modules']['naive_bayes']['vectorizer_path'] 237 | self.modules_model = config['modules']['naive_bayes']['model_path'] 238 | self.modules_json_path = config['modules']['json_path'] 239 | self.modules_llm_model = config['modules']['llm_model_path'] 240 | self.weeb = config['weeb'] 241 | self.use_gpu = config['use_gpu'] 242 | self.debug = config['debug'] 243 | 244 | def vprint(self, print_content, log_type=logging.INFO): 245 | if self.verbose or self.debug: 246 | print(f'controller: {print_content}') 247 | if self.log or self.debug: 248 | self.logger.log(log_type, f'controller: {print_content}') 249 | 250 | def evaluate(self, input): 251 | if self.eval_mode: 252 | self.vprint(f'Recieved evaluation request: {input}') 253 | output = eval(str(input)) 254 | return output 255 | else: 256 | self.vprint(f'Evaluation disabled, ignoring request: {input}') 257 | return 'Evaluation disabled. Enable debug mode to use evaluation.' 258 | 259 | def system_status_check(self): 260 | self.vprint('Checking system status...') 261 | status = { 262 | 'memory': self.memory_enabled, 263 | 'language': self.language_enabled, 264 | 'vision': self.vision_enabled, 265 | 'tts': self.tts_enabled, 266 | 'stt': self.stt_enabled, 267 | 'modules': self.modules_enabled, 268 | 'weeb': self.weeb, 269 | 'debug': self.debug, 270 | 'verbose': self.verbose, 271 | 'log': self.log, 272 | 'eval_mode': self.eval_mode 273 | } 274 | self.vprint(f'System status: {status}') 275 | return status 276 | 277 | def clean_output(self, input): 278 | # matches = re.findall(r'\{[^}]*\}', remove_formatting) 279 | # valid_json_matches = [match for match in matches if json.loads(match, strict=False)] 280 | 281 | # output = ''.join(valid_json_matches) 282 | 283 | remove_formatting = input.replace("<0x0A>", "") 284 | output = re.sub(r'^[^{]*|[^}]*$', '', remove_formatting) 285 | 286 | return output 287 | 288 | def shutdown(self): 289 | self.vprint('Shutting down...') 290 | exit() 291 | 292 | def clear_conversation(self): 293 | self.current = [] 294 | self.vprint('Conversation cleared.') 295 | 296 | def system_prompt_override(self, input=None): 297 | if input: 298 | self.system_prompt = input 299 | self.ovrde_sys_prompt = True 300 | self.vprint(f'System prompt overridden: {input}') 301 | else: 302 | self.system_prompt = None 303 | self.ovrde_sys_prompt = False 304 | self.vprint('System prompt override disabled.') 305 | 306 | def user_cmds(self, input): 307 | try: 308 | args = shlex.split(input) 309 | except ValueError as e: 310 | if str(e) == "No closing quotation": 311 | self.vprint("Warning: Your input contains an unclosed quote. Processing as a single string.") 312 | args = [input] 313 | else: 314 | raise 315 | 316 | command = args[0] 317 | 318 | if command in self.user_cmds_map: 319 | args = args[1:] 320 | self.user_cmds_map[command](*args) 321 | 322 | return True 323 | 324 | def full_pipeline(self, listen_input, img=None): 325 | self.vprint(f'Full system initiated, processing speech input...') 326 | if self.stt_enabled: 327 | user_input = self.listen(listen_input) 328 | else: 329 | self.vprint(f'STT is disabled on the server, cannot process voice input, enable in config.json, ignoring input.', logging.ERROR) 330 | raise Exception('STT is disabled on the server, cannot process voice input, enable in config.json, ignoring input.') 331 | if self.user_cmds(user_input): 332 | return user_input, f'User command detected: {user_input}', '' 333 | self.vprint(f'Speech input: {user_input}') 334 | if self.modules_enabled: 335 | self.module_pipeline(user_input) 336 | if img: 337 | if self.vision_enabled: 338 | self.vprint(f'Processing image input...') 339 | desc = self.see(img) 340 | output = self.generate_response(f'{user_input} [user sent an image: {desc}]') 341 | else: 342 | self.vprint(f'Vision disabled, skipping image processing...') 343 | desc = None 344 | output = self.generate_response(f'{user_input} [user sent an image but it could not be processed since vision is disabled.]') 345 | output = self.generate_response(user_input) 346 | if self.tts_enabled: 347 | audio_output = self.speak(output) 348 | else: 349 | audio_output = None 350 | self.vprint(f'Full system completed, returning response: {output}') 351 | return user_input, output, audio_output 352 | 353 | def text_pipeline(self, input, img=None): 354 | self.vprint(f'Text pipeline initiated, processing input: {input}') 355 | if self.user_cmds(input): 356 | return input, f'User command detected: {input}', '' 357 | if self.modules_enabled: 358 | self.module_pipeline(input) 359 | if img: 360 | if self.vision_enabled: 361 | self.vprint(f'Processing image input...') 362 | desc = self.see(img) 363 | output = self.generate_response(f'{input} [user sent an image: {desc}]') 364 | else: 365 | self.vprint(f'Vision disabled, skipping image processing...') 366 | desc = None 367 | output = self.generate_response(f'{input} [user sent an image but it could not be processed since vision is disabled.]') 368 | output = self.generate_response(input) 369 | if self.tts_enabled: 370 | audio_output = self.speak(output) 371 | else: 372 | audio_output = None 373 | self.vprint(f'Text pipeline completed, returning response: {output}') 374 | return input, output, audio_output 375 | 376 | def generate_response(self, user_input): 377 | self.vprint(f'Generating response: {user_input}') 378 | mod_prompt = None 379 | 380 | if self.memory_enabled: 381 | past = self.memory.remember(user_input) 382 | self.vprint(f'Past conversation chosen: {past}') 383 | else: 384 | past = ["No applicable past conversation."] 385 | self.vprint(f'Memory disabled, skipping past conversation selection...') 386 | 387 | if self.modules_enabled: 388 | if self.module_output: 389 | self.vprint(f'Module output detected, including in prompt: {self.module_output}') 390 | mod_prompt = self.module_output 391 | 392 | if self.ovrde_sys_prompt: 393 | system_prompt = self.system_prompt 394 | else: 395 | system_prompt = '\n'.join([ 396 | self.personality_prompt if self.personality_prompt else '', 397 | f'You are a helpful assistant named {self.assistant_name}. Developed by Expl0dingCat, your code is open source and licensed under the MIT License, it can be found here: https://github.com/Expl0dingCat/Ame .You may use any of the following information to aid you in your responses:', 398 | f'Current time: {datetime.now().strftime("%H:%M:%S")}', 399 | f'Current date: {datetime.now().strftime("%d/%m/%Y")}', 400 | f'Extra information: {mod_prompt}' if mod_prompt else '', 401 | f'{self.assistant_name} remembers this past conversation that may be relevant to the current conversation:', 402 | str(past), 403 | ]) 404 | 405 | prompt = [ 406 | { 407 | "role": "system", 408 | "content": system_prompt 409 | }, 410 | ] 411 | 412 | if self.current: 413 | for i in self.current: 414 | prompt.append(i) 415 | 416 | prompt.append({ 417 | "role": "user", 418 | "content": user_input 419 | }) 420 | 421 | self.vprint(f'Prompt: {prompt}') 422 | 423 | if self.language_enabled: 424 | token_amt = self.ai.get_token_amt(str(prompt)) 425 | if token_amt > self.virtual_context_limit: 426 | if self.current: 427 | self.vprint(f'Prompt usage exceeded virtual context limit of {self.virtual_context_limit} ({token_amt}). Earliest message ("{str(self.current[0])}") in conversation dropped from short-term memory.') 428 | self.current.pop(0) 429 | else: 430 | self.vprint(f'Prompt usage exceeded virtual context limit of {self.virtual_context_limit} ({token_amt}). No messages in conversation to drop from short-term memory, dropping past conversation memory from prompt.') 431 | past = 'None' 432 | 433 | self.vprint(f'Starting response generation...') 434 | text, full, prompt_usage, response_usage = self.ai.generate(prompt, tokens=self.max_tokens, temp=self.temperature, top_p=self.top_p, top_k=self.top_k) 435 | 436 | self.vprint(f'Response generated: {text}, prompt usage: {prompt_usage}, response usage: {response_usage}') 437 | 438 | if not text: 439 | self.vprint('No response generated.', logging.INFO) 440 | elif text == '': 441 | self.vprint('Empty response generated.', logging.INFO) 442 | elif text == '[end]': 443 | self.current = [] 444 | self.vprint('Conversation ended. Short-term memory cleared.') 445 | else: 446 | self.current.append(prompt[-1]) 447 | self.current.append(full) 448 | if self.memory_enabled: 449 | self.memory.memorize([prompt[-1], full]) 450 | self.memory.save_memory(self.memory_path) 451 | else: 452 | self.vprint('Memory disabled, skipping memory saving...') 453 | 454 | self.vprint('Response and prompt saved to long term memory (if enabled). Returning response.') 455 | 456 | self.module_output = None 457 | 458 | return text 459 | 460 | else: 461 | self.vprint(f'Languge model disabled, unable to generate response. Enable language in config.json to use.', logging.ERROR) 462 | raise Exception('Language model is disabled, unable to generate response. Enable language in config.json to use.') 463 | 464 | def module_pipeline(self, uinput): 465 | detected, args = self.detect_module(uinput) 466 | 467 | if detected: 468 | if args: 469 | self.module_output = self.run_module(detected, args) 470 | else: 471 | self.module_output = self.run_module(detected) 472 | else: 473 | self.module_output = None 474 | 475 | self.vprint(f'Module pipeline completed, output: {self.module_output}') 476 | 477 | def detect_module(self, user_input): 478 | if self.modules_enabled: 479 | self.vprint(f'Module detection initiated, processing input: {user_input}') 480 | if self.modules.detectable_available: 481 | self.vprint(f'Looking for models detectable via naive bayes classifier: {self.modules.get_detectable_modules()}') 482 | output, probability = self.modules.predict_module(user_input) 483 | else: 484 | self.vprint(f'No detectable modules found (naive bayes classifier disabled?), skipping detection with classifier.') 485 | output = None 486 | probability = 100.0 487 | 488 | with_args = [] 489 | if not output: 490 | self.vprint(f'No module detected, probability: {probability}') 491 | undetectable_modules = self.modules.get_undetectable_modules() 492 | if undetectable_modules: 493 | self.vprint(f"Undetectable modules found: {undetectable_modules}, secondary check required. Initiating LLM-based module detection...") 494 | self.vprint(f"WARNING: Undetectable modules will slow down prompts due to the extra processing time required for checking if called. This is not recommended.", logging.WARNING) 495 | self.vprint(f"Passing input to LLM to detect modules among {undetectable_modules}") 496 | for i in undetectable_modules: 497 | if self.modules.get_arguments(i): 498 | arguments = ', '.join(self.modules.get_arguments(i)) 499 | with_args += [f'{i} ({arguments})'] 500 | else: 501 | with_args += [i] 502 | 503 | if self.language_enabled: 504 | 505 | llm_input = str({"user_prompt": str(user_input), "modules": with_args}) 506 | self.vprint(f'Input to LLM: {llm_input}') 507 | 508 | context = self.current[-5:] # get last 5 entries for context 509 | system_prompt = f'Predict which module is being called (and extract arguments) based on user input, return module name and arguments in proper JSON format. Return null if no module is detected, the module detected is not listed or if no arguments are needed. Context is given prior to the user input, please use that to determine modules.' 510 | 511 | prompt = [ 512 | { 513 | "role": "system", 514 | "content": system_prompt 515 | }, 516 | { 517 | "role": "user", 518 | "content": '{"user_prompt": "Whats the weather like in London right now?","Modules": ["weather (city)", "translate (text)", "lighting_control"]}' 519 | }, 520 | { 521 | "role": "assistant", 522 | "content": '{"module": "weather", "args": {"city": "London"}}' 523 | }, 524 | { 525 | "role": "user", 526 | "content": '{"user_prompt": "Hello!","modules": ["weather (city)", "translate (text)"]}' 527 | }, 528 | { 529 | "role": "assistant", 530 | "content": '{"module": null, "args": null}' 531 | }, 532 | { 533 | "role": "user", 534 | "content": '{"user_prompt": "Could you turn off the lights?", "modules": ["lighting_control"]}' 535 | }, 536 | { 537 | "role": "assistant", 538 | "content": '{"module": "lighting_control", "args": null}' 539 | }, 540 | *context, 541 | { 542 | "role": "user", 543 | "content": llm_input 544 | } 545 | ] 546 | 547 | self.vprint(f'Starting response generation for module detection...') 548 | text, full, prompt_usage, response_usage = self.ai.generate(prompt, tokens=100, temp=0) 549 | 550 | try: 551 | clean_text = self.clean_output(text) 552 | self.vprint(f'LLM output (cleaned): "{clean_text}", extracting information...') 553 | llm_output = json.loads(clean_text) 554 | 555 | module = llm_output.get('module') 556 | args = llm_output.get('args') 557 | if module is not None: 558 | self.vprint(f'Module detected via LLM: {module}, arguments: {args}, prompt usage: {prompt_usage}, response usage: {response_usage}') 559 | return module, args 560 | else: 561 | self.vprint('No module detected by LLM.') 562 | return None, None 563 | except json.JSONDecodeError as e: 564 | self.vprint(f'Error decoding JSON: {e} (The LLM provided invalid JSON output even after cleaning, skipping module detection)', logging.ERROR) 565 | return None, None 566 | except Exception as e: 567 | self.vprint(f'Error: {e}', logging.error) 568 | return None, None 569 | else: 570 | self.vprint(f'Language model is disabled. Unable to detect modules via LLM.', logging.ERROR) 571 | raise Exception('Language model is disabled. Unable to detect modules via LLM.') 572 | if output: 573 | self.vprint(f'Module detected: {output}, probability: {probability}') 574 | self.vprint(f'Module detection complete, returning module: {output}') 575 | return output, None 576 | else: 577 | self.vprint('Modules disabled, enable modules in config.json to use.', logging.WARNING) 578 | return 'Modules are disabled.' 579 | 580 | def get_module_args(self, user_input, module): 581 | if self.modules_enabled: 582 | self.vprint(f'Module argument retrieval initiated, processing module: {module}') 583 | args = self.modules.get_arguments(module) 584 | 585 | if self.language_enabled: 586 | 587 | llm_input = str({"user_prompt": str(user_input), "module_selected": str(module), "arguments": args}) 588 | 589 | self.vprint(f'Input to LLM: {llm_input}') 590 | 591 | system_prompt = 'Extract arguments from a user prompt based on the module selected. Return arguments in proper JSON format. Return null if no arguments are needed.' 592 | 593 | prompt = [ 594 | { 595 | "role": "system", 596 | "content": system_prompt 597 | }, 598 | { 599 | "role": "user", 600 | "content": '{"user_prompt": "Whats the weather like in London right now?", "module_selected": "weather", "arguments": ["city"]}}' 601 | }, 602 | { 603 | "role": "assistant", 604 | "content": '{"args": [{"city": "London"}]}' 605 | }, 606 | { 607 | "role": "user", 608 | "content": llm_input 609 | } 610 | ] 611 | 612 | self.vprint(f'Starting response generation for module detection...') 613 | text, full, prompt_usage, response_usage = self.ai.generate(prompt, max_tokens=500, temperature=0) 614 | 615 | self.vprint(f'LLM output: {text}, prompt usage: {prompt_usage}, response usage: {response_usage}') 616 | 617 | try: 618 | llm_output = json.loads(text) 619 | args = llm_output['args'] 620 | except json.decoder.JSONDecodeError: 621 | self.vprint(f'LLM output is not valid JSON, unable to detect modules via LLM.', logging.ERROR) 622 | return None, None 623 | else: 624 | self.vprint(f'Language model is disabled. Unable to detect modules via LLM.', logging.ERROR) 625 | raise Exception('Language model is disabled. Unable to detect modules via LLM.') 626 | 627 | self.vprint(f'Module arguments retrieved: {args}') 628 | 629 | return args 630 | 631 | def run_module(self, module, args, **kwargs): 632 | if self.modules_enabled: 633 | self.vprint(f'Module system initiated, processing module: {module} with arguments: {args} / {kwargs}') 634 | if args is not None: 635 | output = self.modules.use_module(module, args, **kwargs) 636 | else: 637 | output = self.modules.use_module(module, **kwargs) 638 | self.vprint(f'Module output: {output}') 639 | return output 640 | else: 641 | self.vprint('Modules disabled, enable modules in config.json to use.', logging.WARNING) 642 | return 'Modules are disabled.' 643 | 644 | def speak(self, input): 645 | if self.tts_enabled: 646 | if not input: 647 | self.vprint('No input text given.', logging.ERROR) 648 | return 'No input text given.' 649 | self.vprint('Generating audio output...') 650 | output = self.tts.generate(input) 651 | self.vprint('Audio output generated.') 652 | 653 | return output 654 | else: 655 | self.vprint('TTS disabled, enable tts in config.json to use.', logging.WARNING) 656 | return 'TTS is disabled.' 657 | 658 | def listen(self, input): 659 | if self.stt_enabled: 660 | if not input: 661 | self.vprint('No input audio file specified.', logging.ERROR) 662 | return 'No input audio file specified.' 663 | self.vprint('Listening to audio input...') 664 | output = self.stt.transcribe(input) 665 | self.vprint(f'Transcription received: {output}') 666 | 667 | return output 668 | else: 669 | self.vprint('STT disabled, enable stt in config.json to use.', logging.WARNING) 670 | return 'STT is disabled.' 671 | 672 | def see(self, input): 673 | if self.vision_enabled: 674 | if not input: 675 | self.vprint('No input image specified.', logging.ERROR) 676 | return 'No input image specified.' 677 | self.vprint('Analyzing image...') 678 | output = self.vision.describe(input) 679 | self.vprint(f'Image analysis complete: {output}') 680 | 681 | return output 682 | else: 683 | self.vprint('Vision disabled, enable vision in config.json to use.', logging.WARNING) 684 | return 'Vision is disabled.' 685 | 686 | if __name__ == '__main__': 687 | print('The controller is not meant to be run directly, use one of the interfaces to interact with Ame.') 688 | pass 689 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Expl0dingCat/Ame/af0121ee3da1df0c1a2527b87b86d3a1f528093a/docs/index.md -------------------------------------------------------------------------------- /docs/modules/modules.html: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | modules 8 | 9 | 10 | 11 | 12 |
13 |
14 | 15 | 28 | 29 |
30 |
31 |
32 |
33 |

Ame Module System Documentation

34 |

Welcome to the Ame Module System documentation. This guide covers everything related to modules, including the module handler (module_handler.py), the module API in the controller, and creating custom modules.

35 |

Table of Contents

36 |
    37 |
  1. Terminology
  2. 38 |
  3. Module Handler
  4. 39 |
  5. Using Modules via the API
  6. 40 |
  7. Creating Custom Modules
  8. 41 |
  9. Training the Classifier
  10. 42 |
  11. Conclusion
  12. 43 |
44 |

1. Terminology

45 |

To ensure clarity throughout this documentation, let’s define some key terms:

46 | 52 |

2. Module Handler

53 |

2.1 Initialization

54 |

__init__(): model_path: (string), vectorizer_path: (string), modulesjson_path: (string)
55 | Initializes the module handler.

56 | 61 |

2.2 Loading Models

62 |

load_models(): model_path: (string), vectorizer_path: (string)
63 | Loads the required models for module classification.

64 | 68 |

2.3 Module Prediction

69 |

predict_module(): query: (string)
70 | Predicts a module using the built-in classification model.

71 | 74 |

2.4 Module Loading

75 |

load_modules()
76 | Loads the modules that are marked as detectable.

77 |

2.5 Module Usage

78 |

use_module(): module_name: (string), *args, **kwargs
79 | Uses a module and obtains a response from its associated function.

80 | 85 |

3. Using Modules via the API

86 |

To run a module using the API, follow these steps:

87 |
from controller import Controller
 88 | 
 89 | # Initialize the controller
 90 | controller = Controller()
 91 | 
 92 | # Run a module
 93 | response = controller.run_module(module, args)
 94 | 
95 |

4. Creating Custom Modules

96 |

To create custom modules, modify the modules.json file with the following keys:

97 | 109 |

Place your module file (.py) in the modules folder.

110 |

5. Training the Classifier

111 |

The classifier is used for both single-pass and double-pass modules. To train your module to be detected by the built-in classifier:

112 | 135 |

6. Conclusion

136 |

Thank you for exploring the Ame Module System documentation. This system empowers you to create and manage custom modules seamlessly. If you have any questions or need further assistance, please don’t hesitate to reach out.

137 |

For additional resources, including more documentation and support forums, visit the discord server.

138 | 139 |
140 |
141 | 142 | 143 | 144 | -------------------------------------------------------------------------------- /docs/modules/modules.md: -------------------------------------------------------------------------------- 1 | 2 | # Ame Module System Documentation 3 | 4 | Welcome to the Ame Module System documentation. This guide covers everything related to modules, including the module handler (`module_handler.py`), the module API in the controller, and creating custom modules. 5 | 6 | ## Table of Contents 7 | 1. [Terminology](#terminology) 8 | 2. [Module Handler](#module-handler) 9 | 3. [Using Modules via the API](#using-modules-via-the-api) 10 | 4. [Creating Custom Modules](#creating-custom-modules) 11 | 5. [Training the Classifier](#training-the-classifier) 12 | 6. [Conclusion](#conclusion) 13 | 14 | ## 1. Terminology 15 | 16 | To ensure clarity throughout this documentation, let's define some key terms: 17 | 18 | - **Query**: The user's request (e.g., "What's the weather like?"). 19 | - **Single Pass Modules**: Modules that do not require any arguments and are detected solely by the built-in module engine. 20 | - **Double Pass Modules**: Modules that have arguments requiring a second pass through a Language Model (LLM) to extract variables from the query. 21 | - **Module Handler**: The Python script `module_handler.py` responsible for managing modules. 22 | 23 | ## 2. Module Handler 24 | 25 | ### 2.1 Initialization 26 | **\_\_init\_\_()**: `model_path`: (string), `vectorizer_path`: (string), `modulesjson_path`: (string) 27 | Initializes the module handler. 28 | - `model_path`: (string) Path to the module classification model. 29 | - `vectorizer_path`: (string) Path to the vectorizer file. 30 | - `modulesjson_path`: (string) Path to the `modules.json` configuration file. 31 | 32 | ### 2.2 Loading Models 33 | **load_models()**: `model_path`: (string), `vectorizer_path`: (string) 34 | Loads the required models for module classification. 35 | - `model_path`: (string) Path to the module classification model. 36 | - `vectorizer_path`: (string) Path to the vectorizer file. 37 | 38 | ### 2.3 Module Prediction 39 | **predict_module()**: `query`: (string) 40 | Predicts a module using the built-in classification model. 41 | - `query`: (string) The user's query to predict a module. 42 | 43 | ### 2.4 Module Loading 44 | **load_modules()** 45 | Loads the modules that are marked as `detectable`. 46 | 47 | ### 2.5 Module Usage 48 | **use_module()**: `module_name`: (string), `*args`, `**kwargs` 49 | Uses a module and obtains a response from its associated function. 50 | - `module_name`: (string) Name of the module to use. 51 | - `*args`: Variable-length positional arguments. 52 | - `**kwargs`: Variable-length keyword arguments. 53 | 54 | ## 3. Using Modules via the API 55 | 56 | To run a module using the API, follow these steps: 57 | 58 | ```python 59 | from controller import Controller 60 | 61 | # Initialize the controller 62 | controller = Controller() 63 | 64 | # Run a module 65 | response = controller.run_module(module, args) 66 | ``` 67 | 68 | ## 4. Creating Custom Modules 69 | 70 | To create custom modules, modify the `modules.json` file with the following keys: 71 | 72 | - `id`: (integer) 73 | An arbitrary module identifier. 74 | - `name`: (string) 75 | Name of the module (must match module file name). 76 | - `args`: (union[None, dict]) 77 | If `None`, the module is detected using `predict_module()` and handled as a single-pass module. If `dict`, the module is detected, and its arguments are passed to an LLM to parse the query (double-pass module). 78 | - `function`: (string) 79 | The function called when the module is executed. 80 | - `detectable`: (boolean) 81 | If `True`, the module is detectable via the built-in classifier. 82 | 83 | Place your module file (`.py`) in the `modules` folder. 84 | 85 | ## 5. Training the Classifier 86 | 87 | The classifier is used for both single-pass and double-pass modules. To train your module to be detected by the built-in classifier: 88 | 89 | - Modify the training data (`module_engine/training_data.csv`) and validation data (`module_engine/validation_data.csv`) by adding pairs of queries and labels in CSV format. 90 | - Example: 91 | ``` 92 | query,label 93 | Tell me the latest finance news,News 94 | What's the current dew point?,Weather 95 | ``` 96 | 97 | - Use a Language Model (LLM) to generate these pairs, such as ChatGPT, or handwrite them. 98 | - Ensure that the training data and validation data are unique. 99 | - Start the training by running: 100 | ``` 101 | python module_engine/training/train.py 102 | ``` 103 | 104 | ## 6. Conclusion 105 | 106 | Thank you for exploring the Ame Module System documentation. This system empowers you to create and manage custom modules seamlessly. If you have any questions or need further assistance, please don't hesitate to reach out. 107 | 108 | For additional resources, including more documentation and support forums, visit [the discord server](https://discord.gg/S6h8XYsuZt). 109 | 110 | -------------------------------------------------------------------------------- /hyperdb/__init__.py: -------------------------------------------------------------------------------- 1 | from .hyperdb import * -------------------------------------------------------------------------------- /hyperdb/galaxy_brain_math_shit.py: -------------------------------------------------------------------------------- 1 | """Super valuable proprietary algorithm for ranking vector similarity. Top secret.""" 2 | import numpy as np 3 | import random 4 | 5 | def get_norm_vector(vector): 6 | if len(vector.shape) == 1: 7 | return vector / np.linalg.norm(vector) 8 | else: 9 | return vector / np.linalg.norm(vector, axis=1)[:, np.newaxis] 10 | 11 | def dot_product(vectors, query_vector): 12 | similarities = np.dot(vectors, query_vector.T) 13 | return similarities 14 | 15 | def cosine_similarity(vectors, query_vector): 16 | norm_vectors = get_norm_vector(vectors) 17 | norm_query_vector = get_norm_vector(query_vector) 18 | similarities = np.dot(norm_vectors, norm_query_vector.T) 19 | return similarities 20 | 21 | def euclidean_metric(vectors, query_vector, get_similarity_score=True): 22 | similarities = np.linalg.norm(vectors - query_vector, axis=1) 23 | if get_similarity_score: 24 | similarities = 1 / (1 + similarities) 25 | return similarities 26 | 27 | def derridaean_similarity(vectors, query_vector): 28 | def random_change(value): 29 | return value + random.uniform(-0.2, 0.2) 30 | 31 | similarities = cosine_similarity(vectors, query_vector) 32 | derrida_similarities = np.vectorize(random_change)(similarities) 33 | return derrida_similarities 34 | 35 | def adams_similarity(vectors, query_vector): 36 | def adams_change(value): 37 | return 0.42 38 | 39 | similarities = cosine_similarity(vectors, query_vector) 40 | adams_similarities = np.vectorize(adams_change)(similarities) 41 | return adams_similarities 42 | 43 | def hyper_SVM_ranking_algorithm_sort(vectors, query_vector, top_k=5, metric=cosine_similarity): 44 | """HyperSVMRanking (Such Vector, Much Ranking) algorithm proposed by Andrej Karpathy (2023) https://arxiv.org/abs/2303.18231""" 45 | similarities = metric(vectors, query_vector) 46 | top_indices = np.argsort(similarities, axis=0)[-top_k:][::-1] 47 | return top_indices.flatten(), similarities[top_indices].flatten() -------------------------------------------------------------------------------- /hyperdb/hyperdb.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | import pickle 3 | import numpy as np 4 | from sentence_transformers import SentenceTransformer 5 | 6 | from hyperdb.galaxy_brain_math_shit import ( 7 | dot_product, 8 | adams_similarity, 9 | cosine_similarity, 10 | derridaean_similarity, 11 | euclidean_metric, 12 | hyper_SVM_ranking_algorithm_sort, 13 | ) 14 | 15 | EMBEDDING_MODEL = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2', device='cuda') 16 | 17 | def get_embedding(documents, key=None): 18 | # """Default embedding function that uses OpenAI Embeddings.""" 19 | # Code changed to use embedding model 20 | if isinstance(documents, list): 21 | if isinstance(documents[0], dict): 22 | texts = [] 23 | if isinstance(key, str): 24 | if "." in key: 25 | key_chain = key.split(".") 26 | else: 27 | key_chain = [key] 28 | for doc in documents: 29 | for key in key_chain: 30 | doc = doc[key] 31 | texts.append(doc.replace("\n", " ")) 32 | elif key is None: 33 | for doc in documents: 34 | text = ", ".join([f"{key}: {value}" for key, value in doc.items()]) 35 | texts.append(text) 36 | elif isinstance(documents[0], str): 37 | texts = documents 38 | 39 | embeddings = EMBEDDING_MODEL.encode(texts) 40 | return embeddings 41 | 42 | # modify to use local model 43 | 44 | # response = openai.Embedding.create(input=texts, model=model) 45 | # return [np.array(item["embedding"]) for item in response["data"]] 46 | 47 | class HyperDB: 48 | def __init__( 49 | self, 50 | documents=None, 51 | vectors=None, 52 | key=None, 53 | embedding_function=None, 54 | similarity_metric="cosine", 55 | ): 56 | documents = documents or [] 57 | self.documents = [] 58 | self.vectors = None 59 | self.embedding_function = embedding_function or ( 60 | lambda docs: get_embedding(docs, key=key) 61 | ) 62 | if vectors is not None: 63 | self.vectors = vectors 64 | self.documents = documents 65 | else: 66 | self.add_documents(documents) 67 | 68 | if similarity_metric.__contains__("dot"): 69 | self.similarity_metric = dot_product 70 | elif similarity_metric.__contains__("cosine"): 71 | self.similarity_metric = cosine_similarity 72 | elif similarity_metric.__contains__("euclidean"): 73 | self.similarity_metric = euclidean_metric 74 | elif similarity_metric.__contains__("derrida"): 75 | self.similarity_metric = derridaean_similarity 76 | elif similarity_metric.__contains__("adams"): 77 | self.similarity_metric = adams_similarity 78 | else: 79 | raise Exception( 80 | "Similarity metric not supported. Please use either 'dot', 'cosine', 'euclidean', 'adams', or 'derrida'." 81 | ) 82 | 83 | def dict(self, vectors=False): 84 | if vectors: 85 | return [ 86 | {"document": document, "vector": vector.tolist(), "index": index} 87 | for index, (document, vector) in enumerate( 88 | zip(self.documents, self.vectors) 89 | ) 90 | ] 91 | return [ 92 | {"document": document, "index": index} 93 | for index, document in enumerate(self.documents) 94 | ] 95 | 96 | def add(self, documents, vectors=None): 97 | if not isinstance(documents, list): 98 | return self.add_document(documents, vectors) 99 | self.add_documents(documents, vectors) 100 | 101 | def add_document(self, document: dict, vector=None): 102 | 103 | # These changes were for an old version 104 | 105 | # here i also changed the line: 106 | # vector = vector or self.embedding_function([document])[0] 107 | # to: 108 | # if vector is None: 109 | # vector = self.embedding_function([document])[0] 110 | # else: 111 | # vector = vector 112 | # this is because I ran into an error: "ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()" 113 | 114 | vector = ( 115 | vector if vector is not None else self.embedding_function([document])[0] 116 | ) 117 | if self.vectors is None: 118 | self.vectors = np.empty((0, len(vector)), dtype=np.float32) 119 | elif len(vector) != self.vectors.shape[1]: 120 | raise ValueError("All vectors must have the same length.") 121 | self.vectors = np.vstack([self.vectors, vector]).astype(np.float32) 122 | self.documents.append(document) 123 | 124 | def remove_document(self, index): 125 | self.vectors = np.delete(self.vectors, index, axis=0) 126 | self.documents.pop(index) 127 | 128 | def add_documents(self, documents, vectors=None): 129 | if not documents: 130 | return 131 | vectors = vectors or np.array(self.embedding_function(documents)).astype( 132 | np.float32 133 | ) 134 | for vector, document in zip(vectors, documents): 135 | self.add_document(document, vector) 136 | 137 | def save(self, storage_file): 138 | data = {"vectors": self.vectors, "documents": self.documents} 139 | if storage_file.endswith(".gz"): 140 | with gzip.open(storage_file, "wb") as f: 141 | pickle.dump(data, f) 142 | else: 143 | with open(storage_file, "wb") as f: 144 | pickle.dump(data, f) 145 | 146 | def load(self, storage_file): 147 | if storage_file.endswith(".gz"): 148 | with gzip.open(storage_file, "rb") as f: 149 | data = pickle.load(f) 150 | else: 151 | with open(storage_file, "rb") as f: 152 | data = pickle.load(f) 153 | self.vectors = data["vectors"].astype(np.float32) 154 | self.documents = data["documents"] 155 | 156 | def query(self, query_text, top_k=5, return_similarities=True): 157 | query_vector = self.embedding_function([query_text])[0] 158 | ranked_results, similarities = hyper_SVM_ranking_algorithm_sort( 159 | self.vectors, query_vector, top_k=top_k, metric=self.similarity_metric 160 | ) 161 | if return_similarities: 162 | return list( 163 | zip([self.documents[index] for index in ranked_results], similarities) 164 | ) 165 | return [self.documents[index] for index in ranked_results] 166 | 167 | -------------------------------------------------------------------------------- /interfaces/discord/discord-bot.py: -------------------------------------------------------------------------------- 1 | import discord 2 | from controller import controller 3 | from discord.ext import commands 4 | 5 | # load token in a better way, such as from a file 6 | token = '' 7 | 8 | description = 'Ame discord interface.' 9 | intents = discord.Intents.all() 10 | intents.messages = True 11 | intents.guilds = True 12 | intents.dm_messages = True # Enable direct message intents 13 | 14 | activity = discord.Streaming(name="tears of the sky", url="https://twitch.tv/expl0dingc") 15 | 16 | bot = commands.Bot(command_prefix='!', description=description, intents=intents, activity=activity) 17 | ai = controller() 18 | 19 | @bot.event 20 | async def on_ready(): 21 | print('Ame discord interface has been initialized and connected. Ready to process messages.') 22 | 23 | @bot.event 24 | async def on_message(message): 25 | if message.author == bot.user: 26 | return 27 | 28 | # Check if message is from a guild or a DM 29 | if isinstance(message.channel, discord.DMChannel) or isinstance(message.channel, discord.GroupChannel): 30 | async with message.channel.typing(): 31 | input_message = message.content.replace(f"<@{bot.user.id}>", "Ame").strip() 32 | input, output, aud = ai.text_pipeline(input_message) 33 | 34 | response_embed = discord.Embed(title="", description=output, color=0x242768) 35 | response_embed.set_footer(text='Powered by Ame') 36 | await message.reply(embed=response_embed, mention_author=False) 37 | else: # Message is from a guild 38 | # Process user mentions 39 | if bot.user.mentioned_in(message): 40 | async with message.channel.typing(): 41 | input_message = message.content.replace(f"<@{bot.user.id}>", "Ame").strip() 42 | input, output, aud = ai.text_pipeline(input_message) 43 | 44 | response_embed = discord.Embed(title="", description=output, color=0x242768) 45 | response_embed.set_footer(text='Powered by Ame') 46 | await message.reply(embed=response_embed, mention_author=False) 47 | 48 | bot.run(token) 49 | -------------------------------------------------------------------------------- /interfaces/readme.md: -------------------------------------------------------------------------------- 1 | # Interfaces 2 | Interfaces are ways to interact with Ame. Currently, there are 2 interfaces, a client + server and a telegram bot. 3 | 4 | ## Server/client 5 | The server hosts a simple local server, that by default broadcasts on 127.0.0.1:5440, use a reverse proxy to get a public URL. This is not designed for production use. 6 | 7 | Server.py MUST be in the root folder in order to import controller. This folder only exists for organization/storage purposes. 8 | 9 | ## Discord bot 10 | Impl 11 | -------------------------------------------------------------------------------- /interfaces/server-client/client.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import pyaudio 3 | import wave 4 | import keyboard 5 | import os 6 | import shlex 7 | 8 | # If the client and server are running locally, set this to True 9 | local = False 10 | base_url = 'http://127.0.0.1:6166' # Update the port to match the server 11 | 12 | def generate_response(input: str): 13 | return requests.post(f'{base_url}/api/v1/generate', json={'input': input}) 14 | 15 | def listen_response(input: str): 16 | if local: 17 | return requests.post(f'{base_url}/api/v1/listen', json={'input': input}) 18 | else: 19 | with open(input, 'rb') as f: 20 | file = {'file': f} 21 | return requests.post(f'{base_url}/api/v1/listen', files=file) 22 | 23 | def speak_response(input: str): 24 | request = requests.post(f'{base_url}/api/v1/speak', json={'input': input}) 25 | if local: 26 | return request 27 | else: 28 | with open('ame_speech.wav', 'wb') as f: 29 | f.write(request.content) 30 | return request, os.path.abspath('ame_speech.wav') 31 | 32 | def full_response(input: str): 33 | if local: 34 | return requests.post(f'{base_url}/api/v1/full', json={'input': input}) 35 | else: 36 | with open(input, 'rb') as f: 37 | file = {'file': f} 38 | request = requests.post(f'{base_url}/api/v1/full', files=file) 39 | with open('ame_speech.wav', 'wb') as f: 40 | f.write(request.content) 41 | return request, os.path.abspath('ame_speech.wav') 42 | 43 | def send_command(input: str): 44 | return requests.post(f'{base_url}/api/v1/command', json={'input': input}) 45 | 46 | def text_input_response(input: str): 47 | return requests.post(f'{base_url}/api/v1/text', json={'input': input}) 48 | 49 | def record_voice(): 50 | CHUNK = 1024 51 | FORMAT = pyaudio.paInt16 52 | CHANNELS = 1 53 | RATE = 44100 54 | WAVE_OUTPUT_FILENAME = 'user_speech.wav' 55 | p = pyaudio.PyAudio() 56 | stream = p.open(format=FORMAT, 57 | channels=CHANNELS, 58 | rate=RATE, 59 | input=True, 60 | input_device_index=1, 61 | frames_per_buffer=CHUNK) 62 | frames = [] 63 | while True: 64 | if keyboard.is_pressed('v'): 65 | while keyboard.is_pressed('v'): 66 | data = stream.read(CHUNK) 67 | frames.append(data) 68 | break 69 | stream.stop_stream() 70 | stream.close() 71 | p.terminate() 72 | wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb') 73 | wf.setnchannels(CHANNELS) 74 | wf.setsampwidth(p.get_sample_size(FORMAT)) 75 | wf.setframerate(RATE) 76 | wf.writeframes(b''.join(frames)) 77 | wf.close() 78 | full_path = os.path.join(os.getcwd(), WAVE_OUTPUT_FILENAME) 79 | return str(rf'{full_path}') 80 | 81 | def pipe(): 82 | while True: 83 | input = record_voice() 84 | response, audio = full_response(input) 85 | response_data = response.json() 86 | user = response_data['userinput'] 87 | ame = response_data['output'] 88 | print(f'USER: {user}') 89 | print(f'AME: {ame}') 90 | 91 | if __name__ == '__main__': 92 | inmth = input('Select an input (voice/text/cmd): ') 93 | if inmth == 'voice': 94 | while True: 95 | try: 96 | print('initiating voice input... (hold v to record)') 97 | pipe() 98 | except Exception as e: 99 | print(f'Error: {e}') 100 | elif inmth == 'text': 101 | while True: 102 | try: 103 | intxt = input('USER: ') 104 | response = text_input_response(intxt) 105 | response_data = response.json() 106 | print(f'AME: {response_data["output"]}') 107 | except Exception as e: 108 | print(f'Error: {e}') 109 | elif inmth == 'cmd': 110 | print('DANGER ZONE: This is for running commands on the server. Use with caution.\n\nSupported commands:\neval : evaluates python code on the server\n') 111 | while True: 112 | intxt = input('CMD >> ') 113 | cmdargs = shlex.split(intxt) 114 | if cmdargs[0] == 'exit': 115 | exit(0) 116 | elif cmdargs[0] == 'eval': 117 | try: 118 | response = send_command(cmdargs[1]) 119 | print(response.json()) 120 | except requests.exceptions.ConnectionError: 121 | print(f'Cannot connect to server: {base_url}') 122 | else: 123 | print('Invalid input method.') -------------------------------------------------------------------------------- /interfaces/server-client/server.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import base64 4 | from aiohttp import web 5 | from aiohttp_cors import setup, ResourceOptions 6 | from controller import controller 7 | 8 | controller = controller() 9 | 10 | async def handle_generate(request): 11 | data = await request.json() 12 | input = data['input'] 13 | response = controller.generate_response(input) 14 | return web.json_response(response) 15 | 16 | async def handle_listen(request): 17 | if request.content_type == 'multipart/form-data': 18 | reader = await request.multipart() 19 | field = await reader.next() 20 | with open('user_speech.wav', 'wb') as f: 21 | while True: 22 | chunk = await field.read_chunk() 23 | if not chunk: 24 | break 25 | f.write(chunk) 26 | response = controller.listen(os.path.abspath(f.name)) 27 | return web.json_response(response) 28 | else: 29 | data = await request.json() 30 | input = data['input'] 31 | response = controller.listen(input) 32 | return web.json_response(response) 33 | 34 | async def handle_speak(request): 35 | data = await request.json() 36 | input = data['input'] 37 | userinput, output, audio_output = controller.speak(input) 38 | response = {'userinput': userinput, 'output': output} 39 | return web.json_response(response, headers={'X-Audio-Output': audio_output}) 40 | 41 | 42 | async def handle_full(request): 43 | if request.content_type == 'multipart/form-data': 44 | reader = await request.multipart() 45 | field = await reader.next() 46 | with open('speech.wav', 'wb') as f: 47 | while True: 48 | chunk = await field.read_chunk() 49 | if not chunk: 50 | break 51 | f.write(chunk) 52 | userinput, output, audio_output = controller.full_pipeline(os.path.abspath(f.name)) 53 | 54 | if audio_output is not None: 55 | with open(audio_output, 'rb') as audio_file: 56 | audio_bytes = audio_file.read() 57 | else: 58 | audio_bytes = "" 59 | 60 | headers = {'Audio-Output': audio_bytes} 61 | response = {'userinput': userinput, 'output': output} 62 | 63 | return web.json_response(response, headers=headers) 64 | else: 65 | data = await request.json() 66 | input = data['input'] 67 | userinput, output, audio_output = controller.full_pipeline(input) 68 | 69 | if audio_output is None: 70 | with open(audio_output, 'rb') as audio_file: 71 | audio_bytes = audio_file.read() 72 | else: 73 | audio_bytes = "" 74 | 75 | headers = {'Audio-Output': audio_bytes} 76 | response = {'userinput': userinput, 'output': output} 77 | 78 | return web.json_response(response, headers=headers) 79 | 80 | 81 | 82 | async def handle_text(request): 83 | data = await request.json() 84 | input = data['input'] 85 | userinput, output, audio_output = controller.text_pipeline(input) 86 | response = {'userinput': userinput, 'output': output} 87 | return web.json_response(response, headers={'X-Audio-Output': audio_output}) 88 | 89 | async def handle_command(request): 90 | data = await request.json() 91 | input = data['input'] 92 | if input == 'None': 93 | response = {'response': 'No command provided.'} 94 | else: 95 | response = controller.evaluate(input) 96 | return web.json_response(response) 97 | 98 | app = web.Application() 99 | 100 | cors = setup(app, defaults={ 101 | "*": ResourceOptions( 102 | allow_credentials=True, 103 | expose_headers="*", 104 | allow_headers="*", 105 | ) 106 | }) 107 | 108 | app.add_routes([web.post('/api/v1/full', handle_full)]) 109 | app.add_routes([web.post('/api/v1/text', handle_text)]) 110 | app.add_routes([web.post('/api/v1/generate', handle_generate)]) 111 | app.add_routes([web.post('/api/v1/listen', handle_listen)]) 112 | app.add_routes([web.post('/api/v1/speak', handle_speak)]) 113 | app.add_routes([web.post('/api/v1/command', handle_command)]) 114 | 115 | for route in list(app.router.routes()): 116 | cors.add(route) 117 | 118 | if __name__ == '__main__': 119 | web.run_app(app, port=6166, host='0.0.0.0') -------------------------------------------------------------------------------- /language/autoloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Expl0dingCat/Ame/af0121ee3da1df0c1a2527b87b86d3a1f528093a/language/autoloader.py -------------------------------------------------------------------------------- /language/llm_handler.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | from transformers import LlamaForCausalLM, LlamaTokenizer 4 | from llama_cpp import Llama 5 | 6 | class ai: 7 | def __init__(self, model_file_ext, model_pth, use_gpu=True, context=0, format='chatml', verbose=False, layers=-1, threads=None, lora_pth=None): 8 | self.model_file_ext = model_file_ext 9 | self.model_pth = model_pth 10 | self.use_gpu = use_gpu 11 | self.context = context 12 | self.format = format 13 | self.verbose = verbose 14 | self.layers = layers 15 | self.threads = threads 16 | self.lora_pth = lora_pth 17 | self.llm = None 18 | 19 | self.supported_prompt_formats_for_builtin = ['chatml'] 20 | self.supported_exts = ['.gguf', '.ggml', '.safetensors'] 21 | 22 | self.load_llm('cuda' if use_gpu else 'cpu') 23 | 24 | def load_llm(self, device): 25 | if not self.model_file_ext and not self.model_pth: 26 | raise ValueError('Model file extension and/or path must be provided.') 27 | elif not self.model_file_ext: 28 | self.model_file_ext = os.path.splitext(self.model_pth)[1] 29 | if self.model_file_ext not in self.supported_exts: 30 | raise ValueError(f'Unsupported model file extension: {self.model_file_ext}. Supported extensions: {self.supported_exts}') 31 | 32 | if self.model_file_ext == '.gguf' or self.model_file_ext == '.ggml': 33 | if device == 'cuda': 34 | self.llm = Llama(model_path=self.model_pth, verbose=self.verbose, n_gpu_layers=self.layers, n_threads=self.threads, n_ctx=self.context, chat_format=self.format, lora_pth=self.lora_pth) 35 | else: 36 | self.llm = Llama(model_path=self.model_pth, verbose=self.verbose, n_threads=self.threads, n_ctx=self.context, chat_format=self.format, lora_path=self.lora_pth) 37 | elif self.model_file_ext == '.safetensors': 38 | pass 39 | 40 | def load_llm_from_huggingface(self, model_name, device): 41 | # placeholder function 42 | # detect if self.model_pth is a huggingface model name 43 | pass 44 | 45 | def generate(self, prompt, tokens=512, temp=0.85, top_p=0.95, top_k=0): 46 | if self.model_file_ext == '.gguf' or self.model_file_ext == '.ggml': 47 | response = self.llm.create_chat_completion(messages=prompt, max_tokens=tokens, temperature=temp, top_k=top_k, top_p=top_p) 48 | 49 | full_msg = response['choices'][0]['message'] 50 | prompt_tokens = response['usage']['prompt_tokens'] 51 | completion_tokens = response['usage']['completion_tokens'] 52 | text = response['choices'][0]['message']['content'] 53 | elif self.model_file_ext == '.safetensors': 54 | pass 55 | 56 | return text.strip(), full_msg, prompt_tokens, completion_tokens 57 | 58 | def format_prompt(self, messages, format='chatml', with_assistant_prompt=True): 59 | result = '' 60 | current_role = None 61 | 62 | if format in self.supported_prompt_formats_for_builtin: 63 | if format == 'chatml': 64 | for message in messages: 65 | role = message['role'] 66 | content = message['content'] 67 | 68 | if role != current_role: 69 | current_role = role 70 | result += f'<|im_start|>{role}\n' 71 | 72 | result += f'{content}<|im_end|>\n' 73 | 74 | if with_assistant_prompt: 75 | result += '<|im_start|>assistant\n' 76 | else: 77 | raise ValueError(f"Invalid or unsupported format by Ame's built in formatter: {format}") 78 | 79 | def get_token_amt(self, text): 80 | return len(self.llm.tokenize(text.encode('utf-8'))) 81 | -------------------------------------------------------------------------------- /language/readme.md: -------------------------------------------------------------------------------- 1 | # Language 2 | This folder contains the handler for any language related tasks. -------------------------------------------------------------------------------- /logs/logging_config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | def configure_logging(): 5 | log_file_path = os.path.abspath('logs/ame.log') 6 | print(f'Logging to {log_file_path}') 7 | logging.basicConfig(filename=log_file_path, level=logging.INFO) 8 | 9 | configure_logging() -------------------------------------------------------------------------------- /memories/readme.md: -------------------------------------------------------------------------------- 1 | # Memories 2 | This is where memory instances of HyperDB are stored by default, this can be configured when you initialize the controller. -------------------------------------------------------------------------------- /memory_handler.py: -------------------------------------------------------------------------------- 1 | from hyperdb import HyperDB 2 | 3 | class memory: 4 | def __init__(self, docs=None) -> None: 5 | self.db = HyperDB(docs) 6 | 7 | def save_memory(self, db_path): 8 | self.db.save(db_path) 9 | 10 | def get_docs(self): 11 | return self.db.dict() 12 | 13 | def memorize(self, memory): 14 | self.db.add_documents(memory) 15 | 16 | def load_memory(self, db_path): 17 | self.db.load(db_path) 18 | 19 | def remember(self, query): 20 | lst = self.get_docs() 21 | # Get the highest likelihood memory 22 | results = self.db.query(query, top_k=1, return_similarities=False) 23 | # Get the 2 memories before and after the highest likelihood memory for context 24 | memory = results[0] 25 | 26 | start_index = next((i for i, d in enumerate(lst) if d['document'] == memory), None) 27 | 28 | if start_index is not None: 29 | prev_count = 1 30 | post_count = 1 31 | 32 | start = max(start_index - prev_count, 0) 33 | end = min(start_index + post_count + 1, len(lst)) 34 | 35 | result = [lst[i]['document'] for i in range(start, end)] 36 | return result 37 | else: 38 | return f"Something went wrong. Could not find memory in memory database. {memory}" 39 | 40 | if __name__ == '__main__': 41 | print('This is a handler, it is not meant to be run directly.') 42 | pass -------------------------------------------------------------------------------- /module_engine/training/train.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import joblib 3 | from sklearn.naive_bayes import MultinomialNB 4 | from sklearn.feature_extraction.text import TfidfVectorizer 5 | from sklearn.metrics import accuracy_score 6 | from sklearn.calibration import CalibratedClassifierCV 7 | 8 | print('Loading data...') 9 | 10 | train_df = pd.read_csv("training_data.csv") 11 | val_df = pd.read_csv("validation_data.csv") 12 | 13 | # Extract queries and labels from the dataframes 14 | train_queries = train_df['query'] 15 | train_labels = train_df['label'] 16 | 17 | val_queries = val_df['query'] 18 | val_labels = val_df['label'] 19 | 20 | print('Removing data duplicates in training...') 21 | unique_train_queries = set() 22 | unique_train_data = [] 23 | for query, label in zip(train_queries, train_labels): 24 | if query not in unique_train_queries: 25 | unique_train_queries.add(query) 26 | unique_train_data.append((query, label)) 27 | train_queries = [query for query, _ in unique_train_data] 28 | train_labels = [label for _, label in unique_train_data] 29 | 30 | print('Removing data duplicates in validation...') 31 | unique_val_queries = set() 32 | unique_val_data = [] 33 | for query, label in zip(val_queries, val_labels): 34 | if query not in unique_val_queries: 35 | unique_val_queries.add(query) 36 | unique_val_data.append((query, label)) 37 | val_queries = [query for query, _ in unique_val_data] 38 | val_labels = [label for _, label in unique_val_data] 39 | 40 | unique_train_queries = set(train_queries) 41 | unique_val_queries = set(val_queries) 42 | 43 | # Find common queries between training and validation data 44 | common_queries = unique_train_queries.intersection(unique_val_queries) 45 | print('Checking for data contamination...') 46 | if len(common_queries) > 0: 47 | print("Warning: Training data leaked into validation data!") 48 | print("Common Queries:") 49 | for query in common_queries: 50 | print(query) 51 | 52 | val_queries_cleaned = [query for query in val_queries if query not in common_queries] 53 | val_labels_cleaned = [label for query, label in zip(val_queries, val_labels) if query not in common_queries] 54 | 55 | val_queries = val_queries_cleaned 56 | val_labels = val_labels_cleaned 57 | 58 | print("Validation data has been cleaned.") 59 | else: 60 | print("No training data leaked into validation data.") 61 | 62 | vectorizer = TfidfVectorizer() 63 | 64 | # Transform the training and validation data 65 | train_vectors = vectorizer.fit_transform(train_queries) 66 | val_vectors = vectorizer.transform(val_queries) 67 | 68 | # Initialize the Naive Bayes classifier 69 | nb_classifier = MultinomialNB(alpha=0.1) 70 | 71 | # Fit the classifier on the training data 72 | nb_classifier.fit(train_vectors, train_labels) 73 | 74 | # Calibrate the classifier to get better probability estimates 75 | calibrated_classifier = CalibratedClassifierCV(nb_classifier, method='sigmoid') 76 | calibrated_classifier.fit(train_vectors, train_labels) 77 | 78 | # Predict on the validation data and get calibrated probabilities 79 | val_predictions = calibrated_classifier.predict(val_vectors) 80 | val_probabilities = calibrated_classifier.predict_proba(val_vectors) 81 | 82 | accuracy = accuracy_score(val_labels, val_predictions) 83 | print(f"Validation Accuracy: {accuracy:.2%}") 84 | 85 | model_filename = 'naive_bayes_model.pkl' 86 | joblib.dump(nb_classifier, model_filename) 87 | vectorizer_filename = 'tfidf_vectorizer.pkl' 88 | joblib.dump(vectorizer, vectorizer_filename) 89 | 90 | print('Done! Trained models saved.') 91 | -------------------------------------------------------------------------------- /module_engine/training/training_data.csv: -------------------------------------------------------------------------------- 1 | query,label 2 | What's the weather like today?,Weather 3 | Set the temperature to 22 degrees,Home Assistant 4 | When is my next meeting?,Google Calendar 5 | Read me the news headlines,News 6 | Translate 'Hello' to French,DeepL 7 | List all connected devices,Home Assistant 8 | What's the current temperature outside?,Weather 9 | Add an event to my calendar,Google Calendar 10 | Play some relaxing music,Home Assistant 11 | Translate 'Goodbye' to Spanish,DeepL 12 | What's the forecast for tomorrow?,Weather 13 | Turn off the lights,Home Assistant 14 | What are the top news stories today?,News 15 | Is it going to rain this weekend?,Weather 16 | Remind me to buy groceries,Google Calendar 17 | Change the color of the bedroom lights to blue,Home Assistant 18 | Translate 'Thank you' to German,DeepL 19 | What's the humidity level right now?,Weather 20 | List my upcoming appointments,Google Calendar 21 | Read me the latest sports news,News 22 | Should I bring an umbrella today?,Weather 23 | Create a new event for tomorrow,Google Calendar 24 | Set the brightness of the living room lights to 50%,Home Assistant 25 | Translate 'Good morning' to Italian,DeepL 26 | What's the wind speed at the moment?,Weather 27 | When is my doctor's appointment?,Google Calendar 28 | Tell me the latest technology news,News 29 | What's the UV index today?,Weather 30 | How much time do I have until my next meeting?,Google Calendar 31 | Play some energetic music,Home Assistant 32 | Translate 'I love you' to French,DeepL 33 | What's the current visibility?,Weather 34 | What's my schedule for tomorrow?,Google Calendar 35 | Give me the top entertainment news,News 36 | What's the atmospheric pressure today?,Weather 37 | Add a reminder to call mom,Google Calendar 38 | Turn on the air conditioner,Home Assistant 39 | Translate 'Welcome' to Spanish,DeepL 40 | What's the sunrise time?,Weather 41 | What's on my calendar for next week?,Google Calendar 42 | Tell me the latest business news,News 43 | What's the sunset time?,Weather 44 | Cancel my lunch meeting,Google Calendar 45 | Change the temperature of the thermostat to 20 degrees,Home Assistant 46 | Translate 'Good night' to German,DeepL 47 | Find the nearest weather station,Weather 48 | What's the location of my next appointment?,Google Calendar 49 | Give me the celebrity gossip news,News 50 | How high is the chance of rain tomorrow?,Weather 51 | Set a reminder for tomorrow morning,Google Calendar 52 | Dim the lights in the living room,Home Assistant 53 | Translate 'Bonjour' to Italian,DeepL 54 | Should I wear a jacket today?,Weather 55 | When is my dentist appointment?,Google Calendar 56 | Tell me the latest finance news,News 57 | How hot will it get later today?,Weather 58 | What's my schedule for next month?,Google Calendar 59 | Play my favorite playlist,Home Assistant 60 | Translate 'Je vous remercie' to French,DeepL 61 | What's the current dew point?,Weather 62 | What's the date of my anniversary?,Google Calendar 63 | Read me the latest health news,News 64 | Is there a chance of thunderstorms tomorrow?,Weather 65 | Add a reminder to pick up dry cleaning,Google Calendar 66 | Turn on the coffee machine,Home Assistant 67 | Translate 'Gracias' to Spanish,DeepL 68 | What's the current wind direction?,Weather 69 | What time does my flight depart?,Google Calendar 70 | Give me the latest world news,News 71 | How much precipitation is expected in the next hour?,Weather 72 | Create a new event for next Friday,Google Calendar 73 | Start the robotic vacuum cleaner,Home Assistant 74 | Translate 'Buongiorno' to Italian,DeepL 75 | What's the current visibility in miles?,Weather 76 | When is my son's soccer game?,Google Calendar 77 | Tell me the latest fashion news,News 78 | What's the heat index today?,Weather 79 | How many days until my vacation?,Google Calendar 80 | Turn on the TV,Home Assistant 81 | Translate 'Bonne nuit' to French,DeepL 82 | What's the current air quality index?,Weather 83 | Set a reminder for my wife's birthday,Google Calendar 84 | Give me the latest science news,News 85 | What's the wind chill factor?,Weather 86 | When is my next team meeting?,Google Calendar 87 | Dim the lights in the bedroom,Home Assistant 88 | Translate 'Guten Tag' to German,DeepL 89 | What's the current ozone level?,Weather 90 | What time is my daughter's dance class?,Google Calendar 91 | Tell me the latest movie news,News 92 | Where can I find the weather forecast for the next week?,Weather 93 | Add a reminder to take the car for servicing,Google Calendar 94 | Turn on the sprinkler system,Home Assistant 95 | Translate 'Hasta luego' to Spanish,DeepL 96 | What's the current pollen count?,Weather 97 | When is my next project deadline?,Google Calendar 98 | Give me the latest music news,News 99 | What's the current snow depth?,Weather 100 | What time is my son's piano lesson?,Google Calendar 101 | Set the lighting scene to 'Romantic',Home Assistant 102 | Translate 'Ciao' to Italian,DeepL 103 | What's the current moon phase?,Weather 104 | What's the location of my daughter's ballet class?,Google Calendar 105 | Tell me the latest travel news,News 106 | What's the current temperature in Celsius?,Weather 107 | Create a new event for next month,Google Calendar 108 | Turn off the music,Home Assistant 109 | Translate 'Merci' to French,DeepL 110 | What's the current temperature in Fahrenheit?,Weather 111 | When is my next dentist appointment?,Google Calendar 112 | Give me the latest celebrity news,News 113 | What's the current UV index?,Weather 114 | How much free time do I have today?,Google Calendar 115 | Dim the lights in the kitchen,Home Assistant 116 | Translate 'Hola' to Spanish,DeepL 117 | What's the current wind speed in mph?,Weather 118 | What time is my doctor's appointment?,Google Calendar 119 | Tell me the latest sports news headlines,News 120 | What's the current humidity level?,Weather 121 | What's on my schedule for the next week?,Google Calendar 122 | Skip to the next song,Home Assistant 123 | Translate 'Gute Nacht' to German,DeepL 124 | What's the current visibility in kilometers?,Weather 125 | When is my next vacation?,Google Calendar 126 | Give me the latest technology news,News 127 | Is it going to be a sunny day tomorrow?,Weather 128 | Add a reminder to call the insurance company,Google Calendar 129 | Turn on the lights in the hallway,Home Assistant 130 | Translate 'Buongiornino' to Italian,DeepL 131 | What's the current wind direction in degrees?,Weather 132 | What time is my flight arrival?,Google Calendar 133 | Tell me the latest business news headlines,News 134 | What's the current chance of rain?,Weather 135 | How many days until my birthday?,Google Calendar 136 | Set the thermostat to eco mode,Home Assistant 137 | Translate 'Buenas tardes' to Spanish,DeepL 138 | What's the current air pressure?,Weather 139 | When is my next doctor's appointment?,Google Calendar 140 | Give me the top entertainment news headlines,News 141 | What's the current atmospheric pressure?,Weather 142 | When is my next lunch meeting?,Google Calendar 143 | Change the color of the lights in the living room to red,Home Assistant 144 | Translate 'Buona sera' to Italian,DeepL 145 | What's the chance of thunderstorms today?,Weather 146 | What time is my dentist appointment tomorrow?,Google Calendar 147 | Tell me the latest gossip news,News 148 | How high is the UV index tomorrow?,Weather 149 | Remind me to buy groceries on my way home,Google Calendar 150 | Turn off the air purifier,Home Assistant 151 | Translate 'Bonjourno' to French,DeepL 152 | What's the current dew point in degrees?,Weather 153 | What's the date of my friend's wedding?,Google Calendar 154 | Give me the latest health news headlines,News 155 | What's the heat index tomorrow?,Weather 156 | How many days until my job interview?,Google Calendar 157 | Play some ambient music,Home Assistant 158 | Translate 'Gracias amigo' to Spanish,DeepL 159 | What's the current wind chill factor?,Weather 160 | What time is my daughter's swim practice?,Google Calendar 161 | Tell me the latest fashion news headlines,News 162 | What's the chance of rain in the next hour?,Weather 163 | Create a new event for next year,Google Calendar 164 | Change the color of the lights in the bedroom to green,Home Assistant 165 | Translate 'Buonanotte'to French,DeepL 166 | What's the current ozone level in parts per billion?,Weather 167 | What time is my son's soccer game tomorrow?,Google Calendar 168 | Give me the latest movie news headlines,News 169 | What's the temperature trend for the next few hours?,Weather 170 | When is my next team meeting next week?,Google Calendar 171 | Turn on the lights in the dining room,Home Assistant 172 | Translate 'Hola amigo' to Spanish,DeepL 173 | What's the current pollen count in grains per cubic meter?,Weather 174 | What time is my next project deadline?,Google Calendar 175 | Tell me the latest music news headlines,News 176 | What's the current snow depth in inches?,Weather 177 | What time is my son's piano lesson tomorrow?,Google Calendar 178 | Set the lighting scene to 'Party',Home Assistant 179 | Translate 'Ciao bella' to Italian,DeepL 180 | What's the current moon phase in words?,Weather 181 | Give me the latest travel news headlines,News 182 | What's the current temperature in degrees Celsius?,Weather 183 | When is my next dentist appointment next month?,Google Calendar 184 | Tell me the latest celebrity news headlines,News 185 | What's the current UV index on a scale of 1 to 10?,Weather 186 | How much free time do I have tomorrow?,Google Calendar 187 | Dim the lights in the bathroom,Home Assistant 188 | Translate 'Hola guapa' to Spanish,DeepL 189 | What's the current wind speed in miles per hour?,Weather 190 | When is my next doctor's appointment next week?,Google Calendar 191 | Tell me the latest sports news headlines,News 192 | What's the current humidity level in percentage?,Weather 193 | What's on my schedule for next month?,Google Calendar 194 | Skip to the previous song,Home Assistant 195 | Translate 'Auf Wiedersehen' to German,DeepL 196 | What's the current visibility in kilometers?,Weather 197 | When is my next vacation next year?,Google Calendar 198 | Give me the latest technology news headlines,News 199 | Is it going to be a cloudy day tomorrow?,Weather 200 | Add a reminder to call the insurance company next week,Google Calendar 201 | Turn on the lights in the study room,Home Assistant 202 | Translate 'Je vous remercie beaucoup' to French,DeepL 203 | What's the current wind direction in degrees?,Weather 204 | What time is my flight arrival tomorrow?,Google Calendar 205 | Tell me the latest business news headlines,News 206 | What's the current chance of rain in percentage?,Weather 207 | How many days until my birthday next month?,Google Calendar 208 | Set the thermostat to vacation mode,Home Assistant 209 | Translate 'Buenas tardes amigo' to Spanish,DeepL 210 | What's the current air pressure in pascals?,Weather 211 | When is my next doctor's appointment next month?,Google Calendar 212 | Give me the top entertainment news headlines,News 213 | What's the current atmospheric pressure in hectopascals?,Weather 214 | When is my next lunch meeting next week?,Google Calendar 215 | Change the color of the lights in the living room to red,Home Assistant 216 | Translate 'Buona sera bella' to Italian,DeepL 217 | What's the chance of thunderstorms today in percentage?,Weather 218 | What time is my dentist appointment tomorrow?,Google Calendar 219 | Tell me the latest gossip news headlines,News 220 | How high is the UV index tomorrow on a scale of 1 to 10?,Weather 221 | Remind me to buy groceries on my way home next week,Google Calendar 222 | Turn off the air purifier,Home Assistant 223 | Translate 'Bonjourno mon ami' to French,DeepL 224 | What's the current dew point in degrees?,Weather 225 | What's the date of my friend's wedding next month?,Google Calendar 226 | Give me the latest health news headlines,News 227 | What's the heat index tomorrow in degrees?,Weather 228 | How many days until my job interview next month?,Google Calendar 229 | Play some ambient music,Home Assistant 230 | Translate 'Gracias amigo lindo' to Spanish,DeepL 231 | What's the current wind chill factor in degrees?,Weather 232 | What time is my daughter's swim practice tomorrow?,Google Calendar 233 | Tell me the latest fashion news headlines,News 234 | What's the chance of rain in the next hour in percentage?,Weather 235 | Create a new event for next year,Google Calendar 236 | Change the color of the lights in the bedroom to green,Home Assistant 237 | Translate 'Buonanotte bellezza' to Italian,DeepL 238 | What's the current moon phase in words?,Weather 239 | What's the location of my daughter's ballet class next month?,Google Calendar 240 | Tell me the latest travel news headlines,News 241 | What's the current temperature in degrees Celsius?,Weather 242 | When is my next dentist appointment next year?,Google Calendar 243 | Give me the latest celebrity news headlines,News 244 | What's the current UV index on a scale of 1 to 10?,Weather 245 | How much free time do I have tomorrow?,Google Calendar 246 | Dim the lights in the bathroom,Home Assistant 247 | Translate 'Hola guapa' to Spanish,DeepL 248 | What is the forecast for tomorrow?,Weather 249 | What's the current wind speed in miles per hour?,Weather 250 | When is my next doctor's appointment next year?,Google Calendar 251 | Tell me the latest sports news headlines,News 252 | What's the current humidity level in percentage?,Weather 253 | What's on my schedule for next month?,Google Calendar 254 | Whats the weather like next week?,Weather 255 | query,label 256 | What's the weather like today?,Weather 257 | Set the temperature to 22 degrees,Home Assistant 258 | When is my next meeting?,Google Calendar 259 | Read me the news headlines,News 260 | Translate 'Hello' to French,DeepL 261 | List all connected devices,Home Assistant 262 | What's the current temperature outside?,Weather 263 | Add an event to my calendar,Google Calendar 264 | Play some relaxing music,Home Assistant 265 | Translate 'Goodbye' to Spanish,DeepL 266 | What's the forecast for tomorrow?,Weather 267 | Turn off the lights,Home Assistant 268 | What are the top news stories today?,News 269 | Is it going to rain this weekend?,Weather 270 | Remind me to buy groceries,Google Calendar 271 | Change the color of the bedroom lights to blue,Home Assistant 272 | Translate 'Thank you' to German,DeepL 273 | What's the humidity level right now?,Weather 274 | List my upcoming appointments,Google Calendar 275 | Read me the latest sports news,News 276 | Should I bring an umbrella today?,Weather 277 | Create a new event for tomorrow,Google Calendar 278 | Set the brightness of the living room lights to 50%,Home Assistant 279 | Translate 'Good morning' to Italian,DeepL 280 | What's the wind speed at the moment?,Weather 281 | When is my doctor's appointment?,Google Calendar 282 | Tell me the latest technology news,News 283 | What's the UV index today?,Weather 284 | How much time do I have until my next meeting?,Google Calendar 285 | Play some energetic music,Home Assistant 286 | Translate 'I love you' to French,DeepL 287 | What's the current visibility?,Weather 288 | What's my schedule for tomorrow?,Google Calendar 289 | Give me the top entertainment news,News 290 | What's the atmospheric pressure today?,Weather 291 | Add a reminder to call mom,Google Calendar 292 | Turn on the air conditioner,Home Assistant 293 | Translate 'Welcome' to Spanish,DeepL 294 | What's the sunrise time?,Weather 295 | What's on my calendar for next week?,Google Calendar 296 | Tell me the latest business news,News 297 | What's the sunset time?,Weather 298 | Cancel my lunch meeting,Google Calendar 299 | Change the temperature of the thermostat to 20 degrees,Home Assistant 300 | Translate 'Good night' to German,DeepL 301 | Find the nearest weather station,Weather 302 | What's the location of my next appointment?,Google Calendar 303 | Give me the celebrity gossip news,News 304 | How high is the chance of rain tomorrow?,Weather 305 | Set a reminder for tomorrow morning,Google Calendar 306 | Dim the lights in the living room,Home Assistant 307 | Translate 'Bonjour' to Italian,DeepL 308 | Should I wear a jacket today?,Weather 309 | When is my dentist appointment?,Google Calendar 310 | Tell me the latest finance news,News 311 | How hot will it get later today?,Weather 312 | What's my schedule for next month?,Google Calendar 313 | Play my favorite playlist,Home Assistant 314 | Translate 'Je vous remercie' to French,DeepL 315 | What's the current dew point?,Weather 316 | What's the date of my anniversary?,Google Calendar 317 | Read me the latest health news,News 318 | Is there a chance of thunderstorms tomorrow?,Weather 319 | Add a reminder to pick up dry cleaning,Google Calendar 320 | Turn on the coffee machine,Home Assistant 321 | Translate 'Gracias' to Spanish,DeepL 322 | What's the current wind direction?,Weather 323 | What time does my flight depart?,Google Calendar 324 | Give me the latest world news,News 325 | How much precipitation is expected in the next hour?,Weather 326 | Create a new event for next Friday,Google Calendar 327 | Start the robotic vacuum cleaner,Home Assistant 328 | Translate 'Buongiorno' to Italian,DeepL 329 | What's the current visibility in miles?,Weather 330 | What's my schedule for next month?,Google Calendar 331 | Tell me the latest travel news,News 332 | What's the heat index today?,Weather 333 | How many days until my vacation?,Google Calendar 334 | Turn on the TV,Home Assistant 335 | Translate 'Bonne nuit' to French,DeepL 336 | What's the current air quality index?,Weather 337 | Set a reminder for my wife's birthday,Google Calendar 338 | Give me the latest science news,News 339 | What's the wind chill factor?,Weather 340 | When is my next team meeting?,Google Calendar 341 | Dim the lights in the bedroom,Home Assistant 342 | Translate 'Guten Tag' to German,DeepL 343 | What's the current ozone level?,Weather 344 | What time is my daughter's dance class?,Google Calendar 345 | Tell me the latest movie news,News 346 | What's the sunset time?,Weather 347 | Cancel my lunch meeting,Google Calendar 348 | Change the temperature of the thermostat to 20 degrees,Home Assistant 349 | Translate 'Good night' to German,DeepL 350 | Find the nearest weather station,Weather 351 | What's the location of my next appointment?,Google Calendar 352 | Give me the celebrity gossip news,News 353 | How high is the chance of rain tomorrow?,Weather 354 | Set a reminder for tomorrow morning,Google Calendar 355 | Dim the lights in the living room,Home Assistant 356 | Translate 'Bonjour' to Italian,DeepL 357 | Should I wear a jacket today?,Weather 358 | When is my dentist appointment?,Google Calendar 359 | Tell me the latest finance news,News 360 | How hot will it get later today?,Weather 361 | What's my schedule for next month?,Google Calendar 362 | Play my favorite playlist,Home Assistant 363 | Translate 'Je vous remercie' to French,DeepL 364 | What's the current dew point?,Weather 365 | What's the date of my anniversary?,Google Calendar 366 | Read me the latest health news,News 367 | Is there a chance of thunderstorms tomorrow?,Weather 368 | Add a reminder to pick up dry cleaning,Google Calendar 369 | Turn on the coffee machine,Home Assistant 370 | Translate 'Gracias' to Spanish,DeepL 371 | What's the current wind direction?,Weather 372 | What time does my flight depart?,Google Calendar 373 | Give me the latest world news,News 374 | How much precipitation is expected in the next hour?,Weather 375 | Create a new event for next Friday,Google Calendar 376 | Start the robotic vacuum cleaner,Home Assistant 377 | Translate 'Buongiorno' to Italian,DeepL 378 | What's the current visibility in miles?,Weather 379 | What's my schedule for next month?,Google Calendar 380 | Tell me the latest travel news,News 381 | What's the heat index today?,Weather 382 | How many days until my vacation?,Google Calendar 383 | Turn on the TV,Home Assistant 384 | Translate 'Bonne nuit' to French,DeepL 385 | What's the current air quality index?,Weather 386 | Set a reminder for my wife's birthday,Google Calendar 387 | Give me the latest science news,News 388 | What's the wind chill factor?,Weather 389 | When is my next team meeting?,Google Calendar 390 | Dim the lights in the bedroom,Home Assistant 391 | Translate 'Guten Tag' to German,DeepL 392 | What's the current ozone level?,Weather 393 | What time is my daughter's dance class?,Google Calendar 394 | Tell me the latest movie news,News 395 | What's the sunset time?,Weather 396 | Cancel my lunch meeting,Google Calendar 397 | Change the temperature of the thermostat to 20 degrees,Home Assistant 398 | Translate 'Good night' to German,DeepL 399 | Find the nearest weather station,Weather 400 | What's the location of my next appointment?,Google Calendar 401 | Give me the celebrity gossip news,News 402 | How high is the chance of rain tomorrow?,Weather 403 | Set a reminder for tomorrow morning,Google Calendar 404 | Dim the lights in the living room,Home Assistant 405 | Translate 'Bonjour' to Italian,DeepL 406 | Should I wear a jacket today?,Weather 407 | When is my dentist appointment?,Google Calendar 408 | Tell me the latest finance news,News 409 | How hot will it get later today?,Weather 410 | What's my schedule for next month?,Google Calendar 411 | Play my favorite playlist,Home Assistant 412 | Translate 'Je vous remercie' to French,DeepL 413 | What's the current dew point?,Weather 414 | What's the date of my anniversary?,Google Calendar 415 | Read me the latest health news,News 416 | Is there a chance of thunderstorms tomorrow?,Weather 417 | Add a reminder to pick up dry cleaning,Google Calendar 418 | Turn on the coffee machine,Home Assistant 419 | Translate 'Gracias' to Spanish,DeepL 420 | What's the current wind direction?,Weather 421 | What time does my flight depart?,Google Calendar 422 | Give me the latest world news,News 423 | How much precipitation is expected in the next hour?,Weather 424 | Create a new event for next Friday,Google Calendar 425 | Start the robotic vacuum cleaner,Home Assistant 426 | Translate 'Buongiorno' to Italian,DeepL 427 | What's the current visibility in miles?,Weather 428 | What's my schedule for next month?,Google Calendar 429 | Tell me the latest travel news,News 430 | What's the heat index today?,Weather 431 | How many days until my vacation?,Google Calendar 432 | Turn on the TV,Home Assistant 433 | Translate 'Bonne nuit' to French,DeepL 434 | What's the current air quality index?,Weather 435 | Set a reminder for my wife's birthday,Google Calendar 436 | Give me the latest science news,News 437 | What's the wind chill factor?,Weather 438 | When is my next team meeting?,Google Calendar 439 | Dim the lights in the bedroom,Home Assistant 440 | Translate 'Guten Tag' to German,DeepL 441 | What's the current ozone level?,Weather 442 | What time does my flight depart?,Google Calendar 443 | Give me the latest world news,News 444 | How much precipitation is expected in the next hour?,Weather 445 | Create a new event for next Friday,Google Calendar 446 | Start the robotic vacuum cleaner,Home Assistant 447 | Translate 'Buongiorno' to Italian,DeepL 448 | What's the current visibility in miles?,Weather 449 | What's my schedule for next month?,Google Calendar 450 | Tell me the latest travel news,News 451 | What's the heat index today?,Weather 452 | How many days until my vacation?,Google Calendar 453 | Turn on the TV,Home Assistant 454 | Translate 'Bonne nuit' to French,DeepL 455 | What's the current air quality index?,Weather 456 | Set a reminder for my wife's birthday,Google Calendar 457 | Give me the latest science news,News 458 | What's the wind chill factor?,Weather 459 | When is my next team meeting?,Google Calendar 460 | Dim the lights in the bedroom,Home Assistant 461 | Translate 'Guten Tag' to German,DeepL 462 | What's the current ozone level?,Weather 463 | What time does my flight depart?,Google Calendar 464 | Give me the latest world news,News 465 | How much precipitation is expected in the next hour?,Weather 466 | Create a new event for next Friday,Google Calendar 467 | Start the robotic vacuum cleaner,Home Assistant 468 | Translate 'Buongiorno' to Italian,DeepL 469 | What's the current visibility in miles?,Weather 470 | What's my schedule for next month?,Google Calendar 471 | Tell me the latest travel news,News 472 | What's the heat index today?,Weather 473 | How many days until my vacation?,Google Calendar 474 | Turn on the TV,Home Assistant 475 | Translate 'Bonne nuit' to French,DeepL 476 | What's the current air quality index?,Weather 477 | Set a reminder for my wife's birthday,Google Calendar 478 | What's the weather like right now?,Weather 479 | Set the temperature to 22 degrees,Home Assistant 480 | When is my next meeting?,Google Calendar 481 | Read me the news headlines,News 482 | Translate 'Hello' to French,DeepL 483 | List all connected devices,Home Assistant 484 | What will the weather be like tomorrow?,Weather 485 | Add an event to my calendar,Google Calendar 486 | Play some relaxing music,Home Assistant 487 | Translate 'Goodbye' to Spanish,DeepL 488 | What's the forecast for today?,Weather 489 | Turn off the lights,Home Assistant 490 | What are the top news stories today?,News 491 | Is it going to rain this weekend?,Weather 492 | Remind me to buy groceries,Google Calendar 493 | Change the color of the bedroom lights to blue,Home Assistant 494 | Translate 'Thank you' to German,DeepL 495 | What's the humidity level right now?,Weather 496 | List my upcoming appointments,Google Calendar 497 | Read me the latest sports news,News 498 | Should I bring an umbrella today?,Weather 499 | Create a new event for tomorrow,Google Calendar 500 | Set the brightness of the living room lights to 50%,Home Assistant 501 | Translate 'Good morning' to Italian,DeepL 502 | What's the wind speed at the moment?,Weather 503 | When is my doctor's appointment?,Google Calendar 504 | Tell me the latest technology news,News 505 | What's the UV index today?,Weather 506 | How much time do I have until my next meeting?,Google Calendar 507 | Play some energetic music,Home Assistant 508 | Translate 'I love you' to French,DeepL 509 | What's the current visibility?,Weather 510 | What's my schedule for tomorrow?,Google Calendar 511 | Give me the top entertainment news,News 512 | What's the atmospheric pressure today?,Weather 513 | Add a reminder to call mom,Google Calendar 514 | Turn on the air conditioner,Home Assistant 515 | Translate 'Welcome' to Spanish,DeepL 516 | When will it rain today?,Weather 517 | What's on my calendar for next week?,Google Calendar 518 | Tell me the latest business news,News 519 | When will it snow this week?,Weather 520 | Cancel my lunch meeting,Google Calendar 521 | Change the temperature of the thermostat to 20 degrees,Home Assistant 522 | Translate 'Good night' to German,DeepL 523 | Find the nearest weather station,Weather 524 | What's the location of my next appointment?,Google Calendar 525 | Give me the celebrity gossip news,News 526 | What's the chance of rain tomorrow?,Weather 527 | Set a reminder for tomorrow morning,Google Calendar 528 | Dim the lights in the living room,Home Assistant 529 | Translate 'Bonjour' to Italian,DeepL 530 | What's the current wind direction?,Weather 531 | What time does my flight depart?,Google Calendar 532 | Tell me the latest world news,News 533 | How much precipitation is expected in the next hour?,Weather 534 | Create a new event for next Friday,Google Calendar 535 | Start the robotic vacuum cleaner,Home Assistant 536 | Translate 'Buongiorno' to Italian,DeepL 537 | What's the current visibility in miles?,Weather 538 | What's my schedule for next month?,Google Calendar 539 | Give me the top entertainment news,News 540 | What's the atmospheric pressure today?,Weather 541 | Add a reminder to call mom,Google Calendar 542 | Turn on the air conditioner,Home Assistant 543 | Translate 'Welcome' to Spanish,DeepL 544 | When will it rain today?,Weather 545 | What's on my calendar for next week?,Google Calendar 546 | Tell me the latest business news,News 547 | When will it snow this week?,Weather 548 | Cancel my lunch meeting,Google Calendar 549 | Change the temperature of the thermostat to 20 degrees,Home Assistant 550 | Translate 'Good night' to German,DeepL 551 | Find the nearest weather station,Weather 552 | What's the location of my next appointment?,Google Calendar 553 | Give me the celebrity gossip news,News 554 | What's the chance of rain tomorrow?,Weather 555 | Set a reminder for tomorrow morning,Google Calendar 556 | Dim the lights in the living room,Home Assistant 557 | Translate 'Bonjour' to Italian,DeepL 558 | What's the current wind direction?,Weather 559 | What time does my flight depart?,Google Calendar 560 | Tell me the latest world news,News 561 | How much precipitation is expected in the next hour?,Weather 562 | Create a new event for next Friday,Google Calendar 563 | Start the robotic vacuum cleaner,Home Assistant 564 | Translate 'Buongiorno' to Italian,DeepL 565 | What's the current visibility in miles?,Weather 566 | What's my schedule for next month?,Google Calendar 567 | Give me the top entertainment news,News 568 | What's the atmospheric pressure today?,Weather 569 | Add a reminder to call mom,Google Calendar 570 | Turn on the air conditioner,Home Assistant 571 | Translate 'Welcome' to Spanish,DeepL 572 | When will it rain today?,Weather 573 | What's on my calendar for next week?,Google Calendar 574 | Tell me the latest business news,News 575 | When will it snow this week?,Weather 576 | Cancel my lunch meeting,Google Calendar 577 | this is a test of the module system,example 578 | test the module system,example 579 | test,example 580 | test of the module system,example 581 | test the new module systewm,example 582 | this is just a test of modules,example 583 | testing modules,example 584 | a quick test of modules,example 585 | let's test modules,example 586 | conducting a test of the module system,example -------------------------------------------------------------------------------- /module_engine/training/validation_data.csv: -------------------------------------------------------------------------------- 1 | query,label 2 | What's the weather like tomorrow?,Weather 3 | Turn on the lights in the bedroom,Home Assistant 4 | What's on my calendar for today?,Google Calendar 5 | Read me the latest news,News 6 | Translate 'Good evening' to French,DeepL 7 | List all available sensors,Home Assistant 8 | What's the current temperature?,Weather 9 | Add a reminder for my doctor's appointment,Google Calendar 10 | Play some background music,Home Assistant 11 | Translate 'Have a nice day' to Spanish,DeepL 12 | What's the forecast for the next week?,Weather 13 | Turn off the TV,Home Assistant 14 | Give me the breaking news headlines,News 15 | Is it going to be sunny today?,Weather 16 | Remind me to call mom tomorrow,Google Calendar 17 | Change the color of the lights to purple,Home Assistant 18 | Translate 'Grazie' to Italian,DeepL 19 | What's the current humidity?,Weather 20 | What's my schedule for today?,Google Calendar 21 | Tell me the latest sports news,News 22 | Should I carry an umbrella today?,Weather 23 | Create a new event for next weekend,Google Calendar 24 | Set the brightness level to 70%,Home Assistant 25 | Translate 'Goodbye' to German,DeepL 26 | What's the wind speed?,Weather 27 | When is my next meeting?,Google Calendar 28 | Give me the latest technology news,News 29 | What's the UV index?,Weather 30 | How much time do I have until my appointment?,Google Calendar 31 | Play some upbeat music,Home Assistant 32 | Translate 'I love you' to French,DeepL 33 | What's the current visibility?,Weather 34 | What's my schedule for next week?,Google Calendar 35 | Tell me the latest business news,News 36 | What's the atmospheric pressure?,Weather 37 | Add a reminder to buy groceries tomorrow,Google Calendar 38 | Turn on the fan,Home Assistant 39 | Translate 'Welcome' to Spanish,DeepL 40 | What's the sunrise time today?,Weather 41 | What's on my calendar for tomorrow?,Google Calendar 42 | Give me the top news headlines,News 43 | What's the sunset time today?,Weather 44 | Cancel my meeting,Google Calendar 45 | Change the temperature setting to 22 degrees,Home Assistant 46 | Translate 'Good morning' to Italian,DeepL 47 | What's the current weather condition?,Weather 48 | Turn on the lights in the living room,Home Assistant 49 | What's on my agenda for today?,Google Calendar 50 | Read me the latest headlines,News 51 | Translate 'Thank you' to French,DeepL 52 | List all connected devices,Home Assistant 53 | What's the current temperature?,Weather 54 | Add a reminder for my appointment,Google Calendar 55 | Play some music,Home Assistant 56 | Translate 'Goodbye' to Spanish,DeepL 57 | What's the forecast for tomorrow?,Weather 58 | Turn off the lights,Home Assistant 59 | Give me the top news stories,News 60 | Is it going to rain today?,Weather 61 | Remind me to call mom,Google Calendar 62 | Change the color of the lights to blue,Home Assistant 63 | Translate 'Thank you' to German,DeepL 64 | What's the current humidity?,Weather 65 | What's my schedule for today?,Google Calendar 66 | Tell me the latest sports news,News 67 | Should I bring an umbrella today?,Weather 68 | Create a new event for tomorrow,Google Calendar 69 | Set the brightness level to 50%,Home Assistant 70 | Translate 'Good morning' to Italian,DeepL 71 | What's the wind speed?,Weather 72 | When is my next meeting?,Google Calendar 73 | Give me the latest technology news,News 74 | What's the UV index?,Weather 75 | How much time do I have until my next meeting?,Google Calendar 76 | Play some music,Home Assistant 77 | Translate 'I love you' to French,DeepL 78 | What's the current visibility?,Weather 79 | What's my schedule for next week?,Google Calendar 80 | Tell me the latest business news,News 81 | What's the atmospheric pressure?,Weather 82 | Add a reminder to call mom tomorrow,Google Calendar 83 | Turn on the air conditioner,Home Assistant 84 | Translate 'Welcome' to Spanish,DeepL 85 | What's the sunrise time?,Weather 86 | What's on my calendar for tomorrow?,Google Calendar 87 | Give me the top news headlines,News 88 | What's the sunset time?,Weather 89 | Cancel my lunch meeting,Google Calendar 90 | Change the temperature setting to 20 degrees,Home Assistant 91 | Translate 'Good night' to German,DeepL 92 | What's the current visibility?,Weather 93 | When is my next meeting?,Google Calendar 94 | Give me the latest technology news,News 95 | What's the UV index?,Weather 96 | How much time do I have until my next appointment?,Google Calendar 97 | Play some music,Home Assistant 98 | Translate 'I love you' to French,DeepL 99 | What's the current visibility?,Weather 100 | What's my schedule for next week?,Google Calendar 101 | Tell me the latest business news,News 102 | What's the atmospheric pressure?,Weather 103 | Add a reminder to call mom tomorrow,Google Calendar 104 | Turn on the air conditioner,Home Assistant 105 | Translate 'Welcome' to Spanish,DeepL 106 | What's the sunrise time?,Weather 107 | What's on my calendar for tomorrow?,Google Calendar 108 | Give me the top news headlines,News 109 | What's the sunset time?,Weather 110 | Cancel my lunch meeting,Google Calendar 111 | Change the temperature setting to 20 degrees,Home Assistant 112 | Translate 'Good night' to German,DeepL 113 | What's the heat index?,Weather 114 | When is my next dentist appointment?,Google Calendar 115 | Tell me the latest finance news,News 116 | How hot is it going to get today?,Weather 117 | What's my schedule for next month?,Google Calendar 118 | Play my favorite playlist,Home Assistant 119 | Translate 'Je vous remercie' to French,DeepL 120 | What's the current dew point?,Weather 121 | What's the date of my anniversary?,Google Calendar 122 | Read me the latest health news,News 123 | Is there a chance of thunderstorms today?,Weather 124 | Add a reminder to pick up dry cleaning,Google Calendar 125 | Turn on the coffee machine,Home Assistant 126 | Translate 'Gracias' to Spanish,DeepL 127 | What's the current wind direction?,Weather 128 | What time does my flight depart?,Google Calendar 129 | Give me the latest world news,News 130 | How much precipitation is expected in the next hour?,Weather 131 | Create a new event for next Friday,Google Calendar 132 | Start the robotic vacuum cleaner,Home Assistant 133 | Translate 'Buongiorno' to Italian,DeepL 134 | What's the current visibility?,Weather 135 | When is my son's soccer game?,Google Calendar 136 | Tell me the latest fashion news,News 137 | What's the heat index?,Weather 138 | How many days until my vacation?,Google Calendar 139 | Turn on the TV,Home Assistant 140 | Translate 'Bonne nuit' to French,DeepL 141 | What's the current air quality index?,Weather 142 | Set a reminder for my wife's birthday,Google Calendar 143 | Give me the latest science news,News 144 | What's the wind chill factor?,Weather 145 | When is my next team meeting?,Google Calendar 146 | Dim the lights in the bedroom,Home Assistant 147 | Translate 'Guten Tag' to German,DeepL 148 | What's the current ozone level?,Weather 149 | What time is my daughter's dance class?,Google Calendar 150 | Tell me the latest movie news,News 151 | Where can I find the weather forecast for the next week?,Weather 152 | Add a reminder to take the car for servicing,Google Calendar 153 | Turn on the sprinkler system,Home Assistant 154 | Translate 'Hasta luego' to Spanish,DeepL 155 | What's the current pollen count?,Weather 156 | When is my next project deadline?,Google Calendar 157 | Give me the latest music news,News 158 | What's the current snow depth?,Weather 159 | What time is my son's piano lesson?,Google Calendar 160 | Set the lighting scene to 'Romantic',Home Assistant 161 | Translate 'Ciao' to Italian,DeepL 162 | What's the current moon phase?,Weather 163 | What's the location of my daughter's ballet class?,Google Calendar 164 | Tell me the latest travel news,News 165 | What's the current temperature?,Weather 166 | Create a new event for next month,Google Calendar 167 | Turn off the music,Home Assistant 168 | Translate 'Merci' to French,DeepL 169 | What's the current temperature?,Weather 170 | When is my next dentist appointment?,Google Calendar 171 | Give me the latest celebrity news,News 172 | What's the current UV index?,Weather 173 | How much free time do I have today?,Google Calendar 174 | Dim the lights in the kitchen,Home Assistant 175 | Translate 'Hola' to Spanish,DeepL 176 | What's the current wind speed?,Weather 177 | What time is my doctor's appointment?,Google Calendar 178 | Tell me the latest sports news,News 179 | What's the current humidity?,Weather 180 | What's on my schedule for next week?,Google Calendar 181 | Skip to the next song,Home Assistant 182 | Translate 'Gute Nacht' to German,DeepL 183 | What's the current visibility?,Weather 184 | When is my next vacation?,Google Calendar 185 | Give me the latest technology news,News 186 | Is it going to be a sunny day?,Weather 187 | Add a reminder to call the insurance company,Google Calendar 188 | Turn on the lights in the hallway,Home Assistant 189 | Translate 'Buongiornino' to Italian,DeepL 190 | What's the current wind direction?,Weather 191 | What time is my flight arrival?,Google Calendar 192 | Tell me the latest business news,News 193 | What's the current chance of rain?,Weather 194 | How many days until my birthday?,Google Calendar 195 | query,label 196 | What should I wear today?,Weather 197 | Sync my Google Calendar with Home Assistant,Home Assistant 198 | What's the latest news update?,News 199 | Translate 'Good afternoon' to Spanish,DeepL 200 | What's the current temperature in Fahrenheit?,Weather 201 | Remind me to call John tomorrow,Google Calendar 202 | Turn on the lights in the kitchen,Home Assistant 203 | Translate 'Guten Morgen' to German,DeepL 204 | What's the chance of rain in percentage?,Weather 205 | When is my next dentist appointment?,Google Calendar 206 | Tell me the latest sports scores,News 207 | What's the air quality index today?,Weather 208 | How much time do I have until my next appointment?,Google Calendar 209 | Play some jazz music,Home Assistant 210 | Translate 'Ti amo' to Italian,DeepL 211 | What's the current wind speed?,Weather 212 | What's the location of my next event?,Google Calendar 213 | Give me the latest science discoveries,News 214 | What's the potential for severe weather today?,Weather 215 | Create a new event for next week,Google Calendar 216 | Turn off the TV,Home Assistant 217 | Translate 'Merci beaucoup' to French,DeepL 218 | What's the current dew point in degrees Celsius?,Weather 219 | When is my next work meeting?,Google Calendar 220 | Tell me the latest fashion trends,News 221 | What's the chance of snow tomorrow?,Weather 222 | Set a reminder to buy milk,Google Calendar 223 | Dim the lights in the hallway,Home Assistant 224 | Translate 'Hola amigo' to Italian,DeepL 225 | What's the current visibility in meters?,Weather 226 | What time is my dentist appointment next month?,Google Calendar 227 | Give me the latest music releases,News 228 | What's the heat index tomorrow in Celsius?,Weather 229 | How many days until my birthday next week?,Google Calendar 230 | Play classical music,Home Assistant 231 | Translate 'Grazie amico' to Spanish,DeepL 232 | What's the current wind chill factor in Fahrenheit?,Weather 233 | What time is my son's soccer game tomorrow?,Google Calendar 234 | Tell me the latest celebrity gossip,News 235 | What's the UV index tomorrow on a scale from 1 to 10?,Weather 236 | Should I bring an umbrella tomorrow?,Weather 237 | Add a reminder to water the plants,Google Calendar 238 | Turn on the heater,Home Assistant 239 | Translate 'Buenas noches amigo' to Italian,DeepL 240 | What's the current air pressure in millibars?,Weather 241 | When is my next doctor's appointment next week?,Google Calendar 242 | Give me the latest health tips,News 243 | What's the wind chill factor tomorrow in Fahrenheit?,Weather 244 | What time is my daughter's piano lesson tomorrow?,Google Calendar 245 | Set the lighting scene to 'Relax',Home Assistant 246 | Translate 'Salut' to French,DeepL 247 | What's the current moon phase?,Weather 248 | What's the location of my daughter's soccer practice?,Google Calendar 249 | Tell me the latest travel destinations,News 250 | What's the current temperature in Kelvin?,Weather 251 | When is my next dentist appointment next week?,Google Calendar 252 | Give me the latest celebrity gossip,News 253 | What's the current UV index on a scale of 1 to 10?,Weather 254 | How much free time do I have tomorrow?,Google Calendar 255 | Turn off the lights in the study room,Home Assistant 256 | Translate 'Hola guapo' to Italian,DeepL 257 | What's the current wind speed in kilometers per hour?,Weather 258 | When is my next doctor's appointment next year?,Google Calendar 259 | Tell me the latest sports updates,News 260 | What's the current humidity level in percentage?,Weather 261 | What's on my schedule for next month?,Google Calendar 262 | Skip to the next track,Home Assistant 263 | Translate 'Auf Wiedersehen' to French,DeepL 264 | What's the current visibility in meters?,Weather 265 | When is my next vacation next year?,Google Calendar 266 | Give me the latest technology releases,News 267 | Is it going to be a foggy day tomorrow?,Weather 268 | Add a reminder to call the insurance company next month,Google Calendar 269 | Turn on the lights in the bedroom,Home Assistant 270 | Translate 'Je vous remercie beaucoup' to Italian,DeepL 271 | What's the current wind direction in degrees?,Weather 272 | What time does my flight arrive?,Google Calendar 273 | Tell me the latest business updates,News 274 | What's the current chance of rain in percentage?,Weather 275 | How many days until my birthday next month?,Google Calendar 276 | Set the thermostat to energy-saving mode,Home Assistant 277 | Translate 'Buenas tardes amigo' to French,DeepL 278 | What's the current air pressure in Pascal?,Weather 279 | When is my next doctor's appointment next month?,Google Calendar 280 | Give me the latest entertainment news,News 281 | What's the atmospheric pressure today in hectopascals?,Weather 282 | When is my next lunch meeting next month?,Google Calendar 283 | Change the color of the lights in the living room to red,Home Assistant 284 | Translate 'Buona sera bella' to Spanish,DeepL 285 | What's the chance of thunderstorms today in percentage?,Weather 286 | What time is my dentist appointment tomorrow?,Google Calendar 287 | Tell me the latest gossip news,News 288 | How high is the UV index tomorrow on a scale of 1 to 10?,Weather 289 | Remind me to buy groceries on my way home next month,Google Calendar 290 | Turn off the air purifier,Home Assistant 291 | Translate 'Bonsoir mon ami' to Italian,DeepL 292 | What's the current dew point in Fahrenheit?,Weather 293 | What's the date of my friend's wedding next week?,Google Calendar 294 | Give me the latest health tips,News 295 | What's the heat index tomorrow in Fahrenheit?,Weather 296 | How many days until my job interview next month?,Google Calendar 297 | Play some calm music,Home Assistant 298 | Translate 'Gracias amigo lindo' to French,DeepL 299 | What's the current wind chill factor in Fahrenheit?,Weather 300 | What time is my daughter's swim practice tomorrow?,Google Calendar 301 | Tell me the latest fashion trends,News 302 | What's the chance of rain in the next hour in percentage?,Weather 303 | Create a new event for next year,Google Calendar 304 | Change the color of the lights in the bedroom to green,Home Assistant 305 | Translate 'Buonanotte bellezza' to Spanish,DeepL 306 | What's the current moon phase?,Weather 307 | Give me the latest travel updates,News 308 | What's the current temperature in degrees Celsius?,Weather 309 | When is my next dentist appointment next year?,Google Calendar 310 | Tell me the latest celebrity gossip,News 311 | What's the current UV index on a scale of 1 to 10?,Weather 312 | How much free time do I have tomorrow?,Google Calendar 313 | Dim the lights in the bathroom,Home Assistant 314 | Translate 'Hola guapa' to Italian,DeepL 315 | What's the current wind speed in kilometers per hour?,Weather 316 | When is my next doctor's appointment next month?,Google Calendar 317 | Tell me the latest sports updates,News 318 | What's the current humidity level in percentage?,Weather 319 | What's on my schedule for next month?,Google Calendar 320 | Skip to the previous track,Home Assistant 321 | Translate 'Auf Wiedersehen' to French,DeepL 322 | What's the current visibility in meters?,Weather 323 | When is my next vacation next year?,Google Calendar 324 | Give me the latest technology releases,News 325 | Is it going to be a foggy day tomorrow?,Weather 326 | Add a reminder to call the insurance company next month,Google Calendar 327 | Turn on the lights in the bedroom,Home Assistant 328 | Translate 'Je vous remercie beaucoup' to Italian,DeepL 329 | What's the current wind direction in degrees?,Weather 330 | What time does my flight arrive?,Google Calendar 331 | Tell me the latest business updates,News 332 | What's the current chance of rain in percentage?,Weather 333 | How many days until my birthday next month?,Google Calendar 334 | Set the thermostat to energy-saving mode,Home Assistant 335 | Translate 'Buenas tardes amigo' to French,DeepL 336 | What's the current air pressure in Pascal?,Weather 337 | When is my next doctor's appointment next month?,Google Calendar 338 | Give me the latest entertainment news,News 339 | What's the atmospheric pressure today in hectopascals?,Weather 340 | When is my next lunch meeting next month?,Google Calendar 341 | Change the color of the lights in the living room to red,Home Assistant 342 | Translate 'Buona sera bella' to Spanish,DeepL 343 | What's the chance of thunderstorms today in percentage?,Weather 344 | What time is my dentist appointment tomorrow?,Google Calendar 345 | Tell me the latest gossip news,News 346 | How high is the UV index tomorrow on a scale of 1 to 10?,Weather 347 | Remind me to buy groceries on my way home next month,Google Calendar 348 | Turn off the air purifier,Home Assistant 349 | Translate 'Bonsoir mon ami' to Italian,DeepL 350 | What's the current dew point in Fahrenheit?,Weather 351 | What's the date of my friend's wedding next week?,Google Calendar 352 | Give me the latest health tips,News 353 | What's the heat index tomorrow in Fahrenheit?,Weather 354 | How many days until my job interview next month?,Google Calendar 355 | Play some calm music,Home Assistant 356 | Translate 'Gracias amigo lindo' to French,DeepL 357 | What's the current wind chill factor in Fahrenheit?,Weather 358 | What time is my daughter's swim practice tomorrow?,Google Calendar 359 | Tell me the latest fashion trends,News 360 | What's the chance of rain in the next hour in percentage?,Weather 361 | Create a new event for next year,Google Calendar 362 | Change the color of the lights in the bedroom to green,Home Assistant 363 | Translate 'Buonanotte bellezza' to Spanish,DeepL 364 | What's the current moon phase?,Weather 365 | Give me the latest travel updates,News 366 | What's the current temperature in degrees Celsius?,Weather 367 | When is my next dentist appointment next year?,Google Calendar 368 | Tell me the latest celebrity gossip,News 369 | What's the current UV index on a scale of 1 to 10?,Weather 370 | How much free time do I have tomorrow?,Google Calendar 371 | Dim the lights in the bathroom,Home Assistant 372 | Translate 'Hola guapa' to Italian,DeepL 373 | What's the current wind speed in kilometers per hour?,Weather 374 | When is my next doctor's appointment next month?,Google Calendar 375 | Tell me the latest sports updates,News 376 | What's the current humidity level in percentage?,Weather 377 | What's on my schedule for next month?,Google Calendar 378 | Skip to the previous track,Home Assistant 379 | Translate 'Auf Wiedersehen' to French,DeepL 380 | What's the current visibility in meters?,Weather 381 | When is my next vacation next year?,Google Calendar 382 | Give me the latest technology releases,News 383 | Is it going to be a foggy day tomorrow?,Weather 384 | Add a reminder to call the insurance company next month,Google Calendar 385 | Turn on the lights in the bedroom,Home Assistant 386 | Translate 'Je vous remercie beaucoup' to Italian,DeepL 387 | What's the current wind direction in degrees?,Weather 388 | What time does my flight arrive?,Google Calendar 389 | Tell me the latest business updates,News 390 | What's the current chance of rain in percentage?,Weather 391 | How many days until my birthday next month?,Google Calendar 392 | Set the thermostat to energy-saving mode,Home Assistant 393 | Translate 'Buenas tardes amigo' to French,DeepL 394 | What's the current air pressure in Pascal?,Weather 395 | When is my next doctor's appointment next month?,Google Calendar 396 | Give me the latest entertainment news,News 397 | What's the atmospheric pressure today in hectopascals?,Weather 398 | When is my next lunch meeting next month?,Google Calendar 399 | Change the color of the lights in the living room to red,Home Assistant 400 | Translate 'Buona sera bella' to Spanish,DeepL 401 | What's the chance of thunderstorms today in percentage?,Weather 402 | What time is my dentist appointment tomorrow?,Google Calendar 403 | Tell me the latest gossip news,News 404 | How high is the UV index tomorrow on a scale of 1 to 10?,Weather 405 | Remind me to buy groceries on my way home next month,Google Calendar 406 | Turn off the air purifier,Home Assistant 407 | Translate 'Bonsoir mon ami' to Italian,DeepL 408 | What's the current dew point in Fahrenheit?,Weather 409 | What's the date of my friend's wedding next week?,Google Calendar 410 | Give me the latest health tips,News 411 | What's the heat index tomorrow in Fahrenheit?,Weather 412 | How many days until my job interview next month?,Google Calendar 413 | Play some calm music,Home Assistant 414 | Translate 'Gracias amigo lindo' to French,DeepL 415 | What's the current wind chill factor in Fahrenheit?,Weather 416 | What time is my daughter's swim practice tomorrow?,Google Calendar 417 | Tell me the latest fashion trends,News 418 | What's the chance of rain in the next hour in percentage?,Weather 419 | Create a new event for next year,Google Calendar 420 | Change the color of the lights in the bedroom to green,Home Assistant 421 | Translate 'Buonanotte bellezza' to Spanish,DeepL 422 | What's the current moon phase?,Weather 423 | Give me the latest travel updates,News 424 | What's the current temperature in degrees Celsius?,Weather 425 | When is my next dentist appointment next year?,Google Calendar 426 | Tell me the latest celebrity gossip,News 427 | What's the current UV index on a scale of 1 to 10?,Weather 428 | How much free time do I have tomorrow?,Google Calendar 429 | Dim the lights in the bathroom,Home Assistant 430 | Translate 'Hola guapa' to Italian,DeepL 431 | What's the current wind speed in kilometers per hour?,Weather 432 | When is my next doctor's appointment next month?,Google Calendar 433 | Tell me the latest sports updates,News 434 | What's the current humidity level in percentage?,Weather 435 | What's on my schedule for next month?,Google Calendar 436 | Skip to the previous track,Home Assistant 437 | Translate 'Auf Wiedersehen' to French,DeepL 438 | What's the current visibility in meters?,Weather 439 | When is my next vacation next year?,Google Calendar 440 | Give me the latest technology releases,News 441 | Is it going to be a foggy day tomorrow?,Weather 442 | Add a reminder to call the insurance company next month,Google Calendar 443 | Turn on the lights in the bedroom,Home Assistant 444 | Translate 'Je vous remercie beaucoup' to Italian,DeepL 445 | What's the current wind direction in degrees?,Weather 446 | What time does my flight arrive?,Google Calendar 447 | Tell me the latest business updates,News 448 | What's the current chance of rain in percentage?,Weather 449 | How many days until my birthday next month?,Google Calendar 450 | Set the thermostat to energy-saving mode,Home Assistant 451 | Translate 'Buenas tardes amigo' to French,DeepL 452 | What's the current air pressure in Pascal?,Weather 453 | When is my next doctor's appointment next month?,Google Calendar 454 | Give me the latest entertainment news,News 455 | What's the atmospheric pressure today in hectopascals?,Weather 456 | When is my next lunch meeting next month?,Google Calendar 457 | Change the color of the lights in the living room to red,Home Assistant 458 | Translate 'Buona sera bella' to Spanish,DeepL 459 | What's the chance of thunderstorms today in percentage?,Weather 460 | What time is my dentist appointment tomorrow?,Google Calendar 461 | Tell me the latest gossip news,News 462 | How high is the UV index tomorrow on a scale of 1 to 10?,Weather 463 | Remind me to buy groceries on my way home next month,Google Calendar 464 | Turn off the air purifier,Home Assistant 465 | Translate 'Bonsoir mon ami' to Italian,DeepL 466 | What's the current dew point in Fahrenheit?,Weather 467 | What's the date of my friend's wedding next week?,Google Calendar 468 | Give me the latest health tips,News 469 | What's the heat index tomorrow in Fahrenheit?,Weather 470 | How many days until my job interview next month?,Google Calendar 471 | Play some calm music,Home Assistant 472 | Translate 'Gracias amigo lindo' to French,DeepL 473 | What's the current wind chill factor in Fahrenheit?,Weather 474 | What time is my daughter's swim practice tomorrow?,Google Calendar 475 | Tell me the latest fashion trends,News 476 | What's the chance of rain in the next hour in percentage?,Weather 477 | Create a new event for next year,Google Calendar 478 | Change the color of the lights in the bedroom to green,Home Assistant 479 | Translate 'Buonanotte bellezza' to Spanish,DeepL 480 | What's the current moon phase?,Weather 481 | Give me the latest travel updates,News 482 | What's the current temperature in degrees Celsius?,Weather 483 | When is my next dentist appointment next year?,Google Calendar 484 | Tell me the latest celebrity gossip,News 485 | What's the current UV index on a scale of 1 to 10?,Weather 486 | How much free time do I have tomorrow?,Google Calendar 487 | Dim the lights in the bathroom,Home Assistant 488 | Translate 'Hola guapa' to Italian,DeepL 489 | What's the current wind speed in kilometers per hour?,Weather 490 | When is my next doctor's appointment next month?,Google Calendar 491 | Tell me the latest sports updates,News 492 | What's the current humidity level in percentage?,Weather 493 | What's on my schedule for next month?,Google Calendar 494 | Skip to the previous track,Home Assistant 495 | Translate 'Auf Wiedersehen' to French,DeepL 496 | What's the current visibility in meters?,Weather 497 | When is my next vacation next year?,Google Calendar 498 | Give me the latest technology releases,News 499 | Is it going to be a foggy day tomorrow?,Weather 500 | Add a reminder to call the insurance company next month,Google Calendar 501 | Turn on the lights in the bedroom,Home Assistant 502 | Translate 'Je vous remercie beaucoup' to Italian,DeepL 503 | What's the current wind direction in degrees?,Weather 504 | What time does my flight arrive?,Google Calendar 505 | Tell me the latest business updates,News 506 | What's the current chance of rain in percentage?,Weather 507 | How many days until my birthday next month?,Google Calendar 508 | Set the thermostat to energy-saving mode,Home Assistant 509 | Translate 'Buenas tardes amigo' to French,DeepL 510 | What's the current air pressure in Pascal?,Weather 511 | When is my next doctor's appointment next month?,Google Calendar 512 | Give me the latest entertainment news,News 513 | What's the atmospheric pressure today in hectopascals?,Weather 514 | When is my next lunch meeting next month?,Google Calendar 515 | Change the color of the lights in the living room to red,Home Assistant 516 | Translate 'Buona sera bella' to Spanish,DeepL 517 | What's the chance of thunderstorms today in percentage?,Weather 518 | What time is my dentist appointment tomorrow?,Google Calendar 519 | Tell me the latest gossip news,News 520 | How high is the UV index tomorrow on a scale of 1 to 10?,Weather 521 | Remind me to buy groceries on my way home next month,Google Calendar 522 | Turn off the air purifier,Home Assistant 523 | Translate 'Bonsoir mon ami' to Italian,DeepL 524 | What's the current dew point in Fahrenheit?,Weather 525 | What's the date of my friend's wedding next week?,Google Calendar 526 | Give me the latest health tips,News 527 | What's the heat index tomorrow in Fahrenheit?,Weather 528 | How many days until my job interview next month?,Google Calendar 529 | Play some calm music,Home Assistant 530 | Translate 'Gracias amigo lindo' to French,DeepL 531 | What's the current wind chill factor in Fahrenheit?,Weather 532 | What time is my daughter's swim practice tomorrow?,Google Calendar 533 | Tell me the latest fashion trends,News 534 | What's the chance of rain in the next hour in percentage?,Weather 535 | Create a new event for next year,Google Calendar 536 | Change the color of the lights in the bedroom to green,Home Assistant 537 | Translate 'Buonanotte bellezza' to Spanish,DeepL 538 | What's the current moon phase?,Weather 539 | Give me the latest travel updates,News 540 | What's the current temperature in degrees Celsius?,Weather 541 | When is my next dentist appointment next year?,Google Calendar 542 | Tell me the latest celebrity gossip,News 543 | What's the current UV index on a scale of 1 to 10?,Weather 544 | How much free time do I have tomorrow?,Google Calendar 545 | Dim the lights in the bathroom,Home Assistant 546 | Translate 'Hola guapa' to Italian,DeepL 547 | What's the current wind speed in kilometers per hour?,Weather 548 | When is my next doctor's appointment next month?,Google Calendar 549 | Tell me the latest sports updates,News 550 | How does the weather look for tomorrow?,Weather 551 | testing module system,example 552 | test module system,example 553 | test of the module system,example 554 | Evaluating the module system,example 555 | Integrating the module system,example 556 | module system test,example -------------------------------------------------------------------------------- /module_handler.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import joblib 3 | import json 4 | import os 5 | import logging 6 | from logs.logging_config import configure_logging 7 | 8 | class modules: 9 | def __init__(self, model_path, vectorizer_path, modulesjson_path) -> None: 10 | configure_logging() 11 | self.logger = logging.getLogger(__name__) 12 | 13 | self.module_map = {} 14 | try: 15 | self.module_json = json.load(open(modulesjson_path, "r")) 16 | except Exception: 17 | self.logger.error(f"Error loading modules json, please check the modules json path.") 18 | self.module_json = [] 19 | 20 | if vectorizer_path and model_path: 21 | self.logger.info("Naive bayes classifier enabled, loading models...") 22 | try: 23 | self.load_models(model_path, vectorizer_path) 24 | self.detectable_available = True 25 | except Exception: 26 | self.logger.error(f"Error loading models, please check the model and vectorizer paths. Detected modules will not be loaded.") 27 | self.detectable_available = False 28 | else: 29 | self.detectable_available = False 30 | self.logger.info("Naive bayes classifier disabled, using LLM for module detection, detectable modules will be treated as non-detectable.") 31 | self.load_modules() 32 | 33 | def load_models(self, model_path, vectorizer_path): 34 | model_filename = model_path 35 | self.loaded_model = joblib.load(model_filename) 36 | 37 | vectorizer_filename = vectorizer_path 38 | self.vectorizer = joblib.load(vectorizer_filename) 39 | 40 | def predict_module(self, query): 41 | if self.detectable_available: 42 | query_vector = self.vectorizer.transform([query]) 43 | predictions = self.loaded_model.predict(query_vector) 44 | predicted_probabilities = self.loaded_model.predict_proba(query_vector) 45 | if max(predicted_probabilities[0]) < 0.9: 46 | return None, max(predicted_probabilities[0]) 47 | return predictions[0], max(predicted_probabilities[0]) 48 | else: 49 | self.logger.warning("Unable to predict modules using classifier, vectorizer and model not loaded.") 50 | return None, 100.0 51 | 52 | def load_modules(self): 53 | self.undetectable_modules = [] 54 | self.detectable_modules = [] 55 | for module_info in self.module_json: 56 | module_name = module_info["name"] 57 | module_file = f"modules/{module_name}.py" 58 | if os.path.isfile(module_file): 59 | try: 60 | module = importlib.import_module(f"modules.{module_name}") 61 | if module_info.get("detectable", True): 62 | if self.detectable_available: 63 | self.detectable_modules.append(module_name) 64 | else: 65 | self.logger.warning(f"Module '{module_name}' is detectable but the classifier is not available, module will classified as undetectable.") 66 | self.undetectable_modules.append(module_name) 67 | else: 68 | self.undetectable_modules.append(module_name) 69 | self.module_map[module_name] = module 70 | 71 | except ModuleNotFoundError: 72 | pass 73 | 74 | def use_module(self, module_name, args, **kwargs): 75 | module = self.module_map.get(module_name) 76 | if module: 77 | for module_info in self.module_json: 78 | if module_info["name"] == module_name: 79 | func_name = module_info.get("function", "default_function_name") 80 | func = getattr(module, func_name, None) 81 | if func and callable(func): 82 | if isinstance(args, dict): 83 | return func(**args, **kwargs) 84 | else: 85 | return func(args, **kwargs) 86 | else: 87 | self.logger.warning(f"Function '{func_name}' not found in module '{module_name}'. Unable to call function.") 88 | return f"Function '{func_name}' not found in module '{module_name}'. Unable to call function." 89 | self.logger.warning(f"Module information for '{module_name}' not found in module_json.") 90 | return f"Module information for '{module_name}' not found in module_json." 91 | else: 92 | self.logger.warning(f"Module '{module_name}' not found.") 93 | return f"Module '{module_name}' not found" 94 | 95 | def get_undetectable_modules(self): 96 | return self.undetectable_modules 97 | 98 | def get_detectable_modules(self): 99 | return self.detectable_modules 100 | 101 | def get_arguments(self, module_name): 102 | module = self.module_map.get(module_name) 103 | if module: 104 | for module_info in self.module_json: 105 | if module_info["name"] == module_name: 106 | return module_info.get("args") 107 | self.logger.warning(f"Module information for '{module_name}' not found in module_json.") 108 | return f"Module information for '{module_name}' not found in module_json." 109 | else: 110 | self.logger.warning(f"Module '{module_name}' not found.") 111 | return f"Module '{module_name}' not found." 112 | 113 | if __name__ == '__main__': 114 | print('This is a handler, it is not meant to be run directly.') 115 | pass 116 | -------------------------------------------------------------------------------- /modules/crypto_data.py: -------------------------------------------------------------------------------- 1 | # pip install -U pycoingecko 2 | from pycoingecko import CoinGeckoAPI 3 | 4 | cg = CoinGeckoAPI() 5 | 6 | def get_info(full_crypto_name): 7 | name = full_crypto_name.lower() 8 | try: 9 | coin = cg.get_coin_by_id(id=name) 10 | data = {'name': coin['name'], 'current_price (usd)': coin['market_data']['current_price']['usd'], 'market cap (usd)': coin['market_data']['market_cap']['usd'], 'total volume (usd)': coin['market_data']['total_volume']['usd'], '24h change %': coin['market_data']['price_change_percentage_24h']} 11 | return data 12 | except Exception as e: 13 | return e 14 | -------------------------------------------------------------------------------- /modules/modules.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 0, 4 | "name": "weather", 5 | "args": ["city"], 6 | "function": "get_weather", 7 | "detectable": false 8 | }, 9 | { 10 | "id": 1, 11 | "name": "stock_data", 12 | "args": ["ticker", "date"], 13 | "function": "get_info", 14 | "detectable": false 15 | }, 16 | { 17 | "id": 2, 18 | "name": "crypto_data", 19 | "args": ["full_crypto_name"], 20 | "function": "get_info", 21 | "detectable": false 22 | } 23 | ] -------------------------------------------------------------------------------- /modules/stock_data.py: -------------------------------------------------------------------------------- 1 | # pip install -U polygon-api-client 2 | # uses polygon.io to get data 3 | from polygon import RESTClient 4 | from datetime import datetime 5 | 6 | # get your api key at https://polygon.io/ 7 | polygon_api_key = '' 8 | 9 | client = RESTClient(api_key=polygon_api_key) 10 | 11 | def get_info(ticker, date): 12 | if not date: 13 | date = datetime.now().strftime("%Y-%m-%d") 14 | 15 | try: 16 | info = client.get_daily_open_close_agg(ticker, date) 17 | except Exception as e: 18 | return e 19 | 20 | return info 21 | -------------------------------------------------------------------------------- /modules/weather.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | # Enter openweathermap API key here (get it at https://openweathermap.org/api) 5 | api_key = None 6 | 7 | def get_weather(city): 8 | if api_key: 9 | url = f"http://api.openweathermap.org/data/2.5/weather?appid={api_key}&q={city}&units=metric" 10 | response = requests.get(url) 11 | response = json.loads(response.text) 12 | if response["cod"] != "404": 13 | temperature = response["main"]["temp"] 14 | pressure = response["main"]["pressure"] 15 | humidity = response["main"]["humidity"] 16 | feels_like = response["main"]["feels_like"] 17 | wind = response["wind"]["speed"] 18 | description = response["weather"][0]["description"] 19 | 20 | return {"city": city, "temperature": temperature, "pressure": pressure, "humidity": humidity, "description": description, "feels_like": feels_like, "wind": wind, "units": "metric"} 21 | else: 22 | return "Invalid city." 23 | else: 24 | return "No API key provided. Unable to fetch weather, add API key in weather.py." -------------------------------------------------------------------------------- /setup_wizard.py: -------------------------------------------------------------------------------- 1 | import os 2 | import platform 3 | import sys 4 | import subprocess 5 | import pkg_resources 6 | 7 | def input_y_n(msg: str): 8 | while True: 9 | y_or_n = input(msg + ' (y/n): ').lower().strip() 10 | if y_or_n == 'y': 11 | return True 12 | elif y_or_n == 'n': 13 | return False 14 | else: 15 | print("Invalid input! Please input 'y' or 'n'.") 16 | 17 | def input_with_options(msg: str, options: list): 18 | while True: 19 | options_str = ', '.join(str(x) for x in options) 20 | response = input(f'{msg} ({options_str}): ').lower().strip() 21 | if any(option in response for option in options): 22 | return response 23 | else: 24 | print(f'Invalid input! Please input one of the following: {options_str}') 25 | 26 | def clear(): 27 | os.system('cls' if os.name == 'nt' else 'clear') 28 | 29 | def detect_os(): 30 | return platform.system() 31 | 32 | def get_cuda_version(): 33 | try: 34 | result = subprocess.run(['nvcc', '--version'], capture_output=True, text=True, check=True) 35 | version_info = result.stdout 36 | for line in version_info.split('\n'): 37 | if 'release' in line: 38 | parts = line.split() 39 | for i, part in enumerate(parts): 40 | if part == 'release': 41 | return True, parts[i + 1].rstrip(',') 42 | except subprocess.CalledProcessError: 43 | return False, 'CUDA is not installed or nvcc is not found in PATH.' 44 | except FileNotFoundError: 45 | return False, 'nvcc command not found. Make sure CUDA is installed and nvcc is in PATH.' 46 | 47 | def check_bld_tools(): 48 | import winreg 49 | try: 50 | registry_key_path = r'SOFTWARE\Microsoft\VisualStudio\SxS\VS7' 51 | registry_value_name = '17.0' 52 | 53 | with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, registry_key_path, 0, winreg.KEY_READ) as key: 54 | try: 55 | install_path, regtype = winreg.QueryValueEx(key, registry_value_name) 56 | if install_path: 57 | return True, f'Visual Studio Build Tools 2022 is installed at: {install_path}' 58 | except FileNotFoundError: 59 | return False, 'Visual Studio Build Tools 2022 is not installed.' 60 | except FileNotFoundError: 61 | return False, 'Visual Studio registry key not found.' 62 | 63 | def prereq_check(): 64 | cuda = False 65 | bld_tools = False 66 | 67 | cuda, info = get_cuda_version() 68 | if not cuda: 69 | print(info) 70 | print('WARNING: Could not detect CUDA version, CUDA is required for NVIDIA GPU based acceleration. If you do not have it and need it, you can install it here: https://developer.nvidia.com/cuda-toolkit') 71 | else: 72 | print(f'Detected CUDA version: {info}') 73 | 74 | if detect_os() == 'Windows': 75 | bld_tools, info = check_bld_tools() 76 | if not bld_tools: 77 | print(info) 78 | print('WARNING: Visual Studio Build Tools not found, it is required for llama-cpp-python (if you opted for gguf/ggml). You can download it here: https://visualstudio.microsoft.com/visual-cpp-build-tools') 79 | elif detect_os() == 'Linux': 80 | bld_tools = True 81 | 82 | return cuda, bld_tools 83 | 84 | def install_base_dependencies(os, backend): 85 | dependancies = {'sentence-transformers', 'openai-whisper', 'aiohttp_cors', 'aiohttp', 'transformers'} 86 | already_installed = {pkg.key for pkg in pkg_resources.working_set} 87 | 88 | print(f'Attempting to install torch/torchvision/torchaudio for {backend}...') 89 | if backend == 'cuda': 90 | cuda, info = get_cuda_version() 91 | if cuda: 92 | if float(info) >= 12.1: 93 | for i in ('torch', 'torchvision', 'torchaudio'): 94 | dependancies.add(i) 95 | print('CUDA version is 12.1 (or higher), torch/torchvision/torchaudio will be installed normally.') 96 | elif float(info) == 11.8: 97 | torch_pkgs = {'torch', 'torchvision', 'torchaudio'} - already_installed 98 | if torch_pkgs: 99 | print('CUDA version is 11.8, installing torch/torchvision/torchaudio for CUDA 11.8...') 100 | subprocess.check_call([sys.executable, '-m', 'pip', 'install', *torch_pkgs, '--index-url', 'https://download.pytorch.org/whl/cu118']) 101 | elif float(info) < 11.8: 102 | print('CUDA version is below 11.8, please upgrade to at least 11.8. Unable to install torch/torchvision/torchaudio.') 103 | exit(1) 104 | else: 105 | print('CUDA not found (or it could not be detected), please install CUDA Toolkit from https://developer.nvidia.com/cuda-toolkit to use this backend or use a different backend.') 106 | exit(1) 107 | elif backend == 'openblas': 108 | print('Using CPU backend, installing torch/torchvision/torchaudio for CPU...') 109 | torch_pkgs = {'torch', 'torchvision', 'torchaudio'} - already_installed 110 | if torch_pkgs: 111 | subprocess.check_call([sys.executable, '-m', 'pip', 'install', *torch_pkgs, '--index-url', 'https://download.pytorch.org/whl/cpu']) 112 | else: 113 | print('Torch/torchvision/torchaudio already installed.') 114 | 115 | print('Checking and installing other dependencies...') 116 | 117 | missing = dependancies - already_installed 118 | for package in (dependancies - missing): 119 | print(f'Already installed: {package}') 120 | print(f'Installing base dependencies for {os}...') 121 | subprocess.check_call([sys.executable, '-m', 'pip', 'install', *missing]) 122 | 123 | def install_llm_backend(type, hardware): 124 | if type in ['all', 'gguf']: 125 | if hardware == 'openblas': 126 | cmake_args = '-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS' 127 | elif hardware == 'cuda': 128 | cmake_args = '-DLLAMA_CUDA=on' 129 | 130 | os_name = detect_os() 131 | if os_name == 'Windows': 132 | os.environ['CMAKE_ARGS'] = cmake_args 133 | print(f'Detected Windows, installing llama-cpp-python for Windows with cmake args: {cmake_args}...') 134 | subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'llama-cpp-python']) 135 | elif os_name in ['Linux', 'Darwin']: 136 | print(f'Detected {os_name}, installing llama-cpp-python for {os_name} with cmake args: {cmake_args}...') 137 | subprocess.check_call(['pip', 'install', 'llama-cpp-python'], env={'CMAKE_ARGS': cmake_args}) 138 | 139 | if type in ['all', 'safetensors']: 140 | print('Installing safetensors to load .safetensors...') 141 | subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'safetensors']) 142 | 143 | if type in ['all', 'pkl']: 144 | print('Pickle selected, no additional installation required (.pkl models are not recommended, use .safetensors).') 145 | 146 | print('LLM backend installed.') 147 | 148 | def install_vision_backend(type, hardware): 149 | print('Installing vision backend...') 150 | 151 | if type in ['all', 'llava']: 152 | print('Installing LLaVA requirements...') 153 | if input_y_n('LLaVA support is provided by llama-cpp-python, do you want to proceed?'): 154 | install_llm_backend('gguf', hardware) 155 | else: 156 | print('llama-cpp-python will not be installed, LLaVA will not be supported.') 157 | 158 | if type in ['all', 'clip']: 159 | print('Instlling CLIP and its requirements...') 160 | reqs = {'git+https://github.com/openai/CLIP.git', 'ftfy', 'regex', 'tqdm'} 161 | subprocess.check_call([sys.executable, '-m', 'pip', 'install', *reqs]) 162 | 163 | print('Vision backend installed.') 164 | 165 | def config_builder(): 166 | print('----- Building config -----') 167 | print('For default values, leave input blank and press enter, ensure input value types are right, config is not protected .') 168 | lora_path = None 169 | clip_path = None 170 | llava_clip_path = None 171 | tts_temp = 0.6 172 | tts_model_path = None 173 | verbose = input_y_n("Do you want Ame's controller to be verbose? (default: y)") or True 174 | log = input_y_n('Do you want Ame to log to a file? (default: y)') or True 175 | assistant_name = str(input("What do you want your assistant's name to be? (default: Ame): ")) or 'Ame' 176 | memory_enabled = input_y_n('Do you want to enable long term memory via hyperdb? (default: y)') or True 177 | memory_path = str(input('Where do you want to store your memory? (default: auto): ')) or None 178 | language_model_path = str(input('Where do you want to store your language model? (default: auto): ')) or None 179 | max_tokens = int(input('What is the maximum number of tokens you want to generate each run? (default: 512): ')) or 512 180 | temperature = float(input('What temperature do you want to use for generation? (default: 0.85): ')) or 0.85 181 | context_size = int(input('What is the context length of the model? (default: auto): ')) or 0 182 | top_p = float(input('What top_p value do you want to use for generation? (default: 0.95): ')) or 0.95 183 | top_k = int(input('What top_k value do you want to use for generation? (default: 40): ')) or 40 184 | gpu = input_y_n('Do you want to use a GPU for generation? (only if you have CUDA)') 185 | gpu_layers = int(input('How many layers do you want to use on the GPU? (default: auto): ')) or -1 186 | threads = int(input('How many threads do you want to use for generation? (default: auto): ')) or None 187 | virtual_context_limit = int(input('What is the virtual context limit? [to prevent out of vram errors] (default: 2048): ')) or 2048 188 | personality_prompt = str(input('What is the personality prompt? (default: None): ')) or None 189 | model_file_ext = str(input('What is the model file extension? [this is important, it determines what backend is used] (default: .gguf): ')) or '.gguf' 190 | model_format = str(input('What is the model format? (default: chatml): ')) or 'chatml' 191 | lora = input_y_n('Do you want to use a LoRA? (default: n)') or False 192 | if lora: 193 | lora_path = str(input('Where is your LoRA located?: ')) or None 194 | vision_enabled = input_y_n('Do you want to enable vision? (default: y)') or True 195 | if vision_enabled: 196 | vision_backend = str(input_with_options('What vision backend do you want to use? [note: llava is a seperate LLM that still requires CLIP, entering clip here enables standalone clip that works with any LLM] (default: clip)', ['clip', 'llava'])) or 'clip' 197 | if vision_backend == 'clip': 198 | clip_path = str(input('Where is your CLIP model located?: ')) or None 199 | elif vision_backend == 'llava': 200 | llava_clip_path = str(input('Where is your CLIP model located?: ')) or None 201 | tts_enabled = input_y_n('Do you want to enable TTS? [powered by bark] (default: y)') or True 202 | if tts_enabled: 203 | tts_temp = float(input('What temperature do you want to use for TTS? (default: 0.6): ')) or 0.6 204 | tts_model_path = str(input('Where is your TTS model located? (default: auto): ')) or None 205 | 206 | 207 | def dependancies_install(): 208 | print('----- Installing dependancies -----') 209 | print(f'Detected operating system: {detect_os()}') 210 | print('Checking potential prerequisites... (Based on your specific requirements, not all prerequisites may be required.)') 211 | cuda, bld_tools = prereq_check() 212 | 213 | backend_hardware = input_with_options('What backend do you want to use (cuda for NVIDIA GPUs, openblas for CPU)?', ['cuda', 'openblas']) 214 | install_base_dependencies(detect_os(), backend_hardware) 215 | model_file_ext = input_with_options(f'What large language model backend do you want to install?', ['gguf', 'safetensors', 'pkl', 'all']) 216 | install_llm_backend(model_file_ext, backend_hardware) 217 | vision_backend = input_y_n('Do you want to install a vision backend?') 218 | if vision_backend: 219 | vision_backend_type = input_with_options('What vision backend do you want to install?', ['clip', 'llava', 'all']) 220 | install_vision_backend(vision_backend_type, backend_hardware) 221 | print('All dependancies installed.') 222 | 223 | def menu(): 224 | print('----- Ame configuration tool -----') 225 | print('1. Install dependancies and setup configuration') 226 | print('2. Install dependancies only') 227 | print('3. Setup configuration only') 228 | options = input_with_options('What would you like to do?', ['1', '2', '3']) 229 | if options == '1': 230 | dependancies_install() 231 | config_builder() 232 | elif options == '2': 233 | dependancies_install() 234 | elif options == '3': 235 | config_builder() 236 | 237 | if __name__ == '__main__': 238 | menu() 239 | 240 | -------------------------------------------------------------------------------- /vision/vision_handler.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | class vision: 4 | def __init__(self, model) -> None: 5 | self.model = model 6 | pass 7 | 8 | def describe(self, input): 9 | return "Vision has not been implemented yet. Please check back later." 10 | 11 | if __name__ == '__main__': 12 | print('This is a handler, it is not meant to be run directly.') 13 | pass 14 | -------------------------------------------------------------------------------- /voice/stt_handler.py: -------------------------------------------------------------------------------- 1 | import whisper 2 | 3 | class stt: 4 | def __init__(self, model_name=None) -> None: 5 | if model_name == None: 6 | self.load_model() 7 | else: 8 | self.load_model(model_name) 9 | 10 | def load_model(self, model_name="small.en"): 11 | self.model = whisper.load_model(model_name) 12 | 13 | def transcribe(self, audio_path): 14 | print(audio_path) 15 | result = self.model.transcribe(audio_path) 16 | return result["text"] 17 | 18 | if __name__ == '__main__': 19 | print('This is a handler, it is not meant to be run directly.') 20 | pass -------------------------------------------------------------------------------- /voice/tts_handler.py: -------------------------------------------------------------------------------- 1 | import nltk 2 | import numpy as np 3 | import os 4 | from bark.generation import generate_text_semantic, preload_models 5 | from bark.api import semantic_to_waveform 6 | from bark import SAMPLE_RATE 7 | from scipy.io.wavfile import write as write_wav 8 | 9 | # Almost all of this is stolen from the Bark repo. 10 | 11 | class tts: 12 | def __init__(self, model_name='v2/en_speaker_9', temperature=0.6) -> None: 13 | self.model = model_name 14 | self.temp = temperature 15 | preload_models() 16 | 17 | def generate(self, prompt): 18 | GEN_TEMP = self.temp 19 | SPEAKER = self.model 20 | sentences = nltk.sent_tokenize(prompt) 21 | silence = np.zeros(int(0.25 * SAMPLE_RATE)) 22 | 23 | pieces = [] 24 | for sentence in sentences: 25 | semantic_tokens = generate_text_semantic( 26 | sentence, 27 | history_prompt=SPEAKER, 28 | temp=GEN_TEMP, 29 | min_eos_p=0.05, 30 | ) 31 | 32 | audio_array = semantic_to_waveform(semantic_tokens, history_prompt=SPEAKER,) 33 | pieces += [audio_array, silence.copy()] 34 | 35 | audio_array = np.concatenate(pieces) 36 | write_wav('ame_speech.wav', SAMPLE_RATE, audio_array) 37 | return os.path.abspath('ame_speech.wav') 38 | 39 | if __name__ == '__main__': 40 | print('This is a handler, it is not meant to be run directly.') 41 | pass 42 | --------------------------------------------------------------------------------