├── .env.example ├── .github └── CODEOWNERS ├── LICENSE ├── README.md ├── config.py ├── endpoint.py ├── functions.py ├── pyproject.toml ├── requirements.txt ├── setup_pixeltable.py ├── static ├── css │ └── style.css ├── image │ ├── overview.gif │ ├── pixelbot.png │ └── pixeltable-logo.png ├── js │ ├── api.js │ └── ui.js ├── manifest.json ├── robots.txt └── sitemap.xml └── templates └── index.html /.env.example: -------------------------------------------------------------------------------- 1 | # Required for Core LLM Functionality * 2 | ANTHROPIC_API_KEY=sk-ant-api03-... # For main reasoning/tool use (Claude 3.5 Sonnet) 3 | OPENAI_API_KEY=sk-... # For audio transcription (Whisper) & image generation (DALL-E 3) 4 | MISTRAL_API_KEY=... # For follow-up question suggestions (Mistral Small) 5 | 6 | # Optional (Enable specific tools by providing keys) 7 | NEWS_API_KEY=... # Enables the NewsAPI tool 8 | # Note: yfinance and DuckDuckGo Search tools do not require API keys. 9 | 10 | # --- !!**Authentication Mode (required to run locally)**!! --- 11 | # Set to 'local' to bypass the WorkOS authentication used at agent.pixeltable.com and to leverage a default user. 12 | # Leaving unset will result in errors 13 | AUTH_MODE=local 14 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Global owners 2 | * @mkornacker @aaron-siegel @pierrebrunelle @jacobweiss2305 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | Pixelbot 4 | 5 | [![License](https://img.shields.io/badge/License-Apache%202.0-0530AD.svg)](https://opensource.org/licenses/Apache-2.0) [![My Discord (1306431018890166272)](https://img.shields.io/badge/💬-Discord-%235865F2.svg)](https://discord.gg/QPyqFYx2UN) 6 |
7 |
8 | 9 | [Pixelbot](http://agent.pixeltable.com/), a multimodal context-aware AI agent built using [Pixeltable](https://github.com/pixeltable/pixeltable) — open-source AI data infrastructure. The agent can process and reason about various data types (documents, images, videos, audio), use external tools, search its knowledge base derived from uploaded files, generate images, maintain a chat history, and leverage a selective memory bank. 10 | 11 | ![Overview](/static/image/overview.gif) 12 | 13 | The endpoint is built with Flask (Python) and the frontend with vanilla JS. This open source code replicates entirely what you can find at https://agent.pixeltable.com/ that is hosted on AWS EC2 instances. 14 | 15 | ## 🚀 How Pixeltable Powers This App 16 | 17 | Pixeltable acts as AI Data Infrastructure, simplifying the development of this complex, infinite-memory multimodal agent: 18 | 19 | - 📜 **Declarative Workflows**: The entire agent logic—from data ingestion and processing to LLM calls and tool execution—is defined declaratively using Pixeltable **tables**, **views**, and **computed columns** (`setup_pixeltable.py`). Pixeltable automatically manages dependencies and execution order. 20 | - 🔀 **Unified Data Handling**: Natively handles diverse data types (documents, images, videos, audio) within its tables, eliminating the need for separate storage solutions. 21 | - ⚙️ **Automated Processing**: **Computed columns** automatically trigger functions (like thumbnail generation, audio extraction, transcription via Whisper, image generation via DALL-E) when new data arrives or dependencies change. 22 | - ✨ **Efficient Transformations**: **Views** and **Iterators** (like `DocumentSplitter`, `FrameIterator`, `AudioSplitter`) process data on-the-fly (e.g., chunking documents, extracting video frames) without duplicating the underlying data. 23 | - 🔎 **Integrated Search**: **Embedding indexes** are easily added to tables/views, enabling powerful semantic search across text, images, and frames with simple syntax (`.similarity()`). 24 | - 🔌 **Seamless Tool Integration**: Any Python function (`@pxt.udf`) or Pixeltable query function (`@pxt.query`) can be registered as a tool for the LLM using `pxt.tools()`. Pixeltable handles the invocation (`pxt.invoke_tools()`) based on the LLM's decision. 25 | - 💾 **State Management**: Persistently stores all relevant application state (uploaded files, chat history, memory, generated images, workflow runs) within its managed tables. 26 | 27 | ```mermaid 28 | flowchart TD 29 | %% User Interaction 30 | User([User]) -->|Query| ToolsTable[agents.tools] 31 | User -->|Selective Memory| MemoryBankTable[agents.memory_bank] 32 | User -->|Upload Files| SourceTables["agents.collection, agents.images, agents.videos, agents.audios"] 33 | User -->|Generate Image| ImageGenTable[agents.image_generation_tasks] 34 | 35 | %% Main Agent Workflow 36 | ToolsTable -->|Prompt| DocSearch[Search Documents] 37 | ToolsTable -->|Prompt| ImageSearch[Search Images] 38 | ToolsTable -->|Prompt| VideoFrameSearch[Search Video Frames] 39 | 40 | ToolsTable -->|Prompt, Tools| InitialLLM[Claude 3.5 - Tools] 41 | AvailableTools["**Available Tools**: 42 | get_latest_news 43 | fetch_financial_data 44 | search_news 45 | search_video_transcripts 46 | search_audio_transcripts"] -.-> InitialLLM 47 | InitialLLM -->|Tool Choice| ToolExecution[pxt.invoke_tools] 48 | ToolExecution --> ToolOutput[Tool Output] 49 | 50 | %% Context Assembly 51 | DocSearch -->|Context| AssembleTextContext[Assemble Text Context] 52 | ImageSearch -->|Context| AssembleFinalMessages[Assemble Final Messages] 53 | VideoFrameSearch -->|Context| AssembleFinalMessages 54 | 55 | ToolOutput -->|Context| AssembleTextContext 56 | AssembleTextContext -->|Text Summary| AssembleFinalMessages 57 | ToolsTable -->|Recent History| AssembleFinalMessages 58 | MemIndex -->|Context| AssembleTextContext 59 | ChatHistIndex -->|Context| AssembleTextContext 60 | 61 | %% Final LLM Call & Output 62 | AssembleFinalMessages -->|Messages| FinalLLM[Claude 3.5 - Answer] 63 | FinalLLM -->|Answer| ExtractAnswer[Extract Answer] 64 | ExtractAnswer -->|Answer| User 65 | ExtractAnswer -->|Answer| LogChat[agents.chat_history] 66 | ToolsTable -->|User Prompt| LogChat 67 | 68 | %% Follow-up Generation 69 | FinalLLM -->|Answer| FollowUpLLM[Mistral Small - Follow-up] 70 | FollowUpLLM -->|Suggestions| User 71 | 72 | %% Image Generation Workflow 73 | ImageGenTable -->|Prompt| OpenAI_Dalle[DALL-E 3] 74 | OpenAI_Dalle -->|Image Data| ImageGenTable 75 | ImageGenTable -->|Retrieve Image| User 76 | 77 | %% Supporting Structures 78 | SourceTables --> Views[**Materialized Views** 79 | Chunks, Frames, Sentences] 80 | Views --> Indexes[Embedding Indexes 81 | E5, CLIP] 82 | MemoryBankTable --> MemIndex[Search Memory] 83 | LogChat --> ChatHistIndex[Search Conversations] 84 | 85 | %% Styling 86 | classDef table fill:#E1C1E9,stroke:#333,stroke-width:1px 87 | classDef view fill:#C5CAE9,stroke:#333,stroke-width:1px 88 | classDef llm fill:#FFF9C4,stroke:#333,stroke-width:1px 89 | classDef workflow fill:#E1F5FE,stroke:#333,stroke-width:1px 90 | classDef search fill:#C8E6C9,stroke:#333,stroke-width:1px 91 | classDef tool fill:#FFCCBC,stroke:#333,stroke-width:1px 92 | classDef io fill:#fff,stroke:#000,stroke-width:2px 93 | 94 | class User io 95 | class ToolsTable,,SourceTables,ImageGenTable,LogChat,MemoryBankTable table 96 | class Views view 97 | class Indexes,MemIndex,ChatHistIndex search 98 | class InitialLLM,FinalLLM,FollowUpLLM,OpenAI_Dalle llm 99 | class DocSearch,ImageSearch,VideoFrameSearch,MemorySearch,ChatHistorySearch search 100 | class ToolExecution,AvailableTools,ToolOutput tool 101 | class AssembleTextContext,AssembleFinalMessages,ExtractAnswer workflow 102 | ``` 103 | 104 | ## 📁 Project Structure 105 | 106 | ``` 107 | . 108 | ├── .env # Environment variables (API keys, AUTH_MODE) 109 | ├── .venv/ # Virtual environment files (if created here) 110 | ├── data/ # Default directory for uploaded/source media files 111 | ├── logs/ # Application logs 112 | │ └── app.log 113 | ├── static/ # Static assets for Flask frontend (CSS, JS, Images) 114 | │ ├── css/style.css 115 | │ ├── image/*.png 116 | │ ├── js/ 117 | │ │ ├── api.js 118 | │ │ └── ui.js 119 | │ └── manifest.json 120 | │ └── robots.txt 121 | │ └── sitemap.xml 122 | ├── templates/ # HTML templates for Flask frontend 123 | │ └── index.html 124 | ├── endpoint.py # Flask backend: API endpoints and UI rendering 125 | ├── functions.py # Python UDFs and context assembly logic 126 | ├── config.py # Central configuration (model IDs, defaults, personas) 127 | ├── requirements.txt # Python dependencies 128 | └── setup_pixeltable.py # Pixeltable schema definition script 129 | ``` 130 | 131 | ## 📊 Pixeltable Schema Overview 132 | 133 | Pixeltable organizes data in directories, tables, and views. This application uses the following structure within the `agents` directory: 134 | 135 | ``` 136 | agents/ 137 | ├── collection # Table: Source documents (PDF, TXT, etc.) 138 | │ ├── document: pxt.Document 139 | │ ├── uuid: pxt.String 140 | │ └── timestamp: pxt.Timestamp 141 | ├── images # Table: Source images 142 | │ ├── image: pxt.Image 143 | │ ├── uuid: pxt.String 144 | │ ├── timestamp: pxt.Timestamp 145 | │ └── thumbnail: pxt.String(computed) # Base64 sidebar thumbnail 146 | ├── videos # Table: Source videos 147 | │ ├── video: pxt.Video 148 | │ ├── uuid: pxt.String 149 | │ ├── timestamp: pxt.Timestamp 150 | │ └── audio: pxt.Audio(computed) # Extracted audio (used by audio_chunks view) 151 | ├── audios # Table: Source audio files (MP3, WAV) 152 | │ ├── audio: pxt.Audio 153 | │ ├── uuid: pxt.String 154 | │ └── timestamp: pxt.Timestamp 155 | ├── chat_history # Table: Stores conversation turns 156 | │ ├── role: pxt.String # 'user' or 'assistant' 157 | │ ├── content: pxt.String 158 | │ └── timestamp: pxt.Timestamp 159 | ├── memory_bank # Table: Saved text/code snippets 160 | │ ├── content: pxt.String 161 | │ ├── type: pxt.String # 'code' or 'text' 162 | │ ├── language: pxt.String # e.g., 'python' 163 | │ ├── context_query: pxt.String # Original query or note 164 | │ └── timestamp: pxt.Timestamp 165 | ├── image_generation_tasks # Table: Image generation requests & results 166 | │ ├── prompt: pxt.String 167 | │ ├── timestamp: pxt.Timestamp 168 | │ └── generated_image: pxt.Image(computed) # DALL-E 3 output 169 | ├── user_personas # Table: User-defined personas 170 | │ ├── persona_name: pxt.String 171 | │ ├── initial_prompt: pxt.String 172 | │ ├── final_prompt: pxt.String 173 | │ ├── llm_params: pxt.Json 174 | │ └── timestamp: pxt.Timestamp 175 | ├── tools # Table: Main agent workflow orchestration 176 | │ ├── prompt: pxt.String 177 | │ ├── timestamp: pxt.Timestamp 178 | │ ├── user_id: pxt.String 179 | │ ├── initial_system_prompt: pxt.String 180 | │ ├── final_system_prompt: pxt.String 181 | │ ├── max_tokens, stop_sequences, temperature, top_k, top_p # LLM Params 182 | │ ├── initial_response: pxt.Json(computed) # Claude tool choice output 183 | │ ├── tool_output: pxt.Json(computed) # Output from executed tools (UDFs or Queries) 184 | │ ├── doc_context: pxt.Json(computed) # Results from document search 185 | │ ├── image_context: pxt.Json(computed) # Results from image search 186 | │ ├── video_frame_context: pxt.Json(computed) # Results from video frame search 187 | │ ├── memory_context: pxt.Json(computed) # Results from memory bank search 188 | │ ├── chat_memory_context: pxt.Json(computed) # Results from chat history search 189 | │ ├── history_context: pxt.Json(computed) # Recent chat turns 190 | │ ├── multimodal_context_summary: pxt.String(computed) # Assembled text context for final LLM 191 | │ ├── final_prompt_messages: pxt.Json(computed) # Fully assembled messages (incl. images/frames) for final LLM 192 | │ ├── final_response: pxt.Json(computed) # Claude final answer generation output 193 | │ ├── answer: pxt.String(computed) # Extracted text answer 194 | │ ├── follow_up_input_message: pxt.String(computed) # Formatted prompt for Mistral 195 | │ ├── follow_up_raw_response: pxt.Json(computed) # Raw Mistral response 196 | │ └── follow_up_text: pxt.String(computed) # Extracted follow-up suggestions 197 | ├── chunks # View: Document chunks via DocumentSplitter 198 | │ └── (Implicit: EmbeddingIndex: E5-large-instruct on text) 199 | ├── video_frames # View: Video frames via FrameIterator (1 FPS) 200 | │ └── (Implicit: EmbeddingIndex: CLIP on frame) 201 | ├── video_audio_chunks # View: Audio chunks from video table via AudioSplitter 202 | │ └── transcription: pxt.Json(computed) # Whisper transcription 203 | ├── video_transcript_sentences # View: Sentences from video transcripts via StringSplitter 204 | │ └── (Implicit: EmbeddingIndex: E5-large-instruct on text) 205 | ├── audio_chunks # View: Audio chunks from audio table via AudioSplitter 206 | │ └── transcription: pxt.Json(computed) # Whisper transcription 207 | └── audio_transcript_sentences # View: Sentences from direct audio transcripts via StringSplitter 208 | └── (Implicit: EmbeddingIndex: E5-large-instruct on text) 209 | 210 | # Available Tools (Registered via pxt.tools()): 211 | # - functions.get_latest_news (UDF) 212 | # - functions.fetch_financial_data (UDF) 213 | # - functions.search_news (UDF) 214 | # - search_video_transcripts (@pxt.query function) 215 | # - search_audio_transcripts (@pxt.query function) 216 | 217 | # Embedding Indexes Enabled On: 218 | # - agents.chunks.text 219 | # - agents.images.image 220 | # - agents.video_frames.frame 221 | # - agents.video_transcript_sentences.text 222 | # - agents.audio_transcript_sentences.text 223 | # - agents.memory_bank.content 224 | # - agents.chat_history.content 225 | ``` 226 | 227 | ## ▶️ Getting Started 228 | 229 | ### Prerequisites 230 | 231 | You are welcome to swap any of the below calls, e.g. [WhisperX](https://docs.pixeltable.com/docs/examples/search/audio) instead of OpenAI Whisper, [Llama.cpp](https://docs.pixeltable.com/docs/integrations/frameworks#local-llm-runtimes) instead of Mistral... either through our built-in modules or by bringing your own models, frameworks, and API calls. See our [integration](https://docs.pixeltable.com/docs/integrations/frameworks) and [UDFs](https://docs.pixeltable.com/docs/datastore/custom-functions) pages to learn more. You can easily make this applicaiton entirely local if you decide to rely on local LLM runtimes and local embedding/transcription solutions. 232 | 233 | - Python 3.9+ 234 | - API Keys: 235 | - [Anthropic](https://console.anthropic.com/) 236 | - [OpenAI](https://platform.openai.com/api-keys) 237 | - [Mistral AI](https://console.mistral.ai/api-keys/) 238 | - [NewsAPI](https://newsapi.org/) (100 requests per day free) 239 | 240 | ### Installation 241 | 242 | ```bash 243 | # 1. Create and activate a virtual environment (recommended) 244 | python -m venv .venv 245 | # Windows: .venv\Scripts\activate 246 | # macOS/Linux: source .venv/bin/activate 247 | 248 | # 2. Install dependencies 249 | pip install -r requirements.txt 250 | ``` 251 | 252 | ### Environment Setup 253 | 254 | Create a `.env` file in the project root and add your API keys. Keys marked with `*` are required for core LLM functionality. 255 | 256 | ```dotenv 257 | # Required for Core LLM Functionality * 258 | ANTHROPIC_API_KEY=sk-ant-api03-... # For main reasoning/tool use (Claude 3.5 Sonnet) 259 | OPENAI_API_KEY=sk-... # For audio transcription (Whisper) & image generation (DALL-E 3) 260 | MISTRAL_API_KEY=... # For follow-up question suggestions (Mistral Small) 261 | 262 | # Optional (Enable specific tools by providing keys) 263 | NEWS_API_KEY=... # Enables the NewsAPI tool 264 | # Note: yfinance and DuckDuckGo Search tools do not require API keys. 265 | 266 | # --- !!**Authentication Mode (required to run locally)**!! --- 267 | # Set to 'local' to bypass the WorkOS authentication used at agent.pixeltable.com and to leverage a default user. 268 | # Leaving unset will result in errors 269 | AUTH_MODE=local 270 | ``` 271 | 272 | ### Running the Application 273 | 274 | 1. **Initialize Pixeltable Schema:** 275 | This script creates the necessary Pixeltable directories, tables, views, and computed columns defined in `setup_pixeltable.py`. Run this *once* initially. 276 | 277 | *Why run this?* This defines the data structures and the declarative AI workflow within Pixeltable. It tells Pixeltable how to store, transform, index, and process your data automatically. 278 | 279 | ```bash 280 | python setup_pixeltable.py 281 | ``` 282 | 283 | 2. **Start the Web Server:** 284 | This runs the Flask application using the Waitress production server by default. 285 | 286 | ```bash 287 | python endpoint.py 288 | ``` 289 | 290 | The application will be available at `http://localhost:5000`. 291 | 292 | **Data Persistence Note:** Pixeltable stores all its data (file references, tables, views, indexes) locally, typically in a `.pixeltable` directory created within your project workspace. This means your uploaded files, generated images, chat history, and memory bank are persistent across application restarts. 293 | 294 | ## 🖱️ Usage Overview 295 | 296 | The web interface provides several tabs: 297 | 298 | - **Chat Interface**: Main interaction area. Ask questions, switch between chat and image generation modes. View results, including context retrieved (images, video frames) and follow-up suggestions. Save responses to the Memory Bank. 299 | - **Agent Settings**: Configure the system prompts (initial for tool use, final for answer generation) and LLM parameters (temperature, max tokens, etc.) used by Claude. 300 | - **Chat History**: View past queries and responses. Search history and view detailed execution metadata for each query. Download history as JSON. 301 | - **Generated Images**: View images created using the image generation mode. Search by prompt, view details, download, or delete images. 302 | - **Memory Bank**: View, search, manually add, and delete saved text/code snippets. Download memory as JSON. 303 | - **How it Works**: Provides a technical overview of how Pixeltable powers the application's features. 304 | 305 | ## ⭐ Key Features 306 | 307 | - 💾 **Unified Multimodal Data Management**: Ingests, manages, process, and index documents (text, PDFs, markdown), images (JPG, PNG), videos (MP4), and audio files (MP3, WAV) using Pixeltable's specialized [data types](https://docs.pixeltable.com/docs/datastore/bringing-data). 308 | - ⚙️ **Declarative AI Workloads**: Leverages Pixeltable's **[computed columns](https://docs.pixeltable.com/docs/datastore/computed-columns)** and **[views](https://docs.pixeltable.com/docs/datastore/views)** to declaratively define complex conditional workflows including data processing (chunking, frame extraction, audio extraction), embedding generation, AI model inference, and context assembly while maintaining data lineage and versioning. 309 | - 🧠 **Agentic RAG & Tool Use**: The agent dynamically decides which tools to use based on the query. Available tools include: 310 | - **External APIs**: Fetching news (NewsAPI, DuckDuckGo), financial data (yfinance). 311 | - **Internal Knowledge Search**: Pixeltable `@pxt.query` functions are registered as tools, allowing the agent to search video transcripts and audio transcripts on demand, as an example. 312 | - 🔍 **Semantic Search**: Implements [vector search](https://docs.pixeltable.com/docs/datastore/vector-database) across multiple modalities, powered by any **embedding indexes** that Pixeltable incrementally and automatically maintain: 313 | - Document Chunks (`sentence-transformers`) 314 | - Images & Video Frames (`CLIP`) 315 | - Chat History (`sentence-transformers`) 316 | - Memory Bank items (`sentence-transformers`) 317 | - 🔌 **LLM Integration**: Seamlessly [integrates](https://docs.pixeltable.com/docs/integrations/frameworks) multiple LLMs for different tasks within the Pixeltable workflow: 318 | - **Reasoning & Tool Use**: Anthropic Claude 3.5 Sonnet 319 | - **Audio Transcription**: OpenAI Whisper (via computed columns on audio chunks) 320 | - **Image Generation**: OpenAI DALL-E 3 (via computed columns on image prompts) 321 | - **Follow-up Suggestions**: Mistral Small Latest 322 | - 💬 **Chat History**: Persistently stores conversation turns in a Pixeltable [table](https://docs.pixeltable.com/docs/datastore/tables-and-operations) (`agents.chat_history`), enabling retrieval and semantic search over past interactions. 323 | - 📝 **Memory Bank**: Allows saving and semantically searching important text snippets or code blocks stored in a dedicated Pixeltable table (`agents.memory_bank`). 324 | - 🖼️ **Image Generation**: Generates images based on user prompts using DALL-E 3, orchestrated via a Pixeltable table (`agents.image_generation_tasks`). 325 | - 🏠 **Local Mode**: Supports running locally without external authentication ([WorkOS](https://github.com/workos/python-flask-example-applications)) (`AUTH_MODE=local`) for easier setup and development. 326 | - 🖥️ **Responsive UI**: A clean web interface built with Flask, Tailwind CSS, and JavaScript. 327 | - 🛠️ **Centralized Configuration**: Uses an arbitraty `config.py` to manage model IDs, default system prompts, LLM parameters, and persona presets. 328 | 329 | ## ⚠️ Disclaimer 330 | 331 | This application serves as a comprehensive demonstration of Pixeltable's capabilities for managing complex multimodal AI workflows, covering data storage, transformation, indexing, retrieval, and serving. 332 | 333 | The primary focus is on illustrating Pixeltable patterns and best practices within the `setup_pixeltable.py` script and related User-Defined Functions (`functions.py`). 334 | 335 | While functional, less emphasis was placed on optimizing the Flask application (`endpoint.py`) and the associated frontend components (`style.css`, `index.html`, `ui.js`...). These parts should not necessarily be considered exemplars of web development best practices. 336 | 337 | For simpler examples demonstrating Pixeltable integration with various frameworks (FastAPI, React, TypeScript, Gradio, etc.), please refer to the [Pixeltable Examples Documentation](https://docs.pixeltable.com/docs/examples/use-cases). 338 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | # config.py - Centralized configuration for the application 2 | 3 | # --- LLM & Model IDs --- 4 | # Embedding model for text-based semantic search (documents, transcripts, memory, history) 5 | EMBEDDING_MODEL_ID = "intfloat/multilingual-e5-large-instruct" 6 | # Vision-language model for image/frame semantic search 7 | CLIP_MODEL_ID = "openai/clip-vit-base-patch32" 8 | # Audio transcription model 9 | WHISPER_MODEL_ID = "whisper-1" 10 | # Image generation model 11 | DALLE_MODEL_ID = "dall-e-3" 12 | # Main reasoning LLM (tool use, final answer) 13 | CLAUDE_MODEL_ID = "claude-3-5-sonnet-latest" 14 | # Follow-up question LLM 15 | MISTRAL_MODEL_ID = "mistral-small-latest" 16 | 17 | # --- Default System Prompts --- 18 | # Initial prompt for tool selection/analysis 19 | INITIAL_SYSTEM_PROMPT = """Identify the best tool(s) to answer the user's query based on the available data sources (documents, images, news, financial data).""" 20 | # Final prompt for synthesizing the answer 21 | FINAL_SYSTEM_PROMPT = """Based on the provided context and the user's query, provide a very concise answer, ideally just a few words.""" 22 | 23 | # --- Default LLM Parameters --- 24 | # Set to None to use the underlying API defaults 25 | DEFAULT_MAX_TOKENS: int | None = 1024 26 | DEFAULT_STOP_SEQUENCES: list[str] | None = None # e.g., ["\n\nHuman:"] 27 | DEFAULT_TEMPERATURE: float | None = 0.7 28 | DEFAULT_TOP_K: int | None = None 29 | DEFAULT_TOP_P: float | None = None 30 | 31 | # --- Consolidated Parameters Dictionary --- 32 | DEFAULT_PARAMETERS = { 33 | "max_tokens": DEFAULT_MAX_TOKENS, 34 | "stop_sequences": DEFAULT_STOP_SEQUENCES, 35 | "temperature": DEFAULT_TEMPERATURE, 36 | "top_k": DEFAULT_TOP_K, 37 | "top_p": DEFAULT_TOP_P, 38 | } 39 | 40 | # --- Persona Presets Definition --- # 41 | # These will be added to a new user's persona table on first login. 42 | PERSONA_PRESETS = { 43 | "Personal Assistant": { 44 | "initial_prompt": "You are a helpful and friendly personal assistant. Use available tools and context to answer the user's questions clearly and concisely.", 45 | "final_prompt": "Provide a clear, friendly, and concise answer based on the gathered information and the user's query.", 46 | "llm_params": { 47 | "max_tokens": 1500, 48 | "stop_sequences": None, 49 | "temperature": 0.7, 50 | "top_k": None, 51 | "top_p": None, 52 | } 53 | }, 54 | "Visual Analyst": { 55 | "initial_prompt": "You are an expert visual analyst. Prioritize information extracted from images and video frames to answer the user's query. Describe visual elements and patterns accurately.", 56 | "final_prompt": "Based primarily on the provided visual context (images, video frames), generate a detailed analysis answering the user's query. Mention specific visual details observed.", 57 | "llm_params": { 58 | "max_tokens": 2000, 59 | "stop_sequences": None, 60 | "temperature": 0.4, 61 | "top_k": None, 62 | "top_p": None, 63 | } 64 | }, 65 | "Research Assistant": { 66 | "initial_prompt": "You are a meticulous research assistant. Synthesize information from various sources (documents, news, financial data, web searches) to construct a comprehensive answer. Identify key findings and cite sources where applicable.", 67 | "final_prompt": "Compile the research findings from the provided context into a well-structured and informative summary that directly addresses the user's query. Highlight key data points or conclusions.", 68 | "llm_params": { 69 | "max_tokens": 2500, 70 | "stop_sequences": None, 71 | "temperature": 0.6, 72 | "top_k": None, 73 | "top_p": None, 74 | } 75 | }, 76 | "Technical Guide": { 77 | "initial_prompt": "You are a technical guide. Focus on providing accurate technical details, explanations, or code examples based on the user's query and available context.", 78 | "final_prompt": "Generate a technically accurate and precise response, potentially including code snippets or step-by-step instructions, based on the user's query and the gathered information.", 79 | "llm_params": { 80 | "max_tokens": 2000, 81 | "stop_sequences": None, 82 | "temperature": 0.3, 83 | "top_k": None, 84 | "top_p": None, 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /functions.py: -------------------------------------------------------------------------------- 1 | # functions.py - User-Defined Functions (UDFs) for the Pixeltable Agent 2 | # --------------------------------------------------------------------------- 3 | # This file contains Python functions decorated with `@pxt.udf`. 4 | # These UDFs define custom logic (e.g., API calls, data processing) 5 | # that can be seamlessly integrated into Pixeltable workflows. 6 | # Pixeltable automatically calls these functions when computing columns 7 | # in tables or views (as defined in setup_pixeltable.py). 8 | # --------------------------------------------------------------------------- 9 | 10 | # Standard library imports 11 | import os 12 | import traceback 13 | from datetime import datetime 14 | from typing import Optional, Dict, Any, List, Union 15 | 16 | # Third-party library imports 17 | import requests 18 | import yfinance as yf 19 | from duckduckgo_search import DDGS 20 | 21 | # Pixeltable library 22 | import pixeltable as pxt 23 | 24 | # Pixeltable UDFs (User-Defined Functions) extend the platform's capabilities. 25 | # They allow you to wrap *any* Python code and use it within Pixeltable's 26 | # declarative data processing and workflow engine. 27 | # Pixeltable handles scheduling, execution, caching, and error handling. 28 | 29 | 30 | # Tool UDF: Fetches latest news using NewsAPI. 31 | # Registered as a tool for the LLM via `pxt.tools()` in setup_pixeltable.py. 32 | @pxt.udf 33 | def get_latest_news(topic: str) -> str: 34 | """Fetch latest news for a given topic using NewsAPI.""" 35 | try: 36 | api_key = os.environ.get("NEWS_API_KEY") 37 | if not api_key: 38 | return "Error: NewsAPI key not found in environment variables." 39 | 40 | url = "https://newsapi.org/v2/everything" 41 | params = { 42 | "q": topic, 43 | "apiKey": api_key, 44 | "sortBy": "publishedAt", 45 | "language": "en", 46 | "pageSize": 5, 47 | } 48 | 49 | response = requests.get(url, params=params, timeout=10) 50 | 51 | if response.status_code != 200: 52 | return f"Error: NewsAPI request failed ({response.status_code}): {response.text}" 53 | 54 | data = response.json() 55 | articles = data.get("articles", []) 56 | 57 | if not articles: 58 | return f"No recent news found for '{topic}'." 59 | 60 | # Format multiple articles 61 | formatted_news = [] 62 | for i, article in enumerate(articles[:3], 1): 63 | pub_date = datetime.fromisoformat( 64 | article["publishedAt"].replace("Z", "+00:00") 65 | ).strftime("%Y-%m-%d") 66 | formatted_news.append( 67 | f"{i}. [{pub_date}] {article['title']}\n {article['description']}" 68 | ) 69 | 70 | return "\n\n".join(formatted_news) 71 | 72 | except requests.Timeout: 73 | return "Error: NewsAPI request timed out." 74 | except requests.RequestException as e: 75 | return f"Error making NewsAPI request: {str(e)}" 76 | except Exception as e: 77 | return f"Unexpected error fetching news: {str(e)}." 78 | 79 | 80 | # Tool UDF: Searches news using DuckDuckGo. 81 | # Registered as a tool for the LLM via `pxt.tools()` in setup_pixeltable.py. 82 | @pxt.udf 83 | def search_news(keywords: str, max_results: int = 5) -> str: 84 | """Search news using DuckDuckGo and return results.""" 85 | try: 86 | # DDGS requires entering the context manager explicitly 87 | with DDGS() as ddgs: 88 | results = list( 89 | ddgs.news( # Convert iterator to list for processing 90 | keywords=keywords, 91 | region="wt-wt", 92 | safesearch="off", 93 | timelimit="m", # Limit search to the last month 94 | max_results=max_results, 95 | ) 96 | ) 97 | if not results: 98 | return "No news results found." 99 | 100 | # Format results for readability 101 | formatted_results = [] 102 | for i, r in enumerate(results, 1): 103 | formatted_results.append( 104 | f"{i}. Title: {r.get('title', 'N/A')}\n" 105 | f" Source: {r.get('source', 'N/A')}\n" 106 | f" Published: {r.get('date', 'N/A')}\n" 107 | f" URL: {r.get('url', 'N/A')}\n" 108 | f" Snippet: {r.get('body', 'N/A')}\n" 109 | ) 110 | return "\n".join(formatted_results) 111 | except Exception as e: 112 | print(f"DuckDuckGo search failed: {str(e)}") 113 | return f"Search failed: {str(e)}." 114 | 115 | 116 | # Tool UDF: Fetches financial data using yfinance. 117 | # Integrates external Python libraries into the Pixeltable workflow. 118 | # Registered as a tool for the LLM via `pxt.tools()` in setup_pixeltable.py. 119 | @pxt.udf 120 | def fetch_financial_data(ticker: str) -> str: 121 | """Fetch financial summary data for a given company ticker using yfinance.""" 122 | try: 123 | if not ticker: 124 | return "Error: No ticker symbol provided." 125 | 126 | stock = yf.Ticker(ticker) 127 | 128 | # Get the info dictionary - this is the primary source now 129 | info = stock.info 130 | if ( 131 | not info or info.get("quoteType") == "MUTUALFUND" 132 | ): # Basic check if info exists and isn't a mutual fund (less relevant fields) 133 | # Attempt history for basic validation if info is sparse 134 | hist = stock.history(period="1d") 135 | if hist.empty: 136 | return f"Error: No data found for ticker '{ticker}'. It might be delisted or incorrect." 137 | else: # Sometimes info is missing but history works, provide minimal info 138 | return f"Limited info for '{ticker}'. Previous Close: {hist['Close'].iloc[-1]:.2f} (if available)." 139 | 140 | # Select and format key fields from the info dictionary 141 | data_points = { 142 | "Company Name": info.get("shortName") or info.get("longName"), 143 | "Symbol": info.get("symbol"), 144 | "Exchange": info.get("exchange"), 145 | "Quote Type": info.get("quoteType"), 146 | "Currency": info.get("currency"), 147 | "Current Price": info.get("currentPrice") 148 | or info.get("regularMarketPrice") 149 | or info.get("bid"), 150 | "Previous Close": info.get("previousClose"), 151 | "Open": info.get("open"), 152 | "Day Low": info.get("dayLow"), 153 | "Day High": info.get("dayHigh"), 154 | "Volume": info.get("volume") or info.get("regularMarketVolume"), 155 | "Market Cap": info.get("marketCap"), 156 | "Trailing P/E": info.get("trailingPE"), 157 | "Forward P/E": info.get("forwardPE"), 158 | "Dividend Yield": info.get("dividendYield"), 159 | "52 Week Low": info.get("fiftyTwoWeekLow"), 160 | "52 Week High": info.get("fiftyTwoWeekHigh"), 161 | "Avg Volume (10 day)": info.get("averageDailyVolume10Day"), 162 | # Add more fields if desired 163 | } 164 | 165 | formatted_data = [ 166 | f"Financial Summary for {data_points.get('Company Name', ticker)} ({data_points.get('Symbol', ticker).upper()}) - {data_points.get('Quote Type', 'N/A')}" 167 | ] 168 | formatted_data.append("-" * 40) 169 | 170 | for key, value in data_points.items(): 171 | if value is not None: # Only show fields that have a value 172 | formatted_value = value 173 | # Format specific types for readability 174 | if key in [ 175 | "Current Price", 176 | "Previous Close", 177 | "Open", 178 | "Day Low", 179 | "Day High", 180 | "52 Week Low", 181 | "52 Week High", 182 | ] and isinstance(value, (int, float)): 183 | formatted_value = ( 184 | f"{value:.2f} {data_points.get('Currency', '')}".strip() 185 | ) 186 | elif key in [ 187 | "Volume", 188 | "Market Cap", 189 | "Avg Volume (10 day)", 190 | ] and isinstance(value, (int, float)): 191 | if value > 1_000_000_000: 192 | formatted_value = f"{value / 1_000_000_000:.2f}B" 193 | elif value > 1_000_000: 194 | formatted_value = f"{value / 1_000_000:.2f}M" 195 | elif value > 1_000: 196 | formatted_value = f"{value / 1_000:.2f}K" 197 | else: 198 | formatted_value = f"{value:,}" 199 | elif key == "Dividend Yield" and isinstance(value, (int, float)): 200 | formatted_value = f"{value * 100:.2f}%" 201 | elif ( 202 | key == "Trailing P/E" 203 | or key == "Forward P/E") and isinstance(value, (int, float) 204 | ): 205 | formatted_value = f"{value:.2f}" 206 | 207 | formatted_data.append(f"{key}: {formatted_value}") 208 | 209 | # Optionally, add a line about latest financials if easily available 210 | try: 211 | latest_financials = stock.financials.iloc[:, 0] 212 | revenue = latest_financials.get("Total Revenue") 213 | net_income = latest_financials.get("Net Income") 214 | if revenue is not None or net_income is not None: 215 | formatted_data.append("-" * 40) 216 | fin_date = latest_financials.name.strftime("%Y-%m-%d") 217 | if revenue: 218 | formatted_data.append( 219 | f"Latest Revenue ({fin_date}): ${revenue / 1e6:.2f}M" 220 | ) 221 | if net_income: 222 | formatted_data.append( 223 | f"Latest Net Income ({fin_date}): ${net_income / 1e6:.2f}M" 224 | ) 225 | except Exception: 226 | pass # Ignore errors fetching/parsing financials for this summary 227 | 228 | return "\n".join(formatted_data) 229 | 230 | except Exception as e: 231 | traceback.print_exc() # Log the full error for debugging 232 | return f"Error fetching financial data for {ticker}: {str(e)}." 233 | 234 | 235 | # Context Assembly UDF: Combines various text-based search results and tool outputs. 236 | # This function is called by a computed column in the `agents.tools` table 237 | # to prepare the summarized context before the final LLM call. 238 | # Demonstrates processing results from multiple Pixeltable search queries. 239 | @pxt.udf 240 | def assemble_multimodal_context( 241 | question: str, 242 | tool_outputs: Optional[List[Dict[str, Any]]], 243 | doc_context: Optional[List[Union[Dict[str, Any], str]]], 244 | memory_context: Optional[List[Dict[str, Any]]] = None, 245 | chat_memory_context: Optional[List[Dict[str, Any]]] = None, 246 | ) -> str: 247 | """ 248 | Constructs a single text block summarizing various context types 249 | (documents, memory bank items, chat history search results, and generic tool outputs) 250 | relevant to the user's question. 251 | Video/Audio transcript results will appear in 'tool_outputs' if the LLM chose to call those tools. 252 | Does NOT include recent chat history or image/video frame details. 253 | """ 254 | # --- Image Handling Note --- 255 | # Image/Video frame context is handled in `assemble_final_messages` 256 | # as it requires specific formatting for multimodal LLM input. 257 | 258 | # Format document context inline 259 | doc_context_str = "N/A" 260 | if doc_context: 261 | doc_items = [] 262 | for item in doc_context: 263 | # Safely extract text and source filename 264 | text = item.get("text", "") if isinstance(item, dict) else str(item) 265 | source = ( 266 | item.get("source_doc", "Unknown Document") 267 | if isinstance(item, dict) 268 | else "Unknown Document" 269 | ) 270 | source_name = os.path.basename(str(source)) 271 | if text: 272 | doc_items.append(f"- [Source: {source_name}] {text}") 273 | if doc_items: 274 | doc_context_str = "\n".join(doc_items) 275 | 276 | # Format memory bank context 277 | memory_context_str = "N/A" 278 | if memory_context: 279 | memory_items = [] 280 | for item in memory_context: 281 | # Safely extract details 282 | content = item.get("content", "") 283 | item_type = item.get("type", "unknown") 284 | language = item.get("language") 285 | sim = item.get("sim") 286 | context_query = item.get("context_query", "Unknown Query") 287 | # Use triple quotes for multi-line f-string clarity 288 | item_desc = f"""- [Memory Item | Type: {item_type}{f" ({language})" if language else ""} | Original Query: '{context_query}' {f"| Sim: {sim:.3f}" if sim is not None else ""}] 289 | Content: {content[:100]}{"..." if len(content) > 100 else ""}""" 290 | memory_items.append(item_desc) 291 | if memory_items: 292 | memory_context_str = "\n".join(memory_items) 293 | 294 | # Format chat history search context 295 | chat_memory_context_str = "N/A" 296 | if chat_memory_context: 297 | chat_memory_items = [] 298 | for item in chat_memory_context: 299 | content = item.get("content", "") 300 | role = item.get("role", "unknown") 301 | sim = item.get("sim") 302 | timestamp = item.get("timestamp") 303 | ts_str = ( 304 | timestamp.strftime("%Y-%m-%d %H:%M") if timestamp else "Unknown Time" 305 | ) 306 | item_desc = f"""- [Chat History | Role: {role} | Time: {ts_str} {f"| Sim: {sim:.3f}" if sim is not None else ""}] 307 | Content: {content[:150]}{"..." if len(content) > 150 else ""}""" 308 | chat_memory_items.append(item_desc) 309 | if chat_memory_items: 310 | chat_memory_context_str = "\n".join(chat_memory_items) 311 | 312 | # Format tool outputs 313 | tool_outputs_str = str(tool_outputs) if tool_outputs else "N/A" 314 | 315 | # Construct the final summary text block 316 | text_content = f""" 317 | ORIGINAL QUESTION: 318 | {question} 319 | 320 | AVAILABLE CONTEXT: 321 | 322 | [TOOL RESULTS] 323 | {tool_outputs_str} 324 | 325 | [DOCUMENT CONTEXT] 326 | {doc_context_str} 327 | 328 | [MEMORY BANK CONTEXT] 329 | {memory_context_str} 330 | 331 | [CHAT HISTORY SEARCH CONTEXT] (Older messages relevant to the query) 332 | {chat_memory_context_str} 333 | """ 334 | 335 | return text_content.strip() 336 | 337 | 338 | # Final Message Assembly UDF: Creates the structured message list for the main LLM. 339 | # This handles the specific format required by multimodal models (like Claude 3.5 Sonnet) 340 | # incorporating text, images, and potentially video frames. 341 | # It is called by a computed column in the `agents.tools` table. 342 | @pxt.udf 343 | def assemble_final_messages( 344 | history_context: Optional[List[Dict[str, Any]]], 345 | multimodal_context_text: str, 346 | image_context: Optional[List[Dict[str, Any]]] = None, # Input image results 347 | video_frame_context: Optional[ 348 | List[Dict[str, Any]] 349 | ] = None, # Input video frame results 350 | ) -> List[Dict[str, Any]]: 351 | """ 352 | Constructs the final list of messages for the LLM, incorporating: 353 | - Recent chat history (user/assistant turns). 354 | - The main text context summary (docs, memory, tool outputs, etc.). 355 | - Image context (base64 encoded images). 356 | - Video frame context (base64 encoded video frames). 357 | 358 | This structure is required for multimodal LLMs like Claude 3. 359 | 360 | Args: 361 | history_context: Recent chat messages. 362 | multimodal_context_text: The combined text context from `assemble_multimodal_context`. 363 | image_context: List of image search results (containing base64 data). 364 | video_frame_context: List of video frame search results (containing base64 data). 365 | 366 | Returns: 367 | A list of messages formatted for the LLM API. 368 | """ 369 | messages = [] 370 | 371 | # 1. Add recent chat history (if any) in chronological order 372 | if history_context: 373 | for item in reversed(history_context): 374 | role = item.get("role") 375 | content = item.get("content") 376 | if role and content: 377 | messages.append({"role": role, "content": content}) 378 | 379 | # 2. Prepare the content block for the final user message 380 | final_user_content = [] 381 | 382 | # 2a. Add image blocks (if any) 383 | if image_context: 384 | for item in image_context: 385 | # Safely extract base64 encoded image data 386 | if isinstance(item, dict) and "encoded_image" in item: 387 | image_data = item["encoded_image"] 388 | # Ensure it's a string 389 | if isinstance(image_data, bytes): 390 | image_data = image_data.decode("utf-8") 391 | elif not isinstance(image_data, str): 392 | continue # Skip invalid data 393 | 394 | # Append in the format required by the LLM API 395 | final_user_content.append( 396 | { 397 | "type": "image", 398 | "source": { 399 | "type": "base64", 400 | "media_type": "image/png", # Assuming PNG, adjust if needed 401 | "data": image_data, 402 | }, 403 | } 404 | ) 405 | 406 | # 2b. Add video frame blocks (if any) - NOTE: Currently illustrative, LLM support varies 407 | if video_frame_context: 408 | for item in video_frame_context: 409 | # Safely extract base64 encoded video frame data 410 | if isinstance(item, dict) and "encoded_video_frame" in item: 411 | video_frame_data = item["encoded_video_frame"] 412 | if isinstance(video_frame_data, bytes): 413 | video_frame_data = video_frame_data.decode("utf-8") 414 | elif not isinstance(video_frame_data, str): 415 | continue # Skip invalid data 416 | 417 | # Append in the format required by the LLM API (adjust if API differs) 418 | final_user_content.append( 419 | { 420 | "type": "video_frame", # Hypothetical type, check LLM docs 421 | "source": { 422 | "type": "base64", 423 | "media_type": "image/png", # Frames are usually images 424 | "data": video_frame_data, 425 | }, 426 | } 427 | ) 428 | 429 | # 2c. Add the main text context block 430 | final_user_content.append( 431 | { 432 | "type": "text", 433 | "text": multimodal_context_text, # Use the pre-formatted summary 434 | } 435 | ) 436 | 437 | # 3. Append the complete user message (potentially multimodal) 438 | messages.append({"role": "user", "content": final_user_content}) 439 | 440 | return messages 441 | 442 | 443 | # Follow-up Prompt Assembly UDF: Creates the input prompt for the follow-up LLM. 444 | # Encapsulates the prompt template structure, making the workflow definition 445 | # in setup_pixeltable.py cleaner and focusing it on data flow. 446 | @pxt.udf 447 | def assemble_follow_up_prompt(original_prompt: str, answer_text: str) -> str: 448 | """Constructs the formatted prompt string for the follow-up question LLM. 449 | 450 | This function encapsulates the prompt template to make it reusable and 451 | easier to see the input being sent to the LLM in the Pixeltable trace. 452 | Includes a few-shot example to guide the model. 453 | """ 454 | # Updated template with clearer instructions and an example 455 | follow_up_system_prompt_template = """You are an expert assistant tasked with generating **exactly 3** relevant and concise follow-up questions based on an original user query and the provided answer. Focus *only* on the content provided. 456 | 457 | **Instructions:** 458 | 1. Read the and sections carefully. 459 | 2. Generate 3 distinct questions that logically follow from the information presented. 460 | 3. The questions should encourage deeper exploration of the topic discussed. 461 | 4. **Output ONLY the 3 questions**, one per line. Do NOT include numbering, bullet points, or any other text. 462 | 463 | **Example:** 464 | 465 | 466 | What are the main benefits of using Pixeltable for AI workflows? 467 | 468 | 469 | 470 | Pixeltable simplifies AI workflows by providing automated data orchestration, native multimodal support (text, images, video, audio), a declarative interface, and integrations with LLMs and ML models. It handles complex tasks like data versioning, incremental computation, and vector indexing automatically. 471 | 472 | 473 | How does Pixeltable handle data versioning specifically? 474 | Can you elaborate on the declarative interface of Pixeltable? 475 | What specific LLMs and ML models does Pixeltable integrate with? 476 | 477 | **Now, generate questions for the following input:** 478 | 479 | 480 | {original_prompt} 481 | 482 | 483 | 484 | {answer_text} 485 | 486 | """ 487 | return follow_up_system_prompt_template.format( 488 | original_prompt=original_prompt, answer_text=answer_text 489 | ) 490 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "pixelbot-agent" 7 | version = "0.1.0" 8 | description = "Multimodal AI agent built using Pixeltable." 9 | readme = "README.md" 10 | requires-python = ">=3.9" 11 | license = { text = "Apache-2.0" } 12 | classifiers = [ 13 | "Programming Language :: Python :: 3", 14 | "License :: OSI Approved :: Apache Software License", 15 | "Operating System :: OS Independent", 16 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 17 | ] 18 | dependencies = [ 19 | # Web Framework & Server 20 | "flask", 21 | "flask-cors", 22 | "Flask-Limiter", 23 | "waitress", 24 | 25 | # AI and ML (Models, LLMs) 26 | "anthropic", 27 | "sentence-transformers", 28 | "tiktoken", 29 | "spacy", 30 | "openai", 31 | "mistralai", 32 | "numpy", 33 | "openpyxl", 34 | "pixeltable==0.3.12", # Specify version as found in requirements.txt 35 | 36 | # Utilities 37 | "python-dotenv", 38 | "duckduckgo-search", 39 | "serpapi", 40 | "yfinance", 41 | "workos", 42 | 43 | # Add development tools here if needed, e.g., under [project.optional-dependencies] 44 | "ruff" # Assuming ruff is used for linting/formatting 45 | ] 46 | 47 | [project.urls] 48 | "Homepage" = "https://github.com/pixeltable/pixelbot-main" # Replace with actual URL if different 49 | "Bug Tracker" = "https://github.com/pixeltable/pixelbot-main/issues" # Replace if different 50 | 51 | [tool.setuptools] 52 | # Optional: Specify packages to include if not automatically discovered 53 | # packages = find: {} 54 | 55 | [tool.ruff] 56 | # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. 57 | # Add others as needed (e.g., `I` for isort). 58 | select = ["E", "F"] 59 | ignore = [] 60 | 61 | # Allow autofix for all enabled rules (when `--fix`) 62 | fixable = ["ALL"] 63 | unfixable = [] 64 | 65 | # Exclude a few common directories. 66 | exclude = [ 67 | ".bzr", 68 | ".direnv", 69 | ".eggs", 70 | ".git", 71 | ".git-rewrite", 72 | ".hg", 73 | ".mypy_cache", 74 | ".nox", 75 | ".pants.d", 76 | ".pytype", 77 | ".ruff_cache", 78 | ".svn", 79 | ".tox", 80 | ".venv", 81 | "__pypackages__", 82 | "_build", 83 | "buck-out", 84 | "build", 85 | "dist", 86 | "node_modules", 87 | "venv", 88 | ] 89 | 90 | # Same as Black. 91 | line-length = 88 92 | indent-width = 4 93 | 94 | # Assume Python 3.9+ 95 | target-version = "py39" 96 | 97 | [tool.ruff.format] 98 | # Use Python 3.9 formatting. 99 | target-version = "py39" 100 | 101 | # Like Black, use double quotes for strings. 102 | quote-style = "double" 103 | 104 | # Like Black, indent with spaces, rather than tabs. 105 | indent-style = "space" 106 | 107 | # Like Black, respect magic trailing commas. 108 | skip-magic-trailing-comma = false 109 | 110 | # Like Black, automatically detect the appropriate line ending. 111 | line-ending = "auto" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pixeltable/pixelbot/3b5b426007fa5f45beb3d6f499ae98ef460b366f/requirements.txt -------------------------------------------------------------------------------- /setup_pixeltable.py: -------------------------------------------------------------------------------- 1 | # Third-party library imports 2 | from dotenv import load_dotenv 3 | 4 | # Import centralized configuration **before** Pixeltable/UDF imports that might use it 5 | import config 6 | 7 | # Pixeltable core imports 8 | import pixeltable as pxt 9 | 10 | # Pixeltable function imports - organized by category 11 | # - Image and video processing 12 | from pixeltable.functions import image as pxt_image 13 | from pixeltable.functions.video import extract_audio 14 | 15 | # - LLM and AI model integrations 16 | from pixeltable.functions.anthropic import invoke_tools, messages 17 | from pixeltable.functions.huggingface import sentence_transformer, clip 18 | from pixeltable.functions import openai 19 | from pixeltable.functions.mistralai import chat_completions as mistral 20 | 21 | # - Data transformation tools 22 | from pixeltable.iterators import ( 23 | DocumentSplitter, 24 | FrameIterator, 25 | AudioSplitter, 26 | StringSplitter, 27 | ) 28 | from pixeltable.functions import string as pxt_str 29 | 30 | # Custom function imports (UDFs) 31 | import functions 32 | 33 | # Load environment variables 34 | load_dotenv() 35 | 36 | # Initialize the Pixeltable directory structure 37 | # This provides a clean, hierarchical organization for related tables and views. 38 | 39 | # WARNING: The following line will DELETE ALL DATA (TABLES, VIEWS, INDEXES) in the 'agents' directory. 40 | pxt.drop_dir("agents", force=True) 41 | pxt.create_dir("agents", if_exists="ignore") # Use if_exists='ignore' to avoid errors if the directory already exists 42 | 43 | # === DOCUMENT PROCESSING === 44 | # Create a table to store uploaded documents. 45 | # Pixeltable tables manage schema and efficiently store references to data. 46 | documents = pxt.create_table( 47 | "agents.collection", 48 | {"document": pxt.Document, "uuid": pxt.String, "timestamp": pxt.Timestamp, "user_id": pxt.String}, 49 | if_exists="ignore", 50 | ) 51 | print("Created/Loaded 'agents.collection' table") 52 | 53 | # Create a view to chunk documents using a Pixeltable Iterator. 54 | # Views transform data on-demand without duplicating storage. 55 | # Iterators like DocumentSplitter handle the generation of new rows (chunks). 56 | chunks = pxt.create_view( 57 | "agents.chunks", 58 | documents, 59 | iterator=DocumentSplitter.create( 60 | document=documents.document, 61 | separators="paragraph", 62 | metadata="title, heading, page" # Include metadata from the document 63 | ), 64 | if_exists="ignore", 65 | ) 66 | 67 | # Add an embedding index to the 'text' column of the chunks view. 68 | # This enables fast semantic search using vector similarity. 69 | chunks.add_embedding_index( 70 | "text", # The column containing text to index 71 | string_embed=sentence_transformer.using( # Specify the embedding function 72 | model_id=config.EMBEDDING_MODEL_ID 73 | ), # Use model from config 74 | if_exists="ignore", 75 | ) 76 | 77 | 78 | # Define a reusable search query function using the @pxt.query decorator. 79 | # This allows calling complex search logic easily from other parts of the application. 80 | @pxt.query 81 | def search_documents(query_text: str, user_id: str): 82 | # Calculate semantic similarity between the query and indexed text chunks. 83 | sim = chunks.text.similarity(query_text) 84 | # Use Pixeltable's fluent API (similar to SQL) to filter, order, and select results. 85 | return ( 86 | chunks.where( 87 | (chunks.user_id == user_id) 88 | & (sim > 0.5) # Filter by similarity threshold 89 | & (pxt_str.len(chunks.text) > 30) # Filter by minimum length 90 | ) 91 | .order_by(sim, asc=False) 92 | .select( 93 | chunks.text, 94 | source_doc=chunks.document, # Include reference to the original document 95 | sim=sim, 96 | title=chunks.title, 97 | heading=chunks.heading, 98 | page_number=chunks.page 99 | ) 100 | .limit(20) 101 | ) 102 | 103 | # === IMAGE PROCESSING === 104 | # Create a table for images using Pixeltable's built-in Image type. 105 | images = pxt.create_table( 106 | "agents.images", 107 | {"image": pxt.Image, "uuid": pxt.String, "timestamp": pxt.Timestamp, "user_id": pxt.String}, 108 | if_exists="ignore", 109 | ) 110 | print("Created/Loaded 'agents.images' table") 111 | 112 | # Add a computed column to automatically generate image thumbnails. 113 | # Pixeltable runs the specified function(s) whenever new data is added or dependencies change. 114 | THUMB_SIZE_SIDEBAR = (96, 96) 115 | images.add_computed_column( 116 | thumbnail=pxt_image.b64_encode( # Encode the resized image as Base64 117 | pxt_image.resize(images.image, size=THUMB_SIZE_SIDEBAR) # Resize the image 118 | ), 119 | if_exists="ignore", 120 | ) 121 | print("Added/verified thumbnail computed column for images.") 122 | 123 | # Add an embedding index for images using CLIP. 124 | # This enables cross-modal search (text-to-image and image-to-image). 125 | images.add_embedding_index( 126 | "image", 127 | embedding=clip.using(model_id=config.CLIP_MODEL_ID), # Use CLIP model from config 128 | if_exists="ignore", 129 | ) 130 | 131 | 132 | # Define an image search query. 133 | @pxt.query 134 | def search_images(query_text: str, user_id: str): 135 | # Calculate similarity between the query text embedding and image embeddings. 136 | sim = images.image.similarity(query_text) # Cross-modal similarity search 137 | print(f"Image search query: {query_text} for user: {user_id}") 138 | return ( 139 | images.where((images.user_id == user_id) & (sim > 0.25)) 140 | .order_by(sim, asc=False) 141 | .select( 142 | # Return Base64 encoded, resized images for direct display in the UI. 143 | encoded_image=pxt_image.b64_encode( 144 | pxt_image.resize(images.image, size=(224, 224)), "png" 145 | ), 146 | sim=sim, 147 | ) 148 | .limit(5) 149 | ) 150 | 151 | 152 | # === VIDEO PROCESSING === 153 | # Create a table for videos. 154 | videos = pxt.create_table( 155 | "agents.videos", 156 | {"video": pxt.Video, "uuid": pxt.String, "timestamp": pxt.Timestamp, "user_id": pxt.String}, 157 | if_exists="ignore", 158 | ) 159 | print("Created/Loaded 'agents.videos' table") 160 | 161 | # Create a view to extract frames from videos using FrameIterator. 162 | print("Creating video frames view...") 163 | video_frames_view = pxt.create_view( 164 | "agents.video_frames", 165 | videos, 166 | iterator=FrameIterator.create(video=videos.video, fps=1), # Extract 1 frame per second 167 | if_exists="ignore", 168 | ) 169 | print("Created/Loaded 'agents.video_frames' view") 170 | 171 | # Add an embedding index to video frames using CLIP. 172 | print("Adding video frame embedding index (CLIP)...") 173 | video_frames_view.add_embedding_index( 174 | column="frame", 175 | embedding=clip.using(model_id=config.CLIP_MODEL_ID), 176 | if_exists="ignore", 177 | ) 178 | print("Video frame embedding index created/verified.") 179 | 180 | 181 | # Define a video frame search query. 182 | @pxt.query 183 | def search_video_frames(query_text: str, user_id: str): 184 | sim = video_frames_view.frame.similarity(query_text) 185 | print(f"Video Frame search query: {query_text} for user: {user_id}") 186 | return ( 187 | video_frames_view.where((video_frames_view.user_id == user_id) & (sim > 0.25)) 188 | .order_by(sim, asc=False) 189 | .select( 190 | encoded_frame=pxt_image.b64_encode(video_frames_view.frame, "png"), 191 | source_video=video_frames_view.video, # Link back to the original video 192 | sim=sim, 193 | ) 194 | .limit(5) 195 | ) 196 | 197 | 198 | # Add a computed column to automatically extract audio from videos. 199 | videos.add_computed_column( 200 | audio=extract_audio(videos.video, format="mp3"), if_exists="ignore" 201 | ) 202 | 203 | # === AUDIO TRANSCRIPTION AND SEARCH === 204 | # Create a view to chunk audio extracted from videos using AudioSplitter. 205 | video_audio_chunks_view = pxt.create_view( 206 | "agents.video_audio_chunks", 207 | videos, 208 | iterator=AudioSplitter.create( 209 | audio=videos.audio, # Input column with extracted audio 210 | chunk_duration_sec=30.0 211 | ), 212 | if_exists="ignore", 213 | ) 214 | 215 | # Add a computed column to transcribe video audio chunks using OpenAI Whisper. 216 | print("Adding/Computing video audio transcriptions (OpenAI Whisper API)...") 217 | video_audio_chunks_view.add_computed_column( 218 | transcription=openai.transcriptions( 219 | audio=video_audio_chunks_view.audio, 220 | model=config.WHISPER_MODEL_ID, 221 | ), 222 | if_exists="replace", # 'replace' ensures updates if the function or model changes 223 | ) 224 | print("Video audio transcriptions column added/updated.") 225 | 226 | # Create a view to split video transcriptions into sentences using StringSplitter. 227 | video_transcript_sentences_view = pxt.create_view( 228 | "agents.video_transcript_sentences", 229 | video_audio_chunks_view.where( 230 | video_audio_chunks_view.transcription != None # Process only chunks with transcriptions 231 | ), 232 | iterator=StringSplitter.create( 233 | text=video_audio_chunks_view.transcription.text, # Access the 'text' field from the JSON result 234 | separators="sentence", 235 | ), 236 | if_exists="ignore", 237 | ) 238 | 239 | # Define the embedding model once for reuse. 240 | sentence_embed_model = sentence_transformer.using( 241 | model_id=config.EMBEDDING_MODEL_ID 242 | ) 243 | 244 | # Add an embedding index to video transcript sentences. 245 | print("Adding video transcript sentence embedding index...") 246 | video_transcript_sentences_view.add_embedding_index( 247 | column="text", 248 | string_embed=sentence_embed_model, 249 | if_exists="ignore", 250 | ) 251 | print("Video transcript sentence embedding index created/verified.") 252 | 253 | 254 | # Define video transcript search query. 255 | @pxt.query 256 | def search_video_transcripts(query_text: str): 257 | """ Search for video transcripts by text query. 258 | Args: 259 | query_text (str): The text query to search for. 260 | Returns: 261 | A list of video transcript sentences and their source video files. 262 | """ 263 | sim = video_transcript_sentences_view.text.similarity(query_text) 264 | return ( 265 | video_transcript_sentences_view.where((video_transcript_sentences_view.user_id == 'local_user') & (sim > 0.7)) 266 | .order_by(sim, asc=False) 267 | .select( 268 | video_transcript_sentences_view.text, 269 | source_video=video_transcript_sentences_view.video, # Link back to the video 270 | sim=sim, 271 | ) 272 | .limit(20) 273 | ) 274 | 275 | 276 | # === DIRECT AUDIO FILE PROCESSING === 277 | # Create a table for directly uploaded audio files. 278 | audios = pxt.create_table( 279 | "agents.audios", 280 | {"audio": pxt.Audio, "uuid": pxt.String, "timestamp": pxt.Timestamp, "user_id": pxt.String}, 281 | if_exists="ignore", 282 | ) 283 | print("Created/Loaded 'agents.audios' table") 284 | 285 | # Sample data insertion (disabled by default) 286 | print("Sample audio insertion is disabled.") 287 | 288 | # Create view to chunk directly uploaded audio files. 289 | audio_chunks_view = pxt.create_view( 290 | "agents.audio_chunks", 291 | audios, 292 | iterator=AudioSplitter.create( 293 | audio=audios.audio, 294 | chunk_duration_sec=60.0 295 | ), 296 | if_exists="ignore", 297 | ) 298 | 299 | # Add computed column to transcribe direct audio chunks. 300 | print("Adding/Computing direct audio transcriptions (OpenAI Whisper API)...") 301 | audio_chunks_view.add_computed_column( 302 | transcription=openai.transcriptions( 303 | audio=audio_chunks_view.audio, 304 | model=config.WHISPER_MODEL_ID, 305 | ), 306 | if_exists="replace", 307 | ) 308 | print("Direct audio transcriptions column added/updated.") 309 | 310 | # Create view to split direct audio transcriptions into sentences. 311 | audio_transcript_sentences_view = pxt.create_view( 312 | "agents.audio_transcript_sentences", 313 | audio_chunks_view.where(audio_chunks_view.transcription != None), 314 | iterator=StringSplitter.create( 315 | text=audio_chunks_view.transcription.text, separators="sentence" 316 | ), 317 | if_exists="ignore", 318 | ) 319 | 320 | # Add embedding index to direct audio transcript sentences. 321 | print("Adding direct audio transcript sentence embedding index...") 322 | audio_transcript_sentences_view.add_embedding_index( 323 | column="text", 324 | string_embed=sentence_embed_model, # Reuse the same sentence model 325 | if_exists="ignore", 326 | ) 327 | print("Direct audio transcript sentence embedding index created/verified.") 328 | 329 | 330 | # Define direct audio transcript search query. 331 | @pxt.query 332 | def search_audio_transcripts(query_text: str): 333 | """ Search for audio transcripts by text query. 334 | Args: 335 | query_text (str): The text query to search for. 336 | Returns: 337 | A list of audio transcript sentences and their source audio files. 338 | """ 339 | sim = audio_transcript_sentences_view.text.similarity(query_text) 340 | print(f"Direct Audio Transcript search query: {query_text}") 341 | return ( 342 | audio_transcript_sentences_view.where((audio_transcript_sentences_view.user_id == 'local_user') & (sim > 0.6)) 343 | .order_by(sim, asc=False) 344 | .select( 345 | audio_transcript_sentences_view.text, 346 | source_audio=audio_transcript_sentences_view.audio, # Link back to the audio file 347 | sim=sim, 348 | ) 349 | .limit(30) 350 | ) 351 | 352 | 353 | # === SELECTIVE MEMORY BANK (Code & Text) === 354 | # Create table for storing user-saved text or code snippets. 355 | memory_bank = pxt.create_table( 356 | "agents.memory_bank", 357 | { 358 | "content": pxt.String, # The saved text or code 359 | "type": pxt.String, # 'code' or 'text' 360 | "language": pxt.String, # Programming language (if type='code') 361 | "context_query": pxt.String, # User note or query that generated the content 362 | "timestamp": pxt.Timestamp, 363 | "user_id": pxt.String 364 | }, 365 | if_exists="ignore", 366 | ) 367 | 368 | # Add embedding index for semantic search on memory bank content. 369 | print("Adding memory bank content embedding index...") 370 | memory_bank.add_embedding_index( 371 | column="content", 372 | string_embed=sentence_embed_model, # Reuse the sentence model 373 | if_exists="ignore", 374 | ) 375 | print("Memory bank content embedding index created/verified.") 376 | 377 | 378 | # Query to retrieve all memory items for a user. 379 | @pxt.query 380 | def get_all_memory(user_id: str): 381 | return memory_bank.where(memory_bank.user_id == user_id).select( 382 | content=memory_bank.content, 383 | type=memory_bank.type, 384 | language=memory_bank.language, 385 | context_query=memory_bank.context_query, 386 | timestamp=memory_bank.timestamp, 387 | ).order_by(memory_bank.timestamp, asc=False) 388 | 389 | 390 | # Query for semantic search on memory bank content. 391 | @pxt.query 392 | def search_memory(query_text: str, user_id: str): 393 | sim = memory_bank.content.similarity(query_text) 394 | print(f"Memory Bank search query: {query_text} for user: {user_id}") 395 | return ( 396 | memory_bank.where((memory_bank.user_id == user_id) & (sim > 0.8)) 397 | .order_by(sim, asc=False) 398 | .select( 399 | content=memory_bank.content, 400 | type=memory_bank.type, 401 | language=memory_bank.language, 402 | context_query=memory_bank.context_query, 403 | sim=sim, 404 | ) 405 | .limit(10) 406 | ) 407 | 408 | 409 | # === CHAT HISTORY TABLE & QUERY === 410 | # Create table specifically for storing conversation turns. 411 | chat_history = pxt.create_table( 412 | "agents.chat_history", 413 | { 414 | "role": pxt.String, # 'user' or 'assistant' 415 | "content": pxt.String, 416 | "timestamp": pxt.Timestamp, 417 | "user_id": pxt.String 418 | }, 419 | if_exists="ignore", 420 | ) 421 | 422 | # Add embedding index to chat history content for semantic search over conversations. 423 | print("Adding chat history content embedding index...") 424 | chat_history.add_embedding_index( 425 | column="content", 426 | string_embed=sentence_embed_model, # Reuse sentence model 427 | if_exists="ignore", 428 | ) 429 | print("Chat history content embedding index created/verified.") 430 | 431 | 432 | # Query to retrieve the N most recent chat messages for context. 433 | @pxt.query 434 | def get_recent_chat_history(user_id: str, limit: int = 4): # Default to last 4 messages 435 | return ( 436 | chat_history.where(chat_history.user_id == user_id) 437 | .order_by(chat_history.timestamp, asc=False) 438 | .select(role=chat_history.role, content=chat_history.content) 439 | .limit(limit) 440 | ) 441 | 442 | 443 | # Query for semantic search across the entire chat history. 444 | @pxt.query 445 | def search_chat_history(query_text: str, user_id: str): 446 | sim = chat_history.content.similarity(query_text) 447 | print(f"Chat History search query: {query_text} for user: {user_id}") 448 | return ( 449 | chat_history.where((chat_history.user_id == user_id) & (sim > 0.8)) 450 | .order_by(sim, asc=False) 451 | .select(role=chat_history.role, content=chat_history.content, sim=sim) 452 | .limit(10) 453 | ) 454 | 455 | 456 | # === USER PERSONAS TABLE === 457 | # Create table to store user-defined agent personas (prompts + parameters). 458 | user_personas = pxt.create_table( 459 | "agents.user_personas", 460 | { 461 | "user_id": pxt.String, 462 | "persona_name": pxt.String, 463 | "initial_prompt": pxt.String, # System prompt for tool selection stage 464 | "final_prompt": pxt.String, # System prompt for final answer generation stage 465 | "llm_params": pxt.Json, # LLM parameters (temperature, max_tokens, etc.) 466 | "timestamp": pxt.Timestamp, 467 | }, 468 | if_exists="ignore", 469 | ) 470 | print("Created/Loaded 'agents.user_personas' table") 471 | 472 | 473 | # === IMAGE GENERATION PIPELINE === 474 | # Create table to store image generation requests. 475 | image_gen_tasks = pxt.create_table( 476 | "agents.image_generation_tasks", 477 | {"prompt": pxt.String, "timestamp": pxt.Timestamp, "user_id": pxt.String}, 478 | if_exists="ignore", 479 | ) 480 | 481 | # Add computed column to generate image using OpenAI DALL-E 3. 482 | # This column calls the Pixeltable OpenAI integration function. 483 | image_gen_tasks.add_computed_column( 484 | generated_image=openai.image_generations( 485 | prompt=image_gen_tasks.prompt, 486 | model=config.DALLE_MODEL_ID, 487 | size="1024x1024", 488 | # Add other DALL-E parameters like quality, style if desired 489 | ), 490 | if_exists="ignore", 491 | ) 492 | print("Image generation table and computed column created/verified.") 493 | 494 | # === AGENT WORKFLOW DEFINITION === 495 | # Register User-Defined Functions (UDFs) from functions.py AND reusable @pxt.query functions as tools. 496 | # Pixeltable's `pxt.tools()` helper facilitates this integration. 497 | tools = pxt.tools( 498 | # UDFs - External API Calls 499 | functions.get_latest_news, 500 | functions.fetch_financial_data, 501 | functions.search_news, 502 | # Query Functions registered as Tools - Agentic RAG 503 | search_video_transcripts, 504 | search_audio_transcripts 505 | ) 506 | 507 | # Create the main workflow table (`agents.tools`). 508 | # Rows are inserted here when a user submits a query. 509 | # Computed columns define the agent's reasoning and action sequence. 510 | tool_agent = pxt.create_table( 511 | "agents.tools", 512 | { 513 | # Input fields from the user query 514 | "prompt": pxt.String, 515 | "timestamp": pxt.Timestamp, 516 | "user_id": pxt.String, 517 | "initial_system_prompt": pxt.String, # Persona-specific or default 518 | "final_system_prompt": pxt.String, # Persona-specific or default 519 | # LLM parameters (from persona or defaults) 520 | "max_tokens": pxt.Int, 521 | "stop_sequences": pxt.Json, 522 | "temperature": pxt.Float, 523 | "top_k": pxt.Int, 524 | "top_p": pxt.Float, 525 | }, 526 | if_exists="ignore", 527 | ) 528 | 529 | # === DECLARATIVE WORKFLOW WITH COMPUTED COLUMNS === 530 | # Define the agent's processing pipeline declaratively. 531 | # Pixeltable automatically executes these steps based on data dependencies. 532 | 533 | # Step 1: Initial LLM Reasoning (Tool Selection) 534 | # Calls Claude via the Pixeltable `messages` function, providing available tools. 535 | tool_agent.add_computed_column( 536 | initial_response=messages( 537 | model=config.CLAUDE_MODEL_ID, 538 | system=tool_agent.initial_system_prompt, 539 | messages=[{"role": "user", "content": tool_agent.prompt}], 540 | tools=tools, # Pass the registered tools 541 | tool_choice=tools.choice(required=True), # Force the LLM to choose a tool 542 | # Pass LLM parameters from the input row 543 | max_tokens=tool_agent.max_tokens, 544 | stop_sequences=tool_agent.stop_sequences, 545 | temperature=tool_agent.temperature, 546 | top_k=tool_agent.top_k, 547 | top_p=tool_agent.top_p, 548 | ), 549 | if_exists="replace", # Replace if the function definition changes 550 | ) 551 | 552 | # Step 2: Tool Execution 553 | # Calls the tool selected by the LLM in the previous step using `invoke_tools`. 554 | tool_agent.add_computed_column( 555 | tool_output=invoke_tools(tools, tool_agent.initial_response), if_exists="replace" 556 | ) 557 | 558 | # Step 3: Context Retrieval (Parallel Execution) 559 | # These computed columns call the @pxt.query functions defined earlier. 560 | # Pixeltable can execute these searches in parallel if resources allow. 561 | 562 | tool_agent.add_computed_column( 563 | doc_context=search_documents(tool_agent.prompt, tool_agent.user_id), 564 | if_exists="replace", 565 | ) 566 | 567 | tool_agent.add_computed_column( 568 | image_context=search_images(tool_agent.prompt, tool_agent.user_id), if_exists="replace" 569 | ) 570 | 571 | # Add Video Frame Search Context 572 | tool_agent.add_computed_column( 573 | video_frame_context=search_video_frames(tool_agent.prompt, tool_agent.user_id), if_exists="ignore" 574 | ) 575 | 576 | tool_agent.add_computed_column( 577 | memory_context=search_memory(tool_agent.prompt, tool_agent.user_id), if_exists="ignore" 578 | ) 579 | 580 | tool_agent.add_computed_column( 581 | chat_memory_context=search_chat_history(tool_agent.prompt, tool_agent.user_id), if_exists="ignore" 582 | ) 583 | 584 | # Step 4: Retrieve Recent Chat History 585 | tool_agent.add_computed_column( 586 | history_context=get_recent_chat_history(tool_agent.user_id), 587 | if_exists="ignore", 588 | ) 589 | 590 | # Step 5: Assemble Multimodal Context Summary (Text Only) 591 | # Calls a UDF to combine text-based context. Video/Audio transcript context 592 | # will now be part of tool_output if the LLM chose to call those tools. 593 | tool_agent.add_computed_column( 594 | multimodal_context_summary=functions.assemble_multimodal_context( 595 | tool_agent.prompt, 596 | tool_agent.tool_output, 597 | tool_agent.doc_context, 598 | tool_agent.memory_context, 599 | tool_agent.chat_memory_context, 600 | ), 601 | if_exists="replace", 602 | ) 603 | 604 | # Step 6: Assemble Final LLM Messages 605 | # Calls a UDF to create the structured message list, including image/frame data. 606 | tool_agent.add_computed_column( 607 | final_prompt_messages=functions.assemble_final_messages( 608 | tool_agent.history_context, 609 | tool_agent.multimodal_context_summary, 610 | image_context=tool_agent.image_context, 611 | video_frame_context=tool_agent.video_frame_context, 612 | ), 613 | if_exists="replace", 614 | ) 615 | 616 | # Step 7: Final LLM Reasoning (Answer Generation) 617 | # Calls Claude again with the fully assembled context and history. 618 | tool_agent.add_computed_column( 619 | final_response=messages( 620 | model=config.CLAUDE_MODEL_ID, 621 | system=tool_agent.final_system_prompt, 622 | messages=tool_agent.final_prompt_messages, # Use the assembled message list 623 | max_tokens=tool_agent.max_tokens, 624 | stop_sequences=tool_agent.stop_sequences, 625 | temperature=tool_agent.temperature, 626 | top_k=tool_agent.top_k, 627 | top_p=tool_agent.top_p, 628 | ), 629 | if_exists="replace", 630 | ) 631 | 632 | # Step 8: Extract Final Answer Text 633 | # Simple transformation using Pixeltable expressions. 634 | tool_agent.add_computed_column( 635 | answer=tool_agent.final_response.content[0].text, 636 | if_exists="replace", 637 | ) 638 | 639 | # Step 9: Prepare Prompt for Follow-up LLM 640 | # Calls a UDF to format the input for Mistral. 641 | tool_agent.add_computed_column( 642 | follow_up_input_message=functions.assemble_follow_up_prompt( 643 | original_prompt=tool_agent.prompt, answer_text=tool_agent.answer 644 | ), 645 | if_exists="replace", 646 | ) 647 | 648 | # Step 10: Generate Follow-up Suggestions (Mistral) 649 | # Calls Mistral via the Pixeltable integration. 650 | tool_agent.add_computed_column( 651 | follow_up_raw_response=mistral( 652 | model=config.MISTRAL_MODEL_ID, 653 | messages=[ 654 | { 655 | "role": "user", 656 | "content": tool_agent.follow_up_input_message, 657 | } 658 | ], 659 | max_tokens=150, 660 | temperature=0.6, 661 | ), 662 | if_exists="replace", 663 | ) 664 | 665 | # Step 11: Extract Follow-up Text 666 | # Simple transformation using Pixeltable expressions. 667 | tool_agent.add_computed_column( 668 | follow_up_text=tool_agent.follow_up_raw_response.choices[0].message.content, 669 | if_exists="replace", 670 | ) 671 | -------------------------------------------------------------------------------- /static/css/style.css: -------------------------------------------------------------------------------- 1 | /* Base Styles */ 2 | body { 3 | background-color: #FAF9F5 !important; 4 | min-height: 100vh; 5 | font-family: system-ui, -apple-system, sans-serif; 6 | } 7 | 8 | /* Pixel Loader Animation */ 9 | .pixel-loader { 10 | display: flex; 11 | gap: 4px; 12 | } 13 | 14 | .pixel-loader::before, 15 | .pixel-loader::after { 16 | content: ''; 17 | width: 8px; 18 | height: 8px; 19 | border-radius: 2px; 20 | animation: pixel-pulse 1.5s infinite ease-in-out both; 21 | } 22 | 23 | .pixel-loader::before { 24 | animation-delay: -0.32s; 25 | } 26 | 27 | .pixel-loader::after { 28 | animation-delay: -0.16s; 29 | } 30 | 31 | @keyframes pixel-pulse { 32 | 0%, 80%, 100% { 33 | opacity: 0.4; 34 | background-color: #fcd34d; /* amber-300 */ 35 | transform: scale(0.9); 36 | } 37 | 40% { 38 | opacity: 1; 39 | background-color: #f59e0b; /* amber-500 */ 40 | transform: scale(1.1); 41 | } 42 | } 43 | 44 | /* Layout Components */ 45 | .container { 46 | max-width: 1440px !important; 47 | margin-left: auto; 48 | margin-right: auto; 49 | padding-left: 1rem; 50 | padding-right: 1rem; 51 | } 52 | 53 | /* Card Base Styles */ 54 | .card-base { 55 | background-color: white; 56 | border: 1px solid #f3f4f6; 57 | border-radius: 0.5rem; 58 | box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05); 59 | } 60 | 61 | /* Content Areas */ 62 | .content-area { 63 | min-height: 200px; 64 | width: 100%; 65 | max-width: 100%; 66 | background-color: #FAF9F5; 67 | } 68 | 69 | /* Individual content cards within panels */ 70 | .content-area .bg-white { 71 | background-color: white; 72 | border: 1px solid #f3f4f6; 73 | border-radius: 0.5rem; 74 | box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.05), 0 2px 4px -2px rgba(0, 0, 0, 0.05); 75 | } 76 | 77 | .sidebar-section { 78 | margin-top: 0; 79 | margin-bottom: 1.25rem; 80 | } 81 | 82 | .result-card { 83 | margin-bottom: 1.25rem; 84 | overflow: hidden; 85 | } 86 | 87 | /* Navigation */ 88 | .nav-tabs { 89 | margin-bottom: 1.5rem; 90 | padding: 1rem 1.5rem 0.5rem 1.5rem; 91 | } 92 | 93 | .nav-item { 94 | margin-right: 1.5rem; 95 | } 96 | 97 | .nav-link { 98 | position: relative; 99 | color: #6b7280; 100 | padding: 0.75rem 0; 101 | font-size: 0.875rem; 102 | font-weight: 500; 103 | border: none; 104 | background: none; 105 | transition: color 0.2s ease-in-out; 106 | cursor: pointer; 107 | } 108 | 109 | .nav-link:hover, 110 | .nav-link:focus-visible { 111 | color: #374151; 112 | } 113 | 114 | .nav-link.active { 115 | color: #374151; 116 | } 117 | 118 | .nav-link.active::after { 119 | content: ''; 120 | position: absolute; 121 | bottom: -1px; 122 | left: 0; 123 | right: 0; 124 | height: 2px; 125 | background-color: #374151; 126 | } 127 | 128 | /* Tab Panels */ 129 | [role="tabpanel"] { 130 | display: none; 131 | } 132 | 133 | [role="tabpanel"]:not(.hidden) { 134 | display: block; 135 | } 136 | 137 | /* Grid Layout */ 138 | .main-grid { 139 | display: grid; 140 | gap: 2rem; 141 | grid-template-columns: minmax(0, 1fr); 142 | margin-bottom: 2rem; 143 | } 144 | 145 | @media (min-width: 1024px) { 146 | .main-grid { 147 | grid-template-columns: minmax(0, 3fr) minmax(0, 1fr); 148 | } 149 | } 150 | 151 | /* Animations */ 152 | @keyframes fadeIn { 153 | 0% { opacity: 0; transform: translateY(10px); } 154 | 100% { opacity: 1; transform: translateY(0); } 155 | } 156 | 157 | .animate-fade-in { 158 | animation: fadeIn 0.5s ease-out forwards; 159 | } 160 | 161 | /* Utility Classes */ 162 | .sidebar-sticky { 163 | position: sticky; 164 | top: 1.5rem; 165 | } 166 | 167 | .clickable-item:hover { 168 | background-color: #f9fafb; 169 | } 170 | 171 | .query-history { 172 | max-height: 300px; 173 | overflow-y: auto; 174 | } 175 | .result-header { 176 | padding: 1rem 1.25rem; 177 | background-color: #f9fafb; 178 | border-bottom: 1px solid #e5e7eb; 179 | display: flex; 180 | align-items: center; 181 | justify-content: space-between; 182 | cursor: pointer; 183 | margin-bottom: 0.5rem; 184 | } 185 | .result-header-content { 186 | display: flex; 187 | align-items: center; 188 | gap: 0.5rem; 189 | overflow: hidden; 190 | flex-grow: 1; 191 | min-width: 0; 192 | margin-right: 1rem; 193 | transition: background-color 0.2s ease-in-out; 194 | } 195 | 196 | /* Target the right-side div holding timestamp and icon */ 197 | .result-header > div:last-child { 198 | display: flex; /* Already there via Tailwind */ 199 | align-items: center; /* Already there via Tailwind */ 200 | gap: 0.75rem; /* Match Tailwind gap-3 */ 201 | flex-shrink: 0; /* *** Crucial: Prevent shrinking *** */ 202 | } 203 | 204 | /* Style for truncated query text in header */ 205 | .result-query-truncated { 206 | display: none; /* Hidden by default (when expanded) */ 207 | white-space: nowrap; 208 | overflow: hidden; 209 | text-overflow: ellipsis; 210 | vertical-align: bottom; 211 | color: #374151; /* Match old query text color */ 212 | font-weight: 400; /* Match old query text weight */ 213 | } 214 | 215 | /* Show truncated query only when header is collapsed */ 216 | .result-header[aria-expanded="false"] .result-query-truncated { 217 | display: inline-block; /* Or block, match previous */ 218 | /* max-width needs careful tuning depending on layout, let flexbox handle for now */ 219 | } 220 | 221 | /* Styling for the Query Label itself */ 222 | .result-header span.font-medium { 223 | font-size: 1rem; 224 | color: #4b5563; 225 | flex-shrink: 0; 226 | } 227 | 228 | .result-timestamp { 229 | color: #6b7280; 230 | font-size: 0.875rem; 231 | white-space: nowrap; /* Prevent timestamp from wrapping */ 232 | } 233 | 234 | /* Ensure collapse icon itself doesn't shrink */ 235 | .result-header > div:last-child > svg { 236 | flex-shrink: 0; 237 | } 238 | 239 | /* NEW: Style for the full query content block */ 240 | .result-full-query-content { 241 | display: block; /* Shown by default (when expanded) */ 242 | } 243 | 244 | /* Hide full query content block when header is collapsed */ 245 | .result-header[aria-expanded="false"] + .result-card-content > .result-full-query-content { 246 | display: none; 247 | } 248 | 249 | .result-content { 250 | padding: 1rem 1.5rem 1.5rem 1.5rem; 251 | } 252 | .result-answer { 253 | color: #374151; 254 | line-height: 1.7; 255 | padding-top: 0.5rem; 256 | } 257 | .result-answer p { 258 | margin-bottom: 0.75rem; 259 | } 260 | .result-answer h2, .result-answer h3 { 261 | margin-bottom: 0.75rem; 262 | margin-top: 1.25rem; 263 | } 264 | .result-answer .flex.gap-2 { 265 | margin-bottom: 0.35rem; 266 | } 267 | .sidebar { 268 | width: 100%; 269 | background-color: #FAF9F5; 270 | padding: 1.5rem; 271 | border-radius: 0.5rem; 272 | border: 1px solid #f3f4f6; 273 | box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.05), 0 2px 4px -2px rgba(0, 0, 0, 0.05); 274 | } 275 | .sidebar-header { 276 | padding: 1rem; 277 | display: flex; 278 | align-items: center; 279 | justify-content: space-between; 280 | cursor: pointer; 281 | border-radius: 0.5rem 0.5rem 0 0; 282 | transition: background-color 0.2s; 283 | } 284 | 285 | /* ADDED: Style for the chevron icon */ 286 | .sidebar-header i.fas.fa-chevron-down, 287 | .sidebar-header i.fas.fa-chevron-up { 288 | transition: transform 0.3s ease-in-out; 289 | } 290 | 291 | .sidebar-header[aria-expanded="true"] i.fas.fa-chevron-down { 292 | transform: rotate(180deg); 293 | } 294 | 295 | .sidebar-content { 296 | padding: 1rem 1.25rem; 297 | /* ADDED: Subtle top border */ 298 | border-top: 1px solid #f3f4f6; 299 | border-radius: 0 0 0.5rem 0.5rem; 300 | } 301 | 302 | /* ADDED: Style for empty state text */ 303 | .sidebar-content .empty-state-message { 304 | text-align: center; 305 | padding: 1.5rem 1rem; 306 | font-size: 0.875rem; 307 | color: #9ca3af; /* gray-400 */ 308 | font-style: italic; 309 | } 310 | 311 | /* ADDED: Optional icon for empty state */ 312 | .sidebar-content .empty-state-message i { 313 | display: block; 314 | font-size: 1.5rem; 315 | margin-bottom: 0.5rem; 316 | color: #d1d5db; /* gray-300 */ 317 | } 318 | 319 | .sidebar-icon { 320 | display: inline-flex; 321 | align-items: center; 322 | justify-content: center; 323 | width: 1.5rem; 324 | height: 1.5rem; 325 | margin-right: 0.5rem; 326 | } 327 | /* Ensure proper spacing and alignment */ 328 | .query-section { 329 | margin-bottom: 1.5rem; 330 | } 331 | .results-section { 332 | margin-top: 1.5rem; 333 | } 334 | /* Fix overflow issues */ 335 | .result-content { 336 | max-width: 100%; 337 | overflow-x: auto; 338 | word-wrap: break-word; 339 | } 340 | /* Improve responsive behavior */ 341 | @media (max-width: 1023px) { 342 | .main-grid > * { 343 | width: 100%; 344 | } 345 | } 346 | /* New styles for better responsiveness */ 347 | .results-grid { 348 | display: flex; 349 | flex-direction: column; 350 | gap: 1.5rem; 351 | } 352 | .recent-query-item { 353 | position: relative; 354 | padding-left: 24px; 355 | } 356 | 357 | .recent-query-item::before { 358 | content: "•"; 359 | position: absolute; 360 | left: 8px; 361 | color: #6b7280; 362 | } 363 | 364 | .example-query::before { 365 | content: "→"; 366 | margin-right: 8px; 367 | color: #9CA3AF; 368 | } 369 | .answer-section { 370 | display: block; 371 | width: 100%; 372 | } 373 | /* Workflow Table Responsiveness */ 374 | #workflow-table-container table { 375 | table-layout: fixed; 376 | width: 100%; 377 | } 378 | #workflow-table-container td { 379 | word-break: break-word; 380 | vertical-align: top; 381 | } 382 | /* Adjust column widths for 3 cols + button */ 383 | #workflow-table-container th:nth-child(1), 384 | #workflow-table-container td:nth-child(1) { /* Timestamp */ 385 | width: 15%; 386 | } 387 | #workflow-table-container th:nth-child(2), 388 | #workflow-table-container td:nth-child(2) { /* Prompt */ 389 | width: 25%; 390 | } 391 | #workflow-table-container th:nth-child(3), 392 | #workflow-table-container td:nth-child(3) { /* Answer */ 393 | width: 45%; 394 | } 395 | #workflow-table-container th:nth-child(4), 396 | #workflow-table-container td:nth-child(4) { /* Actions */ 397 | width: 15%; 398 | text-align: right; 399 | } 400 | 401 | #workflow-table-container td div.line-clamp-3 { 402 | display: -webkit-box; 403 | -webkit-line-clamp: 3; 404 | -webkit-box-orient: vertical; 405 | overflow: hidden; 406 | text-overflow: ellipsis; 407 | } 408 | 409 | .history-delete-btn { 410 | transition: color 0.2s, background-color 0.2s; 411 | } 412 | 413 | /* Modal Styles */ 414 | .modal-overlay { 415 | transition: opacity 0.3s ease; 416 | } 417 | .modal-content { 418 | transition: transform 0.3s ease; 419 | } 420 | /* Suggestion Buttons */ 421 | .suggestion-btn { 422 | display: inline-block; 423 | background-color: #f3f4f6; 424 | color: #374151; 425 | border: 1px solid #e5e7eb; 426 | padding: 0.5rem 1rem; 427 | margin: 0.25rem; 428 | border-radius: 0.375rem; 429 | font-size: 0.875rem; 430 | cursor: pointer; 431 | transition: background-color 0.2s, border-color 0.2s; 432 | } 433 | .suggestion-btn:hover { 434 | background-color: #e5e7eb; 435 | border-color: #d1d5db; 436 | } 437 | /* Suggestion Cards - UPDATED STYLES */ 438 | .suggestion-card { 439 | display: inline-flex; 440 | align-items: center; 441 | background-color: white; 442 | border: 1px solid #e5e7eb; 443 | padding: 0.75rem 1.25rem; 444 | border-radius: 0.5rem; 445 | cursor: pointer; 446 | transition: border-color 0.2s, background-color 0.2s, box-shadow 0.2s ease-in-out, transform 0.2s ease-in-out; 447 | font-size: 0.875rem; 448 | color: #374151; 449 | box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05); 450 | flex-grow: 0; 451 | flex-shrink: 0; 452 | } 453 | .suggestion-card:hover, 454 | .suggestion-card:focus-visible { 455 | border-color: #d1d5db; 456 | background-color: #f9fafb; 457 | box-shadow: 0 4px 8px rgba(0,0,0,0.08); 458 | transform: translateY(-2px); 459 | } 460 | .suggestion-card i { 461 | margin-right: 0.75rem; 462 | color: #6b7280; 463 | width: 1.1em; 464 | text-align: center; 465 | } 466 | /* Code Block Copy Button */ 467 | .code-block-wrapper { 468 | position: relative; 469 | } 470 | .copy-code-btn { 471 | position: absolute; 472 | top: 0.75rem; 473 | right: 0.75rem; 474 | padding: 0.25rem 0.5rem; 475 | background-color: #4a5568; 476 | color: #e2e8f0; 477 | border: none; 478 | border-radius: 0.375rem; 479 | cursor: pointer; 480 | font-size: 0.75rem; 481 | opacity: 0; 482 | transition: opacity 0.2s, background-color 0.2s; 483 | } 484 | .code-block-wrapper:hover .copy-code-btn { 485 | opacity: 1; 486 | } 487 | .copy-code-btn:hover, 488 | .copy-code-btn:focus-visible { 489 | background-color: #2d3748; 490 | outline: none; /* Prevent default focus outline if custom is not needed */ 491 | } 492 | .copy-code-btn .fa-check { 493 | color: #68d391; 494 | } 495 | /* Preset Buttons Styling */ 496 | .preset-btn { 497 | display: inline-flex; 498 | align-items: center; 499 | padding: 0.375rem 0.75rem; 500 | border: 1px solid #d1d5db; 501 | border-radius: 0.375rem; 502 | background-color: white; 503 | font-size: 0.875rem; 504 | font-weight: 500; 505 | color: #374151; 506 | box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05); 507 | cursor: pointer; 508 | transition: background-color 0.2s ease-in-out, border-color 0.2s ease-in-out, box-shadow 0.2s ease-in-out; 509 | } 510 | .preset-btn:hover, 511 | .preset-btn:focus-visible { 512 | background-color: #f9fafb; 513 | border-color: #9ca3af; 514 | outline: none; 515 | } 516 | 517 | /* Remove generic active style */ 518 | /* 519 | .preset-btn.active { 520 | background-color: #eef2ff; 521 | border-color: #a5b4fc; 522 | color: #3730a3; 523 | font-weight: 600; 524 | box-shadow: inset 0 1px 1px rgba(0,0,0,0.05); 525 | } 526 | */ 527 | 528 | /* Specific Active Preset Styles */ 529 | .preset-btn[data-preset="personal_assistant"].active { 530 | background-color: #e0e7ff; /* indigo-100 */ 531 | border-color: #a5b4fc; /* indigo-300 */ 532 | color: #3730a3; /* indigo-800 */ 533 | font-weight: 600; 534 | box-shadow: inset 0 1px 1px rgba(0,0,0,0.05); 535 | } 536 | 537 | .preset-btn[data-preset="visual_analyst"].active { 538 | background-color: #d1fae5; /* green-100 */ 539 | border-color: #6ee7b7; /* green-300 */ 540 | color: #065f46; /* green-800 */ 541 | font-weight: 600; 542 | box-shadow: inset 0 1px 1px rgba(0,0,0,0.05); 543 | } 544 | 545 | .preset-btn[data-preset="research_assistant"].active { 546 | background-color: #fef3c7; /* amber-100 */ 547 | border-color: #fcd34d; /* amber-300 */ 548 | color: #92400e; /* amber-800 */ 549 | font-weight: 600; 550 | box-shadow: inset 0 1px 1px rgba(0,0,0,0.05); 551 | } 552 | 553 | .preset-btn[data-preset="technical_guide"].active { 554 | background-color: #e0f2fe; /* sky-100 */ 555 | border-color: #7dd3fc; /* sky-300 */ 556 | color: #075985; /* sky-800 */ 557 | font-weight: 600; 558 | box-shadow: inset 0 1px 1px rgba(0,0,0,0.05); 559 | } 560 | 561 | .result-header i.fas:not(.hiw-chevron) { 562 | margin-right: 0.5rem; 563 | } 564 | 565 | /* Specific adjustments for How It Works accordion */ 566 | #how-it-works-panel .result-header { 567 | background-color: #f9fafb; 568 | border-bottom: 1px solid #e5e7eb; 569 | } 570 | 571 | #how-it-works-panel .result-header:hover { 572 | background-color: #f3f4f6; 573 | } 574 | 575 | #how-it-works-panel .result-content { 576 | border-top: none; 577 | background-color: #ffffff; 578 | } 579 | 580 | #how-it-works-panel .code-block-wrapper pre { 581 | background-color: #282c34; 582 | padding: 0; 583 | border-radius: 0.5rem; 584 | overflow: hidden; 585 | } 586 | 587 | #how-it-works-panel .code-block-wrapper pre code { 588 | padding: 1rem 1.25rem !important; 589 | } 590 | 591 | #how-it-works-panel a i.fa-external-link-alt { 592 | opacity: 0.6; 593 | transition: opacity 0.2s; 594 | } 595 | 596 | #how-it-works-panel a:hover i.fa-external-link-alt { 597 | opacity: 1; 598 | } 599 | 600 | /* Make sidebar delete-all buttons red on hover */ 601 | .sidebar-header button[title^="Delete all"] .fa-trash-alt { 602 | transition: color 0.2s; 603 | } 604 | 605 | .sidebar-header button[title^="Delete all"]:hover .fa-trash-alt { 606 | color: #ef4444; 607 | } 608 | 609 | /* NEW: Prevent hover effect on non-collapsible Upload header */ 610 | #upload-header { 611 | cursor: default; 612 | padding-top: 0.6rem; 613 | padding-bottom: 0.6rem; 614 | padding-left: 1rem; 615 | padding-right: 1rem; 616 | margin-bottom: 0.25rem; 617 | } 618 | 619 | /* Target the first paragraph inside the upload section */ 620 | #upload-section > p:first-child { 621 | margin-bottom: 0.75rem; 622 | } 623 | 624 | /* Added save-snippet-btn */ 625 | .save-snippet-btn { 626 | position: absolute; 627 | top: 0.75rem; 628 | right: 5.75rem; 629 | padding: 0.25rem 0.5rem; 630 | background-color: #38a169; 631 | color: #e6fffa; 632 | border: none; 633 | border-radius: 0.375rem; 634 | cursor: pointer; 635 | font-size: 0.75rem; 636 | opacity: 0; 637 | transition: opacity 0.2s, background-color 0.2s; 638 | } 639 | 640 | .code-block-wrapper:hover .save-snippet-btn { 641 | opacity: 1; 642 | } 643 | 644 | .save-snippet-btn:hover, 645 | .save-snippet-btn:focus-visible { 646 | background-color: #2f855a; 647 | outline: none; 648 | } 649 | 650 | .save-snippet-btn .fa-check { 651 | color: #f0fff4; 652 | } 653 | .save-snippet-btn .fa-exclamation-triangle { 654 | color: #fed7d7; 655 | } 656 | 657 | .copy-code-btn .fa-check { 658 | color: #68d391; 659 | } 660 | 661 | /* Snippet Display Styles */ 662 | #snippets-display-area .code-block-wrapper pre { 663 | max-height: 200px; 664 | overflow-y: auto; 665 | } 666 | 667 | #snippets-display-area .copy-code-btn { 668 | right: 0.5rem; 669 | } 670 | 671 | .snippet-delete-btn i.fa-trash-alt { 672 | transition: color 0.2s; 673 | } 674 | .snippet-delete-btn:hover i.fa-trash-alt { 675 | color: #c53030; 676 | } 677 | 678 | /* NEW: Inline Save Selection Button */ 679 | .inline-save-selection-btn { 680 | display: inline-block; 681 | margin-left: 1rem; 682 | margin-top: 0.5rem; 683 | margin-bottom: 0.5rem; 684 | padding: 0.25rem 0.6rem; 685 | font-size: 0.8rem; 686 | font-weight: 500; 687 | background-color: #ebf4ff; 688 | color: #2c5282; 689 | border: 1px solid #bee3f8; 690 | border-radius: 0.375rem; 691 | cursor: pointer; 692 | transition: background-color 0.2s, border-color 0.2s; 693 | vertical-align: middle; 694 | } 695 | 696 | .inline-save-selection-btn:hover, 697 | .inline-save-selection-btn:focus-visible { 698 | background-color: #bee3f8; 699 | border-color: #90cdf4; 700 | outline: none; 701 | } 702 | 703 | .inline-save-selection-btn i { 704 | margin-right: 0.3rem; 705 | } 706 | 707 | .inline-save-selection-btn:disabled { 708 | cursor: default; 709 | opacity: 0.7; 710 | } 711 | 712 | /* NEW: Tools Dropdown Button Styles */ 713 | #tools-dropdown-button { 714 | padding: 0; 715 | background-color: transparent; 716 | border: 1px solid #e5e7eb; 717 | transition: border-color 0.2s, color 0.2s; 718 | } 719 | 720 | #tools-dropdown-button:hover, 721 | #tools-dropdown-button:focus-visible { 722 | border-color: #d1d5db; 723 | color: #374151; 724 | outline: none; 725 | } 726 | 727 | /* NEW: Tools Dropdown Content Styles */ 728 | #tools-dropdown-content { 729 | background-color: #fff; 730 | border: 1px solid #e5e7eb; 731 | border-radius: 0.5rem; 732 | box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06); 733 | z-index: 20; 734 | max-height: 15rem; 735 | overflow-y: auto; 736 | width: 18rem; 737 | } 738 | 739 | /* NEW: Styling for individual tool items within the dropdown */ 740 | #tools-dropdown-content .tool-item { 741 | padding: 0.75rem 1rem; 742 | border-bottom: 1px solid #f3f4f6; 743 | } 744 | 745 | #tools-dropdown-content .tool-item:last-child { 746 | border-bottom: none; 747 | } 748 | 749 | #tools-dropdown-content .tool-item .tool-name { 750 | font-weight: 500; 751 | color: #374151; 752 | font-size: 0.875rem; 753 | display: flex; 754 | align-items: center; 755 | margin-bottom: 0.25rem; 756 | } 757 | 758 | #tools-dropdown-content .tool-item .tool-name i { 759 | margin-right: 0.5rem; 760 | color: #6b7280; 761 | width: 1rem; 762 | text-align: center; 763 | } 764 | 765 | #tools-dropdown-content .tool-item .tool-description { 766 | font-size: 0.75rem; 767 | color: #6b7280; 768 | padding-left: 0; 769 | } 770 | 771 | /* Hide scrollbar for Webkit browsers */ 772 | #tools-dropdown-content::-webkit-scrollbar { 773 | display: none; 774 | } 775 | 776 | /* Hide scrollbar for IE, Edge and Firefox */ 777 | #tools-dropdown-content { 778 | -ms-overflow-style: none; /* IE and Edge */ 779 | scrollbar-width: none; /* Firefox */ 780 | } 781 | 782 | /* NEW: History Detail Modal Styles */ 783 | #modal-metadata { 784 | overflow-x: auto; 785 | white-space: pre; 786 | background-color: #f3f4f6; 787 | padding: 1rem 1.25rem; 788 | border-radius: 0.375rem; 789 | font-size: 0.8rem; 790 | color: #4b5563; 791 | line-height: 1.6; 792 | } 793 | 794 | #modal-query { 795 | white-space: pre-wrap; 796 | word-break: break-word; 797 | } 798 | 799 | #modal-response .prose { 800 | line-height: 1.7; 801 | } 802 | 803 | #modal-response .prose p { 804 | margin-bottom: 0.75rem; 805 | } 806 | 807 | /* NEW: Styles for Answer subsection headers */ 808 | .result-subsection-header { 809 | padding: 0.75rem 0; 810 | margin-bottom: 0.5rem; 811 | cursor: pointer; 812 | display: flex; 813 | align-items: center; 814 | justify-content: space-between; 815 | } 816 | 817 | .result-subsection-header > div:first-child { 818 | display: flex; 819 | align-items: center; 820 | gap: 0.5rem; 821 | } 822 | 823 | .result-subsection-header i { 824 | font-size: 1rem; 825 | color: #6b7280; 826 | width: 1.1em; 827 | text-align: center; 828 | margin-right: 0; 829 | } 830 | 831 | .result-subsection-header h4 { 832 | font-size: 1rem; 833 | font-weight: 500; 834 | color: #4b5563; 835 | margin: 0; 836 | line-height: inherit; 837 | } 838 | 839 | .result-subsection-header > svg { 840 | flex-shrink: 0; 841 | } 842 | 843 | .result-content { 844 | padding: 1rem 1.5rem 1.5rem 1.5rem; 845 | } 846 | 847 | #results .result-answer { 848 | padding-top: 0; 849 | color: #374151; 850 | } 851 | 852 | .result-header span.font-medium { 853 | font-size: 1rem; 854 | color: #4b5563; 855 | flex-shrink: 0; 856 | } 857 | 858 | #results .result-subsection-header:has(h4:contains('Answer')) { 859 | cursor: default; 860 | } 861 | 862 | #results .result-answer p, 863 | #results .result-answer ul, 864 | #results .result-answer ol, 865 | #results .result-answer pre, 866 | #results .result-answer blockquote, 867 | #results .result-answer h1, 868 | #results .result-answer h2, 869 | #results .result-answer h3, 870 | #results .result-answer h4, 871 | #results .result-answer h5, 872 | #results .result-answer h6 { 873 | margin-bottom: 0.75rem; 874 | } 875 | 876 | #results .result-answer h1 { margin-top: 1.5rem; } 877 | #results .result-answer h2 { margin-top: 1.25rem; } 878 | #results .result-answer h3 { margin-top: 1rem; } 879 | 880 | #results .result-answer li { 881 | margin-bottom: 0.25rem; 882 | } 883 | 884 | #results .result-answer p > strong:only-child { 885 | display: block; 886 | margin-bottom: 0.5rem; 887 | font-weight: 600; 888 | } 889 | 890 | #results .result-answer ul { 891 | list-style-type: disc; 892 | padding-left: 1.5rem; 893 | } 894 | 895 | #results .result-answer ol { 896 | list-style-type: decimal; 897 | padding-left: 1.5rem; 898 | } 899 | 900 | .code-block-wrapper pre { 901 | background-color: #282c34; 902 | padding: 0; 903 | margin: 0.75rem 0; 904 | border-radius: 0.5rem; 905 | overflow-x: auto; 906 | } 907 | 908 | #how-it-works-panel .code-block-wrapper pre { 909 | background-color: #282c34; 910 | padding: 0; 911 | border-radius: 0.5rem; 912 | overflow: hidden; 913 | } 914 | 915 | #memory-display-area .code-block-wrapper pre { 916 | max-height: 250px; 917 | overflow-y: auto; 918 | background-color: #282c34; 919 | border-radius: 0.5rem; 920 | margin: 0.5rem 0; 921 | padding: 0; 922 | } 923 | 924 | .answer-actions { 925 | display: none; 926 | } 927 | 928 | .result-header[aria-expanded="true"] + .result-card-content .answer-actions { 929 | display: flex; 930 | } 931 | 932 | .action-btn { 933 | display: inline-flex; /* Use inline-flex for alignment */ 934 | align-items: center; /* Center content vertically */ 935 | justify-content: center; /* Center content horizontally */ 936 | background-color: transparent; 937 | border: 1px solid #e5e7eb; /* gray-200 */ 938 | color: #6b7280; /* gray-500 */ 939 | padding: 0.3rem 0.6rem; /* Adjusted padding slightly */ 940 | border-radius: 0.375rem; /* rounded-md */ 941 | cursor: pointer; 942 | transition: all 0.2s ease-in-out; 943 | font-size: 0.8rem; 944 | line-height: 1; 945 | white-space: nowrap; /* Prevent text wrapping */ 946 | } 947 | 948 | /* Hover and Focus for .action-btn */ 949 | .action-btn:hover, 950 | .action-btn:focus-visible { 951 | background-color: #f3f4f6; /* gray-100 */ 952 | border-color: #d1d5db; /* gray-300 */ 953 | color: #374151; /* gray-700 */ 954 | outline: none; 955 | } 956 | 957 | /* Style for icons inside action buttons */ 958 | .action-btn i { 959 | margin-right: 0.3rem; /* Space between icon and text if any */ 960 | font-size: 0.9em; /* Slightly smaller icon */ 961 | line-height: 1; /* Ensure icon aligns well */ 962 | } 963 | /* Remove margin if only icon exists */ 964 | .action-btn i:only-child { 965 | margin-right: 0; 966 | } 967 | 968 | .action-btn .fa-check { 969 | color: #38a169; 970 | } 971 | 972 | .result-subsection-header i { 973 | flex-shrink: 0; 974 | } 975 | 976 | #query { 977 | padding-bottom: 3.5rem; 978 | overflow-y: auto; 979 | max-height: 300px; 980 | transition: height 0.1s ease-out; 981 | } 982 | 983 | @keyframes pulse-border { 984 | 0%, 100% { 985 | border-color: #e5e7eb; 986 | } 987 | 50% { 988 | border-color: #d1d5db; 989 | } 990 | } 991 | 992 | .loading-tip-pulse { 993 | animation: pulse-border 2s cubic-bezier(0.4, 0, 0.6, 1) infinite; 994 | } 995 | 996 | .mode-toggle-bg { 997 | transition: background-color 0.2s ease-in-out; 998 | position: relative; 999 | display: inline-block; 1000 | } 1001 | .mode-toggle-bg::after { 1002 | transition: transform 0.2s ease-in-out; 1003 | content: ''; 1004 | position: absolute; 1005 | top: 2px; 1006 | left: 2px; 1007 | background-color: white; 1008 | border: 1px solid #d1d5db; 1009 | border-radius: 9999px; 1010 | height: 1rem; 1011 | width: 1rem; 1012 | } 1013 | 1014 | input#mode-toggle:checked + .mode-toggle-bg { 1015 | background-color: #374151; 1016 | } 1017 | 1018 | input#mode-toggle:checked + .mode-toggle-bg::after { 1019 | transform: translateX(100%); 1020 | border-color: white; 1021 | } 1022 | 1023 | .sidebar-file-list { 1024 | max-height: 200px; 1025 | overflow-y: auto; 1026 | padding-right: 0.5rem; 1027 | } 1028 | 1029 | .sidebar-grid-container { 1030 | display: grid; 1031 | grid-template-columns: repeat(auto-fill, minmax(80px, 1fr)); 1032 | gap: 0.75rem; 1033 | max-height: 300px; 1034 | overflow-y: auto; 1035 | padding-right: 0.5rem; 1036 | } 1037 | 1038 | .sidebar-grid-item { 1039 | position: relative; 1040 | padding: 4px; 1041 | border-radius: 4px; 1042 | transition: background-color 0.2s ease-in-out; 1043 | } 1044 | 1045 | .sidebar-grid-item img { 1046 | width: 100%; 1047 | height: 100%; 1048 | object-fit: cover; 1049 | display: block; 1050 | } 1051 | 1052 | .sidebar-grid-item .filename-overlay { 1053 | position: absolute; 1054 | bottom: 0; 1055 | left: 0; 1056 | right: 0; 1057 | background-color: rgba(0, 0, 0, 0.6); 1058 | color: white; 1059 | font-size: 0.65rem; 1060 | padding: 2px 4px; 1061 | text-align: center; 1062 | white-space: nowrap; 1063 | overflow: hidden; 1064 | text-overflow: ellipsis; 1065 | opacity: 0; 1066 | /* MODIFIED: Added visibility to transition */ 1067 | transition: opacity 0.2s ease-in-out, visibility 0.2s ease-in-out; 1068 | } 1069 | 1070 | .sidebar-grid-item:hover .filename-overlay { 1071 | opacity: 1; 1072 | visibility: visible; /* Ensure visibility changes with opacity */ 1073 | } 1074 | 1075 | .sidebar-grid-item:hover { 1076 | background-color: #f9fafb; 1077 | /* ADDED: Subtle border highlight on hover */ 1078 | border: 1px solid #e5e7eb; 1079 | } 1080 | 1081 | .sidebar-grid-item .sidebar-file-delete-btn { 1082 | position: absolute; 1083 | top: 2px; 1084 | right: 2px; 1085 | width: 24px; 1086 | height: 24px; 1087 | padding: 0; 1088 | border: none; 1089 | background-color: #f3f4f6; 1090 | color: #ef4444; 1091 | border-radius: 50%; 1092 | cursor: pointer; 1093 | display: flex; 1094 | align-items: center; 1095 | justify-content: center; 1096 | opacity: 0; 1097 | visibility: hidden; 1098 | transition: opacity 0.2s ease-in-out, visibility 0.2s ease-in-out, background-color 0.2s ease-in-out; 1099 | z-index: 10; 1100 | box-shadow: 0 1px 3px rgba(0,0,0,0.1); 1101 | } 1102 | 1103 | .sidebar-grid-item:hover .sidebar-file-delete-btn { 1104 | opacity: 1; 1105 | visibility: visible; 1106 | } 1107 | 1108 | .sidebar-grid-item .sidebar-file-delete-btn:hover { 1109 | background-color: #dc2626; 1110 | color: white; 1111 | } 1112 | 1113 | .sidebar-grid-item .sidebar-file-delete-btn i { 1114 | font-size: 0.75rem; 1115 | line-height: 1; 1116 | } 1117 | 1118 | .sidebar-text-item { 1119 | position: relative; 1120 | overflow-x: hidden; 1121 | list-style-type: none; 1122 | padding: 0.5rem 0.75rem; 1123 | border-radius: 0.375rem; 1124 | cursor: default; 1125 | transition: background-color 0.2s ease-in-out; 1126 | border-bottom: 1px solid transparent; 1127 | } 1128 | 1129 | /* ADDED: File type icon style */ 1130 | .sidebar-text-item .file-icon { 1131 | margin-right: 0.6rem; /* Space between icon and text */ 1132 | color: #9ca3af; /* gray-400 */ 1133 | width: 1em; /* Ensure consistent width */ 1134 | text-align: center; 1135 | } 1136 | 1137 | .sidebar-text-item:hover, 1138 | .sidebar-text-item:focus-visible { 1139 | background-color: #f9fafb; /* gray-50 */ 1140 | /* ADDED: Show border on hover */ 1141 | border-bottom-color: #e5e7eb; /* gray-200 */ 1142 | outline: none; 1143 | } 1144 | 1145 | .sidebar-text-item .truncate { 1146 | margin-right: 30px; 1147 | } 1148 | 1149 | #sidebar-search-container { 1150 | padding: 0 1rem 0.75rem 1rem; 1151 | } 1152 | 1153 | #sidebar-search { 1154 | width: 100%; 1155 | padding: 0.5rem 0.75rem; 1156 | border: 1px solid #d1d5db; 1157 | border-radius: 0.375rem; 1158 | font-size: 0.875rem; 1159 | box-shadow: inset 0 1px 2px rgba(0,0,0,0.05); 1160 | } 1161 | 1162 | #sidebar-search:focus { 1163 | outline: none; 1164 | border-color: #a5b4fc; 1165 | box-shadow: 0 0 0 2px rgba(199, 210, 254, 0.5); 1166 | } 1167 | 1168 | .tooltip-container { 1169 | position: relative; 1170 | } 1171 | 1172 | .custom-tooltip { 1173 | visibility: hidden; 1174 | opacity: 0; 1175 | position: absolute; 1176 | background-color: #2d3748; /* gray-800 */ 1177 | color: #edf2f7; /* gray-200 */ 1178 | text-align: center; 1179 | padding: 6px 10px; 1180 | border-radius: 0.375rem; /* rounded-md */ 1181 | font-size: 0.75rem; /* text-xs */ 1182 | font-weight: 500; 1183 | white-space: nowrap; 1184 | z-index: 1000; /* Increased z-index */ 1185 | 1186 | /* Align to the left edge */ 1187 | bottom: 115%; 1188 | left: 0; 1189 | 1190 | transition: opacity 0.2s ease-in-out; 1191 | } 1192 | 1193 | .custom-tooltip::after { 1194 | content: ""; 1195 | position: absolute; 1196 | top: 100%; 1197 | /* Adjust arrow position slightly if needed now that tooltip isn't centered */ 1198 | left: 10px; /* Example: position arrow near the start */ 1199 | margin-left: -5px; /* Keep centering the arrow itself */ 1200 | border-width: 5px; 1201 | border-style: solid; 1202 | border-color: #2d3748 transparent transparent transparent; 1203 | } 1204 | 1205 | .tooltip-container:hover .custom-tooltip { 1206 | visibility: visible; 1207 | opacity: 1; 1208 | } 1209 | 1210 | .suggestion-link { 1211 | display: block; 1212 | width: 100%; 1213 | padding: 4px 0; 1214 | margin-bottom: 4px; 1215 | text-align: left; 1216 | font-size: 0.875rem; 1217 | color: #D97757; 1218 | background-color: transparent; 1219 | border: none; 1220 | border-radius: 0; 1221 | cursor: pointer; 1222 | transition: color 0.2s, text-decoration 0.2s; 1223 | } 1224 | 1225 | .suggestion-link:hover, 1226 | .suggestion-link:focus-visible { 1227 | background-color: transparent; 1228 | border-color: transparent; 1229 | color: #c2410c; 1230 | text-decoration: underline; 1231 | outline: none; 1232 | } 1233 | 1234 | .result-answer a { 1235 | color: #d97706; 1236 | text-decoration: none; 1237 | font-weight: 500; 1238 | transition: color 0.2s, text-decoration 0.2s; 1239 | } 1240 | 1241 | .result-answer a:hover { 1242 | color: #b45309; 1243 | text-decoration: underline; 1244 | } 1245 | 1246 | .sidebar-file-delete-btn { 1247 | position: absolute; 1248 | right: 0.75rem; 1249 | top: 50%; 1250 | transform: translateY(-50%); 1251 | opacity: 0; 1252 | visibility: hidden; 1253 | padding: 0.15rem; 1254 | line-height: 1; 1255 | color: #9ca3af; 1256 | background-color: transparent; 1257 | border: none; 1258 | border-radius: 50%; 1259 | cursor: pointer; 1260 | transition: opacity 0.2s ease-in-out, visibility 0.2s ease-in-out, color 0.2s, background-color 0.2s; 1261 | } 1262 | 1263 | .sidebar-text-item:hover .sidebar-file-delete-btn { 1264 | opacity: 1; 1265 | visibility: visible; 1266 | } 1267 | 1268 | .sidebar-file-delete-btn:hover { 1269 | color: #ef4444; 1270 | background-color: #fee2e2; 1271 | } 1272 | 1273 | .sidebar-grid-item .sidebar-file-delete-btn { 1274 | position: absolute; 1275 | top: 4px; 1276 | right: 4px; 1277 | background-color: rgba(255, 255, 255, 0.7); 1278 | padding: 2px; 1279 | border-radius: 50%; 1280 | opacity: 0; 1281 | visibility: hidden; 1282 | /* MODIFIED: Ensure all transitions are present */ 1283 | transition: opacity 0.2s ease-in-out, visibility 0.2s ease-in-out, background-color 0.2s, color 0.2s; 1284 | z-index: 5; 1285 | } 1286 | 1287 | .sidebar-grid-item:hover .sidebar-file-delete-btn { 1288 | opacity: 1; 1289 | visibility: visible; 1290 | } 1291 | 1292 | .sidebar-grid-item .sidebar-file-delete-btn:hover { 1293 | background-color: rgba(255, 255, 255, 0.9); 1294 | color: #ef4444; 1295 | } 1296 | 1297 | .sidebar-file-delete-btn i { 1298 | font-size: 0.7rem; 1299 | vertical-align: middle; 1300 | } 1301 | 1302 | .modal-flex-layout { 1303 | display: flex; 1304 | flex-wrap: wrap; 1305 | gap: 1.5rem; 1306 | } 1307 | 1308 | .modal-flex-column { 1309 | flex: 1 1 0%; 1310 | min-width: 300px; 1311 | } 1312 | 1313 | .modal-section-header { 1314 | font-size: 0.75rem; 1315 | font-weight: 600; 1316 | color: #4b5563; 1317 | text-transform: uppercase; 1318 | letter-spacing: 0.05em; 1319 | margin-bottom: 0.5rem; 1320 | border-bottom: 1px solid #e5e7eb; 1321 | padding-bottom: 0.25rem; 1322 | } 1323 | 1324 | .modal-content-box { 1325 | background-color: #f9fafb; 1326 | padding: 0.75rem 1rem; 1327 | border: 1px solid #e5e7eb; 1328 | border-radius: 0.375rem; 1329 | font-size: 0.875rem; 1330 | line-height: 1.6; 1331 | } 1332 | 1333 | #modal-response.modal-content-box { 1334 | padding: 1rem 1.25rem; 1335 | } 1336 | 1337 | #modal-query.modal-content-box { 1338 | background-color: transparent; 1339 | border: none; 1340 | padding: 0; 1341 | font-size: 0.875rem; 1342 | } 1343 | 1344 | #confirmation-modal #confirmation-message { 1345 | line-height: 1.6; 1346 | } 1347 | 1348 | *:focus-visible { 1349 | outline: 2px solid #fbbf24; /* amber-400 */ 1350 | outline-offset: 2px; 1351 | border-radius: 2px; /* Optional: slightly round the outline */ 1352 | } 1353 | *:focus:not(:focus-visible) { 1354 | outline: none; 1355 | } 1356 | 1357 | .sidebar-header.sidebar-toggle-button:hover, 1358 | .sidebar-header.sidebar-toggle-button:focus-visible { 1359 | background-color: #f9fafb; 1360 | outline: none; 1361 | } 1362 | 1363 | /* === Improved Table Styling for Results === */ 1364 | #results .prose table { 1365 | width: 100%; 1366 | margin-top: 1.5rem; /* 24px */ 1367 | margin-bottom: 1.5rem; /* 24px */ 1368 | border-collapse: collapse; 1369 | box-shadow: 0 1px 2px 0 rgb(0 0 0 / 0.05); /* shadow-sm */ 1370 | border-radius: 0.5rem; /* rounded-lg */ 1371 | border: 1px solid #e5e7eb; /* gray-200 */ 1372 | } 1373 | 1374 | #results .prose thead { 1375 | background-color: #f9fafb; /* gray-50 */ 1376 | } 1377 | 1378 | #results .prose th { 1379 | padding-left: 1rem; /* px-4 */ 1380 | padding-right: 1rem; /* px-4 */ 1381 | padding-top: 0.75rem; /* py-3 */ 1382 | padding-bottom: 0.75rem; /* py-3 */ 1383 | text-align: left; 1384 | font-size: 0.75rem; /* text-xs */ 1385 | line-height: 1rem; 1386 | font-weight: 600; /* font-semibold */ 1387 | color: #4b5563; /* text-gray-600 */ 1388 | text-transform: uppercase; 1389 | letter-spacing: 0.05em; /* tracking-wider */ 1390 | border-bottom-width: 2px; 1391 | border-bottom-color: #e5e7eb; /* border-gray-200 */ 1392 | } 1393 | 1394 | #results .prose td { 1395 | padding-left: 1rem; /* px-4 */ 1396 | padding-right: 1rem; /* px-4 */ 1397 | padding-top: 0.75rem; /* py-3 */ 1398 | padding-bottom: 0.75rem; /* py-3 */ 1399 | font-size: 0.875rem; /* text-sm */ 1400 | line-height: 1.25rem; 1401 | color: #374151; /* text-gray-700 */ 1402 | border-bottom-width: 1px; 1403 | border-bottom-color: #f3f4f6; /* border-gray-100 */ 1404 | vertical-align: top; 1405 | } 1406 | 1407 | /* Zebra-striping for better readability */ 1408 | #results .prose tbody tr:nth-child(even) { 1409 | background-color: rgba(249, 250, 251, 0.5); /* bg-gray-50/50 */ 1410 | } 1411 | 1412 | #results .prose tbody tr:last-child td { 1413 | border-bottom-width: 0px; 1414 | } 1415 | 1416 | /* Remove prose's default quotes if they interfere */ 1417 | #results .prose table td::before, 1418 | #results .prose table td::after, 1419 | #results .prose table th::before, 1420 | #results .prose table th::after { 1421 | content: none !important; 1422 | } 1423 | /* === End Table Styling === */ 1424 | 1425 | /* === Improved Header Styling for Results === */ 1426 | #results .prose h1 { /* Less likely to appear, keep simple */ 1427 | font-weight: 700; /* bold */ 1428 | color: #111827; /* gray-900 */ 1429 | margin-top: 1.5rem; 1430 | margin-bottom: 1rem; 1431 | } 1432 | 1433 | #results .prose h2 { 1434 | font-weight: 600; /* semibold */ 1435 | color: #1f2937; /* gray-800 */ 1436 | margin-top: 2rem; /* Add more space above h2 */ 1437 | margin-bottom: 0.75rem; 1438 | padding-bottom: 0.25rem; 1439 | border-bottom: 1px solid #e5e7eb; /* Add subtle separator */ 1440 | } 1441 | 1442 | #results .prose h3 { 1443 | font-weight: 600; /* semibold */ 1444 | color: #374151; /* gray-700 */ 1445 | margin-top: 1.5rem; /* Space above h3 */ 1446 | margin-bottom: 0.5rem; 1447 | } 1448 | 1449 | #results .prose h4 { 1450 | font-weight: 600; /* semibold */ 1451 | color: #4b5563; /* gray-600 */ 1452 | margin-top: 1.25rem; /* Space above h4 */ 1453 | margin-bottom: 0.5rem; 1454 | } 1455 | /* === End Header Styling === */ 1456 | 1457 | /* === NEW: Sidebar Empty State Styles === */ 1458 | .sidebar-empty-state { 1459 | text-align: center; 1460 | padding: 1.5rem 1rem; 1461 | } 1462 | 1463 | .sidebar-empty-icon { 1464 | display: block; 1465 | font-size: 1.75rem; /* Slightly larger icon */ 1466 | margin-bottom: 0.75rem; /* More space below icon */ 1467 | color: #d1d5db; /* gray-300 */ 1468 | opacity: 0.8; 1469 | } 1470 | 1471 | .sidebar-empty-text { 1472 | font-size: 0.8rem; /* Slightly smaller text */ 1473 | color: #9ca3af; /* gray-400 */ 1474 | font-style: italic; 1475 | } 1476 | /* === END: Sidebar Empty State Styles === */ 1477 | -------------------------------------------------------------------------------- /static/image/overview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pixeltable/pixelbot/3b5b426007fa5f45beb3d6f499ae98ef460b366f/static/image/overview.gif -------------------------------------------------------------------------------- /static/image/pixelbot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pixeltable/pixelbot/3b5b426007fa5f45beb3d6f499ae98ef460b366f/static/image/pixelbot.png -------------------------------------------------------------------------------- /static/image/pixeltable-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pixeltable/pixelbot/3b5b426007fa5f45beb3d6f499ae98ef460b366f/static/image/pixeltable-logo.png -------------------------------------------------------------------------------- /static/js/api.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Sends a query to the backend (Chat Mode). 3 | * Handles request abortion, timeout, and calls UI functions for updates. 4 | * @param {URLSearchParams} formData - The form data including query and optional persona_id. 5 | */ 6 | async function sendQuery(formData) { 7 | // Abort any previous request if a new one is started 8 | if (window.currentRequest) { 9 | window.currentRequest.abort(); 10 | console.log("Aborted previous request."); 11 | } 12 | 13 | const queryForDisplay = formData.get('query'); 14 | 15 | const controller = new AbortController(); 16 | window.currentRequest = controller; 17 | 18 | try { 19 | const timeoutId = setTimeout(() => { 20 | controller.abort(); 21 | console.warn("Request timed out after 90 seconds."); 22 | }, 90000); // 90 seconds timeout 23 | 24 | const response = await fetch('/query', { 25 | method: 'POST', 26 | headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, 27 | body: formData, 28 | signal: controller.signal 29 | }); 30 | 31 | clearTimeout(timeoutId); 32 | 33 | if (!response.ok) { 34 | const errorData = await response.json().catch(() => ({ error: `HTTP error! status: ${response.status}` })); 35 | throw new Error(errorData.error || `HTTP error! status: ${response.status}`); 36 | } 37 | 38 | const data = await response.json(); 39 | 40 | if (typeof hideLoadingState === 'function') hideLoadingState(); 41 | if (typeof displayResult === 'function') displayResult(queryForDisplay, data); 42 | if (typeof checkInput === 'function') checkInput(); 43 | if (typeof setLoading === 'function') setLoading(false, 'chat'); 44 | 45 | } catch (error) { 46 | console.error('Fetch Error (Query):', error); 47 | handleFetchError(error, controller, queryForDisplay, 'chat', displayResult); 48 | } finally { 49 | if (window.currentRequest === controller) { 50 | window.currentRequest = null; 51 | } 52 | } 53 | } 54 | 55 | /** 56 | * Sends an image generation request to the backend. 57 | * Handles request abortion, timeout, and calls UI functions for updates. 58 | * @param {string} prompt - The user's prompt text. 59 | */ 60 | async function sendImageGenerationRequest(prompt) { 61 | if (window.currentRequest) { 62 | window.currentRequest.abort(); 63 | console.log("Aborted previous request (Image Gen)."); 64 | } 65 | 66 | const formData = new URLSearchParams(); 67 | formData.append('prompt', prompt); 68 | const controller = new AbortController(); 69 | window.currentRequest = controller; 70 | 71 | try { 72 | const timeoutId = setTimeout(() => { 73 | controller.abort(); 74 | console.warn("Image generation request timed out after 90 seconds."); 75 | }, 90000); 76 | 77 | const response = await fetch('/generate_image', { 78 | method: 'POST', 79 | headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, 80 | body: formData, 81 | signal: controller.signal 82 | }); 83 | 84 | clearTimeout(timeoutId); 85 | 86 | const data = await response.json().catch(() => { 87 | return { error: `Image Generation Failed. Status: ${response.status}`, details: 'Could not parse server response.' }; 88 | }); 89 | 90 | if (!response.ok) { 91 | throw new Error(data.error || `HTTP error! status: ${response.status}`); 92 | } 93 | 94 | if (typeof hideLoadingState === 'function') hideLoadingState(); 95 | if (typeof displayImageResult === 'function') displayImageResult(prompt, data); 96 | if (typeof checkInput === 'function') checkInput(); 97 | if (typeof setLoading === 'function') setLoading(false, 'image'); 98 | 99 | // --- ADDED: Refresh image history data --- // 100 | if (typeof loadImageHistory === 'function') { 101 | console.log("Refreshing image history after successful generation..."); 102 | await loadImageHistory(); 103 | } 104 | // --- END ADDED --- // 105 | 106 | } catch (error) { 107 | console.error('Image Generation Fetch Error:', error); 108 | handleFetchError(error, controller, prompt, 'image', displayImageResult); 109 | } finally { 110 | if (window.currentRequest === controller) { 111 | window.currentRequest = null; 112 | } 113 | } 114 | } 115 | 116 | /** 117 | * Fetches initial application context (tools, files, prompts, history, params) from the backend. 118 | * Calls UI functions to populate the interface. 119 | */ 120 | async function loadContextInfo() { 121 | console.log('loadContextInfo called'); 122 | if (typeof showLoadingIndicator === 'function') showLoadingIndicator(true); 123 | if (typeof hideError === 'function') hideError(); 124 | 125 | try { 126 | const response = await fetch('/context_info'); 127 | console.log(`Fetch finished loading context: GET "${response.url}". Status: ${response.status}`); 128 | 129 | if (!response.ok) { 130 | let errorData = { error: `HTTP error ${response.status}` }; 131 | try { 132 | errorData = await response.json(); 133 | } catch (jsonError) { 134 | console.warn("Could not parse error response as JSON."); 135 | } 136 | throw new Error(errorData.error || `Failed to fetch context info. Status: ${response.status}`); 137 | } 138 | 139 | const data = await response.json(); 140 | console.log('Context data received:', data); 141 | 142 | if (typeof updateSidebarLists === 'function') updateSidebarLists(data); 143 | if (typeof updateToolsDropdown === 'function') updateToolsDropdown(data.tools); 144 | 145 | if (typeof initializeChatHistoryPagination === 'function') { 146 | initializeChatHistoryPagination(data.workflow_data || []); 147 | } 148 | 149 | if (typeof loadImageHistory === 'function') { 150 | await loadImageHistory(); 151 | } 152 | if (typeof loadMemoryDisplay === 'function') { 153 | await loadMemoryDisplay(); 154 | } 155 | 156 | } catch (error) { 157 | console.error('Error loading context info:', error); 158 | if (error instanceof TypeError && error.message === 'Failed to fetch') { 159 | console.error("'Failed to fetch' likely means the backend server isn't running or is unreachable."); 160 | } else { 161 | console.error("Error details:", error.message, error.stack); 162 | } 163 | 164 | if (typeof showError === 'function') showError(`Error loading application context: ${error.message}`); 165 | } finally { 166 | if (typeof showLoadingIndicator === 'function') showLoadingIndicator(false); 167 | } 168 | } 169 | 170 | /** 171 | * Fetches image history from the backend. 172 | * Calls UI functions to display the history. 173 | */ 174 | async function loadImageHistory() { 175 | if (typeof showImageHistoryLoading === 'function') showImageHistoryLoading(true); 176 | if (typeof showImageHistoryError === 'function') showImageHistoryError(false); 177 | if (typeof clearImageHistoryContent === 'function') clearImageHistoryContent(); 178 | 179 | try { 180 | const response = await fetch('/image_history'); 181 | if (!response.ok) { 182 | const errorData = await response.json().catch(() => ({ error: `HTTP error ${response.status}` })); 183 | throw new Error(errorData.error || `Failed to fetch image history`); 184 | } 185 | const fetchedImageHistoryData = await response.json(); 186 | 187 | if (typeof initializeImageHistory === 'function') { 188 | initializeImageHistory(fetchedImageHistoryData); 189 | } 190 | 191 | } catch (error) { 192 | console.error('Error loading image history:', error); 193 | if (typeof showImageHistoryError === 'function') showImageHistoryError(true, `Error: ${error.message}`); 194 | } finally { 195 | if (typeof showImageHistoryLoading === 'function') showImageHistoryLoading(false); 196 | } 197 | } 198 | 199 | /** 200 | * Deletes a generated image from the backend using its timestamp. 201 | * @param {string} timestampToDelete - The precise timestamp of the image to delete. 202 | * @returns {Promise} - True if deletion was successful, false otherwise. 203 | */ 204 | async function deleteGeneratedImageAPI(timestampToDelete) { 205 | try { 206 | const encodedTimestamp = encodeURIComponent(timestampToDelete); 207 | const response = await fetch(`/delete_generated_image/${encodedTimestamp}`, { 208 | method: 'DELETE' 209 | }); 210 | 211 | const data = await response.json(); 212 | 213 | if (!response.ok) { 214 | throw new Error(data.error || `Failed to delete image (Status: ${response.status})`); 215 | } 216 | console.log(`Image ${timestampToDelete} deleted successfully via API.`); 217 | return true; 218 | 219 | } catch (error) { 220 | console.error('Error deleting image via API:', error); 221 | if (typeof showErrorMessage === 'function') showErrorMessage(`Error deleting image: ${error.message}`); 222 | return false; 223 | } 224 | } 225 | 226 | 227 | /** 228 | * Fetches memory items from the backend, optionally filtering by search query. 229 | * @param {string} [searchQuery] - Optional search term. 230 | * @returns {Promise} - A promise that resolves to an array of memory items or empty array on error. 231 | */ 232 | async function fetchMemoryItems(searchQuery = '') { 233 | let url = '/get_memory'; 234 | if (searchQuery) { 235 | url += `?search=${encodeURIComponent(searchQuery)}`; 236 | console.log(`Searching memory bank with query: ${searchQuery}`); 237 | } else { 238 | console.log("Fetching all memory items"); 239 | } 240 | 241 | try { 242 | const response = await fetch(url); 243 | if (!response.ok) { 244 | const errorData = await response.json().catch(() => ({})); 245 | throw new Error(errorData.error || `Failed to fetch memory: Status ${response.status}`); 246 | } 247 | const memoryData = await response.json(); 248 | console.log(`Successfully fetched ${memoryData.length} memory items.`); 249 | return memoryData; 250 | } catch (error) { 251 | console.error('Error fetching memory items:', error); 252 | if (typeof showErrorMessage === 'function') showErrorMessage(`Error fetching memories: ${error.message}`); 253 | return []; 254 | } 255 | } 256 | 257 | 258 | /** 259 | * Deletes a specific memory item from the backend by its timestamp. 260 | * @param {string} timestamp - The precise timestamp of the memory item to delete. 261 | * @returns {Promise} - True if successful, false otherwise. 262 | */ 263 | async function deleteMemoryAPI(timestamp) { 264 | console.log(`Attempting to delete memory item via API with timestamp: ${timestamp}`); 265 | try { 266 | const encodedTimestamp = encodeURIComponent(timestamp); 267 | const response = await fetch(`/delete_memory/${encodedTimestamp}`, { 268 | method: 'DELETE', 269 | }); 270 | const data = await response.json(); 271 | 272 | if (!response.ok) { 273 | throw new Error(data.error || `Failed to delete memory item (Status ${response.status})`); 274 | } 275 | 276 | console.log(`Successfully deleted memory item for timestamp: ${timestamp}`); 277 | return true; 278 | } catch (error) { 279 | console.error(`Error deleting memory item ${timestamp}:`, error); 280 | if (typeof showErrorMessage === 'function') showErrorMessage(`Error deleting memory: ${error.message}`); 281 | return false; 282 | } 283 | } 284 | 285 | 286 | /** 287 | * Deletes a specific chat history entry from the backend by its timestamp. 288 | * @param {string} timestamp - The precise timestamp of the history entry to delete. 289 | * @returns {Promise} - True if successful, false otherwise. 290 | */ 291 | async function deleteHistoryEntryAPI(timestamp) { 292 | console.log(`Attempting to delete history entry via API with timestamp: ${timestamp}`); 293 | try { 294 | const encodedTimestamp = encodeURIComponent(timestamp); 295 | const response = await fetch(`/delete_history_entry/${encodedTimestamp}`, { 296 | method: 'DELETE', 297 | }); 298 | const data = await response.json(); 299 | 300 | if (!response.ok) { 301 | if (response.status === 404) { 302 | console.warn(`No history entry found matching timestamp: ${timestamp}`); 303 | if (typeof showInfoMessage === 'function') showInfoMessage('History entry not found.'); 304 | return false; 305 | } 306 | throw new Error(data.error || `Failed to delete history entry (Status ${response.status})`); 307 | } 308 | 309 | console.log(`Successfully deleted history entry for timestamp: ${timestamp}`); 310 | return true; 311 | } catch (error) { 312 | console.error(`Error deleting history entry ${timestamp}:`, error); 313 | if (typeof showErrorMessage === 'function') showErrorMessage(`Error deleting history: ${error.message}`); 314 | return false; 315 | } 316 | } 317 | 318 | /** 319 | * Saves a memory item (from inline selection or manual entry) to the backend. 320 | * @param {object} memoryData - Object containing { content, type, language?, context_query? } 321 | * @param {boolean} isManual - Flag indicating if it's from the manual add form. 322 | * @returns {Promise} - True if successful, false otherwise. 323 | */ 324 | async function saveMemoryAPI(memoryData, isManual = false) { 325 | const endpoint = isManual ? '/add_memory_manual' : '/save_memory'; 326 | const action = isManual ? 'manual memory item' : 'memory snippet'; 327 | console.log(`Attempting to save ${action} via API:`, memoryData); 328 | 329 | if (!memoryData || !memoryData.content || !memoryData.type) { 330 | console.error("Missing required fields for saving memory:", memoryData); 331 | if (typeof showErrorMessage === 'function') showErrorMessage('Cannot save memory: Missing content or type.'); 332 | return false; 333 | } 334 | 335 | try { 336 | const response = await fetch(endpoint, { 337 | method: 'POST', 338 | headers: { 'Content-Type': 'application/json' }, 339 | body: JSON.stringify(memoryData), 340 | }); 341 | const result = await response.json(); 342 | 343 | if (!response.ok) { 344 | throw new Error(result.error || `Failed to save ${action} (Status ${response.status})`); 345 | } 346 | 347 | console.log(`Successfully saved ${action}.`); 348 | return true; 349 | } catch (error) { 350 | console.error(`Error saving ${action}:`, error); 351 | if (typeof showErrorMessage === 'function') showErrorMessage(`Error saving memory: ${error.message}`); 352 | return false; 353 | } 354 | } 355 | 356 | /** 357 | * Uploads a file to the backend using XMLHttpRequest for progress tracking. 358 | * @param {File} file - The file object to upload. 359 | * @param {number | null} [currentFileNum=null] - The 1-based index of the current file in a batch. 360 | * @param {number | null} [totalFiles=null] - The total number of files in the batch. 361 | * @returns {Promise} - True if upload was successful, false otherwise. 362 | */ 363 | async function uploadFileAPI(file, currentFileNum = null, totalFiles = null) { 364 | return new Promise((resolve) => { 365 | const formData = new FormData(); 366 | formData.append('file', file); 367 | 368 | const xhr = new XMLHttpRequest(); 369 | 370 | const progressBarContainer = document.getElementById('upload-progress-container'); 371 | const progressBar = document.getElementById('upload-progress-bar'); 372 | const percentageSpan = document.getElementById('upload-percentage'); 373 | const successIcon = document.getElementById('upload-complete-icon'); 374 | const errorIcon = document.getElementById('upload-error-icon'); 375 | const errorMessageP = document.getElementById('upload-error-message'); 376 | 377 | const resetProgressUI = () => { 378 | if (progressBarContainer) progressBarContainer.classList.add('hidden'); 379 | if (progressBar) progressBar.value = 0; 380 | if (percentageSpan) percentageSpan.classList.add('hidden'); 381 | if (successIcon) successIcon.classList.add('hidden'); 382 | if (errorIcon) errorIcon.classList.add('hidden'); 383 | if (errorMessageP) errorMessageP.textContent = ''; 384 | }; 385 | 386 | xhr.open('POST', '/upload', true); 387 | 388 | xhr.upload.onprogress = (event) => { 389 | // Progress no longer shown granularly 390 | }; 391 | 392 | xhr.onload = () => { 393 | try { 394 | if (xhr.status >= 200 && xhr.status < 300) { 395 | const response = JSON.parse(xhr.responseText); 396 | console.log('Upload Success:', response); 397 | if (typeof updateUploadProgress === 'function') { 398 | updateUploadProgress(file.name, true, false, null, currentFileNum, totalFiles); 399 | } 400 | resolve({ success: true, message: response.message }); 401 | } else { 402 | let errorMsg = `Upload failed (HTTP ${xhr.status})`; 403 | try { 404 | const errorResponse = JSON.parse(xhr.responseText); 405 | errorMsg = errorResponse.error || errorMsg; 406 | } catch (e) { /* Ignore */ } 407 | console.error('Upload Error:', errorMsg, xhr.statusText); 408 | if (typeof updateUploadProgress === 'function') { 409 | updateUploadProgress(file.name, true, true, errorMsg, currentFileNum, totalFiles); 410 | } 411 | resolve({ success: false, message: errorMsg }); 412 | } 413 | } catch (error) { 414 | console.error('Error processing upload response:', error); 415 | if (typeof updateUploadProgress === 'function') { 416 | updateUploadProgress(file.name, true, true, 'Error processing server response.', currentFileNum, totalFiles); 417 | } 418 | resolve({ success: false, message: 'Error processing server response.' }); 419 | } 420 | }; 421 | 422 | xhr.onerror = () => { 423 | console.error('Upload Network Error'); 424 | if (typeof updateUploadProgress === 'function') { 425 | updateUploadProgress(file.name, true, true, 'Network error during upload.', currentFileNum, totalFiles); 426 | } 427 | resolve({ success: false, message: 'Network error during upload.' }); 428 | }; 429 | 430 | xhr.onabort = () => { 431 | console.log('Upload aborted'); 432 | if (typeof updateUploadProgress === 'function') { 433 | updateUploadProgress(file.name, true, true, 'Upload aborted.', currentFileNum, totalFiles); 434 | } 435 | resolve({ success: false, message: 'Upload aborted.' }); 436 | }; 437 | 438 | xhr.ontimeout = () => { 439 | console.error('Upload timed out'); 440 | if (typeof updateUploadProgress === 'function') { 441 | updateUploadProgress(file.name, true, true, 'Upload timed out.', currentFileNum, totalFiles); 442 | } 443 | resolve({ success: false, message: 'Upload timed out.' }); 444 | }; 445 | xhr.timeout = 120000; 446 | 447 | if (typeof updateUploadProgress === 'function') { 448 | updateUploadProgress(file.name, false, false, null, currentFileNum, totalFiles); 449 | } 450 | xhr.send(formData); 451 | }); 452 | } 453 | 454 | /** 455 | * Submits a URL to be added by the backend. 456 | * @param {string} url - The URL to add. 457 | * @returns {Promise} - The server response on success, or null on error. 458 | */ 459 | async function addUrlAPI(url) { 460 | try { 461 | const response = await fetch('/add_url', { 462 | method: 'POST', 463 | headers: { 'Content-Type': 'application/json' }, 464 | body: JSON.stringify({ url: url }), 465 | }); 466 | 467 | const data = await response.json(); 468 | 469 | if (!response.ok) { 470 | throw new Error(data.error || `Adding URL failed (Status: ${response.status})`); 471 | } 472 | 473 | console.log(`URL ${url} added successfully:`, data); 474 | return data; 475 | } catch (error) { 476 | return null; 477 | } 478 | } 479 | 480 | /** 481 | * Sends a request to delete all files of a specific type. 482 | * @param {string} fileType - The type of file to delete ('document', 'image', 'video', 'audio'). 483 | * @returns {Promise} - True if successful, false otherwise. 484 | */ 485 | async function deleteAllFilesAPI(fileType) { 486 | console.log(`Attempting to delete all ${fileType}s via API`); 487 | try { 488 | const response = await fetch('/delete_all', { 489 | method: 'POST', 490 | headers: { 'Content-Type': 'application/json' }, 491 | body: JSON.stringify({ type: fileType }), 492 | }); 493 | const data = await response.json(); 494 | 495 | if (!response.ok) { 496 | throw new Error(data.error || `Failed to delete all ${fileType}s (Status ${response.status})`); 497 | } 498 | 499 | console.log(`Successfully deleted all ${fileType}s:`, data.message); 500 | return true; 501 | } catch (error) { 502 | console.error(`Error deleting all ${fileType}s:`, error); 503 | if (typeof showErrorMessage === 'function') showErrorMessage(`Delete Error: ${error.message}`); 504 | return false; 505 | } 506 | } 507 | 508 | /** 509 | * Sends a request to delete a specific file by its UUID and type. 510 | * @param {string} uuid - The UUID of the file to delete. 511 | * @param {string} fileType - The type of the file ('document', 'image', 'video', 'audio'). 512 | * @returns {Promise} - True if successful, false otherwise. 513 | */ 514 | async function deleteFileByUuidAPI(uuid, fileType) { 515 | console.log(`Attempting to delete ${fileType} file with UUID ${uuid} via API`); 516 | try { 517 | const response = await fetch(`/delete_file/${encodeURIComponent(uuid)}/${encodeURIComponent(fileType)}`, { 518 | method: 'DELETE', 519 | }); 520 | const data = await response.json(); 521 | 522 | if (!response.ok) { 523 | if (response.status === 404) { 524 | console.warn(`File not found for deletion: ${fileType} UUID ${uuid}`); 525 | if (typeof showInfoMessage === 'function') showInfoMessage('File not found.'); 526 | return false; 527 | } 528 | throw new Error(data.error || `Failed to delete ${fileType} file (Status ${response.status})`); 529 | } 530 | 531 | console.log(`${fileType.charAt(0).toUpperCase() + fileType.slice(1)} deleted successfully (UUID: ${uuid})`); 532 | return true; 533 | } catch (error) { 534 | console.error(`Error deleting ${fileType} ${uuid}:`, error); 535 | if (typeof showErrorMessage === 'function') showErrorMessage(`Delete Error: ${error.message}`); 536 | return false; 537 | } 538 | } 539 | 540 | /** 541 | * Fetches details for a specific workflow entry from the backend. 542 | * @param {string} timestampStr - The precise timestamp string. 543 | * @returns {Promise} - The details object or null on error. 544 | */ 545 | async function fetchWorkflowDetailAPI(timestampStr) { 546 | console.log(`Fetching workflow detail for timestamp: ${timestampStr}`); 547 | try { 548 | const response = await fetch(`/workflow_detail/${encodeURIComponent(timestampStr)}`); 549 | const data = await response.json(); 550 | 551 | if (!response.ok) { 552 | if (response.status === 404) { 553 | console.warn(`Workflow entry not found for timestamp: ${timestampStr}`); 554 | throw new Error('Query details not found.'); 555 | } 556 | throw new Error(data.error || `Failed to fetch details (Status ${response.status})`); 557 | } 558 | console.log(`Successfully retrieved details for timestamp: ${timestampStr}`); 559 | return data; 560 | } catch (error) { 561 | console.error(`Error fetching workflow detail for ${timestampStr}:`, error); 562 | throw error; 563 | } 564 | } 565 | 566 | /** 567 | * Fetches only chat history data (by calling /context_info) and updates the UI. 568 | */ 569 | async function fetchAndRefreshChatHistory() { 570 | console.log("Fetching and refreshing chat history..."); 571 | try { 572 | const response = await fetch('/context_info'); 573 | if (!response.ok) { 574 | const errorData = await response.json().catch(() => ({ error: `HTTP error ${response.status}` })); 575 | throw new Error(errorData.error || `Failed to fetch context for history`); 576 | } 577 | const data = await response.json(); 578 | 579 | if (typeof initializeChatHistoryPagination === 'function') { 580 | initializeChatHistoryPagination(data.workflow_data || []); 581 | } else { 582 | console.error("initializeChatHistoryPagination function not found!"); 583 | } 584 | 585 | } catch (error) { 586 | console.error('Error fetching/refreshing chat history:', error); 587 | if (typeof showError === 'function') showError(`Error refreshing chat history: ${error.message}`); 588 | } 589 | } 590 | 591 | // Make API functions globally available (simple approach) 592 | window.sendQuery = sendQuery; 593 | window.sendImageGenerationRequest = sendImageGenerationRequest; 594 | window.loadContextInfo = loadContextInfo; 595 | window.loadImageHistory = loadImageHistory; 596 | window.deleteGeneratedImageAPI = deleteGeneratedImageAPI; 597 | window.fetchMemoryItems = fetchMemoryItems; 598 | window.deleteMemoryAPI = deleteMemoryAPI; 599 | window.deleteHistoryEntryAPI = deleteHistoryEntryAPI; 600 | window.saveMemoryAPI = saveMemoryAPI; 601 | window.uploadFileAPI = uploadFileAPI; 602 | window.addUrlAPI = addUrlAPI; 603 | window.deleteAllFilesAPI = deleteAllFilesAPI; 604 | window.deleteFileByUuidAPI = deleteFileByUuidAPI; 605 | window.fetchWorkflowDetailAPI = fetchWorkflowDetailAPI; 606 | window.fetchAndRefreshChatHistory = fetchAndRefreshChatHistory; 607 | 608 | console.log("--- api.js finished executing and functions attached to window --- "); 609 | 610 | // --- User Persona Management API Calls --- 611 | 612 | /** 613 | * Fetches the list of personas saved by the current user. 614 | * @returns {Promise>} - Promise resolving with an array of persona objects. 615 | */ 616 | async function fetchUserPersonasAPI() { 617 | try { 618 | const response = await fetch('/user_personas', { 619 | method: 'GET', 620 | headers: {}, 621 | }); 622 | const data = await response.json(); 623 | if (!response.ok) { 624 | console.error('Error fetching user personas:', data.error, data.details); 625 | throw new Error(data.error || `HTTP error! status: ${response.status}`); 626 | } 627 | console.log('Fetched user personas:', data); 628 | return data; 629 | } catch (error) { 630 | console.error('Failed to fetch user personas:', error); 631 | throw error; 632 | } 633 | } 634 | 635 | /** 636 | * Saves a new user persona to the backend. 637 | * @param {object} personaData - The persona data { persona_name, initial_prompt, final_prompt, llm_params }. 638 | * @returns {Promise} - Promise resolving with the success/error message. 639 | */ 640 | async function savePersonaAPI(personaData) { 641 | try { 642 | const response = await fetch('/save_persona', { 643 | method: 'POST', 644 | headers: { 'Content-Type': 'application/json' }, 645 | body: JSON.stringify(personaData), 646 | }); 647 | const data = await response.json(); 648 | if (!response.ok) { 649 | console.error('Error saving persona:', data.error, data.details); 650 | throw new Error(data.error || `HTTP error! status: ${response.status}`); 651 | } 652 | console.log('Save persona response:', data); 653 | return data; 654 | } catch (error) { 655 | console.error('Failed to save persona:', error); 656 | throw new Error(error.message || 'Failed to save persona due to a network or server issue.'); 657 | } 658 | } 659 | 660 | /** 661 | * Deletes a specific user persona by name. 662 | * @param {string} personaName - The name of the persona to delete. 663 | * @returns {Promise} - Promise resolving with the deletion status. 664 | */ 665 | async function deletePersonaAPI(personaName) { 666 | try { 667 | const response = await fetch(`/delete_persona/${encodeURIComponent(personaName)}`, { 668 | method: 'DELETE', 669 | headers: {}, 670 | }); 671 | const data = await response.json(); 672 | if (!response.ok) { 673 | console.error('Error deleting persona:', data.error, data.details); 674 | throw new Error(data.error || `HTTP error! status: ${response.status}`); 675 | } 676 | console.log('Delete persona response:', data); 677 | return data; 678 | } catch (error) { 679 | console.error('Failed to delete persona:', error); 680 | throw error; 681 | } 682 | } 683 | 684 | /** 685 | * Updates an existing user persona on the backend. 686 | * @param {string} originalPersonaName - The original name of the persona to update (used in URL). 687 | * @param {object} updatedData - The updated persona data { initial_prompt, final_prompt, llm_params }. 688 | * @returns {Promise} - Promise resolving with the success/error message. 689 | */ 690 | async function updatePersonaAPI(originalPersonaName, updatedData) { 691 | try { 692 | // We don't include persona_name in the body, as renaming isn't supported via this endpoint 693 | const { initial_prompt, final_prompt, llm_params } = updatedData; 694 | const bodyPayload = { initial_prompt, final_prompt, llm_params }; 695 | 696 | const response = await fetch(`/update_persona/${encodeURIComponent(originalPersonaName)}`, { 697 | method: 'PUT', 698 | headers: { 'Content-Type': 'application/json' }, 699 | body: JSON.stringify(bodyPayload), 700 | }); 701 | const data = await response.json(); 702 | if (!response.ok) { 703 | console.error('Error updating persona:', data.error, data.details); 704 | throw new Error(data.error || `HTTP error! status: ${response.status}`); 705 | } 706 | console.log('Update persona response:', data); 707 | return data; 708 | } catch (error) { 709 | console.error('Failed to update persona:', error); 710 | throw new Error(error.message || 'Failed to update persona due to a network or server issue.'); 711 | } 712 | } 713 | 714 | // --- Make Persona API functions globally available --- // 715 | window.fetchUserPersonasAPI = fetchUserPersonasAPI; 716 | window.savePersonaAPI = savePersonaAPI; 717 | window.deletePersonaAPI = deletePersonaAPI; 718 | window.updatePersonaAPI = updatePersonaAPI; -------------------------------------------------------------------------------- /static/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Pixelbot: Pixeltable Agent", 3 | "short_name": "Pixeltable Agent", 4 | "description": "Ask questions about your documents, videos, images, audios, and get insights using Pixeltable's multimodal AI.", 5 | "start_url": "/", 6 | "scope": "/", 7 | "display": "standalone", 8 | "background_color": "#ffffff", 9 | "theme_color": "#3b82f6", 10 | "icons": [ 11 | { 12 | "src": "/static/image/pixelbot.png", 13 | "sizes": "192x192", 14 | "type": "image/png", 15 | "purpose": "any maskable" 16 | } 17 | ], 18 | "related_applications": [ 19 | { 20 | "platform": "web", 21 | "url": "https://pixeltable.com" 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /static/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Allow: / 3 | Sitemap: https://agent.pixeltable.com/sitemap.xml 4 | 5 | # Prevent crawling of non-essential paths 6 | Disallow: /static/logs/ 7 | Disallow: /static/temp/ -------------------------------------------------------------------------------- /static/sitemap.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | https://agent.pixeltable.com/ 5 | 2024-07-31 6 | weekly 7 | 1.0 8 | 9 | --------------------------------------------------------------------------------