140 | Each session is a distinct workspace with its own chat history and knowledge base. 141 |
142 |Some description text here.
395 | 396 | 400 |Missing 'session_id' or 'source' parameter.
", 400 777 | 778 | try: 779 | chunks_cursor = config.collection.find( 780 | { 781 | f"metadata.{SESSION_FIELD}": session_id, 782 | "metadata.source": source 783 | }, 784 | {"text": 1, "_id": 0} 785 | ).sort("metadata.chunk_index", pymongo.ASCENDING) 786 | 787 | full_content = "".join([chunk.get('text', '') for chunk in chunks_cursor]) 788 | 789 | if not full_content: 790 | return "Source not found or has no content.
", 404 791 | 792 | # Return a simple, styled HTML page instead of JSON 793 | html_template = """ 794 | 795 | 796 | 797 | 798 |An unexpected error occurred: {str(e)}
", 500 843 | 844 | 845 | # --- 9. DB INDEX SETUP & LAUNCH --- 846 | def setup_database_and_index(): 847 | print_log("--- 🚀 Initializing DB and Vector Search Indexes ---") 848 | if COLLECTION_NAME not in config.db.list_collection_names(): 849 | config.db.create_collection(COLLECTION_NAME) 850 | 851 | for model_name, model_cfg in EMBEDDING_CONFIG.items(): 852 | if model_name not in config.embedding_clients: 853 | print_log(f"[WARN] Model '{model_name}' is not loaded, skipping index creation.") 854 | continue 855 | 856 | index_name = model_cfg["index_name"] 857 | vector_field = model_cfg["vector_field"] 858 | dims = model_cfg["dimensions"] 859 | 860 | definition = { 861 | "fields": [ 862 | { 863 | "type": "vector", 864 | "path": vector_field, 865 | "numDimensions": dims, 866 | "similarity": "cosine" 867 | }, 868 | { 869 | "type": "filter", 870 | "path": f"metadata.{SESSION_FIELD}" 871 | } 872 | ] 873 | } 874 | 875 | try: 876 | existing = next(config.collection.list_search_indexes(name=index_name), None) 877 | if not existing: 878 | print_log(f"[ACTION] Creating index '{index_name}' for model '{model_name}'...") 879 | config.collection.create_search_index( 880 | model=SearchIndexModel(name=index_name, type="vectorSearch", definition=definition) 881 | ) 882 | print_log(f"[INFO] Finished creating index '{index_name}'.") 883 | else: 884 | print_log(f"[INFO] Index '{index_name}' already exists.") 885 | except OperationFailure as e: 886 | if "already exists" in str(e).lower(): 887 | print_log(f"[INFO] Index '{index_name}' already exists. OK.") 888 | else: 889 | print_log(f"[ERROR] Creating index '{index_name}' failed: {e}") 890 | raise 891 | 892 | @app.route("/search", methods=["POST"]) 893 | def search_web(): 894 | data = request.json 895 | query = data.get("query") 896 | num_results = data.get("num_results", 5) 897 | if not query: 898 | return jsonify({"error": "Query is required"}), 400 899 | try: 900 | print_log(f"[INFO] Web search for: '{query}'") 901 | with DDGS() as ddgs: 902 | results = [r for r in ddgs.text(query, max_results=num_results)] 903 | return jsonify({"status": "success", "results": results}) 904 | except Exception as e: 905 | print_log(f"[ERROR] Web search failed: {e}\n{traceback.format_exc()}") 906 | return jsonify({"error": f"Web search error: {str(e)}"}), 500 907 | 908 | @app.route("/chunk_preview", methods=["POST"]) 909 | def chunk_preview(): 910 | data = request.json 911 | content = data.get("content") 912 | chunk_size = data.get("chunk_size", 1000) 913 | chunk_overlap = data.get("chunk_overlap", 150) 914 | 915 | if not content: 916 | return jsonify({"error": "Content is required"}), 400 917 | 918 | if chunk_overlap >= chunk_size: 919 | return jsonify({"error": "Chunk overlap must be smaller than chunk size."}), 400 920 | 921 | try: 922 | text_splitter = RecursiveCharacterTextSplitter( 923 | chunk_size=chunk_size, 924 | chunk_overlap=chunk_overlap 925 | ) 926 | chunks = text_splitter.split_text(content) 927 | return jsonify({"chunks": chunks}) 928 | except Exception as e: 929 | print_log(f"[ERROR] Chunk preview failed: {e}\n{traceback.format_exc()}") 930 | return jsonify({"error": f"An unexpected error occurred: {str(e)}"}), 500 931 | 932 | if __name__ == "__main__": 933 | setup_database_and_index() 934 | print_log("--- ✅ Setup complete. Starting server at http://127.0.0.1:5001 ---") 935 | app.run(debug=True, port=5001) -------------------------------------------------------------------------------- /rag/static/script.js: -------------------------------------------------------------------------------- 1 | // --------------------------- 2 | // Global state and references 3 | // --------------------------- 4 | let currentSessionId = "default"; 5 | let allSessions = []; 6 | let availableModels = []; 7 | let chunkCache = new Map(); 8 | 9 | // Dom references 10 | const chatBox = document.getElementById("chat-box"); 11 | const userInput = document.getElementById("user-input"); 12 | const chatForm = document.getElementById("chat-form"); 13 | const sessionSelector = document.getElementById("session-selector"); 14 | const newSessionBtn = document.getElementById("new-session-btn"); 15 | const clearHistoryBtn = document.getElementById("clear-history-btn"); 16 | const toolButtonsContainer = document.getElementById("tool-buttons"); 17 | const thinkingIndicator = document.getElementById("thinking-indicator"); 18 | 19 | const embeddingModelSelector = document.getElementById("embedding-model-selector"); 20 | const numSourcesInput = document.getElementById("num-sources-input"); 21 | const minScoreInput = document.getElementById("min-score-input"); 22 | const minScoreValue = document.getElementById("min-score-value"); 23 | const maxCharsInput = document.getElementById("max-chunk-length-input"); 24 | const maxCharsValue = document.getElementById("max-chars-value"); 25 | 26 | const previewRagBtn = document.getElementById("preview-rag-btn"); 27 | 28 | // Modal references 29 | const modalOverlay = document.getElementById("modal-overlay"); 30 | const modalContainer = document.getElementById("modal-container"); 31 | const modalTitle = document.getElementById("modal-title"); 32 | const modalText = document.getElementById("modal-text"); 33 | const modalContentHost = document.getElementById("modal-content-host"); 34 | const modalCancelBtn = document.getElementById("modal-btn-cancel"); 35 | const modalSubmitBtn = document.getElementById("modal-btn-submit"); 36 | 37 | // Source browser references 38 | const sourceBrowserOverlay = document.getElementById("source-browser-overlay"); 39 | const sourceBrowserContainer = document.getElementById("source-browser-container"); 40 | const sourceBrowserCloseBtn = document.getElementById("source-browser-close-btn"); 41 | const sourceListEl = document.getElementById("source-list"); 42 | const chunkListEl = document.getElementById("chunk-list"); 43 | const chunkListPlaceholder = document.getElementById("chunk-list-placeholder"); 44 | const sourceBrowserTotalChunks = document.getElementById("source-browser-total-chunks"); 45 | const sourceBrowserSourceCount = document.getElementById("source-browser-source-count"); 46 | const sourceBrowserSelectedChunkCount = document.getElementById("source-browser-selected-chunk-count"); 47 | 48 | // ----------- 49 | // Modal Logic 50 | // ----------- 51 | function showModal({ title, text, contentHTML, onSubmit, onCancel }) { 52 | modalTitle.textContent = title || "Modal Title"; 53 | modalText.textContent = text || ""; 54 | modalContentHost.innerHTML = contentHTML || ""; 55 | 56 | if (onSubmit) { 57 | modalSubmitBtn.onclick = () => { 58 | onSubmit(); 59 | }; 60 | } else { 61 | modalSubmitBtn.onclick = () => { 62 | hideModal(); 63 | }; 64 | } 65 | 66 | if (onCancel) { 67 | modalCancelBtn.onclick = () => { 68 | onCancel(); 69 | }; 70 | } else { 71 | modalCancelBtn.onclick = () => { 72 | hideModal(); 73 | }; 74 | } 75 | 76 | modalOverlay.classList.remove("invisible", "opacity-0"); 77 | modalContainer.classList.remove("scale-95", "opacity-0"); 78 | } 79 | 80 | function hideModal() { 81 | modalOverlay.classList.add("opacity-0", "invisible"); 82 | modalContainer.classList.add("scale-95", "opacity-0"); 83 | 84 | modalTitle.textContent = ""; 85 | modalText.textContent = ""; 86 | modalContentHost.innerHTML = ""; 87 | modalSubmitBtn.onclick = null; 88 | modalCancelBtn.onclick = null; 89 | } 90 | 91 | modalOverlay.addEventListener("click", (e) => { 92 | if (e.target === modalOverlay) { 93 | hideModal(); 94 | } 95 | }); 96 | 97 | // ------------------------ 98 | // Source Browser Functions 99 | // ------------------------ 100 | function openSourceBrowser() { 101 | sourceBrowserOverlay.classList.remove("opacity-0", "invisible"); 102 | sourceBrowserContainer.classList.remove("scale-95", "opacity-0"); 103 | sourceBrowserTotalChunks.textContent = ""; 104 | sourceBrowserSourceCount.textContent = "Total: 0"; 105 | sourceBrowserSelectedChunkCount.textContent = "Selected: 0"; 106 | 107 | fetch(`/sources?session_id=${encodeURIComponent(currentSessionId)}`) 108 | .then((r) => r.json()) 109 | .then((data) => { 110 | sourceListEl.innerHTML = ""; 111 | chunkListEl.innerHTML = ""; 112 | chunkListPlaceholder.style.display = "block"; 113 | 114 | if (!data || data.length === 0) { 115 | sourceListEl.innerHTML = "No sources found.
"; 116 | return; 117 | } 118 | 119 | sourceBrowserSourceCount.textContent = `Total: ${data.length}`; 120 | let totalChunks = 0; 121 | data.forEach(src => totalChunks += (src.chunk_count || 0)); 122 | sourceBrowserTotalChunks.textContent = `(${totalChunks.toLocaleString()} Total Chunks)`; 123 | 124 | data.forEach((src) => { 125 | const btn = document.createElement("button"); 126 | btn.className = "source-item"; 127 | 128 | const chunkCount = src.chunk_count !== undefined ? `${src.chunk_count}` : '?'; 129 | const sourceName = src.name + (src.type ? ` (${src.type})` : ""); 130 | 131 | btn.innerHTML = ` 132 | ${escapeHtml(sourceName)} 133 | ${chunkCount} 134 | `; 135 | 136 | btn.onclick = () => { 137 | document.querySelectorAll('.source-item').forEach(b => b.classList.remove('active')); 138 | btn.classList.add('active'); 139 | loadChunksForSource(src.name); 140 | }; 141 | sourceListEl.appendChild(btn); 142 | }); 143 | }) 144 | .catch((err) => { 145 | console.error("Failed to list sources:", err); 146 | sourceListEl.innerHTML = `Error: ${err.message}
`; 147 | }); 148 | } 149 | 150 | function loadChunksForSource(sourceUrl) { 151 | sourceBrowserSelectedChunkCount.textContent = "Loading..."; 152 | fetch(`/chunks?session_id=${encodeURIComponent(currentSessionId)}&source_url=` + encodeURIComponent(sourceUrl)) 153 | .then((r) => r.json()) 154 | .then((data) => { 155 | chunkListEl.innerHTML = ""; 156 | chunkCache.clear(); 157 | 158 | if (data.error) { 159 | chunkListEl.innerHTML = `Error: ${data.error}
`; 160 | sourceBrowserSelectedChunkCount.textContent = "Error"; 161 | return; 162 | } 163 | if (!data || data.length === 0) { 164 | chunkListEl.innerHTML = "No chunks found for this source.
"; 165 | sourceBrowserSelectedChunkCount.textContent = "Selected: 0"; 166 | return; 167 | } 168 | 169 | sourceBrowserSelectedChunkCount.textContent = `Selected: ${data.length}`; 170 | 171 | chunkListPlaceholder.style.display = "none"; 172 | data.forEach((ch) => { 173 | chunkCache.set(ch._id, ch); 174 | const card = document.createElement("div"); 175 | card.className = "chunk-card"; 176 | card.setAttribute("data-chunk-id", ch._id); // Add ID for easier selection 177 | card.innerHTML = ` 178 |Error: ${err.message}
`; 193 | sourceBrowserSelectedChunkCount.textContent = "Error"; 194 | }); 195 | } 196 | 197 | // ----------------------------------------------------------- 198 | // CORRECTED: A single, robust event listener for all chunk buttons 199 | // (Handles Edit, Delete, Save, and Cancel) 200 | // ----------------------------------------------------------- 201 | chunkListEl.addEventListener('click', (event) => { 202 | const editButton = event.target.closest('.chunk-edit-btn'); 203 | if (editButton) { 204 | const chunkId = editButton.getAttribute('data-id'); 205 | startChunkEdit(chunkId); 206 | return; 207 | } 208 | 209 | const deleteButton = event.target.closest('.chunk-delete-btn'); 210 | if (deleteButton) { 211 | const chunkId = deleteButton.getAttribute('data-id'); 212 | onDeleteChunkClick(chunkId); // Uses existing delete logic 213 | return; 214 | } 215 | 216 | const saveButton = event.target.closest('.chunk-save-btn'); 217 | if (saveButton) { 218 | const chunkId = saveButton.getAttribute('data-id'); 219 | saveChunkEdit(chunkId); 220 | return; 221 | } 222 | 223 | const cancelButton = event.target.closest('.chunk-cancel-btn'); 224 | if (cancelButton) { 225 | const chunkId = cancelButton.getAttribute('data-id'); 226 | cancelChunkEdit(chunkId); 227 | return; 228 | } 229 | }); 230 | 231 | 232 | sourceBrowserCloseBtn.addEventListener("click", () => { 233 | closeSourceBrowser(); 234 | }); 235 | 236 | function closeSourceBrowser() { 237 | sourceBrowserOverlay.classList.add("opacity-0", "invisible"); 238 | sourceBrowserContainer.classList.add("scale-95", "opacity-0"); 239 | sourceListEl.innerHTML = ""; 240 | chunkListEl.innerHTML = ""; 241 | chunkListPlaceholder.style.display = "block"; 242 | sourceBrowserTotalChunks.textContent = ""; 243 | sourceBrowserSourceCount.textContent = "Total: 0"; 244 | sourceBrowserSelectedChunkCount.textContent = "Selected: 0"; 245 | } 246 | 247 | // ------------------------------------------------- 248 | // --- NEW IN-PLACE CHUNK EDITING LOGIC --- 249 | // ------------------------------------------------- 250 | 251 | function startChunkEdit(chunkId) { 252 | const chunkCard = chunkListEl.querySelector(`.chunk-card[data-chunk-id='${chunkId}']`); 253 | if (!chunkCard || chunkCard.classList.contains('is-editing')) return; 254 | 255 | const chunkData = chunkCache.get(chunkId); 256 | if (!chunkData) { 257 | alert("Error: Could not find chunk data to edit."); 258 | return; 259 | } 260 | 261 | chunkCard.classList.add('is-editing'); 262 | const contentHost = chunkCard.querySelector('.chunk-content'); 263 | const actionsHost = chunkCard.querySelector('.chunk-actions'); 264 | 265 | // Store original HTML for cancellation 266 | chunkCard.dataset.originalContent = contentHost.innerHTML; 267 | chunkCard.dataset.originalActions = actionsHost.innerHTML; 268 | 269 | // Inject the textarea and new buttons 270 | contentHost.innerHTML = ` 271 | 272 | `; 273 | actionsHost.innerHTML = ` 274 | 275 | 276 | `; 277 | 278 | // Auto-resize and focus the textarea 279 | const textarea = contentHost.querySelector('textarea'); 280 | const autoResize = () => { 281 | textarea.style.height = 'auto'; 282 | textarea.style.height = (textarea.scrollHeight) + 'px'; 283 | }; 284 | textarea.addEventListener('input', autoResize); 285 | autoResize(); 286 | textarea.focus(); 287 | } 288 | 289 | function cancelChunkEdit(chunkId) { 290 | const chunkCard = chunkListEl.querySelector(`.chunk-card[data-chunk-id='${chunkId}']`); 291 | if (!chunkCard || !chunkCard.classList.contains('is-editing')) return; 292 | 293 | const contentHost = chunkCard.querySelector('.chunk-content'); 294 | const actionsHost = chunkCard.querySelector('.chunk-actions'); 295 | 296 | // Restore original content from dataset 297 | contentHost.innerHTML = chunkCard.dataset.originalContent; 298 | actionsHost.innerHTML = chunkCard.dataset.originalActions; 299 | 300 | chunkCard.classList.remove('is-editing'); 301 | delete chunkCard.dataset.originalContent; 302 | delete chunkCard.dataset.originalActions; 303 | } 304 | 305 | function saveChunkEdit(chunkId) { 306 | const chunkCard = chunkListEl.querySelector(`.chunk-card[data-chunk-id='${chunkId}']`); 307 | if (!chunkCard) return; 308 | 309 | const textarea = chunkCard.querySelector('.chunk-edit-textarea'); 310 | const newText = textarea.value; 311 | const saveBtn = chunkCard.querySelector('.chunk-save-btn'); 312 | saveBtn.textContent = 'Saving...'; 313 | saveBtn.disabled = true; 314 | 315 | fetch("/chunk/" + encodeURIComponent(chunkId), { 316 | method: "PUT", 317 | headers: { "Content-Type": "application/json" }, 318 | body: JSON.stringify({ content: newText }), 319 | }) 320 | .then(r => r.json()) 321 | .then(resp => { 322 | if (resp.error) { 323 | alert("Error updating chunk: " + resp.error); 324 | saveBtn.textContent = 'Save'; 325 | saveBtn.disabled = false; // Re-enable on failure 326 | return; 327 | } 328 | // Update local cache 329 | const chunkData = chunkCache.get(chunkId); 330 | chunkData.text = newText; 331 | chunkCache.set(chunkId, chunkData); 332 | 333 | // Restore view mode with the *new* content 334 | const contentHost = chunkCard.querySelector('.chunk-content'); 335 | const actionsHost = chunkCard.querySelector('.chunk-actions'); 336 | 337 | contentHost.innerHTML = marked.parse(newText); 338 | actionsHost.innerHTML = chunkCard.dataset.originalActions; // Restore original buttons 339 | 340 | chunkCard.classList.remove('is-editing'); 341 | delete chunkCard.dataset.originalContent; 342 | delete chunkCard.dataset.originalActions; 343 | }) 344 | .catch(err => { 345 | alert("Error updating chunk: " + err.message); 346 | saveBtn.textContent = 'Save'; 347 | saveBtn.disabled = false; 348 | }); 349 | } 350 | 351 | function onDeleteChunkClick(chunkId) { 352 | if (!confirm("Are you sure you want to delete this chunk?")) return; 353 | fetch(`/chunk/${encodeURIComponent(chunkId)}`, { method: "DELETE" }) 354 | .then((r) => r.json()) 355 | .then((resp) => { 356 | if (resp.error) { 357 | alert("Error deleting chunk: " + resp.error); 358 | return; 359 | } 360 | const chunkCard = chunkListEl.querySelector(`.chunk-card[data-chunk-id='${chunkId}']`); 361 | if (chunkCard) { 362 | chunkCard.remove(); 363 | chunkCache.delete(chunkId); 364 | } 365 | }) 366 | .catch((err) => { 367 | alert("Error deleting chunk: " + err.message); 368 | }); 369 | } 370 | 371 | // -------- 372 | // Helpers 373 | // -------- 374 | function escapeHtml(unsafe) { 375 | if (!unsafe) return ""; 376 | return unsafe.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'"); 377 | } 378 | 379 | function escapeHtmlForTextarea(str) { 380 | return str.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """); 381 | } 382 | 383 | // --------------------------------------------- 384 | // Chat Rendering (add messages to the chat box) 385 | // --------------------------------------------- 386 | function addBotMessage(message) { 387 | const content = message.content; 388 | const sources = message.sources || []; 389 | 390 | const messageEl = document.createElement("div"); 391 | messageEl.className = "message bot-message flex flex-col p-4 bg-gray-700 rounded-lg animate-fade-in-up"; 392 | 393 | const contentDiv = document.createElement("div"); 394 | contentDiv.className = "prose prose-invert max-w-none"; 395 | 396 | if (content.trim().startsWith('Error: Chunk overlap must be smaller than chunk size.
'; 722 | countEl.textContent = 'Total Chunks: 0'; 723 | return false; 724 | } 725 | 726 | try { 727 | const response = await fetch("/chunk_preview", { 728 | method: "POST", 729 | headers: { "Content-Type": "application/json" }, 730 | body: JSON.stringify({ content, chunk_size: chunkSize, chunk_overlap: chunkOverlap }), 731 | }); 732 | const data = await response.json(); 733 | 734 | if (data.error) { 735 | targetEl.innerHTML = `Error chunking: ${escapeHtml(data.error)}
`; 736 | countEl.textContent = 'Total Chunks: 0'; 737 | return false; 738 | } 739 | 740 | if (!data.chunks || data.chunks.length === 0) { 741 | targetEl.innerHTML = 'Could not generate any chunks from the source content.
'; 742 | countEl.textContent = 'Total Chunks: 0'; 743 | return false; 744 | } 745 | 746 | const chunkHtml = data.chunks.map((c, i) => ` 747 |Request error: ${escapeHtml(err.message)}
`; 758 | countEl.textContent = 'Total: 0'; 759 | return false; 760 | } 761 | } 762 | 763 | function handleReadFile() { 764 | let sourceName = ''; 765 | let currentFile = null; 766 | 767 | const modalHTML = ` 768 |Drag & drop your file here
775 |or click to browse
776 |Click again or drop another file to replace
783 |Chunks will appear here.
799 |Chunks will appear here.
946 |${escapeHtml(r.body)}
1096 | ${host} 1097 |Web Search Results:
${escapeHtml(previewContent)}`,
1165 | });
1166 | })
1167 | .catch((err) => {
1168 | alert(`Preview request failed: ${err.message}`);
1169 | });
1170 | });
1171 |
1172 | minScoreInput.addEventListener("input", () => {
1173 | minScoreValue.textContent = parseFloat(minScoreInput.value).toFixed(2);
1174 | });
1175 | maxCharsInput.addEventListener("input", () => {
1176 | maxCharsValue.textContent = parseInt(maxCharsInput.value);
1177 | });
1178 |
1179 | function pollIngestionTask(taskId) {
1180 | const checkStatus = () => {
1181 | fetch(`/ingest/status/${taskId}`)
1182 | .then(r => r.json())
1183 | .then(data => {
1184 | if (data.status === 'complete') {
1185 | addSystemMessage(`Ingestion successful! ${data.message}`);
1186 | loadSessionsAndState();
1187 | } else if (data.status === 'failed') {
1188 | addSystemMessage(`Ingestion failed: ${data.message}`);
1189 | } else {
1190 | setTimeout(checkStatus, 2000);
1191 | }
1192 | })
1193 | .catch(err => {
1194 | addSystemMessage(`Failed to get ingestion status: ${err.message}`);
1195 | });
1196 | };
1197 | addSystemMessage(`Ingestion started with Task ID: ${taskId}. This may take a moment.`);
1198 | setTimeout(checkStatus, 2000);
1199 | }
--------------------------------------------------------------------------------