├── .gitignore
├── README.md
├── package-lock.json
├── package.json
├── public
├── favicon.ico
├── index.html
├── logo192.png
├── logo512.png
├── manifest.json
├── robots.txt
└── uploads
│ ├── anime-dataset-2023.csv
│ ├── movie_data.json
│ ├── repos_graph.json
│ └── repos_readme.json
├── rag.py
├── requirements.txt
└── src
├── App.css
├── App.js
├── App.test.js
├── Navbar
├── Navbar.css
└── Navbar.js
├── NavbarHook
├── NavbarHook.css
└── NavbarHook.js
├── assets
├── VECTR8-demo-ezgif.com-video-to-gif-converter.gif
├── VECTR8-demo.mp4
├── audio-logo.svg
├── code-logo.svg
├── csv-logo.svg
├── excel-logo.svg
├── img-logo.svg
├── json-logo.svg
├── logo.gif
├── logo.png
├── pdf-logo.svg
├── powerpoint-logo.svg
├── torch.png
├── txt-logo.svg
├── video-logo.svg
└── word-logo.svg
├── index.css
├── index.js
├── logo.svg
├── pages
├── Upload.js
├── embed
│ ├── Embed.css
│ └── Embed.js
├── overlay
│ ├── Overlay.css
│ ├── Overlay.js
│ └── graph
│ │ └── Graph.js
├── preview
│ ├── Preview.css
│ └── Preview.js
└── query
│ ├── Query.css
│ └── Query.js
├── reportWebVitals.js
├── setupTests.js
├── transition
├── Transition.css
└── Transition.js
└── utils
├── FileInfo.js
└── mindmapData.js
/.gitignore:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2 |
3 | # dependencies
4 | /node_modules
5 | /.pnp
6 | .pnp.js
7 |
8 | # testing
9 | /coverage
10 |
11 | # production
12 | /build
13 |
14 | # misc
15 | .DS_Store
16 | .env.local
17 | .env.development.local
18 | .env.test.local
19 | .env.production.local
20 |
21 | npm-debug.log*
22 | yarn-debug.log*
23 | yarn-error.log*
24 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | VECT.R8 (Vector Embeddings Creation, Transformation & Retrieval) 🚀
2 |
3 |
4 |
6 |
7 |
8 | A Web UI where you can upload CSV/JSON files, create vector embeddings, and query them. Soon, you'll be able to convert unstructured data to JSON/CSV using an integrated LLM.
9 |
10 |
11 |
12 |
14 |
15 |
16 |
17 | Project under heavy/active development [may be] unstable. Embeddings and Query pages WIP ⚠️
18 |
19 |
20 |
21 | Table of Contents
22 |
23 |
105 |
106 | -----
107 |
108 | Prerequisites
109 |
110 |
146 | Installation
147 |
148 |
149 |
150 |
151 | Step
152 | Instructions
153 |
154 |
155 | Clone the repository
156 | git clone https://github.com/itsPreto/VECTR8.git
157 | cd VECTR8
159 |
160 | Install the required packages
161 | pip install -r requirements.txt
163 |
164 |
Running the Application
167 |
168 |
169 |
170 |
171 | Step
172 | Instructions
173 |
174 |
175 | Start the Flask server
176 | python3 rag.py
178 |
179 | Automatically launch React frontend
180 | The Python endpoint will launch the React frontend in a separate subprocess.
181 |
182 |
183 | Open your web browser
184 | Navigate to http://127.0.0.1:4000
185 |
186 |
187 |
Uploading Files 📂
190 |
191 |
192 |
193 |
194 |
195 | Step
196 | Instructions
197 |
198 |
199 | Drag and Drop a File
200 | Drag and drop a CSV or JSON file into the upload area or click to select a file from your computer.
201 |
202 |
203 | View Uploaded File Information
204 | Once uploaded, the file information such as name and size will be displayed.
205 |
206 |
207 |
Command Line
208 |
To upload a file using curl:
209 |
curl -X POST -F 'file=@/path/to/your/file.csv' http://127.0.0.1:4000/upload_file
210 |
Previewing Data 🧐
213 |
214 |
215 |
216 |
217 | Step
218 | Instructions
219 |
220 |
221 | Select Embedding Keys
222 | After uploading a file, select the keys (columns) you want to include in the embeddings.
223 |
224 |
225 | Preview Document
226 | View a preview of the document created from the selected keys.
227 |
228 |
229 | Preview Embeddings
230 | View the generated embeddings and token count for the selected document.
231 |
232 |
233 |
Command Line
234 |
To preview a file's keys using curl:
235 |
curl -X POST -H "Content-Type: application/json" -d '{"file_path":"uploads/your-file.csv"}' http://127.0.0.1:4000/preview_file
236 |
To preview a document's embeddings using curl:
237 |
curl -X POST -H "Content-Type: application/json" -d '{"file_path":"uploads/your-file.csv", "selected_keys":["key1", "key2"]}' http://127.0.0.1:4000/preview_document
238 |
Creating Vector Embeddings 🧩
241 |
242 |
243 |
244 |
245 | Step
246 | Instructions
247 |
248 |
249 | Start Embedding Creation
250 | Click the "Create Vector DB" button to start the embedding creation process.
251 |
252 |
253 | View Progress
254 | Monitor the progress of the embedding creation with a circular progress indicator. 📈
255 |
256 |
257 |
Command Line
258 |
To create a vector database using curl:
259 |
curl -X POST -H "Content-Type: application/json" -d '{"file_path":"uploads/your-file.csv", "selected_keys":["key1", "key2"]}' http://127.0.0.1:4000/create_vector_database
260 |
Querying the Vector Database 🔍
263 |
264 |
265 |
266 |
267 | Step
268 | Instructions
269 |
270 |
271 | Enter Query
272 | Type your query into the input field.
273 |
274 |
275 | Select Similarity Metric
276 | Choose between cosine similarity or Euclidean distance.
277 |
278 |
279 | Submit Query
280 | Click the "Submit" button to query the vector database.
281 |
282 |
283 | View Results
284 | Inspect the results, which display the document, score, and a button to view detailed data.
285 |
286 |
287 |
Command Line
288 |
To query the vector database using curl:
289 |
curl -X POST -H "Content-Type: application/json" -d '{"query_text":"Your query text here", "similarity_metric":"cosine"}' http://127.0.0.1:4000/query
290 |
Managing the Vector Database 🛠️
293 |
294 |
295 |
296 |
297 | Step
298 | Instructions
299 |
300 |
301 | Backup Database
302 | Click the "Backup Database" button to create a backup of the current vector database.
303 |
304 |
305 | Delete Database
306 | Click the "Delete Database" button to delete the current vector database.
307 |
308 |
309 | View Database Statistics
310 | View statistics such as total documents and average vector length.
311 |
312 |
313 |
Command Line
314 |
To check if the vector database exists using curl:
315 |
curl -X GET http://127.0.0.1:4000/check_vector_db
316 |
To view database statistics using curl:
317 |
curl -X GET http://127.0.0.1:4000/db_stats
318 |
To backup the database using curl:
319 |
curl -X POST http://127.0.0.1:4000/backup_db
320 |
To delete the database using curl:
321 |
curl -X POST http://127.0.0.1:4000/delete_db
322 |
UI Walkthrough 🎨
325 |
326 |
327 |
328 |
329 | Feature
330 | Description
331 |
332 |
333 | Uploading Files
335 |
336 | Drag and drop a file into the upload area or click to select a file.
337 | File information will be displayed after a successful upload.
338 |
339 |
340 |
341 |
342 | Previewing Data
344 |
345 | Select the keys you want to include in the embeddings.
346 | View a preview of the document and generated embeddings.
347 |
348 |
349 |
350 |
351 | Creating Vector Embeddings
353 |
354 | Click the "Create Vector DB" button to start the embedding creation.
355 | Monitor the progress with the circular progress indicator.
356 |
357 |
358 |
359 |
360 | Querying the Vector Database
362 |
363 | Enter your query text and select a similarity metric.
364 | Click "Submit" to query the database and view the results.
365 |
366 |
367 |
368 |
369 | Managing the Vector Database
371 |
372 | Backup the database by clicking "Backup Database".
373 | Delete the database by clicking "Delete Database".
374 | View database statistics such as total documents and average vector length.
375 |
376 |
377 |
378 |
379 |
React App
28 |
29 |
30 | You need to enable JavaScript to run this app.
31 |
32 |
42 |
43 |
44 |
--------------------------------------------------------------------------------
/public/logo192.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itsPreto/VECTR8/7fee734bcc9b5ba66a1ff8b2a09155a595b87f37/public/logo192.png
--------------------------------------------------------------------------------
/public/logo512.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itsPreto/VECTR8/7fee734bcc9b5ba66a1ff8b2a09155a595b87f37/public/logo512.png
--------------------------------------------------------------------------------
/public/manifest.json:
--------------------------------------------------------------------------------
1 | {
2 | "short_name": "React App",
3 | "name": "Create React App Sample",
4 | "icons": [
5 | {
6 | "src": "favicon.ico",
7 | "sizes": "64x64 32x32 24x24 16x16",
8 | "type": "image/x-icon"
9 | },
10 | {
11 | "src": "logo192.png",
12 | "type": "image/png",
13 | "sizes": "192x192"
14 | },
15 | {
16 | "src": "logo512.png",
17 | "type": "image/png",
18 | "sizes": "512x512"
19 | }
20 | ],
21 | "start_url": ".",
22 | "display": "standalone",
23 | "theme_color": "#000000",
24 | "background_color": "#ffffff"
25 | }
26 |
--------------------------------------------------------------------------------
/public/robots.txt:
--------------------------------------------------------------------------------
1 | # https://www.robotstxt.org/robotstxt.html
2 | User-agent: *
3 | Disallow:
4 |
--------------------------------------------------------------------------------
/public/uploads/repos_graph.json:
--------------------------------------------------------------------------------
1 | {
2 | "links": [
3 | {
4 | "source": "spring-ai-chroma",
5 | "target": "spring-ai-openai"
6 | },
7 | {
8 | "source": "spring-ai-pinecone",
9 | "target": "spring-ai-gemfire"
10 | },
11 | {
12 | "source": "spring-ai-cassandra",
13 | "target": "spring-ai-transformers"
14 | },
15 | {
16 | "source": "spring-ai-openai",
17 | "target": "spring-ai-retry"
18 | },
19 | {
20 | "source": "spring-ai-elasticsearch-store",
21 | "target": "spring-ai-neo4j-store"
22 | },
23 | {
24 | "source": "spring-ai-spring-boot-autoconfigure",
25 | "target": "spring-ai-neo4j-store"
26 | },
27 | {
28 | "source": "spring-ai-pgvector-store",
29 | "target": "spring-ai-neo4j-store"
30 | },
31 | {
32 | "source": "spring-ai-hanadb-store",
33 | "target": "spring-ai-openai"
34 | },
35 | {
36 | "source": "spring-ai-ollama",
37 | "target": "spring-ai-core"
38 | },
39 | {
40 | "source": "spring-ai-chroma",
41 | "target": "spring-ai-gemfire"
42 | },
43 | {
44 | "source": "spring-ai-spring-boot-autoconfigure",
45 | "target": "spring-ai-anthropic"
46 | },
47 | {
48 | "source": "spring-ai-spring-boot-autoconfigure",
49 | "target": "spring-ai-qdrant"
50 | },
51 | {
52 | "source": "spring-ai-azure",
53 | "target": "spring-ai-core"
54 | },
55 | {
56 | "source": "spring-ai-spring-boot-autoconfigure",
57 | "target": "spring-ai-cassandra"
58 | },
59 | {
60 | "source": "spring-ai-spring-boot-autoconfigure",
61 | "target": "spring-ai-hanadb-store"
62 | },
63 | {
64 | "source": "spring-ai-anthropic",
65 | "target": "spring-ai-retry"
66 | },
67 | {
68 | "source": "spring-ai-mistral-ai",
69 | "target": "spring-ai-retry"
70 | },
71 | {
72 | "source": "spring-ai-weaviate",
73 | "target": "spring-ai-core"
74 | },
75 | {
76 | "source": "spring-ai-stability-ai",
77 | "target": "spring-ai-retry"
78 | },
79 | {
80 | "source": "spring-ai-azure",
81 | "target": "spring-ai-transformers"
82 | },
83 | {
84 | "source": "spring-ai-mongodb-atlas-store",
85 | "target": "spring-ai-neo4j-store"
86 | },
87 | {
88 | "source": "spring-ai-weaviate",
89 | "target": "spring-ai-transformers"
90 | },
91 | {
92 | "source": "spring-ai-pinecone",
93 | "target": "spring-ai-neo4j-store"
94 | },
95 | {
96 | "source": "spring-ai-postgresml",
97 | "target": "spring-ai-core"
98 | },
99 | {
100 | "source": "spring-ai-spring-boot-testcontainers",
101 | "target": "spring-ai-weaviate"
102 | },
103 | {
104 | "source": "spring-ai-gemfire",
105 | "target": "spring-ai-core"
106 | },
107 | {
108 | "source": "spring-ai-chroma",
109 | "target": "spring-ai-neo4j-store"
110 | },
111 | {
112 | "source": "spring-ai-spring-boot-autoconfigure",
113 | "target": "spring-ai-pgvector-store"
114 | },
115 | {
116 | "source": "spring-ai-milvus-store",
117 | "target": "spring-ai-openai"
118 | },
119 | {
120 | "source": "spring-ai-vertex-ai-gemini",
121 | "target": "spring-ai-core"
122 | },
123 | {
124 | "source": "spring-ai-spring-boot-autoconfigure",
125 | "target": "spring-ai-bedrock"
126 | },
127 | {
128 | "source": "spring-ai-milvus-store",
129 | "target": "spring-ai-core"
130 | },
131 | {
132 | "source": "spring-ai-core",
133 | "target": "spring-ai-gemfire"
134 | },
135 | {
136 | "source": "spring-ai-spring-boot-autoconfigure",
137 | "target": "spring-ai-chroma"
138 | },
139 | {
140 | "source": "spring-ai-gemfire",
141 | "target": "spring-ai-transformers"
142 | },
143 | {
144 | "source": "spring-ai-cassandra",
145 | "target": "spring-ai-gemfire"
146 | },
147 | {
148 | "source": "spring-ai-spring-boot-testcontainers",
149 | "target": "spring-ai-core"
150 | },
151 | {
152 | "source": "spring-ai-watsonx-ai",
153 | "target": "spring-ai-retry"
154 | },
155 | {
156 | "source": "spring-ai-qdrant",
157 | "target": "spring-ai-core"
158 | },
159 | {
160 | "source": "spring-ai-spring-boot-testcontainers",
161 | "target": "spring-ai-transformers"
162 | },
163 | {
164 | "source": "spring-ai-mongodb-atlas-store",
165 | "target": "spring-ai-core"
166 | },
167 | {
168 | "source": "spring-ai-spring-boot-autoconfigure",
169 | "target": "spring-ai-mistral-ai"
170 | },
171 | {
172 | "source": "spring-ai-spring-boot-testcontainers",
173 | "target": "spring-ai-ollama"
174 | },
175 | {
176 | "source": "spring-ai-spring-boot-autoconfigure",
177 | "target": "spring-ai-elasticsearch-store"
178 | },
179 | {
180 | "source": "spring-ai-azure-openai",
181 | "target": "spring-ai-core"
182 | },
183 | {
184 | "source": "spring-ai-azure",
185 | "target": "spring-ai-gemfire"
186 | },
187 | {
188 | "source": "spring-ai-spring-boot-autoconfigure",
189 | "target": "spring-ai-azure"
190 | },
191 | {
192 | "source": "spring-ai-weaviate",
193 | "target": "spring-ai-gemfire"
194 | },
195 | {
196 | "source": "spring-ai-redis",
197 | "target": "spring-ai-core"
198 | },
199 | {
200 | "source": "spring-ai-spring-boot-autoconfigure",
201 | "target": "spring-ai-weaviate"
202 | },
203 | {
204 | "source": "spring-ai-anthropic",
205 | "target": "spring-ai-core"
206 | },
207 | {
208 | "source": "spring-ai-cassandra",
209 | "target": "spring-ai-neo4j-store"
210 | },
211 | {
212 | "source": "spring-ai-elasticsearch-store",
213 | "target": "spring-ai-openai"
214 | },
215 | {
216 | "source": "spring-ai-redis",
217 | "target": "spring-ai-transformers"
218 | },
219 | {
220 | "source": "spring-ai-elasticsearch-store",
221 | "target": "spring-ai-core"
222 | },
223 | {
224 | "source": "spring-ai-spring-boot-autoconfigure",
225 | "target": "spring-ai-openai"
226 | },
227 | {
228 | "source": "spring-ai-spring-boot-autoconfigure",
229 | "target": "spring-ai-azure-openai"
230 | },
231 | {
232 | "source": "spring-ai-spring-boot-autoconfigure",
233 | "target": "spring-ai-core"
234 | },
235 | {
236 | "source": "spring-ai-pgvector-store",
237 | "target": "spring-ai-core"
238 | },
239 | {
240 | "source": "spring-ai-vertex-ai-palm2",
241 | "target": "spring-ai-core"
242 | },
243 | {
244 | "source": "spring-ai-huggingface",
245 | "target": "spring-ai-core"
246 | },
247 | {
248 | "source": "spring-ai-neo4j-store",
249 | "target": "spring-ai-core"
250 | },
251 | {
252 | "source": "spring-ai-spring-boot-autoconfigure",
253 | "target": "spring-ai-transformers"
254 | },
255 | {
256 | "source": "spring-ai-azure",
257 | "target": "spring-ai-neo4j-store"
258 | },
259 | {
260 | "source": "spring-ai-spring-boot-autoconfigure",
261 | "target": "spring-ai-postgresml"
262 | },
263 | {
264 | "source": "spring-ai-qdrant",
265 | "target": "spring-ai-openai"
266 | },
267 | {
268 | "source": "spring-ai-spring-boot-autoconfigure",
269 | "target": "spring-ai-pinecone"
270 | },
271 | {
272 | "source": "spring-ai-milvus-store",
273 | "target": "spring-ai-gemfire"
274 | },
275 | {
276 | "source": "spring-ai-mongodb-atlas-store",
277 | "target": "spring-ai-openai"
278 | },
279 | {
280 | "source": "spring-ai-weaviate",
281 | "target": "spring-ai-neo4j-store"
282 | },
283 | {
284 | "source": "spring-ai-openai",
285 | "target": "spring-ai-core"
286 | },
287 | {
288 | "source": "spring-ai-spring-boot-testcontainers",
289 | "target": "spring-ai-gemfire"
290 | },
291 | {
292 | "source": "pdf-reader",
293 | "target": "spring-ai-core"
294 | },
295 | {
296 | "source": "spring-ai-spring-boot-autoconfigure",
297 | "target": "spring-ai-milvus-store"
298 | },
299 | {
300 | "source": "spring-ai-pinecone",
301 | "target": "spring-ai-core"
302 | },
303 | {
304 | "source": "spring-ai-qdrant",
305 | "target": "spring-ai-gemfire"
306 | },
307 | {
308 | "source": "spring-ai-spring-boot-autoconfigure",
309 | "target": "spring-ai-stability-ai"
310 | },
311 | {
312 | "source": "spring-ai-mongodb-atlas-store",
313 | "target": "spring-ai-gemfire"
314 | },
315 | {
316 | "source": "spring-ai-pinecone",
317 | "target": "spring-ai-transformers"
318 | },
319 | {
320 | "source": "spring-ai-core",
321 | "target": "spring-ai-neo4j-store"
322 | },
323 | {
324 | "source": "spring-ai-mistral-ai",
325 | "target": "spring-ai-core"
326 | },
327 | {
328 | "source": "spring-ai-chroma",
329 | "target": "spring-ai-core"
330 | },
331 | {
332 | "source": "spring-ai-stability-ai",
333 | "target": "spring-ai-core"
334 | },
335 | {
336 | "source": "spring-ai-spring-boot-autoconfigure",
337 | "target": "spring-ai-mongodb-atlas-store"
338 | },
339 | {
340 | "source": "spring-ai-pgvector-store",
341 | "target": "spring-ai-openai"
342 | },
343 | {
344 | "source": "spring-ai-spring-boot-autoconfigure",
345 | "target": "spring-ai-vertex-ai-gemini"
346 | },
347 | {
348 | "source": "spring-ai-spring-boot-autoconfigure",
349 | "target": "spring-ai-vertex-ai-palm2"
350 | },
351 | {
352 | "source": "spring-ai-redis",
353 | "target": "spring-ai-gemfire"
354 | },
355 | {
356 | "source": "spring-ai-neo4j-store",
357 | "target": "spring-ai-openai"
358 | },
359 | {
360 | "source": "spring-ai-milvus-store",
361 | "target": "spring-ai-neo4j-store"
362 | },
363 | {
364 | "source": "spring-ai-transformers",
365 | "target": "spring-ai-core"
366 | },
367 | {
368 | "source": "spring-ai-hanadb-store",
369 | "target": "spring-ai-core"
370 | },
371 | {
372 | "source": "spring-ai-spring-boot-autoconfigure",
373 | "target": "spring-ai-huggingface"
374 | },
375 | {
376 | "source": "spring-ai-test",
377 | "target": "spring-ai-core"
378 | },
379 | {
380 | "source": "spring-ai-elasticsearch-store",
381 | "target": "spring-ai-gemfire"
382 | },
383 | {
384 | "source": "spring-ai-spring-boot-autoconfigure",
385 | "target": "spring-ai-gemfire"
386 | },
387 | {
388 | "source": "spring-ai-pgvector-store",
389 | "target": "spring-ai-gemfire"
390 | },
391 | {
392 | "source": "spring-ai-spring-boot-autoconfigure",
393 | "target": "spring-ai-redis"
394 | },
395 | {
396 | "source": "spring-ai-spring-boot-autoconfigure",
397 | "target": "spring-ai-ollama"
398 | },
399 | {
400 | "source": "spring-ai-neo4j-store",
401 | "target": "spring-ai-gemfire"
402 | },
403 | {
404 | "source": "spring-ai-bedrock",
405 | "target": "spring-ai-core"
406 | },
407 | {
408 | "source": "spring-ai-watsonx-ai",
409 | "target": "spring-ai-core"
410 | },
411 | {
412 | "source": "spring-ai-spring-boot-autoconfigure",
413 | "target": "spring-ai-retry"
414 | },
415 | {
416 | "source": "spring-ai-spring-boot-testcontainers",
417 | "target": "spring-ai-spring-boot-autoconfigure"
418 | },
419 | {
420 | "source": "spring-ai-spring-boot-autoconfigure",
421 | "target": "spring-ai-watsonx-ai"
422 | },
423 | {
424 | "source": "spring-ai-hanadb-store",
425 | "target": "pdf-reader"
426 | },
427 | {
428 | "source": "spring-ai-redis",
429 | "target": "spring-ai-neo4j-store"
430 | },
431 | {
432 | "source": "tika-reader",
433 | "target": "spring-ai-core"
434 | },
435 | {
436 | "source": "spring-ai-cassandra",
437 | "target": "spring-ai-core"
438 | },
439 | {
440 | "source": "spring-ai-openai",
441 | "target": "spring-ai-gemfire"
442 | }
443 | ],
444 | "nodes": [
445 | {
446 | "description": "spring-ai-weaviate",
447 | "fileCount": 4,
448 | "fileSize": 44839,
449 | "id": "spring-ai-weaviate"
450 | },
451 | {
452 | "description": "spring-ai-ollama",
453 | "fileCount": 16,
454 | "fileSize": 86427,
455 | "id": "spring-ai-ollama"
456 | },
457 | {
458 | "description": "spring-ai-anthropic",
459 | "fileCount": 16,
460 | "fileSize": 85563,
461 | "id": "spring-ai-anthropic"
462 | },
463 | {
464 | "description": "spring-ai-transformers",
465 | "fileCount": 5,
466 | "fileSize": 31142,
467 | "id": "spring-ai-transformers"
468 | },
469 | {
470 | "description": "spring-ai-elasticsearch-store",
471 | "fileCount": 4,
472 | "fileSize": 32427,
473 | "id": "spring-ai-elasticsearch-store"
474 | },
475 | {
476 | "description": "spring-ai-pinecone",
477 | "fileCount": 2,
478 | "fileSize": 22927,
479 | "id": "spring-ai-pinecone"
480 | },
481 | {
482 | "description": "spring-ai-stability-ai",
483 | "fileCount": 8,
484 | "fileSize": 36079,
485 | "id": "spring-ai-stability-ai"
486 | },
487 | {
488 | "description": "spring-ai-test",
489 | "fileCount": 1,
490 | "fileSize": 3431,
491 | "id": "spring-ai-test"
492 | },
493 | {
494 | "description": "spring-ai-watsonx-ai",
495 | "fileCount": 12,
496 | "fileSize": 49759,
497 | "id": "spring-ai-watsonx-ai"
498 | },
499 | {
500 | "description": "spring-ai-milvus-store",
501 | "fileCount": 3,
502 | "fileSize": 31500,
503 | "id": "spring-ai-milvus-store"
504 | },
505 | {
506 | "description": "tika-reader",
507 | "fileCount": 2,
508 | "fileSize": 8028,
509 | "id": "tika-reader"
510 | },
511 | {
512 | "description": "spring-ai-postgresml",
513 | "fileCount": 4,
514 | "fileSize": 25282,
515 | "id": "spring-ai-postgresml"
516 | },
517 | {
518 | "description": "pdf-reader",
519 | "fileCount": 10,
520 | "fileSize": 53677,
521 | "id": "pdf-reader"
522 | },
523 | {
524 | "description": "spring-ai-qdrant",
525 | "fileCount": 5,
526 | "fileSize": 34651,
527 | "id": "spring-ai-qdrant"
528 | },
529 | {
530 | "description": "spring-ai-hanadb-store",
531 | "fileCount": 8,
532 | "fileSize": 22451,
533 | "id": "spring-ai-hanadb-store"
534 | },
535 | {
536 | "description": "spring-ai-bedrock",
537 | "fileCount": 49,
538 | "fileSize": 279623,
539 | "id": "spring-ai-bedrock"
540 | },
541 | {
542 | "description": "spring-ai-gemfire",
543 | "fileCount": 2,
544 | "fileSize": 21891,
545 | "id": "spring-ai-gemfire"
546 | },
547 | {
548 | "description": "spring-ai-retry",
549 | "fileCount": 3,
550 | "fileSize": 5431,
551 | "id": "spring-ai-retry"
552 | },
553 | {
554 | "description": "spring-ai-vertex-ai-palm2",
555 | "fileCount": 11,
556 | "fileSize": 54728,
557 | "id": "spring-ai-vertex-ai-palm2"
558 | },
559 | {
560 | "description": "spring-ai-cassandra",
561 | "fileCount": 7,
562 | "fileSize": 90622,
563 | "id": "spring-ai-cassandra"
564 | },
565 | {
566 | "description": "spring-ai-spring-boot-testcontainers",
567 | "fileCount": 12,
568 | "fileSize": 35202,
569 | "id": "spring-ai-spring-boot-testcontainers"
570 | },
571 | {
572 | "description": "spring-ai-azure",
573 | "fileCount": 4,
574 | "fileSize": 40370,
575 | "id": "spring-ai-azure"
576 | },
577 | {
578 | "description": "spring-ai-azure-openai",
579 | "fileCount": 17,
580 | "fileSize": 88015,
581 | "id": "spring-ai-azure-openai"
582 | },
583 | {
584 | "description": "spring-ai-chroma",
585 | "fileCount": 7,
586 | "fileSize": 47095,
587 | "id": "spring-ai-chroma"
588 | },
589 | {
590 | "description": "spring-ai-mistral-ai",
591 | "fileCount": 18,
592 | "fileSize": 109079,
593 | "id": "spring-ai-mistral-ai"
594 | },
595 | {
596 | "description": "spring-ai-vertex-ai-gemini",
597 | "fileCount": 10,
598 | "fileSize": 52356,
599 | "id": "spring-ai-vertex-ai-gemini"
600 | },
601 | {
602 | "description": "spring-ai-neo4j-store",
603 | "fileCount": 4,
604 | "fileSize": 33698,
605 | "id": "spring-ai-neo4j-store"
606 | },
607 | {
608 | "description": "spring-ai-core",
609 | "fileCount": 152,
610 | "fileSize": 483667,
611 | "id": "spring-ai-core"
612 | },
613 | {
614 | "description": "spring-ai-openai",
615 | "fileCount": 64,
616 | "fileSize": 316162,
617 | "id": "spring-ai-openai"
618 | },
619 | {
620 | "description": "spring-ai-mongodb-atlas-store",
621 | "fileCount": 7,
622 | "fileSize": 31349,
623 | "id": "spring-ai-mongodb-atlas-store"
624 | },
625 | {
626 | "description": "spring-ai-spring-boot-autoconfigure",
627 | "fileCount": 168,
628 | "fileSize": 550300,
629 | "id": "spring-ai-spring-boot-autoconfigure"
630 | },
631 | {
632 | "description": "spring-ai-pgvector-store",
633 | "fileCount": 3,
634 | "fileSize": 28633,
635 | "id": "spring-ai-pgvector-store"
636 | },
637 | {
638 | "description": "spring-ai-huggingface",
639 | "fileCount": 3,
640 | "fileSize": 8294,
641 | "id": "spring-ai-huggingface"
642 | },
643 | {
644 | "description": "spring-ai-redis",
645 | "fileCount": 4,
646 | "fileSize": 36706,
647 | "id": "spring-ai-redis"
648 | }
649 | ]
650 | }
--------------------------------------------------------------------------------
/public/uploads/repos_readme.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "id": "spring-ai-test",
4 | "content": "TODO:\n\tDocumentation and sample tests using the `BasicEvaluationTest`."
5 | }
6 | ]
--------------------------------------------------------------------------------
/rag.py:
--------------------------------------------------------------------------------
1 | import subprocess
2 | from flask import Flask, request, jsonify, stream_with_context, Response
3 | from flask_cors import CORS
4 | from transformers import GPT2Tokenizer
5 | import logging
6 | import torch
7 | import numpy as np
8 | import gzip
9 | import pickle
10 | import json
11 | import os
12 | import pandas as pd
13 | import time
14 | from openai import OpenAI
15 |
16 | # Configure logging
17 | logging.basicConfig(level=logging.INFO)
18 | logger = logging.getLogger(__name__)
19 |
20 | def start_react_app():
21 | subprocess.Popen(["npm", "start"])
22 |
23 | app = Flask(__name__)
24 | CORS(app, resources={r"/*": {"origins": "*"}}) # Enable CORS for all routes
25 |
26 | client = OpenAI(base_url="http://localhost:8081/v1", api_key="lm-studio")
27 | chat_history = []
28 | current_progress = 0
29 | library = None
30 | db_filename = None
31 |
32 | class MBedFastAF:
33 | def __init__(self, model_name, similarity_metric='cosine'):
34 | self.device = torch.device("cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu")
35 | self.documents = []
36 | self.full_data = [] # Store full data for each document
37 | self.vectors = np.array([])
38 | self.tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
39 | self.set_similarity_metric(similarity_metric)
40 |
41 | def set_similarity_metric(self, metric):
42 | metrics = {
43 | 'cosine': self.cosine_similarity,
44 | 'euclidean': self.euclidean_metric
45 | }
46 | if metric in metrics:
47 | self.similarity_metric = metrics[metric]
48 | else:
49 | raise ValueError(f"Unsupported similarity metric '{metric}'")
50 |
51 | def add_documents(self, documents, full_data):
52 | global current_progress
53 | total_docs = len(documents)
54 | processed_docs = 0
55 | embeddings = []
56 | for doc, data in zip(documents, full_data):
57 | # Ensure doc is a dictionary before proceeding
58 | if not isinstance(doc, dict):
59 | raise ValueError('Document must be a dictionary.')
60 |
61 | # Concatenate all values in the dictionary to create a single string
62 | doc_text = " ".join(str(value) for value in doc.values())
63 |
64 | embedding = self.get_embedding(doc_text)
65 | embeddings.append(embedding)
66 | self.full_data.append(data)
67 | processed_docs += 1
68 | embeddings = np.vstack(embeddings)
69 | self.documents.extend(documents)
70 | if self.vectors.size == 0:
71 | self.vectors = embeddings
72 | else:
73 | self.vectors = np.vstack((self.vectors, embeddings))
74 |
75 |
76 | def get_embedding(self, text):
77 | text = text.replace("\n", " ")
78 | response = client.embeddings.create(input=[text], model="nomic-ai/nomic-embed-text-v1.5-GGUF")
79 | return response.data[0].embedding
80 |
81 | def get_embeddings(self, documents):
82 | embeddings = [self.get_embedding(doc) for doc in documents]
83 | return np.array(embeddings)
84 |
85 | def query(self, query_document, top_k=5):
86 | query_vector = self.get_embeddings([query_document])[0]
87 | similarities = self.similarity_metric(self.vectors, query_vector)
88 | top_indices = np.argsort(similarities)[::-1]
89 |
90 | unique_documents = {}
91 | results = []
92 | for index in top_indices:
93 | doc_id = self.documents[index]
94 | doc_id_str = str(doc_id) # Convert doc_id to a string
95 | if doc_id_str not in unique_documents:
96 | unique_documents[doc_id_str] = True
97 | results.append((self.documents[index], self.full_data[index], similarities[index]))
98 | if len(results) >= top_k:
99 | break
100 |
101 | return results
102 |
103 | @staticmethod
104 | def cosine_similarity(vectors, query_vector):
105 | vectors_norm = np.linalg.norm(vectors, axis=1)
106 | query_norm = np.linalg.norm(query_vector)
107 | return np.dot(vectors, query_vector) / (vectors_norm * query_norm)
108 |
109 | @staticmethod
110 | def euclidean_metric(vectors, query_vector):
111 | distances = np.linalg.norm(vectors - query_vector, axis=1)
112 | return 1 / (1 + distances)
113 |
114 | def save(self, filename):
115 | with gzip.open(filename, 'wb') as f:
116 | data = {'documents': self.documents, 'full_data': self.full_data, 'vectors': self.vectors}
117 | pickle.dump(data, f)
118 |
119 | def load(self, filename):
120 | with gzip.open(filename, 'rb') as f:
121 | data = pickle.load(f)
122 | self.documents = data['documents']
123 | self.full_data = data['full_data']
124 | self.vectors = data['vectors']
125 |
126 | db_path = os.path.join(os.path.dirname(__file__), 'public', 'databases')
127 |
128 | def extract_keys_from_json(data, parent_key=''):
129 | keys = {}
130 | if isinstance(data, list):
131 | for item in data:
132 | keys.update(extract_keys_from_json(item, parent_key))
133 | elif isinstance(data, dict):
134 | for k, v in data.items():
135 | full_key = f"{parent_key}.{k}" if parent_key else k
136 | if isinstance(v, (dict, list)):
137 | keys.update(extract_keys_from_json(v, full_key))
138 | else:
139 | keys[full_key] = type(v).__name__
140 | return keys
141 |
142 | def trim_chat_history():
143 | total_tokens = sum(len(message['content'].split()) for message in chat_history)
144 | while total_tokens > 2000:
145 | removed_message = chat_history.pop(0)
146 | total_tokens -= len(removed_message['content'].split())
147 |
148 | def extract_texts_from_csv(df, keys):
149 | documents = []
150 | for _, row in df.iterrows():
151 | doc = {}
152 | for key in keys:
153 | if key in row:
154 | doc[key] = str(row[key])
155 | else:
156 | return jsonify({'error': f'Column "{key}" not found in CSV file'}), 400
157 | documents.append(doc)
158 | return documents
159 |
160 | def extract_texts_from_json(data, keys):
161 | documents = []
162 | for item in data:
163 | doc = {}
164 | for key in keys:
165 | parts = key.split('.')
166 | value = item
167 | try:
168 | for part in parts:
169 | value = value[part]
170 | doc[key] = value
171 | except (KeyError, TypeError):
172 | doc[key] = None
173 | documents.append(doc)
174 | return documents
175 |
176 | @app.route('/preview_document', methods=['POST'])
177 | def preview_document():
178 | global library
179 | data = request.get_json()
180 | file_path = data.get('file_path')
181 | selected_keys = data.get('selected_keys', [])
182 |
183 | file_path = os.path.join('./public/uploads', os.path.basename(file_path))
184 |
185 | if not file_path or not os.path.exists(file_path):
186 | return jsonify({'error': 'Invalid file path'}), 400
187 |
188 | if not selected_keys:
189 | return jsonify({'error': 'No keys provided'}), 400
190 |
191 | if library is None:
192 | library = MBedFastAF('nomic-ai/nomic-embed-text-v1.5-GGUF', 'euclidean')
193 |
194 | file_extension = os.path.splitext(file_path)[1].lower()
195 |
196 | documents = []
197 | if file_extension == '.json':
198 | with open(file_path, 'r') as file:
199 | json_data = json.load(file)
200 | documents = extract_texts_from_json(json_data, selected_keys)
201 | elif file_extension == '.csv':
202 | df = pd.read_csv(file_path)
203 | documents = extract_texts_from_csv(df, selected_keys)
204 | else:
205 | return jsonify({'error': 'Unsupported file format. Only JSON and CSV are supported.'}), 400
206 |
207 | if not documents:
208 | return jsonify({'error': 'No documents extracted'}), 400
209 |
210 | document = documents[0]
211 | print("Generated document: ", document)
212 |
213 | # Ensure document is a dictionary before proceeding
214 | if not isinstance(document, dict):
215 | return jsonify({'error': 'Extracted document is not a dictionary'}), 400
216 |
217 | document_values = " ".join(str(value) for value in document.values())
218 | embeddings = library.get_embeddings([document_values])
219 | token_count = sum([len(library.tokenizer.tokenize(doc)) for doc in [document_values]])
220 |
221 | return jsonify({"document": document, "embeddings": embeddings.tolist(), "token_count": token_count})
222 |
223 | @app.route('/create_vector_database', methods=['POST'])
224 | def create_vector_database():
225 | global current_progress, library
226 | current_progress = 0
227 | data = request.get_json()
228 | file_path = data.get('file_path')
229 | selected_keys = data.get('selected_keys', [])
230 |
231 | file_path = os.path.join('./public/uploads', os.path.basename(file_path))
232 |
233 | if not file_path or not os.path.exists(file_path):
234 | return jsonify({'error': 'Invalid file path'}), 400
235 |
236 | print("File path exists: ", os.path.exists(file_path))
237 | print("Selected keys: ", selected_keys)
238 | print("File path: ", file_path)
239 |
240 | file_extension = os.path.splitext(file_path)[1].lower()
241 |
242 | documents = []
243 | full_data = []
244 | if file_extension == '.json':
245 | with open(file_path, 'r') as file:
246 | json_data = json.load(file)
247 | documents = extract_texts_from_json(json_data, selected_keys)
248 | full_data = json_data
249 | elif file_extension == '.csv':
250 | df = pd.read_csv(file_path)
251 | documents = extract_texts_from_csv(df, selected_keys)
252 | full_data = df.to_dict(orient='records')
253 | else:
254 | return jsonify({'error': 'Unsupported file format. Only JSON and CSV are supported.'}), 400
255 |
256 | # Extract the base name of the file (without extension) to use as the database file name
257 | base_name = os.path.splitext(os.path.basename(file_path))[0]
258 | db_filename = os.path.join(db_path, f"{base_name}_vector_database.pkl.gz")
259 |
260 | print(f"Creating {db_filename} with {len(documents)} documents from the selected keys: {selected_keys}...")
261 | # Create a new instance of MBedFastAF
262 | library = MBedFastAF('nomic-ai/nomic-embed-text-v1.5-GGUF', 'euclidean')
263 |
264 | def generate_progress():
265 | global current_progress
266 | total_docs = len(documents)
267 | processed_docs = 0
268 |
269 | logger.info(f"Creating vector database using selected keys: {selected_keys}")
270 |
271 | for document, data in zip(documents, full_data):
272 | library.add_documents([document], [data])
273 | processed_docs += 1
274 | current_progress = processed_docs / total_docs * 100
275 | logger.info(f"Processed {processed_docs} out of {total_docs} documents. Progress: {current_progress:.2f}%")
276 | yield f"data: {current_progress:.2f}\n\n"
277 |
278 | library.save(db_filename)
279 | logger.info(f"Created vector database: {db_filename}")
280 | yield f"data: {db_filename}\n\n"
281 |
282 | return Response(stream_with_context(generate_progress()), mimetype='text/event-stream')
283 |
284 | @app.route('/check_vector_db', methods=['GET'])
285 | def check_vector_db():
286 | global db_filename
287 | if os.path.exists(db_filename):
288 | return jsonify({'message': 'Valid Vector DB found'}), 200
289 | else:
290 | return jsonify({'error': 'No Vector DB found. Please create one!'}), 404
291 |
292 | @app.route('/db_stats', methods=['GET'])
293 | def db_stats():
294 | global library, db_filename
295 | if not os.path.exists(db_filename):
296 | return jsonify({'error': 'Database not found'}), 404
297 |
298 | library.load(db_filename)
299 | total_documents = len(library.documents)
300 | avg_vector_length = np.mean([len(vector) for vector in library.vectors])
301 |
302 | return jsonify({
303 | 'total_documents': total_documents,
304 | 'avg_vector_length': avg_vector_length
305 | }), 200
306 |
307 | @app.route('/backup_db', methods=['POST'])
308 | def backup_db():
309 | global db_filename
310 | if not os.path.exists(db_filename):
311 | return jsonify({'error': 'Database not found'}), 404
312 |
313 | backup_filename = f"{db_filename}.bak"
314 | with open(db_filename, 'rb') as f:
315 | with open(backup_filename, 'wb') as backup_f:
316 | backup_f.write(f.read())
317 |
318 | return '', 200
319 |
320 | @app.route('/delete_db', methods=['POST'])
321 | def delete_db():
322 | global db_filename
323 | if os.path.exists(db_filename):
324 | os.remove(db_filename)
325 | return '', 200
326 | else:
327 | return jsonify({'error': 'Database not found'}), 404
328 |
329 | @app.route('/upload_file', methods=['POST'])
330 | def upload_file():
331 | if 'file' not in request.files:
332 | return jsonify({'error': 'No file provided'}), 400
333 |
334 | file = request.files['file']
335 | if file.filename == '':
336 | return jsonify({'error': 'No selected file'}), 400
337 |
338 | if file:
339 | file_path = os.path.join('./public/uploads/', file.filename)
340 | file.save(file_path)
341 | return jsonify({'file_path': file_path}), 200
342 |
343 | @app.route('/rename_file', methods=['POST'])
344 | def rename_file():
345 | data = request.get_json()
346 | old_name = data.get('old_name')
347 | new_name = data.get('new_name')
348 |
349 | if not old_name or not new_name:
350 | return jsonify({'error': 'Invalid filenames provided'}), 400
351 |
352 | old_path = os.path.join('./public/uploads', old_name)
353 | new_path = os.path.join('./public/uploads', new_name)
354 |
355 | if not os.path.exists(old_path):
356 | return jsonify({'error': 'File not found'}), 404
357 |
358 | if os.path.exists(new_path):
359 | return jsonify({'error': 'New filename already exists'}), 400
360 |
361 | try:
362 | os.rename(old_path, new_path)
363 | return jsonify({'message': 'File renamed successfully'}), 200
364 | except Exception as e:
365 | return jsonify({'error': str(e)}), 500
366 |
367 |
368 | @app.route('/preview_file', methods=['POST'])
369 | def preview_file():
370 | data = request.get_json()
371 | file_path = data.get('file_path')
372 |
373 | # Prepend the upload directory to the file path
374 | file_path = os.path.join('./public/uploads', os.path.basename(file_path))
375 |
376 | if not file_path or not os.path.exists(file_path):
377 | return jsonify({'error': 'Invalid file path'}), 400
378 |
379 | file_extension = os.path.splitext(file_path)[1].lower()
380 | file_size = os.path.getsize(file_path)
381 |
382 | preview_data = {}
383 | if file_extension == '.json':
384 | with open(file_path, 'r') as file:
385 | json_data = json.load(file)
386 | preview_data = extract_keys_from_json(json_data)
387 | elif file_extension == '.csv':
388 | df = pd.read_csv(file_path)
389 | preview_data = list(df.columns)
390 | else:
391 | return jsonify({'error': 'Unsupported file format. Only JSON and CSV are supported.'}), 400
392 |
393 | return jsonify({
394 | 'previewData': preview_data,
395 | 'fileSize': file_size
396 | }), 200
397 |
398 | @app.route('/start_tsne', methods=['POST'])
399 | def start_tsne():
400 | global library, db_filename
401 | if not os.path.exists(db_filename):
402 | return jsonify({'error': 'Database not found'}), 404
403 |
404 | library.load(db_filename)
405 | vectors = library.vectors
406 |
407 | # Perform t-SNE in 3D
408 | tsne = TSNE(n_components=3, random_state=42)
409 | tsne_result = tsne.fit_transform(vectors)
410 |
411 | result = {
412 | 'coordinates': tsne_result.tolist(),
413 | 'documents': library.documents
414 | }
415 |
416 | # Save the t-SNE result to a file (optional)
417 | with open('tsne_result.json', 'w') as f:
418 | json.dump(result, f)
419 |
420 | return '', 200
421 |
422 | @app.route('/tsne_progress', methods=['GET'])
423 | def tsne_progress():
424 | if not os.path.exists('tsne_result.json'):
425 | return jsonify({'error': 't-SNE result not found'}), 404
426 |
427 | def generate_tsne_progress():
428 | progress = 0
429 | while progress < 100:
430 | progress += 10
431 | yield f"data: {progress}\n\n"
432 | time.sleep(1)
433 | with open('tsne_result.json', 'r') as f:
434 | result = json.load(f)
435 | yield f"data: {json.dumps(result)}\n\n"
436 |
437 | return Response(stream_with_context(generate_tsne_progress()), mimetype='text/event-stream')
438 |
439 | @app.route('/list_vector_dbs', methods=['GET'])
440 | def list_vector_dbs():
441 | upload_folder = './public/databases'
442 | try:
443 | files = os.listdir(upload_folder)
444 | file_details = []
445 | for file in files:
446 | file_path = os.path.join(upload_folder, file)
447 | if os.path.isfile(file_path):
448 | file_info = {
449 | 'name': file,
450 | 'size': os.path.getsize(file_path),
451 | 'upload_date': time.ctime(os.path.getctime(file_path)),
452 | 'extension': os.path.splitext(file)[1]
453 | }
454 | file_details.append(file_info)
455 | return jsonify(file_details), 200
456 | except Exception as e:
457 | return jsonify({'error': str(e)}), 500
458 |
459 | @app.route('/list_uploads', methods=['GET'])
460 | def list_uploads():
461 | upload_folder = './public/uploads'
462 | try:
463 | files = os.listdir(upload_folder)
464 | file_details = []
465 | for file in files:
466 | file_path = os.path.join(upload_folder, file)
467 | if os.path.isfile(file_path):
468 | file_info = {
469 | 'name': file,
470 | 'size': os.path.getsize(file_path),
471 | 'upload_date': time.ctime(os.path.getctime(file_path)),
472 | 'extension': os.path.splitext(file)[1]
473 | }
474 | file_details.append(file_info)
475 | return jsonify(file_details), 200
476 | except Exception as e:
477 | return jsonify({'error': str(e)}), 500
478 |
479 | @app.route('/progress', methods=['GET'])
480 | def progress():
481 | global db_filename
482 | def generate_progress():
483 | global current_progress
484 | while current_progress < 100:
485 | yield f"data: {current_progress}\n\n"
486 | time.sleep(1)
487 |
488 | if current_progress == 100:
489 | logger.info(f"Finished creating vector database: {db_filename}")
490 | yield f"data: 100\n\n"
491 |
492 | return Response(stream_with_context(generate_progress()), mimetype='text/event-stream')
493 | @app.route('/query', methods=['POST'])
494 | def query():
495 | global library
496 | data = request.get_json()
497 | query_text = data.get('query_text')
498 | similarity_metric = data.get('similarity_metric')
499 | db_filename = data.get('db_filename')
500 |
501 | if not db_filename:
502 | return jsonify({'error': 'No database file specified'}), 400
503 |
504 | db_filepath = os.path.join(db_path, db_filename)
505 | if not os.path.exists(db_filepath):
506 | return jsonify({'error': 'Database file not found'}), 404
507 |
508 | library = MBedFastAF('nomic-ai/nomic-embed-text-v1.5-GGUF', similarity_metric)
509 | library.load(db_filepath)
510 |
511 | logger.info(f"Querying with text: {query_text}")
512 | results = library.query(query_text, top_k=5)
513 | logger.info("Query complete.")
514 |
515 | # Convert float32 to float for JSON serialization
516 | results = [(result[0], result[1], float(result[2])) for result in results]
517 |
518 | return jsonify(results)
519 |
520 |
521 | @app.route('/list_files', methods=['GET'])
522 | def list_files():
523 | upload_folder = 'uploads'
524 | files = os.listdir(upload_folder)
525 | return jsonify(files)
526 |
527 | @app.route('/merge_files', methods=['POST'])
528 | def merge_files():
529 | data = request.get_json()
530 | files = data.get('files', [])
531 |
532 | if not files:
533 | return jsonify({'error': 'No files provided'}), 400
534 |
535 | merged_data = []
536 | for file in files:
537 | file_path = os.path.join('uploads', file)
538 | if os.path.exists(file_path):
539 | file_extension = os.path.splitext(file_path)[1].lower()
540 | if file_extension == '.json':
541 | with open(file_path, 'r') as f:
542 | merged_data.extend(json.load(f))
543 | elif file_extension == '.csv':
544 | df = pd.read_csv(file_path)
545 | merged_data.extend(df.to_dict(orient='records'))
546 | else:
547 | return jsonify({'error': f'Unsupported file format: {file}'}), 400
548 | else:
549 | return jsonify({'error': f'File not found: {file}'}), 400
550 |
551 | merged_file = 'merged_dataset.json'
552 | with open(os.path.join('uploads', merged_file), 'w') as f:
553 | json.dump(merged_data, f)
554 |
555 | return jsonify({'file': merged_file}), 200
556 |
557 | @app.route('/chatbot_format', methods=['POST'])
558 | def chatbot_format():
559 | data = request.get_json()
560 | user_query = data.get('query', '')
561 |
562 | if 'file_path' in data:
563 | file_path = data['file_path']
564 | if not file_path or not os.path.exists(file_path):
565 | return jsonify({'error': 'Invalid file path'}), 400
566 |
567 | with open(file_path, 'r') as file:
568 | raw_data = file.read()
569 |
570 | user_message = {
571 | "role": "user",
572 | "content": f"You are an extremely capable data scientist and you excel at finding programmatic solutions to clean up unstructured data. You'll find below an excerpt from the user data and your task is to analyze it and come up with a script that will clean it up and turn it into a nicely formatted dataset like a json, csv, or whatever else you decide, based on your best judgement:\n\n{raw_data}"
573 | }
574 | else:
575 | user_message = {"role": "user", "content": user_query}
576 |
577 | chat_history.append(user_message)
578 | trim_chat_history()
579 |
580 | def generate():
581 | completion = client.chat.completions.create(
582 | model="lmstudio-ai/codeqwen60k",
583 | messages=chat_history,
584 | temperature=0.7,
585 | stream=True,
586 | )
587 |
588 | assistant_message = {"role": "assistant", "content": ""}
589 | for chunk in completion:
590 | if chunk.choices[0].delta.content:
591 | delta_content = chunk.choices[0].delta.content
592 | assistant_message["content"] += delta_content
593 | yield f"data: {delta_content}\n\n"
594 |
595 | chat_history.append(assistant_message)
596 | trim_chat_history()
597 |
598 | return Response(stream_with_context(generate()), content_type='text/event-stream')
599 |
600 | if __name__ == '__main__':
601 | upload_folder = 'uploads'
602 | if not os.path.exists(upload_folder):
603 | os.makedirs(upload_folder)
604 | # start_react_app()
605 | app.run(debug=True, host='0.0.0.0', port=4000)
606 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | Flask==3.0.3
2 | Flask_Cors==4.0.1
3 | numpy==1.22.0
4 | openai==1.30.1
5 | pandas==1.5.3
6 | Requests==2.31.0
7 | torch==2.3.0
8 | transformers==4.41.0
9 |
--------------------------------------------------------------------------------
/src/App.css:
--------------------------------------------------------------------------------
1 | :root {
2 | --primary-color: #007bff;
3 | --secondary-color: #6c757d;
4 | --accent-color: #ffc107;
5 | --background-color: #f8f9fa;
6 | --text-color: #343a40;
7 | --border-color: #dee2e6;
8 | --header-height: 3.5rem;
9 | --font-family: 'Montserrat', sans-serif;
10 | --font-size-large: 1.5rem;
11 | --font-size-medium: 1rem;
12 | --font-size-small: 0.875rem;
13 | --border-radius: 8px;
14 | --transition: all 0.3s ease;
15 | }
16 |
17 | body {
18 | font-family: var(--font-family);
19 | margin: 0;
20 | padding: 0;
21 | display: flex;
22 | flex-direction: column;
23 | min-height: 100vh;
24 | background-color: var(--background-color);
25 | color: var(--text-color);
26 | }
27 |
28 |
29 | header {
30 | background-color: #00beff73;
31 | color: #fff;
32 | padding-top: 1rem;
33 | padding-right: 1rem;
34 | padding-bottom: 1rem;
35 | padding-left: 1rem;
36 | text-align: center;
37 | }
38 |
39 | nav {
40 | margin-top: 1rem;
41 | display: flex;
42 | justify-content: center;
43 | background-color: var(--primary-color);
44 | padding: 0.5rem;
45 | }
46 |
47 | nav a {
48 | color: #fff;
49 | margin: 0 1rem;
50 | text-decoration: none;
51 | font-size: var(--font-size-medium);
52 | transition: var(--transition);
53 | }
54 |
55 | nav a:hover,
56 | nav a.active {
57 | color: var(--accent-color);
58 | border-bottom: 2px solid var(--accent-color);
59 | }
60 |
61 | .main-content {
62 | padding: 1rem;
63 | margin-top: 80px;
64 | flex: 1;
65 | padding-left: 1rem;
66 | transition: all 1.3s ease-in-out
67 | }
68 |
69 | h2,
70 | h3 {
71 | color: var(--text-color);
72 | text-align: center;
73 | }
74 |
75 |
76 |
77 | .circular-progress {
78 | position: relative;
79 | width: 100px;
80 | height: 100px;
81 | margin: 0 auto;
82 | }
83 |
84 | .circular-progress svg {
85 | width: 100%;
86 | height: 100%;
87 | transform: rotate(-90deg);
88 | overflow: visible;
89 | }
90 |
91 | .circular-progress circle {
92 | fill: none;
93 | stroke-width: 10;
94 | }
95 |
96 | .circular-progress .bg {
97 | stroke: var(--border-color);
98 | }
99 |
100 | .circular-progress .progress {
101 | stroke: var(--primary-color);
102 | stroke-linecap: round;
103 | stroke-dasharray: 314;
104 | stroke-dashoffset: 314;
105 | transition: stroke-dashoffset 0.5s;
106 | }
107 |
108 | .circular-progress .percentage {
109 | position: absolute;
110 | top: 50%;
111 | left: 50%;
112 | transform: translate(-50%, -50%);
113 | font-size: var(--font-size-medium);
114 | color: var(--text-color);
115 | }
116 |
117 | #fileList {
118 | display: flex;
119 | position: relative;
120 | max-height: 50vh;
121 | margin-top: 17px;
122 | margin-right: 17px;
123 | max-width: 43vw;
124 | margin-left: 17px;
125 | padding: 12px;
126 | overflow-wrap: anywhere;
127 | box-shadow: black 0px 8px 20px 1px inset;
128 | border-radius: 16px;
129 | background: rgba(0, 0, 0, 0.5);
130 | flex-direction: row;
131 | justify-content: flex-start;
132 | overflow-y: auto;
133 | overflow-x: clip;
134 | }
135 |
136 | #uploadContainer.dragover {
137 | background-color: rgba(0, 123, 255, 0.2);
138 | }
139 |
140 | #uploadContainer.uploaded {
141 | background-color: rgba(255, 193, 7, 0.2);
142 | }
143 |
144 | #uploadContainer input[type="file"] {
145 | position: relative;
146 | width: 100%;
147 | height: 100%;
148 | opacity: 0;
149 | cursor: pointer;
150 | }
151 |
152 | #textKeys,
153 | #documentPreview,
154 | #vectorEmbeddings {
155 | background-color: #fff;
156 | color: var(--text-color);
157 | border-radius: var(--border-radius);
158 | box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1);
159 | border: 1px solid var(--border-color);
160 | padding: 1rem;
161 | transition: var(--transition);
162 | flex: 1;
163 | width: 100%;
164 | overflow-y: auto;
165 | margin-bottom: 1rem;
166 | }
167 |
168 | footer {
169 | background-color: var(--primary-color);
170 | color: #fff;
171 | padding: 1rem;
172 | text-align: center;
173 | }
174 |
175 | #previewContainer {
176 | display: flex;
177 | flex: 1;
178 | min-height: 63vh;
179 | flex-direction: row;
180 | gap: 1rem;
181 | }
182 |
183 | .preview-column {
184 | flex: 1;
185 | display: flex;
186 | flex-direction: column;
187 | align-items: center;
188 | }
189 |
190 | .result-card {
191 | border: 1px solid var(--border-color);
192 | border-radius: var(--border-radius);
193 | padding: 1rem;
194 | width: 100%;
195 | background-color: #fff;
196 | box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1);
197 | margin-bottom: 1rem;
198 | }
199 |
200 | .result-header {
201 | display: flex;
202 | justify-content: space-between;
203 | align-items: center;
204 | }
205 |
206 | #similarityMetric {
207 | display: flex;
208 | justify-content: space-between;
209 | align-items: center;
210 | background-color: var(--primary-color);
211 | color: #fff;
212 | border-radius: var(--border-radius);
213 | padding: 0.5rem;
214 | margin-bottom: 1rem;
215 | }
216 | .button,
217 | .inspect-button,
218 | .button-container button {
219 | padding: 0.5rem 1rem;
220 | background-color: var(--primary-color);
221 | color: #fff;
222 | border: none;
223 | margin-right: 4px;
224 | margin-left: 4px;
225 | border-radius: var(--border-radius);
226 | cursor: pointer;
227 | transition: var(--transition);
228 | }
229 |
230 | .question-button {
231 | padding: 0.5rem 1rem;
232 | background-color: var(--text-color);
233 | color: #fff;
234 | border: none;
235 | margin-right: 4px;
236 | margin-left: 4px;
237 | border-radius: var(--border-radius);
238 | cursor: pointer;
239 | transition: var(--transition);
240 | }
241 |
242 | .question-button:hover {
243 | transform: scale(0.90);
244 | background-color: var(--secondary-color);
245 | }
246 |
247 |
248 | #sendButton {
249 | background-color: var(--secondary-color);
250 | color: #fff;
251 | border: none;
252 | border-radius: var(--border-radius);
253 | cursor: pointer;
254 | transition: var(--transition);
255 | }
256 |
257 | #sendButton:hover {
258 | background-color: var(--accent-color);
259 | box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);
260 | }
261 |
262 |
263 | .button:hover,
264 | .inspect-button:hover,
265 | .button-container button:hover {
266 | background-color: var(--accent-color);
267 | box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);
268 | }
269 |
270 | .modal {
271 | position: fixed;
272 | top: 0;
273 | left: 0;
274 | width: 100%;
275 | height: 100%;
276 | background-color: rgba(0, 0, 0, 0.5);
277 | display: flex;
278 | justify-content: center;
279 | align-items: center;
280 | }
281 |
282 | .modal-content {
283 | background-color: #fff;
284 | padding: 2rem;
285 | border-radius: var(--border-radius);
286 | max-width: 80%;
287 | max-height: 80%;
288 | overflow-y: auto;
289 | }
290 |
291 | .close-button {
292 | position: absolute;
293 | top: 10px;
294 | right: 10px;
295 | font-size: 1.5rem;
296 | cursor: pointer;
297 | }
298 |
299 | .button-container {
300 | margin-top: 1rem;
301 | display: flex;
302 | justify-content: flex-end;
303 | flex-direction: column;
304 | }
305 |
306 | .button-container button {
307 | margin-left: 0.5rem;
308 | }
309 |
310 | .result {
311 | margin-bottom: 1rem;
312 | }
313 |
314 | .dashboard {
315 | display: flex;
316 | flex-direction: column;
317 | align-items: center;
318 | text-align: center;
319 | min-height: 60vh;
320 | border: 1px solid var(--border-color);
321 | background-color: #ffc30d73;
322 | border-radius: var(--border-radius);
323 | box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1);
324 | padding: 1rem;
325 | }
326 |
327 | #resultsContainer {
328 | max-height: 50vh;
329 | min-width: 96vw;
330 | padding: 1rem;
331 | border-radius: var(--border-radius);
332 | box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1);
333 | overflow-y: auto;
334 | }
335 |
336 | #upload-container {
337 | display: flex;
338 | gap: 1rem;
339 | position: relative;
340 | flex-direction: row;
341 | align-items: stretch;
342 | }
343 |
344 | #leftPanel {
345 | height: 35vh;
346 | }
347 |
348 | #leftPanel,
349 | #rightPanel {
350 | flex: 1;
351 | min-height: 80vh;
352 | text-align: -webkit-center;
353 | background-color: #fff;
354 | box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1);
355 | border: 1px solid var(--border-color);
356 | padding: 1rem;
357 | border-radius: var(--border-radius);
358 | transition: var(--transition);
359 | }
360 |
361 | #rightPanel:hover,
362 | #leftPanel:hover {
363 | transform: scale(1.01);
364 | }
365 |
366 | .file-item {
367 | display: flex;
368 | align-items: center;
369 | margin: 8px 0;
370 | padding: 8px;
371 | border-radius: 8px;
372 | background: #fff;
373 | transition: transform 0.2s, background 0.2s;
374 | }
375 |
376 | .file-item:hover {
377 | transform: scale(1.02);
378 | background: #f0f0f0;
379 | }
380 |
381 | .file-details {
382 | margin-left: 16px;
383 | }
384 |
385 | .file-name {
386 | font-weight: bold;
387 | }
388 |
389 | .file-info {
390 | font-size: 0.9em;
391 | color: #666;
392 | }
393 |
394 | .file-info-box {
395 | text-align: center;
396 | padding: 16px;
397 | margin-left: 32px;
398 | margin-right: 32px;
399 | background: darksalmon;
400 | margin-bottom: 16px;
401 | border-bottom-right-radius: 16px;
402 | border-bottom-left-radius: 16px;
403 | width: fit-content;
404 | }
405 |
406 | .sample-data {
407 | width: 100%;
408 | height: 50%;
409 | }
410 |
411 | .fa-file-upload,
412 | .fa-file-code,
413 | .fa-file-csv,
414 | .fa-file {
415 | font-size: 2em;
416 | margin-right: 8px;
417 | }
--------------------------------------------------------------------------------
/src/App.js:
--------------------------------------------------------------------------------
1 | import React, { useState } from "react";
2 | import { BrowserRouter as Router, Routes, Route, useLocation } from "react-router-dom";
3 | import NavbarHook from "./NavbarHook/NavbarHook";
4 | import Upload from "./pages/Upload";
5 | import Preview from "./pages/preview/Preview";
6 | import Embed from "./pages/embed/Embed";
7 | import Query from "./pages/query/Query";
8 | import TransitionWrapper from "./transition/Transition";
9 | import './App.css';
10 |
11 | const AppRoutes = ({ handleRouteChange, direction, previewData, filePath, fileSize }) => {
12 | const location = useLocation();
13 |
14 | return (
15 |
16 |
17 |
22 |
23 | );
24 | };
25 | const App = () => {
26 | const [previewData, setPreviewData] = useState(null);
27 | const [filePath, setFilePath] = useState(null);
28 | const [fileSize, setFileSize] = useState(null);
29 | const [selectedKeys, setSelectedKeys] = useState([]);
30 | const [navigationDirection, setNavigationDirection] = useState('forward');
31 |
32 | const handlePreviewData = (data, path, size) => {
33 | setPreviewData(data);
34 | setFileSize(size);
35 | setFilePath(path);
36 | setSelectedKeys(data || []); // Ensure data.keys is defined
37 | };
38 |
39 | const determineDirection = (currentPath, nextPath) => {
40 | const paths = ['/', '/preview', '/embed', '/query'];
41 | const currentIndex = paths.indexOf(currentPath);
42 | const nextIndex = paths.indexOf(nextPath);
43 |
44 | return nextIndex > currentIndex ? 'forward' : 'backward';
45 | };
46 |
47 | const handleRouteChange = (nextPath) => {
48 | const currentPath = window.location.pathname;
49 | const direction = determineDirection(currentPath, nextPath);
50 | setNavigationDirection(direction);
51 | };
52 |
53 | return (
54 |
55 |
57 |
59 |
60 | );
61 | };
62 |
63 | export default App;
--------------------------------------------------------------------------------
/src/App.test.js:
--------------------------------------------------------------------------------
1 | import { render, screen } from '@testing-library/react';
2 | import App from './App';
3 |
4 | test('renders learn react link', () => {
5 | render(
20 |
21 |
22 | Navigation Bar
23 |
24 |
25 |
81 |
82 |
83 |
85 |
86 |
87 | );
88 | };
89 |
90 | export default Navbar;
91 |
--------------------------------------------------------------------------------
/src/NavbarHook/NavbarHook.css:
--------------------------------------------------------------------------------
1 | .header {
2 | position: fixed;
3 | width: 100%;
4 | top: 0;
5 | left: 0;
6 | background-color: transparent;
7 | z-index: var(--z-fixed);
8 | }
9 |
10 | .nav {
11 | display: flex;
12 | background-color: #333;
13 | color: #fff;
14 | padding-bottom: 0.5rem;
15 | transition: all 0.8s, ease-in-out;
16 | padding-left: 1rem;
17 | align-items: center;
18 | justify-content: space-between;
19 | position: relative;
20 | height: var(--header-height);
21 | }
22 |
23 | .nav__logo {
24 | color: var(--first-color);
25 | transition: color 0.4s;
26 | font-size: var(--h2-font-size);
27 | font-family: var(--second-font);
28 | }
29 |
30 | .nav__toggle,
31 | .nav__close {
32 | font-size: 1.5rem;
33 | color: var(--title-color);
34 | cursor: pointer;
35 | }
36 |
37 | .nav__cta {
38 | background-color: var(--first-color);
39 | color: var(--title-color);
40 | border: none;
41 | border-radius: 4px;
42 | cursor: pointer;
43 | transition: background-color 0.3s ease;
44 | padding: 0.75rem 1.5rem;
45 | }
46 |
47 | .nav__list {
48 | display: flex;
49 | flex-direction: column;
50 | row-gap: 2.5rem;
51 | }
52 |
53 | .nav__link {
54 | color: var(--title-color);
55 | font-weight: var(--font-semi-bold);
56 | transition: color 0.4s;
57 | }
58 |
59 | .nav__link:hover {
60 | color: var(--first-color);
61 | }
62 |
63 | .nav__close {
64 | position: absolute;
65 | top: 1rem;
66 | right: 1.5rem;
67 | }
68 |
69 | .show-menu {
70 | right: 0;
71 | }
72 |
73 | .nav__list__web {
74 | display: flex;
75 | flex-direction: row;
76 | column-gap: 2.5rem;
77 | align-items: center;
78 | list-style-type: none;
79 | padding: 0;
80 | }
81 |
--------------------------------------------------------------------------------
/src/NavbarHook/NavbarHook.js:
--------------------------------------------------------------------------------
1 | import React, { useState } from "react";
2 | import { NavLink, useNavigate } from "react-router-dom";
3 | import { IoClose, IoMenu } from "react-icons/io5";
4 | import { useMediaQuery } from "react-responsive";
5 | import "./NavbarHook.css";
6 |
7 | const NavbarHook = ({ selectedKeys, filePath, onRouteChange }) => {
8 | const [isMenuOpen, setIsMenuOpen] = useState(false);
9 | const isMobile = useMediaQuery({ maxWidth: "1150px" });
10 | const navigate = useNavigate();
11 |
12 | const toggleMenu = () => {
13 | setIsMenuOpen(!isMenuOpen);
14 | };
15 |
16 | const closeMobileMenu = () => {
17 | if (isMobile) {
18 | setIsMenuOpen(false);
19 | }
20 | };
21 |
22 | const handleNavLinkClick = (path) => {
23 | // onRouteChange(path);
24 | navigate(path);
25 | closeMobileMenu();
26 | };
27 |
28 | const renderNavLinks = () => {
29 | const listClassName = isMobile ? "nav__list" : "nav__list__web";
30 | const linkClassName = "nav__link";
31 | const buttonClassName = "nav__cta";
32 |
33 | return (
34 |
35 |
36 | handleNavLinkClick('/')}>
37 | Upload
38 |
39 |
40 |
41 | handleNavLinkClick('/preview')}>
42 | Preview
43 |
44 |
45 |
46 | handleNavLinkClick('/embed')}>
47 | Embeddings
48 |
49 |
50 |
51 | handleNavLinkClick('/query')}>
52 | Query
53 |
54 |
55 |
56 | );
57 | };
58 |
59 | return (
60 |
61 |
62 |
63 | VECT.R8
64 |
65 |
66 | {isMobile && (
67 |
68 |
70 | )}
71 |
72 | {isMobile ? (
73 |
82 | ) : (
83 | renderNavLinks()
84 | )}
85 |
86 |
87 | );
88 | };
89 |
90 | export default NavbarHook;
91 |
--------------------------------------------------------------------------------
/src/assets/VECTR8-demo-ezgif.com-video-to-gif-converter.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itsPreto/VECTR8/7fee734bcc9b5ba66a1ff8b2a09155a595b87f37/src/assets/VECTR8-demo-ezgif.com-video-to-gif-converter.gif
--------------------------------------------------------------------------------
/src/assets/VECTR8-demo.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itsPreto/VECTR8/7fee734bcc9b5ba66a1ff8b2a09155a595b87f37/src/assets/VECTR8-demo.mp4
--------------------------------------------------------------------------------
/src/assets/audio-logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/src/assets/code-logo.svg:
--------------------------------------------------------------------------------
1 |
2 | file_type_binary
5 |
--------------------------------------------------------------------------------
/src/assets/excel-logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/src/assets/img-logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/src/assets/json-logo.svg:
--------------------------------------------------------------------------------
1 |
2 | file_type_jsonld3
7 |
8 |
11 |
12 |
27 |
28 |
34 |
35 |
39 |
65 |
66 |
--------------------------------------------------------------------------------
/src/assets/powerpoint-logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
7 |
8 |
9 |
15 |
17 |
31 |
32 |
39 |
40 |
44 |
45 |
--------------------------------------------------------------------------------
/src/assets/torch.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/itsPreto/VECTR8/7fee734bcc9b5ba66a1ff8b2a09155a595b87f37/src/assets/torch.png
--------------------------------------------------------------------------------
/src/assets/txt-logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
6 |
9 |
--------------------------------------------------------------------------------
/src/assets/video-logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 | file_type_word
10 |
12 | );
13 |
14 | // If you want to start measuring performance in your app, pass a function
15 | // to log results (for example: reportWebVitals(console.log))
16 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals
17 | reportWebVitals();
18 |
--------------------------------------------------------------------------------
/src/logo.svg:
--------------------------------------------------------------------------------
1 |
386 |
387 |
388 |
Available Datasets
389 |
setSearchTerm(e.target.value)}
394 | style={{ marginBottom: '16px', padding: '8px', width: '100%' }}
395 | />
396 | {loading &&
Loading datasets...
}
397 |
401 | {filteredDatasets.map((dataset, index) => {
402 | const fileExtension = dataset.name.split('.').pop().toLowerCase();
403 | return (
404 | handleDatasetClick(dataset.name, dataset.size)}
415 | >
416 | {['csv', 'json', 'mp3', 'wav', 'mp4', 'avi', 'jpg', 'jpeg', 'png'].includes(fileExtension) ? (
417 |
422 | {editName === dataset.name ? (
423 |
424 | Save
431 |
432 | ) : (
433 |
434 | {dataset.name}
435 | handleEditClick(dataset.name)}
439 | >
440 |
441 | )}
442 |
443 |
Size: {formatBytes(dataset.size)}
444 |
Uploaded: {timeAgo(dataset.upload_date)}
445 |
446 |
448 | );
449 | })}
450 |
451 |
463 |
464 |
465 |
476 |
{fileInfo ? `File Name: ${fileInfo}` : 'No file selected'}
477 |
Drag and drop a file here, or click to select a file
480 |
481 |
482 |
Merge Selected Files
483 |
484 |
485 |
Data Wizard
486 |
499 |
500 |
501 |
512 |
{wizardFileInfo ? `File Name: ${wizardFileInfo}` : 'No file selected'}
513 |
Drag and drop a file here, or click to select a file
516 |
517 |
518 |
519 | {dataWizardLoading &&
Loading...
}
520 | {wizardResponse && (
521 |
522 |
Chatbot Response:
523 |
{wizardResponse}
524 |
525 | )}
526 |
527 |
543 | {chatMessages.map((message, index) => (
544 |
545 |
559 | {message.content}
560 |
561 |
562 | ))}
563 |
564 |
565 |
583 |
588 | Send
589 |
590 |
591 |
592 |
594 | setIpAddress(document.getElementById("ipAddressInput").value)}>
602 | Update IP
603 |
604 |
605 |
606 | );
607 | };
608 |
609 | export default Upload;
610 |
--------------------------------------------------------------------------------
/src/pages/embed/Embed.css:
--------------------------------------------------------------------------------
1 | :root {
2 | --primary-color: #007bff;
3 | --secondary-color: #6c757d;
4 | --accent-color: #ffc107;
5 | --background-color: #f8f9fa;
6 | --text-color: #343a40;
7 | --border-color: #dee2e6;
8 | --header-height: 3.5rem;
9 | --font-family: 'Montserrat', sans-serif;
10 | --font-size-large: 1.5rem;
11 | --font-size-medium: 1rem;
12 | --font-size-small: 0.875rem;
13 | --border-radius: 8px;
14 | --transition: all 0.3s ease;
15 | }
16 |
17 | body {
18 | font-family: var(--font-family);
19 | margin: 0;
20 | padding: 0;
21 | background-color: var(--background-color);
22 | color: var(--text-color);
23 | }
24 |
25 | .embed-section {
26 | padding: 2rem;
27 | background-color: var(--background-color);
28 | border-radius: var(--border-radius);
29 | box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1);
30 | }
31 |
32 | .embed-title {
33 | text-align: center;
34 | font-size: var(--font-size-large);
35 | color: var(--text-color);
36 | margin-bottom: 1rem;
37 | }
38 |
39 | .embed-container {
40 | text-align: center;
41 | background-color: #fff;
42 | padding: 1rem;
43 | border-radius: var(--border-radius);
44 | box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1);
45 | margin-bottom: 1rem;
46 | }
47 |
48 | .button-group {
49 | display: flex;
50 | justify-content: center;
51 | gap: 1rem;
52 | margin-bottom: 1rem;
53 | }
54 |
55 | .button {
56 | padding: 0.5rem 1rem;
57 | background-color: var(--primary-color);
58 | color: #fff;
59 | border: none;
60 | border-radius: var(--border-radius);
61 | cursor: pointer;
62 | transition: var(--transition);
63 | }
64 |
65 | .button:hover {
66 | background-color: var(--accent-color);
67 | box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);
68 | }
69 |
70 | .progress-bar-container {
71 | transition: all 0.5s ease;
72 | display: none;
73 | margin: 1rem auto;
74 | }
75 |
76 | .progress-bar-container.visible {
77 | display: block;
78 | }
79 |
80 | .circular-progress {
81 | position: relative;
82 | width: 100px;
83 | height: 100px;
84 | }
85 |
86 | .circular-progress svg {
87 | width: 100%;
88 | height: 100%;
89 | }
90 |
91 | .circular-progress .bg {
92 | fill: none;
93 | stroke: #e6e6e6;
94 | stroke-width: 10;
95 | }
96 |
97 | .circular-progress .progress {
98 | fill: none;
99 | stroke: #007bff;
100 | stroke-width: 10;
101 | stroke-dasharray: 314;
102 | }
103 |
104 | .circular-progress .percentage {
105 | position: absolute;
106 | top: 50%;
107 | left: 50%;
108 | transform: translate(-50%, -50%);
109 | font-size: 18px;
110 | font-weight: bold;
111 | }
112 |
113 | .dashboard {
114 | display: flex;
115 | flex-direction: column;
116 | align-items: center;
117 | text-align: center;
118 | background-color: #fff;
119 | padding: 1rem;
120 | border-radius: var(--border-radius);
121 | box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1);
122 | }
123 |
124 | .db-list {
125 | display: flex;
126 | flex-direction: column;
127 | gap: 1rem;
128 | max-height: 200px;
129 | overflow-y: auto;
130 | margin-bottom: 1rem;
131 | }
132 |
133 | .db-item {
134 | padding: 0.5rem;
135 | background-color: var(--background-color);
136 | border-radius: var(--border-radius);
137 | cursor: pointer;
138 | position: relative;
139 | transition: var(--transition);
140 | }
141 |
142 | .db-item.active {
143 | border: 2px solid var(--primary-color);
144 | }
145 |
146 | .db-item:hover {
147 | background-color: var(--accent-color);
148 | color: #fff;
149 | }
150 |
151 | .active-indicator {
152 | position: absolute;
153 | top: 10px;
154 | right: 10px;
155 | width: 10px;
156 | height: 10px;
157 | background-color: red;
158 | border-radius: 50%;
159 | animation: pulse 1s infinite;
160 | }
161 |
162 | .db-item.pulse {
163 | animation: pulse 1.5s infinite;
164 | }
165 |
166 | @keyframes pulse {
167 | 0% {
168 | box-shadow: 0 0 5px rgba(0, 0, 0, 0.1);
169 | }
170 |
171 | 50% {
172 | box-shadow: 0 0 20px rgba(0, 0, 0, 0.2);
173 | }
174 |
175 | 100% {
176 | box-shadow: 0 0 5px rgba(0, 0, 0, 0.1);
177 | }
178 | }
179 |
180 |
181 | /* @keyframes pulse {
182 | 0% {
183 | transform: scale(0.9);
184 | opacity: 1;
185 | }
186 |
187 | 70% {
188 | transform: scale(1.2);
189 | opacity: 0.7;
190 | }
191 |
192 | 100% {
193 | transform: scale(0.9);
194 | opacity: 1;
195 | }
196 | } */
197 |
198 | .stats-title {
199 | margin-top: 1rem;
200 | font-size: var(--font-size-medium);
201 | color: var(--text-color);
202 | }
--------------------------------------------------------------------------------
/src/pages/embed/Embed.js:
--------------------------------------------------------------------------------
1 | import React, { useState, useEffect } from "react";
2 | import { useNavigate } from "react-router-dom";
3 | import { formatBytes, timeAgo } from '../../utils/FileInfo';
4 |
5 | import './Embed.css';
6 |
7 | const Embed = () => {
8 | const navigate = useNavigate();
9 | const [selectedKeys, setSelectedKeys] = useState([]);
10 | const [filePath, setFilePath] = useState(null);
11 | const [progress, setProgress] = useState(0);
12 | const [isVectorDbExisting, setIsVectorDbExisting] = useState(false);
13 | const [ipAddress] = useState(() => localStorage.getItem("ipAddress") || "127.0.0.1");
14 | const [totalDocuments, setTotalDocuments] = useState(0);
15 | const [avgVectorLength, setAvgVectorLength] = useState(0);
16 | const [dbList, setDbList] = useState([]);
17 | const [selectedDb, setSelectedDb] = useState(null);
18 |
19 | useEffect(() => {
20 | const storedSelectedKeys = localStorage.getItem('selectedKeys');
21 | const storedFilePath = localStorage.getItem('fileInfo');
22 | const storedSelectedDb = localStorage.getItem('selectedDb');
23 |
24 | if (storedSelectedKeys) {
25 | try {
26 | setSelectedKeys(JSON.parse(storedSelectedKeys));
27 | } catch (error) {
28 | console.error('Error parsing selectedKeys from localStorage', error);
29 | }
30 | }
31 |
32 | if (storedFilePath) {
33 | setFilePath(storedFilePath);
34 | }
35 |
36 | if (storedSelectedDb) {
37 | try {
38 | setSelectedDb(JSON.parse(storedSelectedDb));
39 | } catch (error) {
40 | console.error('Error parsing selectedDb from localStorage', error);
41 | }
42 | }
43 |
44 | if (storedSelectedKeys && storedFilePath) {
45 | checkVectorDb();
46 | fetchAvailableDatabases();
47 | }
48 | }, []);
49 |
50 | const fetchAvailableDatabases = async () => {
51 | try {
52 | const response = await fetch(`http://${ipAddress}:4000/list_vector_dbs`);
53 | if (response.ok) {
54 | const data = await response.json();
55 | setDbList(data);
56 | } else {
57 | console.error('Error fetching database list');
58 | }
59 | } catch (error) {
60 | console.error('Error fetching database list:', error);
61 | }
62 | };
63 |
64 | const checkVectorDb = async () => {
65 | try {
66 | const response = await fetch(`http://${ipAddress}:4000/check_vector_db`);
67 | if (response.ok) {
68 | setIsVectorDbExisting(true);
69 | } else {
70 | setIsVectorDbExisting(false);
71 | }
72 | } catch (error) {
73 | setIsVectorDbExisting(false);
74 | }
75 | };
76 |
77 | const createVectorDatabase = async () => {
78 | if (!filePath || !selectedKeys.length) {
79 | console.error('File path or selected keys are missing');
80 | return;
81 | }
82 | console.log("selected_keys:", selectedKeys);
83 | console.log("file_path:", filePath);
84 |
85 | try {
86 | const response = await fetch(`http://${ipAddress}:4000/create_vector_database`, {
87 | method: 'POST',
88 | headers: { 'Content-Type': 'application/json' },
89 | body: JSON.stringify({ file_path: filePath, selected_keys: selectedKeys }),
90 | });
91 |
92 | if (response.ok) {
93 | console.log("Vector database created successfully");
94 |
95 | // Listen for progress updates via EventSource
96 | const eventSource = new EventSource(`http://${ipAddress}:4000/progress`);
97 |
98 | eventSource.onmessage = (event) => {
99 | const newProgress = parseFloat(event.data);
100 | if (!isNaN(newProgress)) {
101 | setProgress(newProgress);
102 | }
103 | };
104 |
105 | eventSource.onerror = (error) => {
106 | console.error('Error with EventSource:', error);
107 | eventSource.close();
108 | };
109 | } else {
110 | console.error('Error creating vector database', response);
111 | }
112 | } catch (error) {
113 | console.error('Error creating vector database:', error);
114 | }
115 | };
116 |
117 | const backupDatabase = async () => {
118 | try {
119 | const response = await fetch(`http://${ipAddress}:4000/backup_db`, { method: 'POST' });
120 | if (response.ok) {
121 | alert('Database backup created successfully');
122 | } else {
123 | alert('Error backing up database.');
124 | }
125 | } catch (error) {
126 | console.error('Error backing up database:', error);
127 | }
128 | };
129 |
130 | const deleteDatabase = async () => {
131 | if (window.confirm('Are you sure you want to delete the vector database?')) {
132 | try {
133 | const response = await fetch(`http://${ipAddress}:4000/delete_db`, { method: 'POST' });
134 | if (response.ok) {
135 | alert('Database deleted successfully!');
136 | setIsVectorDbExisting(false);
137 | fetchAvailableDatabases(); // Refresh the list of available databases
138 | } else {
139 | alert('Error deleting database.');
140 | }
141 | } catch (error) {
142 | console.error('Error deleting database:', error);
143 | }
144 | }
145 | };
146 |
147 | const getDbStats = async () => {
148 | try {
149 | const response = await fetch(`http://${ipAddress}:4000/db_stats`);
150 | if (response.ok) {
151 | const data = await response.json();
152 | setTotalDocuments(data.total_documents);
153 | setAvgVectorLength(data.avg_vector_length);
154 | } else {
155 | console.error('Error fetching database stats');
156 | }
157 | } catch (error) {
158 | console.error('Error fetching database stats:', error);
159 | }
160 | };
161 |
162 | const handleDbSelect = (db) => {
163 | setSelectedDb(db);
164 | console.log("selected_db:", db);
165 | localStorage.setItem('selectedDb', JSON.stringify(db));
166 | navigate('/query'); // Navigate to the Preview page
167 | };
168 |
169 | return (
170 |
171 | Create Embeddings
172 | {isVectorDbExisting ? (
173 |
174 |
Manage your vector database and view statistics
175 |
176 | Backup Database
177 | Delete Database
178 |
179 |
Database Statistics
180 |
Total Documents: {totalDocuments}
181 |
Average Vector Length: {avgVectorLength}
182 |
Refresh Stats
183 |
186 | Create Vector DB
187 |
188 | )}
189 | 0 ? 'visible' : ''}`}>
190 |
191 |
192 |
195 |
{progress.toFixed(2)}%
196 |
197 |
199 |
200 |
Select Database
201 |
202 | {dbList.map((db, index) => (
203 |
handleDbSelect(db)}
207 | >
208 |
{db.name}
209 |
{formatBytes(db.size)}
210 |
Uploaded: {timeAgo(db.upload_date)}
211 | {selectedDb && selectedDb.name === db.name &&
}
212 |
213 | ))}
214 |
215 | {selectedDb &&
Selected Database: {selectedDb.name}
}
216 |
218 | );
219 | };
220 |
221 | export default Embed;
222 |
--------------------------------------------------------------------------------
/src/pages/overlay/Overlay.css:
--------------------------------------------------------------------------------
1 | /* Overlay.css */
2 |
3 | .overlay {
4 | position: fixed;
5 | top: 100%;
6 | left: 0;
7 | right: 0;
8 | bottom: 0;
9 | display: flex;
10 | justify-content: center;
11 | align-items: center;
12 | z-index: 1000;
13 | transition: all 0.5s ease-in-out;
14 | opacity: 0;
15 | visibility: hidden;
16 | background-color: transparent; /* Initial background is transparent */
17 | }
18 |
19 | .overlay.active {
20 | top: 0;
21 | opacity: 1;
22 | visibility: visible;
23 | }
24 |
25 | .overlay.active.delayed-bg {
26 | background-color: rgba(0, 0, 0, 0.8); /* Apply background after delay */
27 | }
28 |
29 | .overlayContent {
30 | background-color: #fff;
31 | padding: 20px;
32 | border-radius: 12px;
33 | max-width: 80%;
34 | max-height: 80%;
35 | overflow-y: auto;
36 | box-shadow: 0 8px 16px rgba(0, 0, 0, 0.2);
37 | position: relative;
38 | transition: all 0.5s ease-in-out;
39 | transform: translateY(20px);
40 | opacity: 0;
41 | }
42 |
43 | .overlayContent.active {
44 | transform: translateY(0);
45 | opacity: 1;
46 | }
47 |
48 | .featuresContainer {
49 | display: flex;
50 | }
51 |
52 | .featureCard {
53 | display: flex;
54 | justify-content: flex-start;
55 | background-color: #f8f8f8;
56 | border-radius: 8px;
57 | margin-top: 15px;
58 | margin-right: 10px;
59 | margin-left: 10px;
60 | text-align: -webkit-center;
61 | box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
62 | transition: all 0.5s ease-in-out;
63 | flex-direction: column;
64 | align-items: center;
65 | }
66 |
67 | .featureCard:hover {
68 | transform: scale(1.05);
69 | box-shadow: 0 8px 16px rgba(0, 0, 0, 0.2);
70 | }
71 |
72 | .featureIcon {
73 | font-size: 24px;
74 | margin-bottom: 10px;
75 | }
76 |
77 | .featureTitle {
78 | font-size: 18px;
79 | font-weight: bold;
80 | color: #333;
81 | margin-bottom: 5px;
82 | }
83 |
84 | .featureDescription {
85 | text-align: center;
86 | color: #666;
87 | }
88 |
89 | .navigationButtons {
90 | display: flex;
91 | align-items: center;
92 | margin-top: -10px;
93 | margin-right: 0px;
94 | margin-bottom: 10px;
95 | margin-left: 0px;
96 | justify-content: space-between;
97 | }
98 |
99 | .navButton {
100 | padding: 8px 16px;
101 | margin: 0 5px;
102 | border-radius: 4px;
103 | border: 1px solid #007bff;
104 | background-color: #007bff;
105 | color: #fff;
106 | cursor: pointer;
107 | transition: all 0.3s ease;
108 | }
109 |
110 | .navButton:hover {
111 | background-color: #0056b3;
112 | transform: scale(1.05);
113 | }
114 |
115 | .fullScreenButton {
116 | padding: 8px 16px;
117 | margin: 0 5px;
118 | border-radius: 4px;
119 | border: 1px solid #007bff;
120 | background-color: #007bff;
121 | color: #fff;
122 | cursor: pointer;
123 | transition: all 0.3s ease;
124 | }
125 |
126 | .fullScreenButton:hover {
127 | background-color: #0056b3;
128 | transform: scale(1.05);
129 | }
130 |
131 | .closeButton {
132 | padding: 8px 16px;
133 | margin: 0 5px;
134 | border-radius: 4px;
135 | border: 1px solid #e00000;
136 | background-color: #e00000;
137 | color: #fff;
138 | cursor: pointer;
139 | transition: all 0.3s ease;
140 | }
141 |
142 | .closeButton:hover {
143 | background-color: var(--accent-color);
144 | border: 1px solid var(--accent-color);
145 |
146 | transform: scale(1.05);
147 | }
148 |
--------------------------------------------------------------------------------
/src/pages/overlay/Overlay.js:
--------------------------------------------------------------------------------
1 | import React, { useRef, useState, useCallback, useEffect } from "react";
2 | import screenfull from 'screenfull';
3 | import GraphPage from './graph/Graph';
4 | import './Overlay.css';
5 | import mindmapData from '../../utils/mindmapData';
6 |
7 | const Overlay = ({ showOverlay, handleToggleOverlay }) => {
8 | const graphRef = useRef();
9 | const graphContainerRef = useRef();
10 | const [isFullscreen, setIsFullscreen] = useState(false);
11 |
12 | const graphData = {
13 | nodes: mindmapData.map(node => ({ id: node.id, name: node.label, color: node.color })),
14 | links: mindmapData
15 | .filter(node => node.parent)
16 | .map(node => ({ source: node.parent, target: node.id }))
17 | };
18 |
19 | useEffect(() => {
20 | if (screenfull.isEnabled) {
21 | screenfull.on('change', () => {
22 | setIsFullscreen(screenfull.isFullscreen);
23 | handleResize();
24 | });
25 | }
26 | return () => {
27 | if (screenfull.isEnabled) {
28 | screenfull.off('change');
29 | }
30 | };
31 | }, []);
32 |
33 | useEffect(() => {
34 | let bgTimeout;
35 | if (showOverlay) {
36 | bgTimeout = setTimeout(() => {
37 | document.querySelector('.overlay').classList.add('delayed-bg');
38 | }, 500);
39 | } else {
40 | document.querySelector('.overlay').classList.remove('delayed-bg');
41 | }
42 |
43 | return () => clearTimeout(bgTimeout);
44 | }, [showOverlay]);
45 |
46 | const handleToggleFullScreen = () => {
47 | if (screenfull.isEnabled) {
48 | screenfull.toggle(graphContainerRef.current);
49 | }
50 | };
51 |
52 | const handleResize = useCallback(() => {
53 | if (graphContainerRef.current && graphRef.current) {
54 | graphRef.current.width = graphContainerRef.current.clientWidth;
55 | graphRef.current.height = graphContainerRef.current.clientHeight;
56 | }
57 | }, []);
58 |
59 | const handleFeatureClick = (nodeId) => {
60 | if (graphRef.current) {
61 | graphRef.current.focusOnNode(nodeId);
62 | }
63 | };
64 |
65 | return (
66 |
67 |
68 |
69 | ⛶
70 |
Woah, that's a lot of dimensions...
71 | X
72 |
73 |
74 | {[
75 | { title: "Search", description: "Perform semantic search using vector similarity", icon: "🔍" },
76 | { title: "Clustering", description: "Group similar vectors based on their proximity in the embedding space", icon: "🔗" },
77 | { title: "Recommendations", description: "Generate personalized recommendations using vector similarity", icon: "✨" },
78 | { title: "Anomaly Detection", description: "Identify outlier vectors that deviate from the normal patterns", icon: "🚨" },
79 | { title: "Diversity Measurement", description: "Quantify the diversity of vectors in the embedding space", icon: "📊" },
80 | { title: "Classification", description: "Classify vectors into predefined categories based on their embeddings", icon: "🏷️" },
81 | ].map((feature, index) => (
82 |
handleFeatureClick(feature.title.toLowerCase().replace(' ', '-'))}>
83 |
{feature.icon}
84 |
{feature.title}
85 |
{feature.description}
86 |
87 | ))}
88 |
89 |
90 |
92 |
93 |
86 | {
96 | node.fx = node.x;
97 | node.fy = node.y;
98 | node.fz = node.z;
99 | }}
100 | linkColor={() => '#cccccc'}
101 | linkDirectionalParticles="2"
102 | linkDirectionalParticleSpeed={d => 2 * 0.001}
103 | nodeLabel="label"
104 | nodeCanvasObject={(node, ctx, globalScale) => {
105 | const label = node.name;
106 | const fontSize = 12 / globalScale;
107 | ctx.font = `${fontSize}px Sans-Serif`;
108 | const textWidth = ctx.measureText(label).width;
109 | const bckgDimensions = [textWidth, fontSize].map(n => n + fontSize * 1.6);
110 |
111 | ctx.textAlign = 'center';
112 | ctx.textBaseline = 'middle';
113 | ctx.fillStyle = node.color;
114 |
115 | const words = label.split(' ');
116 | let line = '';
117 | const lines = [];
118 | const lineHeight = fontSize * 1.2;
119 | words.forEach(word => {
120 | const testLine = line + word + ' ';
121 | const testWidth = ctx.measureText(testLine).width;
122 | if (testWidth > 100) {
123 | lines.push(line);
124 | line = word + ' ';
125 | } else {
126 | line = testLine;
127 | }
128 | });
129 | lines.push(line);
130 |
131 | lines.forEach((line, i) => {
132 | ctx.fillText(line, node.x, node.y - (lines.length - 1) * lineHeight / 2 + i * lineHeight);
133 | });
134 |
135 | node.__bckgDimensions = bckgDimensions;
136 | }}
137 |
138 | nodePointerAreaPaintExtend={6}
139 | nodePointerAreaPaint={(node, color, ctx) => {
140 | ctx.fillStyle = color;
141 | const bckgDimensions = node.__bckgDimensions;
142 | bckgDimensions && ctx.fillRect(node.x - bckgDimensions[0] / 2, node.y - bckgDimensions[1] / 2, ...bckgDimensions);
143 | }}
144 | onNodeClick={node => {
145 | }}
146 | />
147 |
148 | );
149 | });
150 |
151 | export default GraphPage;
152 |
--------------------------------------------------------------------------------
/src/pages/preview/Preview.css:
--------------------------------------------------------------------------------
1 | .preview-section {
2 | display: flex;
3 | flex-direction: column;
4 | align-items: center;
5 | border-radius: 12px;
6 | padding: 20px;
7 | transition: background-color 0.3s ease;
8 | }
9 |
10 | .file-details {
11 | text-align: center;
12 | padding: 16px;
13 | background: #cce5ff;
14 | width: 400px;
15 | border-radius: 12px;
16 | transition: background-color 0.3s ease;
17 | }
18 |
19 | .controls {
20 | display: flex;
21 | justify-content: center;
22 | align-items: center;
23 | margin: 10px 0;
24 | }
25 |
26 | .filter-input {
27 | padding: 8px;
28 | margin-right: 10px;
29 | border-radius: 4px;
30 | border: 1px solid #ccc;
31 | transition: border-color 0.3s ease;
32 | }
33 |
34 | .button,
35 | .question-mark-button {
36 | padding: 8px 16px;
37 | margin: 0 5px;
38 | border-radius: 4px;
39 | border: 1px solid #007bff;
40 | background-color: #007bff;
41 | color: #fff;
42 | cursor: pointer;
43 | transition: all 0.3s ease;
44 | }
45 |
46 | .button:hover,
47 | .question-mark-button:hover {
48 | background-color: #0056b3;
49 | transform: scale(1.05);
50 | }
51 |
52 | .preview-container {
53 | display: flex;
54 | flex-direction: row;
55 | justify-content: space-between;
56 | max-height: 40vh;
57 | width: 100%;
58 | }
59 |
60 | .preview-column {
61 | flex: 0 0 30%;
62 | margin: 10px;
63 | border-radius: 12px;
64 | padding: 20px;
65 | overflow: hidden;
66 | transition: all 0.3s ease;
67 | }
68 |
69 | .heading {
70 | margin-bottom: 20px;
71 | color: #333333;
72 | font-family: 'Roboto, sans-serif';
73 | transition: color 0.3s ease;
74 | }
75 |
76 | .document-preview,
77 | .vector-embeddings {
78 | overflow-y: auto;
79 | white-space: pre-wrap;
80 | word-wrap: break-word;
81 | border: 1px solid #ddd;
82 | border-radius: 12px;
83 | padding: 10px;
84 | background-color: #fff;
85 | transition: border-color 0.3s ease, background-color 0.3s ease;
86 | }
87 |
88 | .cards-container {
89 | display: flex;
90 | flex-wrap: wrap;
91 | overflow-x: hidden;
92 | gap: 10px;
93 | flex-direction: column;
94 | align-items: center;
95 | transition: gap 0.3s ease;
96 | }
97 |
98 | .card {
99 | padding: 10px;
100 | border: 1px solid rgb(221, 221, 221);
101 | border-radius: 8px;
102 | cursor: pointer;
103 | margin-left: 6px;
104 | margin-right: 6px;
105 | box-shadow: rgba(0, 0, 0, 0.1) 0px 2px 4px;
106 | display: flex;
107 | align-items: center;
108 | justify-content: center;
109 | text-align: center;
110 | flex-flow: nowrap;
111 | transition: all 0.3s ease;
112 | }
113 |
114 | .card.selected {
115 | background-color: rgb(240, 194, 57);
116 | }
117 |
118 | .preview-column {
119 | max-height: 90vh; /* Adjust based on your needs */
120 | overflow-y: auto;
121 | padding: 10px;
122 | border-radius: 8px;
123 | }
124 |
125 | .datasetListItem {
126 | list-style: none;
127 | padding: 8px;
128 | height: fit-content;
129 | border-radius: 8px;
130 | background: #ffffff;
131 | flex-direction: column;
132 | transition: all 0.3s ease;
133 | cursor: pointer;
134 | display: flex;
135 | align-items: center;
136 | }
137 |
138 | .datasetListItem:hover {
139 | background-color: #cce5ff
140 | }
141 |
142 | .datasetListItem.selected {
143 | background-color: #fae29b;
144 | }
145 |
146 | .heading {
147 | text-align: center;
148 | font-size: 1.5em;
149 | margin-bottom: 10px;
150 | }
151 |
152 | .cards-container {
153 | display: grid;
154 | grid-template-columns: repeat(auto-fill, minmax(150px, 1fr));
155 | gap: 10px;
156 | }
157 |
158 | .card {
159 | background-color: white;
160 | border-radius: 8px;
161 | box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
162 | padding: 10px;
163 | text-align: center;
164 | transition: transform 0.2s;
165 | }
166 |
167 | .card:hover {
168 | transform: scale(1.05);
169 | }
170 |
171 | .card-text {
172 | margin: 0;
173 | font-weight: bold;
174 | color: #333;
175 | }
176 |
177 | .selected {
178 | border: 2px solid #007bff;
179 | }
180 |
181 | .no-data-available-label {
182 | text-align: -webkit-center;
183 | width: 100%;
184 | }
185 |
186 | .card:hover {
187 | transform: scale(1.05);
188 | background-color: #f0f0f0;
189 | box-shadow: 0 8px 16px rgba(0, 0, 0, 0.2);
190 | }
191 |
192 | .card-text {
193 | margin: 0;
194 | font-family: 'Roboto, sans-serif';
195 | color: #555;
196 | transition: color 0.3s ease;
197 | }
198 |
--------------------------------------------------------------------------------
/src/pages/preview/Preview.js:
--------------------------------------------------------------------------------
1 | import React, { useState, useEffect, useCallback } from "react";
2 | import { JSONTree } from "react-json-tree";
3 | import { motion } from 'framer-motion';
4 | import { formatBytes, timeAgo } from '../../utils/FileInfo';
5 | import Overlay from '../overlay/Overlay';
6 | import Embed from '../embed/Embed';
7 | import './Preview.css';
8 |
9 | const customTheme = {
10 | scheme: 'material',
11 | author: 'Material Theme',
12 | base00: '#FFFFFF',
13 | base01: '#FAFAFA',
14 | base02: '#E0E0E0',
15 | base03: '#BDBDBD',
16 | base04: '#90A4AE',
17 | base05: '#212121',
18 | base06: '#757575',
19 | base07: '#263238',
20 | base08: '#FF5252',
21 | base09: '#FF9800',
22 | base0A: '#FFEB3B',
23 | base0B: '#8BC34A',
24 | base0C: '#00BCD4',
25 | base0D: '#2196F3',
26 | base0E: '#673AB7',
27 | base0F: '#E91E63',
28 | };
29 |
30 | const Preview = ({ previewData, filePath, fileSize }) => {
31 | const [selectedKeys, setSelectedKeys] = useState(() => {
32 | const savedKeys = localStorage.getItem('selectedKeys');
33 | return savedKeys ? JSON.parse(savedKeys) : (previewData.keys || []);
34 | });
35 |
36 | const [selectableKeys, setSelectableKeys] = useState([]);
37 |
38 | const [document, setDocument] = useState(() => {
39 | const savedDocument = localStorage.getItem('document');
40 | return savedDocument ? JSON.parse(savedDocument) : {};
41 | });
42 |
43 | const [embeddings, setEmbeddings] = useState(() => {
44 | const savedEmbeddings = localStorage.getItem('embeddings');
45 | return savedEmbeddings ? JSON.parse(savedEmbeddings) : null;
46 | });
47 |
48 | useEffect(() => {
49 | if (previewData) {
50 | const keys = Array.isArray(previewData) ? previewData : Object.keys(previewData || {});
51 | setSelectableKeys(keys);
52 | }
53 | }, [previewData]);
54 |
55 | useEffect(() => {
56 | localStorage.setItem('selectedKeys', JSON.stringify(selectedKeys));
57 | }, [selectedKeys]);
58 |
59 | const [ipAddress] = useState(() => localStorage.getItem("ipAddress") || "127.0.0.1");
60 | const [tokenCount, setTokenCount] = useState(null);
61 | const [filterText, setFilterText] = useState("");
62 | const [showOverlay, setShowOverlay] = useState(false);
63 | const [showEmbed] = useState(false);
64 |
65 | const updatePreviews = useCallback(async () => {
66 | if (!filePath) {
67 | console.error('File path is undefined');
68 | return;
69 | }
70 | try {
71 | const data = { file_path: filePath, selected_keys: selectedKeys };
72 | console.log('Sending data:', data);
73 |
74 | const response = await fetch(`http://${ipAddress}:4000/preview_document`, {
75 | method: 'POST',
76 | headers: { 'Content-Type': 'application/json' },
77 | body: JSON.stringify(data)
78 | });
79 | const result = await response.json();
80 | if (response.ok) {
81 | console.log("Previewed document successfully.", result);
82 | const docObject = selectedKeys.reduce((acc, key) => {
83 | acc[key] = result.document[key];
84 | return acc;
85 | }, {});
86 | setDocument(docObject);
87 | setEmbeddings(result.embeddings);
88 | setTokenCount(result.token_count);
89 | } else {
90 | alert(result.error);
91 | }
92 | } catch (error) {
93 | console.error('Error previewing document:', error);
94 | }
95 | }, [filePath, ipAddress, selectedKeys]);
96 |
97 | useEffect(() => {
98 | if (selectedKeys.length > 0) {
99 | updatePreviews();
100 | }
101 | }, [selectedKeys, filePath, updatePreviews]);
102 |
103 | useEffect(() => {
104 | localStorage.setItem('selectedKeys', JSON.stringify(selectedKeys));
105 | }, [selectedKeys]);
106 |
107 | useEffect(() => {
108 | localStorage.setItem('document', JSON.stringify(document));
109 | }, [document]);
110 |
111 | useEffect(() => {
112 | localStorage.setItem('embeddings', JSON.stringify(embeddings));
113 | }, [embeddings]);
114 |
115 | useEffect(() => {
116 | if (previewData) {
117 | const keys = Array.isArray(previewData) ? previewData : Object.keys(previewData || {});
118 | setSelectableKeys(keys);
119 | }
120 | }, [previewData]);
121 |
122 |
123 | const handleKeySelection = (key) => {
124 | setSelectedKeys((prevKeys) =>
125 | prevKeys.includes(key)
126 | ? prevKeys.filter((k) => k !== key)
127 | : [...prevKeys, key]
128 | );
129 | };
130 |
131 | const displayPreview = () => {
132 | if (!previewData || !Array.isArray(selectableKeys)) {
133 | return No preview data available
;
134 | }
135 | const filteredKeys = selectableKeys.filter((key) =>
136 | key.toLowerCase().includes(filterText.toLowerCase())
137 | );
138 | const selectedKeysSet = new Set(selectedKeys);
139 | const sortedKeys = [
140 | ...selectedKeys.filter((key) => filteredKeys.includes(key)),
141 | ...filteredKeys.filter((key) => !selectedKeysSet.has(key)),
142 | ];
143 |
144 | return sortedKeys.map((key) => (
145 | handleKeySelection(key)}
150 | >
151 | {key}
152 |
153 | ));
154 | };
155 |
156 | const displayDocument = () => {
157 | return (
158 |
184 |
185 |
186 |
File: {filePath}
187 |
188 |
{formatBytes(fileSize)}
189 | {tokenCount !== null && (
190 | <>
191 |
192 |
Tokens: {tokenCount}
193 |
194 | )}
195 |
196 |
197 | Select All
205 | Deselect All
206 | ?
207 |
208 |
210 |
211 |
Select Embedding Keys
212 |
213 | {displayPreview()}
214 |
215 |
216 |
217 |
Generated Document
218 |
219 | {document && displayDocument()}
220 |
221 |
222 |
223 |
Generated Embedding
224 |
225 | {embeddings && displayEmbeddings()}
226 |
227 |
228 |
232 | );
233 | };
234 |
235 | export default Preview;
236 |
--------------------------------------------------------------------------------
/src/pages/query/Query.css:
--------------------------------------------------------------------------------
1 | body {
2 | font-family: 'Roboto', sans-serif;
3 | background: #f0f2f5;
4 | margin: 0;
5 | padding: 0;
6 | }
7 |
8 | #query {
9 | display: flex;
10 | flex-direction: column;
11 | align-items: center;
12 | text-align: -webkit-center;
13 | border-radius: 10px;
14 | margin: 20px auto;
15 | max-width: 800px;
16 | width: 100%;
17 | transition: transform 0.3s ease;
18 | }
19 |
20 | #query:hover {
21 | transform: scale(1.02);
22 | }
23 |
24 | h2 {
25 | color: #333;
26 | margin-bottom: 20px;
27 | font-size: 24px;
28 | text-align: center;
29 | }
30 |
31 | input, select {
32 | padding: 10px;
33 | border: 1px solid #ddd;
34 | border-radius: 4px;
35 | font-size: 16px;
36 | width: 50%;
37 | transition: all 0.3s ease;
38 | }
39 |
40 | input:focus, select:focus {
41 | border-color: #007bff;
42 | box-shadow: 0 0 8px rgba(0, 123, 255, 0.2);
43 | }
44 |
45 | .button {
46 | padding: 10px 20px;
47 | margin: 10px 0;
48 | border: none;
49 | border-radius: 4px;
50 | background-color: #007bff;
51 | color: #fff;
52 | cursor: pointer;
53 | font-size: 16px;
54 | transition: background-color 0.3s ease, transform 0.3s ease;
55 | }
56 |
57 | .button:hover {
58 | background-color: #0056b3;
59 | transform: scale(1.05);
60 | }
61 |
62 | .carousel {
63 | display: flex;
64 | overflow-x: auto;
65 | padding: 20px 0;
66 | scroll-behavior: smooth;
67 | }
68 |
69 | .carousel::-webkit-scrollbar {
70 | display: none;
71 | }
72 |
73 | .query-result-card {
74 | background-color: #fff;
75 | border: 1px solid #ddd;
76 | border-radius: 8px;
77 | overflow-y: auto;
78 | height: 400px;
79 | margin: 0 10px;
80 | cursor: pointer;
81 | padding: 15px;
82 | transition: transform 0.3s ease, background-color 0.3s ease;
83 | box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
84 | flex: 0 0 auto;
85 | width: 28%;
86 | }
87 |
88 | .query-result-card:hover {
89 | transform: translateY(-5px);
90 | background-color: #ffc10787;
91 | }
92 |
93 | .result-header {
94 | display: flex;
95 | justify-content: space-between;
96 | align-items: center;
97 | }
98 |
99 | .result-content {
100 | margin-top: 10px;
101 | }
102 |
103 | .result-content p {
104 | margin: 5px 0;
105 | overflow-y: auto;
106 | }
107 |
108 | .modal {
109 | position: fixed;
110 | top: 0;
111 | left: 0;
112 | width: 100%;
113 | height: 100%;
114 | background-color: rgba(0, 0, 0, 0.7);
115 | display: flex;
116 | align-items: center;
117 | justify-content: center;
118 | animation: fadeIn 0.5s;
119 | z-index: 1000;
120 | }
121 |
122 | .modal-content {
123 | background-color: #fff;
124 | padding: 20px;
125 | border-radius: 8px;
126 | max-width: 80%;
127 | max-height: 80%;
128 | overflow-y: auto;
129 | animation: scaleIn 0.3s;
130 | }
131 |
132 | .result-details {
133 | text-wrap: balance;
134 | }
135 |
136 | .close-inspect-button {
137 | float: right;
138 | background-color: #dc3545;
139 | color: #fff;
140 | border: none;
141 | padding: 10px;
142 | border-radius: 4px;
143 | cursor: pointer;
144 | }
145 |
146 | .close-inspect-button:hover {
147 | background-color: #c82333;
148 | }
149 |
150 | @keyframes fadeIn {
151 | from { opacity: 0; }
152 | to { opacity: 1; }
153 | }
154 |
155 | @keyframes scaleIn {
156 | from { transform: scale(0.8); }
157 | to { transform: scale(1); }
158 | }
159 |
--------------------------------------------------------------------------------
/src/pages/query/Query.js:
--------------------------------------------------------------------------------
1 | import React, { useState, useEffect } from "react";
2 | import { useNavigate } from "react-router-dom";
3 | import { CircularProgressbar, buildStyles } from "react-circular-progressbar";
4 | import "react-circular-progressbar/dist/styles.css";
5 | import "./Query.css";
6 |
7 | const Query = () => {
8 | const [queryText, setQueryText] = useState(() => localStorage.getItem("queryText") || "");
9 | const [similarityMetric, setSimilarityMetric] = useState(() => localStorage.getItem("similarityMetric") || "cosine");
10 | const [results, setResults] = useState(() => JSON.parse(localStorage.getItem("results")) || []);
11 | const [ipAddress] = useState(() => localStorage.getItem("ipAddress") || "127.0.0.1");
12 | const [selectedDb] = useState(() => {
13 | const db = localStorage.getItem("selectedDb");
14 | return db ? JSON.parse(db) : "";
15 | });
16 | const [isLoading] = useState(false);
17 | const [progress] = useState(0);
18 | const navigate = useNavigate();
19 |
20 | useEffect(() => {
21 | if (!selectedDb) {
22 | alert("No database selected. Please select a database first.");
23 | }
24 | console.log("Query.js: selectedDb", selectedDb);
25 | }, [selectedDb, navigate]);
26 |
27 | useEffect(() => {
28 | localStorage.setItem("queryText", queryText);
29 | }, [queryText]);
30 |
31 | useEffect(() => {
32 | localStorage.setItem("similarityMetric", similarityMetric);
33 | }, [similarityMetric]);
34 |
35 | useEffect(() => {
36 | localStorage.setItem("results", JSON.stringify(results));
37 | }, [results]);
38 |
39 | useEffect(() => {
40 | localStorage.setItem("ipAddress", ipAddress);
41 | }, [ipAddress]);
42 |
43 | useEffect(() => {
44 | localStorage.setItem("selectedDb", JSON.stringify(selectedDb));
45 | }, [selectedDb]);
46 |
47 | const handleQuery = async () => {
48 | if (!selectedDb) {
49 | alert("No database selected. Please select a database first.");
50 | navigate('/embed');
51 |
52 | return;
53 | }
54 |
55 | try {
56 | const response = await fetch(`http://${ipAddress}:4000/query`, {
57 | method: "POST",
58 | headers: { "Content-Type": "application/json" },
59 | body: JSON.stringify({ query_text: queryText, similarity_metric: similarityMetric, db_filename: selectedDb.name }),
60 | });
61 | const results = await response.json();
62 | if (response.ok) {
63 | console.log("Query results:", results);
64 | setResults(results);
65 | } else {
66 | alert(results.error);
67 | }
68 | } catch (error) {
69 | console.error("Error querying database:", error);
70 | }
71 | };
72 |
73 | const displayFullData = (data) => {
74 | const modal = document.createElement("div");
75 | modal.classList.add("modal");
76 | modal.innerHTML = `
77 |
78 |
×
79 |
${JSON.stringify(data, null, 2)}
80 |
97 | {results.map((result, index) => {
98 | const [doc, fullData, score] = result;
99 | return (
100 |
displayFullData(fullData)}>
101 |
102 |
Score: {score.toFixed(2)}
103 |
104 |
105 |
Title: {doc.title}
106 |
Overview: {doc.overview}
107 |
Release Date: {doc.release_date}
108 |
Popularity: {doc.popularity}
109 |
110 |
111 | );
112 | })}
113 |
119 | Query Vector Database
120 |
121 | setSimilarityMetric(e.target.value)}>
129 | Cosine Similarity
130 | Euclidean Distance
131 |
132 |
133 |
134 | Submit
135 |
136 | {isLoading && (
137 |
138 |
148 | )}
149 | {displayResults()}
150 |
151 | );
152 | };
153 |
154 | export default Query;
155 |
--------------------------------------------------------------------------------
/src/reportWebVitals.js:
--------------------------------------------------------------------------------
1 | const reportWebVitals = onPerfEntry => {
2 | if (onPerfEntry && onPerfEntry instanceof Function) {
3 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => {
4 | getCLS(onPerfEntry);
5 | getFID(onPerfEntry);
6 | getFCP(onPerfEntry);
7 | getLCP(onPerfEntry);
8 | getTTFB(onPerfEntry);
9 | });
10 | }
11 | };
12 |
13 | export default reportWebVitals;
14 |
--------------------------------------------------------------------------------
/src/setupTests.js:
--------------------------------------------------------------------------------
1 | // jest-dom adds custom jest matchers for asserting on DOM nodes.
2 | // allows you to do things like:
3 | // expect(element).toHaveTextContent(/react/i)
4 | // learn more: https://github.com/testing-library/jest-dom
5 | import '@testing-library/jest-dom';
6 |
--------------------------------------------------------------------------------
/src/transition/Transition.css:
--------------------------------------------------------------------------------
1 | /* TransitionWrapper.css */
2 | .slide-enter {
3 | opacity: 0;
4 | transform: translateX(100%);
5 | }
6 | .slide-enter-active {
7 | opacity: 1;
8 | transform: translateX(0);
9 | transition: opacity 300ms, transform 300ms;
10 | }
11 | .slide-exit {
12 | opacity: 1;
13 | transform: translateX(0);
14 | }
15 | .slide-exit-active {
16 | opacity: 0;
17 | transform: translateX(-100%);
18 | transition: opacity 300ms, transform 300ms;
19 | }
20 |
21 | .slide-reverse-enter {
22 | opacity: 0;
23 | transform: translateX(-100%);
24 | }
25 | .slide-reverse-enter-active {
26 | opacity: 1;
27 | transform: translateX(0);
28 | transition: opacity 300ms, transform 300ms;
29 | }
30 | .slide-reverse-exit {
31 | opacity: 1;
32 | transform: translateX(0);
33 | }
34 | .slide-reverse-exit-active {
35 | opacity: 0;
36 | transform: translateX(100%);
37 | transition: opacity 300ms, transform 300ms;
38 | }
39 |
--------------------------------------------------------------------------------
/src/transition/Transition.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import { CSSTransition, TransitionGroup } from 'react-transition-group';
3 | import { useLocation } from 'react-router-dom';
4 | import './Transition.css';
5 |
6 | const Transition = ({ children, direction }) => {
7 | const location = useLocation();
8 | const classNames = direction === 'forward' ? 'slide' : 'slide-reverse';
9 |
10 | return (
11 |
12 |
17 | {children}
18 |
19 |
20 | );
21 | };
22 |
23 | export default Transition;
24 |
--------------------------------------------------------------------------------
/src/utils/FileInfo.js:
--------------------------------------------------------------------------------
1 | export const formatBytes = (bytes, decimals = 2) => {
2 | if (bytes === 0) return '0 Bytes';
3 | const k = 1024;
4 | const dm = decimals < 0 ? 0 : decimals;
5 | const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
6 | const i = Math.floor(Math.log(bytes) / Math.log(k));
7 | return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
8 | };
9 |
10 | export const timeAgo = (date) => {
11 | const now = new Date();
12 | const then = new Date(date);
13 | const seconds = Math.floor((now - then) / 1000);
14 | let interval = Math.floor(seconds / 31536000);
15 |
16 | if (interval > 1) return `${interval} years ago`;
17 | interval = Math.floor(seconds / 2592000);
18 | if (interval > 1) return `${interval} months ago`;
19 | interval = Math.floor(seconds / 86400);
20 | if (interval > 1) return `${interval} days ago`;
21 | interval = Math.floor(seconds / 3600);
22 | if (interval > 1) return `${interval} hours ago`;
23 | interval = Math.floor(seconds / 60);
24 | if (interval > 1) return `${interval} minutes ago`;
25 | return `${Math.floor(seconds)} seconds ago`;
26 | };
--------------------------------------------------------------------------------
/src/utils/mindmapData.js:
--------------------------------------------------------------------------------
1 | // utils/mindmapData.js
2 |
3 | const mindmapData = [
4 | { id: 'root', label: 'Embeddings Dimensions', parent: null, color: '#ffffff' },
5 | { id: 'search', label: 'SEARCH', parent: 'root', color: '#ffff00' },
6 | { id: 'search-detail1', label: 'High-dimensional embeddings improve search accuracy', parent: 'search', color: '#ffff66' },
7 | { id: 'search-detail2', label: 'Better relevance ranking for complex queries', parent: 'search', color: '#ffff66' },
8 | { id: 'search-example', label: 'Example: Google search engine optimizing results', parent: 'search', color: '#ffff66' },
9 | { id: 'clustering', label: 'CLUSTERING', parent: 'root', color: '#daccff' },
10 | { id: 'clustering-detail1', label: 'Enable finer distinctions between clusters', parent: 'clustering', color: '#51962e' },
11 | { id: 'clustering-detail2', label: 'Improved grouping of semantically similar texts', parent: 'clustering', color: '#2fd12f' },
12 | { id: 'clustering-example', label: 'Example: Organizing customer feedback into coherent topics', parent: 'clustering', color: '#6bcf6b' },
13 | { id: 'recommendations', label: 'RECOMMENDATIONS', parent: 'root', color: '#cc99ff' },
14 | { id: 'recommendations-detail1', label: 'Enhance systems by understanding user preferences better', parent: 'recommendations', color: '#cc99ff' },
15 | { id: 'recommendations-detail2', label: 'More accurate and personalized recommendations', parent: 'recommendations', color: '#cc99ff' },
16 | { id: 'recommendations-example', label: 'Example: Netflix recommending shows', parent: 'recommendations', color: '#cc99ff' },
17 | { id: 'anomaly-detection', label: 'ANOMALY DETECTION', parent: 'root', color: '#ff99ff' },
18 | { id: 'anomaly-detection-detail1', label: 'Improve detection of subtle anomalies', parent: 'anomaly-detection', color: '#ff99ff' },
19 | { id: 'anomaly-detection-detail2', label: 'Better identification of outliers', parent: 'anomaly-detection', color: '#ff99ff' },
20 | { id: 'anomaly-detection-example', label: 'Example: Fraud detection in financial transactions', parent: 'anomaly-detection', color: '#ff99ff' },
21 | { id: 'diversity-measurement', label: 'DIVERSITY MEASUREMENT', parent: 'root', color: '#ff6699' },
22 | { id: 'diversity-measurement-detail1', label: 'Provide a detailed measure of diversity', parent: 'diversity-measurement', color: '#ff6699' },
23 | { id: 'diversity-measurement-detail2', label: 'Better analysis of similarity distributions in datasets', parent: 'diversity-measurement', color: '#ff6699' },
24 | { id: 'diversity-measurement-example', label: 'Example: Assessing diversity in hiring practices', parent: 'diversity-measurement', color: '#ff6699' },
25 | { id: 'classification', label: 'CLASSIFICATION', parent: 'root', color: '#ff9966' },
26 | { id: 'classification-detail1', label: 'Lead to more accurate classification of text', parent: 'classification', color: '#ff9966' },
27 | { id: 'classification-detail2', label: 'Improved sentiment analysis', parent: 'classification', color: '#ff9966' },
28 | { id: 'classification-example', label: 'Example: Classifying customer reviews', parent: 'classification', color: '#ff9966' },
29 | { id: 'embedding-dimensionality', label: 'EMBEDDING DIMENSIONALITY', parent: 'root', color: '#ffffff' },
30 | { id: 'model-architecture', label: 'Model Architecture and Training Objectives', parent: 'embedding-dimensionality', color: '#ffcc99' },
31 | { id: 'model-architecture-detail1', label: 'Different models have unique designs that dictate dimensions', parent: 'model-architecture', color: '#ffcc99' },
32 | { id: 'model-architecture-example1', label: 'BERT (768 dimensions)', parent: 'model-architecture', color: '#ffcc99' },
33 | { id: 'model-architecture-example2', label: 'GPT-3 (2048 dimensions)', parent: 'model-architecture', color: '#ffcc99' },
34 | { id: 'task-specific-training', label: 'Task-Specific Training', parent: 'embedding-dimensionality', color: '#ffcc99' },
35 | { id: 'task-specific-training-detail1', label: 'Complex tasks often require higher dimensions for nuanced understanding', parent: 'task-specific-training', color: '#ffcc99' },
36 | { id: 'capacity-expressiveness', label: 'Capacity and Expressiveness', parent: 'embedding-dimensionality', color: '#ffcc99' },
37 | { id: 'higher-dimensionality', label: 'Higher Dimensionality', parent: 'capacity-expressiveness', color: '#ffcc99' },
38 | { id: 'higher-dimensionality-detail1', label: 'Captures more detailed information', parent: 'higher-dimensionality', color: '#ffcc99' },
39 | { id: 'higher-dimensionality-detail2', label: 'Essential for tasks requiring deep semantic understanding', parent: 'higher-dimensionality', color: '#ffcc99' },
40 | { id: 'lower-dimensionality', label: 'Lower Dimensionality', parent: 'capacity-expressiveness', color: '#ffcc99' },
41 | { id: 'lower-dimensionality-detail1', label: 'Sufficient for simpler tasks', parent: 'lower-dimensionality', color: '#ffcc99' },
42 | { id: 'lower-dimensionality-detail2', label: 'Reduces computational cost but may miss finer details', parent: 'lower-dimensionality', color: '#ffcc99' },
43 | { id: 'computational-efficiency', label: 'Computational Efficiency', parent: 'embedding-dimensionality', color: '#ffcc99' },
44 | { id: 'memory-storage', label: 'Memory and Storage', parent: 'computational-efficiency', color: '#ffcc99' },
45 | { id: 'memory-storage-detail1', label: 'Higher dimensions require more memory and storage', parent: 'memory-storage', color: '#ffcc99' },
46 | { id: 'memory-storage-detail2', label: 'Impacts scalability', parent: 'memory-storage', color: '#ffcc99' },
47 | { id: 'processing-time', label: 'Processing Time', parent: 'computational-efficiency', color: '#ffcc99' },
48 | { id: 'processing-time-detail1', label: 'Higher dimensions increase computational load', parent: 'processing-time', color: '#ffcc99' },
49 | { id: 'processing-time-detail2', label: 'Affects real-time application performance', parent: 'processing-time', color: '#ffcc99' }
50 | ];
51 |
52 | export default mindmapData;
53 |
--------------------------------------------------------------------------------
5 | 
Python 3.7+
Flask
Flask-CORS
transformers
torch
numpy
pandas