= ({
15 | options,
16 | placeholder,
17 | defaultValue,
18 | onSelect,
19 | children,
20 | view,
21 | isDisabled,
22 | value,
23 | }) => {
24 | const handleChange = (selectedOption: OptionType | null | void) => {
25 | onSelect(selectedOption);
26 | };
27 | const allOptions = useMemo(() => options, [options]);
28 | return (
29 | <>
30 |
31 | {
37 | const label =
38 | typeof option === 'string'
39 | ? (option.includes('LLM_MODEL_CONFIG_')
40 | ? capitalize(option.split('LLM_MODEL_CONFIG_').at(-1) as string)
41 | : capitalize(option)
42 | )
43 | .split('_')
44 | .join(' ')
45 | : capitalize(option.label);
46 | const value = typeof option === 'string' ? option : option.value;
47 | return {
48 | label,
49 | value,
50 | };
51 | }),
52 | placeholder: placeholder || 'Select an option',
53 | defaultValue: defaultValue ? { label: capitalize(defaultValue), value: defaultValue } : undefined,
54 | menuPlacement: 'auto',
55 | isDisabled: isDisabled,
56 | value: value,
57 | }}
58 | size='medium'
59 | fluid
60 | />
61 | {children}
62 |
63 |
64 | );
65 | };
66 | export default DropdownComponent;
67 |
--------------------------------------------------------------------------------
/frontend/src/components/QuickStarter.tsx:
--------------------------------------------------------------------------------
1 | import Header from './Layout/Header';
2 | import React, { useState } from 'react';
3 | import { ThemeWrapperContext } from '../context/ThemeWrapper';
4 | import PageLayout from './Layout/PageLayout';
5 | import { FileContextProvider } from '../context/UsersFiles';
6 | import UserCredentialsWrapper from '../context/UserCredentials';
7 | import AlertContextWrapper from '../context/Alert';
8 | import { MessageContextWrapper } from '../context/UserMessages';
9 |
10 | const QuickStarter: React.FunctionComponent = () => {
11 | const themeUtils = React.useContext(ThemeWrapperContext);
12 | const [themeMode, setThemeMode] = useState(themeUtils.colorMode);
13 | const [showSettingsModal, setshowSettingsModal] = useState(false);
14 | const [showOrphanNodeDeletionDialog, setshowOrphanNodeDeletionDialog] = useState(false);
15 |
16 | const toggleColorMode = () => {
17 | setThemeMode((prevThemeMode) => {
18 | return prevThemeMode === 'light' ? 'dark' : 'light';
19 | });
20 | themeUtils.toggleColorMode();
21 | };
22 | const openSettingsModal = () => {
23 | setshowSettingsModal(true);
24 | };
25 | const closeSettingModal = () => {
26 | setshowSettingsModal(false);
27 | };
28 | const openOrphanNodeDeletionModal = () => {
29 | setshowOrphanNodeDeletionDialog(true);
30 | };
31 | const closeOrphanNodeDeletionModal = () => {
32 | setshowOrphanNodeDeletionDialog(false);
33 | };
34 | return (
35 |
36 |
37 |
38 |
39 |
40 |
48 |
49 |
50 |
51 |
52 | );
53 | };
54 | export default QuickStarter;
55 |
--------------------------------------------------------------------------------
/backend/src/gemini_llm.py:
--------------------------------------------------------------------------------
1 | from langchain_community.graphs import Neo4jGraph
2 | from dotenv import load_dotenv
3 | from langchain.schema import Document
4 | import logging
5 | import concurrent.futures
6 | from concurrent.futures import ThreadPoolExecutor
7 | from typing import List
8 | from langchain_experimental.graph_transformers import LLMGraphTransformer
9 | from langchain_core.documents import Document
10 | from typing import List
11 | import google.auth
12 | from typing import List
13 | from langchain_core.documents import Document
14 | import vertexai
15 | from src.llm import get_graph_document_list, get_combined_chunks, get_llm
16 |
17 | load_dotenv()
18 | logging.basicConfig(format='%(asctime)s - %(message)s',level='DEBUG')
19 |
20 |
21 | def get_graph_from_Gemini(model_version,
22 | graph: Neo4jGraph,
23 | chunkId_chunkDoc_list: List,
24 | allowedNodes,
25 | allowedRelationship):
26 | """
27 | Extract graph from OpenAI and store it in database.
28 | This is a wrapper for extract_and_store_graph
29 |
30 | Args:
31 | model_version : identify the model of LLM
32 | graph: Neo4jGraph to be extracted.
33 | chunks: List of chunk documents created from input file
34 | Returns:
35 | List of langchain GraphDocument - used to generate graph
36 | """
37 | logging.info(f"Get graphDocuments from {model_version}")
38 | futures = []
39 | graph_document_list = []
40 | location = "us-central1"
41 | #project_id = "llm-experiments-387609"
42 | credentials, project_id = google.auth.default()
43 | if hasattr(credentials, "service_account_email"):
44 | logging.info(credentials.service_account_email)
45 | else:
46 | logging.info("WARNING: no service account credential. User account credential?")
47 | vertexai.init(project=project_id, location=location)
48 |
49 | combined_chunk_document_list = get_combined_chunks(chunkId_chunkDoc_list)
50 |
51 | llm,model_name = get_llm(model_version)
52 | return get_graph_document_list(llm, combined_chunk_document_list, allowedNodes, allowedRelationship)
53 |
54 |
55 |
--------------------------------------------------------------------------------
/frontend/src/logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/frontend/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 | npm-debug.log*
5 | yarn-debug.log*
6 | yarn-error.log*
7 | lerna-debug.log*
8 | .pnpm-debug.log*
9 |
10 | # Diagnostic reports (https://nodejs.org/api/report.html)
11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
12 |
13 | # Runtime data
14 | pids
15 | *.pid
16 | *.seed
17 | *.pid.lock
18 |
19 | # Directory for instrumented libs generated by jscoverage/JSCover
20 | lib-cov
21 |
22 | # Coverage directory used by tools like istanbul
23 | coverage
24 | *.lcov
25 |
26 | # nyc test coverage
27 | .nyc_output
28 |
29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
30 | .grunt
31 |
32 | # Bower dependency directory (https://bower.io/)
33 | bower_components
34 |
35 | # node-waf configuration
36 | .lock-wscript
37 |
38 | # Compiled binary addons (https://nodejs.org/api/addons.html)
39 | build/Release
40 |
41 | # Dependency directories
42 | node_modules/
43 | jspm_packages/
44 |
45 | # Snowpack dependency directory (https://snowpack.dev/)
46 | web_modules/
47 |
48 | # TypeScript cache
49 | *.tsbuildinfo
50 |
51 | # Optional npm cache directory
52 | .npm
53 |
54 | # Optional eslint cache
55 | .eslintcache
56 |
57 | # Optional stylelint cache
58 | .stylelintcache
59 |
60 | # Microbundle cache
61 | .rpt2_cache/
62 | .rts2_cache_cjs/
63 | .rts2_cache_es/
64 | .rts2_cache_umd/
65 |
66 | # Optional REPL history
67 | .node_repl_history
68 |
69 | # Output of 'npm pack'
70 | *.tgz
71 |
72 | # Yarn Integrity file
73 | .yarn-integrity
74 |
75 | # dotenv environment variable files
76 | .env
77 | .env.development.local
78 | .env.test.local
79 | .env.production.local
80 | .env.local
81 |
82 | # parcel-bundler cache (https://parceljs.org/)
83 | .cache
84 | .parcel-cache
85 |
86 | # Next.js build output
87 | .next
88 | out
89 |
90 | # Nuxt.js build / generate output
91 | .nuxt
92 | dist
93 |
94 | # Gatsby files
95 | .cache/
96 | # Comment in the public line in if your project uses Gatsby and not Next.js
97 | # https://nextjs.org/blog/next-9-1#public-directory-support
98 | # public
99 |
100 | # vuepress build output
101 | .vuepress/dist
102 |
103 | # vuepress v2.x temp and cache directory
104 | .temp
105 | .cache
106 |
107 | # Docusaurus cache and generated files
108 | .docusaurus
109 |
110 | # Serverless directories
111 | .serverless/
112 |
113 | # FuseBox cache
114 | .fusebox/
115 |
116 | # DynamoDB Local files
117 | .dynamodb/
118 |
119 | # TernJS port file
120 | .tern-port
121 |
122 | # Stores VSCode versions used for testing VSCode extensions
123 | .vscode-test
124 |
125 | # yarn v2
126 | .yarn/cache
127 | .yarn/unplugged
128 | .yarn/build-state.yml
129 | .yarn/install-state.gz
130 | .pnp.*
131 | package-lock.json
132 |
133 | docs/build
--------------------------------------------------------------------------------
/frontend/src/components/UI/HoverableLink.tsx:
--------------------------------------------------------------------------------
1 | import React, { useState, useEffect, useRef } from 'react';
2 | import { HoverableLinkProps } from '../../types';
3 | const HoverableLink: React.FC = ({ url, children }) => {
4 | const [hovering, setHovering] = useState(false);
5 | const [iframeSrc, setIframeSrc] = useState('');
6 | const [mousePosition, setMousePosition] = useState({ x: 0, y: 0 });
7 | const popupRef = useRef(null);
8 | useEffect(() => {
9 | let timer: NodeJS.Timeout;
10 | if (hovering) {
11 | setIframeSrc('');
12 | timer = setTimeout(() => {
13 | setIframeSrc(url);
14 | }, 100);
15 | }
16 | return () => clearTimeout(timer);
17 | }, [hovering, url]);
18 | const handleMouseEnter = (event: React.MouseEvent) => {
19 | setHovering(true);
20 | setMousePosition({ x: event.clientX, y: event.clientY });
21 | };
22 | const handleMouseMove = (event: React.MouseEvent) => {
23 | if (hovering) {
24 | setMousePosition({ x: event.clientX, y: event.clientY });
25 | }
26 | };
27 | const handleMouseLeave = () => {
28 | setHovering(false);
29 | };
30 | const isYouTubeURL = (url: string): boolean => {
31 | return url.includes('youtube.com') || url.includes('youtu.be');
32 | };
33 | const extractYouTubeVideoId = (url: string): string => {
34 | const videoIdRegex = /(?:\/embed\/|\/watch\?v=|\/(?:embed\/|v\/|watch\?.*v=|youtu\.be\/|embed\/|v=))([^&?#]+)/;
35 | const match = url.match(videoIdRegex);
36 | return match ? match[1] : '';
37 | };
38 | return (
39 |
45 | {children}
46 | {hovering && (
47 |
52 | {isYouTubeURL(url) ? (
53 |
62 | ) : (
63 | iframeSrc &&
64 | )}
65 |
66 | )}
67 |
68 |
31 |
95 |
39 |
40 | Large Document Notice
41 |
42 | One or more of your selected documents are large and may take extra time to process. Please review the
43 | estimated times below
44 |
45 |
46 | {largeFiles.map((f, i) => {
47 | const { minutes, seconds } = calculateProcessingTime(f.size as number, 0.2);
48 | return (
49 |
50 |
51 |
52 | {
55 | if (e.target.checked) {
56 | handleToggle(true, f.id);
57 | } else {
58 | handleToggle(false, f.id);
59 | }
60 | }}
61 | checked={checked.indexOf(f.id) !== -1}
62 | tabIndex={-1}
63 | />
64 |
65 |
66 | {imageIcon[f.fileSource] ? (
67 | 
68 | ) : (
69 |
70 | )}
71 |
72 |
75 |
76 | {f.name} - {Math.floor((f?.size as number) / 1000)?.toFixed(2)}KB
77 | {f.fileSource === 'local file' && minutes === 0 && typeof f.size === 'number'
78 | ? `- ${seconds} Sec `
79 | : f.fileSource === 'local file'
80 | ? `- ${minutes} Min`
81 | : ''}
82 |
83 | {typeof f.size === 'number' && f.size > chunkSize ? (
84 |
85 |
86 |
87 | ) : (
88 | <>
89 | )}
90 |
91 | }
92 | />
93 |
94 |
95 | );
96 | })}
97 |
98 |
99 |
100 |
101 | );
102 | };
103 | export default LargeFilesAlert;
--------------------------------------------------------------------------------
/POC_Documents/V1/Local-to-global-genAI_GraphRAG_V1:
--------------------------------------------------------------------------------
1 | Graph DB Connectors and GenAI Integrations POC_v1
2 |
3 | “” This is the version v1, where main content is taken with the below paper1 and some other contents are added with other papers and blogs.””
4 |
5 | Paper1: From Local to Global: A Graph RAG Approach to
6 | Query-Focused Summarization
7 |
8 | “” Python-based implementation of both global and local Graph RAG
9 | approaches are forthcoming at https://aka.ms/graphrag.””
10 |
11 | Graph RAG approach uses the natural modularity of graphs to partition data for global summarization. It uses an LLM to build a graph-based text index in two stages:
12 | 1. Derive an entity knowledge graph from the source documents,
13 | 2. Pre-generate community summaries for all groups of closely related entities.
14 |
15 | IT can answer such questions like “What are the main themes in the dataset?”, Basically, it is an inherently query focused summarization (QFS) task. Graph RAG approach improves the question answering over private text corpora that scales with both the generality of user questions and the quantity of source text to be indexed. Graph RAG leads to substantial improvements for both the comprehensiveness and diversity of generated answers.
16 |
17 | community descriptions provide complete coverage of the underlying graph index, and the input documents it represents. Query-focused summarization of an entire corpus is then made possible using a map-reduce approach: first using each community summary to answer the query independently and in parallel, then summarizing all relevant partial answers into a final global answer.
18 |
19 |
20 | Figure 1: Graph RAG pipeline using an LLM-derived graph index of source document text
21 |
22 | I. Data Ingestion:
23 | 1. Documents/chunks/Text Preprocessing:
24 | To reduce document size and improve latency use text summarization for heavy documents or multi-documents with the below steps:
25 | Step1: LLM (use a specific LLM embedding to summarize documents)
26 | Step2: Knowledge graph to reduce size with entities, relationship, and their properties with subgraphs.
27 | Note: Above steps can be followed bidirectionally.
28 |
29 | 2. Create vector DB/Embedding/Indexing with LLM embedding.
30 |
31 | II. vector Embedding/Indexing Storage
32 |
33 | Generate KG with the embedding and store in graph DB or store the embedding in FAISS/pinecone to improve latency and accuracy.
34 | or
35 | Both the methods can be combined (KG+ vector embedding) and store DB to handle both structure and unstructured data
36 |
37 | Generate four communities (C0, C1, C2, C3) Graph RAG summary from the embedding/KG of the document/multi-documents/embeddings by using text summarized Map Reduced approach.
38 | C0: Uses root-level community summaries (fewest in number) to answer user queries.
39 | C1: Uses high-level community summaries to answer queries. These are sub-communities.
40 | of C0, if present, otherwise C0 communities projected down.
41 | C2: Uses intermediate-level community summaries to answer queries. These are subcommunities of C1, if present, otherwise C1 communities projected down.
42 | C3: Uses low-level community summaries (greatest in number) to answer queries. These
43 | are sub-communities of C2, if present, otherwise C2 communities projected down.
44 |
45 | Figure 2.1 Communities’ Summary Figure.2.2 Communities Graph
46 |
47 | Figure 3. Summarized Community Graph
48 |
49 | III. Chat Response/Architecture:
50 | Approaches: Multi-hope RAG, memory-based response, Head-to-Head measures.
51 | Head-To-Head measures can use as a performance metrics using an LLM evaluator are as follows:
52 | • Comprehensiveness: How much detail does the answer provide to cover all aspects and
53 | details of the question?
54 | • Diversity: How varied and rich is the answer in providing different perspectives and insights on the question?
55 |
56 |
57 | For a given community levels (Fig.2.2,2.2 & 3), the global answer to any user query is generated as follows:
58 |
59 | • Prepare community summaries. Community summaries are randomly shuffled and divided
60 | into chunks of pre-specified token size. This ensures relevant information is distributed
61 | across chunks, rather than concentrated (and potentially lost) in a single context window.
62 |
63 | • Map community answers. Generate intermediate answers in parallel, one for each chunk.
64 | The LLM is also asked to generate a score between 0-100 indicating how helpful the generated
65 | answer is in answering the target question. Answers with score 0 are filtered out.
66 |
67 | • Reduce to global answer. Intermediate community answers are sorted in descending order
68 | of helpfulness score and iteratively added into a new context window until the token limit
69 | is reached. This final context is used to generate the global answer returned to the user.
70 |
71 |
--------------------------------------------------------------------------------
7 |