├── .DS_Store ├── .history ├── config.ini_20240522181621.bak ├── graphQA_20240522181756.py ├── pdf2graph-JINGKUNZHOU-Mini_20240523153129.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524231024.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524231025.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524231028.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524231218.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234321.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234358.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234401.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234406.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234411.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234414.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234418.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234420.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234422.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234427.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234428.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234430.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234435.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234454.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234455.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234458.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234509.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234512.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234516.py ├── pdf2graph-JINGKUNZHOU-Mini_20240524234517.py ├── pdf2graph_20240523153130.py ├── readme_20240523183034.md ├── readme_20240524012716.md ├── readme_20240524012808.md ├── readme_20240524012811.md ├── readme_20240524012815.md ├── readme_20240524012817.md ├── readme_20240524012823.md ├── readme_20240524012825.md ├── readme_20240524012850.md ├── readme_20240524012852.md ├── readme_20240524012903.md ├── readme_20240524012912.md ├── readme_20240524012917.md ├── readme_20240524013044.md ├── readme_20240524013045.md ├── readme_20240524013048.md ├── readme_20240524013049.md ├── readme_20240524013054.md ├── readme_20240524013055.md ├── readme_20240524013057.md ├── readme_20240524013107.md ├── readme_20240524013113.md ├── readme_20240524013114.md ├── readme_20240524013127.md ├── readme_20240524013132.md ├── readme_20240524013135.md ├── readme_20240524013222.md ├── readme_20240524013228.md ├── readme_20240524013230.md ├── readme_20240524013233.md ├── readme_20240524013242.md ├── readme_20240524013248.md ├── readme_20240524013251.md ├── readme_20240524013256.md ├── readme_20240524013301.md ├── readme_20240524013303.md ├── readme_20240524013307.md ├── readme_20240524013310.md ├── readme_20240524013313.md ├── readme_20240524013316.md ├── readme_20240524013319.md ├── readme_20240524013321.md ├── readme_20240524013324.md ├── readme_20240524013325.md ├── readme_20240524013327.md ├── readme_20240524013328.md ├── readme_20240524013331.md ├── readme_20240524013336.md ├── readme_20240524013407.md ├── readme_20240524013417.md ├── readme_20240524013436.md ├── readme_20240524013443.md ├── readme_20240524013452.md ├── readme_20240524013455.md ├── readme_20240524013457.md ├── readme_20240524013459.md ├── readme_20240524013503.md ├── readme_20240524014526.md ├── readme_20240524014551.md ├── readme_20240524014558.md ├── readme_20240524014739.md ├── readme_20240524014741.md ├── readme_20240524014748.md ├── readme_20240524014752.md ├── readme_20240524014807.md ├── readme_20240524014810.md ├── readme_20240524014821.md ├── readme_20240524014825.md ├── readme_20240524014837.md ├── readme_20240524014844.md ├── readme_20240524014847.md ├── readme_20240524014852.md ├── readme_20240524014902.md ├── readme_20240524014904.md ├── readme_20240524015042.md ├── readme_20240524015045.md ├── readme_20240524015120.md ├── readme_20240524015154.md ├── readme_20240524015158.md ├── readme_20240524015247.md ├── readme_20240524015253.md ├── requirements_20240522013842.txt └── requirements_20240523183147.txt ├── LICENSE ├── config.ini.bak ├── graphQA.py ├── pdf2graph.py ├── readme.md ├── requirements.txt └── res ├── 1.png ├── 2.png ├── 3.png ├── 4.png ├── 5.gif └── graph.png /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjkhurry/Graph-RAG/11bc55d1908dea366cbcb69b485ac787d595cf7f/.DS_Store -------------------------------------------------------------------------------- /.history/config.ini_20240522181621.bak: -------------------------------------------------------------------------------- 1 | [Neo4j] 2 | uri = bolt://localhost:7687 3 | username = neo4j 4 | password = 12345678 5 | 6 | [LLM] 7 | #Ollama or OpenAI (llama.cpp server or OpenAI API) 8 | llm = Ollama 9 | temperature = 0.0 10 | #max_tokens is the maximum number of tokens that can be generated 11 | max_tokens = 2048 12 | stop = ["<|im_end|>"] 13 | 14 | [OpenAI] 15 | model = gpt-3.5-turbo 16 | api_key = sk-123456 17 | #api_base for llama.cpp server, leave it empty for OpenAI API 18 | api_base = http://127.0.0.1:8080/v1 19 | 20 | [Ollama] 21 | model = llama3 22 | #num_ctx is the number of context tokens, which is the maximum number of tokens that can be processed at a time 23 | num_ctx = 2048 24 | 25 | [Zotero] 26 | enabled = True 27 | library_id = 12345678 28 | library_type = user 29 | api_key = xxxxxx 30 | #Zotero_dir is the directory where the attachments are stored 31 | Zotero_dir = /Path/To/Zotero/storage/ 32 | 33 | [PDF] 34 | #extract_images is used to extract images from PDF files 35 | extract_images = False 36 | max_char = 1000 37 | new_after_n_chars = 800 38 | combine_text_under_n_chars = 200 39 | 40 | [Embeddings] 41 | #embedding method, Ollama or OpenAI 42 | embeddings = Ollama 43 | #text-embedding-3-small for OpenAI, mxbai-embed-large for Ollama, left empty to use the llm model 44 | model = mxbai-embed-large 45 | 46 | [Chat] 47 | #chatbot, Ollama or OpenAI, left empty to use the llm model 48 | chatbot = Ollama 49 | #chatbot model, gpt-3.5-turbo for OpenAI, phi3 for Ollama 50 | model = llama3 51 | #chatbot temperature 52 | temperature = 0.8 53 | #chatbot max_tokens 54 | max_tokens = 2048 55 | #chatbot stop 56 | stop = ["<|im_end|>","USER:"] -------------------------------------------------------------------------------- /.history/readme_20240523183034.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | ## Features 8 | 9 | - Create a RAG from PDFs 10 | - Connect to Zotero to retrival PDFs 11 | - Graph visulation through Neo4j Browser 12 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 13 | 14 | ## Installation 15 | 16 | **Step1**, clone this project and install dependencies 17 | ``` 18 | git clone https://github.com/zjkhurry/Graph-RAG.git 19 | cd Graph-RAG 20 | pip install requitements 21 | ``` 22 | Copy and midofy [config.ini](./config.ini) 23 | ``` 24 | cp config.ini.bak config.ini 25 | ``` 26 | 27 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 28 | ``` 29 | brew install --cask neo4j 30 | ``` 31 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 32 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 33 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 34 | ![avatar](res/4.png) 35 | 3. Start the Graph DBMS. 36 | 37 | **Step3 (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 38 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 39 | ``` 40 | ollama pull llama3 41 | ollama pull mxbai-embed-large 42 | ``` 43 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 44 | 45 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 46 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 47 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 48 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 49 | - Are you accessing your own Zotero library? library_type is 'user' 50 | - Are you accessing a shared group library? library_type is 'group'. 51 | 52 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 53 | 54 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 55 | 56 | ## Usage 57 | To convert PDF files into graph, just use the dpf2graph.py 58 | ``` 59 | python pdf2graph.py 60 | ``` 61 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 62 | 63 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 64 | ``` 65 | python graphQA.py 66 | ``` 67 | 68 | ## References 69 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 70 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 71 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 72 | [Pyzotero](https://github.com/urschrei/pyzotero) 73 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) 74 | -------------------------------------------------------------------------------- /.history/readme_20240524012716.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | ## Features 8 | 9 | - Create a RAG from PDFs 10 | - Connect to Zotero to retrival PDFs 11 | - Graph visulation through Neo4j Browser 12 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 13 | 14 | ## Installation 15 | 16 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 17 | ``` 18 | brew install --cask neo4j 19 | ``` 20 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 21 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 22 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 23 | ![avatar](res/4.png) 24 | 3. Start the Graph DBMS. 25 | 26 | Secondly, install dependencies 27 | ``` 28 | pip install requitements 29 | ``` 30 | 31 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 32 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 33 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 34 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 35 | - Are you accessing your own Zotero library? library_type is 'user' 36 | - Are you accessing a shared group library? library_type is 'group'. 37 | 38 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 39 | 40 | ## Usage 41 | To convert PDF files into graph, just use the dpf2graph.py 42 | ``` 43 | python pdf2graph.py 44 | ``` 45 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 46 | 47 | -------------------------------------------------------------------------------- /.history/readme_20240524012808.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | 8 | 9 | ## Features 10 | 11 | - Create a RAG from PDFs 12 | - Connect to Zotero to retrival PDFs 13 | - Graph visulation through Neo4j Browser 14 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 15 | 16 | ## Installation 17 | 18 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 19 | ``` 20 | brew install --cask neo4j 21 | ``` 22 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 23 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 24 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 25 | ![avatar](res/4.png) 26 | 3. Start the Graph DBMS. 27 | 28 | Secondly, install dependencies 29 | ``` 30 | pip install requitements 31 | ``` 32 | 33 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 34 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 35 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 36 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 37 | - Are you accessing your own Zotero library? library_type is 'user' 38 | - Are you accessing a shared group library? library_type is 'group'. 39 | 40 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 41 | 42 | ## Usage 43 | To convert PDF files into graph, just use the dpf2graph.py 44 | ``` 45 | python pdf2graph.py 46 | ``` 47 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 48 | 49 | -------------------------------------------------------------------------------- /.history/readme_20240524012811.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | [[]] 8 | 9 | ## Features 10 | 11 | - Create a RAG from PDFs 12 | - Connect to Zotero to retrival PDFs 13 | - Graph visulation through Neo4j Browser 14 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 15 | 16 | ## Installation 17 | 18 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 19 | ``` 20 | brew install --cask neo4j 21 | ``` 22 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 23 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 24 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 25 | ![avatar](res/4.png) 26 | 3. Start the Graph DBMS. 27 | 28 | Secondly, install dependencies 29 | ``` 30 | pip install requitements 31 | ``` 32 | 33 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 34 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 35 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 36 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 37 | - Are you accessing your own Zotero library? library_type is 'user' 38 | - Are you accessing a shared group library? library_type is 'group'. 39 | 40 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 41 | 42 | ## Usage 43 | To convert PDF files into graph, just use the dpf2graph.py 44 | ``` 45 | python pdf2graph.py 46 | ``` 47 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 48 | 49 | -------------------------------------------------------------------------------- /.history/readme_20240524012815.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | [[res]] 8 | 9 | ## Features 10 | 11 | - Create a RAG from PDFs 12 | - Connect to Zotero to retrival PDFs 13 | - Graph visulation through Neo4j Browser 14 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 15 | 16 | ## Installation 17 | 18 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 19 | ``` 20 | brew install --cask neo4j 21 | ``` 22 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 23 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 24 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 25 | ![avatar](res/4.png) 26 | 3. Start the Graph DBMS. 27 | 28 | Secondly, install dependencies 29 | ``` 30 | pip install requitements 31 | ``` 32 | 33 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 34 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 35 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 36 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 37 | - Are you accessing your own Zotero library? library_type is 'user' 38 | - Are you accessing a shared group library? library_type is 'group'. 39 | 40 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 41 | 42 | ## Usage 43 | To convert PDF files into graph, just use the dpf2graph.py 44 | ``` 45 | python pdf2graph.py 46 | ``` 47 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 48 | 49 | -------------------------------------------------------------------------------- /.history/readme_20240524012817.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | [[res/]] 8 | 9 | ## Features 10 | 11 | - Create a RAG from PDFs 12 | - Connect to Zotero to retrival PDFs 13 | - Graph visulation through Neo4j Browser 14 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 15 | 16 | ## Installation 17 | 18 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 19 | ``` 20 | brew install --cask neo4j 21 | ``` 22 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 23 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 24 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 25 | ![avatar](res/4.png) 26 | 3. Start the Graph DBMS. 27 | 28 | Secondly, install dependencies 29 | ``` 30 | pip install requitements 31 | ``` 32 | 33 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 34 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 35 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 36 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 37 | - Are you accessing your own Zotero library? library_type is 'user' 38 | - Are you accessing a shared group library? library_type is 'group'. 39 | 40 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 41 | 42 | ## Usage 43 | To convert PDF files into graph, just use the dpf2graph.py 44 | ``` 45 | python pdf2graph.py 46 | ``` 47 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 48 | 49 | -------------------------------------------------------------------------------- /.history/readme_20240524012823.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | [[res/l]] 8 | 9 | ## Features 10 | 11 | - Create a RAG from PDFs 12 | - Connect to Zotero to retrival PDFs 13 | - Graph visulation through Neo4j Browser 14 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 15 | 16 | ## Installation 17 | 18 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 19 | ``` 20 | brew install --cask neo4j 21 | ``` 22 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 23 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 24 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 25 | ![avatar](res/4.png) 26 | 3. Start the Graph DBMS. 27 | 28 | Secondly, install dependencies 29 | ``` 30 | pip install requitements 31 | ``` 32 | 33 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 34 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 35 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 36 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 37 | - Are you accessing your own Zotero library? library_type is 'user' 38 | - Are you accessing a shared group library? library_type is 'group'. 39 | 40 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 41 | 42 | ## Usage 43 | To convert PDF files into graph, just use the dpf2graph.py 44 | ``` 45 | python pdf2graph.py 46 | ``` 47 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 48 | 49 | -------------------------------------------------------------------------------- /.history/readme_20240524012825.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | [[res/]] 8 | 9 | ## Features 10 | 11 | - Create a RAG from PDFs 12 | - Connect to Zotero to retrival PDFs 13 | - Graph visulation through Neo4j Browser 14 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 15 | 16 | ## Installation 17 | 18 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 19 | ``` 20 | brew install --cask neo4j 21 | ``` 22 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 23 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 24 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 25 | ![avatar](res/4.png) 26 | 3. Start the Graph DBMS. 27 | 28 | Secondly, install dependencies 29 | ``` 30 | pip install requitements 31 | ``` 32 | 33 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 34 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 35 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 36 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 37 | - Are you accessing your own Zotero library? library_type is 'user' 38 | - Are you accessing a shared group library? library_type is 'group'. 39 | 40 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 41 | 42 | ## Usage 43 | To convert PDF files into graph, just use the dpf2graph.py 44 | ``` 45 | python pdf2graph.py 46 | ``` 47 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 48 | 49 | -------------------------------------------------------------------------------- /.history/readme_20240524012850.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | [[res/5.]] 8 | 9 | ## Features 10 | 11 | - Create a RAG from PDFs 12 | - Connect to Zotero to retrival PDFs 13 | - Graph visulation through Neo4j Browser 14 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 15 | 16 | ## Installation 17 | 18 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 19 | ``` 20 | brew install --cask neo4j 21 | ``` 22 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 23 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 24 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 25 | ![avatar](res/4.png) 26 | 3. Start the Graph DBMS. 27 | 28 | Secondly, install dependencies 29 | ``` 30 | pip install requitements 31 | ``` 32 | 33 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 34 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 35 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 36 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 37 | - Are you accessing your own Zotero library? library_type is 'user' 38 | - Are you accessing a shared group library? library_type is 'group'. 39 | 40 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 41 | 42 | ## Usage 43 | To convert PDF files into graph, just use the dpf2graph.py 44 | ``` 45 | python pdf2graph.py 46 | ``` 47 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 48 | 49 | -------------------------------------------------------------------------------- /.history/readme_20240524012852.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | [[res/5.gif]] 8 | 9 | ## Features 10 | 11 | - Create a RAG from PDFs 12 | - Connect to Zotero to retrival PDFs 13 | - Graph visulation through Neo4j Browser 14 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 15 | 16 | ## Installation 17 | 18 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 19 | ``` 20 | brew install --cask neo4j 21 | ``` 22 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 23 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 24 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 25 | ![avatar](res/4.png) 26 | 3. Start the Graph DBMS. 27 | 28 | Secondly, install dependencies 29 | ``` 30 | pip install requitements 31 | ``` 32 | 33 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 34 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 35 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 36 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 37 | - Are you accessing your own Zotero library? library_type is 'user' 38 | - Are you accessing a shared group library? library_type is 'group'. 39 | 40 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 41 | 42 | ## Usage 43 | To convert PDF files into graph, just use the dpf2graph.py 44 | ``` 45 | python pdf2graph.py 46 | ``` 47 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 48 | 49 | -------------------------------------------------------------------------------- /.history/readme_20240524012903.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | 8 | [[res/5.gif]] 9 | 10 | ## Features 11 | 12 | - Create a RAG from PDFs 13 | - Connect to Zotero to retrival PDFs 14 | - Graph visulation through Neo4j Browser 15 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 16 | 17 | ## Installation 18 | 19 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 20 | ``` 21 | brew install --cask neo4j 22 | ``` 23 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 24 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 25 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 26 | ![avatar](res/4.png) 27 | 3. Start the Graph DBMS. 28 | 29 | Secondly, install dependencies 30 | ``` 31 | pip install requitements 32 | ``` 33 | 34 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 35 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 36 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 37 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 38 | - Are you accessing your own Zotero library? library_type is 'user' 39 | - Are you accessing a shared group library? library_type is 'group'. 40 | 41 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 42 | 43 | ## Usage 44 | To convert PDF files into graph, just use the dpf2graph.py 45 | ``` 46 | python pdf2graph.py 47 | ``` 48 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 49 | 50 | -------------------------------------------------------------------------------- /.history/readme_20240524012912.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Q 8 | [[res/5.gif]] 9 | 10 | ## Features 11 | 12 | - Create a RAG from PDFs 13 | - Connect to Zotero to retrival PDFs 14 | - Graph visulation through Neo4j Browser 15 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 16 | 17 | ## Installation 18 | 19 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 20 | ``` 21 | brew install --cask neo4j 22 | ``` 23 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 24 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 25 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 26 | ![avatar](res/4.png) 27 | 3. Start the Graph DBMS. 28 | 29 | Secondly, install dependencies 30 | ``` 31 | pip install requitements 32 | ``` 33 | 34 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 35 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 36 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 37 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 38 | - Are you accessing your own Zotero library? library_type is 'user' 39 | - Are you accessing a shared group library? library_type is 'group'. 40 | 41 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 42 | 43 | ## Usage 44 | To convert PDF files into graph, just use the dpf2graph.py 45 | ``` 46 | python pdf2graph.py 47 | ``` 48 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 49 | 50 | -------------------------------------------------------------------------------- /.history/readme_20240524012917.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Question is 8 | [[res/5.gif]] 9 | 10 | ## Features 11 | 12 | - Create a RAG from PDFs 13 | - Connect to Zotero to retrival PDFs 14 | - Graph visulation through Neo4j Browser 15 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 16 | 17 | ## Installation 18 | 19 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 20 | ``` 21 | brew install --cask neo4j 22 | ``` 23 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 24 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 25 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 26 | ![avatar](res/4.png) 27 | 3. Start the Graph DBMS. 28 | 29 | Secondly, install dependencies 30 | ``` 31 | pip install requitements 32 | ``` 33 | 34 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 35 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 36 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 37 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 38 | - Are you accessing your own Zotero library? library_type is 'user' 39 | - Are you accessing a shared group library? library_type is 'group'. 40 | 41 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 42 | 43 | ## Usage 44 | To convert PDF files into graph, just use the dpf2graph.py 45 | ``` 46 | python pdf2graph.py 47 | ``` 48 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 49 | 50 | -------------------------------------------------------------------------------- /.history/readme_20240524013044.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Question is 8 | ! 9 | 10 | ## Features 11 | 12 | - Create a RAG from PDFs 13 | - Connect to Zotero to retrival PDFs 14 | - Graph visulation through Neo4j Browser 15 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 16 | 17 | ## Installation 18 | 19 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 20 | ``` 21 | brew install --cask neo4j 22 | ``` 23 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 24 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 25 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 26 | ![avatar](res/4.png) 27 | 3. Start the Graph DBMS. 28 | 29 | Secondly, install dependencies 30 | ``` 31 | pip install requitements 32 | ``` 33 | 34 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 35 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 36 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 37 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 38 | - Are you accessing your own Zotero library? library_type is 'user' 39 | - Are you accessing a shared group library? library_type is 'group'. 40 | 41 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 42 | 43 | ## Usage 44 | To convert PDF files into graph, just use the dpf2graph.py 45 | ``` 46 | python pdf2graph.py 47 | ``` 48 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 49 | 50 | -------------------------------------------------------------------------------- /.history/readme_20240524013045.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Question is 8 | ![] 9 | 10 | ## Features 11 | 12 | - Create a RAG from PDFs 13 | - Connect to Zotero to retrival PDFs 14 | - Graph visulation through Neo4j Browser 15 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 16 | 17 | ## Installation 18 | 19 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 20 | ``` 21 | brew install --cask neo4j 22 | ``` 23 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 24 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 25 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 26 | ![avatar](res/4.png) 27 | 3. Start the Graph DBMS. 28 | 29 | Secondly, install dependencies 30 | ``` 31 | pip install requitements 32 | ``` 33 | 34 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 35 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 36 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 37 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 38 | - Are you accessing your own Zotero library? library_type is 'user' 39 | - Are you accessing a shared group library? library_type is 'group'. 40 | 41 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 42 | 43 | ## Usage 44 | To convert PDF files into graph, just use the dpf2graph.py 45 | ``` 46 | python pdf2graph.py 47 | ``` 48 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 49 | 50 | -------------------------------------------------------------------------------- /.history/readme_20240524013048.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Question is 8 | ![gif] 9 | 10 | ## Features 11 | 12 | - Create a RAG from PDFs 13 | - Connect to Zotero to retrival PDFs 14 | - Graph visulation through Neo4j Browser 15 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 16 | 17 | ## Installation 18 | 19 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 20 | ``` 21 | brew install --cask neo4j 22 | ``` 23 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 24 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 25 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 26 | ![avatar](res/4.png) 27 | 3. Start the Graph DBMS. 28 | 29 | Secondly, install dependencies 30 | ``` 31 | pip install requitements 32 | ``` 33 | 34 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 35 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 36 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 37 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 38 | - Are you accessing your own Zotero library? library_type is 'user' 39 | - Are you accessing a shared group library? library_type is 'group'. 40 | 41 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 42 | 43 | ## Usage 44 | To convert PDF files into graph, just use the dpf2graph.py 45 | ``` 46 | python pdf2graph.py 47 | ``` 48 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 49 | 50 | -------------------------------------------------------------------------------- /.history/readme_20240524013049.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Question is 8 | ![gif]() 9 | 10 | ## Features 11 | 12 | - Create a RAG from PDFs 13 | - Connect to Zotero to retrival PDFs 14 | - Graph visulation through Neo4j Browser 15 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 16 | 17 | ## Installation 18 | 19 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 20 | ``` 21 | brew install --cask neo4j 22 | ``` 23 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 24 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 25 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 26 | ![avatar](res/4.png) 27 | 3. Start the Graph DBMS. 28 | 29 | Secondly, install dependencies 30 | ``` 31 | pip install requitements 32 | ``` 33 | 34 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 35 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 36 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 37 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 38 | - Are you accessing your own Zotero library? library_type is 'user' 39 | - Are you accessing a shared group library? library_type is 'group'. 40 | 41 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 42 | 43 | ## Usage 44 | To convert PDF files into graph, just use the dpf2graph.py 45 | ``` 46 | python pdf2graph.py 47 | ``` 48 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 49 | 50 | -------------------------------------------------------------------------------- /.history/readme_20240524013054.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Question is 8 | ![gif](res/) 9 | 10 | ## Features 11 | 12 | - Create a RAG from PDFs 13 | - Connect to Zotero to retrival PDFs 14 | - Graph visulation through Neo4j Browser 15 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 16 | 17 | ## Installation 18 | 19 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 20 | ``` 21 | brew install --cask neo4j 22 | ``` 23 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 24 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 25 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 26 | ![avatar](res/4.png) 27 | 3. Start the Graph DBMS. 28 | 29 | Secondly, install dependencies 30 | ``` 31 | pip install requitements 32 | ``` 33 | 34 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 35 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 36 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 37 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 38 | - Are you accessing your own Zotero library? library_type is 'user' 39 | - Are you accessing a shared group library? library_type is 'group'. 40 | 41 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 42 | 43 | ## Usage 44 | To convert PDF files into graph, just use the dpf2graph.py 45 | ``` 46 | python pdf2graph.py 47 | ``` 48 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 49 | 50 | -------------------------------------------------------------------------------- /.history/readme_20240524013055.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Question is 8 | ![gif](res/5) 9 | 10 | ## Features 11 | 12 | - Create a RAG from PDFs 13 | - Connect to Zotero to retrival PDFs 14 | - Graph visulation through Neo4j Browser 15 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 16 | 17 | ## Installation 18 | 19 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 20 | ``` 21 | brew install --cask neo4j 22 | ``` 23 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 24 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 25 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 26 | ![avatar](res/4.png) 27 | 3. Start the Graph DBMS. 28 | 29 | Secondly, install dependencies 30 | ``` 31 | pip install requitements 32 | ``` 33 | 34 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 35 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 36 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 37 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 38 | - Are you accessing your own Zotero library? library_type is 'user' 39 | - Are you accessing a shared group library? library_type is 'group'. 40 | 41 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 42 | 43 | ## Usage 44 | To convert PDF files into graph, just use the dpf2graph.py 45 | ``` 46 | python pdf2graph.py 47 | ``` 48 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 49 | 50 | -------------------------------------------------------------------------------- /.history/readme_20240524013057.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Question is 8 | ![gif](res/5.gif) 9 | 10 | ## Features 11 | 12 | - Create a RAG from PDFs 13 | - Connect to Zotero to retrival PDFs 14 | - Graph visulation through Neo4j Browser 15 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 16 | 17 | ## Installation 18 | 19 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 20 | ``` 21 | brew install --cask neo4j 22 | ``` 23 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 24 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 25 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 26 | ![avatar](res/4.png) 27 | 3. Start the Graph DBMS. 28 | 29 | Secondly, install dependencies 30 | ``` 31 | pip install requitements 32 | ``` 33 | 34 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 35 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 36 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 37 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 38 | - Are you accessing your own Zotero library? library_type is 'user' 39 | - Are you accessing a shared group library? library_type is 'group'. 40 | 41 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 42 | 43 | ## Usage 44 | To convert PDF files into graph, just use the dpf2graph.py 45 | ``` 46 | python pdf2graph.py 47 | ``` 48 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 49 | 50 | -------------------------------------------------------------------------------- /.history/readme_20240524013107.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | 8 | Question is 9 | ![gif](res/5.gif) 10 | 11 | ## Features 12 | 13 | - Create a RAG from PDFs 14 | - Connect to Zotero to retrival PDFs 15 | - Graph visulation through Neo4j Browser 16 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 17 | 18 | ## Installation 19 | 20 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 21 | ``` 22 | brew install --cask neo4j 23 | ``` 24 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 25 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 26 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 27 | ![avatar](res/4.png) 28 | 3. Start the Graph DBMS. 29 | 30 | Secondly, install dependencies 31 | ``` 32 | pip install requitements 33 | ``` 34 | 35 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 36 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 37 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 38 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 39 | - Are you accessing your own Zotero library? library_type is 'user' 40 | - Are you accessing a shared group library? library_type is 'group'. 41 | 42 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 43 | 44 | ## Usage 45 | To convert PDF files into graph, just use the dpf2graph.py 46 | ``` 47 | python pdf2graph.py 48 | ``` 49 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 50 | 51 | -------------------------------------------------------------------------------- /.history/readme_20240524013113.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase 8 | Question is 9 | ![gif](res/5.gif) 10 | 11 | ## Features 12 | 13 | - Create a RAG from PDFs 14 | - Connect to Zotero to retrival PDFs 15 | - Graph visulation through Neo4j Browser 16 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 17 | 18 | ## Installation 19 | 20 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 21 | ``` 22 | brew install --cask neo4j 23 | ``` 24 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 25 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 26 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 27 | ![avatar](res/4.png) 28 | 3. Start the Graph DBMS. 29 | 30 | Secondly, install dependencies 31 | ``` 32 | pip install requitements 33 | ``` 34 | 35 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 36 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 37 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 38 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 39 | - Are you accessing your own Zotero library? library_type is 'user' 40 | - Are you accessing a shared group library? library_type is 'group'. 41 | 42 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 43 | 44 | ## Usage 45 | To convert PDF files into graph, just use the dpf2graph.py 46 | ``` 47 | python pdf2graph.py 48 | ``` 49 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 50 | 51 | -------------------------------------------------------------------------------- /.history/readme_20240524013114.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is 9 | ![gif](res/5.gif) 10 | 11 | ## Features 12 | 13 | - Create a RAG from PDFs 14 | - Connect to Zotero to retrival PDFs 15 | - Graph visulation through Neo4j Browser 16 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 17 | 18 | ## Installation 19 | 20 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 21 | ``` 22 | brew install --cask neo4j 23 | ``` 24 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 25 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 26 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 27 | ![avatar](res/4.png) 28 | 3. Start the Graph DBMS. 29 | 30 | Secondly, install dependencies 31 | ``` 32 | pip install requitements 33 | ``` 34 | 35 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 36 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 37 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 38 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 39 | - Are you accessing your own Zotero library? library_type is 'user' 40 | - Are you accessing a shared group library? library_type is 'group'. 41 | 42 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 43 | 44 | ## Usage 45 | To convert PDF files into graph, just use the dpf2graph.py 46 | ``` 47 | python pdf2graph.py 48 | ``` 49 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 50 | 51 | -------------------------------------------------------------------------------- /.history/readme_20240524013127.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "" 9 | ![gif](res/5.gif) 10 | 11 | ## Features 12 | 13 | - Create a RAG from PDFs 14 | - Connect to Zotero to retrival PDFs 15 | - Graph visulation through Neo4j Browser 16 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 17 | 18 | ## Installation 19 | 20 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 21 | ``` 22 | brew install --cask neo4j 23 | ``` 24 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 25 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 26 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 27 | ![avatar](res/4.png) 28 | 3. Start the Graph DBMS. 29 | 30 | Secondly, install dependencies 31 | ``` 32 | pip install requitements 33 | ``` 34 | 35 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 36 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 37 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 38 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 39 | - Are you accessing your own Zotero library? library_type is 'user' 40 | - Are you accessing a shared group library? library_type is 'group'. 41 | 42 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 43 | 44 | ## Usage 45 | To convert PDF files into graph, just use the dpf2graph.py 46 | ``` 47 | python pdf2graph.py 48 | ``` 49 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 50 | 51 | -------------------------------------------------------------------------------- /.history/readme_20240524013132.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "" 9 | ![gif](res/5.gif) 10 | 11 | ## Features 12 | 13 | - Create a RAG from PDFs 14 | - Connect to Zotero to retrival PDFs 15 | - Graph visulation through Neo4j Browser 16 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 17 | 18 | ## Installation 19 | 20 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 21 | ``` 22 | brew install --cask neo4j 23 | ``` 24 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 25 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 26 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 27 | ![avatar](res/4.png) 28 | 3. Start the Graph DBMS. 29 | 30 | Secondly, install dependencies 31 | ``` 32 | pip install requitements 33 | ``` 34 | 35 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 36 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 37 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 38 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 39 | - Are you accessing your own Zotero library? library_type is 'user' 40 | - Are you accessing a shared group library? library_type is 'group'. 41 | 42 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 43 | 44 | ## Usage 45 | To convert PDF files into graph, just use the dpf2graph.py 46 | ``` 47 | python pdf2graph.py 48 | ``` 49 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 50 | 51 | -------------------------------------------------------------------------------- /.history/readme_20240524013135.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | 11 | ## Features 12 | 13 | - Create a RAG from PDFs 14 | - Connect to Zotero to retrival PDFs 15 | - Graph visulation through Neo4j Browser 16 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 17 | 18 | ## Installation 19 | 20 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 21 | ``` 22 | brew install --cask neo4j 23 | ``` 24 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 25 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 26 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 27 | ![avatar](res/4.png) 28 | 3. Start the Graph DBMS. 29 | 30 | Secondly, install dependencies 31 | ``` 32 | pip install requitements 33 | ``` 34 | 35 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 36 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 37 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 38 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 39 | - Are you accessing your own Zotero library? library_type is 'user' 40 | - Are you accessing a shared group library? library_type is 'group'. 41 | 42 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 43 | 44 | ## Usage 45 | To convert PDF files into graph, just use the dpf2graph.py 46 | ``` 47 | python pdf2graph.py 48 | ``` 49 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 50 | 51 | -------------------------------------------------------------------------------- /.history/readme_20240524013222.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | 11 | 12 | ## Features 13 | 14 | - Create a RAG from PDFs 15 | - Connect to Zotero to retrival PDFs 16 | - Graph visulation through Neo4j Browser 17 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 18 | 19 | ## Installation 20 | 21 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 22 | ``` 23 | brew install --cask neo4j 24 | ``` 25 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 26 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 27 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 28 | ![avatar](res/4.png) 29 | 3. Start the Graph DBMS. 30 | 31 | Secondly, install dependencies 32 | ``` 33 | pip install requitements 34 | ``` 35 | 36 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 37 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 38 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 39 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 40 | - Are you accessing your own Zotero library? library_type is 'user' 41 | - Are you accessing a shared group library? library_type is 'group'. 42 | 43 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 44 | 45 | ## Usage 46 | To convert PDF files into graph, just use the dpf2graph.py 47 | ``` 48 | python pdf2graph.py 49 | ``` 50 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 51 | 52 | -------------------------------------------------------------------------------- /.history/readme_20240524013228.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "" 11 | 12 | ## Features 13 | 14 | - Create a RAG from PDFs 15 | - Connect to Zotero to retrival PDFs 16 | - Graph visulation through Neo4j Browser 17 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 18 | 19 | ## Installation 20 | 21 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 22 | ``` 23 | brew install --cask neo4j 24 | ``` 25 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 26 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 27 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 28 | ![avatar](res/4.png) 29 | 3. Start the Graph DBMS. 30 | 31 | Secondly, install dependencies 32 | ``` 33 | pip install requitements 34 | ``` 35 | 36 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 37 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 38 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 39 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 40 | - Are you accessing your own Zotero library? library_type is 'user' 41 | - Are you accessing a shared group library? library_type is 'group'. 42 | 43 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 44 | 45 | ## Usage 46 | To convert PDF files into graph, just use the dpf2graph.py 47 | ``` 48 | python pdf2graph.py 49 | ``` 50 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 51 | 52 | -------------------------------------------------------------------------------- /.history/readme_20240524013230.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | 12 | ## Features 13 | 14 | - Create a RAG from PDFs 15 | - Connect to Zotero to retrival PDFs 16 | - Graph visulation through Neo4j Browser 17 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 18 | 19 | ## Installation 20 | 21 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 22 | ``` 23 | brew install --cask neo4j 24 | ``` 25 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 26 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 27 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 28 | ![avatar](res/4.png) 29 | 3. Start the Graph DBMS. 30 | 31 | Secondly, install dependencies 32 | ``` 33 | pip install requitements 34 | ``` 35 | 36 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 37 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 38 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 39 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 40 | - Are you accessing your own Zotero library? library_type is 'user' 41 | - Are you accessing a shared group library? library_type is 'group'. 42 | 43 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 44 | 45 | ## Usage 46 | To convert PDF files into graph, just use the dpf2graph.py 47 | ``` 48 | python pdf2graph.py 49 | ``` 50 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 51 | 52 | -------------------------------------------------------------------------------- /.history/readme_20240524013233.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | 12 | 13 | ## Features 14 | 15 | - Create a RAG from PDFs 16 | - Connect to Zotero to retrival PDFs 17 | - Graph visulation through Neo4j Browser 18 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 19 | 20 | ## Installation 21 | 22 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 23 | ``` 24 | brew install --cask neo4j 25 | ``` 26 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 27 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 28 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 29 | ![avatar](res/4.png) 30 | 3. Start the Graph DBMS. 31 | 32 | Secondly, install dependencies 33 | ``` 34 | pip install requitements 35 | ``` 36 | 37 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 38 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 39 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 40 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 41 | - Are you accessing your own Zotero library? library_type is 'user' 42 | - Are you accessing a shared group library? library_type is 'group'. 43 | 44 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 45 | 46 | ## Usage 47 | To convert PDF files into graph, just use the dpf2graph.py 48 | ``` 49 | python pdf2graph.py 50 | ``` 51 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 52 | 53 | -------------------------------------------------------------------------------- /.history/readme_20240524013242.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The 12 | 13 | ## Features 14 | 15 | - Create a RAG from PDFs 16 | - Connect to Zotero to retrival PDFs 17 | - Graph visulation through Neo4j Browser 18 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 19 | 20 | ## Installation 21 | 22 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 23 | ``` 24 | brew install --cask neo4j 25 | ``` 26 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 27 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 28 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 29 | ![avatar](res/4.png) 30 | 3. Start the Graph DBMS. 31 | 32 | Secondly, install dependencies 33 | ``` 34 | pip install requitements 35 | ``` 36 | 37 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 38 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 39 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 40 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 41 | - Are you accessing your own Zotero library? library_type is 'user' 42 | - Are you accessing a shared group library? library_type is 'group'. 43 | 44 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 45 | 46 | ## Usage 47 | To convert PDF files into graph, just use the dpf2graph.py 48 | ``` 49 | python pdf2graph.py 50 | ``` 51 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 52 | 53 | -------------------------------------------------------------------------------- /.history/readme_20240524013248.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference 12 | 13 | ## Features 14 | 15 | - Create a RAG from PDFs 16 | - Connect to Zotero to retrival PDFs 17 | - Graph visulation through Neo4j Browser 18 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 19 | 20 | ## Installation 21 | 22 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 23 | ``` 24 | brew install --cask neo4j 25 | ``` 26 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 27 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 28 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 29 | ![avatar](res/4.png) 30 | 3. Start the Graph DBMS. 31 | 32 | Secondly, install dependencies 33 | ``` 34 | pip install requitements 35 | ``` 36 | 37 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 38 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 39 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 40 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 41 | - Are you accessing your own Zotero library? library_type is 'user' 42 | - Are you accessing a shared group library? library_type is 'group'. 43 | 44 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 45 | 46 | ## Usage 47 | To convert PDF files into graph, just use the dpf2graph.py 48 | ``` 49 | python pdf2graph.py 50 | ``` 51 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 52 | 53 | -------------------------------------------------------------------------------- /.history/readme_20240524013251.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a 12 | 13 | ## Features 14 | 15 | - Create a RAG from PDFs 16 | - Connect to Zotero to retrival PDFs 17 | - Graph visulation through Neo4j Browser 18 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 19 | 20 | ## Installation 21 | 22 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 23 | ``` 24 | brew install --cask neo4j 25 | ``` 26 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 27 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 28 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 29 | ![avatar](res/4.png) 30 | 3. Start the Graph DBMS. 31 | 32 | Secondly, install dependencies 33 | ``` 34 | pip install requitements 35 | ``` 36 | 37 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 38 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 39 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 40 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 41 | - Are you accessing your own Zotero library? library_type is 'user' 42 | - Are you accessing a shared group library? library_type is 'group'. 43 | 44 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 45 | 46 | ## Usage 47 | To convert PDF files into graph, just use the dpf2graph.py 48 | ``` 49 | python pdf2graph.py 50 | ``` 51 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 52 | 53 | -------------------------------------------------------------------------------- /.history/readme_20240524013256.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge gra 12 | 13 | ## Features 14 | 15 | - Create a RAG from PDFs 16 | - Connect to Zotero to retrival PDFs 17 | - Graph visulation through Neo4j Browser 18 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 19 | 20 | ## Installation 21 | 22 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 23 | ``` 24 | brew install --cask neo4j 25 | ``` 26 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 27 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 28 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 29 | ![avatar](res/4.png) 30 | 3. Start the Graph DBMS. 31 | 32 | Secondly, install dependencies 33 | ``` 34 | pip install requitements 35 | ``` 36 | 37 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 38 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 39 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 40 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 41 | - Are you accessing your own Zotero library? library_type is 'user' 42 | - Are you accessing a shared group library? library_type is 'group'. 43 | 44 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 45 | 46 | ## Usage 47 | To convert PDF files into graph, just use the dpf2graph.py 48 | ``` 49 | python pdf2graph.py 50 | ``` 51 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 52 | 53 | -------------------------------------------------------------------------------- /.history/readme_20240524013301.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown blow 12 | 13 | ## Features 14 | 15 | - Create a RAG from PDFs 16 | - Connect to Zotero to retrival PDFs 17 | - Graph visulation through Neo4j Browser 18 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 19 | 20 | ## Installation 21 | 22 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 23 | ``` 24 | brew install --cask neo4j 25 | ``` 26 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 27 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 28 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 29 | ![avatar](res/4.png) 30 | 3. Start the Graph DBMS. 31 | 32 | Secondly, install dependencies 33 | ``` 34 | pip install requitements 35 | ``` 36 | 37 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 38 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 39 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 40 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 41 | - Are you accessing your own Zotero library? library_type is 'user' 42 | - Are you accessing a shared group library? library_type is 'group'. 43 | 44 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 45 | 46 | ## Usage 47 | To convert PDF files into graph, just use the dpf2graph.py 48 | ``` 49 | python pdf2graph.py 50 | ``` 51 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 52 | 53 | -------------------------------------------------------------------------------- /.history/readme_20240524013303.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown bl 12 | 13 | ## Features 14 | 15 | - Create a RAG from PDFs 16 | - Connect to Zotero to retrival PDFs 17 | - Graph visulation through Neo4j Browser 18 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 19 | 20 | ## Installation 21 | 22 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 23 | ``` 24 | brew install --cask neo4j 25 | ``` 26 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 27 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 28 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 29 | ![avatar](res/4.png) 30 | 3. Start the Graph DBMS. 31 | 32 | Secondly, install dependencies 33 | ``` 34 | pip install requitements 35 | ``` 36 | 37 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 38 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 39 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 40 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 41 | - Are you accessing your own Zotero library? library_type is 'user' 42 | - Are you accessing a shared group library? library_type is 'group'. 43 | 44 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 45 | 46 | ## Usage 47 | To convert PDF files into graph, just use the dpf2graph.py 48 | ``` 49 | python pdf2graph.py 50 | ``` 51 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 52 | 53 | -------------------------------------------------------------------------------- /.history/readme_20240524013307.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | 13 | ## Features 14 | 15 | - Create a RAG from PDFs 16 | - Connect to Zotero to retrival PDFs 17 | - Graph visulation through Neo4j Browser 18 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 19 | 20 | ## Installation 21 | 22 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 23 | ``` 24 | brew install --cask neo4j 25 | ``` 26 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 27 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 28 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 29 | ![avatar](res/4.png) 30 | 3. Start the Graph DBMS. 31 | 32 | Secondly, install dependencies 33 | ``` 34 | pip install requitements 35 | ``` 36 | 37 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 38 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 39 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 40 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 41 | - Are you accessing your own Zotero library? library_type is 'user' 42 | - Are you accessing a shared group library? library_type is 'group'. 43 | 44 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 45 | 46 | ## Usage 47 | To convert PDF files into graph, just use the dpf2graph.py 48 | ``` 49 | python pdf2graph.py 50 | ``` 51 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 52 | 53 | -------------------------------------------------------------------------------- /.history/readme_20240524013310.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013313.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![] 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013316.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![fig] 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013319.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![ex] 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013321.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![] 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013324.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg] 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013325.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg]() 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013327.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013328.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013331.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013336.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013407.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013417.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013436.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | > The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013443.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013452.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![1](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013455.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![1](res/1.png) ![2](res/2.png) ![avatar](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013457.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![1](res/1.png) ![2](res/2.png) ![3](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![avatar](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013459.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![1](res/1.png) ![2](res/2.png) ![3](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![4](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524013503.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | First, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 24 | ``` 25 | brew install --cask neo4j 26 | ``` 27 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 28 | ![1](res/1.png) ![2](res/2.png) ![3](res/3.png) 29 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install APOC plugin. 30 | ![4](res/4.png) 31 | 3. Start the Graph DBMS. 32 | 33 | Secondly, install dependencies 34 | ``` 35 | pip install requitements 36 | ``` 37 | 38 | Thirdly, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 39 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 40 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 41 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 42 | - Are you accessing your own Zotero library? library_type is 'user' 43 | - Are you accessing a shared group library? library_type is 'group'. 44 | 45 | Finally, modify the [config.ini](./config.ini), choose to use ollama or openai (llama.cpp), enter the directory where the zotero attachments are stored and so on. 46 | 47 | ## Usage 48 | To convert PDF files into graph, just use the dpf2graph.py 49 | ``` 50 | python pdf2graph.py 51 | ``` 52 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. 53 | 54 | -------------------------------------------------------------------------------- /.history/readme_20240524014526.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 3. Start the Graph DBMS. 43 | 44 | **Step3 (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014551.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | - **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 2. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 3. Start the Graph DBMS. 43 | 44 | **Step3 (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014558.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step3 (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014739.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | <>**Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step3 (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014741.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step3 (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014748.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**<>, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step3 (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014752.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step3 (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014807.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step1**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step3 (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014810.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step3 (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014821.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step1** (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014825.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step3** (optional)** Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014837.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step3** (optional) Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step4**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/readme_20240524014844.md: -------------------------------------------------------------------------------- 1 | # Graph RAG 2 | 3 | ## Introduction 4 | 5 | Graph RAG (Retrieval Augmented Generation) is a technology that uses graph databases to store relationships between data points, replacing traditional text embedding methods like Chuck and enhancing the performance of RAG. This project is base on [Langchain](https://github.com/langchain-ai/langchain/tree/master) and [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file). 6 | 7 | > Showcase: 8 | Question is "how about the performance of llava" 9 | ![gif](res/5.gif) 10 | Answer is "The performance of LLaVA, as mentioned in the structured data, has an overall score of 66.7 + 0.3, with a complex reasoning score of 81.4 + 0.3, a performance score of 58.8 + 0.6, and an improvement score of 49.2 + 0.8." 11 | The reference is a knowledge graph shown below 12 | ![kg](res/graph.png) 13 | 14 | ## Features 15 | 16 | - Create a RAG from PDFs 17 | - Connect to Zotero to retrival PDFs 18 | - Graph visulation through Neo4j Browser 19 | - All data are stored locally, supporting llama.cpp and Ollama local LLM 20 | 21 | ## Installation 22 | 23 | **Step1**, clone this project and install dependencies 24 | ``` 25 | git clone https://github.com/zjkhurry/Graph-RAG.git 26 | cd Graph-RAG 27 | pip install requitements 28 | ``` 29 | Copy and midofy [config.ini](./config.ini) 30 | ``` 31 | cp config.ini.bak config.ini 32 | ``` 33 | 34 | **Step2**, you need to install Neo4j. Download [Neo4j Desktop](https://neo4j.com/download/) or Mac can install with homebrew 35 | ``` 36 | brew install --cask neo4j 37 | ``` 38 | 1. Launch Neo4j Desktop, create a New Project and add a new Graph DBMS. 39 | ![avatar](res/1.png) ![avatar](res/2.png) ![avatar](res/3.png) 40 | 1. Enter the password into the [config.ini](./config.ini) - Neo4j - password. Then click on the Graph DBMS created in the last step and install [APOC plugin](https://github.com/neo4j/apoc). 41 | ![avatar](res/4.png) 42 | 1. Start the Graph DBMS. 43 | 44 | **Step3** (optional) Set up [Ollama](https://ollama.com) or [llama.cpp](https://github.com/ggerganov/llama.cpp) to use local LLM. 45 | - For Ollama, download the app [here](https://ollama.com), run Ollama and then 46 | ``` 47 | ollama pull interstellarninja/hermes-2-pro-llama-3-8b 48 | ollama pull mxbai-embed-large 49 | ``` 50 | *P.S. hermers-2-pro has a better sepport to function calling than original llama3.* 51 | - For llama.cpp, follow the instruction [here](https://github.com/ggerganov/llama.cpp) to build and run llama.cpp server. 52 | 53 | **Step1**, config the zotero. [Pyzotero](https://github.com/urschrei/pyzotero) is used to connect to Zotero library. You'll need the ID of the personal or group library you want to access: 54 | - Your personal library ID is available [here](https://www.zotero.org/settings/keys), in the section Your userID for use in API calls (you may need to login). Enter you ID into [config.ini](./config.ini) 55 | - For group libraries, the ID can be found by opening the group's page: https://www.zotero.org/groups/groupname, and hovering over the group settings link. The ID is the integer after /groups/ 56 | - You'll also need to get an API key [here](https://www.zotero.org/settings/keys/new), enter you API key into [config.ini](./config.ini) 57 | - Are you accessing your own Zotero library? library_type is 'user' 58 | - Are you accessing a shared group library? library_type is 'group'. 59 | 60 | *P.S. I can't make Zotero.file() work properly, maybe because I use WebDAV instead of zotero to store the pdf files, so Zotero_dir is needed to find the PDFs in the file system.* 61 | 62 | **Step5**, modify the config.ini, choose to use ollama or openai (llama.cpp), LLM model, embedding model and so on. You can choose different models for convert PDF, embedding and chat. 63 | 64 | ## Usage 65 | To convert PDF files into graph, just use the dpf2graph.py 66 | ``` 67 | python pdf2graph.py 68 | ``` 69 | Enter the path to the PDF file or just enter the title of the paper stored in Zotero library. Each PDF may take several minutes. 70 | 71 | After convert, you can run graphQA.py to query this graph, enter exit to stop. 72 | ``` 73 | python graphQA.py 74 | ``` 75 | 76 | ## References 77 | [openai cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/RAG_with_graph_db.ipynb) 78 | [Langchain](https://github.com/langchain-ai/langchain/tree/master) 79 | [Neo4j](https://github.com/neo4j/neo4j?tab=readme-ov-file) 80 | [Pyzotero](https://github.com/urschrei/pyzotero) 81 | [blogs](https://github.com/tomasonjo/blogs/blob/master/llm/enhancing_rag_with_graph.ipynb?ref=blog.langchain.dev) -------------------------------------------------------------------------------- /.history/requirements_20240522013842.txt: -------------------------------------------------------------------------------- 1 | json_repair==0.19.1 2 | langchain==0.2.0 3 | langchain_community==0.2.0 4 | langchain_core==0.2.0 5 | langchain_experimental==0.0.59 6 | langchain_openai==0.1.7 7 | pydantic==2.7.1 8 | pyzotero==1.5.19 9 | unstructured==0.13.6 10 | -------------------------------------------------------------------------------- /.history/requirements_20240523183147.txt: -------------------------------------------------------------------------------- 1 | json_repair 2 | langchain 3 | langchain_community 4 | langchain_core 5 | langchain_experimental 6 | langchain_openai 7 | pydantic 8 | pyzotero 9 | unstructured 10 | -------------------------------------------------------------------------------- /config.ini.bak: -------------------------------------------------------------------------------- 1 | [Neo4j] 2 | uri = bolt://localhost:7687 3 | username = neo4j 4 | password = 12345678 5 | 6 | [LLM] 7 | #Ollama or OpenAI (llama.cpp server or OpenAI API) 8 | llm = Ollama 9 | temperature = 0.0 10 | #max_tokens is the maximum number of tokens that can be generated 11 | max_tokens = 2048 12 | stop = ["<|im_end|>"] 13 | 14 | [OpenAI] 15 | model = gpt-3.5-turbo 16 | api_key = sk-123456 17 | #api_base for llama.cpp server, leave it empty for OpenAI API 18 | api_base = http://127.0.0.1:8080/v1 19 | 20 | [Ollama] 21 | model = interstellarninja/hermes-2-pro-llama-3-8b 22 | #num_ctx is the number of context tokens, which is the maximum number of tokens that can be processed at a time 23 | num_ctx = 2048 24 | 25 | [Zotero] 26 | enabled = True 27 | library_id = 12345678 28 | library_type = user 29 | api_key = xxxxxx 30 | #Zotero_dir is the directory where the attachments are stored 31 | Zotero_dir = /Path/To/Zotero/storage/ 32 | 33 | [PDF] 34 | #extract_images is used to extract images from PDF files 35 | extract_images = False 36 | max_char = 1000 37 | new_after_n_chars = 800 38 | combine_text_under_n_chars = 200 39 | 40 | [Embeddings] 41 | #embedding method, Ollama or OpenAI 42 | embeddings = Ollama 43 | #text-embedding-3-small for OpenAI, mxbai-embed-large for Ollama, left empty to use the llm model 44 | model = mxbai-embed-large 45 | 46 | [Chat] 47 | #chatbot, Ollama or OpenAI, left empty to use the llm model 48 | chatbot = Ollama 49 | #chatbot model, gpt-3.5-turbo for OpenAI, phi3 for Ollama 50 | model = interstellarninja/hermes-2-pro-llama-3-8b 51 | #chatbot temperature 52 | temperature = 0.8 53 | #chatbot max_tokens 54 | max_tokens = 2048 55 | #chatbot stop 56 | stop = ["<|im_end|>","USER:"] 57 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | json_repair 2 | langchain 3 | langchain_community 4 | langchain_core 5 | langchain_experimental 6 | langchain_openai 7 | pydantic 8 | pyzotero 9 | unstructured[all-docs] 10 | pytesseract 11 | ollama 12 | chromadb 13 | pytesseract 14 | pdfminer 15 | numpy==1.26.3 16 | neo4j 17 | opencv-python-headless>=4.8.0.76 -------------------------------------------------------------------------------- /res/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjkhurry/Graph-RAG/11bc55d1908dea366cbcb69b485ac787d595cf7f/res/1.png -------------------------------------------------------------------------------- /res/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjkhurry/Graph-RAG/11bc55d1908dea366cbcb69b485ac787d595cf7f/res/2.png -------------------------------------------------------------------------------- /res/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjkhurry/Graph-RAG/11bc55d1908dea366cbcb69b485ac787d595cf7f/res/3.png -------------------------------------------------------------------------------- /res/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjkhurry/Graph-RAG/11bc55d1908dea366cbcb69b485ac787d595cf7f/res/4.png -------------------------------------------------------------------------------- /res/5.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjkhurry/Graph-RAG/11bc55d1908dea366cbcb69b485ac787d595cf7f/res/5.gif -------------------------------------------------------------------------------- /res/graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjkhurry/Graph-RAG/11bc55d1908dea366cbcb69b485ac787d595cf7f/res/graph.png --------------------------------------------------------------------------------