├── requirements.txt ├── LICENSE ├── app.py ├── .gitignore └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | python-dotenv 2 | langchain 3 | openai 4 | pinecone-client 5 | unstructured 6 | tabulate 7 | pdf2image 8 | streamlit 9 | tiktoken -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 AI Anytime 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | from langchain.document_loaders import DirectoryLoader 2 | from langchain.text_splitter import CharacterTextSplitter 3 | import os 4 | import pinecone 5 | from langchain.vectorstores import Pinecone 6 | from langchain.embeddings.openai import OpenAIEmbeddings 7 | from langchain.chains import RetrievalQA 8 | from langchain.chat_models import ChatOpenAI 9 | import streamlit as st 10 | from dotenv import load_dotenv 11 | 12 | load_dotenv() 13 | 14 | 15 | PINECONE_API_KEY = os.getenv('PINECONE_API_KEY') 16 | PINECONE_ENV = os.getenv('PINECONE_ENV') 17 | OPENAI_API_KEY = os.getenv('OPENAI_API_KEY') 18 | 19 | os.environ['OPENAI_API_KEY'] = OPENAI_API_KEY 20 | 21 | 22 | def doc_preprocessing(): 23 | loader = DirectoryLoader( 24 | 'data/', 25 | glob='**/*.pdf', # only the PDFs 26 | show_progress=True 27 | ) 28 | docs = loader.load() 29 | text_splitter = CharacterTextSplitter( 30 | chunk_size=1000, 31 | chunk_overlap=0 32 | ) 33 | docs_split = text_splitter.split_documents(docs) 34 | return docs_split 35 | 36 | @st.cache_resource 37 | def embedding_db(): 38 | # we use the openAI embedding model 39 | embeddings = OpenAIEmbeddings() 40 | pinecone.init( 41 | api_key=PINECONE_API_KEY, 42 | environment=PINECONE_ENV 43 | ) 44 | docs_split = doc_preprocessing() 45 | doc_db = Pinecone.from_documents( 46 | docs_split, 47 | embeddings, 48 | index_name='langchain-demo-indexes' 49 | ) 50 | return doc_db 51 | 52 | llm = ChatOpenAI() 53 | doc_db = embedding_db() 54 | 55 | def retrieval_answer(query): 56 | qa = RetrievalQA.from_chain_type( 57 | llm=llm, 58 | chain_type='stuff', 59 | retriever=doc_db.as_retriever(), 60 | ) 61 | query = query 62 | result = qa.run(query) 63 | return result 64 | 65 | def main(): 66 | st.title("Question and Answering App powered by LLM and Pinecone") 67 | 68 | text_input = st.text_input("Ask your query...") 69 | if st.button("Ask Query"): 70 | if len(text_input)>0: 71 | st.info("Your Query: " + text_input) 72 | answer = retrieval_answer(text_input) 73 | st.success(answer) 74 | 75 | if __name__ == "__main__": 76 | main() 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # QA in PDF using ChatGPT and Pinecone 2 | 3 | A powerful Question-Answering application that enables users to query PDF documents using advanced AI technologies. This project combines OpenAI's ChatGPT for natural language processing with Pinecone's vector database for efficient document retrieval, creating an intelligent document search and Q&A system. 4 | 5 | ## 🚀 Features 6 | 7 | - **PDF Document Processing**: Automatically loads and processes PDF files from a specified directory 8 | - **Intelligent Text Chunking**: Splits documents into manageable chunks for optimal retrieval 9 | - **Vector Embeddings**: Uses OpenAI embeddings to create semantic representations of document content 10 | - **Semantic Search**: Leverages Pinecone vector database for fast and accurate document retrieval 11 | - **Natural Language Q&A**: Powered by ChatGPT for human-like responses 12 | - **Interactive Web Interface**: Built with Streamlit for easy user interaction 13 | - **Caching Optimization**: Implements Streamlit caching for improved performance 14 | 15 | ## 🛠️ Technology Stack 16 | 17 | - **LangChain**: Framework for building LLM applications 18 | - **OpenAI GPT**: Large language model for generating responses 19 | - **Pinecone**: Vector database for similarity search 20 | - **Streamlit**: Web application framework 21 | - **Python**: Core programming language 22 | 23 | ## 📋 Prerequisites 24 | 25 | - Python 3.7+ 26 | - OpenAI API key 27 | - Pinecone API key and environment 28 | 29 | ## 🔧 Installation 30 | 31 | 1. **Clone the repository** 32 | ```bash 33 | git clone https://github.com/danieladdisonorg/QA-in-PDF-using-ChatGPT-and-Pinecone.git 34 | cd QA-in-PDF-using-ChatGPT-and-Pinecone 35 | ``` 36 | 37 | 2. **Install required dependencies** 38 | ```bash 39 | pip install -r requirements.txt 40 | ``` 41 | 42 | 3. **Set up environment variables** 43 | 44 | Create a `.env` file in the root directory and add your API keys: 45 | ``` 46 | OPENAI_API_KEY=your_openai_api_key_here 47 | PINECONE_API_KEY=your_pinecone_api_key_here 48 | PINECONE_ENV=your_pinecone_environment_here 49 | ``` 50 | 51 | 4. **Create data directory** 52 | ```bash 53 | mkdir data 54 | ``` 55 | 56 | 5. **Add your PDF files** 57 | 58 | Place your PDF documents in the `data/` directory. 59 | 60 | ## 🚀 Usage 61 | 62 | 1. **Start the application** 63 | ```bash 64 | streamlit run app.py 65 | ``` 66 | 67 | 2. **Access the web interface** 68 | 69 | Open your browser and navigate to `http://localhost:8501` 70 | 71 | 3. **Ask questions** 72 | 73 | Enter your questions in the text input field and click "Ask Query" to get AI-powered answers based on your PDF documents. 74 | 75 | ## 📁 Project Structure 76 | 77 | ``` 78 | QA-in-PDF-using-ChatGPT-and-Pinecone/ 79 | ├── app.py # Main application file 80 | ├── data/ # Directory for PDF files 81 | ├── .env # Environment variables (create this) 82 | ├── requirements.txt # Python dependencies (create this) 83 | └── README.md # Project documentation 84 | ``` 85 | 86 | ## ⚙️ Configuration 87 | 88 | ### Pinecone Setup 89 | - Create a Pinecone account and get your API key 90 | - Set up an index named `langchain-demo-indexes` 91 | - Note your environment (e.g., `us-west1-gcp`) 92 | 93 | ### OpenAI Setup 94 | - Create an OpenAI account and generate an API key 95 | - Ensure you have sufficient credits for API usage 96 | 97 | ## 🔍 How It Works 98 | 99 | 1. **Document Loading**: The application scans the `data/` directory for PDF files 100 | 2. **Text Processing**: Documents are split into chunks of 1000 characters 101 | 3. **Embedding Generation**: OpenAI creates vector embeddings for each text chunk 102 | 4. **Vector Storage**: Embeddings are stored in Pinecone for fast retrieval 103 | 5. **Query Processing**: User questions are converted to embeddings and matched against stored vectors 104 | 6. **Answer Generation**: Relevant document chunks are retrieved and passed to ChatGPT for answer generation 105 | 106 | ## 🤝 Contributing 107 | 108 | Contributions are welcome! Please feel free to submit a Pull Request. For major changes, please open an issue first to discuss what you would like to change. 109 | 110 | ## 📄 License 111 | 112 | This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. 113 | 114 | ## 🙏 Acknowledgments 115 | 116 | - [LangChain](https://langchain.com/) for the excellent framework 117 | - [OpenAI](https://openai.com/) for the powerful language models 118 | - [Pinecone](https://pinecone.io/) for the vector database 119 | - [Streamlit](https://streamlit.io/) for the web framework 120 | 121 | ## 📞 Support 122 | 123 | If you encounter any issues or have questions, please open an issue on GitHub or contact the maintainer. 124 | 125 | --- 126 | 127 | **Note**: Make sure to keep your API keys secure and never commit them to version control. 128 | --------------------------------------------------------------------------------