├── public ├── demo.png ├── favicon.ico ├── vercel.svg └── next.svg ├── styles └── globals.css ├── jsconfig.json ├── postcss.config.js ├── next.config.js ├── pages ├── api │ ├── hello.js │ ├── my-files.js │ ├── query.js │ ├── process.js │ └── upload.js ├── _document.js ├── _app.js └── index.js ├── env-example ├── components ├── Intro.js ├── ChatBox │ ├── ChooseFileAlert.js │ ├── ReadyAlert.js │ ├── Chat.js │ ├── FileNotProcessedAlert.js │ └── index.js ├── AnimatedEllipsis.js ├── MyFiles.js └── FileUpload.js ├── apiHooks └── useMyFiles.js ├── src ├── pinecone.js ├── s3services.js ├── models │ └── MyFile.js ├── openaiServices.js └── db.js ├── .gitignore ├── tailwind.config.js ├── package.json └── README.md /public/demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mejaz/paperbot/HEAD/public/demo.png -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mejaz/paperbot/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /styles/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "paths": { 4 | "@/*": ["./*"] 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = { 3 | reactStrictMode: true, 4 | } 5 | 6 | module.exports = nextConfig 7 | -------------------------------------------------------------------------------- /pages/api/hello.js: -------------------------------------------------------------------------------- 1 | // Next.js API route support: https://nextjs.org/docs/api-routes/introduction 2 | 3 | export default function handler(req, res) { 4 | res.status(200).json({ name: 'John Doe' }) 5 | } 6 | -------------------------------------------------------------------------------- /env-example: -------------------------------------------------------------------------------- 1 | # AWS 2 | S3_BUCKET= 3 | AWS_ACCESS_ID= 4 | AWS_ACCESS_KEY= 5 | 6 | # MongoDB 7 | DB_USERNAME= 8 | DB_PASSWORD= 9 | 10 | # OPENAI 11 | OPENAI_API_KEY= 12 | 13 | # PINECONE DB 14 | PDB_ENV= 15 | PDB_KEY= -------------------------------------------------------------------------------- /components/Intro.js: -------------------------------------------------------------------------------- 1 | export default function Intro() { 2 | return ( 3 |
4 | With PaperBot you can upload any pdf and chat with the pdf to get answers. 5 | Just Upload and start chatting! 6 |
7 | ) 8 | } -------------------------------------------------------------------------------- /apiHooks/useMyFiles.js: -------------------------------------------------------------------------------- 1 | import useSWR from "swr"; 2 | 3 | const url = "/api/my-files" 4 | 5 | export default function useMyFiles() { 6 | const {data, error} = useSWR(url) 7 | 8 | return { 9 | files: data, 10 | isLoading: !error && !data, 11 | isError: error 12 | } 13 | } -------------------------------------------------------------------------------- /pages/_document.js: -------------------------------------------------------------------------------- 1 | import { Html, Head, Main, NextScript } from 'next/document' 2 | 3 | export default function Document() { 4 | return ( 5 | 6 | 7 | 8 |
9 | 10 | 11 | 12 | ) 13 | } 14 | -------------------------------------------------------------------------------- /components/ChatBox/ChooseFileAlert.js: -------------------------------------------------------------------------------- 1 | export default function ChooseFileAlert() { 2 | return ( 3 |
4 |

Select file

5 |

Select a file to start chatting.

6 |
7 | ) 8 | } 9 | -------------------------------------------------------------------------------- /components/ChatBox/ReadyAlert.js: -------------------------------------------------------------------------------- 1 | export default function ReadyAlert() { 2 | return ( 3 |
4 |

Start Chatting

5 |

Your file is ready, start the conversation!

6 |
7 | ) 8 | } -------------------------------------------------------------------------------- /src/pinecone.js: -------------------------------------------------------------------------------- 1 | import { PineconeClient } from "@pinecone-database/pinecone"; 2 | 3 | const pinecone = new PineconeClient(); 4 | 5 | export const initialize = async () => { 6 | await pinecone.init({ 7 | environment: process.env.PDB_ENV, 8 | apiKey: process.env.PDB_KEY, 9 | }); 10 | console.log('pinecone initialized') 11 | } 12 | 13 | export default pinecone -------------------------------------------------------------------------------- /components/AnimatedEllipsis.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | const AnimatedEllipsis = () => { 4 | return ( 5 |
6 |
.
7 |
.
8 |
.
9 |
10 | ); 11 | }; 12 | 13 | export default AnimatedEllipsis; -------------------------------------------------------------------------------- /pages/_app.js: -------------------------------------------------------------------------------- 1 | import '@/styles/globals.css' 2 | import {SWRConfig} from 'swr' 3 | import { Toaster } from 'react-hot-toast'; 4 | 5 | 6 | export default function App({Component, pageProps}) { 7 | return ( 8 | fetch(resource, init).then(res => res.json()) 11 | }} 12 | > 13 | 14 | 15 | 16 | ) 17 | } 18 | -------------------------------------------------------------------------------- /src/s3services.js: -------------------------------------------------------------------------------- 1 | import AWS from "aws-sdk"; 2 | import fs from "fs"; 3 | 4 | const s3 = new AWS.S3({ 5 | accessKeyId: process.env.AWS_ACCESS_ID, 6 | secretAccessKey: process.env.AWS_ACCESS_KEY, 7 | region: 'ap-south-1', 8 | }); 9 | 10 | export const s3Upload = async (bucket, file) => { 11 | const params = { 12 | Bucket: bucket, 13 | Key: file.name, 14 | Body: fs.createReadStream(file.path), 15 | }; 16 | 17 | return await s3.upload(params).promise() 18 | } -------------------------------------------------------------------------------- /pages/api/my-files.js: -------------------------------------------------------------------------------- 1 | // Next.js API route support: https://nextjs.org/docs/api-routes/introduction 2 | 3 | import {connectDB, disconnectDB} from "@/src/db"; 4 | import MyFileModel from "@/src/models/MyFile"; 5 | 6 | export default async function handler(req, res) { 7 | try { 8 | await connectDB() 9 | const files = await MyFileModel.find({}) 10 | // await disconnectDB() 11 | return res.status(200).json(files) 12 | } catch (e) { 13 | return res.status(500).json({message: 'error fetching files'}) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | .idea 4 | # dependencies 5 | /node_modules 6 | /.pnp 7 | .pnp.js 8 | 9 | # testing 10 | /coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env.local 30 | .env*.local 31 | 32 | # vercel 33 | .vercel 34 | 35 | # typescript 36 | *.tsbuildinfo 37 | next-env.d.ts 38 | -------------------------------------------------------------------------------- /public/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /components/ChatBox/Chat.js: -------------------------------------------------------------------------------- 1 | import AnimatedEllipsis from "@/components/AnimatedEllipsis"; 2 | 3 | export default function Chat({query, response}) { 4 | return ( 5 |
6 | {query &&
7 |
8 |
{query}
9 |
} 10 | {response &&
11 |
{response ? response : }
12 |
13 |
} 14 |
15 | ) 16 | } 17 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: [ 4 | './pages/**/*.{js,ts,jsx,tsx,mdx}', 5 | './components/**/*.{js,ts,jsx,tsx,mdx}', 6 | './app/**/*.{js,ts,jsx,tsx,mdx}', 7 | ], 8 | theme: { 9 | extend: { 10 | colors: { 11 | primary: { 12 | main: "#64748b", 13 | light: "#94a3b8", 14 | dark: "#475569", 15 | contrastText: "#f8fafc" 16 | } 17 | }, 18 | backgroundImage: { 19 | 'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))', 20 | 'gradient-conic': 21 | 'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))', 22 | }, 23 | }, 24 | }, 25 | plugins: [], 26 | } 27 | -------------------------------------------------------------------------------- /components/MyFiles.js: -------------------------------------------------------------------------------- 1 | import {useState} from "react"; 2 | 3 | 4 | export default function MyFiles({setActiveFile, files}) { 5 | return ( 6 |
7 |
My Files
8 | { 9 | files.map((file, index) => ( 10 | 17 | )) 18 | } 19 |
20 | ) 21 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "paperbot", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@pinecone-database/pinecone": "^0.1.5", 13 | "autoprefixer": "10.4.14", 14 | "aws-sdk": "^2.1379.0", 15 | "formidable": "^2.1.1", 16 | "formidable-serverless": "^1.1.1", 17 | "mongoose": "^7.1.1", 18 | "next": "13.4.2", 19 | "openai": "^3.2.1", 20 | "pdfjs-dist": "^2.16.105", 21 | "postcss": "8.4.23", 22 | "react": "18.2.0", 23 | "react-dom": "18.2.0", 24 | "react-hot-toast": "^2.4.1", 25 | "react-icons": "^4.8.0", 26 | "slugify": "^1.6.6", 27 | "swr": "^2.1.5", 28 | "tailwindcss": "3.3.2" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/models/MyFile.js: -------------------------------------------------------------------------------- 1 | import mongoose from 'mongoose' 2 | 3 | const Schema = mongoose.Schema; 4 | 5 | const MyFileSchema = new Schema({ 6 | fileName: { 7 | type: String, 8 | required: [true, 'Filename is a required field.'], 9 | trim: true, 10 | maxLength: 100, 11 | unique: true, 12 | }, 13 | fileUrl: { 14 | type: String, 15 | required: [true, 'File Url is a required field.'], 16 | trim: true, 17 | maxLength: 100, 18 | unique: true, 19 | }, 20 | isProcessed: { 21 | type: Boolean, 22 | default: false, 23 | }, 24 | vectorIndex: { 25 | type: String, 26 | maxLength: 100, 27 | unique: true, 28 | required: false, 29 | }, 30 | }, { 31 | timestamps: true, 32 | }); 33 | 34 | const MyFileModel = mongoose.models.myFile || mongoose.model('myFile', MyFileSchema); 35 | 36 | 37 | export default MyFileModel; -------------------------------------------------------------------------------- /src/openaiServices.js: -------------------------------------------------------------------------------- 1 | import {Configuration, OpenAIApi} from "openai" 2 | 3 | const configuration = new Configuration({ 4 | apiKey: process.env.OPENAI_API_KEY, 5 | }); 6 | 7 | const openai = new OpenAIApi(configuration); 8 | 9 | const OPEN_AI_EMBEDDING_MODEL = "text-embedding-ada-002" 10 | const OPEN_AI_COMPLETION_MODEL = "text-davinci-003" 11 | 12 | export const getEmbeddings = async (text) => { 13 | const response = await openai.createEmbedding({ 14 | model: OPEN_AI_EMBEDDING_MODEL, 15 | input: text, 16 | }); 17 | return response.data.data[0].embedding 18 | } 19 | 20 | export const getCompletion = async (prompt) => { 21 | const completion = await openai.createCompletion({ 22 | model: OPEN_AI_COMPLETION_MODEL, 23 | prompt: prompt, 24 | max_tokens: 500, 25 | temperature: 0 26 | }); 27 | 28 | console.log(completion.data.choices) 29 | 30 | return completion.data.choices[0].text 31 | } -------------------------------------------------------------------------------- /src/db.js: -------------------------------------------------------------------------------- 1 | import mongoose from 'mongoose'; 2 | 3 | async function connectDB() { 4 | if (mongoose.connections[0].readyState) { 5 | // If a connection is already established, reuse it 6 | console.log('existing connection available') 7 | return; 8 | } 9 | 10 | const MONGO_URI = `mongodb+srv://${process.env.DB_USERNAME}:${process.env.DB_PASSWORD}@cluster0.rmf4y1k.mongodb.net/paperbot?retryWrites=true&w=majority` 11 | 12 | try { 13 | await mongoose.connect(MONGO_URI, { 14 | // Replace 'mydatabase' with your actual database name 15 | useNewUrlParser: true, 16 | useUnifiedTopology: true, 17 | }); 18 | console.log('Connected to MongoDB'); 19 | } catch (error) { 20 | console.error('Error connecting to MongoDB:', error.message); 21 | process.exit(1); // Exit the Node.js process on connection error 22 | } 23 | } 24 | 25 | async function disconnectDB() { 26 | if (mongoose.connections[0].readyState) { 27 | await mongoose.disconnect(); 28 | console.log('Disconnected from MongoDB'); 29 | } 30 | } 31 | 32 | export { connectDB, disconnectDB }; 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PaperBot 2 | 3 | 4 | 5 |

6 | demo 7 |

8 | 9 | Upload any pdf file and ask questions from that pdf. 10 | 11 | PaperBot generates embeddings from OpenAI embedding model and stores them in Pinecone Vector database. 12 | To answer user's questions, PaperBot does a semantic search on Pinecone Vector database and further refines the result with OpenAI completion model. 13 | 14 | ## Pre-requisites to run this project 15 | 16 | 1. OpenAI API Key 17 | 2. Pinecone Database ENV and KEY 18 | 3. MongoDB cluster username and password 19 | 4. AWS S3 bucket, access key and access id 20 | 21 | ## Steps to run this project 22 | 23 | 1. Clone the repo 24 | 2. Run `npm install` to install all the dependencies 25 | 3. Create a `.env.local` from `env-example` file: `cp env-example .env.local` 26 | 4. Update the OpenAI, Pinecone keys, MongoDB creds and AWS S3 bucket and creds in the `.env.local` file 27 | 5. Run `npm run dev` to start the project 28 | 29 | 30 | ### Tech stack 31 | 32 | - NextJS 33 | - TailwindCSS 34 | - MongoDB 35 | - Pinecone (Vector DB) 36 | - OpenAI Models - Embedding and Completion 37 | 38 | ## Contact 39 | 40 | mohdejazsiddiqui@gmail.com 41 | 42 | [![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/Y8Y618ZETK) -------------------------------------------------------------------------------- /public/next.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /components/ChatBox/FileNotProcessedAlert.js: -------------------------------------------------------------------------------- 1 | import toast from "react-hot-toast"; 2 | import {useState} from "react"; 3 | 4 | const PillButton = ({func, disabled}) => ( 5 | 9 | ) 10 | 11 | export default function FileNotProcessedAlert({id}) { 12 | console.log("--id--", id) 13 | const [processing, setProcessing] = useState(false) 14 | 15 | const trigger = async (id) => { 16 | setProcessing(true) 17 | let response = await fetch("/api/process", { 18 | method: 'POST', 19 | body: JSON.stringify({id}), 20 | headers: { 21 | 'Content-type': 'application/json' 22 | } 23 | }) 24 | 25 | if (response.ok) { 26 | response = await response.json() 27 | toast.success(response.message) 28 | } else { 29 | response = await response.json() 30 | toast.error(response.message) 31 | } 32 | setProcessing(false) 33 | } 34 | 35 | return ( 36 |
37 |

Process File

38 |

Please process the file before starting to chat. Click button to process 39 | trigger(id)} disabled={processing}/> 40 |

41 |
42 | ) 43 | } -------------------------------------------------------------------------------- /pages/index.js: -------------------------------------------------------------------------------- 1 | import FileUpload from "@/components/FileUpload"; 2 | import MyFiles from "@/components/MyFiles"; 3 | import Intro from "@/components/Intro"; 4 | import ChatBox from "@/components/ChatBox"; 5 | import {useState} from "react"; 6 | import useMyFiles from "@/apiHooks/useMyFiles"; 7 | import Head from 'next/head' 8 | 9 | export default function Home() { 10 | const [activeFile, setActiveFile] = useState() 11 | const {files, isError, isLoading} = useMyFiles() 12 | 13 | if (isLoading) { 14 | return

Loading...

15 | } 16 | 17 | return ( 18 | <> 19 | 20 | Paperbot - Chat with my PDF 21 | 22 |
25 |
26 |

29 | PaperBot 30 |

31 |
32 |
33 |
34 | 35 | 36 | 37 |
38 |
39 | 40 |
41 |
42 |
43 |
44 | 45 |
46 | 47 | ) 48 | } 49 | -------------------------------------------------------------------------------- /pages/api/query.js: -------------------------------------------------------------------------------- 1 | import pinecone, {initialize} from "@/src/pinecone"; 2 | import {getCompletion, getEmbeddings} from "@/src/openaiServices"; 3 | import {connectDB} from "@/src/db"; 4 | import MyFileModel from "@/src/models/MyFile"; 5 | 6 | export default async function handler(req, res) { 7 | 8 | // 1. check for POST call 9 | 10 | const {query, id} = req.body 11 | 12 | // 2. connect to mongodb 13 | await connectDB() 14 | 15 | // 3. query the file by id 16 | const myFile = await MyFileModel.findById(id) 17 | 18 | if(!myFile) { 19 | return res.status(400).send({message: 'invalid file id'}) 20 | } 21 | 22 | // 4. get embeddings for the query 23 | const questionEmb = await getEmbeddings(query) 24 | 25 | // 5. initialize pinecone 26 | await initialize() 27 | 28 | // 6. connect to index 29 | const index = pinecone.Index(myFile.vectorIndex) 30 | 31 | // 7. query the pinecone db 32 | const queryRequest = { 33 | vector: questionEmb, 34 | topK: 5, 35 | includeValues: true, 36 | includeMetadata: true, 37 | }; 38 | 39 | let result = await index.query({queryRequest}) 40 | 41 | // 8. get the metadata from the results 42 | let contexts = result['matches'].map(item => item['metadata'].text) 43 | 44 | contexts = contexts.join("\n\n---\n\n") 45 | 46 | console.log('--contexts--', contexts) 47 | 48 | // 9. build the prompt 49 | const promptStart = `Answer the question based on the context below:\n\n` 50 | const promptEnd = `\n\nQuestion: ${query} \n\nAnswer:` 51 | 52 | const prompt = `${promptStart} ${contexts} ${promptEnd}` 53 | 54 | console.log('--prompt--', prompt) 55 | 56 | // 10. get the completion from openai 57 | let response = await getCompletion(prompt) 58 | 59 | console.log('--completion--', response) 60 | 61 | // 11. return the response 62 | res.status(200).json({response}) 63 | } 64 | -------------------------------------------------------------------------------- /pages/api/process.js: -------------------------------------------------------------------------------- 1 | import * as PDFJS from 'pdfjs-dist/legacy/build/pdf'; 2 | import MyFileModel from "@/src/models/MyFile"; 3 | import {connectDB, disconnectDB} from "@/src/db"; 4 | import {getEmbeddings} from "@/src/openaiServices"; 5 | import pinecone, {initialize} from "@/src/pinecone"; 6 | 7 | export default async function handler(req, res) { 8 | // 1. check for POST call 9 | if (req.method !== 'POST') { 10 | return res.status(400).json({message: 'http method not allowed'}) 11 | } 12 | 13 | try { 14 | // 2. connect to mongodb 15 | await connectDB() 16 | 17 | // 3. query the file by id 18 | const {id} = req.body 19 | 20 | const myFile = await MyFileModel.findById(id) 21 | 22 | if (!myFile) { 23 | return res.status(400).json({message: 'file not found'}) 24 | } 25 | 26 | if(myFile.isProcessed) { 27 | return res.status(400).json({message: 'file is already processed'}) 28 | } 29 | 30 | // 4. Read PDF and iterate through pages 31 | let vectors = [] 32 | 33 | let myFiledata = await fetch(myFile.fileUrl) 34 | 35 | if (myFiledata.ok) { 36 | let pdfDoc = await PDFJS.getDocument(await myFiledata.arrayBuffer()).promise 37 | const numPages = pdfDoc.numPages 38 | for (let i = 0; i < numPages; i++) { 39 | let page = await pdfDoc.getPage(i + 1) 40 | let textContent = await page.getTextContent() 41 | const text = textContent.items.map(item => item.str).join(''); 42 | 43 | // 5. Get embeddings for each page 44 | const embedding = await getEmbeddings(text) 45 | 46 | // 6. push to vector array 47 | vectors.push({ 48 | id: `page${i + 1}`, 49 | values: embedding, 50 | metadata: { 51 | pageNum: i + 1, 52 | text, 53 | }, 54 | }) 55 | } 56 | 57 | // 7. initialize pinecone 58 | await initialize() // initialize pinecone 59 | 60 | // 8. connect to the index 61 | const index = pinecone.Index(myFile.vectorIndex) 62 | 63 | // 9. upsert to pinecone index 64 | await index.upsert({ 65 | upsertRequest: { 66 | vectors, 67 | } 68 | }); 69 | 70 | // 10. update mongodb with isProcessed to true 71 | myFile.isProcessed = true 72 | await myFile.save() 73 | // await disconnectDB() 74 | 75 | // 11. return the response 76 | return res.status(200).json({message: 'File processed successfully'}) 77 | } else { 78 | // await disconnectDB() 79 | return res.status(500).json({message: 'error getting file contents'}) 80 | } 81 | } catch (e) { 82 | console.log(e) 83 | // await disconnectDB() 84 | return res.status(500).json({message: e.message}) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /pages/api/upload.js: -------------------------------------------------------------------------------- 1 | import formidable from 'formidable-serverless'; 2 | import {connectDB, disconnectDB} from '@/src/db' 3 | import MyFileModel from "@/src/models/MyFile"; 4 | import slugify from 'slugify' 5 | import pinecone, {initialize} from "../../src/pinecone"; 6 | import {s3Upload} from "@/src/s3services"; 7 | 8 | export const config = { 9 | api: { 10 | bodyParser: false, 11 | }, 12 | }; 13 | 14 | const createIndex = async (indexName) => { 15 | const indexes = await pinecone.listIndexes() 16 | if (!indexes.includes(indexName)) { 17 | await pinecone.createIndex({ 18 | createRequest: { 19 | name: indexName, 20 | dimension: 1536, // fixed for OPENAI embeddings 21 | }, 22 | }); 23 | console.log('index created') 24 | } else { 25 | throw new Error(`Index with name ${indexName} already exists`) 26 | } 27 | } 28 | 29 | export default async function handler(req, res) { 30 | // 1. only allow POST methods 31 | if (req.method !== 'POST') { 32 | return res.status(400).send('method not supported') 33 | } 34 | 35 | try { 36 | // 2. connect to the mongodb db 37 | await connectDB() 38 | 39 | // 3. parse the incoming FormData 40 | let form = new formidable.IncomingForm(); 41 | 42 | form.parse(req, async (error, fields, files) => { 43 | if (error) { 44 | console.error('Failed to parse form data:', error); 45 | return res.status(500).json({error: 'Failed to parse form data'}); 46 | } 47 | 48 | const file = files.file; 49 | 50 | // Check if the file object exists 51 | if (!file) { 52 | return res.status(400).json({error: 'No file uploaded'}); 53 | } 54 | 55 | // Check if the necessary file properties are available 56 | if (!file.name || !file.path || !file.type) { 57 | return res.status(400).json({error: 'Invalid file data'}); 58 | } 59 | 60 | // 4. upload the file to s3 61 | let data = await s3Upload(process.env.S3_BUCKET, file) 62 | 63 | // 5. initialize pinecone 64 | const filenameWithoutExt = file.name.split(".")[0] 65 | const filenameSlug = slugify(filenameWithoutExt, { 66 | lower: true, strict: true 67 | }) 68 | 69 | await initialize() // initialize pinecone 70 | 71 | // 6. create a pinecone index 72 | await createIndex(filenameSlug) // create index 73 | 74 | // 7. save file info to the mongodb db 75 | const myFile = new MyFileModel({ 76 | fileName: file.name, 77 | fileUrl: data.Location, 78 | vectorIndex: filenameSlug, 79 | }) 80 | await myFile.save() 81 | // await disconnectDB() 82 | 83 | // 8. return the success response 84 | return res.status(200).json({message: 'File uploaded to S3 and index created'}); 85 | }) 86 | } catch (e) { 87 | console.log("--error--", e) 88 | // await disconnectDB() 89 | return res.status(500).send({message: e.message}) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /components/FileUpload.js: -------------------------------------------------------------------------------- 1 | import {useState} from 'react'; 2 | import toast from "react-hot-toast"; 3 | 4 | export default function FileUpload() { 5 | const [file, setFile] = useState(); 6 | const [uploading, setUploading] = useState(false) 7 | 8 | const handleFileChange = (e) => { 9 | if (e.target.files) { 10 | let file = e.target.files[0] 11 | 12 | if (file.type !== 'application/pdf') { 13 | toast.error('Only PDF files are allowed') 14 | e.target.value = null 15 | return 16 | } 17 | setFile(file); 18 | } 19 | }; 20 | 21 | const handleUploadClick = async () => { 22 | if (!file) { 23 | return; 24 | } 25 | 26 | setUploading(true) 27 | const formData = new FormData(); 28 | formData.append('file', file); 29 | 30 | // 👇 Uploading the file using the fetch API to the server 31 | let response = await fetch('/api/upload', { 32 | method: 'POST', 33 | body: formData 34 | }) 35 | if (response.ok) { 36 | response = await response.json() 37 | toast.success(response.message) 38 | } else { 39 | response = await response.json() 40 | toast.error(response.message) 41 | } 42 | setUploading(false) 43 | }; 44 | 45 | return ( 46 |
47 |
48 | 53 | 59 |
60 | 61 |
62 | 69 |
70 |
71 | ) 72 | } -------------------------------------------------------------------------------- /components/ChatBox/index.js: -------------------------------------------------------------------------------- 1 | import {MdSend} from "react-icons/md" 2 | import {useEffect, useRef, useState} from "react"; 3 | import toast from "react-hot-toast"; 4 | import ChooseFileAlert from "@/components/ChatBox/ChooseFileAlert"; 5 | import ReadyAlert from "@/components/ChatBox/ReadyAlert"; 6 | import Chat from "@/components/ChatBox/Chat"; 7 | import FileNotProcessedAlert from "@/components/ChatBox/FileNotProcessedAlert"; 8 | 9 | export default function ChatBox({activeFile}) { 10 | const divRef = useRef(null); 11 | const [chat, setChat] = useState([]) 12 | const [query, setQuery] = useState() 13 | const [id, setId] = useState() 14 | 15 | const scrollToBottom = () => { 16 | if (divRef.current) { 17 | divRef.current.scrollTop = divRef.current.scrollHeight; 18 | } 19 | } 20 | 21 | useEffect(() => { 22 | scrollToBottom() 23 | console.log('msg added') 24 | }, [chat.length]); 25 | 26 | const addChat = (query, response, id) => { 27 | setChat(prevState => [...prevState, { 28 | query, 29 | response, 30 | }]) 31 | 32 | setQuery(query) 33 | setId(id) 34 | } 35 | 36 | const handleSubmit = async (e) => { 37 | e.preventDefault(); 38 | const query = e.target.query.value; 39 | const id = e.target.id.value; 40 | 41 | e.target.query.value = null; 42 | 43 | console.log(query, id); 44 | addChat(query, null, id); 45 | } 46 | 47 | const updateLastChat = (query, response) => { 48 | console.log("--old chat--", chat) 49 | const oldChats = [...chat] 50 | oldChats.pop() 51 | setChat([...oldChats, { 52 | query, 53 | response, 54 | }]) 55 | } 56 | 57 | useEffect(() => { 58 | const fetchChatResponse = async () => { 59 | let response = await fetch("/api/query", { 60 | method: 'POST', 61 | body: JSON.stringify({query, id}), 62 | headers: { 63 | 'Content-type': 'application/json', 64 | }, 65 | }); 66 | 67 | if (response.ok) { 68 | response = await response.json(); 69 | updateLastChat(query, response.response); 70 | } else { 71 | response = await response.json(); 72 | toast.error(response.message) 73 | } 74 | }; 75 | 76 | if (query) { 77 | fetchChatResponse().then(() => console.log('response received')); 78 | } 79 | }, [query]); 80 | 81 | 82 | return ( 83 |
84 |
85 | {activeFile ? activeFile.fileName : "Choose a file to start chatting"} 86 |
87 |
88 | {activeFile 89 | ? !activeFile.isProcessed 90 | ? 91 | : chat.length > 0 92 | ?
93 | { 94 | chat.map(({query, response}, index) => ( 95 | 96 | )) 97 | } 98 |
99 |
100 | : 101 | : 102 | } 103 |
104 | 105 |
106 |
107 | 108 | 109 | 114 |
115 |
116 |
117 | ) 118 | } --------------------------------------------------------------------------------