├── public
├── demo.png
├── favicon.ico
├── vercel.svg
└── next.svg
├── styles
└── globals.css
├── jsconfig.json
├── postcss.config.js
├── next.config.js
├── pages
├── api
│ ├── hello.js
│ ├── my-files.js
│ ├── query.js
│ ├── process.js
│ └── upload.js
├── _document.js
├── _app.js
└── index.js
├── env-example
├── components
├── Intro.js
├── ChatBox
│ ├── ChooseFileAlert.js
│ ├── ReadyAlert.js
│ ├── Chat.js
│ ├── FileNotProcessedAlert.js
│ └── index.js
├── AnimatedEllipsis.js
├── MyFiles.js
└── FileUpload.js
├── apiHooks
└── useMyFiles.js
├── src
├── pinecone.js
├── s3services.js
├── models
│ └── MyFile.js
├── openaiServices.js
└── db.js
├── .gitignore
├── tailwind.config.js
├── package.json
└── README.md
/public/demo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mejaz/paperbot/HEAD/public/demo.png
--------------------------------------------------------------------------------
/public/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mejaz/paperbot/HEAD/public/favicon.ico
--------------------------------------------------------------------------------
/styles/globals.css:
--------------------------------------------------------------------------------
1 | @tailwind base;
2 | @tailwind components;
3 | @tailwind utilities;
4 |
--------------------------------------------------------------------------------
/jsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "paths": {
4 | "@/*": ["./*"]
5 | }
6 | }
7 | }
8 |
--------------------------------------------------------------------------------
/postcss.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | plugins: {
3 | tailwindcss: {},
4 | autoprefixer: {},
5 | },
6 | }
7 |
--------------------------------------------------------------------------------
/next.config.js:
--------------------------------------------------------------------------------
1 | /** @type {import('next').NextConfig} */
2 | const nextConfig = {
3 | reactStrictMode: true,
4 | }
5 |
6 | module.exports = nextConfig
7 |
--------------------------------------------------------------------------------
/pages/api/hello.js:
--------------------------------------------------------------------------------
1 | // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
2 |
3 | export default function handler(req, res) {
4 | res.status(200).json({ name: 'John Doe' })
5 | }
6 |
--------------------------------------------------------------------------------
/env-example:
--------------------------------------------------------------------------------
1 | # AWS
2 | S3_BUCKET=
3 | AWS_ACCESS_ID=
4 | AWS_ACCESS_KEY=
5 |
6 | # MongoDB
7 | DB_USERNAME=
8 | DB_PASSWORD=
9 |
10 | # OPENAI
11 | OPENAI_API_KEY=
12 |
13 | # PINECONE DB
14 | PDB_ENV=
15 | PDB_KEY=
--------------------------------------------------------------------------------
/components/Intro.js:
--------------------------------------------------------------------------------
1 | export default function Intro() {
2 | return (
3 |
4 | With PaperBot you can upload any pdf and chat with the pdf to get answers.
5 | Just Upload and start chatting!
6 |
7 | )
8 | }
--------------------------------------------------------------------------------
/apiHooks/useMyFiles.js:
--------------------------------------------------------------------------------
1 | import useSWR from "swr";
2 |
3 | const url = "/api/my-files"
4 |
5 | export default function useMyFiles() {
6 | const {data, error} = useSWR(url)
7 |
8 | return {
9 | files: data,
10 | isLoading: !error && !data,
11 | isError: error
12 | }
13 | }
--------------------------------------------------------------------------------
/pages/_document.js:
--------------------------------------------------------------------------------
1 | import { Html, Head, Main, NextScript } from 'next/document'
2 |
3 | export default function Document() {
4 | return (
5 |
6 |
8 |
9 |
10 |
11 |
12 | )
13 | }
14 |
--------------------------------------------------------------------------------
/components/ChatBox/ChooseFileAlert.js:
--------------------------------------------------------------------------------
1 | export default function ChooseFileAlert() {
2 | return (
3 |
4 |
Select file
5 |
Select a file to start chatting.
6 |
7 | )
8 | }
9 |
--------------------------------------------------------------------------------
/components/ChatBox/ReadyAlert.js:
--------------------------------------------------------------------------------
1 | export default function ReadyAlert() {
2 | return (
3 |
4 |
Start Chatting
5 |
Your file is ready, start the conversation!
6 |
7 | )
8 | }
--------------------------------------------------------------------------------
/src/pinecone.js:
--------------------------------------------------------------------------------
1 | import { PineconeClient } from "@pinecone-database/pinecone";
2 |
3 | const pinecone = new PineconeClient();
4 |
5 | export const initialize = async () => {
6 | await pinecone.init({
7 | environment: process.env.PDB_ENV,
8 | apiKey: process.env.PDB_KEY,
9 | });
10 | console.log('pinecone initialized')
11 | }
12 |
13 | export default pinecone
--------------------------------------------------------------------------------
/components/AnimatedEllipsis.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | const AnimatedEllipsis = () => {
4 | return (
5 |
10 | );
11 | };
12 |
13 | export default AnimatedEllipsis;
--------------------------------------------------------------------------------
/pages/_app.js:
--------------------------------------------------------------------------------
1 | import '@/styles/globals.css'
2 | import {SWRConfig} from 'swr'
3 | import { Toaster } from 'react-hot-toast';
4 |
5 |
6 | export default function App({Component, pageProps}) {
7 | return (
8 | fetch(resource, init).then(res => res.json())
11 | }}
12 | >
13 |
14 |
15 |
16 | )
17 | }
18 |
--------------------------------------------------------------------------------
/src/s3services.js:
--------------------------------------------------------------------------------
1 | import AWS from "aws-sdk";
2 | import fs from "fs";
3 |
4 | const s3 = new AWS.S3({
5 | accessKeyId: process.env.AWS_ACCESS_ID,
6 | secretAccessKey: process.env.AWS_ACCESS_KEY,
7 | region: 'ap-south-1',
8 | });
9 |
10 | export const s3Upload = async (bucket, file) => {
11 | const params = {
12 | Bucket: bucket,
13 | Key: file.name,
14 | Body: fs.createReadStream(file.path),
15 | };
16 |
17 | return await s3.upload(params).promise()
18 | }
--------------------------------------------------------------------------------
/pages/api/my-files.js:
--------------------------------------------------------------------------------
1 | // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
2 |
3 | import {connectDB, disconnectDB} from "@/src/db";
4 | import MyFileModel from "@/src/models/MyFile";
5 |
6 | export default async function handler(req, res) {
7 | try {
8 | await connectDB()
9 | const files = await MyFileModel.find({})
10 | // await disconnectDB()
11 | return res.status(200).json(files)
12 | } catch (e) {
13 | return res.status(500).json({message: 'error fetching files'})
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2 |
3 | .idea
4 | # dependencies
5 | /node_modules
6 | /.pnp
7 | .pnp.js
8 |
9 | # testing
10 | /coverage
11 |
12 | # next.js
13 | /.next/
14 | /out/
15 |
16 | # production
17 | /build
18 |
19 | # misc
20 | .DS_Store
21 | *.pem
22 |
23 | # debug
24 | npm-debug.log*
25 | yarn-debug.log*
26 | yarn-error.log*
27 |
28 | # local env files
29 | .env.local
30 | .env*.local
31 |
32 | # vercel
33 | .vercel
34 |
35 | # typescript
36 | *.tsbuildinfo
37 | next-env.d.ts
38 |
--------------------------------------------------------------------------------
/public/vercel.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/components/ChatBox/Chat.js:
--------------------------------------------------------------------------------
1 | import AnimatedEllipsis from "@/components/AnimatedEllipsis";
2 |
3 | export default function Chat({query, response}) {
4 | return (
5 |
6 | {query &&
}
10 | {response &&
}
14 |
15 | )
16 | }
17 |
--------------------------------------------------------------------------------
/tailwind.config.js:
--------------------------------------------------------------------------------
1 | /** @type {import('tailwindcss').Config} */
2 | module.exports = {
3 | content: [
4 | './pages/**/*.{js,ts,jsx,tsx,mdx}',
5 | './components/**/*.{js,ts,jsx,tsx,mdx}',
6 | './app/**/*.{js,ts,jsx,tsx,mdx}',
7 | ],
8 | theme: {
9 | extend: {
10 | colors: {
11 | primary: {
12 | main: "#64748b",
13 | light: "#94a3b8",
14 | dark: "#475569",
15 | contrastText: "#f8fafc"
16 | }
17 | },
18 | backgroundImage: {
19 | 'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
20 | 'gradient-conic':
21 | 'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
22 | },
23 | },
24 | },
25 | plugins: [],
26 | }
27 |
--------------------------------------------------------------------------------
/components/MyFiles.js:
--------------------------------------------------------------------------------
1 | import {useState} from "react";
2 |
3 |
4 | export default function MyFiles({setActiveFile, files}) {
5 | return (
6 |
7 |
My Files
8 | {
9 | files.map((file, index) => (
10 |
17 | ))
18 | }
19 |
20 | )
21 | }
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "paperbot",
3 | "version": "0.1.0",
4 | "private": true,
5 | "scripts": {
6 | "dev": "next dev",
7 | "build": "next build",
8 | "start": "next start",
9 | "lint": "next lint"
10 | },
11 | "dependencies": {
12 | "@pinecone-database/pinecone": "^0.1.5",
13 | "autoprefixer": "10.4.14",
14 | "aws-sdk": "^2.1379.0",
15 | "formidable": "^2.1.1",
16 | "formidable-serverless": "^1.1.1",
17 | "mongoose": "^7.1.1",
18 | "next": "13.4.2",
19 | "openai": "^3.2.1",
20 | "pdfjs-dist": "^2.16.105",
21 | "postcss": "8.4.23",
22 | "react": "18.2.0",
23 | "react-dom": "18.2.0",
24 | "react-hot-toast": "^2.4.1",
25 | "react-icons": "^4.8.0",
26 | "slugify": "^1.6.6",
27 | "swr": "^2.1.5",
28 | "tailwindcss": "3.3.2"
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/src/models/MyFile.js:
--------------------------------------------------------------------------------
1 | import mongoose from 'mongoose'
2 |
3 | const Schema = mongoose.Schema;
4 |
5 | const MyFileSchema = new Schema({
6 | fileName: {
7 | type: String,
8 | required: [true, 'Filename is a required field.'],
9 | trim: true,
10 | maxLength: 100,
11 | unique: true,
12 | },
13 | fileUrl: {
14 | type: String,
15 | required: [true, 'File Url is a required field.'],
16 | trim: true,
17 | maxLength: 100,
18 | unique: true,
19 | },
20 | isProcessed: {
21 | type: Boolean,
22 | default: false,
23 | },
24 | vectorIndex: {
25 | type: String,
26 | maxLength: 100,
27 | unique: true,
28 | required: false,
29 | },
30 | }, {
31 | timestamps: true,
32 | });
33 |
34 | const MyFileModel = mongoose.models.myFile || mongoose.model('myFile', MyFileSchema);
35 |
36 |
37 | export default MyFileModel;
--------------------------------------------------------------------------------
/src/openaiServices.js:
--------------------------------------------------------------------------------
1 | import {Configuration, OpenAIApi} from "openai"
2 |
3 | const configuration = new Configuration({
4 | apiKey: process.env.OPENAI_API_KEY,
5 | });
6 |
7 | const openai = new OpenAIApi(configuration);
8 |
9 | const OPEN_AI_EMBEDDING_MODEL = "text-embedding-ada-002"
10 | const OPEN_AI_COMPLETION_MODEL = "text-davinci-003"
11 |
12 | export const getEmbeddings = async (text) => {
13 | const response = await openai.createEmbedding({
14 | model: OPEN_AI_EMBEDDING_MODEL,
15 | input: text,
16 | });
17 | return response.data.data[0].embedding
18 | }
19 |
20 | export const getCompletion = async (prompt) => {
21 | const completion = await openai.createCompletion({
22 | model: OPEN_AI_COMPLETION_MODEL,
23 | prompt: prompt,
24 | max_tokens: 500,
25 | temperature: 0
26 | });
27 |
28 | console.log(completion.data.choices)
29 |
30 | return completion.data.choices[0].text
31 | }
--------------------------------------------------------------------------------
/src/db.js:
--------------------------------------------------------------------------------
1 | import mongoose from 'mongoose';
2 |
3 | async function connectDB() {
4 | if (mongoose.connections[0].readyState) {
5 | // If a connection is already established, reuse it
6 | console.log('existing connection available')
7 | return;
8 | }
9 |
10 | const MONGO_URI = `mongodb+srv://${process.env.DB_USERNAME}:${process.env.DB_PASSWORD}@cluster0.rmf4y1k.mongodb.net/paperbot?retryWrites=true&w=majority`
11 |
12 | try {
13 | await mongoose.connect(MONGO_URI, {
14 | // Replace 'mydatabase' with your actual database name
15 | useNewUrlParser: true,
16 | useUnifiedTopology: true,
17 | });
18 | console.log('Connected to MongoDB');
19 | } catch (error) {
20 | console.error('Error connecting to MongoDB:', error.message);
21 | process.exit(1); // Exit the Node.js process on connection error
22 | }
23 | }
24 |
25 | async function disconnectDB() {
26 | if (mongoose.connections[0].readyState) {
27 | await mongoose.disconnect();
28 | console.log('Disconnected from MongoDB');
29 | }
30 | }
31 |
32 | export { connectDB, disconnectDB };
33 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # PaperBot
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 | Upload any pdf file and ask questions from that pdf.
10 |
11 | PaperBot generates embeddings from OpenAI embedding model and stores them in Pinecone Vector database.
12 | To answer user's questions, PaperBot does a semantic search on Pinecone Vector database and further refines the result with OpenAI completion model.
13 |
14 | ## Pre-requisites to run this project
15 |
16 | 1. OpenAI API Key
17 | 2. Pinecone Database ENV and KEY
18 | 3. MongoDB cluster username and password
19 | 4. AWS S3 bucket, access key and access id
20 |
21 | ## Steps to run this project
22 |
23 | 1. Clone the repo
24 | 2. Run `npm install` to install all the dependencies
25 | 3. Create a `.env.local` from `env-example` file: `cp env-example .env.local`
26 | 4. Update the OpenAI, Pinecone keys, MongoDB creds and AWS S3 bucket and creds in the `.env.local` file
27 | 5. Run `npm run dev` to start the project
28 |
29 |
30 | ### Tech stack
31 |
32 | - NextJS
33 | - TailwindCSS
34 | - MongoDB
35 | - Pinecone (Vector DB)
36 | - OpenAI Models - Embedding and Completion
37 |
38 | ## Contact
39 |
40 | mohdejazsiddiqui@gmail.com
41 |
42 | [](https://ko-fi.com/Y8Y618ZETK)
--------------------------------------------------------------------------------
/public/next.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/components/ChatBox/FileNotProcessedAlert.js:
--------------------------------------------------------------------------------
1 | import toast from "react-hot-toast";
2 | import {useState} from "react";
3 |
4 | const PillButton = ({func, disabled}) => (
5 |
9 | )
10 |
11 | export default function FileNotProcessedAlert({id}) {
12 | console.log("--id--", id)
13 | const [processing, setProcessing] = useState(false)
14 |
15 | const trigger = async (id) => {
16 | setProcessing(true)
17 | let response = await fetch("/api/process", {
18 | method: 'POST',
19 | body: JSON.stringify({id}),
20 | headers: {
21 | 'Content-type': 'application/json'
22 | }
23 | })
24 |
25 | if (response.ok) {
26 | response = await response.json()
27 | toast.success(response.message)
28 | } else {
29 | response = await response.json()
30 | toast.error(response.message)
31 | }
32 | setProcessing(false)
33 | }
34 |
35 | return (
36 |
37 |
Process File
38 |
Please process the file before starting to chat. Click button to process
39 | trigger(id)} disabled={processing}/>
40 |
41 |
42 | )
43 | }
--------------------------------------------------------------------------------
/pages/index.js:
--------------------------------------------------------------------------------
1 | import FileUpload from "@/components/FileUpload";
2 | import MyFiles from "@/components/MyFiles";
3 | import Intro from "@/components/Intro";
4 | import ChatBox from "@/components/ChatBox";
5 | import {useState} from "react";
6 | import useMyFiles from "@/apiHooks/useMyFiles";
7 | import Head from 'next/head'
8 |
9 | export default function Home() {
10 | const [activeFile, setActiveFile] = useState()
11 | const {files, isError, isLoading} = useMyFiles()
12 |
13 | if (isLoading) {
14 | return Loading...
15 | }
16 |
17 | return (
18 | <>
19 |
20 | Paperbot - Chat with my PDF
21 |
22 |
25 |
26 |
29 | PaperBot
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 | >
47 | )
48 | }
49 |
--------------------------------------------------------------------------------
/pages/api/query.js:
--------------------------------------------------------------------------------
1 | import pinecone, {initialize} from "@/src/pinecone";
2 | import {getCompletion, getEmbeddings} from "@/src/openaiServices";
3 | import {connectDB} from "@/src/db";
4 | import MyFileModel from "@/src/models/MyFile";
5 |
6 | export default async function handler(req, res) {
7 |
8 | // 1. check for POST call
9 |
10 | const {query, id} = req.body
11 |
12 | // 2. connect to mongodb
13 | await connectDB()
14 |
15 | // 3. query the file by id
16 | const myFile = await MyFileModel.findById(id)
17 |
18 | if(!myFile) {
19 | return res.status(400).send({message: 'invalid file id'})
20 | }
21 |
22 | // 4. get embeddings for the query
23 | const questionEmb = await getEmbeddings(query)
24 |
25 | // 5. initialize pinecone
26 | await initialize()
27 |
28 | // 6. connect to index
29 | const index = pinecone.Index(myFile.vectorIndex)
30 |
31 | // 7. query the pinecone db
32 | const queryRequest = {
33 | vector: questionEmb,
34 | topK: 5,
35 | includeValues: true,
36 | includeMetadata: true,
37 | };
38 |
39 | let result = await index.query({queryRequest})
40 |
41 | // 8. get the metadata from the results
42 | let contexts = result['matches'].map(item => item['metadata'].text)
43 |
44 | contexts = contexts.join("\n\n---\n\n")
45 |
46 | console.log('--contexts--', contexts)
47 |
48 | // 9. build the prompt
49 | const promptStart = `Answer the question based on the context below:\n\n`
50 | const promptEnd = `\n\nQuestion: ${query} \n\nAnswer:`
51 |
52 | const prompt = `${promptStart} ${contexts} ${promptEnd}`
53 |
54 | console.log('--prompt--', prompt)
55 |
56 | // 10. get the completion from openai
57 | let response = await getCompletion(prompt)
58 |
59 | console.log('--completion--', response)
60 |
61 | // 11. return the response
62 | res.status(200).json({response})
63 | }
64 |
--------------------------------------------------------------------------------
/pages/api/process.js:
--------------------------------------------------------------------------------
1 | import * as PDFJS from 'pdfjs-dist/legacy/build/pdf';
2 | import MyFileModel from "@/src/models/MyFile";
3 | import {connectDB, disconnectDB} from "@/src/db";
4 | import {getEmbeddings} from "@/src/openaiServices";
5 | import pinecone, {initialize} from "@/src/pinecone";
6 |
7 | export default async function handler(req, res) {
8 | // 1. check for POST call
9 | if (req.method !== 'POST') {
10 | return res.status(400).json({message: 'http method not allowed'})
11 | }
12 |
13 | try {
14 | // 2. connect to mongodb
15 | await connectDB()
16 |
17 | // 3. query the file by id
18 | const {id} = req.body
19 |
20 | const myFile = await MyFileModel.findById(id)
21 |
22 | if (!myFile) {
23 | return res.status(400).json({message: 'file not found'})
24 | }
25 |
26 | if(myFile.isProcessed) {
27 | return res.status(400).json({message: 'file is already processed'})
28 | }
29 |
30 | // 4. Read PDF and iterate through pages
31 | let vectors = []
32 |
33 | let myFiledata = await fetch(myFile.fileUrl)
34 |
35 | if (myFiledata.ok) {
36 | let pdfDoc = await PDFJS.getDocument(await myFiledata.arrayBuffer()).promise
37 | const numPages = pdfDoc.numPages
38 | for (let i = 0; i < numPages; i++) {
39 | let page = await pdfDoc.getPage(i + 1)
40 | let textContent = await page.getTextContent()
41 | const text = textContent.items.map(item => item.str).join('');
42 |
43 | // 5. Get embeddings for each page
44 | const embedding = await getEmbeddings(text)
45 |
46 | // 6. push to vector array
47 | vectors.push({
48 | id: `page${i + 1}`,
49 | values: embedding,
50 | metadata: {
51 | pageNum: i + 1,
52 | text,
53 | },
54 | })
55 | }
56 |
57 | // 7. initialize pinecone
58 | await initialize() // initialize pinecone
59 |
60 | // 8. connect to the index
61 | const index = pinecone.Index(myFile.vectorIndex)
62 |
63 | // 9. upsert to pinecone index
64 | await index.upsert({
65 | upsertRequest: {
66 | vectors,
67 | }
68 | });
69 |
70 | // 10. update mongodb with isProcessed to true
71 | myFile.isProcessed = true
72 | await myFile.save()
73 | // await disconnectDB()
74 |
75 | // 11. return the response
76 | return res.status(200).json({message: 'File processed successfully'})
77 | } else {
78 | // await disconnectDB()
79 | return res.status(500).json({message: 'error getting file contents'})
80 | }
81 | } catch (e) {
82 | console.log(e)
83 | // await disconnectDB()
84 | return res.status(500).json({message: e.message})
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/pages/api/upload.js:
--------------------------------------------------------------------------------
1 | import formidable from 'formidable-serverless';
2 | import {connectDB, disconnectDB} from '@/src/db'
3 | import MyFileModel from "@/src/models/MyFile";
4 | import slugify from 'slugify'
5 | import pinecone, {initialize} from "../../src/pinecone";
6 | import {s3Upload} from "@/src/s3services";
7 |
8 | export const config = {
9 | api: {
10 | bodyParser: false,
11 | },
12 | };
13 |
14 | const createIndex = async (indexName) => {
15 | const indexes = await pinecone.listIndexes()
16 | if (!indexes.includes(indexName)) {
17 | await pinecone.createIndex({
18 | createRequest: {
19 | name: indexName,
20 | dimension: 1536, // fixed for OPENAI embeddings
21 | },
22 | });
23 | console.log('index created')
24 | } else {
25 | throw new Error(`Index with name ${indexName} already exists`)
26 | }
27 | }
28 |
29 | export default async function handler(req, res) {
30 | // 1. only allow POST methods
31 | if (req.method !== 'POST') {
32 | return res.status(400).send('method not supported')
33 | }
34 |
35 | try {
36 | // 2. connect to the mongodb db
37 | await connectDB()
38 |
39 | // 3. parse the incoming FormData
40 | let form = new formidable.IncomingForm();
41 |
42 | form.parse(req, async (error, fields, files) => {
43 | if (error) {
44 | console.error('Failed to parse form data:', error);
45 | return res.status(500).json({error: 'Failed to parse form data'});
46 | }
47 |
48 | const file = files.file;
49 |
50 | // Check if the file object exists
51 | if (!file) {
52 | return res.status(400).json({error: 'No file uploaded'});
53 | }
54 |
55 | // Check if the necessary file properties are available
56 | if (!file.name || !file.path || !file.type) {
57 | return res.status(400).json({error: 'Invalid file data'});
58 | }
59 |
60 | // 4. upload the file to s3
61 | let data = await s3Upload(process.env.S3_BUCKET, file)
62 |
63 | // 5. initialize pinecone
64 | const filenameWithoutExt = file.name.split(".")[0]
65 | const filenameSlug = slugify(filenameWithoutExt, {
66 | lower: true, strict: true
67 | })
68 |
69 | await initialize() // initialize pinecone
70 |
71 | // 6. create a pinecone index
72 | await createIndex(filenameSlug) // create index
73 |
74 | // 7. save file info to the mongodb db
75 | const myFile = new MyFileModel({
76 | fileName: file.name,
77 | fileUrl: data.Location,
78 | vectorIndex: filenameSlug,
79 | })
80 | await myFile.save()
81 | // await disconnectDB()
82 |
83 | // 8. return the success response
84 | return res.status(200).json({message: 'File uploaded to S3 and index created'});
85 | })
86 | } catch (e) {
87 | console.log("--error--", e)
88 | // await disconnectDB()
89 | return res.status(500).send({message: e.message})
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/components/FileUpload.js:
--------------------------------------------------------------------------------
1 | import {useState} from 'react';
2 | import toast from "react-hot-toast";
3 |
4 | export default function FileUpload() {
5 | const [file, setFile] = useState();
6 | const [uploading, setUploading] = useState(false)
7 |
8 | const handleFileChange = (e) => {
9 | if (e.target.files) {
10 | let file = e.target.files[0]
11 |
12 | if (file.type !== 'application/pdf') {
13 | toast.error('Only PDF files are allowed')
14 | e.target.value = null
15 | return
16 | }
17 | setFile(file);
18 | }
19 | };
20 |
21 | const handleUploadClick = async () => {
22 | if (!file) {
23 | return;
24 | }
25 |
26 | setUploading(true)
27 | const formData = new FormData();
28 | formData.append('file', file);
29 |
30 | // 👇 Uploading the file using the fetch API to the server
31 | let response = await fetch('/api/upload', {
32 | method: 'POST',
33 | body: formData
34 | })
35 | if (response.ok) {
36 | response = await response.json()
37 | toast.success(response.message)
38 | } else {
39 | response = await response.json()
40 | toast.error(response.message)
41 | }
42 | setUploading(false)
43 | };
44 |
45 | return (
46 |
47 |
48 |
53 |
59 |
60 |
61 |
62 |
69 |
70 |
71 | )
72 | }
--------------------------------------------------------------------------------
/components/ChatBox/index.js:
--------------------------------------------------------------------------------
1 | import {MdSend} from "react-icons/md"
2 | import {useEffect, useRef, useState} from "react";
3 | import toast from "react-hot-toast";
4 | import ChooseFileAlert from "@/components/ChatBox/ChooseFileAlert";
5 | import ReadyAlert from "@/components/ChatBox/ReadyAlert";
6 | import Chat from "@/components/ChatBox/Chat";
7 | import FileNotProcessedAlert from "@/components/ChatBox/FileNotProcessedAlert";
8 |
9 | export default function ChatBox({activeFile}) {
10 | const divRef = useRef(null);
11 | const [chat, setChat] = useState([])
12 | const [query, setQuery] = useState()
13 | const [id, setId] = useState()
14 |
15 | const scrollToBottom = () => {
16 | if (divRef.current) {
17 | divRef.current.scrollTop = divRef.current.scrollHeight;
18 | }
19 | }
20 |
21 | useEffect(() => {
22 | scrollToBottom()
23 | console.log('msg added')
24 | }, [chat.length]);
25 |
26 | const addChat = (query, response, id) => {
27 | setChat(prevState => [...prevState, {
28 | query,
29 | response,
30 | }])
31 |
32 | setQuery(query)
33 | setId(id)
34 | }
35 |
36 | const handleSubmit = async (e) => {
37 | e.preventDefault();
38 | const query = e.target.query.value;
39 | const id = e.target.id.value;
40 |
41 | e.target.query.value = null;
42 |
43 | console.log(query, id);
44 | addChat(query, null, id);
45 | }
46 |
47 | const updateLastChat = (query, response) => {
48 | console.log("--old chat--", chat)
49 | const oldChats = [...chat]
50 | oldChats.pop()
51 | setChat([...oldChats, {
52 | query,
53 | response,
54 | }])
55 | }
56 |
57 | useEffect(() => {
58 | const fetchChatResponse = async () => {
59 | let response = await fetch("/api/query", {
60 | method: 'POST',
61 | body: JSON.stringify({query, id}),
62 | headers: {
63 | 'Content-type': 'application/json',
64 | },
65 | });
66 |
67 | if (response.ok) {
68 | response = await response.json();
69 | updateLastChat(query, response.response);
70 | } else {
71 | response = await response.json();
72 | toast.error(response.message)
73 | }
74 | };
75 |
76 | if (query) {
77 | fetchChatResponse().then(() => console.log('response received'));
78 | }
79 | }, [query]);
80 |
81 |
82 | return (
83 |
84 |
85 | {activeFile ? activeFile.fileName : "Choose a file to start chatting"}
86 |
87 |
88 | {activeFile
89 | ? !activeFile.isProcessed
90 | ?
91 | : chat.length > 0
92 | ?
93 | {
94 | chat.map(({query, response}, index) => (
95 |
96 | ))
97 | }
98 |
99 |
100 | :
101 | :
102 | }
103 |
104 |
105 |
116 |
117 | )
118 | }
--------------------------------------------------------------------------------