├── .gitignore
├── README.md
├── bun.lockb
├── jsconfig.json
├── next.config.js
├── package.json
├── postcss.config.js
├── public
├── next.svg
└── vercel.svg
├── src
└── app
│ ├── api
│ └── backend
│ │ └── route.js
│ ├── globals.css
│ ├── layout.js
│ └── page.js
└── tailwind.config.js
/.gitignore:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2 |
3 | # dependencies
4 | /node_modules
5 | /.pnp
6 | .pnp.js
7 | .yarn/install-state.gz
8 |
9 | # testing
10 | /coverage
11 |
12 | # next.js
13 | /.next/
14 | /out/
15 |
16 | # production
17 | /build
18 |
19 | # misc
20 | .DS_Store
21 | *.pem
22 |
23 | # debug
24 | npm-debug.log*
25 | yarn-debug.log*
26 | yarn-error.log*
27 |
28 | # local env files
29 | .env*.local
30 | .env
31 |
32 | # vercel
33 | .vercel
34 |
35 | # typescript
36 | *.tsbuildinfo
37 | next-env.d.ts
38 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | More details coming!!
2 |
3 | ## Getting Started
4 |
5 | First, run the development server:
6 |
7 | ```bash
8 | npm run dev
9 | # or
10 | yarn dev
11 | # or
12 | pnpm dev
13 | # or
14 | bun dev
15 | ```
16 |
--------------------------------------------------------------------------------
/bun.lockb:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/developersdigest/Perplexity-Next-JS-Supabase/ea35fbadf228522c8498666317817e851981dda1/bun.lockb
--------------------------------------------------------------------------------
/jsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "paths": {
4 | "@/*": ["./src/*"]
5 | }
6 | }
7 | }
8 |
--------------------------------------------------------------------------------
/next.config.js:
--------------------------------------------------------------------------------
1 | /** @type {import('next').NextConfig} */
2 | const nextConfig = {}
3 |
4 | module.exports = nextConfig
5 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "nextjs14",
3 | "version": "0.1.0",
4 | "private": true,
5 | "scripts": {
6 | "dev": "next dev",
7 | "build": "next build",
8 | "start": "next start",
9 | "lint": "next lint"
10 | },
11 | "dependencies": {
12 | "@phosphor-icons/react": "^2.0.13",
13 | "@supabase/supabase-js": "^2.38.4",
14 | "cheerio": "^1.0.0-rc.12",
15 | "dotenv": "^16.3.1",
16 | "langchain": "^0.0.174",
17 | "next": "14.0.0",
18 | "openai": "^4.14.0",
19 | "react": "^18",
20 | "react-dom": "^18",
21 | "react-markdown": "^9.0.0",
22 | "remark-gfm": "^4.0.0"
23 | },
24 | "devDependencies": {
25 | "autoprefixer": "^10",
26 | "postcss": "^8",
27 | "tailwindcss": "^3"
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/postcss.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | plugins: {
3 | tailwindcss: {},
4 | autoprefixer: {},
5 | },
6 | }
7 |
--------------------------------------------------------------------------------
/public/next.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/public/vercel.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/app/api/backend/route.js:
--------------------------------------------------------------------------------
1 | // 1. Import Dependencies
2 | import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
3 | import { OpenAIEmbeddings } from "langchain/embeddings/openai";
4 | import { MemoryVectorStore } from "langchain/vectorstores/memory";
5 | import { BraveSearch } from "langchain/tools";
6 | import OpenAI from "openai";
7 | import cheerio from "cheerio";
8 | import { createClient } from "@supabase/supabase-js";
9 | // 2. Initialize OpenAI and Supabase clients
10 | const openai = new OpenAI();
11 | const embeddings = new OpenAIEmbeddings();
12 | const supabase = createClient(process.env.SUPABASE_URL, process.env.SUPABASE_API_KEY);
13 | // 3. Send payload to Supabase table
14 | async function sendPayload(content) {
15 | await supabase
16 | .from("message_history")
17 | .insert([{ payload: content }])
18 | .select("id");
19 | }
20 | // 4. Rephrase input using GPT
21 | async function rephraseInput(inputString) {
22 | const gptAnswer = await openai.chat.completions.create({
23 | model: "gpt-3.5-turbo",
24 | messages: [
25 | {
26 | role: "system",
27 | content:
28 | "You are a rephraser and always respond with a rephrased version of the input that is given to a search engine API. Always be succint and use the same words as the input.",
29 | },
30 | { role: "user", content: inputString },
31 | ],
32 | });
33 | return gptAnswer.choices[0].message.content;
34 | }
35 | // 5. Search engine for sources
36 | async function searchEngineForSources(message) {
37 | const loader = new BraveSearch({ apiKey: process.env.BRAVE_SEARCH_API_KEY });
38 | const repahrasedMessage = await rephraseInput(message);
39 | const docs = await loader.call(repahrasedMessage);
40 | // 6. Normalize data
41 | function normalizeData(docs) {
42 | return JSON.parse(docs)
43 | .filter((doc) => doc.title && doc.link && !doc.link.includes("brave.com"))
44 | .slice(0, 4)
45 | .map(({ title, link }) => ({ title, link }));
46 | }
47 | const normalizedData = normalizeData(docs);
48 | // 7. Send normalized data as payload
49 | sendPayload({ type: "Sources", content: normalizedData });
50 | // 8. Initialize vectorCount
51 | let vectorCount = 0;
52 | // 9. Initialize async function for processing each search result item
53 | const fetchAndProcess = async (item) => {
54 | try {
55 | // 10. Create a timer for the fetch promise
56 | const timer = new Promise((_, reject) => setTimeout(() => reject(new Error("Timeout")), 1500));
57 | // 11. Fetch the content of the page
58 | const fetchPromise = fetchPageContent(item.link);
59 | // 12. Wait for either the fetch promise or the timer
60 | const htmlContent = await Promise.race([timer, fetchPromise]);
61 | // 13. Check for insufficient content length
62 | if (htmlContent.length < 250) return null;
63 | // 14. Split the text into chunks
64 | const splitText = await new RecursiveCharacterTextSplitter({ chunkSize: 200, chunkOverlap: 0 }).splitText(
65 | htmlContent
66 | );
67 | // 15. Create a vector store from the split text
68 | const vectorStore = await MemoryVectorStore.fromTexts(splitText, { annotationPosition: item.link }, embeddings);
69 | // 16. Increment the vector count
70 | vectorCount++;
71 | // 17. Perform similarity search on the vectors
72 | return await vectorStore.similaritySearch(message, 1);
73 | } catch (error) {
74 | // 18. Log any error and increment the vector count
75 | console.log(`Failed to fetch content for ${item.link}, skipping!`);
76 | vectorCount++;
77 | return null;
78 | }
79 | };
80 | // 19. Wait for all fetch and process promises to complete
81 | const results = await Promise.all(normalizedData.map(fetchAndProcess));
82 | // 20. Make sure that vectorCount reaches at least 4
83 | while (vectorCount < 4) {
84 | vectorCount++;
85 | }
86 | // 21. Filter out unsuccessful results
87 | const successfulResults = results.filter((result) => result !== null);
88 | // 22. Get top 4 results if there are more than 4, otherwise get all
89 | const topResult = successfulResults.length > 4 ? successfulResults.slice(0, 4) : successfulResults;
90 | // 23. Send a payload message indicating the vector creation process is complete
91 | sendPayload({ type: "VectorCreation", content: `Finished Scanning Sources.` });
92 | // 24. Trigger any remaining logic and follow-up actions
93 | triggerLLMAndFollowup(`Query: ${message}, Top Results: ${JSON.stringify(topResult)}`);
94 | }
95 | // 25. Define fetchPageContent function
96 | async function fetchPageContent(link) {
97 | const response = await fetch(link);
98 | return extractMainContent(await response.text());
99 | }
100 | // 26. Define extractMainContent function
101 | function extractMainContent(html) {
102 | const $ = cheerio.load(html);
103 | $("script, style, head, nav, footer, iframe, img").remove();
104 | return $("body").text().replace(/\s+/g, " ").trim();
105 | }
106 | // 27. Define triggerLLMAndFollowup function
107 | async function triggerLLMAndFollowup(inputString) {
108 | // 28. Call getGPTResults with inputString
109 | await getGPTResults(inputString);
110 | // 29. Generate follow-up with generateFollowup
111 | const followUpResult = await generateFollowup(inputString);
112 | // 30. Send follow-up payload
113 | sendPayload({ type: "FollowUp", content: followUpResult });
114 | // 31. Return JSON response
115 | return Response.json({ message: "Processing request" });
116 | }
117 | // 32. Define getGPTResults function
118 | const getGPTResults = async (inputString) => {
119 | // 33. Initialize accumulatedContent
120 | let accumulatedContent = "";
121 | // 34. Open a streaming connection with OpenAI
122 | const stream = await openai.chat.completions.create({
123 | model: "gpt-3.5-turbo",
124 | messages: [
125 | {
126 | role: "system",
127 | content:
128 | "You are a answer generator, you will receive top results of similarity search, they are optional to use depending how well they help answer the query.",
129 | },
130 | { role: "user", content: inputString },
131 | ],
132 | stream: true,
133 | });
134 | // 35. Create an initial row in the database
135 | let rowId = await createRowForGPTResponse();
136 | // 36. Send initial payload
137 | sendPayload({ type: "Heading", content: "Answer" });
138 | // 37. Iterate through the response stream
139 | for await (const part of stream) {
140 | // 38. Check if delta content exists
141 | if (part.choices[0]?.delta?.content) {
142 | // 39. Accumulate the content
143 | accumulatedContent += part.choices[0]?.delta?.content;
144 | // 40. Update the row with new content
145 | rowId = await updateRowWithGPTResponse(rowId, accumulatedContent);
146 | }
147 | }
148 | };
149 | // 41. Define createRowForGPTResponse function
150 | const createRowForGPTResponse = async () => {
151 | // 42. Generate a unique stream ID
152 | const generateUniqueStreamId = () => {
153 | return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
154 | };
155 | const streamId = generateUniqueStreamId();
156 | // 43. Create the payload
157 | const payload = { type: "GPT", content: "" };
158 | // 44. Insert into database
159 | const { data, error } = await supabase.from("message_history").insert([{ payload }]).select("id");
160 | // 45. Return the ID and stream ID
161 | return { id: data ? data[0].id : null, streamId };
162 | };
163 | // 46. Define updateRowWithGPTResponse function
164 | const updateRowWithGPTResponse = async (prevRowId, content) => {
165 | // 47. Create the payload
166 | const payload = { type: "GPT", content };
167 | // 48. Delete the previous row
168 | await supabase.from("message_history").delete().eq("id", prevRowId);
169 | // 49. Insert updated data
170 | const { data } = await supabase.from("message_history").insert([{ payload }]).select("id");
171 | // 50. Return the new row ID
172 | return data ? data[0].id : null;
173 | };
174 | // 51. Define generateFollowup function
175 | async function generateFollowup(message) {
176 | // 52. Create chat completion with OpenAI API
177 | const chatCompletion = await openai.chat.completions.create({
178 | messages: [
179 | {
180 | role: "system",
181 | content: `You are a follow up answer generator and always respond with 4 follow up questions based on this input "${message}" in JSON format. i.e. { "follow_up": ["QUESTION_GOES_HERE", "QUESTION_GOES_HERE", "QUESTION_GOES_HERE"] }`,
182 | },
183 | {
184 | role: "user",
185 | content: `Generate a 4 follow up questions based on this input ""${message}"" `,
186 | },
187 | ],
188 | model: "gpt-4",
189 | });
190 | // 53. Return the content of the chat completion
191 | return chatCompletion.choices[0].message.content;
192 | }
193 | // 54. Define POST function for API endpoint
194 | export async function POST(req, res) {
195 | // 55. Get message from request payload
196 | const { message } = await req.json();
197 | // 56. Send query payload
198 | sendPayload({ type: "Query", content: message });
199 | // 57. Start the search engine to find sources based on the query
200 | await searchEngineForSources(message);
201 | }
202 |
--------------------------------------------------------------------------------
/src/app/globals.css:
--------------------------------------------------------------------------------
1 | @tailwind base;
2 | @tailwind components;
3 | @tailwind utilities;
4 |
5 | :root {
6 | --foreground-rgb: 0, 0, 0;
7 | --background-start-rgb: 214, 219, 220;
8 | --background-end-rgb: 255, 255, 255;
9 | }
10 |
11 | @media (prefers-color-scheme: dark) {
12 | :root {
13 | --foreground-rgb: 255, 255, 255;
14 | --background-start-rgb: 0, 0, 0;
15 | --background-end-rgb: 0, 0, 0;
16 | }
17 | }
18 |
19 | @keyframes gradient-animation {
20 | 0% {
21 | background-position: 0% 50%;
22 | }
23 | 50% {
24 | background-position: 100% 50%;
25 | }
26 | 100% {
27 | background-position: 0% 50%;
28 | }
29 | }
30 |
31 | .tile-animation {
32 | background: linear-gradient(270deg, white, rgb(237, 242, 244), rgb(223, 242, 248));
33 | background-size: 200% 200%;
34 | animation: gradient-animation 10s linear infinite;
35 | }
36 | .tile-animation:hover {
37 | animation: gradient-animation 1s linear infinite;
38 | }
39 |
40 |
--------------------------------------------------------------------------------
/src/app/layout.js:
--------------------------------------------------------------------------------
1 | import { Inter } from 'next/font/google'
2 | import './globals.css'
3 |
4 | const inter = Inter({ subsets: ['latin'] })
5 |
6 | export const metadata = {
7 | title: 'Next.js Langchain Perplexity',
8 | }
9 |
10 | export default function RootLayout({ children }) {
11 | return (
12 |
13 | {children}
14 |
15 | )
16 | }
17 |
--------------------------------------------------------------------------------
/src/app/page.js:
--------------------------------------------------------------------------------
1 | "use client";
2 | "use strict";
3 | // 1. Import required dependencies
4 | import React, { useEffect, useRef, useState, memo } from "react";
5 | import { ArrowCircleRight, ChatCenteredDots, Stack, GitBranch } from "@phosphor-icons/react";
6 | import ReactMarkdown from "react-markdown";
7 | import remarkGfm from "remark-gfm";
8 | import { createClient } from "@supabase/supabase-js";
9 | // 2. Initialize Supabase client
10 | const SUPABASE_URL = "https://duvslrminwfswhezmqli.supabase.co";
11 | const SUPABASE_ANON_KEY =
12 | "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImR1dnNscm1pbndmc3doZXptcWxpIiwicm9sZSI6ImFub24iLCJpYXQiOjE2OTc4OTc4MDYsImV4cCI6MjAxMzQ3MzgwNn0.lvGJAaLpRr5X7uLVpS5IOqVbN8dXDsAjmkG31F8S380";
13 | const supabase = createClient(SUPABASE_URL, SUPABASE_ANON_KEY);
14 | // 3. Home component
15 | export default function Home() {
16 | // 4. Initialize states and refs
17 | const messagesEndRef = useRef(null);
18 | const [inputValue, setInputValue] = useState("");
19 | const [messageHistory, setMessageHistory] = useState([]);
20 | // 5. Auto-scroll to last message
21 | useEffect(() => {
22 | setTimeout(() => {
23 | messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
24 | }, 0);
25 | }, [messageHistory]);
26 | // 6. Fetch message history from Supabase
27 | useEffect(() => {
28 | // 7. Handle new inserts into the table
29 | const handleInserts = (payload) => {
30 | setMessageHistory((prevMessages) => {
31 | const lastMessage = prevMessages[prevMessages.length - 1];
32 | const isSameType = lastMessage?.payload?.type === "GPT" && payload.new.payload.type === "GPT";
33 | return isSameType ? [...prevMessages.slice(0, -1), payload.new] : [...prevMessages, payload.new];
34 | });
35 | };
36 | // 8. Subscribe to Supabase channel for real-time updates
37 | supabase
38 | .channel("message_history")
39 | .on("postgres_changes", { event: "INSERT", schema: "public", table: "message_history" }, handleInserts)
40 | .subscribe();
41 | // 9. Fetch existing message history from Supabase
42 | supabase
43 | .from("message_history")
44 | .select("*")
45 | .order("created_at", { ascending: true })
46 | .then(({ data: message_history, error }) =>
47 | error ? console.log("error", error) : setMessageHistory(message_history)
48 | );
49 | }, []);
50 | // 10. Function to send a message
51 | const sendMessage = (messageToSend) => {
52 | const message = messageToSend || inputValue;
53 | const body = JSON.stringify({ message: message });
54 | setInputValue("");
55 | // 11. POST message to the backend
56 | fetch("/api/backend", {
57 | method: "POST",
58 | body,
59 | headers: {
60 | "Content-Type": "application/json",
61 | },
62 | })
63 | .then((res) => res.json())
64 | .then((data) => {
65 | console.log("data", data);
66 | })
67 | .catch((err) => console.log("err", err));
68 | };
69 | // 12. Render home component
70 | return (
71 |
72 | {/* 13. Create main container with flex and screen height */}
73 |
74 | {/* 14. Map over message history to display each message */}
75 | {messageHistory.map((message, index) => (
76 | <>
77 |
78 |
79 | ))}
80 | {/* 15. Include InputArea for message input and sending */}
81 |
82 | {/* 16. Add a ref for the end of messages to enable auto-scroll */}
83 |
84 |