├── .env.local.example ├── .eslintrc.json ├── .gitignore ├── .prettierrc ├── README.md ├── components ├── layout.tsx └── ui │ ├── LoadingDots.tsx │ └── TextArea.tsx ├── config └── notionurls.ts ├── next.config.js ├── package.json ├── pages ├── _app.tsx ├── _document.tsx ├── api │ ├── chat.ts │ └── test.ts └── index.tsx ├── pnpm-lock.yaml ├── postcss.config.cjs ├── public ├── Thomas-Frank-Avatar.jpg ├── favicon.ico └── usericon.png ├── schema.sql ├── scripts ├── demo-query.ts └── scrape-embed.ts ├── styles ├── Home.module.css ├── base.css ├── chrome-bug.css └── loading-dots.module.css ├── tailwind.config.cjs ├── tsconfig.json ├── types └── chat.ts ├── utils ├── cn.ts ├── custom_web_loader.ts ├── helpers.ts ├── makechain.ts ├── openai-client.ts └── supabase-client.ts └── visual-guide └── websitechatbotsupabase.png /.env.local.example: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY= 2 | 3 | # Update these with your Supabase details from your project settings > API 4 | NEXT_PUBLIC_SUPABASE_URL= 5 | NEXT_PUBLIC_SUPABASE_ANON_KEY= 6 | 7 | # should only be used on a server. This key can bypass Row Level Security. NEVER use this key in a browser. 8 | SUPABASE_SERVICE_ROLE_KEY= 9 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | .pnpm-debug.log* 27 | 28 | # local env files 29 | .env*.local 30 | .env 31 | 32 | # vercel 33 | .vercel 34 | 35 | # typescript 36 | *.tsbuildinfo 37 | next-env.d.ts 38 | 39 | #Notion_db 40 | /Notion_DB 41 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "all", 3 | "singleQuote": true, 4 | "printWidth": 80, 5 | "tabWidth": 2 6 | } 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LangChain & Supabase - Create a ChatGpt Chatbot for Your Website 2 | 3 | Create a chatgpt chatbot for your website using LangChain, Supabase, Typescript, Openai, and Next.js. LangChain is a framework that makes it easier to build scalable AI/LLM apps. Supabase is an open source Postgres database that can store embeddings using a pg vector extension. 4 | 5 | [Tutorial video](https://www.youtube.com/watch?v=prbloUGlvLE) 6 | 7 | [Get in touch via twitter if you need help](https://twitter.com/mayowaoshin) 8 | 9 | The visual guide of this repo and tutorial is in the `visual guide` folder. 10 | 11 | ## Development 12 | 13 | 1. Clone the repo 14 | 15 | ``` 16 | git clone [github https url] 17 | ``` 18 | 19 | 2. Install packages 20 | 21 | ``` 22 | pnpm install 23 | ``` 24 | 25 | 3. Set up your `.env` file 26 | 27 | - Copy `.env.local.example` into `.env` 28 | Your `.env` file should look like this: 29 | 30 | ``` 31 | OPENAI_API_KEY= 32 | 33 | NEXT_PUBLIC_SUPABASE_URL= 34 | NEXT_PUBLIC_SUPABASE_ANON_KEY= 35 | SUPABASE_SERVICE_ROLE_KEY= 36 | 37 | ``` 38 | 39 | - Visit [openai](https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key) to retrieve API keys and insert into your `.env` file. 40 | - Visit [supabase](https://supabase.com/) to create a database and retrieve your keys in the user dashboard as per [docs instructions](https://supabase.com/docs) 41 | 42 | 4. In the `config` folder, replace the urls in the array with your website urls (the script requires more than one url). 43 | 44 | 5. In the `utils/custom_web_loader.ts` inside the `load` function replace the values of `title`, `date` and `content` with the css elements of text you'd like extract from a given webpage. You can learn more about how to use Cheerio [here](https://cheerio.js.org/) 45 | 46 | You can add your custom elements to the metadata to meet your needs, note however that the default loader format as per below expects at least a string for `pageContent` and `metadata` that contains a `source` property as a returned value: 47 | 48 | ``` 49 | async load(): Promise{ 50 | const $ = await this.scrape(); 51 | const text = $("body").text(); 52 | const metadata = { source: this.webPath }; 53 | return [new Document({ pageContent: text, metadata })]; 54 | } 55 | 56 | ``` 57 | 58 | The `pageContent` and `metadata` will later be stored in your supabase database table. 59 | 60 | 6. Copy and run `schema.sql` in your supabase sql editor 61 | 62 | - cross check the `documents` table exists in the database as well as the `match_documents` function. 63 | 64 | ## 🧑 Instructions for scraping and embedding 65 | 66 | To run the scraping and embedding script in `scripts/scrape-embed.ts` simply run: 67 | 68 | `npm run scrape-embed` 69 | 70 | This script will visit all the urls noted in the `config` folder and extract the data you specified in the `custom_web_loader.ts` file. 71 | 72 | Then it will use OpenAI's Embeddings(`text-embedding-ada-002`) to convert your scraped data into vectors. 73 | 74 | ## Run the app 75 | 76 | Once you've verified that the embeddings and content have been successfully added to your supabase table, you can run the app `npm run dev` and type a question to ask your website. 77 | 78 | ## Credit 79 | 80 | Frontend of this repo is inspired by [langchain-chat-nextjs](https://github.com/zahidkhawaja/langchain-chat-nextjs) 81 | 82 | This repo uses in-depth Notion guides from the [website](https://thomasjfrank.com/) of productivity expert, Thomas Frank. 83 | -------------------------------------------------------------------------------- /components/layout.tsx: -------------------------------------------------------------------------------- 1 | interface LayoutProps { 2 | children?: React.ReactNode; 3 | } 4 | 5 | export default function Layout({ children }: LayoutProps) { 6 | return ( 7 |
8 |
9 |
10 | 15 |
16 |
17 |
18 |
19 | {children} 20 |
21 |
22 |
23 | ); 24 | } 25 | -------------------------------------------------------------------------------- /components/ui/LoadingDots.tsx: -------------------------------------------------------------------------------- 1 | import styles from '@/styles/loading-dots.module.css'; 2 | 3 | const LoadingDots = ({ 4 | color = '#000', 5 | style = 'small', 6 | }: { 7 | color: string; 8 | style: string; 9 | }) => { 10 | return ( 11 | 12 | 13 | 14 | 15 | 16 | ); 17 | }; 18 | 19 | export default LoadingDots; 20 | 21 | LoadingDots.defaultProps = { 22 | style: 'small', 23 | }; 24 | -------------------------------------------------------------------------------- /components/ui/TextArea.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { cn } from '@/utils/cn'; 3 | 4 | export interface TextareaProps 5 | extends React.TextareaHTMLAttributes {} 6 | 7 | const Textarea = React.forwardRef( 8 | ({ className, ...props }, ref) => { 9 | return ( 10 |