├── .env.example ├── .gitattributes ├── .gitignore ├── .prettierrc ├── LICENSE ├── README.md ├── messages ├── de.json ├── en.json ├── es.json ├── fa.json ├── fr.json ├── it.json ├── ja.json ├── llm-prompt.md ├── pt.json ├── ru.json ├── tr.json └── zh.json ├── next.config.mjs ├── package.json ├── pnpm-lock.yaml ├── postcss.config.mjs ├── src ├── app │ ├── api │ │ └── chat-stream │ │ │ └── route.ts │ ├── favicon-32x32.png │ ├── globals.css │ ├── layout.tsx │ ├── page.tsx │ └── providers.tsx ├── components │ ├── chat-tab.tsx │ ├── container.tsx │ ├── debug-drawer.tsx │ ├── empty.tsx │ ├── error.tsx │ ├── header.tsx │ ├── info.tsx │ ├── list.tsx │ ├── locale-select.tsx │ ├── markdown-renderer.tsx │ ├── message.tsx │ ├── primitive │ │ ├── accordion.tsx │ │ ├── collapsible.tsx │ │ ├── dialog.tsx │ │ ├── popover.tsx │ │ ├── select.tsx │ │ ├── tabs.tsx │ │ └── toggle-group.tsx │ ├── search-tab.tsx │ └── search.tsx ├── i18n.ts ├── lib │ ├── actions.ts │ ├── dbs.ts │ ├── message-meta.ts │ ├── rag-chat.ts │ ├── types.ts │ ├── use-fetch-info.tsx │ └── utils.ts ├── middleware.ts └── service.ts ├── tailwind.config.ts └── tsconfig.json /.env.example: -------------------------------------------------------------------------------- 1 | UPSTASH_VECTOR_REST_URL= 2 | UPSTASH_VECTOR_REST_TOKEN= 3 | 4 | UPSTASH_REDIS_REST_TOKEN= 5 | UPSTASH_REDIS_REST_URL= 6 | 7 | QSTASH_TOKEN= 8 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | .env.local 4 | .env 5 | 6 | 7 | **/node_modules 8 | **/.idea 9 | 10 | # dependencies 11 | /node_modules 12 | /.pnp 13 | .pnp.js 14 | .yarn/install-state.gz 15 | 16 | # testing 17 | /coverage 18 | 19 | # next.js 20 | /.next/ 21 | /out/ 22 | 23 | # production 24 | /build 25 | 26 | # misc 27 | .DS_Store 28 | *.pem 29 | 30 | # debug 31 | npm-debug.log* 32 | yarn-debug.log* 33 | yarn-error.log* 34 | 35 | # local env files 36 | .env*.local 37 | 38 | # vercel 39 | .vercel 40 | 41 | # typescript 42 | *.tsbuildinfo 43 | next-env.d.ts 44 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/upstash/wikipedia-semantic-search/df9d79be2adb6f3ad64e8ce79bb0d0e06624721a/.prettierrc -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Adem ilter 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Indexing Millions of Wikipedia Articles With Upstash Vector 2 | 3 | This repository contains the code and documentation for our project on indexing millions of Wikipedia articles using Upstash Vector, as described in our [blog post](https://upstash.com/blog/indexing-wikipedia). 4 | 5 | ## Project Overview 6 | 7 | We've created a semantic search engine and [Upstash RAG Chat SDK](https://github.com/upstash/rag-chat) using Wikipedia data to demonstrate the capabilities of Upstash Vector and RAG Chat SDK. The project involves: 8 | 9 | 1. Preparing and embedding Wikipedia articles 10 | 2. Indexing the vectors using Upstash Vector 11 | 3. Building a Wikipedia semantic search engine 12 | 4. Implementing a RAG chatbot 13 | 14 | ## Key Features 15 | 16 | - Indexed over 144 million vectors from Wikipedia articles in 11 languages 17 | - Used BGE-M3 embedding model for multilingual support 18 | - Implemented semantic search with cross-lingual capabilities 19 | - Created a RAG chatbot using Upstash RAG Chat SDK 20 | 21 | ## Technologies Used 22 | 23 | - [Upstash Vector](https://upstash.com/docs/vector/overall/getstarted): For storing and querying vector embeddings 24 | - [Upstash Redis](https://upstash.com/redis): For storing chat sessions 25 | - [Upstash RAG Chat SDK](https://github.com/upstash/rag-chat): For building the RAG Chat application 26 | - [SentenceTransformers](https://www.sbert.net/): For generating embeddings 27 | - [Meta-Llama-3-8B-Instruct](https://ai.meta.com/blog/llama-3-available/): As the LLM provider through [QStash LLM APIs](https://upstash.com/docs/qstash/features/llm) 28 | 29 | ## Development 30 | 31 | To run the project locally, follow these steps: 32 | 33 | 1. Go to [Upstash Console](https://console.upstash.com/) to manage your databases: 34 | - Create a new Vector database with embedding model support. You can choose the BGE-M3 model for multilingual support. 35 | - Create a new Redis database for storing chat sessions. 36 | - Copy the credentials for both Redis and Vector. Also copy the QStash credentials for using the upstash hosted LLM models. 37 | 38 | Put the credentials in a `.env` file in the root of the project. Your `.env` file should look like this: 39 | 40 | ```bash 41 | UPSTASH_VECTOR_REST_URL= 42 | UPSTASH_VECTOR_REST_TOKEN= 43 | 44 | UPSTASH_REDIS_REST_TOKEN= 45 | UPSTASH_REDIS_REST_URL= 46 | 47 | QSTASH_TOKEN= 48 | ``` 49 | 50 | 2. Populate your Vector index. 51 | 52 | > This project uses namespaces to store articles in different languages. So you have to upsert the vectors in the correct namespace. For english, upsert your vectors into the `en` namespace. 53 | 54 | 3. Install the dependencies: 55 | 56 | ```bash 57 | pnpm install 58 | ``` 59 | 60 | 4. Run the development server: 61 | 62 | ```bash 63 | pnpm dev 64 | ``` 65 | 66 | ## Contributing 67 | 68 | We welcome contributions to improve this project. Please feel free to submit issues or pull requests. 69 | 70 | ## Acknowledgements 71 | 72 | - Wikipedia for providing the dataset 73 | - Upstash for their vector database and RAG Chat SDK 74 | - All contributors to the open-source libraries used in this project 75 | 76 | ## Contact 77 | 78 | For any questions or feedback about the project or Upstash Vector, please reach out to us at (add contact information). 79 | 80 | Check out our [live demo](https://wikipedia-semantic-search.vercel.app/) to see the project in action! 81 | -------------------------------------------------------------------------------- /messages/de.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "Beispielfragen", 3 | "example1": "Längster Fluss der Welt", 4 | "example2": "Bücher von Stephen King", 5 | "example3": "Wer hat das Flugzeug erfunden?" 6 | } 7 | -------------------------------------------------------------------------------- /messages/en.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "Example questions", 3 | "example1": "Longest river in the world", 4 | "example2": "Books by Stephen King", 5 | "example3": "Who invented the airplane?" 6 | } 7 | -------------------------------------------------------------------------------- /messages/es.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "Ejemplos de preguntas", 3 | "example1": "Río más largo del mundo", 4 | "example2": "Libros de Stephen King", 5 | "example3": "¿Quién inventó el avión?" 6 | } 7 | -------------------------------------------------------------------------------- /messages/fa.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "سوالات نمونه", 3 | "example1": "طولانی‌ترین رودخانه جهان", 4 | "example2": "کتاب‌های استیون کینگ", 5 | "example3": "چه کسی هواپیما را اختراع کرد؟" 6 | } 7 | -------------------------------------------------------------------------------- /messages/fr.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "Exemples de questions", 3 | "example1": "Plus long fleuve du monde", 4 | "example2": "Livres de Stephen King", 5 | "example3": "Qui a inventé l'avion?" 6 | } 7 | -------------------------------------------------------------------------------- /messages/it.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "Esempi di domande", 3 | "example1": "Il fiume più lungo del mondo", 4 | "example2": "Libri di Stephen King", 5 | "example3": "Chi ha inventato l'aereo?" 6 | } 7 | -------------------------------------------------------------------------------- /messages/ja.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "例の質問", 3 | "example1": "世界で最も長い川", 4 | "example2": "スティーブン・キングの本", 5 | "example3": "飛行機を発明したのは誰ですか？" 6 | } 7 | -------------------------------------------------------------------------------- /messages/llm-prompt.md: -------------------------------------------------------------------------------- 1 | Here is a list of languages: 2 | de 3 | ru 4 | pt 5 | en (already given) 6 | it 7 | fr 8 | zh 9 | es 10 | ja 11 | fa 12 | tr 13 | 14 | I have a json data in english. 15 | Translate this json data to the languages listed above. 16 | 17 | Json data: 18 | { 19 | "exampleTitle": "Example questions", 20 | "example1": "Longest river in the world", 21 | "example2": "Books by Stephen King", 22 | "example3": "Who invented the airplane?" 23 | } 24 | -------------------------------------------------------------------------------- /messages/pt.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "Exemplos de perguntas", 3 | "example1": "Maior rio do mundo", 4 | "example2": "Livros de Stephen King", 5 | "example3": "Quem inventou o avião?" 6 | } 7 | -------------------------------------------------------------------------------- /messages/ru.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "Примеры вопросов", 3 | "example1": "Самая длинная река в мире", 4 | "example2": "Книги Стивена Кинга", 5 | "example3": "Кто изобрел самолет?" 6 | } 7 | -------------------------------------------------------------------------------- /messages/tr.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "Örnek sorular", 3 | "example1": "Dünyanın en uzun nehri", 4 | "example2": "Stephen King'in kitapları", 5 | "example3": "Uçağı kim icat etti?" 6 | } 7 | -------------------------------------------------------------------------------- /messages/zh.json: -------------------------------------------------------------------------------- 1 | { 2 | "exampleTitle": "示例问题", 3 | "example1": "世界上最长的河流", 4 | "example2": "斯蒂芬·金的书", 5 | "example3": "谁发明了飞机?" 6 | } 7 | -------------------------------------------------------------------------------- /next.config.mjs: -------------------------------------------------------------------------------- 1 | import createNextIntlPlugin from 'next-intl/plugin'; 2 | 3 | const withNextIntl = createNextIntlPlugin(); 4 | 5 | /** @type {import('next').NextConfig} */ 6 | const nextConfig = { 7 | images: { 8 | remotePatterns: [ 9 | { 10 | protocol: "https", 11 | hostname: "image.tmdb.org", 12 | }, 13 | ], 14 | }, 15 | }; 16 | 17 | export default withNextIntl(nextConfig); 18 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "movies", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@radix-ui/react-accordion": "^1.2.0", 13 | "@radix-ui/react-collapsible": "^1.1.0", 14 | "@radix-ui/react-dialog": "^1.1.1", 15 | "@radix-ui/react-icons": "^1.3.0", 16 | "@radix-ui/react-popover": "^1.1.1", 17 | "@radix-ui/react-select": "^2.1.1", 18 | "@radix-ui/react-tabs": "^1.1.0", 19 | "@radix-ui/react-toggle-group": "^1.1.0", 20 | "@tabler/icons-react": "^3.12.0", 21 | "@tanstack/react-query": "^5.51.23", 22 | "@upstash/rag-chat": "^1.4.0", 23 | "@upstash/redis": "^1.34.0", 24 | "@upstash/vector": "^1.1.5", 25 | "ai": "^3.3.5", 26 | "class-variance-authority": "^0.7.0", 27 | "clsx": "^2.1.1", 28 | "luxon": "^3.4.4", 29 | "next": "^14.2.5", 30 | "next-intl": "^3.17.2", 31 | "pretty-ms": "^9.1.0", 32 | "react": "^18", 33 | "react-dom": "^18", 34 | "react-markdown": "^9.0.1", 35 | "rehype-raw": "^7.0.0", 36 | "remark-gfm": "^4.0.0", 37 | "tailwindcss-animate": "^1.0.7", 38 | "zod": "^3.23.8" 39 | }, 40 | "devDependencies": { 41 | "@types/luxon": "^3.4.2", 42 | "@types/node": "^20", 43 | "@types/react": "^18", 44 | "@types/react-dom": "^18", 45 | "postcss": "^8", 46 | "prettier": "^3.3.2", 47 | "tailwind-merge": "^2.4.0", 48 | "tailwindcss": "^3.4.1", 49 | "typescript": "^5" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /postcss.config.mjs: -------------------------------------------------------------------------------- 1 | /** @type {import('postcss-load-config').Config} */ 2 | const config = { 3 | plugins: { 4 | tailwindcss: {}, 5 | }, 6 | }; 7 | 8 | export default config; 9 | -------------------------------------------------------------------------------- /src/app/api/chat-stream/route.ts: -------------------------------------------------------------------------------- 1 | import type { Message } from "ai"; 2 | import { NextRequest } from "next/server"; 3 | import { MessageMetadata } from "@/lib/message-meta"; 4 | import { PROMPT, ragChat } from "@/lib/rag-chat"; 5 | import { aiUseChatAdapter } from "@upstash/rag-chat/nextjs"; 6 | 7 | export const maxDuration = 30; 8 | 9 | export async function POST(request: NextRequest) { 10 | try { 11 | const { messages, namespace } = await request.json(); 12 | 13 | const question = (messages as Message[]).at(-1)?.content; 14 | if (!question) throw new Error("No question in the request"); 15 | 16 | const sessionId = request.cookies.get("sessionId")?.value; 17 | 18 | if (!sessionId) throw new Error("No sessionId found"); 19 | 20 | const response = await ragChat.chat(question, { 21 | streaming: true, 22 | sessionId: sessionId, 23 | namespace: namespace, 24 | onContextFetched(context) { 25 | // only the top 5 results 26 | return context.slice(0, 5); 27 | }, 28 | topK: 50, 29 | }); 30 | 31 | const meta = { 32 | usedPrompt: PROMPT, 33 | usedHistory: response.history.map(({ role, content }) => ({ 34 | role, 35 | content, 36 | })), 37 | usedContext: response.context.map(({ metadata, data }) => ({ 38 | url: (metadata as { url?: string }).url ?? "", 39 | data, 40 | })), 41 | }; 42 | 43 | // send meta along the response, this is consumed in the client 44 | // with the data field of useChat 45 | return aiUseChatAdapter(response, meta); 46 | } catch (error) { 47 | return Response.json("Server error", { 48 | status: 500, 49 | }); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/app/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/upstash/wikipedia-semantic-search/df9d79be2adb6f3ad64e8ce79bb0d0e06624721a/src/app/favicon-32x32.png -------------------------------------------------------------------------------- /src/app/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | @layer base { 6 | html { 7 | color-scheme: light !important; 8 | } 9 | .scrollbar-hide::-webkit-scrollbar { 10 | display: none; 11 | } 12 | 13 | /* Hide scrollbar for IE, Edge and Firefox */ 14 | .scrollbar-hide { 15 | -ms-overflow-style: none; /* IE and Edge */ 16 | scrollbar-width: none; /* Firefox */ 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import "./globals.css"; 2 | import { Metadata, Viewport } from "next"; 3 | import { EB_Garamond, Inter } from "next/font/google"; 4 | import { NextIntlClientProvider } from "next-intl"; 5 | import { getLocale, getMessages } from "next-intl/server"; 6 | import { Providers } from "./providers"; 7 | 8 | export const metadata: Metadata = { 9 | title: "Wikipedia Semantic Search by Upstash Vector", 10 | description: 11 | "An experimental project to demonstrate the scalability of Upstash Vector with large datasets. We vectorized 23 million Wikipedia articles in 11 languages and stored 144 millin vectors in a single Upstash Vector index.", 12 | icons: { 13 | icon: "/favicon-32x32.png", 14 | }, 15 | }; 16 | 17 | export const viewport: Viewport = { 18 | maximumScale: 1, // Disable auto-zoom on mobile Safari 19 | }; 20 | 21 | const serif = EB_Garamond({ 22 | subsets: ["latin", "latin-ext"], 23 | variable: "--font-serif", 24 | style: ["normal"], 25 | weight: ["400", "600"], 26 | display: "swap", 27 | }); 28 | 29 | const sans = Inter({ 30 | subsets: ["latin", "latin-ext"], 31 | variable: "--font-sans", 32 | weight: ["400", "600"], 33 | display: "swap", 34 | }); 35 | 36 | export default async function RootLayout({ 37 | children, 38 | }: { 39 | children: React.ReactNode; 40 | }) { 41 | const locale = await getLocale(); 42 | // Providing all messages to the client 43 | // side is the easiest way to get started 44 | const messages = await getMessages(); 45 | 46 | return ( 47 | 51 | 52 | 53 | 54 | {children} 55 | 56 | 57 | 58 | 59 | ); 60 | } 61 | -------------------------------------------------------------------------------- /src/app/page.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { useState } from "react"; 4 | import { ChatTab } from "@/components/chat-tab"; 5 | import { Header } from "@/components/header"; 6 | import { SearchTab } from "@/components/search-tab"; 7 | import { 8 | Tabs, 9 | TabsContent, 10 | TabsList, 11 | TabsTrigger, 12 | } from "@/components/primitive/tabs"; 13 | import Container from "@/components/container"; 14 | 15 | export default function Page() { 16 | const [tab, setTab] = useState<"chat" | "search">("search"); 17 | 18 | return ( 19 | <> 20 |
21 | 22 |
23 | setTab(value as "chat" | "search")} 26 | > 27 | 28 | 29 | Search 30 | Chat to Wikipedia 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 |
46 | 47 | ); 48 | } 49 | -------------------------------------------------------------------------------- /src/app/providers.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; 4 | import { PropsWithChildren } from "react"; 5 | 6 | export const Providers = ({ children }: PropsWithChildren) => { 7 | return ( 8 | {children} 9 | ); 10 | }; 11 | 12 | const queryClient = new QueryClient({ 13 | defaultOptions: { 14 | queries: { 15 | refetchOnWindowFocus: false, 16 | }, 17 | }, 18 | }); 19 | -------------------------------------------------------------------------------- /src/components/chat-tab.tsx: -------------------------------------------------------------------------------- 1 | import { serverClearMessages, serverGetMessages } from "@/lib/actions"; 2 | import { useQuery } from "@tanstack/react-query"; 3 | import { Message, useChat } from "ai/react"; 4 | import { useLocale } from "next-intl"; 5 | import { useEffect, useRef } from "react"; 6 | import LocaleSelect from "./locale-select"; 7 | import { PaperPlaneIcon } from "@radix-ui/react-icons"; 8 | import ChatMessage from "./message"; 9 | import { cn } from "@/lib/utils"; 10 | import { Info } from "@/components/info"; 11 | import { MarkdownRenderer } from "./markdown-renderer"; 12 | 13 | const LOADING_MSG_ID = "loading-msg"; 14 | 15 | export const ChatTab = () => { 16 | const locale = useLocale(); 17 | const messagesEndRef = useRef(null); 18 | 19 | // These also contain metadata for debugging like the context used 20 | const { data: messageHistory, isLoading: isServerMessages } = useQuery({ 21 | queryKey: ["messages"], 22 | queryFn: async () => { 23 | return await serverGetMessages(); 24 | }, 25 | }); 26 | 27 | const { 28 | data, 29 | messages, 30 | setMessages, 31 | handleInputChange, 32 | handleSubmit, 33 | input, 34 | error, 35 | isLoading, 36 | } = useChat({ 37 | api: "/api/chat-stream", 38 | body: { 39 | namespace: locale, 40 | }, 41 | }); 42 | 43 | const hasMessages = messages.length > 0; 44 | 45 | useEffect(() => { 46 | // When a new metadata comes from the server 47 | // update the last message with it 48 | setMessages((messages) => { 49 | const meta = data?.at(-1); 50 | if (!meta) return messages; 51 | const last = messages.at(-1); 52 | if (!last) return messages; 53 | return [ 54 | ...messages.slice(0, -1), 55 | { 56 | ...last, 57 | metadata: meta, 58 | }, 59 | ]; 60 | }); 61 | }, [data]); 62 | 63 | // Only called once 64 | useEffect(() => { 65 | if (messageHistory) { 66 | setMessages(messageHistory); 67 | } 68 | }, [messageHistory]); 69 | 70 | const messagesWithLoading: Message[] = [ 71 | ...messages, 72 | ...(isLoading && messages.at(-1)?.role !== "assistant" 73 | ? [ 74 | { 75 | id: LOADING_MSG_ID, 76 | role: "assistant", 77 | content: "...", 78 | } as const, 79 | ] 80 | : []), 81 | ]; 82 | 83 | useEffect(() => { 84 | if (messagesEndRef.current) { 85 | messagesEndRef.current.scrollIntoView({ behavior: "instant" }); 86 | } 87 | }, [messagesWithLoading]); 88 | 89 | return ( 90 | <> 91 |
95 |
96 |
97 | {!hasMessages && ( 98 |
99 | Chat with the Wikipedia assistant 100 |
101 | )} 102 | 103 |
104 | {messagesWithLoading.map((message) => { 105 | // @ts-ignore 106 | const meta = message.metadata; 107 | 108 | return ( 109 | 110 | {message.content} 111 | 112 | ); 113 | })} 114 | {/* Scroll buffer */} 115 |
{" "} 116 |
117 |
118 |
119 | 120 |
124 | 132 | 133 | 134 | 143 | 144 | {hasMessages && ( 145 | 154 | )} 155 | 156 | 157 | {error && ( 158 |
Error: {error.message}
159 | )} 160 |
161 | 162 | 163 |

Chat support is implemented with RAG-Chat SDK.

164 | 165 |

166 | 167 | 👉 Check out{" "} 168 | 173 | the repo for more. 174 | 175 | 176 |

177 | 178 | 179 | ); 180 | }; 181 | -------------------------------------------------------------------------------- /src/components/container.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { cn } from "@/lib/utils"; 3 | 4 | const Container = ({ 5 | className, 6 | ...props 7 | }: React.ComponentProps<"div"> & {}) => ( 8 |
12 | ); 13 | export default Container; 14 | -------------------------------------------------------------------------------- /src/components/debug-drawer.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Accordion, 3 | AccordionContent, 4 | AccordionItem, 5 | AccordionTrigger, 6 | } from "@/components/primitive/accordion"; 7 | import { PropsWithChildren, useState } from "react"; 8 | import { MessageMetadata, messageMetadataSchema } from "@/lib/message-meta"; 9 | import { 10 | Dialog, 11 | DialogContent, 12 | DialogTitle, 13 | DialogTrigger, 14 | } from "@/components/primitive/dialog"; 15 | import { ExternalLinkIcon } from "@radix-ui/react-icons"; 16 | 17 | export const DebugDrawer = ({ 18 | metadata, 19 | }: PropsWithChildren<{ 20 | metadata: MessageMetadata; 21 | }>) => { 22 | const [open, setOpen] = useState(false); 23 | const { success: isValid } = messageMetadataSchema.safeParse(metadata); 24 | 25 | return ( 26 | 27 | 28 | 37 | 38 | 39 | {isValid ? ( 40 | 41 | Debug Prompt 42 | 43 |
44 | 45 | 46 | Prompt 47 | 48 | 
49 |                     {metadata.usedPrompt}
50 |
51 | 52 | 53 | 54 | Chat History 55 | 56 | {metadata.usedHistory.map(({ role, content }, i) => { 57 | return ( 58 | 
59 |                         {role}: {content}
60 |
61 | ); 62 | })} 63 | 64 | 65 | 66 | Context 67 | 68 |
69 | {metadata?.usedContext.map(({ url, data }) => ( 70 |
74 |

75 | 76 | {decodeURI(url)} 77 | 78 | 79 |

80 |

{data}

81 |
82 | ))} 83 |
84 | 85 | 86 | 87 |
88 | 89 | ) : ( 90 | 91 | Invalid metadata, message is probably old. Please try this with more 92 | recent messages 93 | 94 | )} 95 | 96 | ); 97 | }; 98 | -------------------------------------------------------------------------------- /src/components/empty.tsx: -------------------------------------------------------------------------------- 1 | import { Result } from "@/lib/types"; 2 | import React from "react"; 3 | import { useTranslations } from "next-intl"; 4 | 5 | export default function EmptyState({ 6 | loading, 7 | state, 8 | onSearch = () => {}, 9 | }: { 10 | loading: boolean; 11 | state: Result | undefined; 12 | onSearch: (query: string) => void; 13 | }) { 14 | const t = useTranslations(); 15 | 16 | const _onSearch = (e: React.MouseEvent) => { 17 | e.preventDefault(); 18 | onSearch(e.currentTarget.textContent ?? ""); 19 | }; 20 | 21 | if (loading) { 22 | return ; 23 | } 24 | 25 | if (state?.data && state?.data.length > 0) { 26 | return null; 27 | } 28 | 29 | return ( 30 | <> 31 |

{t("exampleTitle")}

32 | 33 |
    34 |
  1. 35 | 40 | {t("example1")} 41 | 42 |
    2. 43 |
  2. 44 | 49 | {t("example2")} 50 | 51 |
    3. 52 |
  3. 53 | 58 | {t("example3")} 59 | 60 |
    4. 61 |
62 | 63 | ); 64 | } 65 | 66 | const Skeleton = () => { 67 | return ( 68 |
69 |
70 | {new Array(3).fill(null).map((_, i) => ( 71 |
72 |
73 |
74 |
75 |
76 |
77 | ))} 78 |
79 | ); 80 | }; 81 | -------------------------------------------------------------------------------- /src/components/error.tsx: -------------------------------------------------------------------------------- 1 | import { Result, ResultCode } from "@/lib/types"; 2 | import React from "react"; 3 | 4 | export default function ErrorMessages({ 5 | state, 6 | }: { 7 | state: Result | undefined; 8 | }) { 9 | if (state?.code === ResultCode.UnknownError) { 10 | return ( 11 |
12 |

An error occurred, please try again.

13 |
14 | ); 15 | } 16 | 17 | if (state?.code === ResultCode.MinLengthError) { 18 | return ( 19 |
20 |

21 | Please enter at least 2 characters to start searching wikipedia. 22 |

23 |
24 | ); 25 | } 26 | 27 | return null; 28 | } 29 | -------------------------------------------------------------------------------- /src/components/header.tsx: -------------------------------------------------------------------------------- 1 | import { useRouter } from "next/navigation"; 2 | import Container from "@/components/container"; 3 | 4 | export const Header = () => { 5 | const router = useRouter(); 6 | 7 | return ( 8 |
9 | 10 |

{ 12 | router.replace("/"); 13 | }} 14 | className="font-serif font-bold text-2xl sm:text-3xl hover:underline cursor-pointer" 15 | > 16 | Wikipedia Semantic Search 17 |

18 | 19 |
20 | ); 21 | }; 22 | -------------------------------------------------------------------------------- /src/components/info.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { cn } from "@/lib/utils"; 3 | 4 | export const Info = ({ children, className }: React.ComponentProps<"div">) => { 5 | return ( 6 |
12 | {children} 13 |
14 | ); 15 | }; 16 | -------------------------------------------------------------------------------- /src/components/list.tsx: -------------------------------------------------------------------------------- 1 | import { Result } from "@/lib/types"; 2 | import prettyMilliseconds from "pretty-ms"; 3 | import { formatter } from "@/lib/utils"; 4 | import { useFetchInfo } from "@/lib/use-fetch-info"; 5 | import { ExternalLinkIcon } from "@radix-ui/react-icons"; 6 | 7 | export default function List({ state }: { state: Result | undefined }) { 8 | const { data: info } = useFetchInfo(); 9 | 10 | if (state?.data.length === 0) { 11 | return null; 12 | } 13 | 14 | return ( 15 | <> 16 |
17 |

18 | Search has been completed in{" "} 19 | {prettyMilliseconds(state?.ms ?? 0)} over{" "} 20 | {formatter.format(info?.vectorCount ?? 0)} wikipedia records. 21 |

22 |
23 | 24 |
25 | {state?.data.map((movie, index) => ( 26 | 49 | ))} 50 |
51 | 52 | ); 53 | } 54 | -------------------------------------------------------------------------------- /src/components/locale-select.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { useTransition } from "react"; 4 | import { useLocale } from "next-intl"; 5 | import type { Locale } from "@/service"; 6 | import { setUserLocale } from "@/service"; 7 | import { 8 | Select, 9 | SelectContent, 10 | SelectItem, 11 | SelectTrigger, 12 | SelectValue, 13 | } from "@/components/primitive/select"; 14 | import { useFetchInfo } from "@/lib/use-fetch-info"; 15 | 16 | export default function LocaleSelect() { 17 | const { data: info } = useFetchInfo(); 18 | const [_, startTransition] = useTransition(); 19 | const locale = useLocale(); 20 | 21 | return ( 22 | 48 | ); 49 | } 50 | -------------------------------------------------------------------------------- /src/components/markdown-renderer.tsx: -------------------------------------------------------------------------------- 1 | import Markdown from "react-markdown"; 2 | import rehypeRaw from "rehype-raw"; 3 | import remarkGfm from "remark-gfm"; 4 | 5 | export function MarkdownRenderer({ children: markdown }: { children: string }) { 6 | return ( 7 | {children} 15 | ); 16 | }, 17 | ol({ children }) { 18 | return ( 19 |
    20 | {children} 21 |
22 | ); 23 | }, 24 | li({ children }) { 25 | return
  • {children}
    • ; 26 | }, 27 | code({ children }) { 28 | return {children}; 29 | }, 30 | a({ children, href }) { 31 | return ( 32 | 38 | {children} 39 | 40 | ); 41 | }, 42 | }} 43 | > 44 | {markdown} 45 |     46 | ); 47 | } 48 | -------------------------------------------------------------------------------- /src/components/message.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { cn } from "@/lib/utils"; 3 | import { Message as MessageProps } from "ai/react"; 4 | import { DebugDrawer } from "@/components/debug-drawer"; 5 | 6 | const Message = ({ 7 | role, 8 | children, 9 | className, 10 | meta, 11 | ...props 12 | }: React.ComponentProps<"div"> & { 13 | role: MessageProps["role"]; 14 | meta?: any; 15 | }) => ( 16 |
    24 |
    30 |
    36 |
    {children}
    37 | 38 | {role === "assistant" && meta && ( 39 |
    40 | 41 |
    42 | )} 43 |
    44 |
    45 |
    46 | ); 47 | export default Message; 48 | -------------------------------------------------------------------------------- /src/components/primitive/accordion.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as React from "react"; 4 | import * as AccordionPrimitive from "@radix-ui/react-accordion"; 5 | import { IconChevronDown } from "@tabler/icons-react"; 6 | import { cn } from "@/lib/utils"; 7 | 8 | const Accordion = AccordionPrimitive.Root; 9 | 10 | const AccordionItem = React.forwardRef< 11 | React.ElementRef, 12 | React.ComponentPropsWithoutRef 13 | >(({ className, ...props }, ref) => ( 14 | 23 | )); 24 | AccordionItem.displayName = "AccordionItem"; 25 | 26 | const AccordionTrigger = React.forwardRef< 27 | React.ElementRef, 28 | React.ComponentPropsWithoutRef 29 | >(({ className, children, ...props }, ref) => ( 30 | 31 | svg]:rotate-180", 36 | className, 37 | )} 38 | {...props} 39 | > 40 | {children} 41 | 42 | 43 | 44 | )); 45 | AccordionTrigger.displayName = AccordionPrimitive.Trigger.displayName; 46 | 47 | const AccordionContent = React.forwardRef< 48 | React.ElementRef, 49 | React.ComponentPropsWithoutRef 50 | >(({ className, children, ...props }, ref) => ( 51 | 56 |
    {children}
    57 |     58 | )); 59 | 60 | AccordionContent.displayName = AccordionPrimitive.Content.displayName; 61 | 62 | export { Accordion, AccordionItem, AccordionTrigger, AccordionContent }; 63 | -------------------------------------------------------------------------------- /src/components/primitive/collapsible.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import type { 3 | CollapsibleContentProps, 4 | CollapsibleProps, 5 | CollapsibleTriggerProps, 6 | } from "@radix-ui/react-collapsible"; 7 | import * as CollapsiblePrimitive from "@radix-ui/react-collapsible"; 8 | import { cn } from "@/lib/utils"; 9 | 10 | const Collapsible = ({ 11 | className, 12 | ...props 13 | }: CollapsibleProps & { 14 | className?: string; 15 | }) => { 16 | const [open, setOpen] = React.useState(false); 17 | 18 | return ( 19 | 25 | ); 26 | }; 27 | 28 | const CollapsibleTrigger = ({ 29 | className, 30 | 31 | ...props 32 | }: CollapsibleTriggerProps & { 33 | className?: string; 34 | }) => { 35 | return ( 36 | 40 | ); 41 | }; 42 | 43 | const CollapsibleContent = ({ 44 | className, 45 | ...props 46 | }: CollapsibleContentProps & { 47 | className?: string; 48 | }) => { 49 | return ( 50 | 51 | ); 52 | }; 53 | 54 | export { Collapsible, CollapsibleContent, CollapsibleTrigger }; 55 | -------------------------------------------------------------------------------- /src/components/primitive/dialog.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import type { 3 | DialogContentProps, 4 | DialogDescriptionProps, 5 | DialogProps, 6 | DialogTitleProps, 7 | DialogTriggerProps, 8 | } from "@radix-ui/react-dialog"; 9 | import * as DialogPrimitive from "@radix-ui/react-dialog"; 10 | import { Cross2Icon } from "@radix-ui/react-icons"; 11 | import { cn } from "@/lib/utils"; 12 | 13 | const Dialog = ({ ...props }: DialogProps & {}) => ( 14 | 15 | ); 16 | 17 | const DialogTrigger = ({ ...props }: DialogTriggerProps & {}) => ( 18 | 19 | ); 20 | 21 | const DialogContent = ({ children, ...props }: DialogContentProps & {}) => ( 22 | 23 | 24 | 25 | 36 | {children} 37 | 38 | 39 | 45 | 46 | 47 | 48 | ); 49 | 50 | const DialogTitle = ({ ...props }: DialogTitleProps & {}) => ( 51 | 52 | ); 53 | 54 | const DialogDescription = ({ ...props }: DialogDescriptionProps & {}) => ( 55 | 56 | ); 57 | 58 | export { Dialog, DialogTrigger, DialogContent, DialogTitle, DialogDescription }; 59 | -------------------------------------------------------------------------------- /src/components/primitive/popover.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import type { 3 | PopoverContentProps, 4 | PopoverProps, 5 | } from "@radix-ui/react-popover"; 6 | import * as PopoverPrimitive from "@radix-ui/react-popover"; 7 | import { cn } from "@/lib/utils"; 8 | 9 | const Popover = ({ ...props }: PopoverProps & {}) => ( 10 | 11 | ); 12 | 13 | const PopoverTrigger = ({ 14 | ...props 15 | }: PopoverProps & { 16 | className?: string; 17 | }) => ; 18 | 19 | const PopoverContent = ({ 20 | className, 21 | children, 22 | ...props 23 | }: PopoverContentProps & { 24 | className?: string; 25 | }) => ( 26 | 27 | 32 | {children} 33 | 34 | 35 | ); 36 | 37 | export { Popover, PopoverTrigger, PopoverContent }; 38 | -------------------------------------------------------------------------------- /src/components/primitive/select.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import type { 3 | SelectContentProps, 4 | SelectItemProps, 5 | SelectProps, 6 | SelectTriggerProps, 7 | SelectValueProps, 8 | } from "@radix-ui/react-select"; 9 | import * as SelectPrimitive from "@radix-ui/react-select"; 10 | import { 11 | CheckIcon, 12 | ChevronDownIcon, 13 | ChevronUpIcon, 14 | } from "@radix-ui/react-icons"; 15 | import { cn } from "@/lib/utils"; 16 | 17 | const Select = ({ 18 | className, 19 | ...props 20 | }: SelectProps & { 21 | className?: string; 22 | }) => ; 23 | 24 | const SelectTrigger = ({ 25 | className, 26 | children, 27 | ...props 28 | }: SelectTriggerProps & { 29 | className?: string; 30 | }) => ( 31 | 40 | {children} 41 | 42 | 43 | 44 | 45 | ); 46 | 47 | const SelectValue = ({ 48 | className, 49 | ...props 50 | }: SelectValueProps & { 51 | className?: string; 52 | }) => ; 53 | 54 | const SelectContent = ({ 55 | className, 56 | children, 57 | ...props 58 | }: SelectContentProps & { 59 | className?: string; 60 | }) => ( 61 | 62 | 66 | 67 | 68 | 69 | {/**/} 70 | 71 | 72 | {children} 73 | 74 | 75 | {/**/} 76 | 77 | 78 | 79 | 80 | 81 | ); 82 | 83 | const SelectItem = ({ 84 | className, 85 | children, 86 | ...props 87 | }: SelectItemProps & { 88 | className?: string; 89 | }) => { 90 | return ( 91 | 102 | {children} 103 | 104 | 105 | 106 | 107 | 108 | ); 109 | }; 110 | 111 | export { Select, SelectTrigger, SelectContent, SelectItem, SelectValue }; 112 | -------------------------------------------------------------------------------- /src/components/primitive/tabs.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import * as TabsPrimitive from "@radix-ui/react-tabs"; 3 | import { 4 | TabsContentProps, 5 | TabsListProps, 6 | TabsProps, 7 | TabsTriggerProps, 8 | } from "@radix-ui/react-tabs"; 9 | import { cn } from "@/lib/utils"; 10 | 11 | const Tabs = ({ 12 | className, 13 | ...props 14 | }: TabsProps & { 15 | className?: string; 16 | }) => ( 17 | 18 | ); 19 | 20 | const TabsList = ({ 21 | className, 22 | ...props 23 | }: TabsListProps & { 24 | className?: string; 25 | }) => ( 26 | 30 | ); 31 | 32 | const TabsTrigger = ({ 33 | className, 34 | ...props 35 | }: TabsTriggerProps & { 36 | className?: string; 37 | }) => ( 38 | 49 | ); 50 | 51 | const TabsContent = ({ 52 | className, 53 | ...props 54 | }: TabsContentProps & { 55 | className?: string; 56 | }) => ; 57 | 58 | export { Tabs, TabsList, TabsTrigger, TabsContent }; 59 | -------------------------------------------------------------------------------- /src/components/primitive/toggle-group.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import * as ToggleGroupPrimitive from "@radix-ui/react-toggle-group"; 3 | import { 4 | ToggleGroupItemProps, 5 | ToggleGroupSingleProps, 6 | } from "@radix-ui/react-toggle-group"; 7 | import { cn } from "@/lib/utils"; 8 | 9 | const ToggleGroup = ({ 10 | className, 11 | ...props 12 | }: ToggleGroupSingleProps & { 13 | className?: string; 14 | }) => { 15 | return ( 16 | 23 | ); 24 | }; 25 | 26 | const toggleGroupItemClasses = 27 | "hover:bg-emerald-50 data-[state=on]:bg-emerald-100 " + 28 | "data-[state=on]:text-emerald-700 flex h-10 w-10 " + 29 | "items-center justify-center bg-white text-base leading-4 " + 30 | "first:rounded-l last:rounded-r focus:z-10"; 31 | 32 | const ToggleGroupItem = ({ 33 | className, 34 | ...props 35 | }: ToggleGroupItemProps & { 36 | className?: string; 37 | }) => { 38 | return ( 39 | 43 | ); 44 | }; 45 | 46 | export { ToggleGroup, ToggleGroupItem }; 47 | -------------------------------------------------------------------------------- /src/components/search-tab.tsx: -------------------------------------------------------------------------------- 1 | import { serverQueryIndex } from "@/lib/actions"; 2 | import { ResultCode } from "@/lib/types"; 3 | import { useMutation } from "@tanstack/react-query"; 4 | import { usePathname, useRouter, useSearchParams } from "next/navigation"; 5 | import { useCallback, useEffect, useMemo, useState } from "react"; 6 | import EmptyState from "./empty"; 7 | import ErrorMessages from "./error"; 8 | import List from "./list"; 9 | import Search from "./search"; 10 | import { Info } from "@/components/info"; 11 | import { useFetchInfo } from "@/lib/use-fetch-info"; 12 | import { formatter } from "@/lib/utils"; 13 | 14 | const emptyState = { 15 | data: [], 16 | code: ResultCode.Empty, 17 | }; 18 | 19 | export const SearchTab = () => { 20 | const { data: info } = useFetchInfo(); 21 | 22 | const [search, setSearch] = useState(""); 23 | const [searchParam, setSearchParam] = useQuerySearchParam(); 24 | const [isInitial, setIsInitial] = useState(true); 25 | 26 | const { 27 | data, 28 | mutate: fetchResults, 29 | isPending: isLoading, 30 | reset, 31 | } = useMutation({ 32 | mutationFn: async (query: string) => await serverQueryIndex(query), 33 | }); 34 | 35 | const groupedData = useMemo(() => { 36 | const results = data?.data; 37 | if (!results) return []; 38 | 39 | // Group by url 40 | const map = new Map(); 41 | for (const result of results) { 42 | if (!result.metadata?.url || map.has(result.metadata.url)) continue; 43 | 44 | map.set(result.metadata?.url, result); 45 | } 46 | 47 | return Array.from(map.values()).sort((a, b) => b.score - a.score); 48 | }, [data]); 49 | 50 | const state = data ?? emptyState; 51 | 52 | useEffect(() => { 53 | if (!isInitial) return; 54 | setIsInitial(false); 55 | if (searchParam) fetchResults(searchParam); 56 | setSearch(searchParam); 57 | }, [searchParam, isInitial]); 58 | 59 | useEffect(() => { 60 | if (!searchParam) reset(); 61 | }, [searchParam]); 62 | 63 | const onSubmit = () => { 64 | setSearchParam(search); 65 | fetchResults(search); 66 | }; 67 | 68 | return ( 69 |
    70 | 76 | 77 |
    78 | 79 |
    80 | 81 |
    82 | { 86 | setSearch(query); 87 | setSearchParam(query); 88 | fetchResults(query); 89 | }} 90 | /> 91 |
    92 | 93 |
    94 | 100 |
    101 | 102 | {!isLoading && ( 103 | 104 |

    105 | This project is an experiment to demonstrate the scalability of 106 | Upstash Vector with large datasets. We vectorized{" "} 107 | 23M Wikipedia articles in 11 languages and stored{" "} 108 | {info ? formatter.format(info.vectorCount) : "..."} vectors{" "} 109 | in a single Upstash Vector index. 110 |

    111 | 112 |

    113 | 114 | 👉 Check out the{" "} 115 | 120 | github repo 121 | {" "} 122 | or the{" "} 123 | 128 | blog post 129 | {" "} 130 | for more. 131 | 132 |

    133 |     134 | )} 135 |
    136 | ); 137 | }; 138 | 139 | const useQuerySearchParam = () => { 140 | const searchParams = useSearchParams(); 141 | const router = useRouter(); 142 | const pathname = usePathname(); 143 | 144 | const state = searchParams.get("query") ?? ""; 145 | 146 | const setState = useCallback( 147 | (state: string) => { 148 | if (!state) router.push(pathname); 149 | else 150 | router.push( 151 | `${pathname}?${new URLSearchParams({ 152 | query: state, 153 | })}`, 154 | ); 155 | }, 156 | [searchParams], 157 | ); 158 | 159 | return [state, setState] as const; 160 | }; 161 | -------------------------------------------------------------------------------- /src/components/search.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { cn } from "@/lib/utils"; 4 | import LocaleSelect from "@/components/locale-select"; 5 | import { PaperPlaneIcon } from "@radix-ui/react-icons"; 6 | 7 | export default function Search({ 8 | isLoading, 9 | value, 10 | onChange, 11 | onSubmit = () => {}, 12 | }: { 13 | isLoading: boolean; 14 | value: string; 15 | onChange: (value: string) => void; 16 | onSubmit: () => void; 17 | }) { 18 | return ( 19 |
    { 21 | e.preventDefault(); 22 | onSubmit(); 23 | }} 24 | className="flex gap-2 items-center" 25 | > 26 | onChange(e.target.value)} 31 | placeholder="Ask a question..." 32 | disabled={isLoading} 33 | className="border placeholder:text-yellow-950/50 border-yellow-700/20 rounded-md px-4 h-10 w-full focus:border-yellow-950 outline-none ring-0" 34 | /> 35 | 36 | 37 | 38 | 48 | 49 | ); 50 | } 51 | -------------------------------------------------------------------------------- /src/i18n.ts: -------------------------------------------------------------------------------- 1 | import { getRequestConfig } from "next-intl/server"; 2 | import { getUserLocale } from "./service"; 3 | 4 | const supportedLanguages = [ 5 | "de", 6 | "en", 7 | "es", 8 | "fa", 9 | "fr", 10 | "it", 11 | "ja", 12 | "pt", 13 | "ru", 14 | "tr", 15 | "zn", 16 | ]; 17 | 18 | export default getRequestConfig(async () => { 19 | const userLocale = await getUserLocale(); 20 | 21 | const locale = supportedLanguages.includes(userLocale) ? userLocale : "en"; 22 | 23 | console.log(`User locale: ${locale}`); 24 | 25 | return { 26 | locale: locale, 27 | messages: (await import(`../messages/${locale}.json`)).default, 28 | }; 29 | }); 30 | -------------------------------------------------------------------------------- /src/lib/actions.ts: -------------------------------------------------------------------------------- 1 | "use server"; 2 | 3 | import { z } from "zod"; 4 | import { cookies } from "next/headers"; 5 | import { getUserLocale } from "@/service"; 6 | import { openai, UpstashMessage } from "@upstash/rag-chat"; 7 | import { Info, ResultCode, WikiMetadata } from "@/lib/types"; 8 | import { index } from "./dbs"; 9 | import { MessageMetadata } from "./message-meta"; 10 | import { ragChat } from "./rag-chat"; 11 | 12 | export async function serverGetMessages() { 13 | const sessionId = cookies().get("sessionId")?.value; 14 | 15 | if (!sessionId) throw new Error("No sessionId found"); 16 | 17 | const messages = (await ragChat.history.getMessages({ 18 | sessionId: sessionId, 19 | amount: 10, 20 | })) as UpstashMessage[]; 21 | 22 | return messages; 23 | } 24 | 25 | export async function serverClearMessages() { 26 | const sessionId = cookies().get("sessionId")?.value; 27 | 28 | if (!sessionId) throw new Error("No sessionId found"); 29 | 30 | await ragChat.history.deleteMessages({ sessionId }); 31 | } 32 | 33 | const capitalizeWord = (word: string) => { 34 | return word.charAt(0).toUpperCase() + word.slice(1); 35 | }; 36 | 37 | async function getKeywords(query: string) { 38 | const resp = await openai("gpt-4-turbo", { 39 | apiKey: process.env.OPENAI_API_KEY!, 40 | }).invoke(` 41 | Please provide a list of keywords about the question given in JSON format. 42 | Don't answer with anything else. 43 | 44 | EXAMPLE INPUT: 45 | Ghandi 46 | 47 | EXAMPLE OUTPUT: 48 | ["Ghandi", "India", "peace", "leader", "non-violence", "freedom"] 49 | 50 | INPUT: 51 | ${query.split(" ").map(capitalizeWord).join(" ")} 52 | 53 | OUTPUT: 54 | `); 55 | 56 | console.log(resp); 57 | 58 | try { 59 | // @ts-ignore 60 | return JSON.parse(resp.content) as string[]; 61 | } catch (error) { 62 | console.error("Error parsing keywords, prompt:", resp.content); 63 | return undefined; 64 | } 65 | } 66 | 67 | export async function serverQueryIndex(query: string) { 68 | try { 69 | const keywords = await getKeywords(query); 70 | console.log("query: ", query, "keywords: ", keywords); 71 | 72 | if (keywords && keywords.length > 0) 73 | query = query + " " + keywords.join(" "); 74 | 75 | const namespace = await getUserLocale(); 76 | const parsedCredentials = z 77 | .object({ 78 | query: z.string().min(2), 79 | }) 80 | .required() 81 | .safeParse({ 82 | query, 83 | }); 84 | 85 | if (parsedCredentials.error) { 86 | return { 87 | code: ResultCode.MinLengthError, 88 | data: [], 89 | }; 90 | } 91 | 92 | const q = { 93 | data: query as string, 94 | topK: 100, 95 | includeData: true, 96 | includeVectors: false, 97 | includeMetadata: true, 98 | }; 99 | 100 | const t0 = performance.now(); 101 | const data = await index.query(q, { namespace }); 102 | const t1 = performance.now(); 103 | const ms = t1 - t0; 104 | 105 | return { 106 | code: ResultCode.Success, 107 | data, 108 | ms, 109 | }; 110 | } catch (error) { 111 | console.error("Error querying Upstash:", error); 112 | return { 113 | code: ResultCode.UnknownError, 114 | data: [], 115 | }; 116 | } 117 | } 118 | 119 | export async function serverGetInfo(): Promise { 120 | try { 121 | const data = await index.info(); 122 | return data; 123 | } catch (error) { 124 | console.error("Error querying Upstash:", error); 125 | return undefined; 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/lib/dbs.ts: -------------------------------------------------------------------------------- 1 | import { Redis } from "@upstash/redis"; 2 | import { Index } from "@upstash/vector"; 3 | import { WikiMetadata } from "./types"; 4 | 5 | export const index = new Index(); 6 | export const redis = Redis.fromEnv(); 7 | -------------------------------------------------------------------------------- /src/lib/message-meta.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | 3 | export type MessageMetadata = z.infer; 4 | 5 | export const messageMetadataSchema = z.object({ 6 | usedHistory: z.array( 7 | z.object({ 8 | role: z.string(), 9 | content: z.string(), 10 | }), 11 | ), 12 | usedContext: z.array( 13 | z.object({ 14 | url: z.string(), 15 | data: z.string(), 16 | }), 17 | ), 18 | usedPrompt: z.string(), 19 | }); 20 | -------------------------------------------------------------------------------- /src/lib/rag-chat.ts: -------------------------------------------------------------------------------- 1 | import { RAGChat, openai } from "@upstash/rag-chat"; 2 | import { index, redis } from "./dbs"; 3 | 4 | export const ragChat = new RAGChat({ 5 | model: openai("gpt-4-turbo", { 6 | apiKey: process.env.OPENAI_API_KEY!, 7 | }), 8 | vector: index, 9 | redis: redis, 10 | debug: false, 11 | promptFn: ({ chatHistory, context, question }) => { 12 | return PROMPT.replace("{chatHistory}", chatHistory ?? "") 13 | .replace("{context}", context) 14 | .replace("{question}", question); 15 | }, 16 | }); 17 | 18 | export const PROMPT = `You are a friendly AI assistant augmented with an Upstash Vector Store that contains embeddings from wikipedia. 19 | To help you answer the questions, a context and chat history will be provided. 20 | Answer the question at the end using only the information available in the context or chat history, either one is ok. 21 | 22 | ------------- 23 | Chat history: 24 | {chatHistory} 25 | ------------- 26 | Context: 27 | {context} 28 | ------------- 29 | 30 | Question: {question} 31 | Helpful answer:`; 32 | -------------------------------------------------------------------------------- /src/lib/types.ts: -------------------------------------------------------------------------------- 1 | export type WikiMetadata = { 2 | id: string; 3 | url: string; 4 | title: string; 5 | }; 6 | 7 | export type Wiki = { 8 | id: number | string; 9 | score: number; 10 | metadata?: WikiMetadata | undefined; 11 | data?: string; 12 | }; 13 | 14 | export enum ResultCode { 15 | Empty = "EMPTY", 16 | Success = "SUCCESS", 17 | UnknownError = "UNKNOWN_ERROR", 18 | MinLengthError = "MIN_LENGTH_ERROR", 19 | } 20 | 21 | export interface Result { 22 | code: ResultCode; 23 | data: Wiki[]; 24 | ms?: number; 25 | } 26 | 27 | interface NamespaceData { 28 | vectorCount: number; 29 | pendingVectorCount: number; 30 | } 31 | 32 | export interface Info { 33 | vectorCount: number; 34 | pendingVectorCount: number; 35 | indexSize: number; 36 | dimension: number; 37 | similarityFunction: string; 38 | namespaces: { 39 | [key: string]: NamespaceData; 40 | }; 41 | } 42 | -------------------------------------------------------------------------------- /src/lib/use-fetch-info.tsx: -------------------------------------------------------------------------------- 1 | import { useQuery } from "@tanstack/react-query"; 2 | import { serverGetInfo } from "./actions"; 3 | 4 | export const useFetchInfo = () => { 5 | return useQuery({ 6 | queryKey: ["info"], 7 | queryFn: async () => await serverGetInfo(), 8 | }); 9 | }; 10 | -------------------------------------------------------------------------------- /src/lib/utils.ts: -------------------------------------------------------------------------------- 1 | import { type ClassValue, clsx } from "clsx"; 2 | import { twMerge } from "tailwind-merge"; 3 | 4 | export function cn(...inputs: ClassValue[]) { 5 | return twMerge(clsx(inputs)); 6 | } 7 | 8 | export const formatter = Intl.NumberFormat("en", { notation: "compact" }); 9 | -------------------------------------------------------------------------------- /src/middleware.ts: -------------------------------------------------------------------------------- 1 | import { NextRequest, NextResponse } from "next/server"; 2 | 3 | export function middleware(req: NextRequest) { 4 | const res = NextResponse.next(); 5 | 6 | const cookie = req.cookies.get("sessionId"); 7 | 8 | if (!cookie) { 9 | res.cookies.set("sessionId", crypto.randomUUID()); 10 | } 11 | 12 | return res; 13 | } 14 | -------------------------------------------------------------------------------- /src/service.ts: -------------------------------------------------------------------------------- 1 | "use server"; 2 | 3 | import { cookies } from "next/headers"; 4 | 5 | export type Locale = (typeof locales)[number]; 6 | 7 | const locales = [ 8 | "de", 9 | "en", 10 | "es", 11 | "fa", 12 | "fr", 13 | "it", 14 | "ja", 15 | "pt", 16 | "ru", 17 | "tr", 18 | "zh", 19 | ] as const; 20 | 21 | const defaultLocale: Locale = "en"; 22 | 23 | const COOKIE_NAME = "NEXT_LOCALE"; 24 | 25 | export async function getUserLocale() { 26 | return cookies().get(COOKIE_NAME)?.value || defaultLocale; 27 | } 28 | 29 | export async function setUserLocale(locale: Locale) { 30 | cookies().set(COOKIE_NAME, locale); 31 | } 32 | -------------------------------------------------------------------------------- /tailwind.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from "tailwindcss"; 2 | 3 | const config: Config = { 4 | darkMode: "class", 5 | content: [ 6 | "./src/pages/**/*.{js,ts,jsx,tsx,mdx}", 7 | "./src/components/**/*.{js,ts,jsx,tsx,mdx}", 8 | "./src/app/**/*.{js,ts,jsx,tsx,mdx}", 9 | ], 10 | theme: { 11 | extend: { 12 | fontFamily: { 13 | sans: ["var(--font-sans)"], 14 | serif: ["var(--font-serif)"], 15 | }, 16 | keyframes: { 17 | "accordion-down": { 18 | from: { height: "0" }, 19 | to: { height: "var(--radix-accordion-content-height)" }, 20 | }, 21 | "accordion-up": { 22 | from: { height: "var(--radix-accordion-content-height)" }, 23 | to: { height: "0" }, 24 | }, 25 | }, 26 | animation: { 27 | "accordion-down": "accordion-down 0.2s ease-out", 28 | "accordion-up": "accordion-up 0.2s ease-out", 29 | }, 30 | }, 31 | }, 32 | plugins: [require("tailwindcss-animate")], 33 | }; 34 | export default config; 35 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["dom", "dom.iterable", "esnext"], 4 | "allowJs": true, 5 | "skipLibCheck": true, 6 | "strict": true, 7 | "noEmit": true, 8 | "esModuleInterop": true, 9 | "module": "esnext", 10 | "moduleResolution": "bundler", 11 | "resolveJsonModule": true, 12 | "isolatedModules": true, 13 | "jsx": "preserve", 14 | "incremental": true, 15 | "plugins": [ 16 | { 17 | "name": "next" 18 | } 19 | ], 20 | "paths": { 21 | "@/*": ["./src/*"] 22 | } 23 | }, 24 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], 25 | "exclude": ["node_modules"] 26 | } 27 | --------------------------------------------------------------------------------