├── .dockerignore ├── .env.example ├── .gitignore ├── .vscode └── settings.json ├── Dockerfile ├── README.md ├── app ├── components │ ├── documents │ │ └── upload.tsx │ ├── icons │ │ ├── alert-circle.tsx │ │ ├── alert-triangle.tsx │ │ ├── bot.tsx │ │ ├── calendar.tsx │ │ ├── chat-bubble.tsx │ │ ├── check-circle.tsx │ │ ├── check.tsx │ │ ├── chevron-down.tsx │ │ ├── chevron-left.tsx │ │ ├── chevron-right.tsx │ │ ├── cross.tsx │ │ ├── file-upload.tsx │ │ ├── github.tsx │ │ ├── google.tsx │ │ ├── info.tsx │ │ ├── list-down.tsx │ │ ├── list-up.tsx │ │ ├── minus.tsx │ │ ├── plus.tsx │ │ ├── question-circle.tsx │ │ ├── search.tsx │ │ ├── send.tsx │ │ ├── spinner.tsx │ │ └── user.tsx │ ├── layout │ │ └── navbar.tsx │ ├── misc │ │ └── error-boundary.tsx │ └── ui │ │ ├── avatar.tsx │ │ ├── button.tsx │ │ ├── checkbox.tsx │ │ ├── constants.ts │ │ ├── dropzone.tsx │ │ ├── file-trigger.tsx │ │ ├── heading.tsx │ │ ├── input.tsx │ │ ├── menu.tsx │ │ ├── text.tsx │ │ └── toast.tsx ├── entry.client.tsx ├── entry.server.tsx ├── lib │ ├── auth.ts │ ├── auth │ │ └── session.server.ts │ ├── constants.ts │ ├── db │ │ ├── config.ts │ │ ├── index.ts │ │ ├── migrations │ │ │ ├── 0000_colorful_tusk.sql │ │ │ ├── 0001_charming_stick.sql │ │ │ └── meta │ │ │ │ ├── 0000_snapshot.json │ │ │ │ ├── 0001_snapshot.json │ │ │ │ └── _journal.json │ │ ├── schema.ts │ │ └── utils │ │ │ └── generate-id.ts │ ├── env.ts │ ├── utils │ │ ├── cn.ts │ │ └── get-error-message.ts │ └── vector-db │ │ ├── api-schema.ts │ │ ├── config.ts │ │ ├── index.ts │ │ ├── migrations │ │ ├── 0000_fair_hobgoblin.sql │ │ └── meta │ │ │ ├── 0000_snapshot.json │ │ │ └── _journal.json │ │ └── schema.ts ├── root.tsx ├── routes │ ├── $.tsx │ ├── _index.tsx │ ├── api.auth.google.callback.tsx │ ├── api.auth.google.tsx │ ├── api.auth.logout.tsx │ ├── api.document.chat.tsx │ ├── api.document.upload.tsx │ ├── api.document.vectorize.tsx │ ├── document.new.tsx │ ├── documents.tsx │ ├── documents_.$id.chat.tsx │ └── login.tsx └── styles │ └── globals.css ├── biome.json ├── bun.lockb ├── migrate.ts ├── package.json ├── postcss.config.js ├── public ├── _headers └── favicon.ico ├── tailwind.config.ts ├── tsconfig.json └── vite.config.ts /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | 3 | /.cache 4 | /build 5 | .env 6 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | CLOUDFLARE_R2_ENDPOINT= 2 | CLOUDFLARE_R2_ACCESS_ID= 3 | CLOUDFLARE_R2_SECRET_KEY= 4 | CLOUDFLARE_R2_BUCKET_NAME= 5 | CLOUDFLARE_R2_ACCOUNT_ID= 6 | 7 | SESSION_SECRET= 8 | GOOGLE_CLIENT_ID= 9 | GOOGLE_CLIENT_SECRET= 10 | GOOGLE_CALLBACK_URL=http://localhost:3000/api/auth/google/callback 11 | 12 | NEON_API_KEY= 13 | 14 | DATABASE_URL= 15 | 16 | OPENAI_API_KEY= -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | 3 | /.cache 4 | /build 5 | .env 6 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript.preferences.autoImportFileExcludePatterns": [ 3 | "react-aria-components", 4 | "postcss", 5 | "drizzle-orm/mysql-core" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1 2 | 3 | # Adjust BUN_VERSION as desired 4 | ARG BUN_VERSION=1.1.28 5 | FROM oven/bun:${BUN_VERSION}-slim as base 6 | 7 | # Remix app lives here 8 | WORKDIR /app 9 | 10 | # Set production environment 11 | ENV NODE_ENV="production" 12 | 13 | # Throw-away build stage to reduce size of final image 14 | FROM base as build 15 | 16 | # Install packages needed to build node modules 17 | RUN apt-get update -qq && \ 18 | apt-get install --no-install-recommends -y \ 19 | build-essential \ 20 | pkg-config \ 21 | python-is-python3 \ 22 | libcairo2-dev \ 23 | libpango1.0-dev \ 24 | libjpeg-dev \ 25 | libgif-dev \ 26 | librsvg2-dev \ 27 | libpixman-1-dev 28 | 29 | # Install node modules 30 | COPY --link bun.lockb package.json ./ 31 | # Add this line to use a pre-built version of canvas 32 | RUN echo "canvas_binary_host_mirror=https://github.com/Automattic/node-canvas/releases/download/" > .npmrc 33 | RUN bun install 34 | 35 | # Copy application code 36 | COPY --link . . 37 | 38 | # Build application 39 | RUN bun --bun run build 40 | 41 | # Remove development dependencies 42 | RUN rm -rf node_modules && \ 43 | bun install --ci 44 | 45 | # Final stage for app image 46 | FROM base 47 | 48 | # Install runtime dependencies 49 | RUN apt-get update -qq && \ 50 | apt-get install --no-install-recommends -y \ 51 | libcairo2 \ 52 | libpango-1.0-0 \ 53 | libjpeg62-turbo \ 54 | libgif7 \ 55 | librsvg2-2 \ 56 | libpixman-1-0 && \ 57 | rm -rf /var/lib/apt/lists/* 58 | 59 | # Copy built application 60 | COPY --from=build /app /app 61 | 62 | # Start the server by default, this can be overwritten at runtime 63 | EXPOSE 3000 64 | CMD [ "bun", "run", "start" ] 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## AI app architecture: vector database per tenant 2 | 3 | This repo contains an example of a scalable architecture for AI-powered applications. On the surface, it's an AI app where users can upload PDFs and chat with them. However, under the hood, each user gets a dedicated vector database instance (Postgres on [Neon](https://neon.tech/?ref=github) with pgvector). 4 | 5 | You can check out the live version at https://db-per-tenant.up.railway.app/ 6 | 7 | ![Demo app](https://github.com/user-attachments/assets/d9dee48f-a6d6-4dd5-bb89-fa5d31ca26e3) 8 | 9 | The app is built using the following technologies: 10 | 11 | - [Neon](https://neon.tech/ref=github) - Fully managed Postgres 12 | - [Remix](https://remix.run) - Full-stack React framework 13 | - [Remix Auth](https://github.com/sergiodxa/remix-auth) - Authentication 14 | - [Drizzle ORM](https://drizzle.team/) - TypeScript ORM 15 | - [Railway](https://railway.app) - Deployment Platform 16 | - [Vercel AI SDK](sdk.vercel.ai/) - TypeScript toolkit for building AI-powered applications 17 | - [Cloudflare R2](https://www.cloudflare.com/developer-platform/r2/) - Object storage 18 | - [OpenAI](https://openai.com) with gpt-4o-mini - LLM 19 | - [Upstash](https://upstash.com) - Redis for rate limiting 20 | - [Langchain](https://js.langchain.com/v0.2/docs/introduction/) - Framework for developing applications powered by large language models (LLMs) 21 | 22 | ## How it works 23 | 24 | Rather than having all vector embeddings stored in a single Postgres database, you provide each tenant (a user, an organization, a workspace, or any other entity requiring isolation) with its own dedicated Postgres database instance where you can store and query its embeddings. 25 | 26 | Depending on your application, you will provision a vector database after a specific event (e.g., user signup, organization creation, or upgrade to paid tier). You will then track tenants and their associated vector databases in your application's main database. 27 | 28 | This approach offers several benefits: 29 | 1. Each tenant's data is stored in a separate, isolated database not shared with other tenants. This makes it possible for you to be compliant with data residency requirements (e.g., GDPR) 30 | 2. Database resources can be allocated based on each tenant's requirements. 31 | 3. A tenant with a large workload that can impact the database's performance won't affect other tenants; it would also be easier to manage. 32 | 33 | Here's the database architecture diagram of the demo app that's in this repo: 34 | 35 | ![Architecture Diagram](https://github.com/user-attachments/assets/c788d581-1d0a-4201-842e-a20bd498e3db) 36 | 37 | The main application's database consists of three tables: `documents`, `users`, and `vector_databases`. 38 | 39 | - The `documents` table stores information about files, including their titles, sizes, and timestamps, and is linked to users via a foreign key. 40 | - The `users` table maintains user profiles, including names, emails, and avatar URLs. 41 | - The `vector_databases` table tracks which vector database belongs to which user. 42 | 43 | Then, each vector database that gets provisioned has an `embeddings` table for storing document chunks for retrieval-augmented generation (RAG). 44 | 45 | For this app, vector databases are provisioned when a user signs up. Once they upload a document, it gets chunked and stored in their dedicated vector database. Finally, once the user chats with their document, the vector similarity search runs against their database to retrieve the relevant information to answer their prompt. 46 | 47 |
48 | Code snippet example of provisioning a vector database 49 | 50 | ![Provision Vector database for each signup](https://github.com/user-attachments/assets/01e31752-cddb-45c5-b595-92c3cb815a88) 51 | 52 | ```ts 53 | // Code from app/lib/auth.ts 54 | 55 | authenticator.use( 56 | new GoogleStrategy( 57 | { 58 | clientID: process.env.GOOGLE_CLIENT_ID, 59 | clientSecret: process.env.GOOGLE_CLIENT_SECRET, 60 | callbackURL: process.env.GOOGLE_CALLBACK_URL, 61 | }, 62 | async ({ profile }) => { 63 | const email = profile.emails[0].value; 64 | 65 | try { 66 | const userData = await db 67 | .select({ 68 | user: users, 69 | vectorDatabase: vectorDatabases, 70 | }) 71 | .from(users) 72 | .leftJoin(vectorDatabases, eq(users.id, vectorDatabases.userId)) 73 | .where(eq(users.email, email)); 74 | 75 | if ( 76 | userData.length === 0 || 77 | !userData[0].vectorDatabase || 78 | !userData[0].user 79 | ) { 80 | const { data, error } = await neonApiClient.POST("/projects", { 81 | body: { 82 | project: {}, 83 | }, 84 | }); 85 | 86 | if (error) { 87 | throw new Error(`Failed to create Neon project, ${error}`); 88 | } 89 | 90 | const vectorDbId = data?.project.id; 91 | 92 | const vectorDbConnectionUri = data.connection_uris[0]?.connection_uri; 93 | 94 | const sql = postgres(vectorDbConnectionUri); 95 | 96 | await sql`CREATE EXTENSION IF NOT EXISTS vector;`; 97 | 98 | await migrate(drizzle(sql), { migrationsFolder: "./drizzle" }); 99 | 100 | const newUser = await db 101 | .insert(users) 102 | .values({ 103 | email, 104 | name: profile.displayName, 105 | avatarUrl: profile.photos[0].value, 106 | userId: generateId({ object: "user" }), 107 | }) 108 | .onConflictDoNothing() 109 | .returning(); 110 | 111 | await db 112 | .insert(vectorDatabases) 113 | .values({ 114 | vectorDbId, 115 | userId: newUser[0].id, 116 | }) 117 | .returning(); 118 | 119 | const result = { 120 | ...newUser[0], 121 | vectorDbId, 122 | }; 123 | 124 | return result; 125 | } 126 | 127 | return { 128 | ...userData[0].user, 129 | vectorDbId: userData[0].vectorDatabase.vectorDbId, 130 | }; 131 | } catch (error) { 132 | console.error("User creation error:", error); 133 | throw new Error(getErrorMessage(error)); 134 | } 135 | }, 136 | ), 137 | ); 138 | 139 | ``` 140 |
141 | 142 | 143 |
144 | Code snippet and diagram of RAG 145 | 146 | ![Vector database per tenant RAG](https://github.com/user-attachments/assets/43e0f872-6bab-4a06-8208-7871723f1fd0) 147 | 148 | ```ts 149 | // Code from app/routes/api/document/chat 150 | // Get the user's messages and the document ID from the request body. 151 | const { 152 | messages, 153 | documentId, 154 | }: { 155 | messages: Message[]; 156 | documentId: string; 157 | } = await request.json(); 158 | 159 | const { content: prompt } = messages[messages.length - 1]; 160 | 161 | const { data, error } = await neonApiClient.GET( 162 | "/projects/{project_id}/connection_uri", 163 | { 164 | params: { 165 | path: { 166 | project_id: user.vectorDbId, 167 | }, 168 | query: { 169 | role_name: "neondb_owner", 170 | database_name: "neondb", 171 | }, 172 | }, 173 | }, 174 | ); 175 | 176 | if (error) { 177 | return json({ 178 | error: error, 179 | }); 180 | } 181 | 182 | const embeddings = new OpenAIEmbeddings({ 183 | apiKey: process.env.OPENAI_API_KEY, 184 | dimensions: 1536, 185 | model: "text-embedding-3-small", 186 | }); 187 | 188 | const vectorStore = await NeonPostgres.initialize(embeddings, { 189 | connectionString: data.uri, 190 | tableName: "embeddings", 191 | columns: { 192 | contentColumnName: "content", 193 | metadataColumnName: "metadata", 194 | vectorColumnName: "embedding", 195 | }, 196 | }); 197 | 198 | const result = await vectorStore.similaritySearch(prompt, 2, { 199 | documentId, 200 | }); 201 | 202 | const model = new ChatOpenAI({ 203 | apiKey: process.env.OPENAI_API_KEY, 204 | model: "gpt-4o-mini", 205 | temperature: 0, 206 | }); 207 | 208 | const allMessages = messages.map((message) => 209 | message.role === "user" 210 | ? new HumanMessage(message.content) 211 | : new AIMessage(message.content), 212 | ); 213 | 214 | const systemMessage = new SystemMessage( 215 | `You are a helpful assistant, here's some extra additional context that you can use to answer questions. Only use this information if it's relevant: 216 | 217 | ${result.map((r) => r.pageContent).join(" ")}`, 218 | ); 219 | 220 | allMessages.push(systemMessage); 221 | 222 | const stream = await model.stream(allMessages); 223 | 224 | return LangChainAdapter.toDataStreamResponse(stream); 225 | ``` 226 |
227 | 228 | 229 | While this approach is beneficial, it can also be challenging to implement. You need to manage each database's lifecycle, including provisioning, scaling, and de-provisioning. Fortunately, Postgres on Neon is set up differently: 230 | 231 | 1. Postgres on Neon can be provisioned via the in ~2 seconds, making provisioning a Postgres database for every tenant possible. You don't need to wait several minutes for the database to be ready. 232 | 2. The database's compute can automatically scale up to meet an application's workload and can shut down when the database is unused. 233 | 234 | https://github.com/user-attachments/assets/96500fc3-3efa-4cfa-9339-81eb359ff105 235 | 236 | ![Autoscaling on Neon](https://github.com/user-attachments/assets/7f093ead-d51b-46bc-a473-0df483d91c18) 237 | 238 | This makes the proposed pattern of creating a database per tenant not only possible but also cost-effective. 239 | 240 | ## Managing migrations 241 | 242 | When you have a database per tenant, you need to manage migrations for each database. This project uses [Drizzle](https://drizzle.team/): 243 | 1. The schema is defined in `/app/lib/vector-db/schema.ts` using TypeScript. 244 | 2. Migrations are then generated by running `bun run vector-db:generate`, and stored in `/app/lib/vector-db/migrations`. 245 | 3. Finally, to migrate all databases, you can run `bun run vector-db:migrate`. This command will run a script that connects to each tenant's database and applies the migrations. 246 | 247 | It's important to note that any schema changes you would like to introduce should be backward-compatible. Otherwise, you would need to handle schema migrations differently. 248 | 249 | ## Conclusion 250 | 251 | While this pattern is useful in building AI applications, you can simply use it to provide each tenant with its own database. You could also use a database other than Postgres for your main application's database (e.g., MySQL, MongoDB, MSSQL server, etc.). 252 | 253 | If you have any questions, feel free to reach out to in the [Neon Discord](https://neon.tech/discord) or contact the [Neon Sales team](https://neon.tech/contact-sales). We'd love to hear from you. 254 | 255 | 256 | -------------------------------------------------------------------------------- /app/components/documents/upload.tsx: -------------------------------------------------------------------------------- 1 | import { useMutation } from "@tanstack/react-query"; 2 | import { useState } from "react"; 3 | import { FileUpload } from "../icons/file-upload"; 4 | import { Button } from "../ui/button"; 5 | import { DropZone } from "../ui/dropzone"; 6 | import { FileTrigger } from "../ui/file-trigger"; 7 | import { Heading } from "../ui/heading"; 8 | import { Text } from "../ui/text"; 9 | import { useNavigate } from "@remix-run/react"; 10 | import { toast } from "sonner"; 11 | import type { FileDropItem } from "react-aria"; 12 | import { MAX_FILE_SIZE } from "../../lib/constants"; 13 | import { Spinner } from "../icons/spinner"; 14 | import { CheckCircle } from "../icons/check-circle"; 15 | 16 | export const Upload = () => { 17 | const navigate = useNavigate(); 18 | const [file, setFile] = useState(); 19 | 20 | const upload = useMutation({ 21 | mutationFn: async (file: File) => { 22 | if (file.size > MAX_FILE_SIZE) { 23 | throw new Error("File size exceeds the maximum limit of 10MB"); 24 | } 25 | 26 | const res = await fetch("/api/document/upload", { 27 | method: "POST", 28 | headers: { 29 | "Content-Type": "application/json", 30 | }, 31 | body: JSON.stringify({ 32 | filename: file.name, 33 | fileSize: file.size, 34 | }), 35 | }); 36 | 37 | if (!res.ok) { 38 | const errorData = await res.json(); 39 | throw new Error(errorData.error); 40 | } 41 | 42 | return res.json(); 43 | }, 44 | onSuccess: async (data, file) => { 45 | const uploadRes = await fetch(data.url, { 46 | method: "PUT", 47 | body: file, 48 | }); 49 | 50 | if (!uploadRes.ok) { 51 | throw new Error( 52 | "An error occurred while uploading the file to storage", 53 | ); 54 | } 55 | }, 56 | onError: (error: Error) => { 57 | console.error("Upload error:", error); 58 | toast.error(`Upload failed: ${error.message}`); 59 | }, 60 | }); 61 | 62 | const ingest = useMutation({ 63 | mutationFn: async ({ 64 | filename, 65 | title, 66 | }: { 67 | filename: string; 68 | title: string; 69 | }) => { 70 | const res = await fetch("/api/document/vectorize", { 71 | method: "POST", 72 | headers: { 73 | "Content-Type": "application/json", 74 | }, 75 | body: JSON.stringify({ filename, title }), 76 | }); 77 | 78 | if (!res.ok) { 79 | const errorData = await res.json().catch(() => ({})); 80 | throw new Error( 81 | errorData.message || "An error occurred while ingesting the file", 82 | ); 83 | } 84 | 85 | return res.json(); 86 | }, 87 | onError: (error: Error) => { 88 | console.error("Ingest error:", error); 89 | toast.error(`File processing failed: ${error.message}`); 90 | }, 91 | }); 92 | 93 | const handleSubmit = async (file: File) => { 94 | if (!file) { 95 | toast.error("No file selected"); 96 | return; 97 | } 98 | 99 | try { 100 | const uploadResult = await upload.mutateAsync(file); 101 | const ingestResult = await ingest.mutateAsync({ 102 | filename: uploadResult.filename, 103 | title: uploadResult.title, 104 | }); 105 | navigate(`/documents/${ingestResult.document.documentId}/chat`); 106 | } catch (error) { 107 | console.error("Submit error:", error); 108 | // Error is already handled in the respective mutation's onError 109 | } 110 | }; 111 | 112 | const handleFileSelection = async (selectedFile: File) => { 113 | if (selectedFile.type !== "application/pdf") { 114 | toast.error("Only PDF files are allowed"); 115 | return; 116 | } 117 | setFile(selectedFile); 118 | await handleSubmit(selectedFile); 119 | }; 120 | 121 | return ( 122 | <> 123 | {!file ? ( 124 | <> 125 |

126 | Download sample file -{" "} 127 | 133 | FORM 10-K - Apple Inc.{" "} 134 | 135 |

136 | { 138 | const files = e.items.filter( 139 | (file) => file.kind === "file", 140 | ) as FileDropItem[]; 141 | 142 | if (files && files.length > 0) { 143 | const file = await files[0].getFile(); 144 | await handleFileSelection(file); 145 | } 146 | }} 147 | className="w-full gap-y-5 min-h-[50vh] flex items-center justify-center p-10" 148 | > 149 | { 153 | if (files && files.length > 0) { 154 | await handleFileSelection(files[0]); 155 | } 156 | }} 157 | > 158 | 159 |

Drag & drop your PDF file here

160 |

or

161 | 164 | 165 | Maximum PDF file size is 10MB 166 | 167 | 168 | 169 | ) : ( 170 |
171 | File Selected 172 | {file.name} 173 | {upload.isPending && ( 174 |

175 | Uploading 176 |

177 | )} 178 | {upload.isSuccess && ( 179 |

180 | File uploaded successfully 181 |

182 | )} 183 | {upload.isError && ( 184 |

185 | Upload failed: {upload.error.message} 186 |

187 | )} 188 | 189 | {ingest.isPending && ( 190 |

191 | Processing 192 |

193 | )} 194 | {ingest.isSuccess && ( 195 |

196 | File ingested successfully 197 |

198 | )} 199 | {ingest.isError && ( 200 |

201 | Processing failed: 202 | {ingest.error.message} 203 |

204 | )} 205 |
206 | )} 207 | 208 | ); 209 | }; 210 | -------------------------------------------------------------------------------- /app/components/icons/alert-circle.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const AlertCircle = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/alert-triangle.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const AlertTriangle = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/bot.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Bot = (props: React.JSX.IntrinsicElements["svg"]) => { 4 | return ( 5 | 19 | 20 | 21 | ); 22 | }; 23 | -------------------------------------------------------------------------------- /app/components/icons/calendar.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Calendar = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/chat-bubble.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const ChatBubble = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/check-circle.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const CheckCircle = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/check.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Check = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/chevron-down.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const ChevronDown = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/chevron-left.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const ChevronLeft = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/chevron-right.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const ChevronRight = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/cross.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Cross = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/file-upload.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const FileUpload = (props: React.JSX.IntrinsicElements["svg"]) => { 4 | return ( 5 | 19 | 20 | 21 | ); 22 | }; 23 | -------------------------------------------------------------------------------- /app/components/icons/github.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Github = (props: React.JSX.IntrinsicElements["svg"]) => { 4 | return ( 5 | 19 | 20 | 21 | ); 22 | }; 23 | -------------------------------------------------------------------------------- /app/components/icons/google.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Google = (props: React.JSX.IntrinsicElements["svg"]) => { 4 | return ( 5 | 16 | 20 | 24 | 28 | 32 | 33 | ); 34 | }; 35 | -------------------------------------------------------------------------------- /app/components/icons/info.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Info = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/list-down.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const ListDown = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/list-up.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const ListUp = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/minus.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Minus = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/plus.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Plus = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/question-circle.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const QuestionCircle = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/search.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Search = (props: React.JSX.IntrinsicElements["svg"]) => ( 4 | 18 | 19 | 20 | ); 21 | -------------------------------------------------------------------------------- /app/components/icons/send.tsx: -------------------------------------------------------------------------------- 1 | export const Send = (props: JSX.IntrinsicElements["svg"]) => { 2 | return ( 3 | 17 | 18 | 19 | ); 20 | }; 21 | -------------------------------------------------------------------------------- /app/components/icons/spinner.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const Spinner = (props: React.JSX.IntrinsicElements["svg"]) => { 4 | return ( 5 |
6 | 22 | Loading... 23 |
24 | ); 25 | }; 26 | -------------------------------------------------------------------------------- /app/components/icons/user.tsx: -------------------------------------------------------------------------------- 1 | import type React from "react"; 2 | 3 | export const User = (props: React.JSX.IntrinsicElements["svg"]) => { 4 | return ( 5 | 19 | 20 | 21 | 22 | ); 23 | }; 24 | -------------------------------------------------------------------------------- /app/components/layout/navbar.tsx: -------------------------------------------------------------------------------- 1 | import { useSubmit } from "@remix-run/react"; 2 | import { Button as ReactAriaButton } from "react-aria-components"; 3 | import { Menu, MenuContent, MenuItem } from "../ui/menu"; 4 | import { Avatar, AvatarFallback, AvatarImage } from "../ui/avatar"; 5 | import type { User } from "../../lib/db/schema"; 6 | 7 | export const Navbar = ({ 8 | user, 9 | }: { 10 | user: User | null; 11 | }) => { 12 | const submit = useSubmit(); 13 | 14 | return ( 15 | 132 | ); 133 | }; 134 | -------------------------------------------------------------------------------- /app/components/misc/error-boundary.tsx: -------------------------------------------------------------------------------- 1 | import type { ErrorResponse } from "@remix-run/router"; 2 | 3 | import { 4 | isRouteErrorResponse, 5 | useParams, 6 | useRouteError, 7 | } from "@remix-run/react"; 8 | import { getErrorMessage } from "../../lib/utils/get-error-message"; 9 | 10 | type StatusHandler = (info: { 11 | error: ErrorResponse; 12 | params: Record; 13 | }) => JSX.Element | null; 14 | 15 | type GenericErrorBoundaryProps = { 16 | defaultStatusHandler?: StatusHandler; 17 | statusHandlers?: Record; 18 | unexpectedErrorHandler?: (error: unknown) => JSX.Element | null; 19 | }; 20 | 21 | export function GenericErrorBoundary({ 22 | defaultStatusHandler = ({ error }) => ( 23 |

24 | {error.status} {error.data} 25 |

26 | ), 27 | statusHandlers, 28 | unexpectedErrorHandler = (error) =>

{getErrorMessage(error)}

, 29 | }: GenericErrorBoundaryProps) { 30 | const error = useRouteError(); 31 | const params = useParams(); 32 | 33 | if (typeof document !== "undefined") { 34 | console.error(error); 35 | } 36 | 37 | return ( 38 |
39 | {isRouteErrorResponse(error) 40 | ? (statusHandlers?.[error.status] ?? defaultStatusHandler)({ 41 | error, 42 | params, 43 | }) 44 | : unexpectedErrorHandler(error)} 45 |
46 | ); 47 | } 48 | -------------------------------------------------------------------------------- /app/components/ui/avatar.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import * as AvatarPrimitive from "@radix-ui/react-avatar"; 3 | import { cn } from "../../lib/utils/cn"; 4 | 5 | const Avatar = React.forwardRef< 6 | React.ElementRef, 7 | React.ComponentPropsWithoutRef 8 | >(({ className, ...props }, ref) => ( 9 | 17 | )); 18 | 19 | const AvatarImage = React.forwardRef< 20 | React.ElementRef, 21 | React.ComponentPropsWithoutRef 22 | >(({ className, ...props }, ref) => ( 23 | 28 | )); 29 | 30 | const AvatarFallback = React.forwardRef< 31 | React.ElementRef, 32 | React.ComponentPropsWithoutRef 33 | >(({ className, ...props }, ref) => ( 34 | 42 | )); 43 | 44 | export { Avatar, AvatarImage, AvatarFallback }; 45 | -------------------------------------------------------------------------------- /app/components/ui/button.tsx: -------------------------------------------------------------------------------- 1 | import type { ButtonProps as BtnProps } from "react-aria-components"; 2 | import { Button as ReactAriaButton } from "react-aria-components"; 3 | import { cn } from "../../lib/utils/cn"; 4 | import { buttonSizes } from "./constants"; 5 | 6 | const buttonVariants = { 7 | primary: "bg-primary-solid text-white hover:bg-primary-solid-hover", 8 | danger: "bg-danger-solid text-white hover:bg-danger-solid-hover", 9 | outline: 10 | "border border-muted bg-transparent hover:border-muted-hover hover:bg-muted-element-hover hover:text-muted-high-contrast data-[focus-visible]-visible:text-muted-high-contrast", 11 | secondary: 12 | "bg-primary-element text-primary-high-contrast hover:bg-primary-element-hover", 13 | ghost: 14 | "hover:bg-muted-element-hover hover:text-muted-high-contrast data-[focus-visible]:text-muted-high-contrast", 15 | } as const; 16 | 17 | export type ButtonProps = BtnProps & { 18 | variant?: keyof typeof buttonVariants; 19 | size?: keyof typeof buttonSizes; 20 | className?: string; 21 | }; 22 | 23 | const Button = ({ 24 | variant = "outline", 25 | size = "default", 26 | className, 27 | children, 28 | ...props 29 | }: ButtonProps) => { 30 | return ( 31 | 43 | {children} 44 | 45 | ); 46 | }; 47 | 48 | export { Button, buttonSizes, buttonVariants }; 49 | -------------------------------------------------------------------------------- /app/components/ui/checkbox.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Checkbox as ReactAriaCheckbox, 3 | CheckboxGroup as ReactAriaCheckboxGroup, 4 | type CheckboxProps, 5 | } from "react-aria-components"; 6 | import { cn } from "../../lib/utils/cn"; 7 | import { Check } from "../icons/check"; 8 | import { Minus } from "../icons/minus"; 9 | 10 | const CheckboxGroup = ReactAriaCheckboxGroup; 11 | 12 | const Checkbox = ({ className, children, ...props }: CheckboxProps) => ( 13 | 15 | cn( 16 | values, 17 | "group inline-flex focus:outline-none items-center gap-x-2 data-[disabled]:cursor-not-allowed data-[disabled]:opacity-50", 18 | className, 19 | ) 20 | } 21 | {...props} 22 | > 23 | {(values) => ( 24 | <> 25 |
35 | {values.isIndeterminate ? ( 36 | 37 | ) : values.isSelected ? ( 38 | 39 | ) : null} 40 |
41 | {typeof children === "function" ? children(values) : children} 42 | 43 | )} 44 | 45 | ); 46 | 47 | export { Checkbox, CheckboxGroup }; 48 | -------------------------------------------------------------------------------- /app/components/ui/constants.ts: -------------------------------------------------------------------------------- 1 | export const sizes = { 2 | xs: "text-xs", 3 | sm: "text-sm", 4 | md: "text-base", 5 | lg: "text-lg", 6 | xl: "text-xl", 7 | "2xl": "text-2xl", 8 | "3xl": "text-3xl", 9 | "4xl": "text-4xl", 10 | "5xl": "text-5xl", 11 | "6xl": "text-6xl", 12 | "7xl": "text-7xl", 13 | "8xl": "text-8xl", 14 | } as const; 15 | 16 | export const buttonSizes = { 17 | default: "h-9 px-4 py-3", 18 | sm: "h-8 px-3 text-xs", 19 | lg: "h-10 px-8", 20 | icon: "h-9 w-9", 21 | } as const; 22 | -------------------------------------------------------------------------------- /app/components/ui/dropzone.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | DropZone as ReactAriaDropZone, 3 | type DropZoneProps, 4 | } from "react-aria-components"; 5 | import { cn } from "../../lib/utils/cn"; 6 | 7 | const DropZone = ({ className, ...props }: DropZoneProps) => ( 8 | 10 | cn( 11 | values, 12 | "flex flex-col gap-2 h-[150px] w-[300px] items-center justify-center rounded-md border border-dashed border-muted text-sm data-[drop-target]:border-muted-active data-[drop-target]:bg-muted-element-hover bg-muted-element", 13 | className, 14 | ) 15 | } 16 | {...props} 17 | /> 18 | ); 19 | 20 | export { DropZone }; 21 | -------------------------------------------------------------------------------- /app/components/ui/file-trigger.tsx: -------------------------------------------------------------------------------- 1 | import { FileTrigger as ReactAriaFileTrigger } from "react-aria-components"; 2 | 3 | const FileTrigger = ReactAriaFileTrigger; 4 | 5 | export { FileTrigger }; 6 | -------------------------------------------------------------------------------- /app/components/ui/heading.tsx: -------------------------------------------------------------------------------- 1 | import type { HeadingProps as ReactAriaHeadingProps } from "react-aria-components"; 2 | import { Heading as ReactAriaHeading } from "react-aria-components"; 3 | import { cn } from "../../lib/utils/cn"; 4 | import { sizes } from "./constants"; 5 | 6 | export type HeadingProps = ReactAriaHeadingProps & { 7 | size?: keyof typeof sizes; 8 | }; 9 | 10 | const Heading = ({ 11 | children, 12 | className, 13 | level, 14 | size = "lg", 15 | ...props 16 | }: HeadingProps) => { 17 | return ( 18 | 23 | {children} 24 | 25 | ); 26 | }; 27 | 28 | export { Heading }; 29 | -------------------------------------------------------------------------------- /app/components/ui/input.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Input as ReactAriaInput, 3 | type InputProps, 4 | DateInput as ReactAriaDateInput, 5 | type DateInputProps, 6 | DateSegment, 7 | } from "react-aria-components"; 8 | import { cn } from "../../lib/utils/cn"; 9 | 10 | const Input = ({ className, type = "text", ...props }: InputProps) => { 11 | return ( 12 | 30 | 57 | 58 | ); 59 | }; 60 | 61 | const DateInput = (props: Omit) => { 62 | return ( 63 | 88 | {(segment) => ( 89 | 95 | )} 96 | 97 | ); 98 | }; 99 | 100 | export { Input, DateInput }; 101 | -------------------------------------------------------------------------------- /app/components/ui/menu.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Header as ReactAriaHeader, 3 | Menu as ReactAriaMenu, 4 | MenuItem as ReactAriaMenuItem, 5 | MenuTrigger as ReactAriaMenuTrigger, 6 | Popover as ReactAriaPopover, 7 | Section as ReactAriaSection, 8 | Separator as ReactAriaSeparator, 9 | type MenuItemProps, 10 | type MenuProps, 11 | type MenuTriggerProps, 12 | type PopoverProps, 13 | type SectionProps, 14 | type SeparatorProps, 15 | } from "react-aria-components"; 16 | 17 | import { cn } from "../../lib/utils/cn"; 18 | import { Check } from "../icons/check"; 19 | 20 | const Menu = (props: MenuTriggerProps) => { 21 | return ; 22 | }; 23 | 24 | export interface MenuContentProps 25 | extends Omit, 26 | MenuProps { 27 | className?: string; 28 | popoverClassName?: string; 29 | } 30 | 31 | const MenuContent = ({ 32 | className, 33 | popoverClassName, 34 | ...props 35 | }: MenuContentProps) => { 36 | return ( 37 | 53 | 54 | 55 | ); 56 | }; 57 | 58 | const MenuItem = ({ className, children, ...props }: MenuItemProps) => { 59 | return ( 60 | 74 | {({ selectionMode }) => ( 75 | <> 76 | {selectionMode === "multiple" || 77 | (selectionMode === "single" && ( 78 | 87 | ); 88 | }; 89 | 90 | const MenuSection = (props: SectionProps) => { 91 | return ; 92 | }; 93 | 94 | const MenuHeader = ({ 95 | className, 96 | ...props 97 | }: React.HTMLAttributes) => { 98 | return ( 99 | 106 | ); 107 | }; 108 | 109 | const MenuSeparator = ({ className, ...props }: SeparatorProps) => { 110 | return ( 111 | 115 | ); 116 | }; 117 | 118 | export { Menu, MenuContent, MenuSection, MenuHeader, MenuItem, MenuSeparator }; 119 | -------------------------------------------------------------------------------- /app/components/ui/text.tsx: -------------------------------------------------------------------------------- 1 | import type { TextProps as ReactAriaTextProps } from "react-aria-components"; 2 | import { Text as ReactAriaText } from "react-aria-components"; 3 | import { cn } from "../../lib/utils/cn"; 4 | import { sizes } from "./constants"; 5 | 6 | export type TextProps = ReactAriaTextProps & { 7 | size?: keyof typeof sizes; 8 | }; 9 | 10 | const Text = ({ 11 | children, 12 | className, 13 | size = "md", 14 | elementType = "p", 15 | ...props 16 | }: TextProps) => { 17 | return ( 18 | 23 | {children} 24 | 25 | ); 26 | }; 27 | 28 | export { Text }; 29 | -------------------------------------------------------------------------------- /app/components/ui/toast.tsx: -------------------------------------------------------------------------------- 1 | import { Toaster as Sonner } from "sonner"; 2 | import { toast } from "sonner"; 3 | import { AlertTriangle } from "../icons/alert-triangle"; 4 | import { Info } from "../icons/info"; 5 | import { Check } from "../icons/check"; 6 | import { AlertCircle } from "../icons/alert-circle"; 7 | 8 | type ToasterProps = React.ComponentProps; 9 | 10 | const Toaster = ({ ...props }: ToasterProps) => { 11 | return ( 12 | , 17 | info: , 18 | warning: , 19 | error: , 20 | }} 21 | toastOptions={{ 22 | classNames: { 23 | toast: 24 | "group toast group-[.toaster]:bg-muted-app-subtle group-[.toaster]:text-muted-base group-[.toaster]:border-transparent group-[.toaster]:shadow-lg", 25 | description: "group-[.toast]:text-muted-base", 26 | actionButton: 27 | "group-[.toast]:bg-muted-element group-[.toast]:text-muted-base group-[.toast]:hover:bg-muted-element-hover group-[.toast]:hover:text-muted-high-contrast group-[.toast]:cursor-default", 28 | }, 29 | }} 30 | {...props} 31 | /> 32 | ); 33 | }; 34 | 35 | export { Toaster, toast }; 36 | -------------------------------------------------------------------------------- /app/entry.client.tsx: -------------------------------------------------------------------------------- 1 | import { RemixBrowser } from "@remix-run/react"; 2 | import { startTransition, StrictMode } from "react"; 3 | import { hydrateRoot } from "react-dom/client"; 4 | 5 | startTransition(() => { 6 | hydrateRoot( 7 | document, 8 | 9 | 10 | 11 | ); 12 | }); 13 | -------------------------------------------------------------------------------- /app/entry.server.tsx: -------------------------------------------------------------------------------- 1 | import { PassThrough } from "node:stream"; 2 | 3 | import type { AppLoadContext, EntryContext } from "@remix-run/node"; 4 | import { createReadableStreamFromReadable } from "@remix-run/node"; 5 | import { RemixServer } from "@remix-run/react"; 6 | import * as isbotModule from "isbot"; 7 | import { renderToPipeableStream } from "react-dom/server"; 8 | 9 | const ABORT_DELAY = 5_000; 10 | 11 | export default function handleRequest( 12 | request: Request, 13 | responseStatusCode: number, 14 | responseHeaders: Headers, 15 | remixContext: EntryContext, 16 | loadContext: AppLoadContext 17 | ) { 18 | let prohibitOutOfOrderStreaming = 19 | isBotRequest(request.headers.get("user-agent")) || remixContext.isSpaMode; 20 | 21 | return prohibitOutOfOrderStreaming 22 | ? handleBotRequest( 23 | request, 24 | responseStatusCode, 25 | responseHeaders, 26 | remixContext 27 | ) 28 | : handleBrowserRequest( 29 | request, 30 | responseStatusCode, 31 | responseHeaders, 32 | remixContext 33 | ); 34 | } 35 | 36 | // We have some Remix apps in the wild already running with isbot@3 so we need 37 | // to maintain backwards compatibility even though we want new apps to use 38 | // isbot@4. That way, we can ship this as a minor Semver update to @remix-run/dev. 39 | function isBotRequest(userAgent: string | null) { 40 | if (!userAgent) { 41 | return false; 42 | } 43 | 44 | // isbot >= 3.8.0, >4 45 | if ("isbot" in isbotModule && typeof isbotModule.isbot === "function") { 46 | return isbotModule.isbot(userAgent); 47 | } 48 | 49 | // isbot < 3.8.0 50 | if ("default" in isbotModule && typeof isbotModule.default === "function") { 51 | return isbotModule.default(userAgent); 52 | } 53 | 54 | return false; 55 | } 56 | 57 | function handleBotRequest( 58 | request: Request, 59 | responseStatusCode: number, 60 | responseHeaders: Headers, 61 | remixContext: EntryContext 62 | ) { 63 | return new Promise((resolve, reject) => { 64 | let shellRendered = false; 65 | const { pipe, abort } = renderToPipeableStream( 66 | , 71 | { 72 | onAllReady() { 73 | shellRendered = true; 74 | const body = new PassThrough(); 75 | const stream = createReadableStreamFromReadable(body); 76 | 77 | responseHeaders.set("Content-Type", "text/html"); 78 | 79 | resolve( 80 | new Response(stream, { 81 | headers: responseHeaders, 82 | status: responseStatusCode, 83 | }) 84 | ); 85 | 86 | pipe(body); 87 | }, 88 | onShellError(error: unknown) { 89 | reject(error); 90 | }, 91 | onError(error: unknown) { 92 | responseStatusCode = 500; 93 | // Log streaming rendering errors from inside the shell. Don't log 94 | // errors encountered during initial shell rendering since they'll 95 | // reject and get logged in handleDocumentRequest. 96 | if (shellRendered) { 97 | console.error(error); 98 | } 99 | }, 100 | } 101 | ); 102 | 103 | setTimeout(abort, ABORT_DELAY); 104 | }); 105 | } 106 | 107 | function handleBrowserRequest( 108 | request: Request, 109 | responseStatusCode: number, 110 | responseHeaders: Headers, 111 | remixContext: EntryContext 112 | ) { 113 | return new Promise((resolve, reject) => { 114 | let shellRendered = false; 115 | const { pipe, abort } = renderToPipeableStream( 116 | , 121 | { 122 | onShellReady() { 123 | shellRendered = true; 124 | const body = new PassThrough(); 125 | const stream = createReadableStreamFromReadable(body); 126 | 127 | responseHeaders.set("Content-Type", "text/html"); 128 | 129 | resolve( 130 | new Response(stream, { 131 | headers: responseHeaders, 132 | status: responseStatusCode, 133 | }) 134 | ); 135 | 136 | pipe(body); 137 | }, 138 | onShellError(error: unknown) { 139 | reject(error); 140 | }, 141 | onError(error: unknown) { 142 | responseStatusCode = 500; 143 | // Log streaming rendering errors from inside the shell. Don't log 144 | // errors encountered during initial shell rendering since they'll 145 | // reject and get logged in handleDocumentRequest. 146 | if (shellRendered) { 147 | console.error(error); 148 | } 149 | }, 150 | } 151 | ); 152 | 153 | setTimeout(abort, ABORT_DELAY); 154 | }); 155 | } 156 | -------------------------------------------------------------------------------- /app/lib/auth.ts: -------------------------------------------------------------------------------- 1 | import { GoogleStrategy } from "remix-auth-google"; 2 | import { users, vectorDatabases, type User as TUser } from "./db/schema"; 3 | import { generateId } from "./db/utils/generate-id"; 4 | import { getErrorMessage } from "./utils/get-error-message"; 5 | import { eq } from "drizzle-orm"; 6 | import { migrate } from "drizzle-orm/postgres-js/migrator"; 7 | import postgres from "postgres"; 8 | import { db } from "./db"; 9 | import { drizzle } from "drizzle-orm/postgres-js"; 10 | import { neonApiClient } from "./vector-db"; 11 | import { Authenticator } from "remix-auth"; 12 | import { sessionStorage } from "./auth/session.server"; 13 | 14 | export type User = TUser & { 15 | vectorDbId: string; 16 | }; 17 | 18 | export const authenticator = new Authenticator(sessionStorage); 19 | 20 | authenticator.use( 21 | new GoogleStrategy( 22 | { 23 | clientID: process.env.GOOGLE_CLIENT_ID, 24 | clientSecret: process.env.GOOGLE_CLIENT_SECRET, 25 | callbackURL: process.env.GOOGLE_CALLBACK_URL, 26 | }, 27 | async ({ profile }) => { 28 | const email = profile.emails[0].value; 29 | 30 | try { 31 | const userData = await db 32 | .select({ 33 | user: users, 34 | vectorDatabase: vectorDatabases, 35 | }) 36 | .from(users) 37 | .leftJoin(vectorDatabases, eq(users.id, vectorDatabases.userId)) 38 | .where(eq(users.email, email)); 39 | 40 | if ( 41 | userData.length === 0 || 42 | !userData[0].vectorDatabase || 43 | !userData[0].user 44 | ) { 45 | const { data, error } = await neonApiClient.POST("/projects", { 46 | body: { 47 | project: {}, 48 | }, 49 | }); 50 | 51 | if (error) { 52 | throw new Error(`Failed to create Neon project, ${error}`); 53 | } 54 | 55 | const vectorDbId = data?.project.id; 56 | 57 | const vectorDbConnectionUri = data.connection_uris[0]?.connection_uri; 58 | 59 | const sql = postgres(vectorDbConnectionUri); 60 | 61 | await sql`CREATE EXTENSION IF NOT EXISTS vector;`; 62 | 63 | await migrate(drizzle(sql), { 64 | migrationsFolder: "app/lib/vector-db/migrations", 65 | }); 66 | 67 | const newUser = await db 68 | .insert(users) 69 | .values({ 70 | email, 71 | name: profile.displayName, 72 | avatarUrl: profile.photos[0].value, 73 | userId: generateId({ object: "user" }), 74 | }) 75 | .onConflictDoNothing() 76 | .returning(); 77 | 78 | await db 79 | .insert(vectorDatabases) 80 | .values({ 81 | vectorDbId, 82 | userId: newUser[0].id, 83 | }) 84 | .returning(); 85 | 86 | const result = { 87 | ...newUser[0], 88 | vectorDbId, 89 | }; 90 | 91 | return result; 92 | } 93 | 94 | return { 95 | ...userData[0].user, 96 | vectorDbId: userData[0].vectorDatabase.vectorDbId, 97 | }; 98 | } catch (error) { 99 | console.error("User creation error:", error); 100 | throw new Error(getErrorMessage(error)); 101 | } 102 | }, 103 | ), 104 | ); 105 | -------------------------------------------------------------------------------- /app/lib/auth/session.server.ts: -------------------------------------------------------------------------------- 1 | import { createCookieSessionStorage } from "@remix-run/node"; 2 | 3 | export const sessionStorage = createCookieSessionStorage({ 4 | cookie: { 5 | name: "_session", // use any name you want here 6 | sameSite: "lax", // this helps with CSRF 7 | path: "/", // remember to add this so the cookie will work in all routes 8 | httpOnly: true, // for security reasons, make this cookie http only 9 | secrets: [process.env.SESSION_SECRET], // replace this with an actual secret 10 | secure: process.env.NODE_ENV === "production", // enable this in prod only 11 | }, 12 | }); 13 | 14 | export const { getSession, commitSession, destroySession } = sessionStorage; 15 | -------------------------------------------------------------------------------- /app/lib/constants.ts: -------------------------------------------------------------------------------- 1 | export const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10 MB 2 | 3 | export const MAX_FILE_COUNT = 3; 4 | -------------------------------------------------------------------------------- /app/lib/db/config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "drizzle-kit"; 2 | 3 | export default defineConfig({ 4 | schema: "app/lib/db/schema.ts", 5 | out: "app/lib/db/migrations", 6 | dialect: "postgresql", 7 | dbCredentials: { 8 | url: process.env.DATABASE_URL!, 9 | }, 10 | }); 11 | -------------------------------------------------------------------------------- /app/lib/db/index.ts: -------------------------------------------------------------------------------- 1 | import postgres from "postgres"; 2 | import { drizzle } from "drizzle-orm/postgres-js"; 3 | import * as schema from "./schema"; 4 | 5 | const sql = postgres(process.env.DATABASE_URL); 6 | 7 | export const db = drizzle(sql, { schema }); 8 | -------------------------------------------------------------------------------- /app/lib/db/migrations/0000_colorful_tusk.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "documents" ( 2 | "id" serial PRIMARY KEY NOT NULL, 3 | "document_id" varchar(256) NOT NULL, 4 | "user_id" serial NOT NULL, 5 | "title" varchar(255) NOT NULL, 6 | "file_name" text NOT NULL, 7 | "file_size" numeric NOT NULL, 8 | "created_at" timestamp with time zone DEFAULT now(), 9 | "updated_at" timestamp with time zone DEFAULT now(), 10 | CONSTRAINT "documents_document_id_unique" UNIQUE("document_id") 11 | ); 12 | --> statement-breakpoint 13 | CREATE TABLE IF NOT EXISTS "users" ( 14 | "id" serial PRIMARY KEY NOT NULL, 15 | "user_id" varchar(256) NOT NULL, 16 | "name" varchar(32), 17 | "email" varchar(255) NOT NULL, 18 | "avatar_url" text, 19 | "created_at" timestamp with time zone DEFAULT now(), 20 | "updated_at" timestamp with time zone DEFAULT now(), 21 | CONSTRAINT "users_user_id_unique" UNIQUE("user_id"), 22 | CONSTRAINT "users_email_unique" UNIQUE("email") 23 | ); 24 | --> statement-breakpoint 25 | CREATE TABLE IF NOT EXISTS "vector_databases" ( 26 | "id" serial PRIMARY KEY NOT NULL, 27 | "vector_db_id" varchar(256) NOT NULL, 28 | "user_id" serial NOT NULL, 29 | "created_at" timestamp with time zone DEFAULT now(), 30 | "updated_at" timestamp with time zone DEFAULT now(), 31 | CONSTRAINT "vector_databases_vector_db_id_unique" UNIQUE("vector_db_id"), 32 | CONSTRAINT "vector_databases_user_id_unique" UNIQUE("user_id") 33 | ); 34 | --> statement-breakpoint 35 | DO $$ BEGIN 36 | ALTER TABLE "documents" ADD CONSTRAINT "documents_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action; 37 | EXCEPTION 38 | WHEN duplicate_object THEN null; 39 | END $$; 40 | --> statement-breakpoint 41 | DO $$ BEGIN 42 | ALTER TABLE "vector_databases" ADD CONSTRAINT "vector_databases_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action; 43 | EXCEPTION 44 | WHEN duplicate_object THEN null; 45 | END $$; 46 | --> statement-breakpoint 47 | CREATE INDEX IF NOT EXISTS "document_id_idx" ON "documents" USING btree ("document_id");--> statement-breakpoint 48 | CREATE INDEX IF NOT EXISTS "user_id_idx" ON "users" USING btree ("user_id");--> statement-breakpoint 49 | CREATE INDEX IF NOT EXISTS "vector_db_id_idx" ON "vector_databases" USING btree ("vector_db_id"); -------------------------------------------------------------------------------- /app/lib/db/migrations/0001_charming_stick.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "documents" DROP COLUMN IF EXISTS "file_size"; -------------------------------------------------------------------------------- /app/lib/db/migrations/meta/0000_snapshot.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "d5dc3f32-7e5b-4294-8aa9-cd20323973a4", 3 | "prevId": "00000000-0000-0000-0000-000000000000", 4 | "version": "7", 5 | "dialect": "postgresql", 6 | "tables": { 7 | "public.documents": { 8 | "name": "documents", 9 | "schema": "", 10 | "columns": { 11 | "id": { 12 | "name": "id", 13 | "type": "serial", 14 | "primaryKey": true, 15 | "notNull": true 16 | }, 17 | "document_id": { 18 | "name": "document_id", 19 | "type": "varchar(256)", 20 | "primaryKey": false, 21 | "notNull": true 22 | }, 23 | "user_id": { 24 | "name": "user_id", 25 | "type": "serial", 26 | "primaryKey": false, 27 | "notNull": true 28 | }, 29 | "title": { 30 | "name": "title", 31 | "type": "varchar(255)", 32 | "primaryKey": false, 33 | "notNull": true 34 | }, 35 | "file_name": { 36 | "name": "file_name", 37 | "type": "text", 38 | "primaryKey": false, 39 | "notNull": true 40 | }, 41 | "file_size": { 42 | "name": "file_size", 43 | "type": "numeric", 44 | "primaryKey": false, 45 | "notNull": true 46 | }, 47 | "created_at": { 48 | "name": "created_at", 49 | "type": "timestamp with time zone", 50 | "primaryKey": false, 51 | "notNull": false, 52 | "default": "now()" 53 | }, 54 | "updated_at": { 55 | "name": "updated_at", 56 | "type": "timestamp with time zone", 57 | "primaryKey": false, 58 | "notNull": false, 59 | "default": "now()" 60 | } 61 | }, 62 | "indexes": { 63 | "document_id_idx": { 64 | "name": "document_id_idx", 65 | "columns": [ 66 | { 67 | "expression": "document_id", 68 | "isExpression": false, 69 | "asc": true, 70 | "nulls": "last" 71 | } 72 | ], 73 | "isUnique": false, 74 | "concurrently": false, 75 | "method": "btree", 76 | "with": {} 77 | } 78 | }, 79 | "foreignKeys": { 80 | "documents_user_id_users_id_fk": { 81 | "name": "documents_user_id_users_id_fk", 82 | "tableFrom": "documents", 83 | "tableTo": "users", 84 | "columnsFrom": ["user_id"], 85 | "columnsTo": ["id"], 86 | "onDelete": "cascade", 87 | "onUpdate": "no action" 88 | } 89 | }, 90 | "compositePrimaryKeys": {}, 91 | "uniqueConstraints": { 92 | "documents_document_id_unique": { 93 | "name": "documents_document_id_unique", 94 | "nullsNotDistinct": false, 95 | "columns": ["document_id"] 96 | } 97 | } 98 | }, 99 | "public.users": { 100 | "name": "users", 101 | "schema": "", 102 | "columns": { 103 | "id": { 104 | "name": "id", 105 | "type": "serial", 106 | "primaryKey": true, 107 | "notNull": true 108 | }, 109 | "user_id": { 110 | "name": "user_id", 111 | "type": "varchar(256)", 112 | "primaryKey": false, 113 | "notNull": true 114 | }, 115 | "name": { 116 | "name": "name", 117 | "type": "varchar(32)", 118 | "primaryKey": false, 119 | "notNull": false 120 | }, 121 | "email": { 122 | "name": "email", 123 | "type": "varchar(255)", 124 | "primaryKey": false, 125 | "notNull": true 126 | }, 127 | "avatar_url": { 128 | "name": "avatar_url", 129 | "type": "text", 130 | "primaryKey": false, 131 | "notNull": false 132 | }, 133 | "created_at": { 134 | "name": "created_at", 135 | "type": "timestamp with time zone", 136 | "primaryKey": false, 137 | "notNull": false, 138 | "default": "now()" 139 | }, 140 | "updated_at": { 141 | "name": "updated_at", 142 | "type": "timestamp with time zone", 143 | "primaryKey": false, 144 | "notNull": false, 145 | "default": "now()" 146 | } 147 | }, 148 | "indexes": { 149 | "user_id_idx": { 150 | "name": "user_id_idx", 151 | "columns": [ 152 | { 153 | "expression": "user_id", 154 | "isExpression": false, 155 | "asc": true, 156 | "nulls": "last" 157 | } 158 | ], 159 | "isUnique": false, 160 | "concurrently": false, 161 | "method": "btree", 162 | "with": {} 163 | } 164 | }, 165 | "foreignKeys": {}, 166 | "compositePrimaryKeys": {}, 167 | "uniqueConstraints": { 168 | "users_user_id_unique": { 169 | "name": "users_user_id_unique", 170 | "nullsNotDistinct": false, 171 | "columns": ["user_id"] 172 | }, 173 | "users_email_unique": { 174 | "name": "users_email_unique", 175 | "nullsNotDistinct": false, 176 | "columns": ["email"] 177 | } 178 | } 179 | }, 180 | "public.vector_databases": { 181 | "name": "vector_databases", 182 | "schema": "", 183 | "columns": { 184 | "id": { 185 | "name": "id", 186 | "type": "serial", 187 | "primaryKey": true, 188 | "notNull": true 189 | }, 190 | "vector_db_id": { 191 | "name": "vector_db_id", 192 | "type": "varchar(256)", 193 | "primaryKey": false, 194 | "notNull": true 195 | }, 196 | "user_id": { 197 | "name": "user_id", 198 | "type": "serial", 199 | "primaryKey": false, 200 | "notNull": true 201 | }, 202 | "created_at": { 203 | "name": "created_at", 204 | "type": "timestamp with time zone", 205 | "primaryKey": false, 206 | "notNull": false, 207 | "default": "now()" 208 | }, 209 | "updated_at": { 210 | "name": "updated_at", 211 | "type": "timestamp with time zone", 212 | "primaryKey": false, 213 | "notNull": false, 214 | "default": "now()" 215 | } 216 | }, 217 | "indexes": { 218 | "vector_db_id_idx": { 219 | "name": "vector_db_id_idx", 220 | "columns": [ 221 | { 222 | "expression": "vector_db_id", 223 | "isExpression": false, 224 | "asc": true, 225 | "nulls": "last" 226 | } 227 | ], 228 | "isUnique": false, 229 | "concurrently": false, 230 | "method": "btree", 231 | "with": {} 232 | } 233 | }, 234 | "foreignKeys": { 235 | "vector_databases_user_id_users_id_fk": { 236 | "name": "vector_databases_user_id_users_id_fk", 237 | "tableFrom": "vector_databases", 238 | "tableTo": "users", 239 | "columnsFrom": ["user_id"], 240 | "columnsTo": ["id"], 241 | "onDelete": "no action", 242 | "onUpdate": "no action" 243 | } 244 | }, 245 | "compositePrimaryKeys": {}, 246 | "uniqueConstraints": { 247 | "vector_databases_vector_db_id_unique": { 248 | "name": "vector_databases_vector_db_id_unique", 249 | "nullsNotDistinct": false, 250 | "columns": ["vector_db_id"] 251 | }, 252 | "vector_databases_user_id_unique": { 253 | "name": "vector_databases_user_id_unique", 254 | "nullsNotDistinct": false, 255 | "columns": ["user_id"] 256 | } 257 | } 258 | } 259 | }, 260 | "enums": {}, 261 | "schemas": {}, 262 | "sequences": {}, 263 | "_meta": { 264 | "columns": {}, 265 | "schemas": {}, 266 | "tables": {} 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /app/lib/db/migrations/meta/0001_snapshot.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "623f1340-6051-4d2d-8676-ad06fd1cb9da", 3 | "prevId": "d5dc3f32-7e5b-4294-8aa9-cd20323973a4", 4 | "version": "7", 5 | "dialect": "postgresql", 6 | "tables": { 7 | "public.documents": { 8 | "name": "documents", 9 | "schema": "", 10 | "columns": { 11 | "id": { 12 | "name": "id", 13 | "type": "serial", 14 | "primaryKey": true, 15 | "notNull": true 16 | }, 17 | "document_id": { 18 | "name": "document_id", 19 | "type": "varchar(256)", 20 | "primaryKey": false, 21 | "notNull": true 22 | }, 23 | "user_id": { 24 | "name": "user_id", 25 | "type": "serial", 26 | "primaryKey": false, 27 | "notNull": true 28 | }, 29 | "title": { 30 | "name": "title", 31 | "type": "varchar(255)", 32 | "primaryKey": false, 33 | "notNull": true 34 | }, 35 | "file_name": { 36 | "name": "file_name", 37 | "type": "text", 38 | "primaryKey": false, 39 | "notNull": true 40 | }, 41 | "created_at": { 42 | "name": "created_at", 43 | "type": "timestamp with time zone", 44 | "primaryKey": false, 45 | "notNull": false, 46 | "default": "now()" 47 | }, 48 | "updated_at": { 49 | "name": "updated_at", 50 | "type": "timestamp with time zone", 51 | "primaryKey": false, 52 | "notNull": false, 53 | "default": "now()" 54 | } 55 | }, 56 | "indexes": { 57 | "document_id_idx": { 58 | "name": "document_id_idx", 59 | "columns": [ 60 | { 61 | "expression": "document_id", 62 | "isExpression": false, 63 | "asc": true, 64 | "nulls": "last" 65 | } 66 | ], 67 | "isUnique": false, 68 | "concurrently": false, 69 | "method": "btree", 70 | "with": {} 71 | } 72 | }, 73 | "foreignKeys": { 74 | "documents_user_id_users_id_fk": { 75 | "name": "documents_user_id_users_id_fk", 76 | "tableFrom": "documents", 77 | "tableTo": "users", 78 | "columnsFrom": ["user_id"], 79 | "columnsTo": ["id"], 80 | "onDelete": "cascade", 81 | "onUpdate": "no action" 82 | } 83 | }, 84 | "compositePrimaryKeys": {}, 85 | "uniqueConstraints": { 86 | "documents_document_id_unique": { 87 | "name": "documents_document_id_unique", 88 | "nullsNotDistinct": false, 89 | "columns": ["document_id"] 90 | } 91 | } 92 | }, 93 | "public.users": { 94 | "name": "users", 95 | "schema": "", 96 | "columns": { 97 | "id": { 98 | "name": "id", 99 | "type": "serial", 100 | "primaryKey": true, 101 | "notNull": true 102 | }, 103 | "user_id": { 104 | "name": "user_id", 105 | "type": "varchar(256)", 106 | "primaryKey": false, 107 | "notNull": true 108 | }, 109 | "name": { 110 | "name": "name", 111 | "type": "varchar(32)", 112 | "primaryKey": false, 113 | "notNull": false 114 | }, 115 | "email": { 116 | "name": "email", 117 | "type": "varchar(255)", 118 | "primaryKey": false, 119 | "notNull": true 120 | }, 121 | "avatar_url": { 122 | "name": "avatar_url", 123 | "type": "text", 124 | "primaryKey": false, 125 | "notNull": false 126 | }, 127 | "created_at": { 128 | "name": "created_at", 129 | "type": "timestamp with time zone", 130 | "primaryKey": false, 131 | "notNull": false, 132 | "default": "now()" 133 | }, 134 | "updated_at": { 135 | "name": "updated_at", 136 | "type": "timestamp with time zone", 137 | "primaryKey": false, 138 | "notNull": false, 139 | "default": "now()" 140 | } 141 | }, 142 | "indexes": { 143 | "user_id_idx": { 144 | "name": "user_id_idx", 145 | "columns": [ 146 | { 147 | "expression": "user_id", 148 | "isExpression": false, 149 | "asc": true, 150 | "nulls": "last" 151 | } 152 | ], 153 | "isUnique": false, 154 | "concurrently": false, 155 | "method": "btree", 156 | "with": {} 157 | } 158 | }, 159 | "foreignKeys": {}, 160 | "compositePrimaryKeys": {}, 161 | "uniqueConstraints": { 162 | "users_user_id_unique": { 163 | "name": "users_user_id_unique", 164 | "nullsNotDistinct": false, 165 | "columns": ["user_id"] 166 | }, 167 | "users_email_unique": { 168 | "name": "users_email_unique", 169 | "nullsNotDistinct": false, 170 | "columns": ["email"] 171 | } 172 | } 173 | }, 174 | "public.vector_databases": { 175 | "name": "vector_databases", 176 | "schema": "", 177 | "columns": { 178 | "id": { 179 | "name": "id", 180 | "type": "serial", 181 | "primaryKey": true, 182 | "notNull": true 183 | }, 184 | "vector_db_id": { 185 | "name": "vector_db_id", 186 | "type": "varchar(256)", 187 | "primaryKey": false, 188 | "notNull": true 189 | }, 190 | "user_id": { 191 | "name": "user_id", 192 | "type": "serial", 193 | "primaryKey": false, 194 | "notNull": true 195 | }, 196 | "created_at": { 197 | "name": "created_at", 198 | "type": "timestamp with time zone", 199 | "primaryKey": false, 200 | "notNull": false, 201 | "default": "now()" 202 | }, 203 | "updated_at": { 204 | "name": "updated_at", 205 | "type": "timestamp with time zone", 206 | "primaryKey": false, 207 | "notNull": false, 208 | "default": "now()" 209 | } 210 | }, 211 | "indexes": { 212 | "vector_db_id_idx": { 213 | "name": "vector_db_id_idx", 214 | "columns": [ 215 | { 216 | "expression": "vector_db_id", 217 | "isExpression": false, 218 | "asc": true, 219 | "nulls": "last" 220 | } 221 | ], 222 | "isUnique": false, 223 | "concurrently": false, 224 | "method": "btree", 225 | "with": {} 226 | } 227 | }, 228 | "foreignKeys": { 229 | "vector_databases_user_id_users_id_fk": { 230 | "name": "vector_databases_user_id_users_id_fk", 231 | "tableFrom": "vector_databases", 232 | "tableTo": "users", 233 | "columnsFrom": ["user_id"], 234 | "columnsTo": ["id"], 235 | "onDelete": "no action", 236 | "onUpdate": "no action" 237 | } 238 | }, 239 | "compositePrimaryKeys": {}, 240 | "uniqueConstraints": { 241 | "vector_databases_vector_db_id_unique": { 242 | "name": "vector_databases_vector_db_id_unique", 243 | "nullsNotDistinct": false, 244 | "columns": ["vector_db_id"] 245 | }, 246 | "vector_databases_user_id_unique": { 247 | "name": "vector_databases_user_id_unique", 248 | "nullsNotDistinct": false, 249 | "columns": ["user_id"] 250 | } 251 | } 252 | } 253 | }, 254 | "enums": {}, 255 | "schemas": {}, 256 | "sequences": {}, 257 | "_meta": { 258 | "columns": {}, 259 | "schemas": {}, 260 | "tables": {} 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /app/lib/db/migrations/meta/_journal.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "7", 3 | "dialect": "postgresql", 4 | "entries": [ 5 | { 6 | "idx": 0, 7 | "version": "7", 8 | "when": 1723725443908, 9 | "tag": "0000_colorful_tusk", 10 | "breakpoints": true 11 | }, 12 | { 13 | "idx": 1, 14 | "version": "7", 15 | "when": 1723733161480, 16 | "tag": "0001_charming_stick", 17 | "breakpoints": true 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /app/lib/db/schema.ts: -------------------------------------------------------------------------------- 1 | import { relations, type InferSelectModel } from "drizzle-orm"; 2 | import { 3 | boolean, 4 | index, 5 | jsonb, 6 | numeric, 7 | pgTable, 8 | serial, 9 | text, 10 | timestamp, 11 | varchar, 12 | } from "drizzle-orm/pg-core"; 13 | import { createInsertSchema, createSelectSchema } from "drizzle-zod"; 14 | 15 | export const users = pgTable( 16 | "users", 17 | { 18 | id: serial("id").primaryKey(), 19 | userId: varchar("user_id", { length: 256 }).notNull().unique(), 20 | name: varchar("name", { length: 32 }), 21 | email: varchar("email", { length: 255 }).notNull().unique(), 22 | avatarUrl: text("avatar_url"), 23 | createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), 24 | updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), 25 | }, 26 | (t) => ({ 27 | userIdIdx: index("user_id_idx").on(t.userId), 28 | }), 29 | ); 30 | 31 | export type User = InferSelectModel; 32 | 33 | export const insertUserSchema = createInsertSchema(users); 34 | export const selectUserSchema = createSelectSchema(users); 35 | 36 | export const documents = pgTable( 37 | "documents", 38 | { 39 | id: serial("id").primaryKey(), 40 | documentId: varchar("document_id", { length: 256 }).notNull().unique(), 41 | userId: serial("user_id") 42 | .notNull() 43 | .references(() => users.id, { onDelete: "cascade" }), 44 | title: varchar("title", { length: 255 }).notNull(), 45 | fileName: text("file_name").notNull(), 46 | createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), 47 | updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), 48 | }, 49 | (t) => ({ 50 | documentIdIdx: index("document_id_idx").on(t.documentId), 51 | }), 52 | ); 53 | 54 | export type Document = InferSelectModel; 55 | 56 | export const insertDocumentSchema = createInsertSchema(documents); 57 | export const selectDocumentSchema = createSelectSchema(documents); 58 | 59 | export const vectorDatabases = pgTable( 60 | "vector_databases", 61 | { 62 | id: serial("id").primaryKey(), 63 | vectorDbId: varchar("vector_db_id", { length: 256 }).notNull().unique(), 64 | userId: serial("user_id") 65 | .notNull() 66 | .references(() => users.id) 67 | .unique(), 68 | createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), 69 | updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), 70 | }, 71 | (t) => ({ 72 | vectorDbIdIdx: index("vector_db_id_idx").on(t.vectorDbId), 73 | }), 74 | ); 75 | 76 | export type VectorDatabase = InferSelectModel; 77 | 78 | export const insertVectorDatabaseSchema = createInsertSchema(vectorDatabases); 79 | export const selectVectorDatabaseSchema = createSelectSchema(vectorDatabases); 80 | -------------------------------------------------------------------------------- /app/lib/db/utils/generate-id.ts: -------------------------------------------------------------------------------- 1 | import { customAlphabet } from "nanoid"; 2 | 3 | const NO_LOOKALIKES = "346789ABCDEFGHJKLMNPQRTUVWXYabcdefghijkmnpqrtwxyz"; // Numbers and english alphabet without lookalikes: 1, l, I, 0, O, o, u, v, 5, S, s, 2, Z. 4 | const DEFAULT_ID_LENGTH = 14; 5 | 6 | const prefixMapping = { 7 | user: "usr", 8 | document: "doc", 9 | }; 10 | 11 | type Options = { 12 | object: keyof typeof prefixMapping; 13 | length?: number; 14 | }; 15 | 16 | /** 17 | * Generates a public-facing ID with a specified prefix from a predefined set and a custom length. It excludes visually similar characters to avoid confusion. 18 | * @example 19 | * // Returns an ID string like "acc_X7Yk92ndA31f" 20 | * generateId({ prefix: 'account', length: 14 }); 21 | */ 22 | 23 | export const generateId = ({ object, length = DEFAULT_ID_LENGTH }: Options) => { 24 | const nanoid = customAlphabet(NO_LOOKALIKES, length); 25 | 26 | const prefix = prefixMapping[object]; 27 | 28 | return `${prefix}_${nanoid()}`; 29 | }; 30 | -------------------------------------------------------------------------------- /app/lib/env.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | 3 | export const envSchema = z.object({ 4 | DATABASE_URL: z.string(), 5 | // Neon 6 | NEON_API_KEY: z.string(), 7 | // Auth 8 | SESSION_SECRET: z.string(), 9 | GOOGLE_CLIENT_ID: z.string(), 10 | GOOGLE_CLIENT_SECRET: z.string(), 11 | GOOGLE_CALLBACK_URL: z.string().url(), 12 | // Cloudflare R2 13 | CLOUDFLARE_R2_ENDPOINT: z.string().url(), 14 | CLOUDFLARE_R2_ACCESS_ID: z.string(), 15 | CLOUDFLARE_R2_SECRET_KEY: z.string(), 16 | CLOUDFLARE_R2_BUCKET_NAME: z.string(), 17 | CLOUDFLARE_R2_ACCOUNT_ID: z.string(), 18 | 19 | // Upstash 20 | UPSTASH_REDIS_REST_URL: z.string(), 21 | UPSTASH_REDIS_REST_TOKEN: z.string(), 22 | 23 | OPENAI_API_KEY: z.string(), 24 | }); 25 | 26 | export type Env = z.infer; 27 | 28 | export const env = envSchema.parse(process.env); 29 | -------------------------------------------------------------------------------- /app/lib/utils/cn.ts: -------------------------------------------------------------------------------- 1 | import type { ClassValue } from "clsx"; 2 | import { clsx } from "clsx"; 3 | import { twMerge } from "tailwind-merge"; 4 | 5 | export const cn = (...inputs: ClassValue[]) => twMerge(clsx(inputs)); 6 | -------------------------------------------------------------------------------- /app/lib/utils/get-error-message.ts: -------------------------------------------------------------------------------- 1 | type ErrorWithMessage = { 2 | message: string; 3 | }; 4 | 5 | const isErrorWithMessage = (error: unknown): error is ErrorWithMessage => { 6 | return ( 7 | typeof error === "object" && 8 | error !== null && 9 | "message" in error && 10 | typeof (error as Record).message === "string" 11 | ); 12 | }; 13 | 14 | const toErrorWithMessage = (maybeError: unknown): ErrorWithMessage => { 15 | if (isErrorWithMessage(maybeError)) return maybeError; 16 | 17 | try { 18 | return new Error(JSON.stringify(maybeError)); 19 | } catch { 20 | // fallback in case there's an error stringifying the maybeError 21 | // like with circular references for example. 22 | return new Error(String(maybeError)); 23 | } 24 | }; 25 | 26 | export const getErrorMessage = (error: unknown) => { 27 | return toErrorWithMessage(error).message; 28 | }; 29 | -------------------------------------------------------------------------------- /app/lib/vector-db/config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "drizzle-kit"; 2 | 3 | export default defineConfig({ 4 | schema: "app/lib/vector-db/schema.ts", 5 | out: "app/lib/vector-db/migrations", 6 | dialect: "postgresql", 7 | }); 8 | -------------------------------------------------------------------------------- /app/lib/vector-db/index.ts: -------------------------------------------------------------------------------- 1 | import createClient from "openapi-fetch"; 2 | import type { paths } from "./api-schema"; 3 | 4 | export const neonApiClient = createClient({ 5 | baseUrl: "https://console.neon.tech/api/v2/", 6 | headers: { 7 | "Content-Type": "application/json", 8 | Authorization: `Bearer ${process.env.NEON_API_KEY}`, 9 | }, 10 | }); 11 | -------------------------------------------------------------------------------- /app/lib/vector-db/migrations/0000_fair_hobgoblin.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "embeddings" ( 2 | "id" serial PRIMARY KEY NOT NULL, 3 | "content" text NOT NULL, 4 | "metadata" jsonb NOT NULL, 5 | "embedding" vector(1536), 6 | "created_at" timestamp with time zone DEFAULT now(), 7 | "updated_at" timestamp with time zone DEFAULT now() 8 | ); 9 | --> statement-breakpoint 10 | CREATE INDEX IF NOT EXISTS "embedding_idx" ON "embeddings" USING hnsw ("embedding" vector_cosine_ops); -------------------------------------------------------------------------------- /app/lib/vector-db/migrations/meta/0000_snapshot.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "ba1960e0-3a2e-4870-9c0f-a5b56d693ee7", 3 | "prevId": "00000000-0000-0000-0000-000000000000", 4 | "version": "7", 5 | "dialect": "postgresql", 6 | "tables": { 7 | "public.embeddings": { 8 | "name": "embeddings", 9 | "schema": "", 10 | "columns": { 11 | "id": { 12 | "name": "id", 13 | "type": "serial", 14 | "primaryKey": true, 15 | "notNull": true 16 | }, 17 | "content": { 18 | "name": "content", 19 | "type": "text", 20 | "primaryKey": false, 21 | "notNull": true 22 | }, 23 | "metadata": { 24 | "name": "metadata", 25 | "type": "jsonb", 26 | "primaryKey": false, 27 | "notNull": true 28 | }, 29 | "embedding": { 30 | "name": "embedding", 31 | "type": "vector(1536)", 32 | "primaryKey": false, 33 | "notNull": false 34 | }, 35 | "created_at": { 36 | "name": "created_at", 37 | "type": "timestamp with time zone", 38 | "primaryKey": false, 39 | "notNull": false, 40 | "default": "now()" 41 | }, 42 | "updated_at": { 43 | "name": "updated_at", 44 | "type": "timestamp with time zone", 45 | "primaryKey": false, 46 | "notNull": false, 47 | "default": "now()" 48 | } 49 | }, 50 | "indexes": { 51 | "embedding_idx": { 52 | "name": "embedding_idx", 53 | "columns": [ 54 | { 55 | "expression": "embedding", 56 | "isExpression": false, 57 | "asc": true, 58 | "nulls": "last", 59 | "opclass": "vector_cosine_ops" 60 | } 61 | ], 62 | "isUnique": false, 63 | "concurrently": false, 64 | "method": "hnsw", 65 | "with": {} 66 | } 67 | }, 68 | "foreignKeys": {}, 69 | "compositePrimaryKeys": {}, 70 | "uniqueConstraints": {} 71 | } 72 | }, 73 | "enums": {}, 74 | "schemas": {}, 75 | "sequences": {}, 76 | "_meta": { 77 | "columns": {}, 78 | "schemas": {}, 79 | "tables": {} 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /app/lib/vector-db/migrations/meta/_journal.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "7", 3 | "dialect": "postgresql", 4 | "entries": [ 5 | { 6 | "idx": 0, 7 | "version": "7", 8 | "when": 1726682302332, 9 | "tag": "0000_fair_hobgoblin", 10 | "breakpoints": true 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /app/lib/vector-db/schema.ts: -------------------------------------------------------------------------------- 1 | import type { InferSelectModel } from "drizzle-orm"; 2 | import { 3 | pgTable, 4 | serial, 5 | text, 6 | jsonb, 7 | timestamp, 8 | index, 9 | vector, 10 | } from "drizzle-orm/pg-core"; 11 | 12 | export const embeddings = pgTable( 13 | "embeddings", 14 | { 15 | id: serial("id").primaryKey(), 16 | content: text("content").notNull(), 17 | metadata: jsonb("metadata").notNull(), 18 | embedding: vector("embedding", { dimensions: 1536 }), 19 | createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), 20 | updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow(), 21 | }, 22 | (table) => ({ 23 | embeddingIdx: index("embedding_idx").using( 24 | "hnsw", 25 | table.embedding.op("vector_cosine_ops"), 26 | ), 27 | }), 28 | ); 29 | 30 | export type Embedding = InferSelectModel; 31 | -------------------------------------------------------------------------------- /app/root.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Links, 3 | Meta, 4 | Outlet, 5 | Scripts, 6 | ScrollRestoration, 7 | useLoaderData, 8 | } from "@remix-run/react"; 9 | import "./styles/globals.css"; 10 | import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; 11 | import { json, type LoaderFunctionArgs } from "@remix-run/node"; 12 | 13 | import { Navbar } from "./components/layout/navbar"; 14 | import type { User } from "./lib/db/schema"; 15 | import { GenericErrorBoundary } from "./components/misc/error-boundary"; 16 | import { Toaster } from "sonner"; 17 | import { authenticator } from "./lib/auth"; 18 | 19 | const queryClient = new QueryClient(); 20 | 21 | export const loader = async ({ request }: LoaderFunctionArgs) => { 22 | try { 23 | const user = await authenticator.isAuthenticated(request); 24 | 25 | return json({ user }); 26 | } catch (error) { 27 | console.error(error); 28 | return json({ user: null }); 29 | } 30 | }; 31 | 32 | function Document({ 33 | children, 34 | }: { 35 | children: React.ReactNode; 36 | }) { 37 | const data = useLoaderData(); 38 | 39 | return ( 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | {children} 52 | 53 | 54 | 55 | 56 | 57 | ); 58 | } 59 | 60 | export default function App() { 61 | return ( 62 | 63 | 64 | 65 | ); 66 | } 67 | 68 | export function ErrorBoundary() { 69 | return ( 70 | 71 | 72 | 73 | ); 74 | } 75 | -------------------------------------------------------------------------------- /app/routes/$.tsx: -------------------------------------------------------------------------------- 1 | import { Form, useLocation } from "@remix-run/react"; 2 | import { GenericErrorBoundary } from "../components/misc/error-boundary"; 3 | import { Button } from "../components/ui/button"; 4 | import { Heading } from "../components/ui/heading"; 5 | 6 | export async function loader() { 7 | throw new Response("Not found", { status: 404 }); 8 | } 9 | 10 | export default function NotFound() { 11 | return ; 12 | } 13 | 14 | export function ErrorBoundary() { 15 | const location = useLocation(); 16 | 17 | return ( 18 |
19 |
20 | ( 23 |
24 | 25 | Nothing here. 26 | 27 | 28 | {location.pathname} not found 29 | 30 |
31 | 34 |
35 |
36 | ), 37 | }} 38 | /> 39 |
40 |
41 | ); 42 | } 43 | -------------------------------------------------------------------------------- /app/routes/_index.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | json, 3 | type LoaderFunctionArgs, 4 | type MetaFunction, 5 | redirect, 6 | } from "@remix-run/node"; 7 | import { Form } from "@remix-run/react"; 8 | 9 | import { Github } from "../components/icons/github"; 10 | import { Google } from "../components/icons/google"; 11 | import { Button } from "../components/ui/button"; 12 | import { authenticator } from "~/lib/auth"; 13 | 14 | export const meta: MetaFunction = () => { 15 | return [ 16 | { title: "AI Vector Database Per tenant" }, 17 | { 18 | name: "description", 19 | content: "Example app where each tenant has its own Vector database", 20 | }, 21 | ]; 22 | }; 23 | 24 | export async function loader({ request }: LoaderFunctionArgs) { 25 | const user = await authenticator.isAuthenticated(request); 26 | 27 | if (user) { 28 | return redirect("/documents"); 29 | } 30 | return json({ user: null }); 31 | } 32 | 33 | export default function Index() { 34 | return ( 35 |
36 |
37 |
38 |
39 |

40 | AI vector database per tenant 41 |

42 |

43 | Example chat-with-pdf app showing how to provision a dedicated 44 | vector database instance for each user.{" "} 45 | 49 | Powered by Neon{" "} 50 | 51 |

52 |
53 |
54 | 57 |
58 | 59 | <> 60 | 65 | View code 66 | 67 | 68 |
69 |
70 |
71 |
72 |
73 | ); 74 | } 75 | -------------------------------------------------------------------------------- /app/routes/api.auth.google.callback.tsx: -------------------------------------------------------------------------------- 1 | import type { LoaderFunctionArgs } from "@remix-run/node"; 2 | import { authenticator } from "~/lib/auth"; 3 | 4 | export const loader = ({ request }: LoaderFunctionArgs) => { 5 | return authenticator.authenticate("google", request, { 6 | successRedirect: "/documents", 7 | failureRedirect: "/login", 8 | }); 9 | }; 10 | -------------------------------------------------------------------------------- /app/routes/api.auth.google.tsx: -------------------------------------------------------------------------------- 1 | import { redirect, type ActionFunctionArgs } from "@remix-run/node"; 2 | import { authenticator } from "~/lib/auth"; 3 | 4 | export const loader = () => redirect("/login"); 5 | 6 | export const action = ({ request }: ActionFunctionArgs) => { 7 | return authenticator.authenticate("google", request); 8 | }; 9 | -------------------------------------------------------------------------------- /app/routes/api.auth.logout.tsx: -------------------------------------------------------------------------------- 1 | import type { ActionFunctionArgs } from "@remix-run/node"; 2 | import { authenticator } from "~/lib/auth"; 3 | 4 | export async function action({ request }: ActionFunctionArgs) { 5 | await authenticator.logout(request, { redirectTo: "/login" }); 6 | } 7 | -------------------------------------------------------------------------------- /app/routes/api.document.chat.tsx: -------------------------------------------------------------------------------- 1 | import { type ActionFunctionArgs, json } from "@remix-run/node"; 2 | import { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openai"; 3 | import { LangChainAdapter, type Message } from "ai"; 4 | import { 5 | AIMessage, 6 | HumanMessage, 7 | SystemMessage, 8 | } from "@langchain/core/messages"; 9 | import { NeonPostgres } from "@langchain/community/vectorstores/neon"; 10 | 11 | import { Ratelimit } from "@upstash/ratelimit"; 12 | import { Redis } from "@upstash/redis"; 13 | import { authenticator } from "~/lib/auth"; 14 | import { neonApiClient } from "~/lib/vector-db"; 15 | 16 | export const action = async ({ request }: ActionFunctionArgs) => { 17 | const user = await authenticator.isAuthenticated(request); 18 | 19 | if (!user) { 20 | return json( 21 | { 22 | error: "Unauthorized", 23 | }, 24 | 401, 25 | ); 26 | } 27 | const ip = request.headers.get("CF-Connecting-IP"); 28 | const identifier = ip ?? "global"; 29 | 30 | const ratelimit = new Ratelimit({ 31 | redis: Redis.fromEnv(), 32 | limiter: Ratelimit.fixedWindow(10, "60 s"), 33 | analytics: true, 34 | }); 35 | 36 | const { success, limit, remaining, reset } = 37 | await ratelimit.limit(identifier); 38 | 39 | if (!success) { 40 | return json( 41 | { 42 | error: "Rate limit exceeded", 43 | limit, 44 | remaining, 45 | reset, 46 | }, 47 | { 48 | status: 429, 49 | }, 50 | ); 51 | } 52 | 53 | const { 54 | messages, 55 | documentId, 56 | }: { 57 | messages: Message[]; 58 | documentId: string; 59 | } = await request.json(); 60 | 61 | const { content: prompt } = messages[messages.length - 1]; 62 | 63 | const { data, error } = await neonApiClient.GET( 64 | "/projects/{project_id}/connection_uri", 65 | { 66 | params: { 67 | path: { 68 | project_id: user.vectorDbId, 69 | }, 70 | query: { 71 | role_name: "neondb_owner", 72 | database_name: "neondb", 73 | }, 74 | }, 75 | }, 76 | ); 77 | 78 | if (error) { 79 | return json({ 80 | error: error, 81 | }); 82 | } 83 | 84 | const embeddings = new OpenAIEmbeddings({ 85 | apiKey: process.env.OPENAI_API_KEY, 86 | dimensions: 1536, 87 | model: "text-embedding-3-small", 88 | }); 89 | 90 | const vectorStore = await NeonPostgres.initialize(embeddings, { 91 | connectionString: data.uri, 92 | tableName: "embeddings", 93 | columns: { 94 | contentColumnName: "content", 95 | metadataColumnName: "metadata", 96 | vectorColumnName: "embedding", 97 | }, 98 | }); 99 | 100 | const result = await vectorStore.similaritySearch(prompt, 2, { 101 | documentId, 102 | }); 103 | 104 | const model = new ChatOpenAI({ 105 | apiKey: process.env.OPENAI_API_KEY, 106 | model: "gpt-4o-mini", 107 | temperature: 0, 108 | }); 109 | 110 | const allMessages = messages.map((message) => 111 | message.role === "user" 112 | ? new HumanMessage(message.content) 113 | : new AIMessage(message.content), 114 | ); 115 | 116 | const systemMessage = new SystemMessage( 117 | `You are a helpful assistant, here's some extra additional context that you can use to answer questions. Only use this information if it's relevant: 118 | 119 | ${result.map((r) => r.pageContent).join(" ")}`, 120 | ); 121 | 122 | allMessages.push(systemMessage); 123 | 124 | const stream = await model.stream(allMessages); 125 | 126 | return LangChainAdapter.toDataStreamResponse(stream); 127 | }; 128 | -------------------------------------------------------------------------------- /app/routes/api.document.upload.tsx: -------------------------------------------------------------------------------- 1 | import { type ActionFunctionArgs, json } from "@remix-run/node"; 2 | import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; 3 | import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; 4 | import { documents } from "../lib/db/schema"; 5 | import { count, eq } from "drizzle-orm"; 6 | import { MAX_FILE_COUNT, MAX_FILE_SIZE } from "../lib/constants"; 7 | import { Ratelimit } from "@upstash/ratelimit"; 8 | import { Redis } from "@upstash/redis"; 9 | import { authenticator } from "~/lib/auth"; 10 | import { db } from "~/lib/db"; 11 | 12 | export const action = async ({ request }: ActionFunctionArgs) => { 13 | const ip = request.headers.get("CF-Connecting-IP"); 14 | const identifier = ip ?? "global"; 15 | 16 | const ratelimit = new Ratelimit({ 17 | redis: Redis.fromEnv(), 18 | limiter: Ratelimit.fixedWindow(10, "60 s"), 19 | analytics: true, 20 | }); 21 | 22 | const { success, limit, remaining, reset } = 23 | await ratelimit.limit(identifier); 24 | 25 | if (!success) { 26 | return json( 27 | { 28 | error: "Rate limit exceeded", 29 | limit, 30 | remaining, 31 | reset, 32 | }, 33 | { 34 | status: 429, 35 | }, 36 | ); 37 | } 38 | const user = await authenticator.isAuthenticated(request); 39 | 40 | if (!user) { 41 | return json( 42 | { 43 | error: "Unauthorized", 44 | }, 45 | 401, 46 | ); 47 | } 48 | 49 | const userDocuments = await db 50 | .select({ count: count() }) 51 | .from(documents) 52 | .where(eq(documents.userId, user.id)); 53 | 54 | const userDocumentCount = userDocuments[0].count; 55 | 56 | if (userDocumentCount >= MAX_FILE_COUNT) { 57 | return json( 58 | { 59 | error: "You have reached the maximum limit of 3 documents", 60 | }, 61 | 403, 62 | ); 63 | } 64 | 65 | const { filename, fileSize }: { filename: string; fileSize: number } = 66 | await request.json(); 67 | 68 | // Check if the file size exceeds the limit 69 | if (fileSize > MAX_FILE_SIZE) { 70 | return json( 71 | { 72 | error: "File size exceeds the maximum limit of 10 MB", 73 | }, 74 | 413, // 413 Payload Too Large 75 | ); 76 | } 77 | 78 | try { 79 | const key = `${filename.split(".").slice(0, -1).join(".")}-${crypto.randomUUID()}.${filename.split(".").pop()}`; // e.g "file-usr_oahdfhj.pdf" 80 | 81 | const url = await getSignedUrl( 82 | new S3Client({ 83 | region: "auto", 84 | endpoint: `https://${process.env.CLOUDFLARE_R2_ACCOUNT_ID}.r2.cloudflarestorage.com`, 85 | credentials: { 86 | accessKeyId: process.env.CLOUDFLARE_R2_ACCESS_ID, 87 | secretAccessKey: process.env.CLOUDFLARE_R2_SECRET_KEY, 88 | }, 89 | }), 90 | new PutObjectCommand({ 91 | Bucket: process.env.CLOUDFLARE_R2_BUCKET_NAME, 92 | Key: key, 93 | ContentLength: fileSize, // Set the expected content length 94 | }), 95 | { 96 | expiresIn: 600, // 10 minutes 97 | }, 98 | ); 99 | return json({ 100 | url, 101 | title: filename, 102 | filename: key, 103 | }); 104 | } catch (error) { 105 | return json({ error: error.message }, { status: 500 }); 106 | } 107 | }; 108 | -------------------------------------------------------------------------------- /app/routes/api.document.vectorize.tsx: -------------------------------------------------------------------------------- 1 | import { json, redirect, type ActionFunctionArgs } from "@remix-run/node"; 2 | import { OpenAIEmbeddings } from "@langchain/openai"; 3 | import { NeonPostgres } from "@langchain/community/vectorstores/neon"; 4 | import { Document } from "langchain/document"; 5 | import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; 6 | import { extractText, getDocumentProxy } from "unpdf"; 7 | import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; 8 | import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; 9 | import { eq } from "drizzle-orm"; 10 | import { documents, vectorDatabases } from "../lib/db/schema"; 11 | import { generateId } from "../lib/db/utils/generate-id"; 12 | import { Ratelimit } from "@upstash/ratelimit"; 13 | import { Redis } from "@upstash/redis/cloudflare"; 14 | import { db } from "~/lib/db"; 15 | import { authenticator } from "~/lib/auth"; 16 | import { neonApiClient } from "~/lib/vector-db"; 17 | 18 | // vectorize document using langchain 19 | // request validation using Zod 20 | 21 | // check that user is authenticated, validate request body, generate embeddings and store them in associated vector store for the user, create document 22 | export const action = async ({ request }: ActionFunctionArgs) => { 23 | try { 24 | const ip = request.headers.get("CF-Connecting-IP"); 25 | const identifier = ip ?? "global"; 26 | 27 | const ratelimit = new Ratelimit({ 28 | redis: Redis.fromEnv(process.env), 29 | limiter: Ratelimit.fixedWindow(10, "60 s"), 30 | analytics: true, 31 | }); 32 | 33 | const { success, limit, remaining, reset } = 34 | await ratelimit.limit(identifier); 35 | 36 | if (!success) { 37 | return json( 38 | { 39 | error: "Rate limit exceeded", 40 | limit, 41 | remaining, 42 | reset, 43 | }, 44 | { 45 | status: 429, 46 | }, 47 | ); 48 | } 49 | 50 | const user = await authenticator.isAuthenticated(request); 51 | 52 | if (!user) { 53 | return json( 54 | { 55 | error: "Unauthorized", 56 | }, 57 | 401, 58 | ); 59 | } 60 | 61 | const { filename, title }: { filename: string; title: string } = 62 | await request.json(); 63 | 64 | // get user's project ID, and get the connection string 65 | 66 | const vectorDb = await db 67 | .select() 68 | .from(vectorDatabases) 69 | .where(eq(vectorDatabases.userId, user.id)); 70 | 71 | const { data, error } = await neonApiClient.GET( 72 | "/projects/{project_id}/connection_uri", 73 | { 74 | params: { 75 | path: { 76 | project_id: vectorDb[0].vectorDbId, 77 | }, 78 | query: { 79 | role_name: "neondb_owner", 80 | database_name: "neondb", 81 | }, 82 | }, 83 | }, 84 | ); 85 | 86 | if (error) { 87 | return json({ 88 | error: error, 89 | }); 90 | } 91 | 92 | const embeddings = new OpenAIEmbeddings({ 93 | apiKey: process.env.OPENAI_API_KEY, 94 | dimensions: 1536, 95 | model: "text-embedding-3-small", 96 | }); 97 | 98 | const vectorStore = await NeonPostgres.initialize(embeddings, { 99 | connectionString: data.uri, 100 | tableName: "embeddings", 101 | columns: { 102 | contentColumnName: "content", 103 | metadataColumnName: "metadata", 104 | vectorColumnName: "embedding", 105 | }, 106 | }); 107 | 108 | const url = await getSignedUrl( 109 | new S3Client({ 110 | region: "auto", 111 | endpoint: `https://${process.env.CLOUDFLARE_R2_ACCOUNT_ID}.r2.cloudflarestorage.com`, 112 | credentials: { 113 | accessKeyId: process.env.CLOUDFLARE_R2_ACCESS_ID, 114 | secretAccessKey: process.env.CLOUDFLARE_R2_SECRET_KEY, 115 | }, 116 | }), 117 | new GetObjectCommand({ 118 | Bucket: process.env.CLOUDFLARE_R2_BUCKET_NAME, 119 | Key: filename, 120 | }), 121 | { 122 | expiresIn: 600, 123 | }, 124 | ); 125 | 126 | const buffer = await fetch(url).then((res) => res.arrayBuffer()); 127 | 128 | const pdf = await getDocumentProxy(new Uint8Array(buffer)); 129 | // Extract text from PDF 130 | const { text } = await extractText(pdf, { mergePages: true }); 131 | 132 | const splitter = new RecursiveCharacterTextSplitter({ 133 | chunkSize: 1000, 134 | chunkOverlap: 100, 135 | }); 136 | 137 | const documentId = generateId({ object: "document" }); 138 | 139 | const docOutput = await splitter.splitDocuments([ 140 | new Document({ pageContent: text, metadata: { documentId } }), 141 | ]); 142 | 143 | // add ids and make it the same length as the docOutput 144 | const result = await vectorStore.addDocuments(docOutput); 145 | 146 | const document = await db 147 | .insert(documents) 148 | .values({ 149 | documentId: documentId, 150 | userId: user.id, 151 | title: title, 152 | fileName: filename, 153 | }) 154 | .returning(); 155 | 156 | return json({ document: document[0] }, 201); 157 | } catch (error) { 158 | return json( 159 | { 160 | error: error.message, 161 | }, 162 | 400, 163 | ); 164 | } 165 | }; 166 | -------------------------------------------------------------------------------- /app/routes/document.new.tsx: -------------------------------------------------------------------------------- 1 | import { json, type LoaderFunctionArgs } from "@remix-run/node"; 2 | import { Upload as DocumentUploader } from "../components/documents/upload"; 3 | import { Heading } from "../components/ui/heading"; 4 | import { authenticator } from "~/lib/auth"; 5 | 6 | export async function loader({ request }: LoaderFunctionArgs) { 7 | return await authenticator.isAuthenticated(request, { 8 | failureRedirect: "/login", 9 | }); 10 | } 11 | 12 | export default function NewDocument() { 13 | return ( 14 |
15 |
16 | 17 | Upload document 18 | 19 | 20 |
21 |
22 | ); 23 | } 24 | -------------------------------------------------------------------------------- /app/routes/documents.tsx: -------------------------------------------------------------------------------- 1 | import { json, redirect, type LoaderFunctionArgs } from "@remix-run/node"; 2 | import { Form, Link, useLoaderData } from "@remix-run/react"; 3 | import { desc, eq } from "drizzle-orm"; 4 | import { Plus } from "../components/icons/plus"; 5 | import { documents } from "../lib/db/schema"; 6 | import { Button } from "../components/ui/button"; 7 | import { Heading } from "../components/ui/heading"; 8 | import { Upload as DocumentUploader } from "../components/documents/upload"; 9 | import { formatDistanceToNowStrict } from "date-fns"; 10 | import { MAX_FILE_COUNT } from "../lib/constants"; 11 | import { authenticator } from "~/lib/auth"; 12 | import { db } from "~/lib/db"; 13 | 14 | export async function loader({ request }: LoaderFunctionArgs) { 15 | try { 16 | const user = await authenticator.isAuthenticated(request); 17 | 18 | if (!user) { 19 | return redirect("/login"); 20 | } 21 | 22 | const allDocuments = await db 23 | .select() 24 | .from(documents) 25 | .where(eq(documents.userId, user.id)) 26 | .orderBy(desc(documents.createdAt)); 27 | 28 | return json({ allDocuments }); 29 | } catch (error) { 30 | throw new Error(`Error loading documents: ${error}`); 31 | } 32 | } 33 | 34 | export default function Documents() { 35 | const { allDocuments } = useLoaderData(); 36 | 37 | return ( 38 |
39 |
40 | 41 | Documents 42 | 43 | {allDocuments.length > 0 && ( 44 |
45 | 54 |
55 | )} 56 |
57 | 58 |
59 | {allDocuments.length === 0 ? ( 60 | <> 61 | {" "} 62 | 63 | 64 | ) : ( 65 | <> 66 | {allDocuments.map((document) => ( 67 | 72 | 76 | {document.title} 77 | 78 |

79 | {formatDistanceToNowStrict(new Date(document.createdAt))} ago 80 |

81 | 82 | ))} 83 | 84 | )} 85 |
86 |
87 | ); 88 | } 89 | -------------------------------------------------------------------------------- /app/routes/documents_.$id.chat.tsx: -------------------------------------------------------------------------------- 1 | import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; 2 | import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; 3 | import { json, redirect, type LoaderFunctionArgs } from "@remix-run/node"; 4 | import { useLoaderData } from "@remix-run/react"; 5 | import { eq } from "drizzle-orm"; 6 | import { Button } from "../components/ui/button"; 7 | import { Bot } from "../components/icons/bot"; 8 | import { Send } from "../components/icons/send"; 9 | import { Spinner } from "../components/icons/spinner"; 10 | import { User } from "../components/icons/user"; 11 | 12 | import { documents } from "../lib/db/schema"; 13 | import { useChat } from "ai/react"; 14 | import { Input } from "../components/ui/input"; 15 | import { authenticator } from "~/lib/auth"; 16 | import { db } from "~/lib/db"; 17 | 18 | export const loader = async ({ request, params }: LoaderFunctionArgs) => { 19 | const documentId = params.id as string; 20 | 21 | try { 22 | const user = await authenticator.isAuthenticated(request); 23 | 24 | if (!user) { 25 | return redirect("/login"); 26 | } 27 | 28 | const document = await db 29 | .select() 30 | .from(documents) 31 | .where(eq(documents.documentId, documentId)); 32 | 33 | const url = await getSignedUrl( 34 | new S3Client({ 35 | region: "auto", 36 | endpoint: `https://${process.env.CLOUDFLARE_R2_ACCOUNT_ID}.r2.cloudflarestorage.com`, 37 | credentials: { 38 | accessKeyId: process.env.CLOUDFLARE_R2_ACCESS_ID, 39 | secretAccessKey: process.env.CLOUDFLARE_R2_SECRET_KEY, 40 | }, 41 | }), 42 | new GetObjectCommand({ 43 | Bucket: process.env.CLOUDFLARE_R2_BUCKET_NAME, 44 | Key: document[0].fileName, 45 | }), 46 | { 47 | expiresIn: 600, 48 | }, 49 | ); 50 | 51 | return json({ 52 | documentId, 53 | url, 54 | }); 55 | } catch (error) { 56 | throw new Error(`Error loading document: ${error}`); 57 | } 58 | }; 59 | 60 | export default function DocumentChat() { 61 | const { documentId, url } = useLoaderData(); 62 | 63 | const { messages, input, handleInputChange, handleSubmit, isLoading } = 64 | useChat({ 65 | body: { documentId }, 66 | api: "/api/document/chat", 67 | }); 68 | 69 | return ( 70 |
71 |