├── bun.lockb ├── public ├── favicon.ico └── firecrawl-logo-with-fire.png ├── postcss.config.mjs ├── lib ├── utils.ts ├── context-processor.ts ├── upstash-search.ts ├── rate-limit.ts └── storage.ts ├── vercel.json ├── app ├── api │ ├── check-env │ │ └── route.ts │ ├── indexes │ │ └── route.ts │ ├── scrape │ │ └── route.ts │ ├── firestarter │ │ ├── debug │ │ │ └── route.ts │ │ ├── create │ │ │ └── route.ts │ │ └── query │ │ │ └── route.ts │ └── v1 │ │ └── chat │ │ └── completions │ │ └── route.ts ├── layout.tsx ├── debug │ └── page.tsx ├── indexes │ └── page.tsx ├── globals.css ├── page.tsx └── dashboard │ └── page.tsx ├── eslint.config.mjs ├── components.json ├── next.config.ts ├── .gitignore ├── components └── ui │ ├── sonner.tsx │ ├── separator.tsx │ ├── progress.tsx │ ├── input.tsx │ ├── button.tsx │ └── dialog.tsx ├── tsconfig.json ├── test-sources.md ├── tailwind.config.ts ├── package.json ├── hooks └── useStorage.ts ├── firestarter.config.ts └── README.md /bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mendableai/firestarter/HEAD/bun.lockb -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mendableai/firestarter/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /postcss.config.mjs: -------------------------------------------------------------------------------- 1 | const config = { 2 | plugins: ["@tailwindcss/postcss"], 3 | }; 4 | 5 | export default config; 6 | -------------------------------------------------------------------------------- /public/firecrawl-logo-with-fire.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mendableai/firestarter/HEAD/public/firecrawl-logo-with-fire.png -------------------------------------------------------------------------------- /lib/utils.ts: -------------------------------------------------------------------------------- 1 | import { type ClassValue, clsx } from "clsx" 2 | import { twMerge } from "tailwind-merge" 3 | 4 | export function cn(...inputs: ClassValue[]) { 5 | return twMerge(clsx(inputs)) 6 | } 7 | -------------------------------------------------------------------------------- /vercel.json: -------------------------------------------------------------------------------- 1 | { 2 | "functions": { 3 | "app/api/*/route.ts": { 4 | "maxDuration": 300 5 | }, 6 | "app/api/enrich/route.ts": { 7 | "maxDuration": 300 8 | }, 9 | "app/firesearch/search.tsx": { 10 | "maxDuration": 300 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /app/api/check-env/route.ts: -------------------------------------------------------------------------------- 1 | import { NextResponse } from 'next/server'; 2 | 3 | export async function GET() { 4 | const environmentStatus = { 5 | FIRECRAWL_API_KEY: !!process.env.FIRECRAWL_API_KEY, 6 | OPENAI_API_KEY: !!process.env.OPENAI_API_KEY, 7 | ANTHROPIC_API_KEY: !!process.env.ANTHROPIC_API_KEY, 8 | DISABLE_CHATBOT_CREATION: process.env.DISABLE_CHATBOT_CREATION === 'true', 9 | }; 10 | 11 | return NextResponse.json({ environmentStatus }); 12 | } -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import { dirname } from "path"; 2 | import { fileURLToPath } from "url"; 3 | import { FlatCompat } from "@eslint/eslintrc"; 4 | 5 | const __filename = fileURLToPath(import.meta.url); 6 | const __dirname = dirname(__filename); 7 | 8 | const compat = new FlatCompat({ 9 | baseDirectory: __dirname, 10 | }); 11 | 12 | const eslintConfig = [ 13 | ...compat.extends("next/core-web-vitals", "next/typescript"), 14 | ]; 15 | 16 | export default eslintConfig; 17 | -------------------------------------------------------------------------------- /components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "new-york", 4 | "rsc": true, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "", 8 | "css": "app/globals.css", 9 | "baseColor": "neutral", 10 | "cssVariables": true, 11 | "prefix": "" 12 | }, 13 | "aliases": { 14 | "components": "@/components", 15 | "utils": "@/lib/utils", 16 | "ui": "@/components/ui", 17 | "lib": "@/lib", 18 | "hooks": "@/hooks" 19 | }, 20 | "iconLibrary": "lucide" 21 | } -------------------------------------------------------------------------------- /next.config.ts: -------------------------------------------------------------------------------- 1 | import type { NextConfig } from "next"; 2 | 3 | const nextConfig: NextConfig = { 4 | // Remove assetPrefix to fix image loading issues 5 | images: { 6 | remotePatterns: [ 7 | { 8 | protocol: 'https', 9 | hostname: 'www.google.com', 10 | pathname: '/s2/favicons**', 11 | }, 12 | { 13 | protocol: 'https', 14 | hostname: '**', 15 | }, 16 | { 17 | protocol: 'http', 18 | hostname: '**', 19 | }, 20 | ], 21 | }, 22 | }; 23 | 24 | export default nextConfig; 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.* 7 | .yarn/* 8 | !.yarn/patches 9 | !.yarn/plugins 10 | !.yarn/releases 11 | !.yarn/versions 12 | 13 | # testing 14 | /coverage 15 | 16 | # next.js 17 | /.next/ 18 | /out/ 19 | 20 | # production 21 | /build 22 | 23 | # misc 24 | .DS_Store 25 | *.pem 26 | 27 | # debug 28 | npm-debug.log* 29 | yarn-debug.log* 30 | yarn-error.log* 31 | .pnpm-debug.log* 32 | 33 | # env files (can opt-in for committing if needed) 34 | .env* 35 | 36 | # vercel 37 | .vercel 38 | 39 | # typescript 40 | *.tsbuildinfo 41 | next-env.d.ts 42 | -------------------------------------------------------------------------------- /components/ui/sonner.tsx: -------------------------------------------------------------------------------- 1 | "use client" 2 | 3 | import { useTheme } from "next-themes" 4 | import { Toaster as Sonner, ToasterProps } from "sonner" 5 | 6 | const Toaster = ({ ...props }: ToasterProps) => { 7 | const { theme = "system" } = useTheme() 8 | 9 | return ( 10 | 22 | ) 23 | } 24 | 25 | export { Toaster } 26 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2017", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": true, 8 | "noEmit": true, 9 | "esModuleInterop": true, 10 | "module": "esnext", 11 | "moduleResolution": "bundler", 12 | "resolveJsonModule": true, 13 | "isolatedModules": true, 14 | "jsx": "preserve", 15 | "incremental": true, 16 | "plugins": [ 17 | { 18 | "name": "next" 19 | } 20 | ], 21 | "paths": { 22 | "@/*": ["./*"] 23 | } 24 | }, 25 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], 26 | "exclude": ["node_modules", "hostedTools"] 27 | } 28 | -------------------------------------------------------------------------------- /lib/context-processor.ts: -------------------------------------------------------------------------------- 1 | export interface Source { 2 | url: string; 3 | title: string; 4 | content?: string; 5 | quality?: number; 6 | } 7 | 8 | export class ContextProcessor { 9 | async processSources( 10 | _query: string, 11 | sources: Source[] 12 | ): Promise { 13 | // For now, return sources with basic processing 14 | // In a full implementation, this would use AI to: 15 | // 1. Extract relevant snippets from each source 16 | // 2. Rank sources by relevance 17 | // 3. Summarize key points 18 | 19 | return sources 20 | .filter(source => source.content && source.content.length > 100) 21 | .sort((a, b) => (b.quality || 0) - (a.quality || 0)) 22 | .slice(0, 10); // Return top 10 sources 23 | } 24 | } -------------------------------------------------------------------------------- /components/ui/separator.tsx: -------------------------------------------------------------------------------- 1 | "use client" 2 | 3 | import * as React from "react" 4 | import * as SeparatorPrimitive from "@radix-ui/react-separator" 5 | 6 | import { cn } from "@/lib/utils" 7 | 8 | function Separator({ 9 | className, 10 | orientation = "horizontal", 11 | decorative = true, 12 | ...props 13 | }: React.ComponentProps) { 14 | return ( 15 | 25 | ) 26 | } 27 | 28 | export { Separator } -------------------------------------------------------------------------------- /lib/upstash-search.ts: -------------------------------------------------------------------------------- 1 | import { Search } from '@upstash/search' 2 | 3 | // Initialize Upstash Search client 4 | const searchClient = new Search({ 5 | url: process.env.UPSTASH_SEARCH_REST_URL!, 6 | token: process.env.UPSTASH_SEARCH_REST_TOKEN!, 7 | }) 8 | 9 | // Create a search index for firestarter documents 10 | export const searchIndex = searchClient.index('firestarter') 11 | 12 | export interface FirestarterContent { 13 | text: string 14 | url: string 15 | title: string 16 | [key: string]: unknown // Add index signature for Upstash type compatibility 17 | } 18 | 19 | export interface FirestarterIndex { 20 | namespace: string 21 | url: string 22 | pagesCrawled: number 23 | crawlDate: string 24 | metadata: { 25 | title: string 26 | description?: string 27 | favicon?: string 28 | ogImage?: string 29 | } 30 | } -------------------------------------------------------------------------------- /components/ui/progress.tsx: -------------------------------------------------------------------------------- 1 | "use client" 2 | 3 | import * as React from "react" 4 | import * as ProgressPrimitive from "@radix-ui/react-progress" 5 | 6 | import { cn } from "@/lib/utils" 7 | 8 | function Progress({ 9 | className, 10 | value, 11 | ...props 12 | }: React.ComponentProps) { 13 | return ( 14 | 22 | 27 | 28 | ) 29 | } 30 | 31 | export { Progress } 32 | -------------------------------------------------------------------------------- /components/ui/input.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | 3 | import { cn } from "@/lib/utils" 4 | 5 | function Input({ className, type, ...props }: React.ComponentProps<"input">) { 6 | return ( 7 | 18 | ) 19 | } 20 | 21 | export { Input } 22 | -------------------------------------------------------------------------------- /test-sources.md: -------------------------------------------------------------------------------- 1 | # Testing Sources in Firestarter 2 | 3 | ## Current Implementation 4 | 5 | The sources from RAG results are now included in the streaming response using Vercel AI SDK: 6 | 7 | 1. **API Route** (`/api/firestarter/query/route.ts`): 8 | - Sources are prepared from the search results 9 | - When streaming is enabled, sources are included via `toDataStreamResponse({ data: { sources } })` 10 | - Each source includes: url, title, and snippet 11 | 12 | 2. **Dashboard** (`/app/dashboard/page.tsx`): 13 | - Parses the streaming response for sources data 14 | - Updates messages with sources when found 15 | - Displays sources below each assistant message 16 | 17 | 3. **Source Display**: 18 | - Shows as "References:" section below the answer 19 | - Each source shows: 20 | - Citation number [1], [2], etc. 21 | - Title (truncated to 60 chars) 22 | - Snippet (truncated to 100 chars) 23 | - URL 24 | - Clickable link with external icon 25 | 26 | ## Testing Steps 27 | 28 | 1. Start the dev server: `npm run dev` 29 | 2. Create a new chatbot by entering a URL 30 | 3. After crawling completes, ask a question 31 | 4. Verify that sources appear below the answer 32 | 5. Click on sources to verify they open in new tabs 33 | 34 | ## Expected Behavior 35 | 36 | When you ask a question, the response should: 37 | 1. Stream the answer text 38 | 2. Include relevant sources at the bottom 39 | 3. Sources should be clickable and open in new tabs 40 | 4. Each source should show a preview snippet -------------------------------------------------------------------------------- /app/api/indexes/route.ts: -------------------------------------------------------------------------------- 1 | import { NextRequest, NextResponse } from 'next/server' 2 | import { getIndexes, saveIndex, deleteIndex, IndexMetadata } from '@/lib/storage' 3 | 4 | export async function GET() { 5 | try { 6 | const indexes = await getIndexes() 7 | return NextResponse.json({ indexes: indexes || [] }) 8 | } catch { 9 | // Return empty array instead of error to allow app to function 10 | console.error('Failed to get indexes') 11 | return NextResponse.json({ indexes: [] }) 12 | } 13 | } 14 | 15 | export async function POST(request: NextRequest) { 16 | try { 17 | const index: IndexMetadata = await request.json() 18 | await saveIndex(index) 19 | return NextResponse.json({ success: true }) 20 | } catch { 21 | // Return success anyway to allow app to continue 22 | console.error('Failed to save index') 23 | return NextResponse.json({ success: true, warning: 'Index saved locally only' }) 24 | } 25 | } 26 | 27 | export async function DELETE(request: NextRequest) { 28 | try { 29 | const { searchParams } = new URL(request.url) 30 | const namespace = searchParams.get('namespace') 31 | 32 | if (!namespace) { 33 | return NextResponse.json({ error: 'Namespace is required' }, { status: 400 }) 34 | } 35 | 36 | await deleteIndex(namespace) 37 | return NextResponse.json({ success: true }) 38 | } catch { 39 | // Return success anyway to allow app to continue 40 | console.error('Failed to delete index') 41 | return NextResponse.json({ success: true, warning: 'Index deleted locally only' }) 42 | } 43 | } -------------------------------------------------------------------------------- /lib/rate-limit.ts: -------------------------------------------------------------------------------- 1 | import { Ratelimit } from "@upstash/ratelimit"; 2 | import { Redis } from "@upstash/redis"; 3 | import { NextRequest } from "next/server"; 4 | 5 | // Create a new ratelimiter that allows 50 requests per day per IP per endpoint 6 | export const getRateLimiter = (endpoint: string) => { 7 | // Check if we're in a production environment to apply rate limiting 8 | // In development, we don't want to be rate limited for testing 9 | if (process.env.NODE_ENV !== "production" && !process.env.UPSTASH_REDIS_REST_URL) { 10 | return null; 11 | } 12 | 13 | // Requires the following environment variables: 14 | // UPSTASH_REDIS_REST_URL 15 | // UPSTASH_REDIS_REST_TOKEN 16 | const redis = Redis.fromEnv(); 17 | 18 | return new Ratelimit({ 19 | redis, 20 | limiter: Ratelimit.fixedWindow(50, "1 d"), 21 | analytics: true, 22 | prefix: `ratelimit:${endpoint}`, 23 | }); 24 | }; 25 | 26 | // Helper function to get the IP from a NextRequest or default to a placeholder 27 | export const getIP = (request: NextRequest): string => { 28 | const forwarded = request.headers.get("x-forwarded-for"); 29 | const realIp = request.headers.get("x-real-ip"); 30 | 31 | if (forwarded) { 32 | return forwarded.split(/, /)[0]; 33 | } 34 | 35 | if (realIp) { 36 | return realIp; 37 | } 38 | 39 | // Default to placeholder IP if none found 40 | return "127.0.0.1"; 41 | }; 42 | 43 | // Helper function to check if a request is rate limited 44 | export const isRateLimited = async (request: NextRequest, endpoint: string) => { 45 | const limiter = getRateLimiter(endpoint); 46 | 47 | // If no limiter is available (e.g., in development), allow the request 48 | if (!limiter) { 49 | return { success: true, limit: 50, remaining: 50 }; 50 | } 51 | 52 | // Get the IP from the request 53 | const ip = getIP(request); 54 | 55 | // Check if the IP has exceeded the rate limit 56 | const result = await limiter.limit(ip); 57 | 58 | return { 59 | success: result.success, 60 | limit: result.limit, 61 | remaining: result.remaining, 62 | }; 63 | }; -------------------------------------------------------------------------------- /tailwind.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from "tailwindcss"; 2 | import defaultTheme from "tailwindcss/defaultTheme"; 3 | 4 | export default { 5 | content: [ 6 | "./app/**/*.{js,ts,jsx,tsx}", 7 | "./pages/**/*.{js,ts,jsx,tsx}", 8 | "./components/**/*.{js,ts,jsx,tsx}", 9 | ], 10 | theme: { 11 | container: { 12 | center: true, 13 | padding: "2rem", 14 | screens: { 15 | "2xl": "1400px", 16 | }, 17 | }, 18 | extend: { 19 | colors: { 20 | border: "hsl(var(--border))", 21 | input: "hsl(var(--input))", 22 | ring: "hsl(var(--ring))", 23 | background: "hsl(var(--background))", 24 | foreground: "hsl(var(--foreground))", 25 | primary: { 26 | DEFAULT: "hsl(var(--primary))", 27 | foreground: "hsl(var(--primary-foreground))", 28 | }, 29 | secondary: { 30 | DEFAULT: "hsl(var(--secondary))", 31 | foreground: "hsl(var(--secondary-foreground))", 32 | }, 33 | destructive: { 34 | DEFAULT: "hsl(var(--destructive))", 35 | foreground: "hsl(var(--destructive-foreground))", 36 | }, 37 | muted: { 38 | DEFAULT: "hsl(var(--muted))", 39 | foreground: "hsl(var(--muted-foreground))", 40 | }, 41 | accent: { 42 | DEFAULT: "hsl(var(--accent))", 43 | foreground: "hsl(var(--accent-foreground))", 44 | }, 45 | popover: { 46 | DEFAULT: "hsl(var(--popover))", 47 | foreground: "hsl(var(--popover-foreground))", 48 | }, 49 | card: { 50 | DEFAULT: "hsl(var(--card))", 51 | foreground: "hsl(var(--card-foreground))", 52 | }, 53 | }, 54 | borderRadius: { 55 | lg: "var(--radius)", 56 | md: "calc(var(--radius) - 2px)", 57 | sm: "calc(var(--radius) - 4px)", 58 | }, 59 | fontFamily: { 60 | sans: ["var(--font-inter)", ...defaultTheme.fontFamily.sans], 61 | mono: defaultTheme.fontFamily.mono, 62 | }, 63 | }, 64 | }, 65 | plugins: [require("tailwindcss-animate")], 66 | } satisfies Config; -------------------------------------------------------------------------------- /app/layout.tsx: -------------------------------------------------------------------------------- 1 | import type { Metadata } from "next"; 2 | import { Inter } from "next/font/google"; 3 | import "./globals.css"; 4 | import { cn } from "@/lib/utils"; 5 | import { Analytics } from "@vercel/analytics/next"; 6 | 7 | const inter = Inter({ 8 | variable: "--font-inter", 9 | subsets: ["latin"], 10 | }); 11 | 12 | export const metadata: Metadata = { 13 | title: "Firecrawl Tools - AI-Powered Web Scraping & Data Enrichment", 14 | description: "Transform websites into structured data with Firecrawl's suite of AI tools. Create chatbots, enrich CSVs, search intelligently, and generate images from URLs.", 15 | metadataBase: new URL(process.env.NEXT_PUBLIC_URL || "https://firecrawl.dev"), 16 | openGraph: { 17 | title: "Firecrawl Tools - AI-Powered Web Scraping & Data Enrichment", 18 | description: "Transform websites into structured data with Firecrawl's suite of AI tools. Create chatbots, enrich CSVs, search intelligently, and generate images from URLs.", 19 | url: "/", 20 | siteName: "Firecrawl Tools", 21 | images: [ 22 | { 23 | url: "/firecrawl-logo-with-fire.png", 24 | width: 1200, 25 | height: 630, 26 | alt: "Firecrawl - AI-Powered Web Scraping", 27 | }, 28 | ], 29 | locale: "en_US", 30 | type: "website", 31 | }, 32 | twitter: { 33 | card: "summary_large_image", 34 | title: "Firecrawl Tools - AI-Powered Web Scraping", 35 | description: "Transform websites into structured data with AI", 36 | images: ["/firecrawl-logo-with-fire.png"], 37 | creator: "@firecrawl_dev", 38 | }, 39 | icons: { 40 | icon: "/favicon.ico", 41 | shortcut: "/favicon.ico", 42 | apple: "/favicon.ico", 43 | }, 44 | robots: { 45 | index: true, 46 | follow: true, 47 | googleBot: { 48 | index: true, 49 | follow: true, 50 | "max-video-preview": -1, 51 | "max-image-preview": "large", 52 | "max-snippet": -1, 53 | }, 54 | }, 55 | }; 56 | 57 | export default function RootLayout({ 58 | children, 59 | }: Readonly<{ 60 | children: React.ReactNode; 61 | }>) { 62 | return ( 63 | 64 | 71 |
72 | {children} 73 |
74 | 75 | 76 | 77 | ); 78 | } 79 | -------------------------------------------------------------------------------- /app/api/scrape/route.ts: -------------------------------------------------------------------------------- 1 | import { NextRequest, NextResponse } from 'next/server'; 2 | import FirecrawlApp from '@mendable/firecrawl-js'; 3 | import { serverConfig as config } from '@/firestarter.config'; 4 | 5 | interface ScrapeRequestBody { 6 | url?: string; 7 | urls?: string[]; 8 | [key: string]: unknown; 9 | } 10 | 11 | interface ScrapeResult { 12 | success: boolean; 13 | data?: Record; 14 | error?: string; 15 | } 16 | 17 | interface ApiError extends Error { 18 | status?: number; 19 | } 20 | 21 | export async function POST(request: NextRequest) { 22 | // Check rate limit if enabled 23 | if (config.rateLimits.scrape) { 24 | const ip = request.headers.get('x-forwarded-for')?.split(',')[0] || 25 | request.headers.get('x-real-ip') || 26 | '127.0.0.1'; 27 | 28 | const rateLimit = await config.rateLimits.scrape.limit(ip); 29 | 30 | if (!rateLimit.success) { 31 | return NextResponse.json({ 32 | success: false, 33 | error: 'Rate limit exceeded. Please try again later.' 34 | }, { 35 | status: 429, 36 | headers: { 37 | 'X-RateLimit-Limit': rateLimit.limit.toString(), 38 | 'X-RateLimit-Remaining': rateLimit.remaining.toString(), 39 | } 40 | }); 41 | } 42 | } 43 | 44 | let apiKey = process.env.FIRECRAWL_API_KEY; 45 | 46 | if (!apiKey) { 47 | const headerApiKey = request.headers.get('X-Firecrawl-API-Key'); 48 | 49 | if (!headerApiKey) { 50 | return NextResponse.json({ 51 | success: false, 52 | error: 'API configuration error. Please try again later or contact support.' 53 | }, { status: 500 }); 54 | } 55 | 56 | apiKey = headerApiKey; 57 | } 58 | 59 | try { 60 | const app = new FirecrawlApp({ apiKey }); 61 | const body = await request.json() as ScrapeRequestBody; 62 | const { url, urls, ...params } = body; 63 | 64 | let result: ScrapeResult; 65 | 66 | if (url && typeof url === 'string') { 67 | result = await app.scrapeUrl(url, params) as ScrapeResult; 68 | } else if (urls && Array.isArray(urls)) { 69 | result = await app.batchScrapeUrls(urls, params) as ScrapeResult; 70 | } else { 71 | return NextResponse.json({ success: false, error: 'Invalid request format. Please check your input and try again.' }, { status: 400 }); 72 | } 73 | 74 | return NextResponse.json(result); 75 | 76 | } catch (error: unknown) { 77 | const err = error as ApiError; 78 | const errorStatus = typeof err.status === 'number' ? err.status : 500; 79 | return NextResponse.json({ success: false, error: 'An error occurred while processing your request. Please try again later.' }, { status: errorStatus }); 80 | } 81 | } -------------------------------------------------------------------------------- /app/debug/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import { useState } from 'react' 4 | import { Button } from "@/components/ui/button" 5 | import { Input } from "@/components/ui/input" 6 | 7 | export default function DebugPage() { 8 | const [namespace, setNamespace] = useState('firecrawl-dev-1749845075753') 9 | interface DebugResults { 10 | [key: string]: unknown 11 | } 12 | const [results, setResults] = useState(null) 13 | const [loading, setLoading] = useState(false) 14 | 15 | const runDebug = async () => { 16 | setLoading(true) 17 | try { 18 | const response = await fetch(`/api/firestarter/debug?namespace=${namespace}`) 19 | const data = await response.json() 20 | setResults(data) 21 | } catch (error) { 22 | setResults({ error: error instanceof Error ? error.message : 'Unknown error' }) 23 | } finally { 24 | setLoading(false) 25 | } 26 | } 27 | 28 | return ( 29 |
30 |
31 |

Firestarter Debug

32 | 33 |
34 |
35 |
36 | 37 | setNamespace(e.target.value)} 40 | placeholder="Enter namespace to debug" 41 | /> 42 |
43 | 44 | 47 |
48 | 49 | {results && ( 50 |
51 |

Results:

52 | 
53 |                 {JSON.stringify(results, null, 2)}
54 |
55 |
56 | )} 57 |
58 | 59 |
60 |

Instructions:

61 |
    62 |
  1. First, go to the Indexes page and crawl a website
    2. 63 |
  2. Note the namespace that's returned (it will be shown in the response)
    3. 64 |
  3. Enter that namespace above and click "Run Debug"
    4. 65 |
  4. This will show you what documents are stored in Upstash
    5. 66 |
67 |
68 |
69 |
70 | ) 71 | } -------------------------------------------------------------------------------- /app/api/firestarter/debug/route.ts: -------------------------------------------------------------------------------- 1 | import { NextRequest, NextResponse } from 'next/server' 2 | import { searchIndex } from '@/lib/upstash-search' 3 | 4 | export async function GET(request: NextRequest) { 5 | try { 6 | const searchParams = request.nextUrl.searchParams 7 | const namespace = searchParams.get('namespace') 8 | 9 | 10 | // Try different search approaches 11 | interface DebugResults { 12 | namespaceSearch?: { 13 | count: number 14 | items: unknown[] 15 | } 16 | namespaceSearchError?: string 17 | allSearch?: { 18 | count: number 19 | items: unknown[] 20 | } 21 | allSearchError?: string 22 | semanticSearch?: { 23 | count: number 24 | items: unknown[] 25 | } 26 | semanticSearchError?: string 27 | } 28 | const results: DebugResults = {} 29 | 30 | // 1. Search with namespace filter 31 | if (namespace) { 32 | try { 33 | const namespaceSearch = await searchIndex.search({ 34 | query: '*', 35 | filter: `metadata.namespace = "${namespace}"`, 36 | limit: 10 37 | }) 38 | results.namespaceSearch = { 39 | count: namespaceSearch.length, 40 | items: namespaceSearch 41 | } 42 | } catch (e) { 43 | results.namespaceSearchError = e instanceof Error ? e.message : String(e) 44 | } 45 | } 46 | 47 | // 2. Search without filter 48 | try { 49 | const allSearch = await searchIndex.search({ 50 | query: namespace || 'test', 51 | limit: 10 52 | }) 53 | results.allSearch = { 54 | count: allSearch.length, 55 | items: allSearch 56 | } 57 | } catch (e) { 58 | results.allSearchError = e instanceof Error ? e.message : String(e) 59 | } 60 | 61 | // 3. Try semantic search 62 | try { 63 | const semanticSearch = await searchIndex.search({ 64 | query: 'homepage website content', 65 | limit: 10 66 | }) 67 | results.semanticSearch = { 68 | count: semanticSearch.length, 69 | items: semanticSearch 70 | } 71 | } catch (e) { 72 | results.semanticSearchError = e instanceof Error ? e.message : String(e) 73 | } 74 | 75 | return NextResponse.json({ 76 | success: true, 77 | namespace: namespace, 78 | results: results, 79 | upstashUrl: process.env.UPSTASH_SEARCH_REST_URL ? 'Configured' : 'Not configured', 80 | upstashToken: process.env.UPSTASH_SEARCH_REST_TOKEN ? 'Configured' : 'Not configured' 81 | }) 82 | } catch (error) { 83 | return NextResponse.json( 84 | { 85 | error: 'Debug endpoint error', 86 | details: error instanceof Error ? error.message : 'Unknown error' 87 | }, 88 | { status: 500 } 89 | ) 90 | } 91 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firecrawl-template", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev --turbopack", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@ai-sdk/anthropic": "^1.2.12", 13 | "@ai-sdk/google": "^1.2.18", 14 | "@ai-sdk/groq": "^1.2.9", 15 | "@ai-sdk/openai": "^1.3.22", 16 | "@ai-sdk/openai-compatible": "^0.2.14", 17 | "@fal-ai/client": "^1.4.0", 18 | "@hookform/resolvers": "^5.0.1", 19 | "@langchain/core": "^0.3.57", 20 | "@langchain/langgraph": "^0.2.74", 21 | "@langchain/openai": "^0.5.11", 22 | "@mendable/firecrawl-js": "^1.25.1", 23 | "@openai/agents": "^0.0.1", 24 | "@radix-ui/react-accordion": "^1.2.10", 25 | "@radix-ui/react-alert-dialog": "^1.1.13", 26 | "@radix-ui/react-aspect-ratio": "^1.1.6", 27 | "@radix-ui/react-avatar": "^1.1.9", 28 | "@radix-ui/react-checkbox": "^1.3.1", 29 | "@radix-ui/react-collapsible": "^1.1.10", 30 | "@radix-ui/react-context-menu": "^2.2.14", 31 | "@radix-ui/react-dialog": "^1.1.13", 32 | "@radix-ui/react-dropdown-menu": "^2.1.14", 33 | "@radix-ui/react-hover-card": "^1.1.13", 34 | "@radix-ui/react-label": "^2.1.6", 35 | "@radix-ui/react-menubar": "^1.1.14", 36 | "@radix-ui/react-navigation-menu": "^1.2.12", 37 | "@radix-ui/react-popover": "^1.1.13", 38 | "@radix-ui/react-progress": "^1.1.6", 39 | "@radix-ui/react-radio-group": "^1.3.6", 40 | "@radix-ui/react-scroll-area": "^1.2.8", 41 | "@radix-ui/react-select": "^2.2.4", 42 | "@radix-ui/react-separator": "^1.1.6", 43 | "@radix-ui/react-slider": "^1.3.4", 44 | "@radix-ui/react-slot": "^1.2.2", 45 | "@radix-ui/react-switch": "^1.2.4", 46 | "@radix-ui/react-tabs": "^1.1.11", 47 | "@radix-ui/react-toggle": "^1.1.8", 48 | "@radix-ui/react-toggle-group": "^1.1.9", 49 | "@radix-ui/react-tooltip": "^1.2.6", 50 | "@types/uuid": "^10.0.0", 51 | "@upstash/ratelimit": "^2.0.5", 52 | "@upstash/redis": "^1.34.9", 53 | "@upstash/search": "^0.1.0", 54 | "@vercel/analytics": "^1.5.0", 55 | "ai": "^4.3.16", 56 | "class-variance-authority": "^0.7.1", 57 | "clsx": "^2.1.1", 58 | "cmdk": "^1.1.1", 59 | "date-fns": "^4.1.0", 60 | "embla-carousel-react": "^8.6.0", 61 | "input-otp": "^1.4.2", 62 | "lucide-react": "^0.511.0", 63 | "next": "15.3.2", 64 | "next-themes": "^0.4.6", 65 | "openai": "^4.73.0", 66 | "papaparse": "^5.4.1", 67 | "react": "^19.0.0", 68 | "react-day-picker": "8.10.1", 69 | "react-dom": "^19.0.0", 70 | "react-dropzone": "^14.3.5", 71 | "react-hook-form": "^7.56.4", 72 | "react-markdown": "^10.1.0", 73 | "react-resizable-panels": "^3.0.2", 74 | "recharts": "^2.15.3", 75 | "remark-gfm": "^4.0.1", 76 | "sonner": "^2.0.3", 77 | "tailwind-merge": "^3.3.0", 78 | "tailwindcss-animate": "^1.0.7", 79 | "uuid": "^11.1.0", 80 | "vaul": "^1.1.2", 81 | "zod": "^3.25.3" 82 | }, 83 | "devDependencies": { 84 | "@eslint/eslintrc": "^3", 85 | "@shadcn/ui": "^0.0.4", 86 | "@tailwindcss/postcss": "^4", 87 | "@types/node": "^20", 88 | "@types/papaparse": "^5.3.14", 89 | "@types/react": "^19", 90 | "@types/react-dom": "^19", 91 | "eslint": "^9", 92 | "eslint-config-next": "15.3.2", 93 | "tailwindcss": "^4", 94 | "tw-animate-css": "^1.3.0", 95 | "typescript": "^5" 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /components/ui/button.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | import { Slot } from "@radix-ui/react-slot" 3 | import { cva, type VariantProps } from "class-variance-authority" 4 | 5 | import { cn } from "@/lib/utils" 6 | 7 | const buttonVariants = cva( 8 | "inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50", 9 | { 10 | variants: { 11 | variant: { 12 | default: "bg-primary text-primary-foreground hover:bg-primary/90", 13 | destructive: 14 | "bg-destructive text-destructive-foreground hover:bg-destructive/90", 15 | outline: 16 | "border border-input bg-background hover:bg-accent hover:text-accent-foreground", 17 | secondary: 18 | "bg-secondary text-secondary-foreground hover:bg-secondary/80", 19 | ghost: "hover:bg-accent hover:text-accent-foreground", 20 | link: "text-primary underline-offset-4 hover:underline", 21 | code: "h-9 px-4 rounded-[10px] text-sm font-medium items-center transition-all duration-200 disabled:cursor-not-allowed disabled:opacity-50 bg-[#36322F] text-[#fff] hover:bg-[#4a4542] disabled:bg-[#8c8885] disabled:hover:bg-[#8c8885] [box-shadow:inset_0px_-2.108433723449707px_0px_0px_#171310,_0px_1.2048193216323853px_6.325301647186279px_0px_rgba(58,_33,_8,_58%)] hover:translate-y-[1px] hover:scale-[0.98] hover:[box-shadow:inset_0px_-1px_0px_0px_#171310,_0px_1px_3px_0px_rgba(58,_33,_8,_40%)] active:translate-y-[2px] active:scale-[0.97] active:[box-shadow:inset_0px_1px_1px_0px_#171310,_0px_1px_2px_0px_rgba(58,_33,_8,_30%)] disabled:shadow-none disabled:hover:translate-y-0 disabled:hover:scale-100", 22 | orange: "h-9 px-4 rounded-[10px] text-sm font-medium items-center transition-all duration-200 disabled:cursor-not-allowed disabled:opacity-50 bg-orange-500 text-white hover:bg-orange-300 dark:bg-orange-500 dark:hover:bg-orange-300 dark:text-white [box-shadow:inset_0px_-2.108433723449707px_0px_0px_#c2410c,_0px_1.2048193216323853px_6.325301647186279px_0px_rgba(234,_88,_12,_58%)] hover:translate-y-[1px] hover:scale-[0.98] hover:[box-shadow:inset_0px_-1px_0px_0px_#c2410c,_0px_1px_3px_0px_rgba(234,_88,_12,_40%)] active:translate-y-[2px] active:scale-[0.97] active:[box-shadow:inset_0px_1px_1px_0px_#c2410c,_0px_1px_2px_0px_rgba(234,_88,_12,_30%)] disabled:shadow-none disabled:hover:translate-y-0 disabled:hover:scale-100", 23 | }, 24 | size: { 25 | default: "h-10 px-4 py-2", 26 | sm: "h-9 rounded-md px-3", 27 | lg: "h-11 rounded-md px-8", 28 | icon: "h-10 w-10", 29 | }, 30 | }, 31 | defaultVariants: { 32 | variant: "default", 33 | size: "default", 34 | }, 35 | } 36 | ) 37 | 38 | export interface ButtonProps 39 | extends React.ButtonHTMLAttributes, 40 | VariantProps { 41 | asChild?: boolean 42 | } 43 | 44 | const Button = React.forwardRef( 45 | ({ className, variant, size, asChild = false, ...props }, ref) => { 46 | const Comp = asChild ? Slot : "button" 47 | return ( 48 | 53 | ) 54 | } 55 | ) 56 | Button.displayName = "Button" 57 | 58 | export { Button, buttonVariants } 59 | -------------------------------------------------------------------------------- /hooks/useStorage.ts: -------------------------------------------------------------------------------- 1 | import { useState, useEffect } from 'react' 2 | import { IndexMetadata } from '@/lib/storage' 3 | 4 | // Check if we should use Redis (server-side storage) 5 | const useRedis = !!(process.env.NEXT_PUBLIC_UPSTASH_REDIS_REST_URL && process.env.NEXT_PUBLIC_UPSTASH_REDIS_REST_TOKEN) 6 | 7 | export function useStorage() { 8 | const [indexes, setIndexes] = useState([]) 9 | const [loading, setLoading] = useState(true) 10 | const [error, setError] = useState(null) 11 | 12 | const fetchIndexes = async () => { 13 | setLoading(true) 14 | setError(null) 15 | 16 | try { 17 | if (useRedis) { 18 | // Fetch from API endpoint 19 | const response = await fetch('/api/indexes') 20 | if (!response.ok) { 21 | throw new Error('Failed to fetch indexes') 22 | } 23 | const data = await response.json() 24 | setIndexes(data.indexes || []) 25 | } else { 26 | // Use localStorage 27 | const stored = localStorage.getItem('firestarter_indexes') 28 | setIndexes(stored ? JSON.parse(stored) : []) 29 | } 30 | } catch (err) { 31 | setError(err instanceof Error ? err.message : 'Failed to fetch indexes') 32 | setIndexes([]) 33 | } finally { 34 | setLoading(false) 35 | } 36 | } 37 | 38 | const saveIndex = async (index: IndexMetadata) => { 39 | try { 40 | if (useRedis) { 41 | // Save via API endpoint 42 | const response = await fetch('/api/indexes', { 43 | method: 'POST', 44 | headers: { 'Content-Type': 'application/json' }, 45 | body: JSON.stringify(index) 46 | }) 47 | if (!response.ok) { 48 | throw new Error('Failed to save index') 49 | } 50 | // Refresh indexes 51 | await fetchIndexes() 52 | } else { 53 | // Save to localStorage 54 | const currentIndexes = [...indexes] 55 | const existingIndex = currentIndexes.findIndex(i => i.namespace === index.namespace) 56 | 57 | if (existingIndex !== -1) { 58 | currentIndexes[existingIndex] = index 59 | } else { 60 | currentIndexes.unshift(index) 61 | } 62 | 63 | // Keep only the last 50 indexes 64 | const limitedIndexes = currentIndexes.slice(0, 50) 65 | localStorage.setItem('firestarter_indexes', JSON.stringify(limitedIndexes)) 66 | setIndexes(limitedIndexes) 67 | } 68 | } catch (err) { 69 | throw err 70 | } 71 | } 72 | 73 | const deleteIndex = async (namespace: string) => { 74 | try { 75 | if (useRedis) { 76 | // Delete via API endpoint 77 | const response = await fetch(`/api/indexes?namespace=${namespace}`, { 78 | method: 'DELETE' 79 | }) 80 | if (!response.ok) { 81 | throw new Error('Failed to delete index') 82 | } 83 | // Refresh indexes 84 | await fetchIndexes() 85 | } else { 86 | // Delete from localStorage 87 | const filteredIndexes = indexes.filter(i => i.namespace !== namespace) 88 | localStorage.setItem('firestarter_indexes', JSON.stringify(filteredIndexes)) 89 | setIndexes(filteredIndexes) 90 | } 91 | } catch (err) { 92 | throw err 93 | } 94 | } 95 | 96 | useEffect(() => { 97 | fetchIndexes() 98 | }, []) 99 | 100 | return { 101 | indexes, 102 | loading, 103 | error, 104 | saveIndex, 105 | deleteIndex, 106 | refresh: fetchIndexes, 107 | isUsingRedis: useRedis 108 | } 109 | } -------------------------------------------------------------------------------- /firestarter.config.ts: -------------------------------------------------------------------------------- 1 | import { groq } from '@ai-sdk/groq' 2 | import { openai } from '@ai-sdk/openai' 3 | import { anthropic } from '@ai-sdk/anthropic' 4 | import { Ratelimit } from '@upstash/ratelimit' 5 | import { Redis } from '@upstash/redis' 6 | 7 | // AI provider configuration 8 | const AI_PROVIDERS = { 9 | groq: { 10 | model: groq('meta-llama/llama-4-scout-17b-16e-instruct'), 11 | enabled: !!process.env.GROQ_API_KEY, 12 | }, 13 | openai: { 14 | model: openai('gpt-4o'), 15 | enabled: !!process.env.OPENAI_API_KEY, 16 | }, 17 | anthropic: { 18 | model: anthropic('claude-3-5-sonnet-20241022'), 19 | enabled: !!process.env.ANTHROPIC_API_KEY, 20 | }, 21 | } 22 | 23 | // Get the active AI provider 24 | function getAIModel() { 25 | // Only check on server side 26 | if (typeof window !== 'undefined') { 27 | return null 28 | } 29 | // Priority: OpenAI (GPT-4o) > Anthropic (Claude 3.5 Sonnet) > Groq 30 | if (AI_PROVIDERS.openai.enabled) return AI_PROVIDERS.openai.model 31 | if (AI_PROVIDERS.anthropic.enabled) return AI_PROVIDERS.anthropic.model 32 | if (AI_PROVIDERS.groq.enabled) return AI_PROVIDERS.groq.model 33 | throw new Error('No AI provider configured. Please set OPENAI_API_KEY, ANTHROPIC_API_KEY, or GROQ_API_KEY') 34 | } 35 | 36 | // Rate limiter factory 37 | function createRateLimiter(identifier: string, requests = 50, window = '1 d') { 38 | if (typeof window !== 'undefined') { 39 | return null 40 | } 41 | if (!process.env.UPSTASH_REDIS_REST_URL || !process.env.UPSTASH_REDIS_REST_TOKEN) { 42 | return null 43 | } 44 | 45 | const redis = new Redis({ 46 | url: process.env.UPSTASH_REDIS_REST_URL, 47 | token: process.env.UPSTASH_REDIS_REST_TOKEN, 48 | }) 49 | 50 | return new Ratelimit({ 51 | redis, 52 | limiter: Ratelimit.fixedWindow(requests, window), 53 | analytics: true, 54 | prefix: `firestarter:ratelimit:${identifier}`, 55 | }) 56 | } 57 | 58 | const config = { 59 | app: { 60 | name: 'Firestarter', 61 | url: process.env.NEXT_PUBLIC_URL || 'http://localhost:3000', 62 | logoPath: '/firecrawl-logo-with-fire.png', 63 | }, 64 | 65 | ai: { 66 | model: getAIModel(), 67 | temperature: 0.7, 68 | maxTokens: 800, 69 | systemPrompt: `You are a friendly assistant. If a user greets you or engages in small talk, respond politely without referencing the website. For questions about the website, answer using ONLY the provided context below. Do not use any other knowledge. If the context isn't sufficient to answer, say so explicitly.`, 70 | providers: AI_PROVIDERS, 71 | }, 72 | 73 | crawling: { 74 | defaultLimit: 10, 75 | maxLimit: 100, 76 | minLimit: 10, 77 | limitOptions: [10, 25, 50, 100], 78 | scrapeTimeout: 15000, 79 | cacheMaxAge: 604800, 80 | }, 81 | 82 | search: { 83 | maxResults: 100, 84 | maxContextDocs: 10, 85 | maxContextLength: 1500, 86 | maxSourcesDisplay: 20, 87 | snippetLength: 200, 88 | }, 89 | 90 | storage: { 91 | maxIndexes: 50, 92 | localStorageKey: 'firestarter_indexes', 93 | redisPrefix: { 94 | indexes: 'firestarter:indexes', 95 | index: 'firestarter:index:', 96 | }, 97 | }, 98 | 99 | rateLimits: { 100 | create: createRateLimiter('create', 20, '1 d'), 101 | query: createRateLimiter('query', 100, '1 h'), 102 | scrape: createRateLimiter('scrape', 50, '1 d'), 103 | }, 104 | 105 | features: { 106 | enableCreation: process.env.DISABLE_CHATBOT_CREATION !== 'true', 107 | enableRedis: !!(process.env.UPSTASH_REDIS_REST_URL && process.env.UPSTASH_REDIS_REST_TOKEN), 108 | enableSearch: !!(process.env.UPSTASH_SEARCH_REST_URL && process.env.UPSTASH_SEARCH_REST_TOKEN), 109 | }, 110 | } 111 | 112 | export type Config = typeof config 113 | 114 | // Client-safe config (no AI model initialization) 115 | export const clientConfig = { 116 | app: config.app, 117 | crawling: config.crawling, 118 | search: config.search, 119 | storage: config.storage, 120 | features: config.features, 121 | } 122 | 123 | // Server-only config (includes AI model) 124 | export const serverConfig = config 125 | 126 | // Default export for backward compatibility 127 | export { clientConfig as config } -------------------------------------------------------------------------------- /components/ui/dialog.tsx: -------------------------------------------------------------------------------- 1 | "use client" 2 | 3 | import * as React from "react" 4 | import * as DialogPrimitive from "@radix-ui/react-dialog" 5 | import { XIcon } from "lucide-react" 6 | 7 | import { cn } from "@/lib/utils" 8 | 9 | function Dialog({ 10 | ...props 11 | }: React.ComponentProps) { 12 | return 13 | } 14 | 15 | function DialogTrigger({ 16 | ...props 17 | }: React.ComponentProps) { 18 | return 19 | } 20 | 21 | function DialogPortal({ 22 | ...props 23 | }: React.ComponentProps) { 24 | return 25 | } 26 | 27 | function DialogClose({ 28 | ...props 29 | }: React.ComponentProps) { 30 | return 31 | } 32 | 33 | function DialogOverlay({ 34 | className, 35 | ...props 36 | }: React.ComponentProps) { 37 | return ( 38 | 46 | ) 47 | } 48 | 49 | function DialogContent({ 50 | className, 51 | children, 52 | ...props 53 | }: React.ComponentProps) { 54 | return ( 55 | 56 | 57 | 65 | {children} 66 | 67 | 68 | Close 69 | 70 | 71 | 72 | ) 73 | } 74 | 75 | function DialogHeader({ className, ...props }: React.ComponentProps<"div">) { 76 | return ( 77 |
82 | ) 83 | } 84 | 85 | function DialogFooter({ className, ...props }: React.ComponentProps<"div">) { 86 | return ( 87 |
95 | ) 96 | } 97 | 98 | function DialogTitle({ 99 | className, 100 | ...props 101 | }: React.ComponentProps) { 102 | return ( 103 | 108 | ) 109 | } 110 | 111 | function DialogDescription({ 112 | className, 113 | ...props 114 | }: React.ComponentProps) { 115 | return ( 116 | 121 | ) 122 | } 123 | 124 | export { 125 | Dialog, 126 | DialogClose, 127 | DialogContent, 128 | DialogDescription, 129 | DialogFooter, 130 | DialogHeader, 131 | DialogOverlay, 132 | DialogPortal, 133 | DialogTitle, 134 | DialogTrigger, 135 | } 136 | -------------------------------------------------------------------------------- /lib/storage.ts: -------------------------------------------------------------------------------- 1 | import { Redis } from '@upstash/redis' 2 | 3 | export interface IndexMetadata { 4 | url: string 5 | namespace: string 6 | pagesCrawled: number 7 | createdAt: string 8 | metadata?: { 9 | title?: string 10 | description?: string 11 | favicon?: string 12 | ogImage?: string 13 | } 14 | } 15 | 16 | interface StorageAdapter { 17 | getIndexes(): Promise 18 | getIndex(namespace: string): Promise 19 | saveIndex(index: IndexMetadata): Promise 20 | deleteIndex(namespace: string): Promise 21 | } 22 | 23 | class LocalStorageAdapter implements StorageAdapter { 24 | private readonly STORAGE_KEY = 'firestarter_indexes' 25 | 26 | async getIndexes(): Promise { 27 | if (typeof window === 'undefined') return [] 28 | 29 | try { 30 | const stored = localStorage.getItem(this.STORAGE_KEY) 31 | return stored ? JSON.parse(stored) : [] 32 | } catch { 33 | console.error('Failed to get stored indexes') 34 | return [] 35 | } 36 | } 37 | 38 | async getIndex(namespace: string): Promise { 39 | const indexes = await this.getIndexes() 40 | return indexes.find(i => i.namespace === namespace) || null 41 | } 42 | 43 | async saveIndex(index: IndexMetadata): Promise { 44 | if (typeof window === 'undefined') { 45 | throw new Error('localStorage is not available on the server') 46 | } 47 | 48 | const indexes = await this.getIndexes() 49 | const existingIndex = indexes.findIndex(i => i.namespace === index.namespace) 50 | 51 | if (existingIndex !== -1) { 52 | indexes[existingIndex] = index 53 | } else { 54 | indexes.unshift(index) 55 | } 56 | 57 | // Keep only the last 50 indexes 58 | const limitedIndexes = indexes.slice(0, 50) 59 | 60 | try { 61 | localStorage.setItem(this.STORAGE_KEY, JSON.stringify(limitedIndexes)) 62 | } catch (error) { 63 | throw error 64 | } 65 | } 66 | 67 | async deleteIndex(namespace: string): Promise { 68 | if (typeof window === 'undefined') { 69 | throw new Error('localStorage is not available on the server') 70 | } 71 | 72 | const indexes = await this.getIndexes() 73 | const filteredIndexes = indexes.filter(i => i.namespace !== namespace) 74 | 75 | try { 76 | localStorage.setItem(this.STORAGE_KEY, JSON.stringify(filteredIndexes)) 77 | } catch (error) { 78 | throw error 79 | } 80 | } 81 | } 82 | 83 | class RedisStorageAdapter implements StorageAdapter { 84 | private redis: Redis 85 | private readonly INDEXES_KEY = 'firestarter:indexes' 86 | private readonly INDEX_KEY_PREFIX = 'firestarter:index:' 87 | 88 | constructor() { 89 | if (!process.env.UPSTASH_REDIS_REST_URL || !process.env.UPSTASH_REDIS_REST_TOKEN) { 90 | throw new Error('Redis configuration missing') 91 | } 92 | 93 | this.redis = new Redis({ 94 | url: process.env.UPSTASH_REDIS_REST_URL, 95 | token: process.env.UPSTASH_REDIS_REST_TOKEN, 96 | }) 97 | } 98 | 99 | async getIndexes(): Promise { 100 | try { 101 | const indexes = await this.redis.get(this.INDEXES_KEY) 102 | return indexes || [] 103 | } catch { 104 | console.error('Failed to get indexes from Redis') 105 | return [] 106 | } 107 | } 108 | 109 | async getIndex(namespace: string): Promise { 110 | try { 111 | const index = await this.redis.get(`${this.INDEX_KEY_PREFIX}${namespace}`) 112 | return index 113 | } catch { 114 | console.error('Failed to get index from Redis') 115 | return null 116 | } 117 | } 118 | 119 | async saveIndex(index: IndexMetadata): Promise { 120 | try { 121 | // Save individual index 122 | await this.redis.set(`${this.INDEX_KEY_PREFIX}${index.namespace}`, index) 123 | 124 | // Update indexes list 125 | const indexes = await this.getIndexes() 126 | const existingIndex = indexes.findIndex(i => i.namespace === index.namespace) 127 | 128 | if (existingIndex !== -1) { 129 | indexes[existingIndex] = index 130 | } else { 131 | indexes.unshift(index) 132 | } 133 | 134 | // Keep only the last 50 indexes 135 | const limitedIndexes = indexes.slice(0, 50) 136 | await this.redis.set(this.INDEXES_KEY, limitedIndexes) 137 | } catch (error) { 138 | throw error 139 | } 140 | } 141 | 142 | async deleteIndex(namespace: string): Promise { 143 | try { 144 | // Delete individual index 145 | await this.redis.del(`${this.INDEX_KEY_PREFIX}${namespace}`) 146 | 147 | // Update indexes list 148 | const indexes = await this.getIndexes() 149 | const filteredIndexes = indexes.filter(i => i.namespace !== namespace) 150 | await this.redis.set(this.INDEXES_KEY, filteredIndexes) 151 | } catch (error) { 152 | throw error 153 | } 154 | } 155 | } 156 | 157 | // Factory function to get the appropriate storage adapter 158 | function getStorageAdapter(): StorageAdapter { 159 | // Use Redis if both environment variables are set 160 | if (process.env.UPSTASH_REDIS_REST_URL && process.env.UPSTASH_REDIS_REST_TOKEN) { 161 | return new RedisStorageAdapter() 162 | } 163 | 164 | // Check if we're on the server 165 | if (typeof window === 'undefined') { 166 | throw new Error('No storage adapter available on the server. Please configure Redis by setting UPSTASH_REDIS_REST_URL and UPSTASH_REDIS_REST_TOKEN environment variables.') 167 | } 168 | 169 | // Otherwise, use localStorage (only on client) 170 | return new LocalStorageAdapter() 171 | } 172 | 173 | // Lazy initialization to avoid errors at module load time 174 | let storage: StorageAdapter | null = null 175 | 176 | function getStorage(): StorageAdapter | null { 177 | if (!storage) { 178 | try { 179 | storage = getStorageAdapter() 180 | } catch { 181 | // This is expected on the server without Redis configured 182 | return null 183 | } 184 | } 185 | return storage 186 | } 187 | 188 | export const getIndexes = async (): Promise => { 189 | const adapter = getStorage() 190 | if (!adapter) { 191 | return [] 192 | } 193 | 194 | try { 195 | return await adapter.getIndexes() 196 | } catch { 197 | console.error('Failed to get indexes') 198 | return [] 199 | } 200 | } 201 | 202 | export const getIndex = async (namespace: string): Promise => { 203 | const adapter = getStorage() 204 | if (!adapter) { 205 | return null 206 | } 207 | 208 | try { 209 | return await adapter.getIndex(namespace) 210 | } catch { 211 | console.error('Failed to get index') 212 | return null 213 | } 214 | } 215 | 216 | export const saveIndex = async (index: IndexMetadata): Promise => { 217 | const adapter = getStorage() 218 | if (!adapter) { 219 | console.warn('No storage adapter available - index not saved') 220 | return 221 | } 222 | 223 | try { 224 | return await adapter.saveIndex(index) 225 | } catch { 226 | // Don't throw - this allows the app to continue functioning 227 | console.error('Failed to save index') 228 | } 229 | } 230 | 231 | export const deleteIndex = async (namespace: string): Promise => { 232 | const adapter = getStorage() 233 | if (!adapter) { 234 | console.warn('No storage adapter available - index not deleted') 235 | return 236 | } 237 | 238 | try { 239 | return await adapter.deleteIndex(namespace) 240 | } catch { 241 | // Don't throw - this allows the app to continue functioning 242 | console.error('Failed to delete index') 243 | } 244 | } -------------------------------------------------------------------------------- /app/indexes/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import { useRouter } from 'next/navigation' 4 | import Image from "next/image" 5 | import Link from "next/link" 6 | import { Button } from "@/components/ui/button" 7 | import { Globe, FileText, Database, ExternalLink, Trash2, Calendar } from 'lucide-react' 8 | import { toast } from "sonner" 9 | import { useStorage } from "@/hooks/useStorage" 10 | 11 | interface IndexedSite { 12 | url: string 13 | namespace: string 14 | pagesCrawled: number 15 | createdAt: string 16 | metadata?: { 17 | title?: string 18 | description?: string 19 | favicon?: string 20 | ogImage?: string 21 | } 22 | } 23 | 24 | export default function IndexesPage() { 25 | const router = useRouter() 26 | const { indexes, loading, deleteIndex, isUsingRedis } = useStorage() 27 | 28 | const handleSelectIndex = (index: IndexedSite) => { 29 | // Store the site info in session storage for the dashboard 30 | const siteInfo = { 31 | url: index.url, 32 | namespace: index.namespace, 33 | pagesCrawled: index.pagesCrawled, 34 | crawlDate: index.createdAt, 35 | metadata: index.metadata || {}, 36 | crawlComplete: true, 37 | fromIndex: true // Flag to indicate this is from the index list 38 | } 39 | 40 | sessionStorage.setItem('firestarter_current_data', JSON.stringify(siteInfo)) 41 | 42 | // Navigate to the dashboard with namespace parameter 43 | router.push(`/dashboard?namespace=${index.namespace}`) 44 | } 45 | 46 | const handleDeleteIndex = async (index: IndexedSite, e: React.MouseEvent) => { 47 | e.stopPropagation() 48 | 49 | if (confirm(`Delete chatbot for ${index.metadata?.title || index.url}?`)) { 50 | try { 51 | await deleteIndex(index.namespace) 52 | toast.success('Chatbot deleted successfully') 53 | } catch { 54 | toast.error('Failed to delete chatbot') 55 | console.error('Failed to delete index') 56 | } 57 | } 58 | } 59 | 60 | const formatDate = (dateString: string) => { 61 | const date = new Date(dateString) 62 | return date.toLocaleDateString('en-US', { 63 | month: 'short', 64 | day: 'numeric', 65 | year: 'numeric', 66 | hour: '2-digit', 67 | minute: '2-digit' 68 | }) 69 | } 70 | 71 | return ( 72 |
73 |
74 | 75 | Firecrawl Logo 81 | 82 | 91 |
92 | 93 |
94 |

Your Chatbots

95 |

96 | View and manage all your chatbots 97 | {isUsingRedis && (using Redis storage)} 98 |

99 |
100 | 101 | {loading ? ( 102 |
103 |

Loading indexes...

104 |
105 | ) : indexes.length === 0 ? ( 106 |
107 | 108 |

No Chatbots Yet

109 |

You haven't created any chatbots yet.

110 | 115 |
116 | ) : ( 117 |
118 | {indexes.map((index) => ( 119 |
handleSelectIndex(index)} 122 | className="bg-white rounded-xl border border-gray-200 hover:shadow-md transition-shadow cursor-pointer group overflow-hidden" 123 | > 124 |
125 | {/* Left side - OG Image */} 126 |
127 | {index.metadata?.ogImage ? ( 128 | <> 129 | {index.metadata?.title { 135 | e.currentTarget.style.display = 'none'; 136 | e.currentTarget.parentElement?.querySelector('.fallback-icon')?.classList.remove('hidden'); 137 | }} 138 | /> 139 |
140 |
141 | 142 |
143 | 144 | ) : ( 145 |
146 | 147 |
148 | )} 149 | {index.metadata?.favicon && ( 150 |
151 | favicon { 158 | e.currentTarget.parentElement!.style.display = 'none'; 159 | }} 160 | /> 161 |
162 | )} 163 |
164 | 165 | {/* Right side - Content */} 166 |
167 |
168 |

169 | {index.metadata?.title || new URL(index.url).hostname} 170 |

171 |

{index.url}

172 | {index.metadata?.description && ( 173 |

174 | {index.metadata.description} 175 |

176 | )} 177 | 178 |
179 |
180 | 181 | {index.pagesCrawled} pages 182 |
183 |
184 | 185 | {index.namespace.split('-').slice(0, -1).join('.')} 186 |
187 |
188 | 189 | {formatDate(index.createdAt)} 190 |
191 |
192 |
193 | 194 |
195 | 203 | 204 |
205 |
206 |
207 |
208 | ))} 209 |
210 | )} 211 |
212 | ) 213 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Firestarter - Instant AI Chatbots for Any Website 2 | 3 |
4 | Firestarter Demo 5 |
6 | 7 | Instantly create a knowledgeable AI chatbot for any website. Firestarter crawls your site, indexes the content, and provides a ready-to-use RAG-powered chat interface and an OpenAI-compatible API endpoint. 8 | 9 | ## Technologies 10 | 11 | - **Firecrawl**: Web scraping and content aggregation 12 | - **Upstash Search**: High-performance vector database for semantic search 13 | - **Vercel AI SDK**: For streaming AI responses 14 | - **Next.js 15**: Modern React framework with App Router 15 | - **Groq, OpenAI, Anthropic**: Flexible LLM provider support 16 | 17 | [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fmendableai%2Ffirestarter&env=FIRECRAWL_API_KEY,UPSTASH_SEARCH_REST_URL,UPSTASH_SEARCH_REST_TOKEN,OPENAI_API_KEY&envDescription=API%20keys%20for%20Firecrawl,%20Upstash,%20and%20OpenAI%20are%20required.&envLink=https%3A%2F%2Fgithub.com%2Fmendableai%2Ffirestarter%23required-api-keys) 18 | 19 | ## Setup 20 | 21 | ### Required API Keys 22 | 23 | You need a key from Firecrawl, Upstash, and at least one LLM provider. 24 | 25 | | Service | Purpose | Get Key | 26 | | ---------------- | ------------------------------------- | -------------------------------------------- | 27 | | Firecrawl | Web scraping and content aggregation | [firecrawl.dev/app/api-keys](https://www.firecrawl.dev/app/api-keys) | 28 | | Upstash | Vector DB for semantic search | [console.upstash.com](https://console.upstash.com) | 29 | | OpenAI | AI model provider (default) | [platform.openai.com/api-keys](https://platform.openai.com/api-keys) | 30 | 31 | ### Quick Start 32 | 33 | 1. Clone this repository 34 | 2. Create a `.env.local` file with your API keys: 35 | ``` 36 | FIRECRAWL_API_KEY=your_firecrawl_key 37 | 38 | # Upstash Vector DB Credentials 39 | UPSTASH_SEARCH_REST_URL=your_upstash_search_url 40 | UPSTASH_SEARCH_REST_TOKEN=your_upstash_search_token 41 | 42 | # OpenAI API Key (default provider) 43 | OPENAI_API_KEY=your_openai_key 44 | 45 | # Optional: Disable chatbot creation (for read-only deployments) 46 | # DISABLE_CHATBOT_CREATION=true 47 | ``` 48 | 3. Install dependencies: `npm install` or `yarn install` 49 | 4. Run the development server: `npm run dev` or `yarn dev` 50 | 5. Open [http://localhost:3000](http://localhost:3000) 51 | 52 | ## Example Interaction 53 | 54 | **Input:** 55 | A website URL, like `https://firecrawl.dev` 56 | 57 | **Output:** 58 | A fully functional chat interface and an API endpoint to query your website's content. 59 | 60 | ## How It Works 61 | 62 | ### Architecture Overview: From URL to AI Chatbot 63 | 64 | Let's trace the journey from submitting `https://docs.firecrawl.dev` to asking it "How do I use the API?". 65 | 66 | ### Phase 1: Indexing a Website 67 | 68 | 1. **URL Submission**: You enter a URL. The frontend calls the `/api/firestarter/create` endpoint. 69 | 2. **Smart Crawling**: The backend uses the **Firecrawl API** to crawl the website, fetching the content of each page as clean Markdown. 70 | 3. **Indexing**: The content of each page is sent to **Upstash Search**, which automatically chunks and converts it into vector embeddings. 71 | 4. **Namespace Creation**: The entire crawl is stored under a unique, shareable `namespace` (e.g., `firecrawl-dev-1718394041`). This isolates the data for each chatbot. 72 | 5. **Metadata Storage**: Information about the chatbot (URL, namespace, title, favicon) is saved in Redis (if configured) or the browser's local storage for persistence. 73 | 74 | ### Phase 2: Answering Questions (RAG Pipeline) 75 | 76 | 1. **User Query**: You ask a question in the chat interface. The frontend calls the `/api/firestarter/query` endpoint with your question and the chatbot's `namespace`. 77 | 2. **Semantic Search**: The backend performs a semantic search on the **Upstash Search** index. It looks for the most relevant document chunks based on the meaning of your query within the specific `namespace`. 78 | 3. **Context-Aware Prompting**: The most relevant chunks are compiled into a context block, which is then passed to the LLM (e.g., Groq) along with your original question. The system prompt instructs the LLM to answer *only* using the provided information. 79 | 4. **Streaming Response**: The LLM generates the answer, and the response is streamed back to the UI in real-time using the **Vercel AI SDK**, creating a smooth, "typing" effect. 80 | 81 | ### OpenAI-Compatible API: The Ultimate Power-Up 82 | 83 | Firestarter doesn't just give you a UI; it provides a powerful, OpenAI-compatible API endpoint for each chatbot you create. 84 | 85 | - **Endpoint**: `api/v1/chat/completions` 86 | - **How it works**: When you create a chatbot for `example.com`, Firestarter generates a unique model name like `firecrawl-example-com-12345`. 87 | - **Integration**: You can use this model name with any official or community-built OpenAI library. Just point the client's `baseURL` to your Firestarter instance. 88 | 89 | ```javascript 90 | // Example: Using the OpenAI JS SDK with your new chatbot 91 | import OpenAI from 'openai'; 92 | 93 | const firestarter = new OpenAI({ 94 | apiKey: 'any_string_works_here', // Auth is handled by your deployment 95 | baseURL: 'https://your-firestarter-deployment.vercel.app/api/v1/chat/completions' 96 | }); 97 | 98 | const completion = await firestarter.chat.completions.create({ 99 | model: 'firecrawl-firecrawl-dev-12345', // The model name for your site 100 | messages: [{ role: 'user', content: 'What is Firecrawl?' }], 101 | }); 102 | 103 | console.log(completion.choices[0].message.content); 104 | ``` 105 | This turns any website into a programmatic, queryable data source, perfect for building custom applications. 106 | 107 | ## Key Features 108 | 109 | - **Instant Chatbot Creation**: Go from a URL to a fully-functional AI chatbot in under a minute. 110 | - **High-Performance RAG**: Leverages Firecrawl for clean data extraction and Upstash for fast, serverless semantic search. 111 | - **OpenAI-Compatible API**: Integrate your website's knowledge into any application using the familiar OpenAI SDKs. 112 | - **Streaming Responses**: Real-time answer generation powered by the Vercel AI SDK for a seamless user experience. 113 | - **Flexible LLM Support**: Works out-of-the-box with Groq, OpenAI, and Anthropic. 114 | - **Persistent Indexes**: Your chatbots are saved and can be accessed anytime from the index page. 115 | - **Customizable Crawl Depth**: Easily configure how many pages to crawl for deeper or quicker indexing. 116 | - **Fully Open Source**: Understand, modify, and extend every part of the system. 117 | 118 | ## Configuration 119 | 120 | You can customize the application's behavior by modifying [`firestarter.config.ts`](firestarter.config.ts). When you run this repository locally, the crawling limits are increased. 121 | 122 | ```typescript 123 | // firestarter.config.ts 124 | 125 | const config = { 126 | // ... 127 | crawling: { 128 | defaultLimit: 10, 129 | maxLimit: 100, // Change this for your self-hosted version 130 | minLimit: 10, 131 | limitOptions: [10, 25, 50, 100], 132 | // ... 133 | }, 134 | features: { 135 | // Chatbot creation can be disabled for read-only deployments 136 | enableCreation: process.env.DISABLE_CHATBOT_CREATION !== 'true', 137 | }, 138 | // ... 139 | } 140 | ``` 141 | 142 | ### Changing the LLM Provider 143 | 144 | Firestarter supports multiple LLM providers and uses them based on a priority system. The default priority is **OpenAI (GPT-4o) → Anthropic (Claude 3.5 Sonnet) → Groq**. The system will use the first provider in this list for which it finds a valid API key in your environment. 145 | 146 | To change the provider, simply adjust your `.env.local` file. For example, to use Anthropic instead of OpenAI, comment out your `OPENAI_API_KEY` and ensure your `ANTHROPIC_API_KEY` is set. 147 | 148 | **Example `.env.local` to use Anthropic:** 149 | ``` 150 | # To use Anthropic instead of OpenAI, comment out the OpenAI key: 151 | # OPENAI_API_KEY=sk-... 152 | 153 | ANTHROPIC_API_KEY=sk-ant-... 154 | 155 | # GROQ_API_KEY=gsk_... 156 | ``` 157 | 158 | This provider selection logic is controlled in [`firestarter.config.ts`](firestarter.config.ts). You can modify the `getAIModel` function if you want to implement a different selection strategy. 159 | 160 | ## Our Open Source Philosophy 161 | 162 | Let's be blunt: building a production-grade RAG pipeline is complex. Our goal with Firestarter isn't to be a black box, but a crystal-clear, powerful foundation that anyone can use, understand, and contribute to. 163 | 164 | This is just the start. By open-sourcing it, we're inviting you to join us on this journey. 165 | 166 | - **Want to add a new vector DB?** Fork the repo and show us what you've got. 167 | - **Improve the RAG prompt?** Open a pull request. 168 | - **Have a new feature idea?** Start a discussion in the issues. 169 | 170 | We believe that by building in public, we can create a tool that is more accessible, affordable, and adaptable, thanks to the collective intelligence of the open-source community. 171 | 172 | ## License 173 | 174 | MIT License - see [LICENSE](LICENSE) file for details. 175 | 176 | ## Contributing 177 | 178 | We welcome contributions! Please feel free to submit a Pull Request. 179 | 180 | ## Support 181 | 182 | For questions and issues, please open an issue in this repository. 183 | -------------------------------------------------------------------------------- /app/api/firestarter/create/route.ts: -------------------------------------------------------------------------------- 1 | import { NextRequest, NextResponse } from 'next/server' 2 | import FirecrawlApp from '@mendable/firecrawl-js' 3 | import { searchIndex } from '@/lib/upstash-search' 4 | import { saveIndex } from '@/lib/storage' 5 | import { serverConfig as config } from '@/firestarter.config' 6 | 7 | 8 | export async function POST(request: NextRequest) { 9 | try { 10 | // Check if creation is disabled 11 | if (!config.features.enableCreation) { 12 | return NextResponse.json({ 13 | error: 'Chatbot creation is currently disabled. You can only view existing chatbots.' 14 | }, { status: 403 }) 15 | } 16 | 17 | let body; 18 | try { 19 | body = await request.json() 20 | } catch { 21 | return NextResponse.json({ error: 'Invalid request body' }, { status: 400 }) 22 | } 23 | 24 | const { url, limit = config.crawling.defaultLimit, includePaths, excludePaths } = body 25 | 26 | if (!url) { 27 | return NextResponse.json({ error: 'URL is required' }, { status: 400 }) 28 | } 29 | 30 | // Generate unique namespace with timestamp to avoid collisions 31 | const baseNamespace = new URL(url).hostname.replace(/\./g, '-') 32 | const timestamp = Date.now() 33 | const namespace = `${baseNamespace}-${timestamp}` 34 | 35 | // Initialize Firecrawl with API key from environment or headers 36 | const apiKey = process.env.FIRECRAWL_API_KEY || request.headers.get('X-Firecrawl-API-Key') 37 | if (!apiKey) { 38 | return NextResponse.json({ 39 | error: 'Firecrawl API key is not configured. Please provide your API key.' 40 | }, { status: 500 }) 41 | } 42 | 43 | const firecrawl = new FirecrawlApp({ 44 | apiKey: apiKey 45 | }) 46 | 47 | // Start crawling the website with specified limit 48 | 49 | const crawlOptions = { 50 | limit: limit, 51 | scrapeOptions: { 52 | formats: ['markdown', 'html'] as ('markdown' | 'html')[], 53 | maxAge: config.crawling.cacheMaxAge, // Use config value 54 | }, 55 | includePaths: undefined as string[] | undefined, 56 | excludePaths: undefined as string[] | undefined 57 | } 58 | 59 | // Add include/exclude paths if provided 60 | if (includePaths && Array.isArray(includePaths) && includePaths.length > 0) { 61 | crawlOptions.includePaths = includePaths 62 | } 63 | if (excludePaths && Array.isArray(excludePaths) && excludePaths.length > 0) { 64 | crawlOptions.excludePaths = excludePaths 65 | } 66 | 67 | const crawlResponse = await firecrawl.crawlUrl(url, crawlOptions) as { 68 | success: boolean 69 | data: Array<{ 70 | url?: string 71 | markdown?: string 72 | content?: string 73 | metadata?: { 74 | title?: string 75 | description?: string 76 | ogDescription?: string 77 | sourceURL?: string 78 | favicon?: string 79 | ogImage?: string 80 | 'og:image'?: string 81 | } 82 | }> 83 | } 84 | 85 | 86 | // Store the crawl data for immediate use 87 | const crawlId = 'immediate-' + Date.now() 88 | 89 | // Log first page content preview for debugging 90 | if (crawlResponse.data && crawlResponse.data.length > 0) { 91 | // Find the homepage in the crawled data 92 | const homepage = crawlResponse.data.find((page) => { 93 | const pageUrl = page.metadata?.sourceURL || page.url || '' 94 | // Check if it's the homepage (ends with domain or domain/) 95 | return pageUrl === url || pageUrl === url + '/' || pageUrl === url.replace(/\/$/, '') 96 | }) || crawlResponse.data[0] // Fallback to first page 97 | 98 | // Log homepage info for debugging 99 | console.log('Homepage:', { 100 | title: homepage?.metadata?.title, 101 | url: homepage?.metadata?.sourceURL || homepage?.url 102 | }) 103 | } 104 | 105 | // Store documents in Upstash Search 106 | const documents = crawlResponse.data.map((page, index) => { 107 | // Get the content and metadata 108 | const fullContent = page.markdown || page.content || '' 109 | const title = page.metadata?.title || 'Untitled' 110 | const url = page.metadata?.sourceURL || page.url || '' 111 | const description = page.metadata?.description || page.metadata?.ogDescription || '' 112 | 113 | // Create a searchable text - include namespace for better search filtering 114 | // The limit is 1500 chars for the whole content object when stringified 115 | const searchableText = `namespace:${namespace} ${title} ${description} ${fullContent}`.substring(0, 1000) 116 | 117 | return { 118 | id: `${namespace}-${index}`, 119 | content: { 120 | text: searchableText, // Searchable text 121 | url: url, // Required by FirestarterContent 122 | title: title // Required by FirestarterContent 123 | }, 124 | metadata: { 125 | namespace: namespace, 126 | title: title, 127 | url: url, 128 | sourceURL: page.metadata?.sourceURL || page.url || '', 129 | crawlDate: new Date().toISOString(), 130 | pageTitle: page.metadata?.title, 131 | description: page.metadata?.description || page.metadata?.ogDescription, 132 | favicon: page.metadata?.favicon, 133 | ogImage: page.metadata?.ogImage || page.metadata?.['og:image'], 134 | // Store the full content in metadata for retrieval (not searchable but accessible) 135 | fullContent: fullContent.substring(0, 5000) // Store more content here 136 | } 137 | } 138 | }) 139 | 140 | // Store documents in batches 141 | const batchSize = 10 142 | 143 | try { 144 | for (let i = 0; i < documents.length; i += batchSize) { 145 | const batch = documents.slice(i, i + batchSize) 146 | await searchIndex.upsert(batch) 147 | } 148 | 149 | 150 | // Verify documents were stored - try multiple approaches 151 | 152 | // First try with filter 153 | interface SearchResult { 154 | metadata?: { 155 | namespace?: string 156 | } 157 | } 158 | let verifyResult: SearchResult[] = [] 159 | try { 160 | verifyResult = await searchIndex.search({ 161 | query: documents[0]?.content?.title || 'test', 162 | filter: `metadata.namespace = "${namespace}"`, 163 | limit: 1 164 | }) 165 | } catch { 166 | 167 | // Try without filter 168 | try { 169 | const allResults = await searchIndex.search({ 170 | query: namespace, // Search for the namespace itself 171 | limit: 10 172 | }) 173 | 174 | // Log the structure of the first result for debugging 175 | if (allResults.length > 0) { 176 | } 177 | 178 | // Manual filter check 179 | verifyResult = allResults.filter((doc: SearchResult) => { 180 | const docNamespace = doc.metadata?.namespace 181 | return docNamespace === namespace 182 | }) 183 | } catch { 184 | console.error('Failed to search without filter') 185 | } 186 | } 187 | 188 | if (verifyResult.length === 0) { 189 | } else { 190 | } 191 | } catch (upsertError) { 192 | throw new Error(`Failed to store documents: ${upsertError instanceof Error ? upsertError.message : 'Unknown error'}`) 193 | } 194 | 195 | // Save index metadata to storage 196 | const homepage = crawlResponse.data.find((page) => { 197 | const pageUrl = page.metadata?.sourceURL || page.url || '' 198 | return pageUrl === url || pageUrl === url + '/' || pageUrl === url.replace(/\/$/, '') 199 | }) || crawlResponse.data[0] 200 | 201 | try { 202 | await saveIndex({ 203 | url, 204 | namespace, 205 | pagesCrawled: crawlResponse.data?.length || 0, 206 | createdAt: new Date().toISOString(), 207 | metadata: { 208 | title: homepage?.metadata?.title, 209 | description: homepage?.metadata?.description || homepage?.metadata?.ogDescription, 210 | favicon: homepage?.metadata?.favicon, 211 | ogImage: homepage?.metadata?.ogImage || homepage?.metadata?.['og:image'] 212 | } 213 | }) 214 | } catch { 215 | // Continue execution - storage error shouldn't fail the entire operation 216 | console.error('Failed to save index metadata') 217 | } 218 | 219 | return NextResponse.json({ 220 | success: true, 221 | namespace, 222 | crawlId, 223 | message: `Crawl completed successfully (limited to ${limit} pages)`, 224 | details: { 225 | url, 226 | pagesLimit: limit, 227 | pagesCrawled: crawlResponse.data?.length || 0, 228 | formats: ['markdown', 'html'] 229 | }, 230 | data: crawlResponse.data // Include the actual crawl data 231 | }) 232 | } catch (error) { 233 | 234 | const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred' 235 | const statusCode = error && typeof error === 'object' && 'statusCode' in error ? error.statusCode : undefined 236 | 237 | 238 | // Provide more specific error messages 239 | if (statusCode === 401) { 240 | return NextResponse.json( 241 | { error: 'Firecrawl authentication failed. Please check your API key.' }, 242 | { status: 401 } 243 | ) 244 | } 245 | 246 | return NextResponse.json( 247 | { 248 | error: 'Failed to start crawl', 249 | details: errorMessage 250 | }, 251 | { status: 500 } 252 | ) 253 | } 254 | } 255 | 256 | -------------------------------------------------------------------------------- /app/globals.css: -------------------------------------------------------------------------------- 1 | @import "tailwindcss"; 2 | @import "tw-animate-css"; 3 | 4 | @custom-variant dark (&:is(.dark *)); 5 | 6 | /* Glowing border animation for email column */ 7 | @keyframes glow-pulse { 8 | 0%, 100% { 9 | box-shadow: 10 | 0 0 20px rgba(251, 146, 60, 0.5), 11 | 0 0 40px rgba(251, 146, 60, 0.3), 12 | 0 0 60px rgba(251, 146, 60, 0.1); 13 | } 14 | 50% { 15 | box-shadow: 16 | 0 0 30px rgba(251, 146, 60, 0.8), 17 | 0 0 60px rgba(251, 146, 60, 0.5), 18 | 0 0 90px rgba(251, 146, 60, 0.3); 19 | } 20 | } 21 | 22 | .email-column-glow { 23 | animation: glow-pulse 2s ease-in-out infinite; 24 | position: relative; 25 | } 26 | 27 | /* Rounded corners for email column */ 28 | .email-column-rounded-top { 29 | border-top-left-radius: 0.5rem; 30 | border-top-right-radius: 0.5rem; 31 | } 32 | 33 | .email-column-rounded-bottom { 34 | border-bottom-left-radius: 0.5rem; 35 | border-bottom-right-radius: 0.5rem; 36 | } 37 | 38 | .email-column-glow::before { 39 | content: ''; 40 | position: absolute; 41 | inset: -2px; 42 | border-radius: 0.5rem; 43 | padding: 2px; 44 | background: linear-gradient(45deg, rgba(251, 146, 60, 0.8), rgba(251, 191, 36, 0.8)); 45 | -webkit-mask: linear-gradient(#fff 0 0) content-box, linear-gradient(#fff 0 0); 46 | -webkit-mask-composite: xor; 47 | mask-composite: exclude; 48 | opacity: 0.7; 49 | animation: glow-pulse 2s ease-in-out infinite; 50 | } 51 | 52 | @theme inline { 53 | --color-background: var(--background); 54 | --color-foreground: var(--foreground); 55 | --font-sans: var(--font-geist-sans); 56 | --font-mono: var(--font-geist-mono); 57 | --color-sidebar-ring: var(--sidebar-ring); 58 | --color-sidebar-border: var(--sidebar-border); 59 | --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); 60 | --color-sidebar-accent: var(--sidebar-accent); 61 | --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); 62 | --color-sidebar-primary: var(--sidebar-primary); 63 | --color-sidebar-foreground: var(--sidebar-foreground); 64 | --color-sidebar: var(--sidebar); 65 | --color-chart-5: var(--chart-5); 66 | --color-chart-4: var(--chart-4); 67 | --color-chart-3: var(--chart-3); 68 | --color-chart-2: var(--chart-2); 69 | --color-chart-1: var(--chart-1); 70 | --color-ring: var(--ring); 71 | --color-input: var(--input); 72 | --color-border: var(--border); 73 | --color-destructive: var(--destructive); 74 | --color-accent-foreground: var(--accent-foreground); 75 | --color-accent: var(--accent); 76 | --color-muted-foreground: var(--muted-foreground); 77 | --color-muted: var(--muted); 78 | --color-secondary-foreground: var(--secondary-foreground); 79 | --color-secondary: var(--secondary); 80 | --color-primary-foreground: var(--primary-foreground); 81 | --color-primary: var(--primary); 82 | --color-popover-foreground: var(--popover-foreground); 83 | --color-popover: var(--popover); 84 | --color-card-foreground: var(--card-foreground); 85 | --color-card: var(--card); 86 | --radius-sm: calc(var(--radius) - 4px); 87 | --radius-md: calc(var(--radius) - 2px); 88 | --radius-lg: var(--radius); 89 | --radius-xl: calc(var(--radius) + 4px); 90 | } 91 | 92 | @layer base { 93 | :root { 94 | --background: 0 0% 100%; 95 | --foreground: 222 47% 11%; 96 | --card: 0 0% 100%; 97 | --card-foreground: 222 47% 11%; 98 | --popover: 0 0% 100%; 99 | --popover-foreground: 222 47% 11%; 100 | --primary: 350 100% 62%; 101 | --primary-foreground: 0 0% 100%; 102 | --secondary: 240 5% 96%; 103 | --secondary-foreground: 222 47% 11%; 104 | --muted: 240 5% 96%; 105 | --muted-foreground: 240 4% 46%; 106 | --accent: 240 5% 96%; 107 | --accent-foreground: 222 47% 11%; 108 | --destructive: 0 85% 60%; 109 | --destructive-foreground: 0 0% 100%; 110 | --border: 240 6% 90%; 111 | --input: 240 6% 90%; 112 | --ring: 350 100% 62%; 113 | --radius: 0.75rem; 114 | 115 | /* New gradient variables */ 116 | --gradient-primary: linear-gradient(135deg, #FF6B6B 0%, #FF5E5E 25%, #FF8E53 100%); 117 | --gradient-secondary: linear-gradient(135deg, #667eea 0%, #764ba2 100%); 118 | --gradient-accent: linear-gradient(135deg, #FA8BFF 0%, #2BD2FF 52%, #2BFF88 90%); 119 | --gradient-subtle: linear-gradient(135deg, #FFECD2 0%, #FCB69F 100%); 120 | --gradient-dark: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); 121 | 122 | /* Shadow variables */ 123 | --shadow-sm: 0 1px 2px 0 rgb(0 0 0 / 0.05); 124 | --shadow-md: 0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1); 125 | --shadow-lg: 0 10px 15px -3px rgb(0 0 0 / 0.1), 0 4px 6px -4px rgb(0 0 0 / 0.1); 126 | --shadow-xl: 0 20px 25px -5px rgb(0 0 0 / 0.1), 0 8px 10px -6px rgb(0 0 0 / 0.1); 127 | --shadow-glow: 0 0 20px rgb(255 107 107 / 0.3); 128 | 129 | /* Animation durations and delays */ 130 | --d-1: 150ms; 131 | --d-2: 300ms; 132 | --d-3: 500ms; 133 | --t-1: 200ms; 134 | --t-2: 400ms; 135 | --t-3: 600ms; 136 | --spring: cubic-bezier(0.175, 0.885, 0.32, 1.275); 137 | --ease: cubic-bezier(0.4, 0, 0.2, 1); 138 | } 139 | 140 | .dark { 141 | --background: 222 47% 7%; 142 | --foreground: 0 0% 98%; 143 | --card: 222 47% 9%; 144 | --card-foreground: 0 0% 98%; 145 | --popover: 222 47% 9%; 146 | --popover-foreground: 0 0% 98%; 147 | --primary: 350 100% 62%; 148 | --primary-foreground: 0 0% 100%; 149 | --secondary: 222 47% 15%; 150 | --secondary-foreground: 0 0% 98%; 151 | --muted: 222 47% 15%; 152 | --muted-foreground: 215 20% 65%; 153 | --accent: 222 47% 15%; 154 | --accent-foreground: 0 0% 98%; 155 | --destructive: 0 85% 60%; 156 | --destructive-foreground: 0 0% 100%; 157 | --border: 222 47% 20%; 158 | --input: 222 47% 20%; 159 | --ring: 350 100% 62%; 160 | 161 | /* Dark mode gradient adjustments */ 162 | --gradient-primary: linear-gradient(135deg, #FF6B6B 0%, #FF8E53 100%); 163 | --gradient-secondary: linear-gradient(135deg, #667eea 0%, #764ba2 100%); 164 | --gradient-accent: linear-gradient(135deg, #FA8BFF 0%, #2BD2FF 52%, #2BFF88 90%); 165 | --gradient-subtle: linear-gradient(135deg, rgba(255, 107, 107, 0.1) 0%, rgba(255, 142, 83, 0.1) 100%); 166 | --gradient-dark: linear-gradient(135deg, #0f0f1e 0%, #1a1a2e 100%); 167 | 168 | /* Dark mode shadows */ 169 | --shadow-sm: 0 1px 2px 0 rgb(0 0 0 / 0.5); 170 | --shadow-md: 0 4px 6px -1px rgb(0 0 0 / 0.5), 0 2px 4px -2px rgb(0 0 0 / 0.5); 171 | --shadow-lg: 0 10px 15px -3px rgb(0 0 0 / 0.5), 0 4px 6px -4px rgb(0 0 0 / 0.5); 172 | --shadow-xl: 0 20px 25px -5px rgb(0 0 0 / 0.5), 0 8px 10px -6px rgb(0 0 0 / 0.5); 173 | --shadow-glow: 0 0 30px rgb(255 107 107 / 0.5); 174 | } 175 | } 176 | 177 | @layer base { 178 | * { 179 | @apply border-border; 180 | } 181 | body { 182 | @apply bg-background text-foreground; 183 | font-feature-settings: "rlig" 1, "calt" 1; 184 | } 185 | } 186 | 187 | :root { 188 | --sidebar: hsl(0 0% 98%); 189 | --sidebar-foreground: hsl(240 5.3% 26.1%); 190 | --sidebar-primary: hsl(240 5.9% 10%); 191 | --sidebar-primary-foreground: hsl(0 0% 98%); 192 | --sidebar-accent: hsl(240 4.8% 95.9%); 193 | --sidebar-accent-foreground: hsl(240 5.9% 10%); 194 | --sidebar-border: hsl(220 13% 91%); 195 | --sidebar-ring: hsl(217.2 91.2% 59.8%); 196 | } 197 | 198 | .dark { 199 | --sidebar: hsl(240 5.9% 10%); 200 | --sidebar-foreground: hsl(240 4.8% 95.9%); 201 | --sidebar-primary: hsl(224.3 76.3% 48%); 202 | --sidebar-primary-foreground: hsl(0 0% 100%); 203 | --sidebar-accent: hsl(240 3.7% 15.9%); 204 | --sidebar-accent-foreground: hsl(240 4.8% 95.9%); 205 | --sidebar-border: hsl(240 3.7% 15.9%); 206 | --sidebar-ring: hsl(217.2 91.2% 59.8%); 207 | } 208 | 209 | @layer base { 210 | * { 211 | @apply border-border outline-ring/50; 212 | } 213 | body { 214 | @apply bg-background text-foreground; 215 | } 216 | } 217 | 218 | @keyframes text { 219 | to { 220 | background-position: 200% center; 221 | } 222 | } 223 | 224 | .animate-text { 225 | animation: text 5s ease infinite; 226 | background-size: 200% auto; 227 | } 228 | 229 | @keyframes fade-up { 230 | from { 231 | opacity: 0; 232 | transform: translateY(20px); 233 | } 234 | to { 235 | opacity: 1; 236 | transform: translateY(0); 237 | } 238 | } 239 | 240 | .animate-fade-up { 241 | animation-name: fade-up; 242 | animation-fill-mode: forwards; 243 | } 244 | 245 | @keyframes fade-in { 246 | from { 247 | opacity: 0; 248 | transform: translateY(10px); 249 | } 250 | to { 251 | opacity: 1; 252 | transform: translateY(0); 253 | } 254 | } 255 | 256 | @keyframes fade-in-scale { 257 | from { 258 | opacity: 0; 259 | transform: scale(0.95) translateY(10px); 260 | } 261 | to { 262 | opacity: 1; 263 | transform: scale(1) translateY(0); 264 | } 265 | } 266 | 267 | .animate-fade-in-scale { 268 | animation: fade-in-scale 0.4s ease-out forwards; 269 | } 270 | 271 | @keyframes scale-in { 272 | from { 273 | transform: scale(0); 274 | } 275 | to { 276 | transform: scale(1); 277 | } 278 | } 279 | 280 | .animate-fade-in { 281 | animation: fade-in 0.3s ease-out forwards; 282 | } 283 | 284 | .animate-scale-in { 285 | animation: scale-in 0.2s ease-out; 286 | } 287 | 288 | @keyframes shimmer { 289 | 0% { 290 | background-position: -1000px 0; 291 | } 292 | 100% { 293 | background-position: 1000px 0; 294 | } 295 | } 296 | 297 | .animate-shimmer { 298 | background: linear-gradient( 299 | 90deg, 300 | transparent 0%, 301 | rgba(255, 255, 255, 0.4) 50%, 302 | transparent 100% 303 | ); 304 | background-size: 1000px 100%; 305 | animation: shimmer 2s infinite; 306 | } 307 | 308 | /* New animations */ 309 | @keyframes float { 310 | 0%, 100% { transform: translateY(0px); } 311 | 50% { transform: translateY(-20px); } 312 | } 313 | 314 | @keyframes pulse-glow { 315 | 0%, 100% { 316 | box-shadow: 0 0 20px rgb(255 107 107 / 0.5), 317 | 0 0 40px rgb(255 107 107 / 0.3); 318 | } 319 | 50% { 320 | box-shadow: 0 0 30px rgb(255 107 107 / 0.8), 321 | 0 0 60px rgb(255 107 107 / 0.4); 322 | } 323 | } 324 | 325 | @keyframes gradient-shift { 326 | 0% { background-position: 0% 50%; } 327 | 50% { background-position: 100% 50%; } 328 | 100% { background-position: 0% 50%; } 329 | } 330 | 331 | @keyframes slide-up-fade { 332 | 0% { 333 | opacity: 0; 334 | transform: translateY(40px); 335 | } 336 | 100% { 337 | opacity: 1; 338 | transform: translateY(0); 339 | } 340 | } 341 | 342 | /* Utility classes */ 343 | .glass { 344 | background: rgba(255, 255, 255, 0.7); 345 | backdrop-filter: blur(10px); 346 | -webkit-backdrop-filter: blur(10px); 347 | border: 1px solid rgba(255, 255, 255, 0.2); 348 | } 349 | 350 | .dark .glass { 351 | background: rgba(0, 0, 0, 0.5); 352 | border: 1px solid rgba(255, 255, 255, 0.1); 353 | } 354 | 355 | .gradient-border { 356 | position: relative; 357 | background: linear-gradient(var(--background), var(--background)) padding-box, 358 | var(--gradient-primary) border-box; 359 | border: 2px solid transparent; 360 | } 361 | 362 | .gradient-text { 363 | background: var(--gradient-primary); 364 | -webkit-background-clip: text; 365 | -webkit-text-fill-color: transparent; 366 | background-clip: text; 367 | } 368 | 369 | .hover-lift { 370 | transition: all 0.3s var(--spring); 371 | } 372 | 373 | .hover-lift:hover { 374 | transform: translateY(-4px); 375 | box-shadow: var(--shadow-xl); 376 | } 377 | 378 | .animate-gradient { 379 | background-size: 200% 200%; 380 | animation: gradient-shift 3s ease infinite; 381 | } 382 | 383 | .animate-float { 384 | animation: float 6s ease-in-out infinite; 385 | } 386 | 387 | .animate-pulse-glow { 388 | animation: pulse-glow 2s ease-in-out infinite; 389 | } 390 | 391 | /* Premium button styles */ 392 | .btn-gradient { 393 | background: var(--gradient-primary); 394 | color: white; 395 | font-weight: 600; 396 | position: relative; 397 | overflow: hidden; 398 | transition: all 0.3s var(--ease); 399 | } 400 | 401 | .btn-gradient:hover { 402 | transform: translateY(-2px); 403 | box-shadow: 0 10px 20px rgb(255 107 107 / 0.3); 404 | } 405 | 406 | .btn-gradient::before { 407 | content: ''; 408 | position: absolute; 409 | top: 0; 410 | left: -100%; 411 | width: 100%; 412 | height: 100%; 413 | background: linear-gradient(90deg, transparent, rgba(255,255,255,0.2), transparent); 414 | transition: left 0.5s; 415 | } 416 | 417 | .btn-gradient:hover::before { 418 | left: 100%; 419 | } 420 | 421 | /* Card hover effects */ 422 | .card-hover { 423 | transition: all 0.3s var(--spring); 424 | cursor: pointer; 425 | } 426 | 427 | .card-hover:hover { 428 | transform: translateY(-8px) scale(1.02); 429 | box-shadow: var(--shadow-xl); 430 | } 431 | 432 | /* Smooth number transitions */ 433 | .number-transition { 434 | transition: all 0.8s var(--spring); 435 | } 436 | 437 | /* Custom scrollbar */ 438 | ::-webkit-scrollbar { 439 | width: 8px; 440 | height: 8px; 441 | } 442 | 443 | ::-webkit-scrollbar-track { 444 | background: var(--muted); 445 | border-radius: 4px; 446 | } 447 | 448 | ::-webkit-scrollbar-thumb { 449 | background: var(--muted-foreground); 450 | border-radius: 4px; 451 | } 452 | 453 | ::-webkit-scrollbar-thumb:hover { 454 | background: var(--foreground); 455 | } 456 | -------------------------------------------------------------------------------- /app/api/firestarter/query/route.ts: -------------------------------------------------------------------------------- 1 | import { NextRequest } from 'next/server' 2 | import { streamText } from 'ai' 3 | import { groq } from '@ai-sdk/groq' 4 | import { openai } from '@ai-sdk/openai' 5 | import { anthropic } from '@ai-sdk/anthropic' 6 | import { searchIndex } from '@/lib/upstash-search' 7 | import { serverConfig as config } from '@/firestarter.config' 8 | 9 | // Get AI model at runtime on server 10 | const getModel = () => { 11 | try { 12 | // Initialize models directly here to avoid module-level issues 13 | if (process.env.GROQ_API_KEY) { 14 | return groq('meta-llama/llama-4-scout-17b-16e-instruct') 15 | } 16 | if (process.env.OPENAI_API_KEY) { 17 | return openai('gpt-4o') 18 | } 19 | if (process.env.ANTHROPIC_API_KEY) { 20 | return anthropic('claude-3-5-sonnet-20241022') 21 | } 22 | throw new Error('No AI provider configured. Please set GROQ_API_KEY, OPENAI_API_KEY, or ANTHROPIC_API_KEY') 23 | } catch (error) { 24 | throw error 25 | } 26 | } 27 | 28 | export async function POST(request: NextRequest) { 29 | try { 30 | const body = await request.json() 31 | 32 | // Handle both direct query format and useChat format 33 | let query = body.query 34 | const namespace = body.namespace 35 | const stream = body.stream ?? false 36 | 37 | // If using useChat format, extract query from messages 38 | if (!query && body.messages && Array.isArray(body.messages)) { 39 | const lastUserMessage = body.messages.filter((m: { role: string }) => m.role === 'user').pop() 40 | query = lastUserMessage?.content 41 | } 42 | 43 | if (!query || !namespace) { 44 | return new Response( 45 | JSON.stringify({ error: 'Query and namespace are required' }), 46 | { status: 400, headers: { 'Content-Type': 'application/json' } } 47 | ) 48 | } 49 | 50 | 51 | // Retrieve documents from Upstash Search 52 | interface SearchDocument { 53 | content?: { 54 | text?: string // Searchable text 55 | } 56 | metadata?: { 57 | namespace?: string 58 | title?: string 59 | pageTitle?: string 60 | url?: string 61 | sourceURL?: string 62 | description?: string 63 | fullContent?: string // Full content stored here 64 | } 65 | score?: number 66 | } 67 | 68 | let documents: SearchDocument[] = [] 69 | 70 | try { 71 | // Search for documents - include namespace to improve relevance 72 | 73 | // Include namespace in search to boost relevance 74 | const searchQuery = `${query} ${namespace}` 75 | 76 | const searchResults = await searchIndex.search({ 77 | query: searchQuery, 78 | limit: config.search.maxResults, 79 | reranking: true 80 | }) 81 | 82 | 83 | // Filter to only include documents from the correct namespace 84 | documents = searchResults.filter((doc) => { 85 | const docNamespace = doc.metadata?.namespace 86 | const matches = docNamespace === namespace 87 | if (!matches && doc.metadata?.namespace) { 88 | // Only log first few mismatches to avoid spam 89 | if (documents.length < 3) { 90 | } 91 | } 92 | return matches 93 | }) 94 | 95 | 96 | // If no results, try searching just for documents in this namespace 97 | if (documents.length === 0) { 98 | 99 | const fallbackResults = await searchIndex.search({ 100 | query: namespace, 101 | limit: config.search.maxResults, 102 | reranking: true 103 | }) 104 | 105 | 106 | // Filter for exact namespace match 107 | const namespaceDocs = fallbackResults.filter((doc) => { 108 | return doc.metadata?.namespace === namespace 109 | }) 110 | 111 | 112 | // If we found documents in the namespace, search within their content 113 | if (namespaceDocs.length > 0) { 114 | // Score documents based on query relevance 115 | const queryLower = query.toLowerCase() 116 | documents = namespaceDocs.filter((doc) => { 117 | const content = (doc.content?.text || '').toLowerCase() 118 | const title = (doc.content?.title || '').toLowerCase() 119 | const url = (doc.content?.url || '').toLowerCase() 120 | 121 | return content.includes(queryLower) || 122 | title.includes(queryLower) || 123 | url.includes(queryLower) 124 | }) 125 | 126 | 127 | // If still no results, return all namespace documents 128 | if (documents.length === 0) { 129 | documents = namespaceDocs 130 | } 131 | } 132 | } 133 | 134 | } catch { 135 | console.error('Search failed') 136 | documents = [] 137 | } 138 | 139 | // Check if we have any data for this namespace 140 | if (documents.length === 0) { 141 | 142 | const answer = `I don't have any indexed content for this website. Please make sure the website has been crawled first.` 143 | const sources: never[] = [] 144 | 145 | if (stream) { 146 | // Create a simple text stream for the answer 147 | const result = await streamText({ 148 | model: getModel(), 149 | prompt: answer, 150 | maxTokens: 1, 151 | temperature: 0, 152 | }) 153 | 154 | return result.toDataStreamResponse() 155 | } else { 156 | return new Response( 157 | JSON.stringify({ answer, sources }), 158 | { headers: { 'Content-Type': 'application/json' } } 159 | ) 160 | } 161 | } 162 | 163 | // Check if we have any AI provider configured 164 | try { 165 | const model = getModel() 166 | if (!model) { 167 | throw new Error('No AI model available') 168 | } 169 | } catch { 170 | const answer = 'AI service is not configured. Please set GROQ_API_KEY, OPENAI_API_KEY, or ANTHROPIC_API_KEY in your environment variables.' 171 | return new Response( 172 | JSON.stringify({ answer, sources: [] }), 173 | { headers: { 'Content-Type': 'application/json' } } 174 | ) 175 | } 176 | 177 | // Transform Upstash search results to expected format 178 | interface TransformedDocument { 179 | content: string 180 | url: string 181 | title: string 182 | description: string 183 | score: number 184 | } 185 | 186 | const transformedDocuments: TransformedDocument[] = documents.map((result) => { 187 | const title = result.metadata?.title || result.metadata?.pageTitle || 'Untitled' 188 | const description = result.metadata?.description || '' 189 | const url = result.metadata?.url || result.metadata?.sourceURL || '' 190 | 191 | // Get content from the document - prefer full content from metadata, fallback to searchable text 192 | const rawContent = result.metadata?.fullContent || result.content?.text || '' 193 | 194 | if (!rawContent) { 195 | } 196 | 197 | // Create structured content with clear metadata headers 198 | const structuredContent = `TITLE: ${title} 199 | DESCRIPTION: ${description} 200 | SOURCE: ${url} 201 | 202 | ${rawContent}` 203 | 204 | return { 205 | content: structuredContent, 206 | url: url, 207 | title: title, 208 | description: description, 209 | score: result.score || 0 210 | } 211 | }) 212 | 213 | // Documents from Upstash are already scored by relevance 214 | // Sort by score and take top results 215 | const relevantDocs = transformedDocuments 216 | .sort((a, b) => (b.score || 0) - (a.score || 0)) 217 | .slice(0, config.search.maxSourcesDisplay) // Get many more sources for better coverage 218 | 219 | 220 | // If no matches, use more documents as context 221 | const docsToUse = relevantDocs.length > 0 ? relevantDocs : transformedDocuments.slice(0, 10) 222 | 223 | // Build context from relevant documents - use more content for better answers 224 | const contextDocs = docsToUse.slice(0, config.search.maxContextDocs) // Use top docs for richer context 225 | 226 | // Log document structure for debugging 227 | if (contextDocs.length > 0) { 228 | } 229 | 230 | const context = contextDocs 231 | .map((doc) => { 232 | const content = doc.content || '' 233 | if (!content) { 234 | return null 235 | } 236 | return content.substring(0, config.search.maxContextLength) + '...' 237 | }) 238 | .filter(Boolean) 239 | .join('\n\n---\n\n') 240 | 241 | 242 | // If context is empty, log error 243 | if (!context || context.length < 100) { 244 | 245 | const answer = 'I found some relevant pages but couldn\'t extract enough content to answer your question. This might be due to the way the pages were crawled. Try crawling the website again with a higher page limit.' 246 | const sources = docsToUse.map((doc) => ({ 247 | url: doc.url, 248 | title: doc.title, 249 | snippet: (doc.content || '').substring(0, config.search.snippetLength) + '...' 250 | })) 251 | 252 | return new Response( 253 | JSON.stringify({ answer, sources }), 254 | { headers: { 'Content-Type': 'application/json' } } 255 | ) 256 | } 257 | 258 | // Prepare sources 259 | const sources = docsToUse.map((doc) => ({ 260 | url: doc.url, 261 | title: doc.title, 262 | snippet: (doc.content || '').substring(0, config.search.snippetLength) + '...' 263 | })) 264 | 265 | 266 | // Generate response using Vercel AI SDK 267 | try { 268 | 269 | const systemPrompt = config.ai.systemPrompt 270 | 271 | const userPrompt = `Question: ${query}\n\nRelevant content from the website:\n${context}\n\nPlease provide a comprehensive answer based on this information.` 272 | 273 | 274 | // Log a sample of the actual content being sent 275 | 276 | 277 | if (stream) { 278 | 279 | let result 280 | try { 281 | const model = getModel() 282 | 283 | // Stream the response 284 | result = await streamText({ 285 | model: model, 286 | messages: [ 287 | { role: 'system', content: systemPrompt }, 288 | { role: 'user', content: userPrompt } 289 | ], 290 | temperature: config.ai.temperature, 291 | maxTokens: config.ai.maxTokens 292 | }) 293 | 294 | } catch (streamError) { 295 | throw streamError 296 | } 297 | 298 | // Create a streaming response with sources 299 | 300 | // Always use custom streaming to include sources 301 | // The built-in toDataStreamResponse doesn't include our sources 302 | const encoder = new TextEncoder() 303 | 304 | const stream = new ReadableStream({ 305 | async start(controller) { 306 | // Send sources as initial data 307 | const sourcesData = { sources } 308 | const sourcesLine = `8:${JSON.stringify(sourcesData)}\n` 309 | controller.enqueue(encoder.encode(sourcesLine)) 310 | 311 | // Stream the text 312 | try { 313 | for await (const textPart of result.textStream) { 314 | // Format as Vercel AI SDK expects 315 | const escaped = JSON.stringify(textPart) 316 | controller.enqueue(encoder.encode(`0:${escaped}\n`)) 317 | } 318 | } catch { 319 | console.error('Stream processing failed') 320 | } 321 | 322 | controller.close() 323 | } 324 | }) 325 | 326 | return new Response(stream, { 327 | headers: { 328 | 'Content-Type': 'text/plain; charset=utf-8' 329 | } 330 | }) 331 | } else { 332 | // Non-streaming response 333 | const result = await streamText({ 334 | model: getModel(), 335 | messages: [ 336 | { role: 'system', content: systemPrompt }, 337 | { role: 'user', content: userPrompt } 338 | ], 339 | temperature: config.ai.temperature, 340 | maxTokens: config.ai.maxTokens 341 | }) 342 | 343 | // Get the full text 344 | let answer = '' 345 | for await (const textPart of result.textStream) { 346 | answer += textPart 347 | } 348 | 349 | return new Response( 350 | JSON.stringify({ answer, sources }), 351 | { headers: { 'Content-Type': 'application/json' } } 352 | ) 353 | } 354 | 355 | } catch (groqError) { 356 | 357 | const errorMessage = groqError instanceof Error ? groqError.message : 'Unknown error' 358 | let answer = `Error generating response: ${errorMessage}` 359 | 360 | if (errorMessage.includes('401') || errorMessage.includes('Unauthorized')) { 361 | answer = 'Error: Groq API authentication failed. Please check your GROQ_API_KEY.' 362 | } else if (errorMessage.includes('rate limit')) { 363 | answer = 'Error: Groq API rate limit exceeded. Please try again later.' 364 | } 365 | 366 | return new Response( 367 | JSON.stringify({ answer, sources }), 368 | { headers: { 'Content-Type': 'application/json' } } 369 | ) 370 | } 371 | } catch { 372 | console.error('Query processing failed') 373 | return new Response( 374 | JSON.stringify({ error: 'Failed to process query' }), 375 | { status: 500, headers: { 'Content-Type': 'application/json' } } 376 | ) 377 | } 378 | } -------------------------------------------------------------------------------- /app/api/v1/chat/completions/route.ts: -------------------------------------------------------------------------------- 1 | import { NextRequest, NextResponse } from 'next/server' 2 | import { serverConfig as config } from '@/firestarter.config' 3 | 4 | // CORS headers for API access 5 | export async function OPTIONS() { 6 | return new NextResponse(null, { 7 | status: 200, 8 | headers: { 9 | 'Access-Control-Allow-Origin': '*', 10 | 'Access-Control-Allow-Methods': 'POST, OPTIONS', 11 | 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Use-Groq, X-Use-OpenAI', 12 | 'Access-Control-Max-Age': '86400', 13 | }, 14 | }) 15 | } 16 | 17 | // OpenAI-compatible chat completions endpoint 18 | export async function POST(request: NextRequest) { 19 | try { 20 | const body = await request.json() 21 | const { messages, model, stream = false } = body 22 | 23 | // Check if this is a Groq API request 24 | const useGroq = request.headers.get('X-Use-Groq') === 'true' 25 | const useOpenAI = request.headers.get('X-Use-OpenAI') === 'true' 26 | 27 | if (useGroq) { 28 | // Handle Groq API request 29 | const groqApiKey = process.env.GROQ_API_KEY 30 | 31 | if (!groqApiKey) { 32 | return NextResponse.json( 33 | { 34 | error: { 35 | message: 'Groq API key not configured', 36 | type: 'server_error', 37 | code: 500 38 | } 39 | }, 40 | { status: 500 } 41 | ) 42 | } 43 | 44 | // Forward request to Groq API 45 | const groqResponse = await fetch('https://api.groq.com/openai/v1/chat/completions', { 46 | method: 'POST', 47 | headers: { 48 | 'Authorization': `Bearer ${groqApiKey}`, 49 | 'Content-Type': 'application/json' 50 | }, 51 | body: JSON.stringify({ 52 | messages, 53 | model, 54 | stream, 55 | temperature: body.temperature || config.ai.temperature, 56 | max_tokens: body.max_tokens || 2000 // Keep higher default for OpenAI-compatible endpoint 57 | }) 58 | }) 59 | 60 | if (!groqResponse.ok) { 61 | const errorData = await groqResponse.json() 62 | throw new Error(errorData.error?.message || 'Groq API error') 63 | } 64 | 65 | const groqData = await groqResponse.json() 66 | 67 | return NextResponse.json(groqData, { 68 | headers: { 69 | 'Access-Control-Allow-Origin': '*', 70 | 'Access-Control-Allow-Methods': 'POST, OPTIONS', 71 | 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Use-Groq, X-Use-OpenAI', 72 | } 73 | }) 74 | } 75 | 76 | if (useOpenAI) { 77 | // Handle OpenAI API request with follow-up questions 78 | const openaiApiKey = process.env.OPENAI_API_KEY 79 | 80 | if (!openaiApiKey) { 81 | return NextResponse.json( 82 | { 83 | error: { 84 | message: 'OpenAI API key not configured', 85 | type: 'server_error', 86 | code: 500 87 | } 88 | }, 89 | { status: 500 } 90 | ) 91 | } 92 | 93 | // First, get the main response 94 | // Handle streaming differently 95 | if (stream) { 96 | const openaiResponse = await fetch('https://api.openai.com/v1/chat/completions', { 97 | method: 'POST', 98 | headers: { 99 | 'Authorization': `Bearer ${openaiApiKey}`, 100 | 'Content-Type': 'application/json' 101 | }, 102 | body: JSON.stringify({ 103 | messages, 104 | model, 105 | stream: true, 106 | temperature: body.temperature || config.ai.temperature, 107 | max_tokens: body.max_tokens || 2000 108 | }) 109 | }) 110 | 111 | if (!openaiResponse.ok) { 112 | const errorData = await openaiResponse.json() 113 | throw new Error(errorData.error?.message || 'OpenAI API error') 114 | } 115 | 116 | // Return the streaming response directly 117 | return new Response(openaiResponse.body, { 118 | headers: { 119 | 'Content-Type': 'text/event-stream', 120 | 'Cache-Control': 'no-cache', 121 | 'Connection': 'keep-alive', 122 | 'Access-Control-Allow-Origin': '*', 123 | } 124 | }) 125 | } 126 | 127 | // Non-streaming response 128 | const openaiResponse = await fetch('https://api.openai.com/v1/chat/completions', { 129 | method: 'POST', 130 | headers: { 131 | 'Authorization': `Bearer ${openaiApiKey}`, 132 | 'Content-Type': 'application/json' 133 | }, 134 | body: JSON.stringify({ 135 | messages, 136 | model, 137 | stream: false, 138 | temperature: body.temperature || config.ai.temperature, 139 | max_tokens: body.max_tokens || 2000 140 | }) 141 | }) 142 | 143 | if (!openaiResponse.ok) { 144 | const errorData = await openaiResponse.json() 145 | throw new Error(errorData.error?.message || 'OpenAI API error') 146 | } 147 | 148 | const openaiData = await openaiResponse.json() 149 | 150 | // Generate follow-up questions 151 | const lastUserMessage = messages.filter((m: { role: string }) => m.role === 'user').pop() 152 | const assistantResponse = openaiData.choices[0].message.content 153 | 154 | const followUpResponse = await fetch('https://api.openai.com/v1/chat/completions', { 155 | method: 'POST', 156 | headers: { 157 | 'Authorization': `Bearer ${openaiApiKey}`, 158 | 'Content-Type': 'application/json' 159 | }, 160 | body: JSON.stringify({ 161 | messages: [ 162 | { 163 | role: 'system', 164 | content: 'Generate 3 relevant follow-up questions based on the query and answer. Return only the questions, one per line, no numbering or bullets.' 165 | }, 166 | { 167 | role: 'user', 168 | content: `Original query: "${lastUserMessage?.content}"\n\nAnswer summary: ${assistantResponse.slice(0, 1000)}...\n\nGenerate 3 follow-up questions that explore different aspects or dig deeper into the topic.` 169 | } 170 | ], 171 | model: 'gpt-4o-mini', 172 | temperature: 0.8, // Higher temperature for more diverse follow-up questions 173 | max_tokens: 200 // Limited tokens for concise follow-up questions 174 | }) 175 | }) 176 | 177 | if (followUpResponse.ok) { 178 | const followUpData = await followUpResponse.json() 179 | const followUpText = followUpData.choices[0].message.content 180 | const followUpQuestions = followUpText 181 | .split('\n') 182 | .map((q: string) => q.trim()) 183 | .filter((q: string) => q.length > 0) 184 | .slice(0, 3) 185 | 186 | // Add follow-up questions to the response 187 | openaiData.follow_up_questions = followUpQuestions 188 | } 189 | 190 | return NextResponse.json(openaiData, { 191 | headers: { 192 | 'Access-Control-Allow-Origin': '*', 193 | 'Access-Control-Allow-Methods': 'POST, OPTIONS', 194 | 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Use-Groq, X-Use-OpenAI', 195 | } 196 | }) 197 | } 198 | 199 | // Original Firecrawl namespace logic 200 | let namespace = '' 201 | 202 | if (model?.startsWith('firecrawl-')) { 203 | // Extract the domain part after "firecrawl-" 204 | const domainPart = model.substring('firecrawl-'.length) 205 | // For now, we'll need to look up the actual namespace based on the domain 206 | // This is a simplified version - in production you'd want to store a mapping 207 | namespace = domainPart 208 | } 209 | 210 | if (!namespace) { 211 | return NextResponse.json( 212 | { 213 | error: { 214 | message: 'Invalid model specified. Use format: firecrawl-', 215 | type: 'invalid_request_error', 216 | code: 400 217 | } 218 | }, 219 | { status: 400 } 220 | ) 221 | } 222 | 223 | // Get the last user message for context search 224 | interface Message { 225 | role: string 226 | content: string 227 | } 228 | 229 | const lastUserMessage = messages.filter((m: Message) => m.role === 'user').pop() 230 | const query = lastUserMessage?.content || '' 231 | 232 | // Handle streaming for firecrawl models 233 | if (stream) { 234 | const contextResponse = await fetch(`${request.nextUrl.origin}/api/firestarter/query`, { 235 | method: 'POST', 236 | headers: { 'Content-Type': 'application/json' }, 237 | body: JSON.stringify({ 238 | query, 239 | namespace, 240 | messages: messages.slice(0, -1), 241 | stream: true 242 | }) 243 | }) 244 | 245 | if (!contextResponse.ok) { 246 | const error = await contextResponse.text() 247 | throw new Error(error || 'Failed to retrieve context') 248 | } 249 | 250 | // Transform Vercel AI SDK stream to OpenAI format 251 | const reader = contextResponse.body?.getReader() 252 | if (!reader) throw new Error('No response body') 253 | 254 | const encoder = new TextEncoder() 255 | const decoder = new TextDecoder() 256 | 257 | const stream = new ReadableStream({ 258 | async start(controller) { 259 | let buffer = '' 260 | 261 | // Send initial chunk 262 | controller.enqueue(encoder.encode(`data: {"id":"chatcmpl-${Date.now()}","object":"chat.completion.chunk","created":${Math.floor(Date.now() / 1000)},"model":"${model}","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}\n\n`)) 263 | 264 | while (true) { 265 | const { done, value } = await reader.read() 266 | if (done) break 267 | 268 | const chunk = decoder.decode(value) 269 | buffer += chunk 270 | const lines = buffer.split('\n') 271 | buffer = lines.pop() || '' 272 | 273 | for (const line of lines) { 274 | if (line.trim() === '') continue 275 | 276 | // Handle Vercel AI SDK format 277 | if (line.startsWith('0:')) { 278 | const content = line.slice(2) 279 | if (content.startsWith('"') && content.endsWith('"')) { 280 | try { 281 | const text = JSON.parse(content) 282 | const data = { 283 | id: `chatcmpl-${Date.now()}`, 284 | object: 'chat.completion.chunk', 285 | created: Math.floor(Date.now() / 1000), 286 | model: model, 287 | choices: [{ 288 | index: 0, 289 | delta: { content: text }, 290 | finish_reason: null 291 | }] 292 | } 293 | controller.enqueue(encoder.encode(`data: ${JSON.stringify(data)}\n\n`)) 294 | } catch { 295 | // Skip invalid JSON 296 | } 297 | } 298 | } 299 | } 300 | } 301 | 302 | // Send final chunk 303 | controller.enqueue(encoder.encode(`data: {"id":"chatcmpl-${Date.now()}","object":"chat.completion.chunk","created":${Math.floor(Date.now() / 1000)},"model":"${model}","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n`)) 304 | controller.enqueue(encoder.encode('data: [DONE]\n\n')) 305 | controller.close() 306 | } 307 | }) 308 | 309 | return new Response(stream, { 310 | headers: { 311 | 'Content-Type': 'text/event-stream', 312 | 'Cache-Control': 'no-cache', 313 | 'Connection': 'keep-alive', 314 | 'Access-Control-Allow-Origin': '*', 315 | } 316 | }) 317 | } 318 | 319 | // Non-streaming response 320 | const contextResponse = await fetch(`${request.nextUrl.origin}/api/firestarter/query`, { 321 | method: 'POST', 322 | headers: { 'Content-Type': 'application/json' }, 323 | body: JSON.stringify({ 324 | query, 325 | namespace, 326 | messages: messages.slice(0, -1), 327 | stream: false 328 | }) 329 | }) 330 | 331 | const contextData = await contextResponse.json() 332 | 333 | if (!contextResponse.ok) { 334 | throw new Error(contextData.error || 'Failed to retrieve context') 335 | } 336 | 337 | // Format the response in OpenAI format 338 | const completion = { 339 | id: `chatcmpl-${Date.now()}`, 340 | object: 'chat.completion', 341 | created: Math.floor(Date.now() / 1000), 342 | model: model, 343 | choices: [ 344 | { 345 | index: 0, 346 | message: { 347 | role: 'assistant', 348 | content: contextData.answer 349 | }, 350 | finish_reason: 'stop' 351 | } 352 | ] 353 | } 354 | 355 | // Add sources as metadata if available 356 | if (contextData.sources && contextData.sources.length > 0) { 357 | interface Source { 358 | title: string 359 | url: string 360 | } 361 | 362 | completion.choices[0].message.content += `\n\n**Sources:**\n${contextData.sources.map((s: Source) => `- [${s.title}](${s.url})`).join('\n')}` 363 | } 364 | 365 | return NextResponse.json(completion, { 366 | headers: { 367 | 'Access-Control-Allow-Origin': '*', 368 | 'Access-Control-Allow-Methods': 'POST, OPTIONS', 369 | 'Access-Control-Allow-Headers': 'Content-Type, Authorization', 370 | } 371 | }) 372 | } catch (error) { 373 | return NextResponse.json( 374 | { 375 | error: { 376 | message: error instanceof Error ? error.message : 'Failed to process chat completion', 377 | type: 'server_error', 378 | code: 500 379 | } 380 | }, 381 | { 382 | status: 500, 383 | headers: { 384 | 'Access-Control-Allow-Origin': '*', 385 | 'Access-Control-Allow-Methods': 'POST, OPTIONS', 386 | 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Use-Groq, X-Use-OpenAI', 387 | } 388 | } 389 | ) 390 | } 391 | } -------------------------------------------------------------------------------- /app/page.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { useState, useEffect } from "react"; 4 | import { useRouter } from "next/navigation"; 5 | import Image from "next/image"; 6 | import Link from "next/link"; 7 | import { Button } from "@/components/ui/button"; 8 | import { Input } from "@/components/ui/input"; 9 | import { useStorage } from "@/hooks/useStorage"; 10 | import { clientConfig as config } from "@/firestarter.config"; 11 | import { 12 | Globe, 13 | ArrowRight, 14 | Settings, 15 | Loader2, 16 | CheckCircle2, 17 | FileText, 18 | AlertCircle, 19 | Database, 20 | Zap, 21 | Search, 22 | Sparkles, 23 | Lock, 24 | ExternalLink 25 | } from "lucide-react"; 26 | import { toast } from "sonner"; 27 | import { 28 | Dialog, 29 | DialogContent, 30 | DialogDescription, 31 | DialogFooter, 32 | DialogHeader, 33 | DialogTitle, 34 | } from "@/components/ui/dialog"; 35 | 36 | export default function FirestarterPage() { 37 | const router = useRouter(); 38 | const searchParams = new URLSearchParams(typeof window !== 'undefined' ? window.location.search : ''); 39 | const urlParam = searchParams.get('url'); 40 | const { saveIndex } = useStorage(); 41 | 42 | const [url, setUrl] = useState(urlParam || 'https://docs.firecrawl.dev/'); 43 | const [hasInteracted, setHasInteracted] = useState(false); 44 | const [loading, setLoading] = useState(false); 45 | const [showSettings, setShowSettings] = useState(false); 46 | const [pageLimit, setPageLimit] = useState(config.crawling.defaultLimit); 47 | const [isCreationDisabled, setIsCreationDisabled] = useState(undefined); 48 | const [crawlProgress, setCrawlProgress] = useState<{ 49 | status: string; 50 | pagesFound: number; 51 | pagesScraped: number; 52 | currentPage?: string; 53 | } | null>(null); 54 | const [showApiKeyModal, setShowApiKeyModal] = useState(false); 55 | const [firecrawlApiKey, setFirecrawlApiKey] = useState(''); 56 | const [isValidatingApiKey, setIsValidatingApiKey] = useState(false); 57 | const [hasFirecrawlKey, setHasFirecrawlKey] = useState(false); 58 | 59 | useEffect(() => { 60 | // Check environment and API keys 61 | fetch('/api/check-env') 62 | .then(res => res.json()) 63 | .then(data => { 64 | setIsCreationDisabled(data.environmentStatus.DISABLE_CHATBOT_CREATION || false); 65 | 66 | // Check for Firecrawl API key 67 | const hasEnvFirecrawl = data.environmentStatus.FIRECRAWL_API_KEY; 68 | setHasFirecrawlKey(hasEnvFirecrawl); 69 | 70 | if (!hasEnvFirecrawl) { 71 | // Check localStorage for saved API key 72 | const savedKey = localStorage.getItem('firecrawl_api_key'); 73 | if (savedKey) { 74 | setFirecrawlApiKey(savedKey); 75 | setHasFirecrawlKey(true); 76 | } 77 | } 78 | }) 79 | .catch(() => { 80 | setIsCreationDisabled(false); 81 | }); 82 | }, []); 83 | 84 | const handleSubmit = async (e: React.FormEvent) => { 85 | e.preventDefault(); 86 | if (!url) return; 87 | 88 | // Check if we have Firecrawl API key 89 | 90 | if (!hasFirecrawlKey && !localStorage.getItem('firecrawl_api_key')) { 91 | setShowApiKeyModal(true); 92 | return; 93 | } 94 | 95 | // Normalize URL 96 | let normalizedUrl = url.trim(); 97 | if (!normalizedUrl.startsWith('http://') && !normalizedUrl.startsWith('https://')) { 98 | normalizedUrl = 'https://' + normalizedUrl; 99 | } 100 | 101 | // Validate URL 102 | try { 103 | new URL(normalizedUrl); 104 | } catch { 105 | toast.error('Please enter a valid URL'); 106 | return; 107 | } 108 | 109 | setLoading(true); 110 | setCrawlProgress({ 111 | status: 'Starting crawl...', 112 | pagesFound: 0, 113 | pagesScraped: 0 114 | }); 115 | 116 | interface CrawlResponse { 117 | success: boolean 118 | namespace: string 119 | crawlId?: string 120 | details: { 121 | url: string 122 | pagesCrawled: number 123 | } 124 | data: Array<{ 125 | url?: string 126 | metadata?: { 127 | sourceURL?: string 128 | title?: string 129 | ogTitle?: string 130 | description?: string 131 | ogDescription?: string 132 | favicon?: string 133 | ogImage?: string 134 | 'og:image'?: string 135 | 'twitter:image'?: string 136 | } 137 | }> 138 | } 139 | 140 | let data: CrawlResponse | null = null; 141 | 142 | try { 143 | // Simulate progressive updates 144 | let currentProgress = 0; 145 | 146 | const progressInterval = setInterval(() => { 147 | currentProgress += Math.random() * 3; 148 | if (currentProgress > pageLimit * 0.8) { 149 | clearInterval(progressInterval); 150 | } 151 | 152 | setCrawlProgress(prev => { 153 | if (!prev) return null; 154 | const scraped = Math.min(Math.floor(currentProgress), pageLimit); 155 | return { 156 | ...prev, 157 | status: scraped < pageLimit * 0.3 ? 'Discovering pages...' : 158 | scraped < pageLimit * 0.7 ? 'Scraping content...' : 159 | 'Finalizing...', 160 | pagesFound: pageLimit, 161 | pagesScraped: scraped, 162 | currentPage: scraped > 0 ? `Processing page ${scraped} of ${pageLimit}` : undefined 163 | }; 164 | }); 165 | }, 300); 166 | 167 | // Get API key from localStorage if not in environment 168 | const firecrawlApiKey = localStorage.getItem('firecrawl_api_key'); 169 | 170 | const headers: Record = { 171 | 'Content-Type': 'application/json', 172 | }; 173 | 174 | // Add API key to headers if available from localStorage (and not in env) 175 | if (firecrawlApiKey) { 176 | headers['X-Firecrawl-API-Key'] = firecrawlApiKey; 177 | } 178 | 179 | const response = await fetch('/api/firestarter/create', { 180 | method: 'POST', 181 | headers, 182 | body: JSON.stringify({ url: normalizedUrl, limit: pageLimit }) 183 | }); 184 | 185 | data = await response.json(); 186 | 187 | // Clear the interval 188 | if (progressInterval) clearInterval(progressInterval); 189 | 190 | if (data && data.success) { 191 | // Update progress to show completion 192 | setCrawlProgress({ 193 | status: 'Crawl complete!', 194 | pagesFound: data.details?.pagesCrawled || 0, 195 | pagesScraped: data.details?.pagesCrawled || 0 196 | }); 197 | 198 | // Find the homepage in crawled data for metadata 199 | let homepageMetadata: { 200 | title?: string 201 | ogTitle?: string 202 | description?: string 203 | ogDescription?: string 204 | favicon?: string 205 | ogImage?: string 206 | 'og:image'?: string 207 | 'twitter:image'?: string 208 | } = {}; 209 | if (data.data && data.data.length > 0) { 210 | const homepage = data.data.find((page) => { 211 | const pageUrl = page.metadata?.sourceURL || page.url || ''; 212 | // Check if it's the homepage 213 | return pageUrl === normalizedUrl || pageUrl === normalizedUrl + '/' || pageUrl === normalizedUrl.replace(/\/$/, ''); 214 | }) || data.data[0]; // Fallback to first page 215 | 216 | homepageMetadata = homepage.metadata || {}; 217 | } 218 | 219 | // Store the crawl info and redirect to dashboard 220 | const siteInfo = { 221 | url: normalizedUrl, 222 | namespace: data.namespace, 223 | crawlId: data.crawlId, 224 | pagesCrawled: data.details?.pagesCrawled || 0, 225 | crawlComplete: true, 226 | crawlDate: new Date().toISOString(), 227 | metadata: { 228 | title: homepageMetadata.ogTitle || homepageMetadata.title || new URL(normalizedUrl).hostname, 229 | description: homepageMetadata.ogDescription || homepageMetadata.description || 'Your custom website', 230 | favicon: homepageMetadata.favicon, 231 | ogImage: homepageMetadata.ogImage || homepageMetadata['og:image'] || homepageMetadata['twitter:image'] 232 | } 233 | }; 234 | 235 | // Store only metadata for current session (no crawlData - that's in Upstash) 236 | sessionStorage.setItem('firestarter_current_data', JSON.stringify(siteInfo)); 237 | 238 | // Save index metadata using the storage hook 239 | await saveIndex({ 240 | url: normalizedUrl, 241 | namespace: data.namespace, 242 | pagesCrawled: data.details?.pagesCrawled || 0, 243 | createdAt: new Date().toISOString(), 244 | metadata: { 245 | title: homepageMetadata.ogTitle || homepageMetadata.title || new URL(normalizedUrl).hostname, 246 | description: homepageMetadata.ogDescription || homepageMetadata.description || 'Your custom website', 247 | favicon: homepageMetadata.favicon, 248 | ogImage: homepageMetadata.ogImage || homepageMetadata['og:image'] || homepageMetadata['twitter:image'] 249 | } 250 | }); 251 | 252 | // Small delay to show completion 253 | setTimeout(() => { 254 | router.push(`/dashboard?namespace=${siteInfo.namespace}`); 255 | }, 1000); 256 | } else if (data && 'error' in data) { 257 | setCrawlProgress({ 258 | status: 'Error: ' + (data as { error: string }).error, 259 | pagesFound: 0, 260 | pagesScraped: 0 261 | }); 262 | toast.error((data as { error: string }).error); 263 | } 264 | } catch { 265 | toast.error('Failed to start crawling. Please try again.'); 266 | } finally { 267 | if (!data?.success) { 268 | setLoading(false); 269 | setCrawlProgress(null); 270 | } 271 | } 272 | }; 273 | 274 | const handleApiKeySubmit = async () => { 275 | if (!firecrawlApiKey.trim()) { 276 | toast.error('Please enter a valid Firecrawl API key'); 277 | return; 278 | } 279 | 280 | setIsValidatingApiKey(true); 281 | 282 | try { 283 | // Test the Firecrawl API key 284 | const response = await fetch('/api/scrape', { 285 | method: 'POST', 286 | headers: { 287 | 'Content-Type': 'application/json', 288 | 'X-Firecrawl-API-Key': firecrawlApiKey, 289 | }, 290 | body: JSON.stringify({ url: 'https://example.com' }), 291 | }); 292 | 293 | if (!response.ok) { 294 | throw new Error('Invalid Firecrawl API key'); 295 | } 296 | 297 | // Save the API key to localStorage 298 | localStorage.setItem('firecrawl_api_key', firecrawlApiKey); 299 | setHasFirecrawlKey(true); 300 | 301 | toast.success('API key saved successfully!'); 302 | setShowApiKeyModal(false); 303 | 304 | // Trigger form submission after API key is saved 305 | if (url) { 306 | const form = document.querySelector('form'); 307 | if (form) { 308 | form.requestSubmit(); 309 | } 310 | } 311 | } catch { 312 | toast.error('Invalid API key. Please check and try again.'); 313 | } finally { 314 | setIsValidatingApiKey(false); 315 | } 316 | }; 317 | 318 | return ( 319 |
320 |
321 | 322 | Firecrawl Logo 328 | 329 |
330 | 339 | 355 |
356 |
357 | 358 | {isCreationDisabled === undefined ? ( 359 | // Show loading state while checking environment 360 |
361 |
362 |

363 | Firestarter
364 | 365 | Loading... 366 | 367 |

368 |
369 |
370 | ) : isCreationDisabled === true ? ( 371 |
372 |
373 |

374 | Firestarter
375 | 376 | Read-Only Mode 377 | 378 |

379 |
380 | 381 |
382 | 383 |

384 | Chatbot Creation Disabled 385 |

386 |

387 | Chatbot creation has been disabled by the administrator. You can only view and interact with existing chatbots. 388 |

389 |
390 | 399 | 408 |
409 |
410 |
411 | ) : ( 412 | <> 413 |
414 |

415 | Firestarter
416 | 417 | Chatbots, Instantly. 418 | 419 |

420 |
421 | 422 |
423 |
424 |
425 | { 429 | setUrl(e.target.value); 430 | setHasInteracted(true); 431 | }} 432 | onFocus={() => { 433 | if (!hasInteracted && url === 'https://docs.firecrawl.dev/') { 434 | setUrl(''); 435 | setHasInteracted(true); 436 | } 437 | }} 438 | placeholder="https://example.com" 439 | className="w-full h-14 px-6 text-lg" 440 | required 441 | disabled={loading} 442 | /> 443 | 461 |
462 |
463 | 464 | {/* Loading Progress */} 465 | {loading && crawlProgress && ( 466 |
467 |
468 |

469 | {crawlProgress.status === 'Crawl complete!' ? ( 470 | 471 | ) : crawlProgress.status.includes('Error') ? ( 472 | 473 | ) : ( 474 | 475 | )} 476 | {crawlProgress.status} 477 |

478 |
479 | 480 |
481 |
482 | Pages discovered 483 | 484 | {crawlProgress.pagesFound} 485 | 486 |
487 | 488 |
489 | Pages scraped 490 | 491 | {crawlProgress.pagesScraped} 492 | 493 |
494 | 495 | {crawlProgress.pagesFound > 0 && ( 496 |
497 |
498 |
502 |
503 |
504 | )} 505 | 506 | {crawlProgress.currentPage && ( 507 |
508 |

Currently scraping:

509 |

510 | 511 | {crawlProgress.currentPage} 512 |

513 |
514 | )} 515 |
516 |
517 | )} 518 | 519 | {/* Settings Button */} 520 |
521 | 531 |
532 | 533 | {/* Settings Panel */} 534 | {showSettings && ( 535 |
536 |

Crawl Settings

537 | 538 |
539 |
540 | 543 |
544 | setPageLimit(parseInt(e.target.value))} 551 | className="flex-1 accent-orange-500" 552 | disabled={loading} 553 | /> 554 | {pageLimit} 555 |
556 |

557 | More pages = better coverage but longer crawl time 558 |

559 |

560 | * To set limit higher - feel free to pull the GitHub repo and deploy your own version (with a better copy) 561 |

562 |
563 | 564 |
565 | {config.crawling.limitOptions.map(limit => ( 566 | 576 | ))} 577 |
578 |
579 |
580 | )} 581 | 582 |
583 |
584 |
585 |
586 | 587 |
588 |
589 |

Smart Crawling

590 |
591 |
592 | 593 |
594 |
595 | 596 |
597 |
598 |

Content Extraction

599 |
600 |
601 | 602 |
603 |
604 | 605 |
606 |
607 |

Intelligent Chunking

608 |
609 |
610 | 611 |
612 |
613 | 614 |
615 |
616 |

Semantic Search

617 |
618 |
619 | 620 |
621 |
622 | 623 |
624 |
625 |

RAG Pipeline

626 |
627 |
628 | 629 |
630 |
631 | 632 |
633 |
634 |

Instant API

635 |
636 |
637 |
638 |
639 |
640 | 641 | )} 642 | 643 | {/* API Key Modal */} 644 | 645 | 646 | 647 | Firecrawl API Key Required 648 | 649 | This tool requires a Firecrawl API key to crawl and index websites. 650 | 651 | 652 |
653 | 662 |
663 | 666 | setFirecrawlApiKey(e.target.value)} 672 | onKeyDown={(e) => { 673 | if (e.key === 'Enter' && !isValidatingApiKey) { 674 | handleApiKeySubmit(); 675 | } 676 | }} 677 | disabled={isValidatingApiKey} 678 | /> 679 |
680 |
681 | 682 | 690 | 705 | 706 | 707 | 708 |
709 | ); 710 | } -------------------------------------------------------------------------------- /app/dashboard/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import { useState, useEffect, useRef, Suspense } from 'react' 4 | import { useRouter, useSearchParams } from 'next/navigation' 5 | import { Button } from "@/components/ui/button" 6 | import { Input } from "@/components/ui/input" 7 | import { Send, Globe, Copy, Check, FileText, Database, ArrowLeft, ExternalLink, BookOpen } from 'lucide-react' 8 | import Image from 'next/image' 9 | // Removed useChat - using custom implementation 10 | import { toast } from "sonner" 11 | import { 12 | Dialog, 13 | DialogContent, 14 | DialogDescription, 15 | DialogFooter, 16 | DialogHeader, 17 | DialogTitle, 18 | } from "@/components/ui/dialog" 19 | 20 | interface Source { 21 | url: string 22 | title: string 23 | snippet: string 24 | } 25 | 26 | interface SiteData { 27 | url: string 28 | namespace: string 29 | pagesCrawled: number 30 | metadata: { 31 | title: string 32 | description: string 33 | favicon?: string 34 | ogImage?: string 35 | } 36 | crawlId?: string 37 | crawlComplete?: boolean 38 | crawlDate?: string 39 | createdAt?: string 40 | } 41 | 42 | // Simple markdown renderer component 43 | function MarkdownContent({ content, onSourceClick, isStreaming = false }: { content: string; onSourceClick?: (index: number) => void; isStreaming?: boolean }) { 44 | // Simple markdown parsing 45 | const parseMarkdown = (text: string) => { 46 | // First, handle code blocks to prevent other parsing inside them 47 | const codeBlocks: string[] = []; 48 | let parsed = text.replace(/```([\s\S]*?)```/g, (_, code) => { 49 | const placeholder = `__CODE_BLOCK_${codeBlocks.length}__`; 50 | codeBlocks.push(`
${code.trim()}
`); 51 | return placeholder; 52 | }); 53 | 54 | // Handle inline code 55 | parsed = parsed.replace(/`([^`]+)`/g, '$1'); 56 | 57 | // Handle links [text](url) - must come before citations 58 | parsed = parsed.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '$1'); 59 | 60 | // Handle citations [1], [2], etc. 61 | parsed = parsed.replace(/\[(\d+)\]/g, (_, num) => { 62 | return `[${num}]`; 63 | }); 64 | 65 | // Bold text 66 | parsed = parsed.replace(/\*\*(.+?)\*\*/g, '$1'); 67 | 68 | // Italic text 69 | parsed = parsed.replace(/\*(.+?)\*/g, '$1'); 70 | 71 | // Split into lines for processing 72 | const lines = parsed.split('\n'); 73 | const processedLines = []; 74 | let inList = false; 75 | let listType = ''; 76 | let inParagraph = false; 77 | 78 | for (let i = 0; i < lines.length; i++) { 79 | const line = lines[i].trim(); 80 | const nextLine = i < lines.length - 1 ? lines[i + 1].trim() : ''; 81 | 82 | // Headers 83 | if (line.match(/^#{1,3}\s/)) { 84 | if (inParagraph) { 85 | processedLines.push('

'); 86 | inParagraph = false; 87 | } 88 | if (line.match(/^###\s(.+)$/)) { 89 | processedLines.push(line.replace(/^###\s(.+)$/, '

$1

')); 90 | } else if (line.match(/^##\s(.+)$/)) { 91 | processedLines.push(line.replace(/^##\s(.+)$/, '

$1

')); 92 | } else if (line.match(/^#\s(.+)$/)) { 93 | processedLines.push(line.replace(/^#\s(.+)$/, '

$1

')); 94 | } 95 | continue; 96 | } 97 | 98 | // Lists 99 | const bulletMatch = line.match(/^[-*]\s(.+)$/); 100 | const numberedMatch = line.match(/^(\d+)\.\s(.+)$/); 101 | 102 | if (bulletMatch || numberedMatch) { 103 | if (inParagraph) { 104 | processedLines.push('

'); 105 | inParagraph = false; 106 | } 107 | 108 | const newListType = bulletMatch ? 'ul' : 'ol'; 109 | if (!inList) { 110 | listType = newListType; 111 | processedLines.push(`<${listType} class="${listType === 'ul' ? 'list-disc' : 'list-decimal'} ml-6 my-3 space-y-1">`); 112 | inList = true; 113 | } else if (listType !== newListType) { 114 | processedLines.push(``); 115 | listType = newListType; 116 | processedLines.push(`<${listType} class="${listType === 'ul' ? 'list-disc' : 'list-decimal'} ml-6 my-3 space-y-1">`); 117 | } 118 | 119 | const content = bulletMatch ? bulletMatch[1] : numberedMatch![2]; 120 | processedLines.push(`
  • ${content}
    • `); 121 | continue; 122 | } else if (inList && line === '') { 123 | processedLines.push(``); 124 | inList = false; 125 | continue; 126 | } 127 | 128 | // Empty lines 129 | if (line === '') { 130 | if (inParagraph) { 131 | processedLines.push('

    '); 132 | inParagraph = false; 133 | } 134 | if (inList) { 135 | processedLines.push(``); 136 | inList = false; 137 | } 138 | continue; 139 | } 140 | 141 | // Regular text - start new paragraph if needed 142 | if (!inParagraph && !inList && !line.startsWith('<')) { 143 | processedLines.push('

    '); 144 | inParagraph = true; 145 | } 146 | 147 | // Add line with space if in paragraph 148 | if (inParagraph) { 149 | processedLines.push(line + (nextLine && !nextLine.match(/^[-*#]|\d+\./) ? ' ' : '')); 150 | } else { 151 | processedLines.push(line); 152 | } 153 | } 154 | 155 | // Close any open tags 156 | if (inParagraph) { 157 | processedLines.push('

    '); 158 | } 159 | if (inList) { 160 | processedLines.push(``); 161 | } 162 | 163 | parsed = processedLines.join('\n'); 164 | 165 | // Restore code blocks 166 | codeBlocks.forEach((block, index) => { 167 | parsed = parsed.replace(`__CODE_BLOCK_${index}__`, block); 168 | }); 169 | 170 | return parsed; 171 | }; 172 | 173 | useEffect(() => { 174 | // Add click handlers for citations 175 | const citations = document.querySelectorAll('.citation'); 176 | citations.forEach(citation => { 177 | citation.addEventListener('click', (e) => { 178 | const citationNum = parseInt((e.target as HTMLElement).getAttribute('data-citation') || '0'); 179 | if (onSourceClick && citationNum > 0) { 180 | onSourceClick(citationNum - 1); 181 | } 182 | }); 183 | }); 184 | 185 | return () => { 186 | citations.forEach(citation => { 187 | citation.removeEventListener('click', () => {}); 188 | }); 189 | }; 190 | }, [content, onSourceClick]); 191 | 192 | return ( 193 |
    194 |
    198 | {isStreaming && ( 199 | 200 | )} 201 |
    202 | ); 203 | } 204 | 205 | function DashboardContent() { 206 | const router = useRouter() 207 | const searchParams = useSearchParams() 208 | const [siteData, setSiteData] = useState(null) 209 | const [showDeleteModal, setShowDeleteModal] = useState(false) 210 | const [copiedItem, setCopiedItem] = useState(null) 211 | const [messages, setMessages] = useState>([]) 212 | const [input, setInput] = useState('') 213 | const [isLoading, setIsLoading] = useState(false) 214 | const [showApiModal, setShowApiModal] = useState(false) 215 | const [activeTab, setActiveTab] = useState<'curl' | 'javascript' | 'python' | 'openai-js' | 'openai-python'>('curl') 216 | const scrollAreaRef = useRef(null) 217 | const [autoScroll, setAutoScroll] = useState(true) 218 | 219 | useEffect(() => { 220 | const el = scrollAreaRef.current 221 | if (!el) return 222 | const handleScroll = () => { 223 | const { scrollTop, scrollHeight, clientHeight } = el 224 | const atBottom = scrollHeight - scrollTop - clientHeight < 20 225 | setAutoScroll(atBottom) 226 | } 227 | el.addEventListener('scroll', handleScroll) 228 | return () => { 229 | el.removeEventListener('scroll', handleScroll) 230 | } 231 | }, []) 232 | 233 | // Handle form submission 234 | const handleSubmit = async (e: React.FormEvent) => { 235 | e.preventDefault() 236 | if (!input.trim() || !siteData) return 237 | 238 | let processedInput = input.trim() 239 | 240 | // Check if the input looks like a URL without protocol 241 | const urlPattern = /^(?!https?:\/\/)([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}(\/.*)?$/ 242 | if (urlPattern.test(processedInput)) { 243 | processedInput = 'https://' + processedInput 244 | } 245 | 246 | const userMessage = { role: 'user' as const, content: processedInput } 247 | setMessages(prev => [...prev, userMessage]) 248 | setInput('') 249 | setIsLoading(true) 250 | 251 | try { 252 | const response = await fetch('/api/firestarter/query', { 253 | method: 'POST', 254 | headers: { 'Content-Type': 'application/json' }, 255 | body: JSON.stringify({ 256 | messages: [userMessage], 257 | namespace: siteData.namespace, 258 | stream: true 259 | }) 260 | }) 261 | 262 | if (!response.ok) { 263 | throw new Error('Failed to get response') 264 | } 265 | 266 | const reader = response.body?.getReader() 267 | const decoder = new TextDecoder() 268 | let sources: Source[] = [] 269 | let content = '' 270 | let hasStartedStreaming = false 271 | 272 | if (!reader) throw new Error('No response body') 273 | 274 | while (true) { 275 | const { done, value } = await reader.read() 276 | if (done) break 277 | 278 | const chunk = decoder.decode(value) 279 | const lines = chunk.split('\n') 280 | 281 | for (const line of lines) { 282 | if (line.trim() === '') continue 283 | 284 | 285 | // Handle Vercel AI SDK streaming format 286 | if (line.startsWith('0:')) { 287 | // Text content chunk 288 | const textContent = line.slice(2) 289 | if (textContent.startsWith('"') && textContent.endsWith('"')) { 290 | const text = JSON.parse(textContent) 291 | content += text 292 | 293 | // Add assistant message on first content 294 | if (!hasStartedStreaming) { 295 | hasStartedStreaming = true 296 | setMessages(prev => [...prev, { 297 | role: 'assistant' as const, 298 | content: content, 299 | sources: sources 300 | }]) 301 | } else { 302 | // Update the last message with new content 303 | setMessages(prev => { 304 | const newMessages = [...prev] 305 | const lastMessage = newMessages[newMessages.length - 1] 306 | if (lastMessage && lastMessage.role === 'assistant') { 307 | lastMessage.content = content 308 | lastMessage.sources = sources 309 | } 310 | return newMessages 311 | }) 312 | } 313 | scrollToBottom() 314 | } 315 | } else if (line.startsWith('8:')) { 316 | // Streaming data chunk (sources, etc) 317 | try { 318 | const jsonStr = line.slice(2) 319 | const data = JSON.parse(jsonStr) 320 | 321 | // Check if this is the sources data 322 | if (data && typeof data === 'object' && 'sources' in data) { 323 | sources = data.sources 324 | 325 | // Update the last message with sources 326 | setMessages(prev => { 327 | const newMessages = [...prev] 328 | const lastMessage = newMessages[newMessages.length - 1] 329 | if (lastMessage && lastMessage.role === 'assistant') { 330 | lastMessage.sources = sources 331 | } 332 | return newMessages 333 | }) 334 | } else if (Array.isArray(data)) { 335 | // Legacy format support 336 | const sourcesData = data.find(item => item && typeof item === 'object' && 'type' in item && item.type === 'sources') 337 | if (sourcesData && sourcesData.sources) { 338 | sources = sourcesData.sources 339 | } 340 | } 341 | } catch { 342 | console.error('Failed to parse streaming data') 343 | } 344 | } else if (line.startsWith('e:') || line.startsWith('d:')) { 345 | // End metadata - we can ignore these 346 | } 347 | } 348 | } 349 | } catch { 350 | toast.error('Failed to get response') 351 | console.error('Query failed') 352 | } finally { 353 | setIsLoading(false) 354 | } 355 | } 356 | 357 | useEffect(() => { 358 | // Get namespace from URL params 359 | const namespaceParam = searchParams.get('namespace') 360 | 361 | if (namespaceParam) { 362 | // Try to load data for this specific namespace 363 | const storedIndexes = localStorage.getItem('firestarter_indexes') 364 | if (storedIndexes) { 365 | const indexes = JSON.parse(storedIndexes) 366 | const matchingIndex = indexes.find((idx: { namespace: string }) => idx.namespace === namespaceParam) 367 | if (matchingIndex) { 368 | setSiteData(matchingIndex) 369 | // Also update sessionStorage for consistency 370 | sessionStorage.setItem('firestarter_current_data', JSON.stringify(matchingIndex)) 371 | // Clear messages when namespace changes 372 | setMessages([]) 373 | } else { 374 | // Namespace not found in stored indexes 375 | router.push('/indexes') 376 | } 377 | } else { 378 | router.push('/indexes') 379 | } 380 | } else { 381 | // Fallback to sessionStorage if no namespace param 382 | const data = sessionStorage.getItem('firestarter_current_data') 383 | if (data) { 384 | const parsedData = JSON.parse(data) 385 | setSiteData(parsedData) 386 | // Add namespace to URL for consistency 387 | router.replace(`/dashboard?namespace=${parsedData.namespace}`) 388 | } else { 389 | router.push('/indexes') 390 | } 391 | } 392 | }, [router, searchParams]) 393 | 394 | const scrollToBottom = () => { 395 | if (scrollAreaRef.current && autoScroll) { 396 | scrollAreaRef.current.scrollTo({ 397 | top: scrollAreaRef.current.scrollHeight, 398 | behavior: 'smooth' 399 | }) 400 | } 401 | } 402 | 403 | const handleDelete = () => { 404 | // Remove from localStorage 405 | const storedIndexes = localStorage.getItem('firestarter_indexes') 406 | if (storedIndexes && siteData) { 407 | const indexes = JSON.parse(storedIndexes) 408 | const updatedIndexes = indexes.filter((idx: { namespace: string }) => idx.namespace !== siteData.namespace) 409 | localStorage.setItem('firestarter_indexes', JSON.stringify(updatedIndexes)) 410 | } 411 | 412 | sessionStorage.removeItem('firestarter_current_data') 413 | router.push('/indexes') 414 | } 415 | 416 | const copyToClipboard = (text: string, itemId: string) => { 417 | navigator.clipboard.writeText(text) 418 | setCopiedItem(itemId) 419 | setTimeout(() => setCopiedItem(null), 2000) 420 | } 421 | 422 | 423 | if (!siteData) { 424 | return ( 425 |
    426 |
    Loading...
    427 |
    428 | ) 429 | } 430 | 431 | 432 | const modelName = `firecrawl-${siteData.namespace}` 433 | 434 | // Get dynamic API URL based on current location 435 | const getApiUrl = () => { 436 | if (typeof window === 'undefined') return 'http://localhost:3001/api/v1/chat/completions' 437 | const protocol = window.location.protocol 438 | const host = window.location.host 439 | return `${protocol}//${host}/api/v1/chat/completions` 440 | } 441 | const apiUrl = getApiUrl() 442 | 443 | const curlCommand = `# Standard request 444 | curl ${apiUrl} \\ 445 | -H "Content-Type: application/json" \\ 446 | -H "Authorization: Bearer YOUR_FIRESTARTER_API_KEY" \\ 447 | -d '{ 448 | "model": "${modelName}", 449 | "messages": [ 450 | {"role": "user", "content": "Your question here"} 451 | ] 452 | }' 453 | 454 | # Streaming request (SSE format) 455 | curl ${apiUrl} \\ 456 | -H "Content-Type: application/json" \\ 457 | -H "Authorization: Bearer YOUR_FIRESTARTER_API_KEY" \\ 458 | -H "Accept: text/event-stream" \\ 459 | -N \\ 460 | -d '{ 461 | "model": "${modelName}", 462 | "messages": [ 463 | {"role": "user", "content": "Your question here"} 464 | ], 465 | "stream": true 466 | }'` 467 | 468 | const openaiJsCode = `import OpenAI from 'openai'; 469 | 470 | const openai = new OpenAI({ 471 | apiKey: 'YOUR_FIRESTARTER_API_KEY', 472 | baseURL: '${apiUrl.replace('/chat/completions', '')}', 473 | }); 474 | 475 | const completion = await openai.chat.completions.create({ 476 | model: '${modelName}', 477 | messages: [ 478 | { role: 'user', content: 'Your question here' } 479 | ], 480 | }); 481 | 482 | console.log(completion.choices[0].message.content); 483 | 484 | // Streaming example 485 | const stream = await openai.chat.completions.create({ 486 | model: '${modelName}', 487 | messages: [ 488 | { role: 'user', content: 'Your question here' } 489 | ], 490 | stream: true, 491 | }); 492 | 493 | for await (const chunk of stream) { 494 | process.stdout.write(chunk.choices[0]?.delta?.content || ''); 495 | }` 496 | 497 | const openaiPythonCode = `from openai import OpenAI 498 | 499 | client = OpenAI( 500 | api_key="YOUR_FIRESTARTER_API_KEY", 501 | base_url="${apiUrl.replace('/chat/completions', '')}" 502 | ) 503 | 504 | completion = client.chat.completions.create( 505 | model="${modelName}", 506 | messages=[ 507 | {"role": "user", "content": "Your question here"} 508 | ] 509 | ) 510 | 511 | print(completion.choices[0].message.content) 512 | 513 | # Streaming example 514 | stream = client.chat.completions.create( 515 | model="${modelName}", 516 | messages=[ 517 | {"role": "user", "content": "Your question here"} 518 | ], 519 | stream=True 520 | ) 521 | 522 | for chunk in stream: 523 | if chunk.choices[0].delta.content is not None: 524 | print(chunk.choices[0].delta.content, end="")` 525 | 526 | const jsCode = `// Using fetch API 527 | const response = await fetch('${apiUrl}', { 528 | method: 'POST', 529 | headers: { 530 | 'Content-Type': 'application/json', 531 | 'Authorization': 'Bearer YOUR_FIRESTARTER_API_KEY' 532 | }, 533 | body: JSON.stringify({ 534 | model: '${modelName}', 535 | messages: [ 536 | { role: 'user', content: 'Your question here' } 537 | ] 538 | }) 539 | }); 540 | 541 | const data = await response.json(); 542 | console.log(data.choices[0].message.content);` 543 | 544 | const pythonCode = `import requests 545 | 546 | response = requests.post( 547 | '${apiUrl}', 548 | headers={ 549 | 'Content-Type': 'application/json', 550 | 'Authorization': 'Bearer YOUR_FIRESTARTER_API_KEY' 551 | }, 552 | json={ 553 | 'model': '${modelName}', 554 | 'messages': [ 555 | {'role': 'user', 'content': 'Your question here'} 556 | ] 557 | } 558 | ) 559 | 560 | data = response.json() 561 | print(data['choices'][0]['message']['content'])` 562 | 563 | 564 | return ( 565 |
    566 | {/* Header */} 567 |
    568 |
    569 |
    570 |
    571 | 577 |
    578 | {siteData.metadata.favicon ? ( 579 | {siteData.metadata.title} { 586 | e.currentTarget.style.display = 'none'; 587 | e.currentTarget.parentElement?.querySelector('.fallback-icon')?.classList.remove('hidden'); 588 | }} 589 | /> 590 | ) : ( 591 |
    592 | 593 |
    594 | )} 595 |
    596 |

    597 | {siteData.metadata.title.length > 50 598 | ? siteData.metadata.title.substring(0, 47) + '...' 599 | : siteData.metadata.title} 600 |

    601 |

    {siteData.url}

    602 |
    603 |
    604 |
    605 | 606 | 613 |
    614 |
    615 |
    616 | 617 |
    618 |
    619 | {/* Stats Cards - Show at top on mobile */} 620 |
    621 |
    622 | {/* OG Image Background */} 623 | {siteData.metadata.ogImage && ( 624 |
    625 | { 631 | e.currentTarget.parentElement!.style.display = 'none'; 632 | }} 633 | /> 634 |
    635 |
    636 | )} 637 | 638 |
    639 |
    640 |

    641 | {siteData.metadata.title.length > 30 642 | ? siteData.metadata.title.substring(0, 27) + '...' 643 | : siteData.metadata.title} 644 |

    645 |

    Knowledge Base

    646 |
    647 | 648 |
    649 |
    650 |
    651 | 652 | Pages 653 |
    654 | {siteData.pagesCrawled} 655 |
    656 | 657 |
    658 |
    659 | 660 | Chunks 661 |
    662 | {Math.round(siteData.pagesCrawled * 3)} 663 |
    664 | 665 |
    666 |
    667 | 668 | Namespace 669 |
    670 | {siteData.namespace.split('-').slice(0, -1).join('.')} 671 |
    672 |
    673 |
    674 |
    675 | 676 |
    677 |

    Quick Start

    678 |
    679 |
    680 |

    1. Test in Dashboard

    681 |

    Use the chat panel to test responses and refine your queries

    682 |
    683 |
    684 |

    2. Get API Access

    685 |

    Click below to see integration code in multiple languages

    686 |
    687 |
    688 |

    3. Deploy Anywhere

    689 |

    Deploy chatbot script OR OpenAI-compatible endpoint API

    690 |
    691 |
    692 |
    693 | 700 |
    701 |
    702 |
    703 | 704 | {/* Chat Panel and Sources - Show below on mobile */} 705 |
    706 |
    707 | {/* Chat Panel */} 708 |
    709 |
    710 | {messages.length === 0 && ( 711 |
    712 |
    713 | {siteData.metadata.favicon && ( 714 | {siteData.metadata.title} { 721 | e.currentTarget.style.display = 'none'; 722 | }} 723 | /> 724 | )} 725 |
    726 |

    727 | Chat with {siteData.metadata.title} 728 |

    729 |

    730 | Ask anything about their {siteData.pagesCrawled} indexed pages 731 |

    732 |
    733 | )} 734 | 735 | {messages.map((message, index) => ( 736 |
    740 |
    741 |
    748 | {message.role === 'user' ? ( 749 |

    {message.content}

    750 | ) : ( 751 |
    752 | 756 |
    757 | )} 758 |
    759 | 760 |
    761 |
    762 | ))} 763 | 764 | {isLoading && messages[messages.length - 1]?.role !== 'assistant' && ( 765 |
    766 |
    767 |
    768 |
    769 |
    770 |
    771 |
    772 |
    773 |
    774 |
    775 |
    776 | )} 777 |
    778 | 779 | 780 |
    781 |
    782 | setInput(e.target.value)} 786 | placeholder={`Ask about ${siteData.metadata.title}...`} 787 | className="w-full pr-12 placeholder:text-gray-400" 788 | disabled={isLoading} 789 | /> 790 | 797 |
    798 |
    799 |
    800 | 801 | {/* Sources Panel - Shows on right side when available */} 802 |
    803 |
    804 | {(() => { 805 | const lastAssistantMessage = messages.filter(m => m.role === 'assistant').pop() 806 | const hasSources = lastAssistantMessage?.sources && lastAssistantMessage.sources.length > 0 807 | 808 | if (hasSources) { 809 | return ( 810 | <> 811 |
    812 |

    813 | 814 | Sources 815 |

    816 | 817 | {lastAssistantMessage.sources?.length || 0} references 818 | 819 |
    820 | 821 | 857 | 858 | ) 859 | } 860 | 861 | // Default knowledge base view when no sources 862 | return ( 863 | <> 864 |
    865 |

    866 | 867 | Knowledge Base 868 |

    869 |
    870 | 871 |
    872 |
    873 |
    874 | 875 |
    876 |
    877 |
    878 |
    879 |

    880 | {siteData.pagesCrawled} pages indexed 881 |

    882 |

    883 | Ready to answer questions about {siteData.metadata.title} 884 |

    885 |
    886 |
    887 | Total chunks 888 | 889 | {Math.round(siteData.pagesCrawled * 3)} 890 | 891 |
    892 |
    893 | Crawl date 894 | 895 | {(() => { 896 | const dateString = siteData.crawlDate || siteData.createdAt; 897 | return dateString ? new Date(dateString).toLocaleDateString() : 'N/A'; 898 | })()} 899 | 900 |
    901 |
    902 | Namespace 903 | 904 | {siteData.namespace.split('-').slice(0, -1).join('.')} 905 | 906 |
    907 |
    908 |
    909 |
    910 | 911 | ) 912 | })()} 913 |
    914 |
    915 |
    916 |
    917 |
    918 |
    919 | 920 | {/* Delete Confirmation Modal */} 921 | 922 | 923 | 924 | Delete Index 925 | 926 | Are you sure you want to delete the index for {siteData.metadata.title}? This action cannot be undone. 927 | 928 | 929 | 930 | 937 | 944 | 945 | 946 | 947 | 948 | {/* API Modal */} 949 | 950 | 951 | 952 | API Access 953 | 954 | Use this index with any OpenAI-compatible API client. 955 | 956 | 957 | 958 |
    959 |
    960 | Model Name: 961 | {modelName} 962 |
    963 |
    964 | Endpoint: 965 | /api/v1/chat/completions 966 |
    967 |
    968 | 969 | {/* Language tabs */} 970 |
    971 |
    972 | 982 | 992 | 1002 | 1012 | 1022 |
    1023 | 1024 | {/* Tab content */} 1025 |
    1026 |
    1027 | 1028 | {activeTab === 'curl' && 'cURL Command'} 1029 | {activeTab === 'javascript' && 'JavaScript (Fetch API)'} 1030 | {activeTab === 'python' && 'Python (Requests)'} 1031 | {activeTab === 'openai-js' && 'OpenAI SDK for JavaScript'} 1032 | {activeTab === 'openai-python' && 'OpenAI SDK for Python'} 1033 | 1034 | 1048 |
    1049 | 
    1050 |                 
1051 |                   {activeTab === 'curl' && curlCommand}
1052 |                   {activeTab === 'javascript' && jsCode}
1053 |                   {activeTab === 'python' && pythonCode}
1054 |                   {activeTab === 'openai-js' && openaiJsCode}
1055 |                   {activeTab === 'openai-python' && openaiPythonCode}
1056 |                 
1057 |
    1058 |
    1059 |
    1060 |     1061 |     1062 |
    1063 | ) 1064 | } 1065 | 1066 | export default function DashboardPage() { 1067 | return ( 1068 | 1070 |
    1071 |
    1072 |
    1073 |
    1074 | 1075 |
    1076 |

    Loading dashboard...

    1077 |
    1078 |
    1079 |
    1080 |
    1081 | }> 1082 | 1083 | 1084 | ) 1085 | } --------------------------------------------------------------------------------