├── .eslintrc.json ├── public ├── favicon.png └── screenshot.png ├── styles └── globals.css ├── postcss.config.js ├── next.config.js ├── pages ├── _app.tsx ├── _document.tsx ├── api │ ├── answer.ts │ └── sources.ts └── index.tsx ├── tailwind.config.js ├── types └── index.ts ├── utils ├── sources.ts └── answer.ts ├── .gitignore ├── tsconfig.json ├── package.json ├── license ├── README.md └── components ├── Answer.tsx └── Search.tsx /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /public/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code/app-clarity-ai/main/public/favicon.png -------------------------------------------------------------------------------- /styles/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | -------------------------------------------------------------------------------- /public/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code/app-clarity-ai/main/public/screenshot.png -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = { 3 | reactStrictMode: true, 4 | } 5 | 6 | module.exports = nextConfig 7 | -------------------------------------------------------------------------------- /pages/_app.tsx: -------------------------------------------------------------------------------- 1 | import '@/styles/globals.css' 2 | import type { AppProps } from 'next/app' 3 | 4 | export default function App({ Component, pageProps }: AppProps) { 5 | return 6 | } 7 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: ["./app/**/*.{js,ts,jsx,tsx}", "./pages/**/*.{js,ts,jsx,tsx}", "./components/**/*.{js,ts,jsx,tsx}"], 4 | theme: { 5 | extend: {} 6 | }, 7 | plugins: [] 8 | }; 9 | -------------------------------------------------------------------------------- /types/index.ts: -------------------------------------------------------------------------------- 1 | export enum OpenAIModel { 2 | DAVINCI_TURBO = "gpt-3.5-turbo" 3 | } 4 | 5 | export type Source = { 6 | url: string; 7 | text: string; 8 | }; 9 | 10 | export type SearchQuery = { 11 | query: string; 12 | sourceLinks: string[]; 13 | }; 14 | -------------------------------------------------------------------------------- /utils/sources.ts: -------------------------------------------------------------------------------- 1 | export const cleanSourceText = (text: string) => { 2 | return text 3 | .trim() 4 | .replace(/(\n){4,}/g, "\n\n\n") 5 | .replace(/\n\n/g, " ") 6 | .replace(/ {3,}/g, " ") 7 | .replace(/\t/g, "") 8 | .replace(/\n+(\s*\n)*/g, "\n"); 9 | }; 10 | -------------------------------------------------------------------------------- /pages/_document.tsx: -------------------------------------------------------------------------------- 1 | import { Html, Head, Main, NextScript } from 'next/document' 2 | 3 | export default function Document() { 4 | return ( 5 | 6 | 7 | 8 |
9 | 10 | 11 | 12 | ) 13 | } 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | .pnpm-debug.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts 37 | -------------------------------------------------------------------------------- /pages/api/answer.ts: -------------------------------------------------------------------------------- 1 | import { OpenAIStream } from "@/utils/answer"; 2 | 3 | export const config = { 4 | runtime: "edge" 5 | }; 6 | 7 | const handler = async (req: Request): Promise => { 8 | try { 9 | const { prompt, apiKey } = (await req.json()) as { 10 | prompt: string; 11 | apiKey: string; 12 | }; 13 | 14 | const stream = await OpenAIStream(prompt, apiKey); 15 | 16 | return new Response(stream); 17 | } catch (error) { 18 | console.error(error); 19 | return new Response("Error", { status: 500 }); 20 | } 21 | }; 22 | 23 | export default handler; 24 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "noEmit": true, 10 | "esModuleInterop": true, 11 | "module": "esnext", 12 | "moduleResolution": "node", 13 | "resolveJsonModule": true, 14 | "isolatedModules": true, 15 | "jsx": "preserve", 16 | "incremental": true, 17 | "baseUrl": ".", 18 | "paths": { 19 | "@/*": ["./*"] 20 | } 21 | }, 22 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], 23 | "exclude": ["node_modules"] 24 | } 25 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frontend", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@mozilla/readability": "^0.4.2", 13 | "@next/font": "13.1.6", 14 | "@tabler/icons": "^2.4.0", 15 | "@tabler/icons-react": "^2.4.0", 16 | "@types/node": "18.14.0", 17 | "@types/react": "18.0.28", 18 | "@types/react-dom": "18.0.11", 19 | "cheerio": "^1.0.0-rc.12", 20 | "endent": "^2.1.0", 21 | "eslint": "8.34.0", 22 | "eslint-config-next": "13.1.6", 23 | "eventsource-parser": "^0.1.0", 24 | "jsdom": "^21.1.0", 25 | "next": "13.1.6", 26 | "openai": "^3.1.0", 27 | "react": "18.2.0", 28 | "react-dom": "18.2.0", 29 | "typescript": "4.9.5" 30 | }, 31 | "devDependencies": { 32 | "@types/jsdom": "^21.1.0", 33 | "autoprefixer": "^10.4.13", 34 | "postcss": "^8.4.21", 35 | "tailwindcss": "^3.2.7" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Mckay Wrigley 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Clarity AI 2 | 3 | Clarity is simple [perplexity.ai](https://www.perplexity.ai/) clone. Use the code for whatever you like! :) 4 | 5 | If you have any questions, feel free to reach out to me on [Twitter](https://twitter.com/mckaywrigley). 6 | 7 | [![Clarity AI](./public/screenshot.png)](https://clarity-ai.vercel.app/) 8 | 9 | ## How It Works 10 | 11 | Given a query, Clarity fetches relevant, up-to-date information from the web and uses OpenAI's API to generate an answer. 12 | 13 | The app works as follows: 14 | 15 | 1. Get query from user 16 | 2. Scrape Google for relevant webpages 17 | 3. Parse webpages for text 18 | 4. Build prompt using query + webpage text 19 | 5. Call OpenAI API to generate answer 20 | 6. Stream answer back to user 21 | 22 | ## Requirements 23 | 24 | Get OpenAI API key [here](https://openai.com/api/). 25 | 26 | ## Running Locally 27 | 28 | 1. Clone repo 29 | 30 | ```bash 31 | git clone https://github.com/mckaywrigley/clarity-ai.git 32 | ``` 33 | 34 | 2. Install dependencies 35 | 36 | ```bash 37 | npm i 38 | ``` 39 | 40 | 3. Run app 41 | 42 | ```bash 43 | npm run dev 44 | ``` 45 | 46 | ## Improvement Ideas 47 | 48 | Here are some ideas for how to improve Clarity: 49 | 50 | - [ ] Speed up answers by replacing link scraping with the Google Search API (scraping was used to circumvent cost + rate limits) 51 | - [ ] Add "follow up" searches 52 | - [ ] Improve the prompt 53 | - [ ] Get sources working in non text-davinci-003 models 54 | - [ ] Train your own model to use for answer synthesis 55 | 56 | ## Credits 57 | 58 | Shoutout to [Perplexity AI](https://www.perplexity.ai/) for the inspiration. I highly recommend checking their product out. 59 | 60 | This repo is meant to show people that you can build powerful apps like Perplexity even if you don't have a large, experienced team. 61 | 62 | LLMs are amazing, and I hope Clarity inspires you to build something cool! 63 | -------------------------------------------------------------------------------- /utils/answer.ts: -------------------------------------------------------------------------------- 1 | import { OpenAIModel } from "@/types"; 2 | import { createParser, ParsedEvent, ReconnectInterval } from "eventsource-parser"; 3 | 4 | export const OpenAIStream = async (prompt: string, apiKey: string) => { 5 | const encoder = new TextEncoder(); 6 | const decoder = new TextDecoder(); 7 | 8 | const res = await fetch("https://api.openai.com/v1/chat/completions", { 9 | headers: { 10 | "Content-Type": "application/json", 11 | Authorization: `Bearer ${apiKey}` 12 | }, 13 | method: "POST", 14 | body: JSON.stringify({ 15 | model: OpenAIModel.DAVINCI_TURBO, 16 | messages: [ 17 | { role: "system", content: "You are a helpful assistant that accurately answers the user's queries based on the given text." }, 18 | { role: "user", content: prompt } 19 | ], 20 | max_tokens: 120, 21 | temperature: 0.0, 22 | stream: true 23 | }) 24 | }); 25 | 26 | if (res.status !== 200) { 27 | throw new Error("OpenAI API returned an error"); 28 | } 29 | 30 | const stream = new ReadableStream({ 31 | async start(controller) { 32 | const onParse = (event: ParsedEvent | ReconnectInterval) => { 33 | if (event.type === "event") { 34 | const data = event.data; 35 | 36 | if (data === "[DONE]") { 37 | controller.close(); 38 | return; 39 | } 40 | 41 | try { 42 | const json = JSON.parse(data); 43 | const text = json.choices[0].delta.content; 44 | const queue = encoder.encode(text); 45 | controller.enqueue(queue); 46 | } catch (e) { 47 | controller.error(e); 48 | } 49 | } 50 | }; 51 | 52 | const parser = createParser(onParse); 53 | 54 | for await (const chunk of res.body as any) { 55 | parser.feed(decoder.decode(chunk)); 56 | } 57 | } 58 | }); 59 | 60 | return stream; 61 | }; 62 | -------------------------------------------------------------------------------- /pages/index.tsx: -------------------------------------------------------------------------------- 1 | import { Answer } from "@/components/Answer"; 2 | import { Search } from "@/components/Search"; 3 | import { SearchQuery } from "@/types"; 4 | import { IconBrandGithub, IconBrandTwitter } from "@tabler/icons-react"; 5 | import Head from "next/head"; 6 | import { useState } from "react"; 7 | 8 | export default function Home() { 9 | const [searchQuery, setSearchQuery] = useState({ query: "", sourceLinks: [] }); 10 | const [answer, setAnswer] = useState(""); 11 | const [done, setDone] = useState(false); 12 | 13 | return ( 14 | <> 15 | 16 | Clarity AI 17 | 21 | 25 | 29 | 30 |
31 | 37 | 38 | 39 | 40 | 46 | 47 | 48 | 49 | {answer ? ( 50 | { 55 | setAnswer(""); 56 | setSearchQuery({ query: "", sourceLinks: [] }); 57 | setDone(false); 58 | }} 59 | /> 60 | ) : ( 61 | setAnswer((prev) => prev + value)} 64 | onDone={setDone} 65 | /> 66 | )} 67 |
68 | 69 | ); 70 | } 71 | -------------------------------------------------------------------------------- /pages/api/sources.ts: -------------------------------------------------------------------------------- 1 | import { OpenAIModel, Source } from "@/types"; 2 | import { Readability } from "@mozilla/readability"; 3 | import * as cheerio from "cheerio"; 4 | import { JSDOM } from "jsdom"; 5 | import type { NextApiRequest, NextApiResponse } from "next"; 6 | import { cleanSourceText } from "../../utils/sources"; 7 | 8 | type Data = { 9 | sources: Source[]; 10 | }; 11 | 12 | const searchHandler = async (req: NextApiRequest, res: NextApiResponse) => { 13 | try { 14 | const { query, model } = req.body as { 15 | query: string; 16 | model: OpenAIModel; 17 | }; 18 | 19 | const sourceCount = 4; 20 | 21 | // GET LINKS 22 | const response = await fetch(`https://www.google.com/search?q=${query}`); 23 | const html = await response.text(); 24 | const $ = cheerio.load(html); 25 | const linkTags = $("a"); 26 | 27 | let links: string[] = []; 28 | 29 | linkTags.each((i, link) => { 30 | const href = $(link).attr("href"); 31 | 32 | if (href && href.startsWith("/url?q=")) { 33 | const cleanedHref = href.replace("/url?q=", "").split("&")[0]; 34 | 35 | if (!links.includes(cleanedHref)) { 36 | links.push(cleanedHref); 37 | } 38 | } 39 | }); 40 | 41 | const filteredLinks = links.filter((link, idx) => { 42 | const domain = new URL(link).hostname; 43 | 44 | const excludeList = ["google", "facebook", "twitter", "instagram", "youtube", "tiktok"]; 45 | if (excludeList.some((site) => domain.includes(site))) return false; 46 | 47 | return links.findIndex((link) => new URL(link).hostname === domain) === idx; 48 | }); 49 | 50 | const finalLinks = filteredLinks.slice(0, sourceCount); 51 | 52 | // SCRAPE TEXT FROM LINKS 53 | const sources = (await Promise.all( 54 | finalLinks.map(async (link) => { 55 | const response = await fetch(link); 56 | const html = await response.text(); 57 | const dom = new JSDOM(html); 58 | const doc = dom.window.document; 59 | const parsed = new Readability(doc).parse(); 60 | 61 | if (parsed) { 62 | let sourceText = cleanSourceText(parsed.textContent); 63 | 64 | return { url: link, text: sourceText }; 65 | } 66 | }) 67 | )) as Source[]; 68 | 69 | const filteredSources = sources.filter((source) => source !== undefined); 70 | 71 | for (const source of filteredSources) { 72 | source.text = source.text.slice(0, 1500); 73 | } 74 | 75 | res.status(200).json({ sources: filteredSources }); 76 | } catch (err) { 77 | console.log(err); 78 | res.status(500).json({ sources: [] }); 79 | } 80 | }; 81 | 82 | export default searchHandler; 83 | -------------------------------------------------------------------------------- /components/Answer.tsx: -------------------------------------------------------------------------------- 1 | import { SearchQuery } from "@/types"; 2 | import { IconReload } from "@tabler/icons-react"; 3 | import { FC } from "react"; 4 | 5 | interface AnswerProps { 6 | searchQuery: SearchQuery; 7 | answer: string; 8 | done: boolean; 9 | onReset: () => void; 10 | } 11 | 12 | export const Answer: FC = ({ searchQuery, answer, done, onReset }) => { 13 | return ( 14 |
15 |
{searchQuery.query}
16 | 17 |
18 |
Answer
19 | 20 |
{replaceSourcesWithLinks(answer, searchQuery.sourceLinks)}
21 |
22 | 23 | {done && ( 24 | <> 25 |
26 |
Sources
27 | 28 | {searchQuery.sourceLinks.map((source, index) => ( 29 | 43 | ))} 44 |
45 | 46 | 53 | 54 | )} 55 |
56 | ); 57 | }; 58 | 59 | const replaceSourcesWithLinks = (answer: string, sourceLinks: string[]) => { 60 | const elements = answer.split(/(\[[0-9]+\])/).map((part, index) => { 61 | if (/\[[0-9]+\]/.test(part)) { 62 | const link = sourceLinks[parseInt(part.replace(/[\[\]]/g, "")) - 1]; 63 | 64 | return ( 65 | 72 | {part} 73 | 74 | ); 75 | } else { 76 | return part; 77 | } 78 | }); 79 | 80 | return elements; 81 | }; 82 | -------------------------------------------------------------------------------- /components/Search.tsx: -------------------------------------------------------------------------------- 1 | import { SearchQuery, Source } from "@/types"; 2 | import { IconArrowRight, IconBolt, IconSearch } from "@tabler/icons-react"; 3 | import endent from "endent"; 4 | import { FC, KeyboardEvent, useEffect, useRef, useState } from "react"; 5 | 6 | interface SearchProps { 7 | onSearch: (searchResult: SearchQuery) => void; 8 | onAnswerUpdate: (answer: string) => void; 9 | onDone: (done: boolean) => void; 10 | } 11 | 12 | export const Search: FC = ({ onSearch, onAnswerUpdate, onDone }) => { 13 | const inputRef = useRef(null); 14 | 15 | const [query, setQuery] = useState(""); 16 | const [apiKey, setApiKey] = useState(""); 17 | const [showSettings, setShowSettings] = useState(false); 18 | const [loading, setLoading] = useState(false); 19 | 20 | const handleSearch = async () => { 21 | if (!query) { 22 | alert("Please enter a query"); 23 | return; 24 | } 25 | 26 | setLoading(true); 27 | const sources = await fetchSources(); 28 | await handleStream(sources); 29 | }; 30 | 31 | const fetchSources = async () => { 32 | const response = await fetch("/api/sources", { 33 | method: "POST", 34 | headers: { 35 | "Content-Type": "application/json" 36 | }, 37 | body: JSON.stringify({ query }) 38 | }); 39 | 40 | if (!response.ok) { 41 | setLoading(false); 42 | throw new Error(response.statusText); 43 | } 44 | 45 | const { sources }: { sources: Source[] } = await response.json(); 46 | 47 | return sources; 48 | }; 49 | 50 | const handleStream = async (sources: Source[]) => { 51 | try { 52 | const prompt = endent`Provide a 2-3 sentence answer to the query based on the following sources. Be original, concise, accurate, and helpful. Cite sources as [1] or [2] or [3] after each sentence (not just the very end) to back up your answer (Ex: Correct: [1], Correct: [2][3], Incorrect: [1, 2]). 53 | 54 | ${sources.map((source, idx) => `Source [${idx + 1}]:\n${source.text}`).join("\n\n")} 55 | `; 56 | 57 | const response = await fetch("/api/answer", { 58 | method: "POST", 59 | headers: { 60 | "Content-Type": "application/json" 61 | }, 62 | body: JSON.stringify({ prompt, apiKey }) 63 | }); 64 | 65 | if (!response.ok) { 66 | setLoading(false); 67 | throw new Error(response.statusText); 68 | } 69 | 70 | setLoading(false); 71 | onSearch({ query, sourceLinks: sources.map((source) => source.url) }); 72 | 73 | const data = response.body; 74 | 75 | if (!data) { 76 | return; 77 | } 78 | 79 | const reader = data.getReader(); 80 | const decoder = new TextDecoder(); 81 | let done = false; 82 | 83 | while (!done) { 84 | const { value, done: doneReading } = await reader.read(); 85 | done = doneReading; 86 | const chunkValue = decoder.decode(value); 87 | onAnswerUpdate(chunkValue); 88 | } 89 | 90 | onDone(true); 91 | } catch (err) { 92 | onAnswerUpdate("Error"); 93 | } 94 | }; 95 | 96 | const handleKeyDown = (e: KeyboardEvent) => { 97 | if (e.key === "Enter") { 98 | handleSearch(); 99 | } 100 | }; 101 | 102 | const handleSave = () => { 103 | if (apiKey.length !== 51) { 104 | alert("Please enter a valid API key."); 105 | return; 106 | } 107 | 108 | localStorage.setItem("CLARITY_KEY", apiKey); 109 | 110 | setShowSettings(false); 111 | inputRef.current?.focus(); 112 | }; 113 | 114 | const handleClear = () => { 115 | localStorage.removeItem("CLARITY_KEY"); 116 | 117 | setApiKey(""); 118 | }; 119 | 120 | useEffect(() => { 121 | const CLARITY_KEY = localStorage.getItem("CLARITY_KEY"); 122 | 123 | if (CLARITY_KEY) { 124 | setApiKey(CLARITY_KEY); 125 | } else { 126 | setShowSettings(true); 127 | } 128 | 129 | inputRef.current?.focus(); 130 | }, []); 131 | 132 | return ( 133 | <> 134 | {loading ? ( 135 |
136 |
137 |
Getting answer...
138 |
139 | ) : ( 140 |
141 |
142 | 143 |
Clarity
144 |
145 | 146 | {apiKey.length === 51 ? ( 147 |
148 | 149 | 150 | setQuery(e.target.value)} 157 | onKeyDown={handleKeyDown} 158 | /> 159 | 160 | 166 |
167 | ) : ( 168 |
Please enter your OpenAI API key.
169 | )} 170 | 171 | 177 | 178 | {showSettings && ( 179 | <> 180 | { 185 | setApiKey(e.target.value); 186 | 187 | if (e.target.value.length !== 51) { 188 | setShowSettings(true); 189 | } 190 | }} 191 | /> 192 | 193 |
194 |
198 | Save 199 |
200 | 201 |
205 | Clear 206 |
207 |
208 | 209 | )} 210 |
211 | )} 212 | 213 | ); 214 | }; 215 | --------------------------------------------------------------------------------