├── .eslintrc.json
├── public
├── favicon.png
└── screenshot.png
├── styles
└── globals.css
├── postcss.config.js
├── next.config.js
├── pages
├── _app.tsx
├── _document.tsx
├── api
│ ├── answer.ts
│ └── sources.ts
└── index.tsx
├── tailwind.config.js
├── types
└── index.ts
├── utils
├── sources.ts
└── answer.ts
├── .gitignore
├── tsconfig.json
├── package.json
├── license
├── README.md
└── components
├── Answer.tsx
└── Search.tsx
/.eslintrc.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "next/core-web-vitals"
3 | }
4 |
--------------------------------------------------------------------------------
/public/favicon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/code/app-clarity-ai/main/public/favicon.png
--------------------------------------------------------------------------------
/styles/globals.css:
--------------------------------------------------------------------------------
1 | @tailwind base;
2 | @tailwind components;
3 | @tailwind utilities;
4 |
--------------------------------------------------------------------------------
/public/screenshot.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/code/app-clarity-ai/main/public/screenshot.png
--------------------------------------------------------------------------------
/postcss.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | plugins: {
3 | tailwindcss: {},
4 | autoprefixer: {},
5 | },
6 | }
7 |
--------------------------------------------------------------------------------
/next.config.js:
--------------------------------------------------------------------------------
1 | /** @type {import('next').NextConfig} */
2 | const nextConfig = {
3 | reactStrictMode: true,
4 | }
5 |
6 | module.exports = nextConfig
7 |
--------------------------------------------------------------------------------
/pages/_app.tsx:
--------------------------------------------------------------------------------
1 | import '@/styles/globals.css'
2 | import type { AppProps } from 'next/app'
3 |
4 | export default function App({ Component, pageProps }: AppProps) {
5 | return
6 | }
7 |
--------------------------------------------------------------------------------
/tailwind.config.js:
--------------------------------------------------------------------------------
1 | /** @type {import('tailwindcss').Config} */
2 | module.exports = {
3 | content: ["./app/**/*.{js,ts,jsx,tsx}", "./pages/**/*.{js,ts,jsx,tsx}", "./components/**/*.{js,ts,jsx,tsx}"],
4 | theme: {
5 | extend: {}
6 | },
7 | plugins: []
8 | };
9 |
--------------------------------------------------------------------------------
/types/index.ts:
--------------------------------------------------------------------------------
1 | export enum OpenAIModel {
2 | DAVINCI_TURBO = "gpt-3.5-turbo"
3 | }
4 |
5 | export type Source = {
6 | url: string;
7 | text: string;
8 | };
9 |
10 | export type SearchQuery = {
11 | query: string;
12 | sourceLinks: string[];
13 | };
14 |
--------------------------------------------------------------------------------
/utils/sources.ts:
--------------------------------------------------------------------------------
1 | export const cleanSourceText = (text: string) => {
2 | return text
3 | .trim()
4 | .replace(/(\n){4,}/g, "\n\n\n")
5 | .replace(/\n\n/g, " ")
6 | .replace(/ {3,}/g, " ")
7 | .replace(/\t/g, "")
8 | .replace(/\n+(\s*\n)*/g, "\n");
9 | };
10 |
--------------------------------------------------------------------------------
/pages/_document.tsx:
--------------------------------------------------------------------------------
1 | import { Html, Head, Main, NextScript } from 'next/document'
2 |
3 | export default function Document() {
4 | return (
5 |
6 |
8 |
9 |
10 |
11 |
12 | )
13 | }
14 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2 |
3 | # dependencies
4 | /node_modules
5 | /.pnp
6 | .pnp.js
7 |
8 | # testing
9 | /coverage
10 |
11 | # next.js
12 | /.next/
13 | /out/
14 |
15 | # production
16 | /build
17 |
18 | # misc
19 | .DS_Store
20 | *.pem
21 |
22 | # debug
23 | npm-debug.log*
24 | yarn-debug.log*
25 | yarn-error.log*
26 | .pnpm-debug.log*
27 |
28 | # local env files
29 | .env*.local
30 |
31 | # vercel
32 | .vercel
33 |
34 | # typescript
35 | *.tsbuildinfo
36 | next-env.d.ts
37 |
--------------------------------------------------------------------------------
/pages/api/answer.ts:
--------------------------------------------------------------------------------
1 | import { OpenAIStream } from "@/utils/answer";
2 |
3 | export const config = {
4 | runtime: "edge"
5 | };
6 |
7 | const handler = async (req: Request): Promise => {
8 | try {
9 | const { prompt, apiKey } = (await req.json()) as {
10 | prompt: string;
11 | apiKey: string;
12 | };
13 |
14 | const stream = await OpenAIStream(prompt, apiKey);
15 |
16 | return new Response(stream);
17 | } catch (error) {
18 | console.error(error);
19 | return new Response("Error", { status: 500 });
20 | }
21 | };
22 |
23 | export default handler;
24 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es5",
4 | "lib": ["dom", "dom.iterable", "esnext"],
5 | "allowJs": true,
6 | "skipLibCheck": true,
7 | "strict": true,
8 | "forceConsistentCasingInFileNames": true,
9 | "noEmit": true,
10 | "esModuleInterop": true,
11 | "module": "esnext",
12 | "moduleResolution": "node",
13 | "resolveJsonModule": true,
14 | "isolatedModules": true,
15 | "jsx": "preserve",
16 | "incremental": true,
17 | "baseUrl": ".",
18 | "paths": {
19 | "@/*": ["./*"]
20 | }
21 | },
22 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
23 | "exclude": ["node_modules"]
24 | }
25 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "frontend",
3 | "version": "0.1.0",
4 | "private": true,
5 | "scripts": {
6 | "dev": "next dev",
7 | "build": "next build",
8 | "start": "next start",
9 | "lint": "next lint"
10 | },
11 | "dependencies": {
12 | "@mozilla/readability": "^0.4.2",
13 | "@next/font": "13.1.6",
14 | "@tabler/icons": "^2.4.0",
15 | "@tabler/icons-react": "^2.4.0",
16 | "@types/node": "18.14.0",
17 | "@types/react": "18.0.28",
18 | "@types/react-dom": "18.0.11",
19 | "cheerio": "^1.0.0-rc.12",
20 | "endent": "^2.1.0",
21 | "eslint": "8.34.0",
22 | "eslint-config-next": "13.1.6",
23 | "eventsource-parser": "^0.1.0",
24 | "jsdom": "^21.1.0",
25 | "next": "13.1.6",
26 | "openai": "^3.1.0",
27 | "react": "18.2.0",
28 | "react-dom": "18.2.0",
29 | "typescript": "4.9.5"
30 | },
31 | "devDependencies": {
32 | "@types/jsdom": "^21.1.0",
33 | "autoprefixer": "^10.4.13",
34 | "postcss": "^8.4.21",
35 | "tailwindcss": "^3.2.7"
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/license:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2023 Mckay Wrigley
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Clarity AI
2 |
3 | Clarity is simple [perplexity.ai](https://www.perplexity.ai/) clone. Use the code for whatever you like! :)
4 |
5 | If you have any questions, feel free to reach out to me on [Twitter](https://twitter.com/mckaywrigley).
6 |
7 | [](https://clarity-ai.vercel.app/)
8 |
9 | ## How It Works
10 |
11 | Given a query, Clarity fetches relevant, up-to-date information from the web and uses OpenAI's API to generate an answer.
12 |
13 | The app works as follows:
14 |
15 | 1. Get query from user
16 | 2. Scrape Google for relevant webpages
17 | 3. Parse webpages for text
18 | 4. Build prompt using query + webpage text
19 | 5. Call OpenAI API to generate answer
20 | 6. Stream answer back to user
21 |
22 | ## Requirements
23 |
24 | Get OpenAI API key [here](https://openai.com/api/).
25 |
26 | ## Running Locally
27 |
28 | 1. Clone repo
29 |
30 | ```bash
31 | git clone https://github.com/mckaywrigley/clarity-ai.git
32 | ```
33 |
34 | 2. Install dependencies
35 |
36 | ```bash
37 | npm i
38 | ```
39 |
40 | 3. Run app
41 |
42 | ```bash
43 | npm run dev
44 | ```
45 |
46 | ## Improvement Ideas
47 |
48 | Here are some ideas for how to improve Clarity:
49 |
50 | - [ ] Speed up answers by replacing link scraping with the Google Search API (scraping was used to circumvent cost + rate limits)
51 | - [ ] Add "follow up" searches
52 | - [ ] Improve the prompt
53 | - [ ] Get sources working in non text-davinci-003 models
54 | - [ ] Train your own model to use for answer synthesis
55 |
56 | ## Credits
57 |
58 | Shoutout to [Perplexity AI](https://www.perplexity.ai/) for the inspiration. I highly recommend checking their product out.
59 |
60 | This repo is meant to show people that you can build powerful apps like Perplexity even if you don't have a large, experienced team.
61 |
62 | LLMs are amazing, and I hope Clarity inspires you to build something cool!
63 |
--------------------------------------------------------------------------------
/utils/answer.ts:
--------------------------------------------------------------------------------
1 | import { OpenAIModel } from "@/types";
2 | import { createParser, ParsedEvent, ReconnectInterval } from "eventsource-parser";
3 |
4 | export const OpenAIStream = async (prompt: string, apiKey: string) => {
5 | const encoder = new TextEncoder();
6 | const decoder = new TextDecoder();
7 |
8 | const res = await fetch("https://api.openai.com/v1/chat/completions", {
9 | headers: {
10 | "Content-Type": "application/json",
11 | Authorization: `Bearer ${apiKey}`
12 | },
13 | method: "POST",
14 | body: JSON.stringify({
15 | model: OpenAIModel.DAVINCI_TURBO,
16 | messages: [
17 | { role: "system", content: "You are a helpful assistant that accurately answers the user's queries based on the given text." },
18 | { role: "user", content: prompt }
19 | ],
20 | max_tokens: 120,
21 | temperature: 0.0,
22 | stream: true
23 | })
24 | });
25 |
26 | if (res.status !== 200) {
27 | throw new Error("OpenAI API returned an error");
28 | }
29 |
30 | const stream = new ReadableStream({
31 | async start(controller) {
32 | const onParse = (event: ParsedEvent | ReconnectInterval) => {
33 | if (event.type === "event") {
34 | const data = event.data;
35 |
36 | if (data === "[DONE]") {
37 | controller.close();
38 | return;
39 | }
40 |
41 | try {
42 | const json = JSON.parse(data);
43 | const text = json.choices[0].delta.content;
44 | const queue = encoder.encode(text);
45 | controller.enqueue(queue);
46 | } catch (e) {
47 | controller.error(e);
48 | }
49 | }
50 | };
51 |
52 | const parser = createParser(onParse);
53 |
54 | for await (const chunk of res.body as any) {
55 | parser.feed(decoder.decode(chunk));
56 | }
57 | }
58 | });
59 |
60 | return stream;
61 | };
62 |
--------------------------------------------------------------------------------
/pages/index.tsx:
--------------------------------------------------------------------------------
1 | import { Answer } from "@/components/Answer";
2 | import { Search } from "@/components/Search";
3 | import { SearchQuery } from "@/types";
4 | import { IconBrandGithub, IconBrandTwitter } from "@tabler/icons-react";
5 | import Head from "next/head";
6 | import { useState } from "react";
7 |
8 | export default function Home() {
9 | const [searchQuery, setSearchQuery] = useState({ query: "", sourceLinks: [] });
10 | const [answer, setAnswer] = useState("");
11 | const [done, setDone] = useState(false);
12 |
13 | return (
14 | <>
15 |
16 | Clarity AI
17 |
21 |
25 |
29 |
30 |
31 |
37 |
38 |
39 |
40 |
46 |
47 |
48 |
49 | {answer ? (
50 |
{
55 | setAnswer("");
56 | setSearchQuery({ query: "", sourceLinks: [] });
57 | setDone(false);
58 | }}
59 | />
60 | ) : (
61 | setAnswer((prev) => prev + value)}
64 | onDone={setDone}
65 | />
66 | )}
67 |
68 | >
69 | );
70 | }
71 |
--------------------------------------------------------------------------------
/pages/api/sources.ts:
--------------------------------------------------------------------------------
1 | import { OpenAIModel, Source } from "@/types";
2 | import { Readability } from "@mozilla/readability";
3 | import * as cheerio from "cheerio";
4 | import { JSDOM } from "jsdom";
5 | import type { NextApiRequest, NextApiResponse } from "next";
6 | import { cleanSourceText } from "../../utils/sources";
7 |
8 | type Data = {
9 | sources: Source[];
10 | };
11 |
12 | const searchHandler = async (req: NextApiRequest, res: NextApiResponse) => {
13 | try {
14 | const { query, model } = req.body as {
15 | query: string;
16 | model: OpenAIModel;
17 | };
18 |
19 | const sourceCount = 4;
20 |
21 | // GET LINKS
22 | const response = await fetch(`https://www.google.com/search?q=${query}`);
23 | const html = await response.text();
24 | const $ = cheerio.load(html);
25 | const linkTags = $("a");
26 |
27 | let links: string[] = [];
28 |
29 | linkTags.each((i, link) => {
30 | const href = $(link).attr("href");
31 |
32 | if (href && href.startsWith("/url?q=")) {
33 | const cleanedHref = href.replace("/url?q=", "").split("&")[0];
34 |
35 | if (!links.includes(cleanedHref)) {
36 | links.push(cleanedHref);
37 | }
38 | }
39 | });
40 |
41 | const filteredLinks = links.filter((link, idx) => {
42 | const domain = new URL(link).hostname;
43 |
44 | const excludeList = ["google", "facebook", "twitter", "instagram", "youtube", "tiktok"];
45 | if (excludeList.some((site) => domain.includes(site))) return false;
46 |
47 | return links.findIndex((link) => new URL(link).hostname === domain) === idx;
48 | });
49 |
50 | const finalLinks = filteredLinks.slice(0, sourceCount);
51 |
52 | // SCRAPE TEXT FROM LINKS
53 | const sources = (await Promise.all(
54 | finalLinks.map(async (link) => {
55 | const response = await fetch(link);
56 | const html = await response.text();
57 | const dom = new JSDOM(html);
58 | const doc = dom.window.document;
59 | const parsed = new Readability(doc).parse();
60 |
61 | if (parsed) {
62 | let sourceText = cleanSourceText(parsed.textContent);
63 |
64 | return { url: link, text: sourceText };
65 | }
66 | })
67 | )) as Source[];
68 |
69 | const filteredSources = sources.filter((source) => source !== undefined);
70 |
71 | for (const source of filteredSources) {
72 | source.text = source.text.slice(0, 1500);
73 | }
74 |
75 | res.status(200).json({ sources: filteredSources });
76 | } catch (err) {
77 | console.log(err);
78 | res.status(500).json({ sources: [] });
79 | }
80 | };
81 |
82 | export default searchHandler;
83 |
--------------------------------------------------------------------------------
/components/Answer.tsx:
--------------------------------------------------------------------------------
1 | import { SearchQuery } from "@/types";
2 | import { IconReload } from "@tabler/icons-react";
3 | import { FC } from "react";
4 |
5 | interface AnswerProps {
6 | searchQuery: SearchQuery;
7 | answer: string;
8 | done: boolean;
9 | onReset: () => void;
10 | }
11 |
12 | export const Answer: FC = ({ searchQuery, answer, done, onReset }) => {
13 | return (
14 |
15 |
{searchQuery.query}
16 |
17 |
18 |
Answer
19 |
20 |
{replaceSourcesWithLinks(answer, searchQuery.sourceLinks)}
21 |
22 |
23 | {done && (
24 | <>
25 |
26 |
Sources
27 |
28 | {searchQuery.sourceLinks.map((source, index) => (
29 |
43 | ))}
44 |
45 |
46 |
53 | >
54 | )}
55 |
56 | );
57 | };
58 |
59 | const replaceSourcesWithLinks = (answer: string, sourceLinks: string[]) => {
60 | const elements = answer.split(/(\[[0-9]+\])/).map((part, index) => {
61 | if (/\[[0-9]+\]/.test(part)) {
62 | const link = sourceLinks[parseInt(part.replace(/[\[\]]/g, "")) - 1];
63 |
64 | return (
65 |
72 | {part}
73 |
74 | );
75 | } else {
76 | return part;
77 | }
78 | });
79 |
80 | return elements;
81 | };
82 |
--------------------------------------------------------------------------------
/components/Search.tsx:
--------------------------------------------------------------------------------
1 | import { SearchQuery, Source } from "@/types";
2 | import { IconArrowRight, IconBolt, IconSearch } from "@tabler/icons-react";
3 | import endent from "endent";
4 | import { FC, KeyboardEvent, useEffect, useRef, useState } from "react";
5 |
6 | interface SearchProps {
7 | onSearch: (searchResult: SearchQuery) => void;
8 | onAnswerUpdate: (answer: string) => void;
9 | onDone: (done: boolean) => void;
10 | }
11 |
12 | export const Search: FC = ({ onSearch, onAnswerUpdate, onDone }) => {
13 | const inputRef = useRef(null);
14 |
15 | const [query, setQuery] = useState("");
16 | const [apiKey, setApiKey] = useState("");
17 | const [showSettings, setShowSettings] = useState(false);
18 | const [loading, setLoading] = useState(false);
19 |
20 | const handleSearch = async () => {
21 | if (!query) {
22 | alert("Please enter a query");
23 | return;
24 | }
25 |
26 | setLoading(true);
27 | const sources = await fetchSources();
28 | await handleStream(sources);
29 | };
30 |
31 | const fetchSources = async () => {
32 | const response = await fetch("/api/sources", {
33 | method: "POST",
34 | headers: {
35 | "Content-Type": "application/json"
36 | },
37 | body: JSON.stringify({ query })
38 | });
39 |
40 | if (!response.ok) {
41 | setLoading(false);
42 | throw new Error(response.statusText);
43 | }
44 |
45 | const { sources }: { sources: Source[] } = await response.json();
46 |
47 | return sources;
48 | };
49 |
50 | const handleStream = async (sources: Source[]) => {
51 | try {
52 | const prompt = endent`Provide a 2-3 sentence answer to the query based on the following sources. Be original, concise, accurate, and helpful. Cite sources as [1] or [2] or [3] after each sentence (not just the very end) to back up your answer (Ex: Correct: [1], Correct: [2][3], Incorrect: [1, 2]).
53 |
54 | ${sources.map((source, idx) => `Source [${idx + 1}]:\n${source.text}`).join("\n\n")}
55 | `;
56 |
57 | const response = await fetch("/api/answer", {
58 | method: "POST",
59 | headers: {
60 | "Content-Type": "application/json"
61 | },
62 | body: JSON.stringify({ prompt, apiKey })
63 | });
64 |
65 | if (!response.ok) {
66 | setLoading(false);
67 | throw new Error(response.statusText);
68 | }
69 |
70 | setLoading(false);
71 | onSearch({ query, sourceLinks: sources.map((source) => source.url) });
72 |
73 | const data = response.body;
74 |
75 | if (!data) {
76 | return;
77 | }
78 |
79 | const reader = data.getReader();
80 | const decoder = new TextDecoder();
81 | let done = false;
82 |
83 | while (!done) {
84 | const { value, done: doneReading } = await reader.read();
85 | done = doneReading;
86 | const chunkValue = decoder.decode(value);
87 | onAnswerUpdate(chunkValue);
88 | }
89 |
90 | onDone(true);
91 | } catch (err) {
92 | onAnswerUpdate("Error");
93 | }
94 | };
95 |
96 | const handleKeyDown = (e: KeyboardEvent) => {
97 | if (e.key === "Enter") {
98 | handleSearch();
99 | }
100 | };
101 |
102 | const handleSave = () => {
103 | if (apiKey.length !== 51) {
104 | alert("Please enter a valid API key.");
105 | return;
106 | }
107 |
108 | localStorage.setItem("CLARITY_KEY", apiKey);
109 |
110 | setShowSettings(false);
111 | inputRef.current?.focus();
112 | };
113 |
114 | const handleClear = () => {
115 | localStorage.removeItem("CLARITY_KEY");
116 |
117 | setApiKey("");
118 | };
119 |
120 | useEffect(() => {
121 | const CLARITY_KEY = localStorage.getItem("CLARITY_KEY");
122 |
123 | if (CLARITY_KEY) {
124 | setApiKey(CLARITY_KEY);
125 | } else {
126 | setShowSettings(true);
127 | }
128 |
129 | inputRef.current?.focus();
130 | }, []);
131 |
132 | return (
133 | <>
134 | {loading ? (
135 |
136 |
137 |
Getting answer...
138 |
139 | ) : (
140 |
141 |
142 |
143 |
Clarity
144 |
145 |
146 | {apiKey.length === 51 ? (
147 |
148 |
149 |
150 | setQuery(e.target.value)}
157 | onKeyDown={handleKeyDown}
158 | />
159 |
160 |
166 |
167 | ) : (
168 |
Please enter your OpenAI API key.
169 | )}
170 |
171 |
177 |
178 | {showSettings && (
179 | <>
180 |
{
185 | setApiKey(e.target.value);
186 |
187 | if (e.target.value.length !== 51) {
188 | setShowSettings(true);
189 | }
190 | }}
191 | />
192 |
193 |
194 |
198 | Save
199 |
200 |
201 |
205 | Clear
206 |
207 |
208 | >
209 | )}
210 |
211 | )}
212 | >
213 | );
214 | };
215 |
--------------------------------------------------------------------------------