├── .eslintrc.json ├── .gitignore ├── README.md ├── components ├── Answer │ ├── Answer.tsx │ └── answer.module.css ├── Footer.tsx └── Navbar.tsx ├── license ├── next.config.js ├── package-lock.json ├── package.json ├── pages ├── _app.tsx ├── _document.tsx ├── api │ ├── answer.ts │ └── search.ts └── index.tsx ├── postcss.config.js ├── public ├── favicon.ico ├── king.png └── wbw.png ├── schema.sql ├── scripts ├── embed.ts ├── images.json ├── images.ts ├── scrape.ts └── wbw.json ├── styles └── globals.css ├── tailwind.config.js ├── tsconfig.json ├── types └── index.ts └── utils ├── images.ts └── index.ts /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | .pnpm-debug.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Wait But Why GPT 2 | 3 | AI-powered search and chat for Tim Urban's blog "Wait But Why." 4 | 5 | [![Wait But Why GPT](./public/wbw.png)](https://waitbutwhy.com/) 6 | 7 | ## Dataset 8 | 9 | The dataset is a CSV file containing all text & embeddings used. 10 | 11 | Download it [here](https://drive.google.com/file/d/1qgNZYOwkqk30PtPBBAyz-u7DBhzin8us/view?usp=sharing). 12 | 13 | I recommend getting familiar with fetching, cleaning, and storing data as outlined in the scraping and embedding scripts below, but feel free to skip those steps and just use the dataset. 14 | 15 | ## How It Works 16 | 17 | Wait But Why GPT provides 2 things: 18 | 19 | 1. A search interface. 20 | 2. A chat interface. 21 | 22 | ### Search 23 | 24 | Search was created with [OpenAI Embeddings](https://platform.openai.com/docs/guides/embeddings) (`text-embedding-ada-002`). 25 | 26 | First, we loop over the essays and generate embeddings for each chunk of text. 27 | 28 | Then in the app we take the user's search query, generate an embedding, and use the result to find the most similar passages from the book. 29 | 30 | The comparison is done using cosine similarity across our database of vectors. 31 | 32 | Our database is a Postgres database with the [pgvector](https://github.com/pgvector/pgvector) extension hosted on [Supabase](https://supabase.com/). 33 | 34 | Results are ranked by similarity score and returned to the user. 35 | 36 | ### Chat 37 | 38 | Chat builds on top of search. It uses search results to create a prompt that is fed into GPT-3.5-turbo. 39 | 40 | This allows for a chat-like experience where the user can ask questions about the book and get answers. 41 | 42 | ## Running Locally 43 | 44 | Here's a quick overview of how to run it locally. 45 | 46 | ### Requirements 47 | 48 | 1. Set up OpenAI 49 | 50 | You'll need an OpenAI API key to generate embeddings. 51 | 52 | 2. Set up Supabase and create a database 53 | 54 | Note: You don't have to use Supabase. Use whatever method you prefer to store your data. But I like Supabase and think it's easy to use. 55 | 56 | There is a schema.sql file in the root of the repo that you can use to set up the database. 57 | 58 | Run that in the SQL editor in Supabase as directed. 59 | 60 | I recommend turning on Row Level Security and setting up a service role to use with the app. 61 | 62 | ### Repo Setup 63 | 64 | 3. Clone repo 65 | 66 | ```bash 67 | git clone https://github.com/mckaywrigley/wait-but-why-gpt.git 68 | ``` 69 | 70 | 4. Install dependencies 71 | 72 | ```bash 73 | npm i 74 | ``` 75 | 76 | 5. Set up environment variables 77 | 78 | Create a .env.local file in the root of the repo with the following variables: 79 | 80 | ```bash 81 | OPENAI_API_KEY= 82 | 83 | NEXT_PUBLIC_SUPABASE_URL= 84 | SUPABASE_SERVICE_ROLE_KEY= 85 | ``` 86 | 87 | ### Dataset 88 | 89 | 6. Run scraping script 90 | 91 | ```bash 92 | npm run scrape 93 | ``` 94 | 95 | This scrapes all of the posts from the Wait But Why website and saves them to a json file. 96 | 97 | 1. Run embedding script 98 | 99 | ```bash 100 | npm run embed 101 | ``` 102 | 103 | This reads the json file, generates embeddings for each chunk of text, and saves the results to your database. 104 | 105 | There is a 100ms delay between each request to avoid rate limiting. 106 | 107 | This process will take 20-30 minutes. 108 | 109 | ### App 110 | 111 | 8. Run app 112 | 113 | ```bash 114 | npm run dev 115 | ``` 116 | 117 | ## Credits 118 | 119 | Thanks to [Tim Urban](https://twitter.com/waitbutwhy) for his writing. 120 | 121 | I don't think you could find a more fun blog on the internet. 122 | 123 | And go buy his new [book](https://www.amazon.com/Whats-Our-Problem-Self-Help-Societies-ebook/dp/B0BTJCTR58/ref=sr_1_1?crid=3HVMHUY8BNF7I&keywords=tim+urban&qid=1677871628&sprefix=tim+urban%2Caps%2C133&sr=8-1)! 124 | 125 | ## Contact 126 | 127 | If you have any questions, feel free to reach out to me on [Twitter](https://twitter.com/mckaywrigley)! 128 | 129 | ## Notes 130 | 131 | I sacrificed composability for simplicity in the app. 132 | 133 | Yes, you can make things more modular and reusable. 134 | 135 | But I kept pretty much everything in the homepage component for the sake of simplicity. 136 | -------------------------------------------------------------------------------- /components/Answer/Answer.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from "react"; 2 | import styles from "./answer.module.css"; 3 | 4 | interface AnswerProps { 5 | text: string; 6 | } 7 | 8 | export const Answer: React.FC = ({ text }) => { 9 | const [words, setWords] = useState([]); 10 | 11 | useEffect(() => { 12 | setWords(text.split(" ")); 13 | }, [text]); 14 | 15 | return ( 16 |
17 | {words.map((word, index) => ( 18 | 23 | {word}{" "} 24 | 25 | ))} 26 |
27 | ); 28 | }; 29 | -------------------------------------------------------------------------------- /components/Answer/answer.module.css: -------------------------------------------------------------------------------- 1 | .fadeIn { 2 | animation: fadeIn 0.5s ease-in-out forwards; 3 | opacity: 0; 4 | } 5 | 6 | @keyframes fadeIn { 7 | from { 8 | opacity: 0; 9 | } 10 | to { 11 | opacity: 1; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /components/Footer.tsx: -------------------------------------------------------------------------------- 1 | import { IconBrandGithub, IconBrandTwitter } from "@tabler/icons-react"; 2 | import { FC } from "react"; 3 | 4 | export const Footer: FC = () => { 5 | return ( 6 |
7 |
8 | 9 |
10 | Created by 11 | 17 | Mckay Wrigley 18 | 19 | based on 20 | 26 | Tim Urban 27 | 28 | {`'s blog`} 29 | 35 | Wait But Why 36 | 37 | . 38 |
39 | 40 |
41 | 47 | 48 | 49 | 50 | 56 | 57 | 58 |
59 |
60 | ); 61 | }; 62 | -------------------------------------------------------------------------------- /components/Navbar.tsx: -------------------------------------------------------------------------------- 1 | import { IconExternalLink } from "@tabler/icons-react"; 2 | import Image from "next/image"; 3 | import { FC } from "react"; 4 | import king from "../public/king.png"; 5 | 6 | export const Navbar: FC = () => { 7 | return ( 8 |
9 |
10 | 14 | The Network State GPT 20 |
Wait But Why GPT
21 | 22 |
23 |
24 | 30 |
WaitButWhy.com
31 | 32 | 36 | 37 |
38 |
39 | ); 40 | }; 41 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Mckay Wrigley 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = { 3 | reactStrictMode: true, 4 | images: { 5 | remotePatterns: [ 6 | { 7 | protocol: "https", 8 | hostname: "**" 9 | } 10 | ] 11 | } 12 | }; 13 | 14 | module.exports = nextConfig; 15 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "app", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint", 10 | "scrape": "tsx scripts/scrape.ts", 11 | "embed": "tsx scripts/embed.ts", 12 | "images": "tsx scripts/images.ts" 13 | }, 14 | "dependencies": { 15 | "@next/font": "^13.2.3", 16 | "@tabler/icons-react": "^2.7.0", 17 | "@types/node": "18.14.2", 18 | "@types/react": "18.0.28", 19 | "@types/react-dom": "18.0.11", 20 | "endent": "^2.1.0", 21 | "eslint": "8.35.0", 22 | "eslint-config-next": "13.2.1", 23 | "eventsource-parser": "^0.1.0", 24 | "next": "13.2.1", 25 | "react": "18.2.0", 26 | "react-dom": "18.2.0", 27 | "typescript": "4.9.5" 28 | }, 29 | "devDependencies": { 30 | "@next/env": "^13.2.3", 31 | "@supabase/supabase-js": "^2.10.0", 32 | "autoprefixer": "^10.4.13", 33 | "axios": "^1.3.4", 34 | "cheerio": "^1.0.0-rc.12", 35 | "gpt-3-encoder": "^1.1.4", 36 | "openai": "^3.2.1", 37 | "postcss": "^8.4.21", 38 | "tailwindcss": "^3.2.7", 39 | "tsx": "^3.12.3" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /pages/_app.tsx: -------------------------------------------------------------------------------- 1 | import "@/styles/globals.css"; 2 | import { Inter } from "@next/font/google"; 3 | import type { AppProps } from "next/app"; 4 | 5 | const inter = Inter({ subsets: ["latin"] }); 6 | 7 | export default function App({ Component, pageProps }: AppProps<{}>) { 8 | return ( 9 |
10 | 11 |
12 | ); 13 | } 14 | -------------------------------------------------------------------------------- /pages/_document.tsx: -------------------------------------------------------------------------------- 1 | import { Html, Head, Main, NextScript } from 'next/document' 2 | 3 | export default function Document() { 4 | return ( 5 | 6 | 7 | 8 |
9 | 10 | 11 | 12 | ) 13 | } 14 | -------------------------------------------------------------------------------- /pages/api/answer.ts: -------------------------------------------------------------------------------- 1 | import { OpenAIStream } from "@/utils"; 2 | 3 | export const config = { 4 | runtime: "edge" 5 | }; 6 | 7 | const handler = async (req: Request): Promise => { 8 | try { 9 | const { prompt, apiKey } = (await req.json()) as { 10 | prompt: string; 11 | apiKey: string; 12 | }; 13 | 14 | const stream = await OpenAIStream(prompt, apiKey); 15 | 16 | return new Response(stream); 17 | } catch (error) { 18 | console.error(error); 19 | return new Response("Error", { status: 500 }); 20 | } 21 | }; 22 | 23 | export default handler; 24 | -------------------------------------------------------------------------------- /pages/api/search.ts: -------------------------------------------------------------------------------- 1 | import { supabaseAdmin } from "@/utils"; 2 | 3 | export const config = { 4 | runtime: "edge" 5 | }; 6 | 7 | const handler = async (req: Request): Promise => { 8 | try { 9 | const { query, apiKey, matches } = (await req.json()) as { 10 | query: string; 11 | apiKey: string; 12 | matches: number; 13 | }; 14 | 15 | const input = query.replace(/\n/g, " "); 16 | 17 | const res = await fetch("https://api.openai.com/v1/embeddings", { 18 | headers: { 19 | "Content-Type": "application/json", 20 | Authorization: `Bearer ${apiKey}` 21 | }, 22 | method: "POST", 23 | body: JSON.stringify({ 24 | model: "text-embedding-ada-002", 25 | input 26 | }) 27 | }); 28 | 29 | const json = await res.json(); 30 | const embedding = json.data[0].embedding; 31 | 32 | const { data: chunks, error } = await supabaseAdmin.rpc("wbw_search", { 33 | query_embedding: embedding, 34 | similarity_threshold: 0.01, 35 | match_count: matches 36 | }); 37 | 38 | if (error) { 39 | console.error(error); 40 | return new Response("Error", { status: 500 }); 41 | } 42 | 43 | return new Response(JSON.stringify(chunks), { status: 200 }); 44 | } catch (error) { 45 | console.error(error); 46 | return new Response("Error", { status: 500 }); 47 | } 48 | }; 49 | 50 | export default handler; 51 | -------------------------------------------------------------------------------- /pages/index.tsx: -------------------------------------------------------------------------------- 1 | import { Answer } from "@/components/Answer/Answer"; 2 | import { Footer } from "@/components/Footer"; 3 | import { Navbar } from "@/components/Navbar"; 4 | import { WBWChunk } from "@/types"; 5 | import { getImage } from "@/utils/images"; 6 | import { IconArrowRight, IconExternalLink, IconSearch } from "@tabler/icons-react"; 7 | import endent from "endent"; 8 | import Head from "next/head"; 9 | import Image from "next/image"; 10 | import { KeyboardEvent, useEffect, useRef, useState } from "react"; 11 | 12 | export default function Home() { 13 | const inputRef = useRef(null); 14 | 15 | const [query, setQuery] = useState(""); 16 | const [chunks, setChunks] = useState([]); 17 | const [answer, setAnswer] = useState(""); 18 | const [loading, setLoading] = useState(false); 19 | 20 | const [showSettings, setShowSettings] = useState(false); 21 | const [mode, setMode] = useState<"search" | "chat">("chat"); 22 | const [matchCount, setMatchCount] = useState(5); 23 | const [apiKey, setApiKey] = useState(""); 24 | 25 | const handleSearch = async () => { 26 | if (!apiKey) { 27 | alert("Please enter an API key."); 28 | return; 29 | } 30 | 31 | if (!query) { 32 | alert("Please enter a query."); 33 | return; 34 | } 35 | 36 | setAnswer(""); 37 | setChunks([]); 38 | 39 | setLoading(true); 40 | 41 | const searchResponse = await fetch("/api/search", { 42 | method: "POST", 43 | headers: { 44 | "Content-Type": "application/json" 45 | }, 46 | body: JSON.stringify({ query, apiKey, matches: matchCount }) 47 | }); 48 | 49 | if (!searchResponse.ok) { 50 | setLoading(false); 51 | throw new Error(searchResponse.statusText); 52 | } 53 | 54 | const results: WBWChunk[] = await searchResponse.json(); 55 | 56 | setChunks(results); 57 | 58 | setLoading(false); 59 | 60 | return results; 61 | }; 62 | 63 | const handleAnswer = async () => { 64 | if (!apiKey) { 65 | alert("Please enter an API key."); 66 | return; 67 | } 68 | 69 | if (!query) { 70 | alert("Please enter a query."); 71 | return; 72 | } 73 | 74 | setAnswer(""); 75 | setChunks([]); 76 | 77 | setLoading(true); 78 | 79 | const searchResponse = await fetch("/api/search", { 80 | method: "POST", 81 | headers: { 82 | "Content-Type": "application/json" 83 | }, 84 | body: JSON.stringify({ query, apiKey, matches: matchCount }) 85 | }); 86 | 87 | if (!searchResponse.ok) { 88 | setLoading(false); 89 | throw new Error(searchResponse.statusText); 90 | } 91 | 92 | const results: WBWChunk[] = await searchResponse.json(); 93 | 94 | setChunks(results); 95 | 96 | const prompt = endent` 97 | Use the following passages to provide an answer to the query: "${query}" 98 | 99 | ${results?.map((d: any) => d.content).join("\n\n")} 100 | `; 101 | 102 | const answerResponse = await fetch("/api/answer", { 103 | method: "POST", 104 | headers: { 105 | "Content-Type": "application/json" 106 | }, 107 | body: JSON.stringify({ prompt, apiKey }) 108 | }); 109 | 110 | if (!answerResponse.ok) { 111 | setLoading(false); 112 | throw new Error(answerResponse.statusText); 113 | } 114 | 115 | const data = answerResponse.body; 116 | 117 | if (!data) { 118 | return; 119 | } 120 | 121 | setLoading(false); 122 | 123 | const reader = data.getReader(); 124 | const decoder = new TextDecoder(); 125 | let done = false; 126 | 127 | while (!done) { 128 | const { value, done: doneReading } = await reader.read(); 129 | done = doneReading; 130 | const chunkValue = decoder.decode(value); 131 | setAnswer((prev) => prev + chunkValue); 132 | } 133 | }; 134 | 135 | const handleKeyDown = (e: KeyboardEvent) => { 136 | if (e.key === "Enter") { 137 | if (mode === "search") { 138 | handleSearch(); 139 | } else { 140 | handleAnswer(); 141 | } 142 | } 143 | }; 144 | 145 | const handleSave = () => { 146 | if (apiKey.length !== 51) { 147 | alert("Please enter a valid API key."); 148 | return; 149 | } 150 | 151 | localStorage.setItem("WBW_KEY", apiKey); 152 | localStorage.setItem("WBW_MATCH_COUNT", matchCount.toString()); 153 | localStorage.setItem("WBW_MODE", mode); 154 | 155 | setShowSettings(false); 156 | }; 157 | 158 | const handleClear = () => { 159 | localStorage.removeItem("WBW_KEY"); 160 | localStorage.removeItem("WBW_MATCH_COUNT"); 161 | localStorage.removeItem("WBW_MODE"); 162 | 163 | setApiKey(""); 164 | setMatchCount(5); 165 | setMode("search"); 166 | }; 167 | 168 | useEffect(() => { 169 | if (matchCount > 10) { 170 | setMatchCount(10); 171 | } else if (matchCount < 1) { 172 | setMatchCount(1); 173 | } 174 | }, [matchCount]); 175 | 176 | useEffect(() => { 177 | const WBW_KEY = localStorage.getItem("WBW_KEY"); 178 | const WBW_MATCH_COUNT = localStorage.getItem("WBW_MATCH_COUNT"); 179 | const WBW_MODE = localStorage.getItem("WBW_MODE"); 180 | 181 | if (WBW_KEY) { 182 | setApiKey(WBW_KEY); 183 | } 184 | 185 | if (WBW_MATCH_COUNT) { 186 | setMatchCount(parseInt(WBW_MATCH_COUNT)); 187 | } 188 | 189 | if (WBW_MODE) { 190 | setMode(WBW_MODE as "search" | "chat"); 191 | } 192 | }, []); 193 | 194 | return ( 195 | <> 196 | 197 | Wait But Why GPT 198 | 202 | 206 | 210 | 211 | 212 |
213 | 214 |
215 |
216 | 222 | 223 | {showSettings && ( 224 |
225 |
226 |
Mode
227 | 235 |
236 | 237 |
238 |
Passage Count
239 | setMatchCount(Number(e.target.value))} 245 | className="max-w-[400px] block w-full rounded-md border border-gray-300 p-2 text-black shadow-sm focus:border-blue-500 focus:outline-none focus:ring-2 focus:ring-blue-500 sm:text-sm" 246 | /> 247 |
248 | 249 |
250 |
OpenAI API Key
251 | { 257 | setApiKey(e.target.value); 258 | 259 | if (e.target.value.length !== 51) { 260 | setShowSettings(true); 261 | } 262 | }} 263 | /> 264 |
265 | 266 |
267 |
271 | Save 272 |
273 | 274 |
278 | Clear 279 |
280 |
281 |
282 | )} 283 | 284 | {apiKey.length === 51 ? ( 285 |
286 | 287 | 288 | setQuery(e.target.value)} 295 | onKeyDown={handleKeyDown} 296 | /> 297 | 298 | 304 |
305 | ) : ( 306 |
307 | Please enter your 308 | 312 | OpenAI API key 313 | 314 | in settings. 315 |
316 | )} 317 | 318 | {loading ? ( 319 |
320 | {mode === "chat" && ( 321 | <> 322 |
Answer
323 |
324 |
325 |
326 |
327 |
328 |
329 |
330 | 331 | )} 332 | 333 |
Passages
334 |
335 |
336 |
337 |
338 |
339 |
340 |
341 |
342 | ) : answer ? ( 343 |
344 |
Answer
345 | 346 | 347 |
348 |
Passages
349 | 350 | {chunks.map((chunk, index) => ( 351 |
352 |
353 |
354 |
355 | {chunk.post_title} 362 |
363 |
{chunk.post_title}
364 |
{chunk.post_date}
365 |
366 |
367 | 373 | 374 | 375 |
376 |
{chunk.content}
377 |
378 |
379 | ))} 380 |
381 |
382 | ) : chunks.length > 0 ? ( 383 |
384 |
Passages
385 | {chunks.map((chunk, index) => ( 386 |
387 |
388 |
389 |
390 | {chunk.post_title} 397 |
398 |
{chunk.post_title}
399 |
{chunk.post_date}
400 |
401 |
402 | 408 | 409 | 410 |
411 |
{chunk.content}
412 |
413 |
414 | ))} 415 |
416 | ) : ( 417 |
{`AI-powered search and chat for Tim Urban's blog "Wait But Why."`}
418 | )} 419 |
420 |
421 |
423 | 424 | ); 425 | } 426 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/wait-but-why-gpt/3002c615d84ad2fad6d7ab7aafa34ffa52356432/public/favicon.ico -------------------------------------------------------------------------------- /public/king.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/wait-but-why-gpt/3002c615d84ad2fad6d7ab7aafa34ffa52356432/public/king.png -------------------------------------------------------------------------------- /public/wbw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/wait-but-why-gpt/3002c615d84ad2fad6d7ab7aafa34ffa52356432/public/wbw.png -------------------------------------------------------------------------------- /schema.sql: -------------------------------------------------------------------------------- 1 | -- RUN 1st 2 | create extension vector; 3 | 4 | -- RUN 2nd 5 | create table wbw ( 6 | id bigserial primary key, 7 | post_title text, 8 | post_url text, 9 | post_date text, 10 | post_type text, 11 | content text, 12 | content_length bigint, 13 | content_tokens bigint, 14 | embedding vector (1536) 15 | ); 16 | 17 | -- RUN 3rd after running the scraping and embedding scripts 18 | create or replace function wbw_search ( 19 | query_embedding vector(1536), 20 | similarity_threshold float, 21 | match_count int 22 | ) 23 | returns table ( 24 | id bigint, 25 | post_title text, 26 | post_url text, 27 | post_date text, 28 | post_type text, 29 | content text, 30 | content_length bigint, 31 | content_tokens bigint, 32 | similarity float 33 | ) 34 | language plpgsql 35 | as $$ 36 | begin 37 | return query 38 | select 39 | wbw.id, 40 | wbw.post_title, 41 | wbw.post_url, 42 | wbw.post_date, 43 | wbw.post_type, 44 | wbw.content, 45 | wbw.content_length, 46 | wbw.content_tokens, 47 | 1 - (wbw.embedding <=> query_embedding) as similarity 48 | from wbw 49 | where 1 - (wbw.embedding <=> query_embedding) > similarity_threshold 50 | order by wbw.embedding <=> query_embedding 51 | limit match_count; 52 | end; 53 | $$; 54 | 55 | -- RUN 4th 56 | create index on wbw 57 | using ivfflat (embedding vector_cosine_ops) 58 | with (lists = 100); -------------------------------------------------------------------------------- /scripts/embed.ts: -------------------------------------------------------------------------------- 1 | import { loadEnvConfig } from "@next/env"; 2 | import { createClient } from "@supabase/supabase-js"; 3 | import fs from "fs"; 4 | import { Configuration, OpenAIApi } from "openai"; 5 | import { WBWJSON, WBWPost } from "./../types/index"; 6 | 7 | loadEnvConfig(""); 8 | 9 | const generateEmbeddings = async (posts: WBWPost[]) => { 10 | const configuration = new Configuration({ apiKey: process.env.OPENAI_API_KEY }); 11 | const openai = new OpenAIApi(configuration); 12 | 13 | const supabase = createClient(process.env.NEXT_PUBLIC_SUPABASE_URL!, process.env.SUPABASE_SERVICE_ROLE_KEY!); 14 | 15 | let chunkNum = 1; 16 | 17 | for (let i = 0; i < posts.length; i++) { 18 | const section = posts[i]; 19 | 20 | for (let j = 0; j < section.chunks.length; j++) { 21 | const chunk = section.chunks[j]; 22 | 23 | const { post_title, post_url, post_date, post_type, content, content_length, content_tokens } = chunk; 24 | 25 | const embeddingResponse = await openai.createEmbedding({ 26 | model: "text-embedding-ada-002", 27 | input: content 28 | }); 29 | 30 | if (embeddingResponse.status !== 200) { 31 | console.log("error", embeddingResponse); 32 | } else { 33 | const [{ embedding }] = embeddingResponse.data.data; 34 | 35 | const { data, error } = await supabase 36 | .from("wbw") 37 | .insert({ 38 | post_title, 39 | post_url, 40 | post_date, 41 | post_type, 42 | content, 43 | content_length, 44 | content_tokens, 45 | embedding 46 | }) 47 | .select("*"); 48 | 49 | if (error) { 50 | console.log("not saved", error); 51 | } else { 52 | console.log("saved", i + 1, j + 1, chunkNum); 53 | } 54 | } 55 | 56 | chunkNum++; 57 | 58 | await new Promise((resolve) => setTimeout(resolve, 100)); 59 | } 60 | } 61 | }; 62 | 63 | (async () => { 64 | const book: WBWJSON = JSON.parse(fs.readFileSync("scripts/wbw.json", "utf8")); 65 | 66 | await generateEmbeddings(book.posts); 67 | })(); 68 | -------------------------------------------------------------------------------- /scripts/images.json: -------------------------------------------------------------------------------- 1 | [{"title":"A Short History of My Last Six Years","src":"https://waitbutwhy.com/wp-content/uploads/2023/02/Feature-WBW-103x70.png"},{"title":"Mailbag #2","src":"https://waitbutwhy.com/wp-content/uploads/2016/08/FEATURE-103x70.png"},{"title":"The Trump-Biden Debate","src":"https://waitbutwhy.com/wp-content/uploads/2020/09/FEATURE-homepage-103x70.png"},{"title":"The Big and the Small","src":"https://waitbutwhy.com/wp-content/uploads/2020/09/FEATURE-site-103x70.png"},{"title":"You Won’t Believe My Morning","src":"https://waitbutwhy.com/wp-content/uploads/2020/03/Feature-103x70.jpg"},{"title":"It’s 2020 and you’re in the future","src":"https://waitbutwhy.com/wp-content/uploads/2020/01/FEATURE-site-103x70.png"},{"title":"How to Pick a Career (That Actually Fits You)","src":"https://waitbutwhy.com/wp-content/uploads/2018/04/career-feature-1-103x70.png"},{"title":"Neuralink and the Brain’s Magical Future","src":"https://waitbutwhy.com/wp-content/uploads/2018/04/FEATURE-103x70.png"},{"title":"100 Blocks a Day","src":"https://waitbutwhy.com/wp-content/uploads/2016/10/100-blocks-a-day-FEATURE-103x70.png"},{"title":"The Second Presidential Debate","src":"https://waitbutwhy.com/wp-content/uploads/2016/10/Feature-2-103x70.png"},{"title":"SpaceX’s Big Fucking Rocket – The Full Story...","src":"https://waitbutwhy.com/wp-content/uploads/2016/09/Feature-2-103x70.jpg"},{"title":"The Marriage Decision: Everything Forever or Nothi...","src":"https://waitbutwhy.com/wp-content/uploads/2016/09/MarriageDecision_Feature-103x70.jpg"},{"title":"Wait But Hi – Full Report","src":"https://waitbutwhy.com/wp-content/uploads/2016/08/WBH16_Report_Feature-103x70.png"},{"title":"Clueyness: A Weird Kind of Sad","src":"https://waitbutwhy.com/wp-content/uploads/2016/05/Clueyness-FEATURE-2-103x70.png"},{"title":"Mailbag #1","src":"https://waitbutwhy.com/wp-content/uploads/2016/08/FEATURE-103x70.png"},{"title":"Why Cryonics Makes Sense","src":"https://waitbutwhy.com/wp-content/uploads/2016/03/FEATURE-3-103x70.png"},{"title":"My TED Talk","src":"https://waitbutwhy.com/wp-content/uploads/2016/03/TED-Vid-FEATURE1-103x70.jpg"},{"title":"How I Handle Long Email Delays","src":"https://waitbutwhy.com/wp-content/uploads/2016/03/FEATURE-1-103x70.png"},{"title":"Everything You Should Know About Sound","src":"https://waitbutwhy.com/wp-content/uploads/2016/03/Feature-2-103x70.png"},{"title":"Doing a TED Talk: The Full Story","src":"https://waitbutwhy.com/wp-content/uploads/2016/03/FEATURE-103x70.png"},{"title":"Horizontal History","src":"https://waitbutwhy.com/wp-content/uploads/2016/01/Renaissance-F-1-103x70.jpg"},{"title":"The Tail End","src":"https://waitbutwhy.com/wp-content/uploads/2015/12/Tail-End-F-103x70.png"},{"title":"The Cook and the Chef: Musk’s Secret Sauce","src":"https://waitbutwhy.com/wp-content/uploads/2015/11/Line-of-cooks-FEATURE-103x70.jpg"},{"title":"How (and Why) SpaceX Will Colonize Mars","src":"https://waitbutwhy.com/wp-content/uploads/2015/08/SpaceX-F-103x70.jpg"},{"title":"Why I’m Always Late","src":"https://waitbutwhy.com/wp-content/uploads/2015/07/F-103x70.png"},{"title":"How Tesla Will Change The World","src":"https://waitbutwhy.com/wp-content/uploads/2015/06/F-103x70.png"},{"title":"Elon Musk: The World’s Raddest Man","src":"https://waitbutwhy.com/wp-content/uploads/2015/05/F-103x70.jpg"},{"title":"The Procrastination Matrix","src":"https://waitbutwhy.com/wp-content/uploads/2015/03/monkey-blog-F-103x70.png"},{"title":"7.3 Billion People, One Building","src":"https://waitbutwhy.com/wp-content/uploads/2015/03/Crowd-F-103x70.png"},{"title":"The American Presidents—Johnson to McKinley","src":"https://waitbutwhy.com/wp-content/uploads/2015/02/F-103x70.jpg"},{"title":"The AI Revolution: Our Immortality or Extinction","src":"https://waitbutwhy.com/wp-content/uploads/2015/01/Feature-103x70.jpg"},{"title":"The AI Revolution: The Road to Superintelligence","src":"https://waitbutwhy.com/wp-content/uploads/2015/01/G1-103x70.jpg"},{"title":"Our Most Popular Posts of 2014","src":"https://waitbutwhy.com/wp-content/uploads/2014/12/Top-10-of-2014-103x70.jpg"},{"title":"The Teen Years: 9 Cringe-Inducing Realizations","src":"https://waitbutwhy.com/wp-content/uploads/2014/12/Cruelty-Scale-FEATURE-103x70.png"},{"title":"What Makes You You?","src":"https://waitbutwhy.com/wp-content/uploads/2014/12/cubicle-beam-FEATURE-103x70.png"},{"title":"10 Types of Odd Friendships You’re Probably ...","src":"https://waitbutwhy.com/wp-content/uploads/2014/12/double-obligated-F-103x70.png"},{"title":"From 1,000,000 to Graham’s Number","src":"https://waitbutwhy.com/wp-content/uploads/2014/11/sun-tower-FEATURE-103x70.png"},{"title":"From 1 to 1,000,000","src":"https://waitbutwhy.com/wp-content/uploads/2014/11/FEATURE2-103x70.png"},{"title":"The Dark Secrets of the Bird World","src":"https://waitbutwhy.com/wp-content/uploads/2014/10/FEATURE-chicken-103x70.jpg"},{"title":"Religion for the Nonreligious","src":"https://waitbutwhy.com/wp-content/uploads/2014/10/chaotic-brain-FEATURE1-103x70.jpg"},{"title":"How Religion Got in the Way","src":"https://waitbutwhy.com/wp-content/uploads/2014/10/Atheism-FEATURE-103x70.png"},{"title":"Odd Things in Odd Places – All 7 Travel Post...","src":"https://waitbutwhy.com/wp-content/uploads/2014/10/Odd-Trips-Package-Feature1-103x70.png"},{"title":"The Genie Question","src":"https://waitbutwhy.com/wp-content/uploads/2014/09/Genie-FEATURE-103x70.jpg"},{"title":"But What About Greenland?","src":"https://waitbutwhy.com/wp-content/uploads/2014/09/Sad-Greenland-FEATURE-black-103x70.png"},{"title":"From Muhammad to ISIS: Iraq’s Full Story","src":"https://waitbutwhy.com/wp-content/uploads/2014/09/Scary-Map-FEATURE2-103x70.png"},{"title":"19 Things I Learned in Nigeria","src":"https://waitbutwhy.com/wp-content/uploads/2014/08/FEATURE-103x70.jpg"},{"title":"Japan, and How I Failed to Figure it Out","src":"https://waitbutwhy.com/wp-content/uploads/2014/07/Japan-Map-FEATURE-103x70.jpg"},{"title":"Russia: What You Didn’t Know You Don’t...","src":"https://waitbutwhy.com/wp-content/uploads/2014/07/FEATURE-103x70.jpg"},{"title":"Odd Things in Odd Places: Intro","src":"https://waitbutwhy.com/wp-content/uploads/2014/07/Odd-Trips-INTRO-FEATURE2-103x70.png"},{"title":"Taming the Mammoth: Why You Should Stop Caring Wha...","src":"https://waitbutwhy.com/wp-content/uploads/2014/06/Mammoth-FEATURE-103x70.png"},{"title":"Why You Secretly Hate Cool Bars","src":"https://waitbutwhy.com/wp-content/uploads/2014/05/Bar-line-no-rope-FEATURE-103x70.png"},{"title":"The Fermi Paradox","src":"https://waitbutwhy.com/wp-content/uploads/2014/05/Night-FEATURE-1-white-head-103x70.png"},{"title":"10 Absurdly Famous People You Probably Don’t...","src":"https://waitbutwhy.com/wp-content/uploads/2014/05/Venn-FEATURE-103x70.png"},{"title":"Your Life in Weeks","src":"https://waitbutwhy.com/wp-content/uploads/2014/05/Weeks-block-LIFE-FEATURE1-e1443808396297-103x70.png"},{"title":"Traveling To The Third World Is Great And Also It ...","src":"https://waitbutwhy.com/wp-content/uploads/2014/04/destroyed-honor-FEATURE-103x70.png"},{"title":"Everything You Don’t Know About Tipping","src":"https://waitbutwhy.com/wp-content/uploads/2014/04/overtip-3-FEATURE-103x70.png"},{"title":"What Could You Buy With $241 Trillion?","src":"https://waitbutwhy.com/wp-content/uploads/2014/03/pizza-FEATURE-103x70.jpg"},{"title":"Why Sports Fans are Sports Fans","src":"https://waitbutwhy.com/wp-content/uploads/2014/03/escape-FEATURE1-103x70.png"},{"title":"Why is My Laptop On?","src":"https://waitbutwhy.com/wp-content/uploads/2014/03/Electricity-FEATURE-103x70.png"},{"title":"The American Presidents—Washington to Lincoln","src":"https://waitbutwhy.com/wp-content/uploads/2014/02/artsy-FEATURE2-103x70.png"},{"title":"How to Pick Your Life Partner – Part 2","src":"https://waitbutwhy.com/wp-content/uploads/2014/02/lineup-FEATURE-103x70.png"},{"title":"How to Pick Your Life Partner – Part 1","src":"https://waitbutwhy.com/wp-content/uploads/2014/02/non-vday-2-FEATURE-103x70.jpg"},{"title":"Why Bugs Ruin Everything","src":"https://waitbutwhy.com/wp-content/uploads/2014/02/Outside-FEATURE-103x70.png"},{"title":"Your Family: Past, Present, and Future","src":"https://waitbutwhy.com/wp-content/uploads/2014/01/tree-128-FEATURE1-103x70.png"},{"title":"The Great Perils of Social Interaction","src":"https://waitbutwhy.com/wp-content/uploads/2014/01/handshake-FEATURE3-103x70.png"},{"title":"Wait But Why Holiday Post","src":"https://waitbutwhy.com/wp-content/uploads/2013/12/FEATURE7-103x70.png"},{"title":"Meet Your Ancestors (All of Them)","src":"https://waitbutwhy.com/wp-content/uploads/2013/12/FEATURED1-103x70.png"},{"title":"How to Name a Baby","src":"https://waitbutwhy.com/wp-content/uploads/2013/12/FEATURE5-103x70.png"},{"title":"11 Awkward Things About Email","src":"https://waitbutwhy.com/wp-content/uploads/2013/12/FEATURE4-103x70.png"},{"title":"Life is a Picture, But You Live in a Pixel","src":"https://waitbutwhy.com/wp-content/uploads/2013/11/FEATURE3-103x70.png"},{"title":"4 Mind-Blowing Things About Stars","src":"https://waitbutwhy.com/wp-content/uploads/2013/11/FEATURED-21-103x70.png"},{"title":"How to Beat Procrastination","src":"https://waitbutwhy.com/wp-content/uploads/2013/11/FEATURE5-103x70.png"},{"title":"Why Procrastinators Procrastinate","src":"https://waitbutwhy.com/wp-content/uploads/2013/10/FEATURE6-103x70.png"},{"title":"The Primate Awards","src":"https://waitbutwhy.com/wp-content/uploads/2013/10/FEATURE-FINAL-small-103x70.jpg"},{"title":"The Battle to Lose the Independent Vote","src":"https://waitbutwhy.com/wp-content/uploads/2013/10/FEATURE5-103x70.png"},{"title":"10 Types of 30-Year-Old Single Guys","src":"https://waitbutwhy.com/wp-content/uploads/2013/10/FEATURED2-103x70.png"},{"title":"What Does a Quadrillion Sour Patch Kids Look Like?...","src":"https://waitbutwhy.com/wp-content/uploads/2013/10/FEATURE4-103x70.png"},{"title":"20 Things I Learned While I Was in North Korea","src":"https://waitbutwhy.com/wp-content/uploads/2013/09/FEATURE-2-small-103x70.jpg"},{"title":"Why Generation Y Yuppies Are Unhappy","src":"https://waitbutwhy.com/wp-content/uploads/2013/09/FEATURED1-103x70.png"},{"title":"All the Weird Toys From Your Childhood","src":"https://waitbutwhy.com/wp-content/uploads/2013/09/FEATURE-multiple-103x70.png"},{"title":"The Apple Game: How Good a Person Are You?","src":"https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE131-103x70.png"},{"title":"7 Asinine Things About Society","src":"https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE12-103x70.png"},{"title":"Putting Time In Perspective – UPDATED","src":"https://waitbutwhy.com/wp-content/uploads/2013/08/Time-FEATURE-103x70.png"},{"title":"Creepy Kids in Creepy Vintage Ads","src":"https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE11-103x70.png"},{"title":"What If All 7.1 Billion People Moved To Tunisia?","src":"https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE10-103x70.png"},{"title":"The Bunny Manifesto","src":"https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE9-103x70.png"},{"title":"The Death Toll Comparison Breakdown","src":"https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE8-103x70.png"},{"title":"14 Shitty Sayings","src":"https://waitbutwhy.com/wp-content/uploads/2013/07/FEATURE13-103x70.png"},{"title":"12 Types Of People You’ll Find In Every Host...","src":"https://waitbutwhy.com/wp-content/uploads/2013/07/FEATURE12-103x70.png"},{"title":"God’s Wounded Ego","src":"https://waitbutwhy.com/wp-content/uploads/2013/07/FEATURE11-103x70.png"},{"title":"Medieval People In Bad Situations","src":"https://waitbutwhy.com/wp-content/uploads/2013/07/FEATURE-2-103x70.png"},{"title":"7 Ways to Be Insufferable on Facebook","src":"https://waitbutwhy.com/wp-content/uploads/2013/07/FEATURE9-103x70.png"},{"title":"Why Going to the Doctor Sucks","src":"https://waitbutwhy.com/wp-content/uploads/2021/04/lanby_feature-wp-782x530.png"},{"title":"Did James make the right Final Jeopardy bet?","src":"https://waitbutwhy.com/wp-content/uploads/2019/06/FEATURE-782x530.png"},{"title":"An Actual Thing That Actually Happened","src":"https://waitbutwhy.com/wp-content/uploads/2018/04/FEATURE-1-782x530.png"},{"title":"It’s Going to Be Okay – Follow Up","src":"https://waitbutwhy.com/wp-content/uploads/2016/11/Zig-Zag-presidents-FEATURE-small-782x530.png"},{"title":"It’s Going to Be Okay","src":"https://waitbutwhy.com/wp-content/uploads/2016/11/Feature-1-782x530.png"},{"title":"Results: WBW Election Survey","src":"https://waitbutwhy.com/wp-content/uploads/2016/11/Results-Feature-782x530.jpg"},{"title":"Mini post: Oceans and Clay","src":"https://waitbutwhy.com/wp-content/uploads/2016/09/Ocean-2-FEATURE-782x530.jpg"},{"title":"Wait But Hi","src":"https://waitbutwhy.com/wp-content/uploads/2016/07/Survey-Feature-782x530.jpg"},{"title":"The Confusing Triangle Situation","src":"https://waitbutwhy.com/wp-content/uploads/2016/06/Triangle-FB-782x530.png"},{"title":"Everything I Ate Last Week","src":"https://waitbutwhy.com/wp-content/uploads/2016/06/Grub-Street-FEATURE-782x530.png"},{"title":"The Puzzle Of The Pirate Booty","src":"https://waitbutwhy.com/wp-content/uploads/2016/06/Pirate-Puzzle-FEATURE-782x530.png"},{"title":"Myers-Briggs: How WBW readers compare to the general population","src":"https://waitbutwhy.com/wp-content/uploads/2016/05/MBTI-Feature-782x530.png"},{"title":"Why I Should Never Drink a Full Cup of Starbucks Coffee","src":"https://waitbutwhy.com/wp-content/uploads/2016/03/FEATURE-2.png"},{"title":"The Jelly Bean Problem","src":"https://waitbutwhy.com/wp-content/uploads/2016/03/Feature-smaller-stump-1-782x530.jpg"},{"title":"You’re in the Future: 2016 Edition","src":"https://waitbutwhy.com/wp-content/uploads/2016/01/Feature.jpg"},{"title":"SpaceX Launch Live Webcast and Explanation (12.21.15)","src":"https://waitbutwhy.com/wp-content/uploads/2015/12/arc-782x530.jpg"},{"title":"Birthdays Are Weird","src":"https://waitbutwhy.com/wp-content/uploads/2015/11/Birthday-F1-782x530.jpg"},{"title":"The SpaceX Post Progress Meter","src":"https://waitbutwhy.com/wp-content/uploads/2015/07/Untitled-4-782x530.png"},{"title":"Three Fascinating Videos With Live Footage of Thomas Edison and Henry Ford","src":"https://waitbutwhy.com/wp-content/uploads/2015/06/Screen-Shot-2015-06-20-at-6.31.41-PM-782x530.jpg"},{"title":"The Deal With the Hyperloop","src":"https://waitbutwhy.com/wp-content/uploads/2015/06/F1-782x530.png"},{"title":"The Deal With Solar","src":"https://waitbutwhy.com/wp-content/uploads/2014/05/Global_energy_potential_perez_2009_en.svg_-782x530.png"},{"title":"Putting Floyd Mayweather’s $210 Million Payout in Perspective","src":"https://waitbutwhy.com/wp-content/uploads/2015/05/Floyd-Mayweather-Chart-F-782x530.png"},{"title":"Trying Something Different With the Schedule","src":"https://waitbutwhy.com/wp-content/uploads/2015/03/Untitled-4.png"},{"title":"The Most Depressing Buzzfeed Article of All Time","src":"https://waitbutwhy.com/wp-content/uploads/2015/01/Depressing-Buzzfeed-Article-782x530.jpg"},{"title":"It’s 2015, and You’re in the Future","src":"https://waitbutwhy.com/wp-content/uploads/2015/01/2015-Feature-782x530.jpg"},{"title":"Where Wait But Why Gets Its Traffic","src":"https://waitbutwhy.com/wp-content/uploads/2014/11/WBW-Referral-Traffic-All-engaged-FEATURE-782x530.png"},{"title":"Introducing the Dinner Table","src":"https://waitbutwhy.com/wp-content/uploads/2014/11/Dinner-Table-Logo-FEATURE-782x530.png"},{"title":"Why I Can’t Post On Time","src":"https://waitbutwhy.com/wp-content/uploads/2014/10/Post-On-Time-FEATURE1-782x530.png"},{"title":"How the Panama Canal Works","src":"https://waitbutwhy.com/wp-content/uploads/2014/09/Panama-FEATURE-782x530.png"},{"title":"Your Life is Worse When You Know About Dust Mites","src":"https://waitbutwhy.com/wp-content/uploads/2014/08/Mites-FEATURE-782x530.jpg"},{"title":"What Makes a Face Trustworthy?","src":"https://waitbutwhy.com/wp-content/uploads/2014/08/Trustworthy-Faces-FEATURE1-782x530.png"},{"title":"Happy Birthday Wait But Why","src":"https://waitbutwhy.com/wp-content/uploads/2014/07/King-bday-782x530.png"},{"title":"If Andromeda Were Brighter, This is What You’d See","src":"https://waitbutwhy.com/wp-content/uploads/2014/06/Andromeda-FEATURE.png"},{"title":"The Titanic Compared With a Modern Cruiseship","src":"https://waitbutwhy.com/wp-content/uploads/2014/05/Titanic-FEATURE.png"},{"title":"Stick Figure Puzzle","src":"https://waitbutwhy.com/wp-content/uploads/2014/04/two-heads-FEATURE.png"},{"title":"Energy for Dummies","src":"https://waitbutwhy.com/wp-content/uploads/2014/03/2012new2012newUSEnergy-782x530.png"},{"title":"Crazy Fact About Population Density","src":"https://waitbutwhy.com/wp-content/uploads/2014/01/Shed-Thumb-782x530.png"},{"title":"200 People’s New Year’s Resolutions","src":"https://waitbutwhy.com/wp-content/uploads/2013/12/FEATURE6-782x530.png"},{"title":"Putting All the World’s Water into a Big Cube","src":"https://waitbutwhy.com/wp-content/uploads/2013/09/FEATURE-tiny-782x530.jpg"},{"title":"A-List Stars In Tiny Roles","src":"https://waitbutwhy.com/wp-content/uploads/2013/08/A-Listers-FEATURE-782x530.png"}] -------------------------------------------------------------------------------- /scripts/images.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import * as cheerio from "cheerio"; 3 | import fs from "fs"; 4 | 5 | const BASE_URL = "https://waitbutwhy.com"; 6 | 7 | const ARCHIVE_PAGE_1 = "/archive"; 8 | const ARCHIVE_PAGE_2 = "/archive/page/2"; 9 | const ARCHIVE_CLASS = ".post-list"; 10 | 11 | const MINIS_PAGE_1 = "/minis"; 12 | const MINIS_PAGE_2 = "/minis/page/2"; 13 | 14 | const getArchiveImages = async (page: string) => { 15 | const fullLink = BASE_URL + page; 16 | const html = await axios.get(fullLink); 17 | const $ = cheerio.load(html.data); 18 | const list = $(ARCHIVE_CLASS); 19 | const firstList = list.first(); 20 | const li = firstList.find("li"); 21 | 22 | let images: { title: string; src: string }[] = []; 23 | 24 | li.each((i, li) => { 25 | const title = $(li).find("h5").text().trim(); 26 | const src = $(li).find("img").attr("src"); 27 | 28 | if (title && src) { 29 | images.push({ title, src }); 30 | } 31 | }); 32 | 33 | return images; 34 | }; 35 | 36 | const getMinisImages = async (page: string) => { 37 | const fullLink = BASE_URL + page; 38 | const html = await axios.get(fullLink); 39 | const $ = cheerio.load(html.data); 40 | const oneHalf = $(".one-half"); 41 | 42 | let images: { title: string; src: string }[] = []; 43 | 44 | oneHalf.each((i, el) => { 45 | const title = $(el).find("h3").text().trim(); 46 | const src = $(el).find("img").attr("src"); 47 | 48 | if (title && src) { 49 | images.push({ title, src }); 50 | } 51 | }); 52 | 53 | return images; 54 | }; 55 | 56 | (async () => { 57 | let images: { title: string; src: string }[] = []; 58 | 59 | const archivePage1Images = await getArchiveImages(ARCHIVE_PAGE_1); 60 | const archivePage2Images = await getArchiveImages(ARCHIVE_PAGE_2); 61 | 62 | const minisPage1Images = await getMinisImages(MINIS_PAGE_1); 63 | const minisPage2Images = await getMinisImages(MINIS_PAGE_2); 64 | 65 | images = [...archivePage1Images, ...archivePage2Images, ...minisPage1Images, ...minisPage2Images]; 66 | 67 | console.log(images); 68 | console.log(images.length); 69 | 70 | fs.writeFileSync("scripts/images.json", JSON.stringify(images)); 71 | })(); 72 | -------------------------------------------------------------------------------- /scripts/scrape.ts: -------------------------------------------------------------------------------- 1 | import { WBWChunk, WBWJSON, WBWPost } from "@/types"; 2 | import axios from "axios"; 3 | import * as cheerio from "cheerio"; 4 | import fs from "fs"; 5 | import { encode } from "gpt-3-encoder"; 6 | 7 | const BASE_URL = "https://waitbutwhy.com"; 8 | 9 | const ARCHIVE_PAGE_1 = "/archive"; 10 | const ARCHIVE_PAGE_2 = "/archive/page/2"; 11 | const ARCHIVE_CLASS = ".archive-page"; 12 | 13 | const MINIS_PAGE_1 = "/minis"; 14 | const MINIS_PAGE_2 = "/minis/page/2"; 15 | const MINIS_CLASS = ".archive-postlist"; 16 | 17 | const CHUNK_SIZE = 200; 18 | 19 | const getLinks = async (page: string, className: string) => { 20 | const fullLink = BASE_URL + page; 21 | const html = await axios.get(fullLink); 22 | const $ = cheerio.load(html.data); 23 | const list = $(className); 24 | 25 | const links: string[] = []; 26 | 27 | list.find("a").map((i, link) => { 28 | const href = $(link).attr("href"); 29 | 30 | if (href) { 31 | links.push(href); 32 | } 33 | }); 34 | 35 | const filteredLinks = [...new Set(links.filter((link) => link.endsWith(".html")))]; 36 | 37 | return filteredLinks; 38 | }; 39 | 40 | const getPost = async (url: string, type: "post" | "mini") => { 41 | let post: WBWPost = { 42 | title: "", 43 | url: "", 44 | date: "", 45 | type, 46 | content: "", 47 | length: 0, 48 | tokens: 0, 49 | chunks: [] 50 | }; 51 | 52 | const html = await axios.get(url); 53 | const $ = cheerio.load(html.data); 54 | 55 | const title = $("header h1").text().trim(); 56 | const date = $("header .date").text().trim(); 57 | const text = $(".entry-content #pico").text().trim(); 58 | 59 | let cleanedText = text.replace(/\s+/g, " "); 60 | cleanedText = cleanedText.replace(/\.([a-zA-Z])/g, ". $1"); 61 | 62 | const trimmedContent = cleanedText.trim(); 63 | 64 | post = { 65 | title, 66 | url, 67 | date, 68 | type, 69 | content: trimmedContent, 70 | length: trimmedContent.length, 71 | tokens: encode(trimmedContent).length, 72 | chunks: [] 73 | }; 74 | 75 | return post; 76 | }; 77 | 78 | const chunkPost = async (post: WBWPost) => { 79 | const { title, url, date, content } = post; 80 | 81 | let postTextChunks = []; 82 | 83 | if (encode(content).length > CHUNK_SIZE) { 84 | const split = content.split(". "); 85 | let chunkText = ""; 86 | 87 | for (let i = 0; i < split.length; i++) { 88 | const sentence = split[i]; 89 | const sentenceTokenLength = encode(sentence); 90 | const chunkTextTokenLength = encode(chunkText).length; 91 | 92 | if (chunkTextTokenLength + sentenceTokenLength.length > CHUNK_SIZE) { 93 | postTextChunks.push(chunkText); 94 | chunkText = ""; 95 | } 96 | 97 | if (sentence[sentence.length - 1] && sentence[sentence.length - 1].match(/[a-z0-9]/i)) { 98 | chunkText += sentence + ". "; 99 | } else { 100 | chunkText += sentence + " "; 101 | } 102 | } 103 | 104 | postTextChunks.push(chunkText.trim()); 105 | } else { 106 | postTextChunks.push(content.trim()); 107 | } 108 | 109 | const postChunks = postTextChunks.map((text) => { 110 | const trimmedText = text.trim(); 111 | 112 | const chunk: WBWChunk = { 113 | post_title: title, 114 | post_url: url, 115 | post_date: date, 116 | post_type: post.type, 117 | content: trimmedText, 118 | content_length: trimmedText.length, 119 | content_tokens: encode(trimmedText).length, 120 | embedding: [] 121 | }; 122 | 123 | return chunk; 124 | }); 125 | 126 | if (postChunks.length > 1) { 127 | for (let i = 0; i < postChunks.length; i++) { 128 | const chunk = postChunks[i]; 129 | const prevChunk = postChunks[i - 1]; 130 | 131 | if (chunk.content_tokens < 100 && prevChunk) { 132 | prevChunk.content += " " + chunk.content; 133 | prevChunk.content_length += chunk.content_length; 134 | prevChunk.content_tokens += chunk.content_tokens; 135 | postChunks.splice(i, 1); 136 | i--; 137 | } 138 | } 139 | } 140 | 141 | const chunkedSection: WBWPost = { 142 | ...post, 143 | chunks: postChunks 144 | }; 145 | 146 | return chunkedSection; 147 | }; 148 | 149 | (async () => { 150 | const archivePage1Links = await getLinks(ARCHIVE_PAGE_1, ARCHIVE_CLASS); 151 | const archivePage2Links = await getLinks(ARCHIVE_PAGE_2, ARCHIVE_CLASS); 152 | const archiveLinks = [...archivePage1Links, ...archivePage2Links]; 153 | 154 | let posts = []; 155 | 156 | for (let i = 0; i < archiveLinks.length; i++) { 157 | const link = archiveLinks[i]; 158 | const post = await getPost(link, "post"); 159 | const chunkedPost = await chunkPost(post); 160 | 161 | posts.push(chunkedPost); 162 | } 163 | 164 | const minisPage1Links = await getLinks(MINIS_PAGE_1, MINIS_CLASS); 165 | const minisPage2Links = await getLinks(MINIS_PAGE_2, MINIS_CLASS); 166 | const minisLinks = [...minisPage1Links, ...minisPage2Links]; 167 | 168 | for (let i = 0; i < minisLinks.length; i++) { 169 | const link = minisLinks[i]; 170 | const post = await getPost(link, "mini"); 171 | const chunkedPost = await chunkPost(post); 172 | 173 | posts.push(chunkedPost); 174 | } 175 | 176 | const todayDate = new Date().toISOString().split("T")[0]; 177 | 178 | const json: WBWJSON = { 179 | current_date: todayDate, 180 | author: "Tim Urban", 181 | url: BASE_URL, 182 | length: posts.reduce((acc, essay) => acc + essay.length, 0), 183 | tokens: posts.reduce((acc, essay) => acc + essay.tokens, 0), 184 | posts 185 | }; 186 | 187 | fs.writeFileSync("scripts/wbw.json", JSON.stringify(json)); 188 | })(); 189 | -------------------------------------------------------------------------------- /styles/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: ["./app/**/*.{js,ts,jsx,tsx}", "./pages/**/*.{js,ts,jsx,tsx}", "./components/**/*.{js,ts,jsx,tsx}"], 4 | theme: { 5 | extend: {} 6 | }, 7 | plugins: [] 8 | }; 9 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2015", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "noEmit": true, 10 | "esModuleInterop": true, 11 | "module": "esnext", 12 | "moduleResolution": "node", 13 | "resolveJsonModule": true, 14 | "isolatedModules": true, 15 | "jsx": "preserve", 16 | "incremental": true, 17 | "baseUrl": ".", 18 | "paths": { 19 | "@/*": ["./*"] 20 | } 21 | }, 22 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], 23 | "exclude": ["node_modules"] 24 | } 25 | -------------------------------------------------------------------------------- /types/index.ts: -------------------------------------------------------------------------------- 1 | export enum OpenAIModel { 2 | DAVINCI_TURBO = "gpt-3.5-turbo" 3 | } 4 | 5 | export type WBWPost = { 6 | title: string; 7 | url: string; 8 | date: string; 9 | type: "post" | "mini"; 10 | content: string; 11 | length: number; 12 | tokens: number; 13 | chunks: WBWChunk[]; 14 | }; 15 | 16 | export type WBWChunk = { 17 | post_title: string; 18 | post_url: string; 19 | post_date: string | undefined; 20 | post_type: "post" | "mini"; 21 | content: string; 22 | content_length: number; 23 | content_tokens: number; 24 | embedding: number[]; 25 | }; 26 | 27 | export type WBWJSON = { 28 | current_date: string; 29 | author: string; 30 | url: string; 31 | length: number; 32 | tokens: number; 33 | posts: WBWPost[]; 34 | }; 35 | -------------------------------------------------------------------------------- /utils/images.ts: -------------------------------------------------------------------------------- 1 | export const getImage = (title: string) => { 2 | const images = [ 3 | { title: "A Short History of My Last Six Years", src: "https://waitbutwhy.com/wp-content/uploads/2023/02/Feature-WBW-103x70.png" }, 4 | { title: "Mailbag #2", src: "https://waitbutwhy.com/wp-content/uploads/2016/08/FEATURE-103x70.png" }, 5 | { title: "The Trump-Biden Debate", src: "https://waitbutwhy.com/wp-content/uploads/2020/09/FEATURE-homepage-103x70.png" }, 6 | { title: "The Big and the Small", src: "https://waitbutwhy.com/wp-content/uploads/2020/09/FEATURE-site-103x70.png" }, 7 | { title: "You Won’t Believe My Morning", src: "https://waitbutwhy.com/wp-content/uploads/2020/03/Feature-103x70.jpg" }, 8 | { title: "It’s 2020 and you’re in the future", src: "https://waitbutwhy.com/wp-content/uploads/2020/01/FEATURE-site-103x70.png" }, 9 | { title: "How to Pick a Career (That Actually Fits You)", src: "https://waitbutwhy.com/wp-content/uploads/2018/04/career-feature-1-103x70.png" }, 10 | { title: "Neuralink and the Brain’s Magical Future", src: "https://waitbutwhy.com/wp-content/uploads/2018/04/FEATURE-103x70.png" }, 11 | { title: "100 Blocks a Day", src: "https://waitbutwhy.com/wp-content/uploads/2016/10/100-blocks-a-day-FEATURE-103x70.png" }, 12 | { title: "The Second Presidential Debate", src: "https://waitbutwhy.com/wp-content/uploads/2016/10/Feature-2-103x70.png" }, 13 | { title: "SpaceX’s Big Fucking Rocket – The Full Story...", src: "https://waitbutwhy.com/wp-content/uploads/2016/09/Feature-2-103x70.jpg" }, 14 | { title: "The Marriage Decision: Everything Forever or Nothi...", src: "https://waitbutwhy.com/wp-content/uploads/2016/09/MarriageDecision_Feature-103x70.jpg" }, 15 | { title: "Wait But Hi – Full Report", src: "https://waitbutwhy.com/wp-content/uploads/2016/08/WBH16_Report_Feature-103x70.png" }, 16 | { title: "Clueyness: A Weird Kind of Sad", src: "https://waitbutwhy.com/wp-content/uploads/2016/05/Clueyness-FEATURE-2-103x70.png" }, 17 | { title: "Mailbag #1", src: "https://waitbutwhy.com/wp-content/uploads/2016/08/FEATURE-103x70.png" }, 18 | { title: "Why Cryonics Makes Sense", src: "https://waitbutwhy.com/wp-content/uploads/2016/03/FEATURE-3-103x70.png" }, 19 | { title: "My TED Talk", src: "https://waitbutwhy.com/wp-content/uploads/2016/03/TED-Vid-FEATURE1-103x70.jpg" }, 20 | { title: "How I Handle Long Email Delays", src: "https://waitbutwhy.com/wp-content/uploads/2016/03/FEATURE-1-103x70.png" }, 21 | { title: "Everything You Should Know About Sound", src: "https://waitbutwhy.com/wp-content/uploads/2016/03/Feature-2-103x70.png" }, 22 | { title: "Doing a TED Talk: The Full Story", src: "https://waitbutwhy.com/wp-content/uploads/2016/03/FEATURE-103x70.png" }, 23 | { title: "Horizontal History", src: "https://waitbutwhy.com/wp-content/uploads/2016/01/Renaissance-F-1-103x70.jpg" }, 24 | { title: "The Tail End", src: "https://waitbutwhy.com/wp-content/uploads/2015/12/Tail-End-F-103x70.png" }, 25 | { title: "The Cook and the Chef: Musk’s Secret Sauce", src: "https://waitbutwhy.com/wp-content/uploads/2015/11/Line-of-cooks-FEATURE-103x70.jpg" }, 26 | { title: "How (and Why) SpaceX Will Colonize Mars", src: "https://waitbutwhy.com/wp-content/uploads/2015/08/SpaceX-F-103x70.jpg" }, 27 | { title: "Why I’m Always Late", src: "https://waitbutwhy.com/wp-content/uploads/2015/07/F-103x70.png" }, 28 | { title: "How Tesla Will Change The World", src: "https://waitbutwhy.com/wp-content/uploads/2015/06/F-103x70.png" }, 29 | { title: "Elon Musk: The World’s Raddest Man", src: "https://waitbutwhy.com/wp-content/uploads/2015/05/F-103x70.jpg" }, 30 | { title: "The Procrastination Matrix", src: "https://waitbutwhy.com/wp-content/uploads/2015/03/monkey-blog-F-103x70.png" }, 31 | { title: "7.3 Billion People, One Building", src: "https://waitbutwhy.com/wp-content/uploads/2015/03/Crowd-F-103x70.png" }, 32 | { title: "The American Presidents—Johnson to McKinley", src: "https://waitbutwhy.com/wp-content/uploads/2015/02/F-103x70.jpg" }, 33 | { title: "The AI Revolution: Our Immortality or Extinction", src: "https://waitbutwhy.com/wp-content/uploads/2015/01/Feature-103x70.jpg" }, 34 | { title: "The AI Revolution: The Road to Superintelligence", src: "https://waitbutwhy.com/wp-content/uploads/2015/01/G1-103x70.jpg" }, 35 | { title: "Our Most Popular Posts of 2014", src: "https://waitbutwhy.com/wp-content/uploads/2014/12/Top-10-of-2014-103x70.jpg" }, 36 | { title: "The Teen Years: 9 Cringe-Inducing Realizations", src: "https://waitbutwhy.com/wp-content/uploads/2014/12/Cruelty-Scale-FEATURE-103x70.png" }, 37 | { title: "What Makes You You?", src: "https://waitbutwhy.com/wp-content/uploads/2014/12/cubicle-beam-FEATURE-103x70.png" }, 38 | { title: "10 Types of Odd Friendships You’re Probably ...", src: "https://waitbutwhy.com/wp-content/uploads/2014/12/double-obligated-F-103x70.png" }, 39 | { title: "From 1,000,000 to Graham’s Number", src: "https://waitbutwhy.com/wp-content/uploads/2014/11/sun-tower-FEATURE-103x70.png" }, 40 | { title: "From 1 to 1,000,000", src: "https://waitbutwhy.com/wp-content/uploads/2014/11/FEATURE2-103x70.png" }, 41 | { title: "The Dark Secrets of the Bird World", src: "https://waitbutwhy.com/wp-content/uploads/2014/10/FEATURE-chicken-103x70.jpg" }, 42 | { title: "Religion for the Nonreligious", src: "https://waitbutwhy.com/wp-content/uploads/2014/10/chaotic-brain-FEATURE1-103x70.jpg" }, 43 | { title: "How Religion Got in the Way", src: "https://waitbutwhy.com/wp-content/uploads/2014/10/Atheism-FEATURE-103x70.png" }, 44 | { title: "Odd Things in Odd Places – All 7 Travel Post...", src: "https://waitbutwhy.com/wp-content/uploads/2014/10/Odd-Trips-Package-Feature1-103x70.png" }, 45 | { title: "The Genie Question", src: "https://waitbutwhy.com/wp-content/uploads/2014/09/Genie-FEATURE-103x70.jpg" }, 46 | { title: "But What About Greenland?", src: "https://waitbutwhy.com/wp-content/uploads/2014/09/Sad-Greenland-FEATURE-black-103x70.png" }, 47 | { title: "From Muhammad to ISIS: Iraq’s Full Story", src: "https://waitbutwhy.com/wp-content/uploads/2014/09/Scary-Map-FEATURE2-103x70.png" }, 48 | { title: "19 Things I Learned in Nigeria", src: "https://waitbutwhy.com/wp-content/uploads/2014/08/FEATURE-103x70.jpg" }, 49 | { title: "Japan, and How I Failed to Figure it Out", src: "https://waitbutwhy.com/wp-content/uploads/2014/07/Japan-Map-FEATURE-103x70.jpg" }, 50 | { title: "Russia: What You Didn’t Know You Don’t...", src: "https://waitbutwhy.com/wp-content/uploads/2014/07/FEATURE-103x70.jpg" }, 51 | { title: "Odd Things in Odd Places: Intro", src: "https://waitbutwhy.com/wp-content/uploads/2014/07/Odd-Trips-INTRO-FEATURE2-103x70.png" }, 52 | { title: "Taming the Mammoth: Why You Should Stop Caring Wha...", src: "https://waitbutwhy.com/wp-content/uploads/2014/06/Mammoth-FEATURE-103x70.png" }, 53 | { title: "Why You Secretly Hate Cool Bars", src: "https://waitbutwhy.com/wp-content/uploads/2014/05/Bar-line-no-rope-FEATURE-103x70.png" }, 54 | { title: "The Fermi Paradox", src: "https://waitbutwhy.com/wp-content/uploads/2014/05/Night-FEATURE-1-white-head-103x70.png" }, 55 | { title: "10 Absurdly Famous People You Probably Don’t...", src: "https://waitbutwhy.com/wp-content/uploads/2014/05/Venn-FEATURE-103x70.png" }, 56 | { title: "Your Life in Weeks", src: "https://waitbutwhy.com/wp-content/uploads/2014/05/Weeks-block-LIFE-FEATURE1-e1443808396297-103x70.png" }, 57 | { title: "Traveling To The Third World Is Great And Also It ...", src: "https://waitbutwhy.com/wp-content/uploads/2014/04/destroyed-honor-FEATURE-103x70.png" }, 58 | { title: "Everything You Don’t Know About Tipping", src: "https://waitbutwhy.com/wp-content/uploads/2014/04/overtip-3-FEATURE-103x70.png" }, 59 | { title: "What Could You Buy With $241 Trillion?", src: "https://waitbutwhy.com/wp-content/uploads/2014/03/pizza-FEATURE-103x70.jpg" }, 60 | { title: "Why Sports Fans are Sports Fans", src: "https://waitbutwhy.com/wp-content/uploads/2014/03/escape-FEATURE1-103x70.png" }, 61 | { title: "Why is My Laptop On?", src: "https://waitbutwhy.com/wp-content/uploads/2014/03/Electricity-FEATURE-103x70.png" }, 62 | { title: "The American Presidents—Washington to Lincoln", src: "https://waitbutwhy.com/wp-content/uploads/2014/02/artsy-FEATURE2-103x70.png" }, 63 | { title: "How to Pick Your Life Partner – Part 2", src: "https://waitbutwhy.com/wp-content/uploads/2014/02/lineup-FEATURE-103x70.png" }, 64 | { title: "How to Pick Your Life Partner – Part 1", src: "https://waitbutwhy.com/wp-content/uploads/2014/02/non-vday-2-FEATURE-103x70.jpg" }, 65 | { title: "Why Bugs Ruin Everything", src: "https://waitbutwhy.com/wp-content/uploads/2014/02/Outside-FEATURE-103x70.png" }, 66 | { title: "Your Family: Past, Present, and Future", src: "https://waitbutwhy.com/wp-content/uploads/2014/01/tree-128-FEATURE1-103x70.png" }, 67 | { title: "The Great Perils of Social Interaction", src: "https://waitbutwhy.com/wp-content/uploads/2014/01/handshake-FEATURE3-103x70.png" }, 68 | { title: "Wait But Why Holiday Post", src: "https://waitbutwhy.com/wp-content/uploads/2013/12/FEATURE7-103x70.png" }, 69 | { title: "Meet Your Ancestors (All of Them)", src: "https://waitbutwhy.com/wp-content/uploads/2013/12/FEATURED1-103x70.png" }, 70 | { title: "How to Name a Baby", src: "https://waitbutwhy.com/wp-content/uploads/2013/12/FEATURE5-103x70.png" }, 71 | { title: "11 Awkward Things About Email", src: "https://waitbutwhy.com/wp-content/uploads/2013/12/FEATURE4-103x70.png" }, 72 | { title: "Life is a Picture, But You Live in a Pixel", src: "https://waitbutwhy.com/wp-content/uploads/2013/11/FEATURE3-103x70.png" }, 73 | { title: "4 Mind-Blowing Things About Stars", src: "https://waitbutwhy.com/wp-content/uploads/2013/11/FEATURED-21-103x70.png" }, 74 | { title: "How to Beat Procrastination", src: "https://waitbutwhy.com/wp-content/uploads/2013/11/FEATURE5-103x70.png" }, 75 | { title: "Why Procrastinators Procrastinate", src: "https://waitbutwhy.com/wp-content/uploads/2013/10/FEATURE6-103x70.png" }, 76 | { title: "The Primate Awards", src: "https://waitbutwhy.com/wp-content/uploads/2013/10/FEATURE-FINAL-small-103x70.jpg" }, 77 | { title: "The Battle to Lose the Independent Vote", src: "https://waitbutwhy.com/wp-content/uploads/2013/10/FEATURE5-103x70.png" }, 78 | { title: "10 Types of 30-Year-Old Single Guys", src: "https://waitbutwhy.com/wp-content/uploads/2013/10/FEATURED2-103x70.png" }, 79 | { title: "What Does a Quadrillion Sour Patch Kids Look Like?...", src: "https://waitbutwhy.com/wp-content/uploads/2013/10/FEATURE4-103x70.png" }, 80 | { title: "20 Things I Learned While I Was in North Korea", src: "https://waitbutwhy.com/wp-content/uploads/2013/09/FEATURE-2-small-103x70.jpg" }, 81 | { title: "Why Generation Y Yuppies Are Unhappy", src: "https://waitbutwhy.com/wp-content/uploads/2013/09/FEATURED1-103x70.png" }, 82 | { title: "All the Weird Toys From Your Childhood", src: "https://waitbutwhy.com/wp-content/uploads/2013/09/FEATURE-multiple-103x70.png" }, 83 | { title: "The Apple Game: How Good a Person Are You?", src: "https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE131-103x70.png" }, 84 | { title: "7 Asinine Things About Society", src: "https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE12-103x70.png" }, 85 | { title: "Putting Time In Perspective – UPDATED", src: "https://waitbutwhy.com/wp-content/uploads/2013/08/Time-FEATURE-103x70.png" }, 86 | { title: "Creepy Kids in Creepy Vintage Ads", src: "https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE11-103x70.png" }, 87 | { title: "What If All 7.1 Billion People Moved To Tunisia?", src: "https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE10-103x70.png" }, 88 | { title: "The Bunny Manifesto", src: "https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE9-103x70.png" }, 89 | { title: "The Death Toll Comparison Breakdown", src: "https://waitbutwhy.com/wp-content/uploads/2013/08/FEATURE8-103x70.png" }, 90 | { title: "14 Shitty Sayings", src: "https://waitbutwhy.com/wp-content/uploads/2013/07/FEATURE13-103x70.png" }, 91 | { title: "12 Types Of People You’ll Find In Every Host...", src: "https://waitbutwhy.com/wp-content/uploads/2013/07/FEATURE12-103x70.png" }, 92 | { title: "God’s Wounded Ego", src: "https://waitbutwhy.com/wp-content/uploads/2013/07/FEATURE11-103x70.png" }, 93 | { title: "Medieval People In Bad Situations", src: "https://waitbutwhy.com/wp-content/uploads/2013/07/FEATURE-2-103x70.png" }, 94 | { title: "7 Ways to Be Insufferable on Facebook", src: "https://waitbutwhy.com/wp-content/uploads/2013/07/FEATURE9-103x70.png" }, 95 | { title: "Why Going to the Doctor Sucks", src: "https://waitbutwhy.com/wp-content/uploads/2021/04/lanby_feature-wp-782x530.png" }, 96 | { title: "Did James make the right Final Jeopardy bet?", src: "https://waitbutwhy.com/wp-content/uploads/2019/06/FEATURE-782x530.png" }, 97 | { title: "An Actual Thing That Actually Happened", src: "https://waitbutwhy.com/wp-content/uploads/2018/04/FEATURE-1-782x530.png" }, 98 | { title: "It’s Going to Be Okay – Follow Up", src: "https://waitbutwhy.com/wp-content/uploads/2016/11/Zig-Zag-presidents-FEATURE-small-782x530.png" }, 99 | { title: "It’s Going to Be Okay", src: "https://waitbutwhy.com/wp-content/uploads/2016/11/Feature-1-782x530.png" }, 100 | { title: "Results: WBW Election Survey", src: "https://waitbutwhy.com/wp-content/uploads/2016/11/Results-Feature-782x530.jpg" }, 101 | { title: "Mini post: Oceans and Clay", src: "https://waitbutwhy.com/wp-content/uploads/2016/09/Ocean-2-FEATURE-782x530.jpg" }, 102 | { title: "Wait But Hi", src: "https://waitbutwhy.com/wp-content/uploads/2016/07/Survey-Feature-782x530.jpg" }, 103 | { title: "The Confusing Triangle Situation", src: "https://waitbutwhy.com/wp-content/uploads/2016/06/Triangle-FB-782x530.png" }, 104 | { title: "Everything I Ate Last Week", src: "https://waitbutwhy.com/wp-content/uploads/2016/06/Grub-Street-FEATURE-782x530.png" }, 105 | { title: "The Puzzle Of The Pirate Booty", src: "https://waitbutwhy.com/wp-content/uploads/2016/06/Pirate-Puzzle-FEATURE-782x530.png" }, 106 | { title: "Myers-Briggs: How WBW readers compare to the general population", src: "https://waitbutwhy.com/wp-content/uploads/2016/05/MBTI-Feature-782x530.png" }, 107 | { title: "Why I Should Never Drink a Full Cup of Starbucks Coffee", src: "https://waitbutwhy.com/wp-content/uploads/2016/03/FEATURE-2.png" }, 108 | { title: "The Jelly Bean Problem", src: "https://waitbutwhy.com/wp-content/uploads/2016/03/Feature-smaller-stump-1-782x530.jpg" }, 109 | { title: "You’re in the Future: 2016 Edition", src: "https://waitbutwhy.com/wp-content/uploads/2016/01/Feature.jpg" }, 110 | { title: "SpaceX Launch Live Webcast and Explanation (12.21.15)", src: "https://waitbutwhy.com/wp-content/uploads/2015/12/arc-782x530.jpg" }, 111 | { title: "Birthdays Are Weird", src: "https://waitbutwhy.com/wp-content/uploads/2015/11/Birthday-F1-782x530.jpg" }, 112 | { title: "The SpaceX Post Progress Meter", src: "https://waitbutwhy.com/wp-content/uploads/2015/07/Untitled-4-782x530.png" }, 113 | { title: "Three Fascinating Videos With Live Footage of Thomas Edison and Henry Ford", src: "https://waitbutwhy.com/wp-content/uploads/2015/06/Screen-Shot-2015-06-20-at-6.31.41-PM-782x530.jpg" }, 114 | { title: "The Deal With the Hyperloop", src: "https://waitbutwhy.com/wp-content/uploads/2015/06/F1-782x530.png" }, 115 | { title: "The Deal With Solar", src: "https://waitbutwhy.com/wp-content/uploads/2014/05/Global_energy_potential_perez_2009_en.svg_-782x530.png" }, 116 | { title: "Putting Floyd Mayweather’s $210 Million Payout in Perspective", src: "https://waitbutwhy.com/wp-content/uploads/2015/05/Floyd-Mayweather-Chart-F-782x530.png" }, 117 | { title: "Trying Something Different With the Schedule", src: "https://waitbutwhy.com/wp-content/uploads/2015/03/Untitled-4.png" }, 118 | { title: "The Most Depressing Buzzfeed Article of All Time", src: "https://waitbutwhy.com/wp-content/uploads/2015/01/Depressing-Buzzfeed-Article-782x530.jpg" }, 119 | { title: "It’s 2015, and You’re in the Future", src: "https://waitbutwhy.com/wp-content/uploads/2015/01/2015-Feature-782x530.jpg" }, 120 | { title: "Where Wait But Why Gets Its Traffic", src: "https://waitbutwhy.com/wp-content/uploads/2014/11/WBW-Referral-Traffic-All-engaged-FEATURE-782x530.png" }, 121 | { title: "Introducing the Dinner Table", src: "https://waitbutwhy.com/wp-content/uploads/2014/11/Dinner-Table-Logo-FEATURE-782x530.png" }, 122 | { title: "Why I Can’t Post On Time", src: "https://waitbutwhy.com/wp-content/uploads/2014/10/Post-On-Time-FEATURE1-782x530.png" }, 123 | { title: "How the Panama Canal Works", src: "https://waitbutwhy.com/wp-content/uploads/2014/09/Panama-FEATURE-782x530.png" }, 124 | { title: "Your Life is Worse When You Know About Dust Mites", src: "https://waitbutwhy.com/wp-content/uploads/2014/08/Mites-FEATURE-782x530.jpg" }, 125 | { title: "What Makes a Face Trustworthy?", src: "https://waitbutwhy.com/wp-content/uploads/2014/08/Trustworthy-Faces-FEATURE1-782x530.png" }, 126 | { title: "Happy Birthday Wait But Why", src: "https://waitbutwhy.com/wp-content/uploads/2014/07/King-bday-782x530.png" }, 127 | { title: "If Andromeda Were Brighter, This is What You’d See", src: "https://waitbutwhy.com/wp-content/uploads/2014/06/Andromeda-FEATURE.png" }, 128 | { title: "The Titanic Compared With a Modern Cruiseship", src: "https://waitbutwhy.com/wp-content/uploads/2014/05/Titanic-FEATURE.png" }, 129 | { title: "Stick Figure Puzzle", src: "https://waitbutwhy.com/wp-content/uploads/2014/04/two-heads-FEATURE.png" }, 130 | { title: "Energy for Dummies", src: "https://waitbutwhy.com/wp-content/uploads/2014/03/2012new2012newUSEnergy-782x530.png" }, 131 | { title: "Crazy Fact About Population Density", src: "https://waitbutwhy.com/wp-content/uploads/2014/01/Shed-Thumb-782x530.png" }, 132 | { title: "200 People’s New Year’s Resolutions", src: "https://waitbutwhy.com/wp-content/uploads/2013/12/FEATURE6-782x530.png" }, 133 | { title: "Putting All the World’s Water into a Big Cube", src: "https://waitbutwhy.com/wp-content/uploads/2013/09/FEATURE-tiny-782x530.jpg" }, 134 | { title: "A-List Stars In Tiny Roles", src: "https://waitbutwhy.com/wp-content/uploads/2013/08/A-Listers-FEATURE-782x530.png" } 135 | ]; 136 | 137 | const match = images.find((post) => post.title === title); 138 | 139 | if (match) { 140 | return match.src; 141 | } else { 142 | return "https://images.pico.tools/production/waitbutwhy_logo_503.png"; 143 | } 144 | }; 145 | -------------------------------------------------------------------------------- /utils/index.ts: -------------------------------------------------------------------------------- 1 | import { OpenAIModel } from "@/types"; 2 | import { createClient } from "@supabase/supabase-js"; 3 | import { createParser, ParsedEvent, ReconnectInterval } from "eventsource-parser"; 4 | 5 | export const supabaseAdmin = createClient(process.env.NEXT_PUBLIC_SUPABASE_URL!, process.env.SUPABASE_SERVICE_ROLE_KEY!); 6 | 7 | export const OpenAIStream = async (prompt: string, apiKey: string) => { 8 | const encoder = new TextEncoder(); 9 | const decoder = new TextDecoder(); 10 | 11 | const res = await fetch("https://api.openai.com/v1/chat/completions", { 12 | headers: { 13 | "Content-Type": "application/json", 14 | Authorization: `Bearer ${apiKey}` 15 | }, 16 | method: "POST", 17 | body: JSON.stringify({ 18 | model: OpenAIModel.DAVINCI_TURBO, 19 | messages: [ 20 | { 21 | role: "system", 22 | content: "You are a helpful assistant that accurately answers queries using Wait But Why posts. Use the text provided to form your answer, but avoid copying word-for-word from the posts. Try to use your own words when possible. Keep your answer under 5 sentences. Be accurate, helpful, concise, and clear." 23 | }, 24 | { 25 | role: "user", 26 | content: prompt 27 | } 28 | ], 29 | max_tokens: 150, 30 | temperature: 0.0, 31 | stream: true 32 | }) 33 | }); 34 | 35 | if (res.status !== 200) { 36 | throw new Error("OpenAI API returned an error"); 37 | } 38 | 39 | const stream = new ReadableStream({ 40 | async start(controller) { 41 | const onParse = (event: ParsedEvent | ReconnectInterval) => { 42 | if (event.type === "event") { 43 | const data = event.data; 44 | 45 | if (data === "[DONE]") { 46 | controller.close(); 47 | return; 48 | } 49 | 50 | try { 51 | const json = JSON.parse(data); 52 | const text = json.choices[0].delta.content; 53 | const queue = encoder.encode(text); 54 | controller.enqueue(queue); 55 | } catch (e) { 56 | controller.error(e); 57 | } 58 | } 59 | }; 60 | 61 | const parser = createParser(onParse); 62 | 63 | for await (const chunk of res.body as any) { 64 | parser.feed(decoder.decode(chunk)); 65 | } 66 | } 67 | }); 68 | 69 | return stream; 70 | }; 71 | --------------------------------------------------------------------------------