├── .gitignore ├── LICENSE ├── README.md ├── data ├── how-to-do-great-work.md ├── how-to-start-google.md ├── how-to-work-hard.md ├── superlinear-returns.md └── weird-languages.md ├── embed.py ├── output-50.jsonl ├── output.jsonl ├── pyproject.toml ├── rag_app ├── README.md ├── __init__.py ├── cli.py ├── evaluate.py ├── generate_synthetic_question.py ├── ingest.py ├── models.py ├── query.py └── src │ ├── chunking.py │ └── metrics.py ├── requirements.txt ├── test.py └── tests ├── __init__.py └── test_models.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | /db -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Jason Liu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Levels of Complexity: RAG Applications 2 | 3 | In summary, this repository serves as a comprehensive guide to understanding and implementing RAG applications across different levels of complexity. Whether you're a beginner eager to learn the basics or an experienced developer looking to deepen your expertise, you'll find valuable insights and practical knowledge to help you on your journey. Let's embark on this exciting exploration together and unlock the full potential of RAG applications. 4 | 5 | ## Level 1: The Basics 6 | 7 | Welcome to the foundational level of RAG applications! Here, we'll start with the basics, laying the groundwork for your journey into the realm of Retrieval-Augmented Generation. This level is designed to introduce you to the core concepts and techniques essential for working with RAG models. By the end of this section, you'll have a solid understanding of how to traverse file systems for text generation, chunk and batch text for processing, and interact with embedding APIs. Let's dive in and explore the exciting capabilities of RAG applications together! 8 | 9 | 10 | 1. Recursively traverse the file system to generate text. 11 | 2. Utilize a generator for text chunking. 12 | 3. Employ a generator to batch requests and asynchronously send them to an embedding API. 13 | 4. Store data in LanceDB. 14 | 5. Implement a CLI for querying, embedding questions, yielding text chunks, and generating responses. 15 | 16 | ### Processing Pipeline 17 | 18 | 19 | ```python 20 | from dataclasses import dataclass 21 | from typing import Iterable, List 22 | import asyncio 23 | 24 | sem = asyncio.Semaphore(10) 25 | 26 | class TextChunk(BaseModel): 27 | id: int 28 | text: str 29 | embedding: np.array 30 | filename: str 31 | uuid: str = Field(default_factory=uuid.uuid4) 32 | 33 | def flatmap(f, items): 34 | for item in items: 35 | for subitem in f(item): 36 | yield subitem 37 | 38 | def get_texts(): 39 | for file in files: 40 | yield TextChunk( 41 | text=file.read(), 42 | embedding=None, 43 | filename=file.name 44 | ) 45 | 46 | def chunk_text(items:Iterable[TextChunk], window_size: int, overlap: int=0): 47 | for i in range(0, len(items), window_size-overlap): 48 | yield TextChunk( 49 | text = items[i:i+window_size], 50 | embedding = None, 51 | filename = items[i].filename 52 | ) 53 | 54 | def batched(iterable, n=1): 55 | l = len(iterable) 56 | for ndx in range(0, l, n): 57 | yield iterable[ndx:min(ndx + n, l)] 58 | 59 | def embed_batch(chunks: List[TextChunk]) -> List[TextChunk]: 60 | texts = [chunk.text for chunk in chunks] 61 | resp = embedding_api( # this is just the openai call 62 | texts=texts 63 | ) 64 | for chunk, embedding in zip(chunks, resp): 65 | chunk.embedding = embedding 66 | yield chunks 67 | 68 | def save_batch(chunks: List[TextChunk]): 69 | for chunk in chunks: 70 | db.insert(chunk) 71 | 72 | if __name__ == "__main__": 73 | # This is the CLI 74 | texts = get_texts() 75 | chunks = flatmap(chunk_text, texts) 76 | batched_chunks = batched(chunks, 10) 77 | for chunks in tqdm(batched_chunks): 78 | chunk_with_embedding = embed_batch(chunks) 79 | save_batch(chunk_with_embedding) 80 | ``` 81 | 82 | ### Search Pipeline 83 | 84 | ```python 85 | def search(question: str) -> List[TextChunk]: 86 | embeddings = embedding_api(texts=[question]) 87 | results = db.search(question) 88 | return results 89 | 90 | if __name__ == "__main__": 91 | question = input("Ask a question: ") 92 | results = search(question) 93 | for chunk in results: 94 | print(chunk.text) 95 | ``` 96 | 97 | ### Answer Pipeline 98 | 99 | ```python 100 | def answer(question: str, results: List[TextChunk]) -> str: 101 | return client.chat.completions.create( 102 | model="gpt-3.5-turbo", 103 | stream=False, 104 | messages=[ 105 | {"role": "system", "content": "You are a helpful assistant."}, 106 | {"role": "user", "content": prompt(question, results)} 107 | ] 108 | ) 109 | 110 | if __name__ == "__main__": 111 | question = input("Ask a question: ") 112 | results = search(question) 113 | response = answer(question, results) 114 | for chunk in response: 115 | print(chunk.text, end="") 116 | ``` 117 | 118 | ## Level 2 Structure RAG Application 119 | 120 | ## Level 2: Advanced Techniques 121 | 122 | Here we delve deeper into the world of Retrieval-Augmented Generation (RAG) applications. This level is designed for those who have grasped the basics and are ready to explore more advanced techniques and optimizations. Here, we focus on enhancing the efficiency and effectiveness of our RAG applications through better asynchronous programming, improved chunking strategies, and robust retry mechanisms in processing pipelines. 123 | 124 | In the search pipeline, we introduce sophisticated methods such as better ranking algorithms, query expansion and rewriting, and executing parallel queries to elevate the quality and relevance of search results. 125 | 126 | Furthermore, the answering pipeline is refined to provide more structured and informative responses, including citing specific text chunks and employing a streaming response model for better interaction. 127 | 128 | ### Processing 129 | 130 | 1. Better Asyncio 131 | 2. Better Chunking 132 | 3. Better Retries 133 | 134 | ### Search 135 | 136 | 1. Better Ranking (Cohere) 137 | 2. Query Expansion / Rewriting 138 | 3. Parallel Queries 139 | 140 | ```python 141 | class SearchQuery(BaseModel): 142 | semantic_search: str 143 | 144 | def extract_query(question: str) -> Iterable[SearchQuery]: 145 | return client.completions.create( 146 | model="gpt-3.5-turbo", 147 | messages=[ 148 | { 149 | "role": "system", 150 | "content": "Extract a query" 151 | }, 152 | { 153 | "role": "user", 154 | "content": question 155 | } 156 | ], 157 | response_model=Iterable[SearchQuery] 158 | ) 159 | 160 | def search(search: Iterable[SearchQuery]) -> List[TextChunk]: 161 | with LanceDB() as db: 162 | results = db.search(search) 163 | return results 164 | 165 | 166 | def rerank(question: str, results: List[TextChunk]) -> List[TextChunk]: 167 | return cohere_api( 168 | question=question, 169 | texts=[chunk.text for chunk in results] 170 | ) 171 | 172 | if __name__ == "__main__": 173 | question = input("Ask a question: ") 174 | search_query = extract_query(question) 175 | results = search(search_query) 176 | ranked_results = rerank(question, results) 177 | for chunk in ranked_results: 178 | print(chunk.text) 179 | ``` 180 | 181 | ### Answering 182 | 183 | 1. Citating specific text chunks 184 | 2. Streaming Response Model for better structure. 185 | 186 | ```python 187 | class MultiPartResponse(BaseModel): 188 | response: str 189 | followups: List[str] 190 | sources: List[int] 191 | 192 | def answer(question: str, results: List[TextChunk]) -> Iterable[MultiPartResponse]: 193 | return client.chat.completions.create( 194 | model="gpt-3.5-turbo", 195 | stream=True, 196 | response_model=instructor.Partial[MultiPartResponse] 197 | messages=[ 198 | {"role": "system", "content": "You are a helpful assistant."}, 199 | {"role": "user", "content": prompt(question, results)} 200 | ] 201 | ) 202 | 203 | if __name__ == "__main__": 204 | from rich.console import Console 205 | 206 | question = input("Ask a question: ") 207 | 208 | search_query = extract_query(question) 209 | results = search(search_query) 210 | results = rerank(question, results) 211 | response = answer(question, results) 212 | 213 | console = Console() 214 | for chunk in response: 215 | console.clear() 216 | console.print(chunk.dump_model_json()) 217 | ``` 218 | 219 | ## Level 3: Observability 220 | 221 | At Level 3, the focus shifts towards the critical practice of observability. This stage emphasizes the importance of implementing comprehensive logging mechanisms to monitor and measure the multifaceted performance of your application. Establishing robust observability allows you to swiftly pinpoint and address any bottlenecks or issues, ensuring optimal functionality. Below, we outline several key types of logs that are instrumental in achieving this goal. 222 | 223 | ### Expanding on Wide Event Tracking 224 | 225 | Wide event tracking 226 | 227 | - [Do it wide](https://isburmistrov.substack.com/p/all-you-need-is-wide-events-not-metrics?utm_source=tldrwebdev) 228 | 229 | ### Log how the queries are being rewritten 230 | 231 | 1. When addressing a complaint we should quickly understand if the query was written correctly 232 | 233 | | Query | Rewritten Query | latency | ... | 234 | | ----- | --------------- | ------- | --- | 235 | | ... | ... | ... | ... | 236 | 237 | example: once we found that for queries with "latest" the dates it was selecting was literally the current date, we were able to quickly fix the issue by including few shot examples that consider latest to be 1 week or more. 238 | 239 | 1. Training a model 240 | 241 | We can also use all the positive examples to figure out how to train a model that does query expansion better. 242 | 243 | ### Log the citations 244 | 245 | By logging the citations, we can quickly understand if the model is citing the correct information, what text chunks are popular, review and understand if the model is citing the correct information. and also potentially build a model in the future that can understand what text chunks are more important. 246 | 247 | | Query | Rewritten Query | ... | Citations | 248 | | ----- | --------------- | --- | --------- | 249 | | ... | ... | ... | [1,2,4] | 250 | 251 | There's a couple ways you can do this. For example, when you cite something, you can include not only what was shown to the language model, but also what was cited. If something was shown as a language model but was not cited, we can include this as part of the dataset. 252 | 253 | | Query | Rewritten Query | ... | sources | cited | 254 | | ----- | --------------- | --- | ------- | ----- | 255 | | ... | ... | ... | [1,2,4] | [1,2] | 256 | 257 | ### Log mean cosine scores and reranker scores 258 | 259 | By attaching this little metadata, we will be able to very cheaply identify queries that may be performing poorly. 260 | 261 | | Query | Rewritten Query | ... | Mean Cosine Score | Reranker Score | 262 | | ------------------------------ | ------------------------------ | --- | ----------------- | -------------- | 263 | | What is the capital of France? | What is the capital of France? | ... | 0.9 | 0.8 | 264 | | Who modified the file last? | Who modified the file last? | ... | 0.2 | 0.1 | 265 | 266 | Here you might see "oh clearly i can't answer questions about file modification" thats not even in my index. 267 | 268 | ### Log user level metadata for the search 269 | 270 | By including other group information information, we can quickly identify if a certain group is having a bad experience 271 | 272 | Examples could be 273 | 274 | 1. Organization ID 275 | 2. User ID 276 | 3. User Role 277 | 4. Signup Date 278 | 5. Device Type 279 | 6. Geo Location 280 | 7. Language 281 | 282 | This could help you understand a lot of different things about how your application is being used. Maybe people on a different device are asking shorter queries and they are performing poorly. Or maybe when a new organization signed up, the types of questions they were asking were being served poorly by the language model. In the future we'll talk about other metrics, but just by implementing the mean cosine score and the free ranker score, you get these things for free without any additional work. 283 | 284 | Just by building up some simple dashboards that are grouped by these attributes and look at the average scores, you can learn a lot. My recommendation is to review these things during stand-up once a week, look at some examples, and figure out what we could do to improve our system. When we see poor scores, we can look at the query and the rewritten query and try to understand what exactly is going on. 285 | 286 | ### Have Users 287 | 288 | By this point you should definitely be having users. You've already set yourself for success by understanding queries, rewriting them, and monitoring how users are actually using your system. The next couple of steps will be around improving specific metrics and also different ways of doing that. 289 | 290 | ## Level 4: Evaluations 291 | 292 | Evaluations at this stage are crucial for understanding the performance and effectiveness of our systems. Primarily, we are dealing with two distinct systems: the search system and the question answering (QA) system. It's common to see a lot of focus on evaluating the QA system, given its direct interaction with the end-user's queries. However, it's equally important to not overlook the search system. The search system acts as the backbone, fetching relevant information upon which the QA system builds its answers. A comprehensive evaluation strategy should include both systems, assessing them individually and how well they integrate and complement each other in providing accurate, relevant answers to the user's queries. 293 | 294 | ### Evaluating the Search System 295 | 296 | The aim here is to enhance our focus on key metrics such as precision and recall at various levels (K). With the comprehensive logging of all citation data, we have a solid foundation to employ a language model for an in-depth evaluation of the search system's efficacy. 297 | 298 | For instances where the dataset might be limited, turning to synthetic data is a practical approach. This method involves selecting random text chunks or documents and then prompting a language model to generate questions that these texts could answer. This process is crucial for verifying the search system's ability to accurately identify and retrieve the text chunks responsible for generating these questions. 299 | 300 | ```python 301 | def test(): 302 | text_chunk = sample_text_chunk() 303 | questions = ask_ai(f"generate questions that could be ansered by {text_chunk.text}") 304 | for question in questions: 305 | search_results = search(question) 306 | 307 | return { 308 | "recall@5": (1 if text_chunk in search_results[:5] else 0), 309 | ... 310 | } 311 | 312 | average_recall = sum(test() for _ in range(n)) / n 313 | ``` 314 | 315 | Your code shouldn't actually look like this, but this generally captures the idea that we can synthetically generate questions and use them as part of our evaluation. You can try to be creative. But ultimately it will be a function of how well you can actually write a generation prompt. 316 | 317 | ### Evaluating the Answering System 318 | 319 | This is a lot trickier, but often times people will use a framework like, I guess, to evaluate the questions. Here I recommend spending some more time building out a data set that actually has answers. 320 | 321 | ```python 322 | def test(): 323 | text_chunk = sample_text_chunks(n=...) 324 | question, answer = ask_ai(f"generate questions and answers for {text_chunk.text}") 325 | 326 | ai_answer = rag_app(question) 327 | return ask_ai(f"for the question {question} is the answer {ai_answer} correct given that {answer} is the correct answer?") 328 | ``` 329 | 330 | #### Evaluating the Answering System: Feedback 331 | 332 | It's also good to build in feedback mechanisms in order to get better scores. I recommend building a thumbs up, thumbs down rating system rather than a five star rating system. I won't go into details right now, but this is something I strongly recommend. 333 | 334 | ### The purpose of synethic data 335 | 336 | The purpose of synthetic data is to help you quickly get some metrics out. It will help you build out this evaluation pipeline in hopes that as you get more users and more real questions, you'll be able to understand where we're performing well and where we're performing poorly using the suite of tests that we have. Precision, recall, mean ranking scores, etc. 337 | 338 | ## Level 5: Understanding Short comings 339 | 340 | At this point you should be able to have a data set that is extremely diverse using both the synthetic data and production data. We should also have a suite of scores that we can use to evaluate the quality of our answers. 341 | 342 | | org_id | query | rewritten | answer | recall@k | precision@k | mean ranking score | reranker score | user feedback | citations | sources | ... | 343 | | ------ | ----- | --------- | ------ | -------- | ----------- | ------------------ | -------------- | ------------- | --------- | ------- | --- | 344 | | org123 | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | 345 | 346 | Now we can do a bunch of different things to understand how we're doing by doing exploratory data analysis. We can look at the mean ranking score and reranker score and see if there are any patterns. We can look at the citations and see if there are any patterns. We can look at the user feedback and see if there are any patterns. We can look at the sources and see if there are any patterns. We can look at the queries and see if there are any patterns. 347 | 348 | ### Clustering Queries 349 | 350 | We can use clustering to understand if there are any patterns in the queries. We can use clustering to understand if there are any patterns in the citations. We can use clustering to understand if there are any patterns in the sources. We can use clustering to understand if there are any patterns in the user feedback. 351 | 352 | We'll go into more depth later, but the general idea is we can also introduce cluster topics. I find that there's usually two different kinds of clutches that we detect. 353 | 354 | 1. Topics 355 | 2. Capabilities 356 | 357 | Topics are captured by the nature of the text chunks and the queries. Capabilities are captured by the nature of the sources or additional metadata that we have. 358 | 359 | Capabilites could be more like: 360 | 361 | 1. Questions that ask about document metadata "who modified the file last" 362 | 2. Quyestions that require summarzation of a document "what are the main points of this document" 363 | 3. Questions that required timeline information "what happened in the last 3 months" 364 | 4. Questions that compare and contrast "what are the differences between these two documents" 365 | 366 | There are all things you'll likely find as you cluster and explore the datasets. 367 | 368 | ### Upcoming Topics 369 | 370 | As we continue to explore the depths of RAG applications, the following areas will be addressed in subsequent levels, each designed to enhance the complexity and functionality of your RAG systems: 371 | 372 | 373 | #### Level 6: Advanced Data Handling 374 | 375 | - Finding Segments and Routing: Techniques for identifying and directing data segments efficiently. 376 | - Processing Tables: Strategies for interpreting and manipulating tabular data. 377 | - Processing Images: Methods for incorporating image data into RAG applications. 378 | 379 | #### Level 7: Query Enhancement 380 | 381 | - Building Up Timeline Queries: Crafting queries that span across different timeframes for dynamic data analysis. 382 | - Adding Additional Metadata: Leveraging metadata to enrich query context and improve response accuracy. 383 | 384 | #### Level 8: Summarization Techniques 385 | 386 | - Summarization and Summary Indices: Developing concise summaries from extensive datasets to aid quick insights. 387 | 388 | #### Level 9: Outcome Modeling 389 | 390 | - Modeling Business Outcomes: Applying RAG techniques to predict and model business outcomes, facilitating strategic decision-making. 391 | -------------------------------------------------------------------------------- /data/how-to-do-great-work.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How to do great work 3 | date: 2023-07 4 | url: https://paulgraham.com/superlinear.html 5 | --- 6 | 7 | If you collected lists of techniques for doing great work in a lot of different fields, what would the intersection look like? I decided to find out by making it. 8 | 9 | Partly my goal was to create a guide that could be used by someone working in any field. But I was also curious about the shape of the intersection. And one thing this exercise shows is that it does have a definite shape; it's not just a point labelled "work hard." 10 | 11 | The following recipe assumes you're very ambitious. 12 | 13 | The first step is to decide what to work on. The work you choose needs to have three qualities: it has to be something you have a natural aptitude for, that you have a deep interest in, and that offers scope to do great work. 14 | 15 | In practice you don't have to worry much about the third criterion. Ambitious people are if anything already too conservative about it. So all you need to do is find something you have an aptitude for and great interest in. 16 | 17 | That sounds straightforward, but it's often quite difficult. When you're young you don't know what you're good at or what different kinds of work are like. Some kinds of work you end up doing may not even exist yet. So while some people know what they want to do at 14, most have to figure it out. 18 | 19 | The way to figure out what to work on is by working. If you're not sure what to work on, guess. But pick something and get going. You'll probably guess wrong some of the time, but that's fine. It's good to know about multiple things; some of the biggest discoveries come from noticing connections between different fields. 20 | 21 | Develop a habit of working on your own projects. Don't let "work" mean something other people tell you to do. If you do manage to do great work one day, it will probably be on a project of your own. It may be within some bigger project, but you'll be driving your part of it. 22 | 23 | What should your projects be? Whatever seems to you excitingly ambitious. As you grow older and your taste in projects evolves, exciting and important will converge. At 7 it may seem excitingly ambitious to build huge things out of Lego, then at 14 to teach yourself calculus, till at 21 you're starting to explore unanswered questions in physics. But always preserve excitingness. 24 | 25 | There's a kind of excited curiosity that's both the engine and the rudder of great work. It will not only drive you, but if you let it have its way, will also show you what to work on. 26 | 27 | What are you excessively curious about — curious to a degree that would bore most other people? That's what you're looking for. 28 | 29 | Once you've found something you're excessively interested in, the next step is to learn enough about it to get you to one of the frontiers of knowledge. Knowledge expands fractally, and from a distance its edges look smooth, but once you learn enough to get close to one, they turn out to be full of gaps. 30 | 31 | The next step is to notice them. This takes some skill, because your brain wants to ignore such gaps in order to make a simpler model of the world. Many discoveries have come from asking questions about things that everyone else took for granted. 32 | 33 | If the answers seem strange, so much the better. Great work often has a tincture of strangeness. You see this from painting to math. It would be affected to try to manufacture it, but if it appears, embrace it. 34 | 35 | Boldly chase outlier ideas, even if other people aren't interested in them — in fact, especially if they aren't. If you're excited about some possibility that everyone else ignores, and you have enough expertise to say precisely what they're all overlooking, that's as good a bet as you'll find. 36 | 37 | Four steps: choose a field, learn enough to get to the frontier, notice gaps, explore promising ones. This is how practically everyone who's done great work has done it, from painters to physicists. 38 | 39 | Steps two and four will require hard work. It may not be possible to prove that you have to work hard to do great things, but the empirical evidence is on the scale of the evidence for mortality. That's why it's essential to work on something you're deeply interested in. Interest will drive you to work harder than mere diligence ever could. 40 | 41 | The three most powerful motives are curiosity, delight, and the desire to do something impressive. Sometimes they converge, and that combination is the most powerful of all. 42 | 43 | The big prize is to discover a new fractal bud. You notice a crack in the surface of knowledge, pry it open, and there's a whole world inside. 44 | 45 | Let's talk a little more about the complicated business of figuring out what to work on. The main reason it's hard is that you can't tell what most kinds of work are like except by doing them. Which means the four steps overlap: you may have to work at something for years before you know how much you like it or how good you are at it. And in the meantime you're not doing, and thus not learning about, most other kinds of work. So in the worst case you choose late based on very incomplete information. 46 | 47 | The nature of ambition exacerbates this problem. Ambition comes in two forms, one that precedes interest in the subject and one that grows out of it. Most people who do great work have a mix, and the more you have of the former, the harder it will be to decide what to do. 48 | 49 | The educational systems in most countries pretend it's easy. They expect you to commit to a field long before you could know what it's really like. And as a result an ambitious person on an optimal trajectory will often read to the system as an instance of breakage. 50 | 51 | It would be better if they at least admitted it — if they admitted that the system not only can't do much to help you figure out what to work on, but is designed on the assumption that you'll somehow magically guess as a teenager. They don't tell you, but I will: when it comes to figuring out what to work on, you're on your own. Some people get lucky and do guess correctly, but the rest will find themselves scrambling diagonally across tracks laid down on the assumption that everyone does. 52 | 53 | What should you do if you're young and ambitious but don't know what to work on? What you should not do is drift along passively, assuming the problem will solve itself. You need to take action. But there is no systematic procedure you can follow. When you read biographies of people who've done great work, it's remarkable how much luck is involved. They discover what to work on as a result of a chance meeting, or by reading a book they happen to pick up. So you need to make yourself a big target for luck, and the way to do that is to be curious. Try lots of things, meet lots of people, read lots of books, ask lots of questions. 54 | 55 | When in doubt, optimize for interestingness. Fields change as you learn more about them. What mathematicians do, for example, is very different from what you do in high school math classes. So you need to give different types of work a chance to show you what they're like. But a field should become increasingly interesting as you learn more about it. If it doesn't, it's probably not for you. 56 | 57 | Don't worry if you find you're interested in different things than other people. The stranger your tastes in interestingness, the better. Strange tastes are often strong ones, and a strong taste for work means you'll be productive. And you're more likely to find new things if you're looking where few have looked before. 58 | 59 | One sign that you're suited for some kind of work is when you like even the parts that other people find tedious or frightening. 60 | 61 | But fields aren't people; you don't owe them any loyalty. If in the course of working on one thing you discover another that's more exciting, don't be afraid to switch. 62 | 63 | If you're making something for people, make sure it's something they actually want. The best way to do this is to make something you yourself want. Write the story you want to read; build the tool you want to use. Since your friends probably have similar interests, this will also get you your initial audience. 64 | 65 | This should follow from the excitingness rule. Obviously the most exciting story to write will be the one you want to read. The reason I mention this case explicitly is that so many people get it wrong. Instead of making what they want, they try to make what some imaginary, more sophisticated audience wants. And once you go down that route, you're lost. 66 | 67 | There are a lot of forces that will lead you astray when you're trying to figure out what to work on. Pretentiousness, fashion, fear, money, politics, other people's wishes, eminent frauds. But if you stick to what you find genuinely interesting, you'll be proof against all of them. If you're interested, you're not astray. 68 | 69 | Following your interests may sound like a rather passive strategy, but in practice it usually means following them past all sorts of obstacles. You usually have to risk rejection and failure. So it does take a good deal of boldness. 70 | 71 | But while you need boldness, you don't usually need much planning. In most cases the recipe for doing great work is simply: work hard on excitingly ambitious projects, and something good will come of it. Instead of making a plan and then executing it, you just try to preserve certain invariants. 72 | 73 | The trouble with planning is that it only works for achievements you can describe in advance. You can win a gold medal or get rich by deciding to as a child and then tenaciously pursuing that goal, but you can't discover natural selection that way. 74 | 75 | I think for most people who want to do great work, the right strategy is not to plan too much. At each stage do whatever seems most interesting and gives you the best options for the future. I call this approach "staying upwind." This is how most people who've done great work seem to have done it. 76 | 77 | Even when you've found something exciting to work on, working on it is not always straightforward. There will be times when some new idea makes you leap out of bed in the morning and get straight to work. But there will also be plenty of times when things aren't like that. 78 | 79 | You don't just put out your sail and get blown forward by inspiration. There are headwinds and currents and hidden shoals. So there's a technique to working, just as there is to sailing. 80 | 81 | For example, while you must work hard, it's possible to work too hard, and if you do that you'll find you get diminishing returns: fatigue will make you stupid, and eventually even damage your health. The point at which work yields diminishing returns depends on the type. Some of the hardest types you might only be able to do for four or five hours a day. 82 | 83 | Ideally those hours will be contiguous. To the extent you can, try to arrange your life so you have big blocks of time to work in. You'll shy away from hard tasks if you know you might be interrupted. 84 | 85 | It will probably be harder to start working than to keep working. You'll often have to trick yourself to get over that initial threshold. Don't worry about this; it's the nature of work, not a flaw in your character. Work has a sort of activation energy, both per day and per project. And since this threshold is fake in the sense that it's higher than the energy required to keep going, it's ok to tell yourself a lie of corresponding magnitude to get over it. 86 | 87 | It's usually a mistake to lie to yourself if you want to do great work, but this is one of the rare cases where it isn't. When I'm reluctant to start work in the morning, I often trick myself by saying "I'll just read over what I've got so far." Five minutes later I've found something that seems mistaken or incomplete, and I'm off. 88 | 89 | Similar techniques work for starting new projects. It's ok to lie to yourself about how much work a project will entail, for example. Lots of great things began with someone saying "How hard could it be?" 90 | 91 | This is one case where the young have an advantage. They're more optimistic, and even though one of the sources of their optimism is ignorance, in this case ignorance can sometimes beat knowledge. 92 | 93 | Try to finish what you start, though, even if it turns out to be more work than you expected. Finishing things is not just an exercise in tidiness or self-discipline. In many projects a lot of the best work happens in what was meant to be the final stage. 94 | 95 | Another permissible lie is to exaggerate the importance of what you're working on, at least in your own mind. If that helps you discover something new, it may turn out not to have been a lie after all. 96 | 97 | Since there are two senses of starting work — per day and per project — there are also two forms of procrastination. Per-project procrastination is far the more dangerous. You put off starting that ambitious project from year to year because the time isn't quite right. When you're procrastinating in units of years, you can get a lot not done. 98 | 99 | One reason per-project procrastination is so dangerous is that it usually camouflages itself as work. You're not just sitting around doing nothing; you're working industriously on something else. So per-project procrastination doesn't set off the alarms that per-day procrastination does. You're too busy to notice it. 100 | 101 | The way to beat it is to stop occasionally and ask yourself: Am I working on what I most want to work on? When you're young it's ok if the answer is sometimes no, but this gets increasingly dangerous as you get older. 102 | 103 | Great work usually entails spending what would seem to most people an unreasonable amount of time on a problem. You can't think of this time as a cost, or it will seem too high. You have to find the work sufficiently engaging as it's happening. 104 | 105 | There may be some jobs where you have to work diligently for years at things you hate before you get to the good part, but this is not how great work happens. Great work happens by focusing consistently on something you're genuinely interested in. When you pause to take stock, you're surprised how far you've come. 106 | 107 | The reason we're surprised is that we underestimate the cumulative effect of work. Writing a page a day doesn't sound like much, but if you do it every day you'll write a book a year. That's the key: consistency. People who do great things don't get a lot done every day. They get something done, rather than nothing. 108 | 109 | If you do work that compounds, you'll get exponential growth. Most people who do this do it unconsciously, but it's worth stopping to think about. Learning, for example, is an instance of this phenomenon: the more you learn about something, the easier it is to learn more. Growing an audience is another: the more fans you have, the more new fans they'll bring you. 110 | 111 | The trouble with exponential growth is that the curve feels flat in the beginning. It isn't; it's still a wonderful exponential curve. But we can't grasp that intuitively, so we underrate exponential growth in its early stages. 112 | 113 | Something that grows exponentially can become so valuable that it's worth making an extraordinary effort to get it started. But since we underrate exponential growth early on, this too is mostly done unconsciously: people push through the initial, unrewarding phase of learning something new because they know from experience that learning new things always takes an initial push, or they grow their audience one fan at a time because they have nothing better to do. If people consciously realized they could invest in exponential growth, many more would do it. 114 | 115 | Work doesn't just happen when you're trying to. There's a kind of undirected thinking you do when walking or taking a shower or lying in bed that can be very powerful. By letting your mind wander a little, you'll often solve problems you were unable to solve by frontal attack. 116 | 117 | You have to be working hard in the normal way to benefit from this phenomenon, though. You can't just walk around daydreaming. The daydreaming has to be interleaved with deliberate work that feeds it questions. 118 | 119 | Everyone knows to avoid distractions at work, but it's also important to avoid them in the other half of the cycle. When you let your mind wander, it wanders to whatever you care about most at that moment. So avoid the kind of distraction that pushes your work out of the top spot, or you'll waste this valuable type of thinking on the distraction instead. (Exception: Don't avoid love.) 120 | 121 | Consciously cultivate your taste in the work done in your field. Until you know which is the best and what makes it so, you don't know what you're aiming for. 122 | 123 | And that is what you're aiming for, because if you don't try to be the best, you won't even be good. This observation has been made by so many people in so many different fields that it might be worth thinking about why it's true. It could be because ambition is a phenomenon where almost all the error is in one direction — where almost all the shells that miss the target miss by falling short. Or it could be because ambition to be the best is a qualitatively different thing from ambition to be good. Or maybe being good is simply too vague a standard. Probably all three are true. 124 | 125 | Fortunately there's a kind of economy of scale here. Though it might seem like you'd be taking on a heavy burden by trying to be the best, in practice you often end up net ahead. It's exciting, and also strangely liberating. It simplifies things. In some ways it's easier to try to be the best than to try merely to be good. 126 | 127 | One way to aim high is to try to make something that people will care about in a hundred years. Not because their opinions matter more than your contemporaries', but because something that still seems good in a hundred years is more likely to be genuinely good. 128 | 129 | Don't try to work in a distinctive style. Just try to do the best job you can; you won't be able to help doing it in a distinctive way. 130 | 131 | Style is doing things in a distinctive way without trying to. Trying to is affectation. 132 | 133 | Affectation is in effect to pretend that someone other than you is doing the work. You adopt an impressive but fake persona, and while you're pleased with the impressiveness, the fakeness is what shows in the work. 134 | 135 | The temptation to be someone else is greatest for the young. They often feel like nobodies. But you never need to worry about that problem, because it's self-solving if you work on sufficiently ambitious projects. If you succeed at an ambitious project, you're not a nobody; you're the person who did it. So just do the work and your identity will take care of itself. 136 | 137 | "Avoid affectation" is a useful rule so far as it goes, but how would you express this idea positively? How would you say what to be, instead of what not to be? The best answer is earnest. If you're earnest you avoid not just affectation but a whole set of similar vices. 138 | 139 | The core of being earnest is being intellectually honest. We're taught as children to be honest as an unselfish virtue — as a kind of sacrifice. But in fact it's a source of power too. To see new ideas, you need an exceptionally sharp eye for the truth. You're trying to see more truth than others have seen so far. And how can you have a sharp eye for the truth if you're intellectually dishonest? 140 | 141 | One way to avoid intellectual dishonesty is to maintain a slight positive pressure in the opposite direction. Be aggressively willing to admit that you're mistaken. Once you've admitted you were mistaken about something, you're free. Till then you have to carry it. 142 | 143 | Another more subtle component of earnestness is informality. Informality is much more important than its grammatically negative name implies. It's not merely the absence of something. It means focusing on what matters instead of what doesn't. 144 | 145 | What formality and affectation have in common is that as well as doing the work, you're trying to seem a certain way as you're doing it. But any energy that goes into how you seem comes out of being good. That's one reason nerds have an advantage in doing great work: they expend little effort on seeming anything. In fact that's basically the definition of a nerd. 146 | 147 | Nerds have a kind of innocent boldness that's exactly what you need in doing great work. It's not learned; it's preserved from childhood. So hold onto it. Be the one who puts things out there rather than the one who sits back and offers sophisticated-sounding criticisms of them. "It's easy to criticize" is true in the most literal sense, and the route to great work is never easy. 148 | 149 | There may be some jobs where it's an advantage to be cynical and pessimistic, but if you want to do great work it's an advantage to be optimistic, even though that means you'll risk looking like a fool sometimes. There's an old tradition of doing the opposite. The Old Testament says it's better to keep quiet lest you look like a fool. But that's advice for seeming smart. If you actually want to discover new things, it's better to take the risk of telling people your ideas. 150 | 151 | Some people are naturally earnest, and with others it takes a conscious effort. Either kind of earnestness will suffice. But I doubt it would be possible to do great work without being earnest. It's so hard to do even if you are. You don't have enough margin for error to accommodate the distortions introduced by being affected, intellectually dishonest, orthodox, fashionable, or cool. 152 | 153 | Great work is consistent not only with who did it, but with itself. It's usually all of a piece. So if you face a decision in the middle of working on something, ask which choice is more consistent. 154 | 155 | You may have to throw things away and redo them. You won't necessarily have to, but you have to be willing to. And that can take some effort; when there's something you need to redo, status quo bias and laziness will combine to keep you in denial about it. To beat this ask: If I'd already made the change, would I want to revert to what I have now? 156 | 157 | Have the confidence to cut. Don't keep something that doesn't fit just because you're proud of it, or because it cost you a lot of effort. 158 | 159 | Indeed, in some kinds of work it's good to strip whatever you're doing to its essence. The result will be more concentrated; you'll understand it better; and you won't be able to lie to yourself about whether there's anything real there. 160 | 161 | Mathematical elegance may sound like a mere metaphor, drawn from the arts. That's what I thought when I first heard the term "elegant" applied to a proof. But now I suspect it's conceptually prior — that the main ingredient in artistic elegance is mathematical elegance. At any rate it's a useful standard well beyond math. 162 | 163 | Elegance can be a long-term bet, though. Laborious solutions will often have more prestige in the short term. They cost a lot of effort and they're hard to understand, both of which impress people, at least temporarily. 164 | 165 | Whereas some of the very best work will seem like it took comparatively little effort, because it was in a sense already there. It didn't have to be built, just seen. It's a very good sign when it's hard to say whether you're creating something or discovering it. 166 | 167 | When you're doing work that could be seen as either creation or discovery, err on the side of discovery. Try thinking of yourself as a mere conduit through which the ideas take their natural shape. 168 | 169 | (Strangely enough, one exception is the problem of choosing a problem to work on. This is usually seen as search, but in the best case it's more like creating something. In the best case you create the field in the process of exploring it.) 170 | 171 | Similarly, if you're trying to build a powerful tool, make it gratuitously unrestrictive. A powerful tool almost by definition will be used in ways you didn't expect, so err on the side of eliminating restrictions, even if you don't know what the benefit will be. 172 | 173 | Great work will often be tool-like in the sense of being something others build on. So it's a good sign if you're creating ideas that others could use, or exposing questions that others could answer. The best ideas have implications in many different areas. 174 | 175 | If you express your ideas in the most general form, they'll be truer than you intended. 176 | 177 | True by itself is not enough, of course. Great ideas have to be true and new. And it takes a certain amount of ability to see new ideas even once you've learned enough to get to one of the frontiers of knowledge. 178 | 179 | In English we give this ability names like originality, creativity, and imagination. And it seems reasonable to give it a separate name, because it does seem to some extent a separate skill. It's possible to have a great deal of ability in other respects — to have a great deal of what's often called "technical ability" — and yet not have much of this. 180 | 181 | I've never liked the term "creative process." It seems misleading. Originality isn't a process, but a habit of mind. Original thinkers throw off new ideas about whatever they focus on, like an angle grinder throwing off sparks. They can't help it. 182 | 183 | If the thing they're focused on is something they don't understand very well, these new ideas might not be good. One of the most original thinkers I know decided to focus on dating after he got divorced. He knew roughly as much about dating as the average 15 year old, and the results were spectacularly colorful. But to see originality separated from expertise like that made its nature all the more clear. 184 | 185 | I don't know if it's possible to cultivate originality, but there are definitely ways to make the most of however much you have. For example, you're much more likely to have original ideas when you're working on something. Original ideas don't come from trying to have original ideas. They come from trying to build or understand something slightly too difficult. 186 | 187 | Talking or writing about the things you're interested in is a good way to generate new ideas. When you try to put ideas into words, a missing idea creates a sort of vacuum that draws it out of you. Indeed, there's a kind of thinking that can only be done by writing. 188 | 189 | Changing your context can help. If you visit a new place, you'll often find you have new ideas there. The journey itself often dislodges them. But you may not have to go far to get this benefit. Sometimes it's enough just to go for a walk. 190 | 191 | It also helps to travel in topic space. You'll have more new ideas if you explore lots of different topics, partly because it gives the angle grinder more surface area to work on, and partly because analogies are an especially fruitful source of new ideas. 192 | 193 | Don't divide your attention evenly between many topics though, or you'll spread yourself too thin. You want to distribute it according to something more like a power law. Be professionally curious about a few topics and idly curious about many more. 194 | 195 | Curiosity and originality are closely related. Curiosity feeds originality by giving it new things to work on. But the relationship is closer than that. Curiosity is itself a kind of originality; it's roughly to questions what originality is to answers. And since questions at their best are a big component of answers, curiosity at its best is a creative force. 196 | 197 | Having new ideas is a strange game, because it usually consists of seeing things that were right under your nose. Once you've seen a new idea, it tends to seem obvious. Why did no one think of this before? 198 | 199 | When an idea seems simultaneously novel and obvious, it's probably a good one. 200 | 201 | Seeing something obvious sounds easy. And yet empirically having new ideas is hard. What's the source of this apparent contradiction? It's that seeing the new idea usually requires you to change the way you look at the world. We see the world through models that both help and constrain us. When you fix a broken model, new ideas become obvious. But noticing and fixing a broken model is hard. That's how new ideas can be both obvious and yet hard to discover: they're easy to see after you do something hard. 202 | 203 | One way to discover broken models is to be stricter than other people. Broken models of the world leave a trail of clues where they bash against reality. Most people don't want to see these clues. It would be an understatement to say that they're attached to their current model; it's what they think in; so they'll tend to ignore the trail of clues left by its breakage, however conspicuous it may seem in retrospect. 204 | 205 | To find new ideas you have to seize on signs of breakage instead of looking away. That's what Einstein did. He was able to see the wild implications of Maxwell's equations not so much because he was looking for new ideas as because he was stricter. 206 | 207 | The other thing you need is a willingness to break rules. Paradoxical as it sounds, if you want to fix your model of the world, it helps to be the sort of person who's comfortable breaking rules. From the point of view of the old model, which everyone including you initially shares, the new model usually breaks at least implicit rules. 208 | 209 | Few understand the degree of rule-breaking required, because new ideas seem much more conservative once they succeed. They seem perfectly reasonable once you're using the new model of the world they brought with them. But they didn't at the time; it took the greater part of a century for the heliocentric model to be generally accepted, even among astronomers, because it felt so wrong. 210 | 211 | Indeed, if you think about it, a good new idea has to seem bad to most people, or someone would have already explored it. So what you're looking for is ideas that seem crazy, but the right kind of crazy. How do you recognize these? You can't with certainty. Often ideas that seem bad are bad. But ideas that are the right kind of crazy tend to be exciting; they're rich in implications; whereas ideas that are merely bad tend to be depressing. 212 | 213 | There are two ways to be comfortable breaking rules: to enjoy breaking them, and to be indifferent to them. I call these two cases being aggressively and passively independent-minded. 214 | 215 | The aggressively independent-minded are the naughty ones. Rules don't merely fail to stop them; breaking rules gives them additional energy. For this sort of person, delight at the sheer audacity of a project sometimes supplies enough activation energy to get it started. 216 | 217 | The other way to break rules is not to care about them, or perhaps even to know they exist. This is why novices and outsiders often make new discoveries; their ignorance of a field's assumptions acts as a source of temporary passive independent-mindedness. Aspies also seem to have a kind of immunity to conventional beliefs. Several I know say that this helps them to have new ideas. 218 | 219 | Strictness plus rule-breaking sounds like a strange combination. In popular culture they're opposed. But popular culture has a broken model in this respect. It implicitly assumes that issues are trivial ones, and in trivial matters strictness and rule-breaking are opposed. But in questions that really matter, only rule-breakers can be truly strict. 220 | 221 | An overlooked idea often doesn't lose till the semifinals. You do see it, subconsciously, but then another part of your subconscious shoots it down because it would be too weird, too risky, too much work, too controversial. This suggests an exciting possibility: if you could turn off such filters, you could see more new ideas. 222 | 223 | One way to do that is to ask what would be good ideas for someone else to explore. Then your subconscious won't shoot them down to protect you. 224 | 225 | You could also discover overlooked ideas by working in the other direction: by starting from what's obscuring them. Every cherished but mistaken principle is surrounded by a dead zone of valuable ideas that are unexplored because they contradict it. 226 | 227 | Religions are collections of cherished but mistaken principles. So anything that can be described either literally or metaphorically as a religion will have valuable unexplored ideas in its shadow. Copernicus and Darwin both made discoveries of this type. 228 | 229 | What are people in your field religious about, in the sense of being too attached to some principle that might not be as self-evident as they think? What becomes possible if you discard it? 230 | 231 | People show much more originality in solving problems than in deciding which problems to solve. Even the smartest can be surprisingly conservative when deciding what to work on. People who'd never dream of being fashionable in any other way get sucked into working on fashionable problems. 232 | 233 | One reason people are more conservative when choosing problems than solutions is that problems are bigger bets. A problem could occupy you for years, while exploring a solution might only take days. But even so I think most people are too conservative. They're not merely responding to risk, but to fashion as well. Unfashionable problems are undervalued. 234 | 235 | One of the most interesting kinds of unfashionable problem is the problem that people think has been fully explored, but hasn't. Great work often takes something that already exists and shows its latent potential. Durer and Watt both did this. So if you're interested in a field that others think is tapped out, don't let their skepticism deter you. People are often wrong about this. 236 | 237 | Working on an unfashionable problem can be very pleasing. There's no hype or hurry. Opportunists and critics are both occupied elsewhere. The existing work often has an old-school solidity. And there's a satisfying sense of economy in cultivating ideas that would otherwise be wasted. 238 | 239 | But the most common type of overlooked problem is not explicitly unfashionable in the sense of being out of fashion. It just doesn't seem to matter as much as it actually does. How do you find these? By being self-indulgent — by letting your curiosity have its way, and tuning out, at least temporarily, the little voice in your head that says you should only be working on "important" problems. 240 | 241 | You do need to work on important problems, but almost everyone is too conservative about what counts as one. And if there's an important but overlooked problem in your neighborhood, it's probably already on your subconscious radar screen. So try asking yourself: if you were going to take a break from "serious" work to work on something just because it would be really interesting, what would you do? The answer is probably more important than it seems. 242 | 243 | Originality in choosing problems seems to matter even more than originality in solving them. That's what distinguishes the people who discover whole new fields. So what might seem to be merely the initial step — deciding what to work on — is in a sense the key to the whole game. 244 | 245 | Few grasp this. One of the biggest misconceptions about new ideas is about the ratio of question to answer in their composition. People think big ideas are answers, but often the real insight was in the question. 246 | 247 | Part of the reason we underrate questions is the way they're used in schools. In schools they tend to exist only briefly before being answered, like unstable particles. But a really good question can be much more than that. A really good question is a partial discovery. How do new species arise? Is the force that makes objects fall to earth the same as the one that keeps planets in their orbits? By even asking such questions you were already in excitingly novel territory. 248 | 249 | Unanswered questions can be uncomfortable things to carry around with you. But the more you're carrying, the greater the chance of noticing a solution — or perhaps even more excitingly, noticing that two unanswered questions are the same. 250 | 251 | Sometimes you carry a question for a long time. Great work often comes from returning to a question you first noticed years before — in your childhood, even — and couldn't stop thinking about. People talk a lot about the importance of keeping your youthful dreams alive, but it's just as important to keep your youthful questions alive. 252 | 253 | This is one of the places where actual expertise differs most from the popular picture of it. In the popular picture, experts are certain. But actually the more puzzled you are, the better, so long as (a) the things you're puzzled about matter, and (b) no one else understands them either. 254 | 255 | Think about what's happening at the moment just before a new idea is discovered. Often someone with sufficient expertise is puzzled about something. Which means that originality consists partly of puzzlement — of confusion! You have to be comfortable enough with the world being full of puzzles that you're willing to see them, but not so comfortable that you don't want to solve them. 256 | 257 | It's a great thing to be rich in unanswered questions. And this is one of those situations where the rich get richer, because the best way to acquire new questions is to try answering existing ones. Questions don't just lead to answers, but also to more questions. 258 | 259 | The best questions grow in the answering. You notice a thread protruding from the current paradigm and try pulling on it, and it just gets longer and longer. So don't require a question to be obviously big before you try answering it. You can rarely predict that. It's hard enough even to notice the thread, let alone to predict how much will unravel if you pull on it. 260 | 261 | It's better to be promiscuously curious — to pull a little bit on a lot of threads, and see what happens. Big things start small. The initial versions of big things were often just experiments, or side projects, or talks, which then grew into something bigger. So start lots of small things. 262 | 263 | Being prolific is underrated. The more different things you try, the greater the chance of discovering something new. Understand, though, that trying lots of things will mean trying lots of things that don't work. You can't have a lot of good ideas without also having a lot of bad ones. 264 | 265 | Though it sounds more responsible to begin by studying everything that's been done before, you'll learn faster and have more fun by trying stuff. And you'll understand previous work better when you do look at it. So err on the side of starting. Which is easier when starting means starting small; those two ideas fit together like two puzzle pieces. 266 | 267 | How do you get from starting small to doing something great? By making successive versions. Great things are almost always made in successive versions. You start with something small and evolve it, and the final version is both cleverer and more ambitious than anything you could have planned. 268 | 269 | It's particularly useful to make successive versions when you're making something for people — to get an initial version in front of them quickly, and then evolve it based on their response. 270 | 271 | Begin by trying the simplest thing that could possibly work. Surprisingly often, it does. If it doesn't, this will at least get you started. 272 | 273 | Don't try to cram too much new stuff into any one version. There are names for doing this with the first version (taking too long to ship) and the second (the second system effect), but these are both merely instances of a more general principle. 274 | 275 | An early version of a new project will sometimes be dismissed as a toy. It's a good sign when people do this. That means it has everything a new idea needs except scale, and that tends to follow. 276 | 277 | The alternative to starting with something small and evolving it is to plan in advance what you're going to do. And planning does usually seem the more responsible choice. It sounds more organized to say "we're going to do x and then y and then z" than "we're going to try x and see what happens." And it is more organized; it just doesn't work as well. 278 | 279 | Planning per se isn't good. It's sometimes necessary, but it's a necessary evil — a response to unforgiving conditions. It's something you have to do because you're working with inflexible media, or because you need to coordinate the efforts of a lot of people. If you keep projects small and use flexible media, you don't have to plan as much, and your designs can evolve instead. 280 | 281 | Take as much risk as you can afford. In an efficient market, risk is proportionate to reward, so don't look for certainty, but for a bet with high expected value. If you're not failing occasionally, you're probably being too conservative. 282 | 283 | Though conservatism is usually associated with the old, it's the young who tend to make this mistake. Inexperience makes them fear risk, but it's when you're young that you can afford the most. 284 | 285 | Even a project that fails can be valuable. In the process of working on it, you'll have crossed territory few others have seen, and encountered questions few others have asked. And there's probably no better source of questions than the ones you encounter in trying to do something slightly too hard. 286 | 287 | Use the advantages of youth when you have them, and the advantages of age once you have those. The advantages of youth are energy, time, optimism, and freedom. The advantages of age are knowledge, efficiency, money, and power. With effort you can acquire some of the latter when young and keep some of the former when old. 288 | 289 | The old also have the advantage of knowing which advantages they have. The young often have them without realizing it. The biggest is probably time. The young have no idea how rich they are in time. The best way to turn this time to advantage is to use it in slightly frivolous ways: to learn about something you don't need to know about, just out of curiosity, or to try building something just because it would be cool, or to become freakishly good at something. 290 | 291 | That "slightly" is an important qualification. Spend time lavishly when you're young, but don't simply waste it. There's a big difference between doing something you worry might be a waste of time and doing something you know for sure will be. The former is at least a bet, and possibly a better one than you think. 292 | 293 | The most subtle advantage of youth, or more precisely of inexperience, is that you're seeing everything with fresh eyes. When your brain embraces an idea for the first time, sometimes the two don't fit together perfectly. Usually the problem is with your brain, but occasionally it's with the idea. A piece of it sticks out awkwardly and jabs you when you think about it. People who are used to the idea have learned to ignore it, but you have the opportunity not to. 294 | 295 | So when you're learning about something for the first time, pay attention to things that seem wrong or missing. You'll be tempted to ignore them, since there's a 99% chance the problem is with you. And you may have to set aside your misgivings temporarily to keep progressing. But don't forget about them. When you've gotten further into the subject, come back and check if they're still there. If they're still viable in the light of your present knowledge, they probably represent an undiscovered idea. 296 | 297 | One of the most valuable kinds of knowledge you get from experience is to know what you don't have to worry about. The young know all the things that could matter, but not their relative importance. So they worry equally about everything, when they should worry much more about a few things and hardly at all about the rest. 298 | 299 | But what you don't know is only half the problem with inexperience. The other half is what you do know that ain't so. You arrive at adulthood with your head full of nonsense — bad habits you've acquired and false things you've been taught — and you won't be able to do great work till you clear away at least the nonsense in the way of whatever type of work you want to do. 300 | 301 | Much of the nonsense left in your head is left there by schools. We're so used to schools that we unconsciously treat going to school as identical with learning, but in fact schools have all sorts of strange qualities that warp our ideas about learning and thinking. 302 | 303 | For example, schools induce passivity. Since you were a small child, there was an authority at the front of the class telling all of you what you had to learn and then measuring whether you did. But neither classes nor tests are intrinsic to learning; they're just artifacts of the way schools are usually designed. 304 | 305 | The sooner you overcome this passivity, the better. If you're still in school, try thinking of your education as your project, and your teachers as working for you rather than vice versa. That may seem a stretch, but it's not merely some weird thought experiment. It's the truth, economically, and in the best case it's the truth intellectually as well. The best teachers don't want to be your bosses. They'd prefer it if you pushed ahead, using them as a source of advice, rather than being pulled by them through the material. 306 | 307 | Schools also give you a misleading impression of what work is like. In school they tell you what the problems are, and they're almost always soluble using no more than you've been taught so far. In real life you have to figure out what the problems are, and you often don't know if they're soluble at all. 308 | 309 | But perhaps the worst thing schools do to you is train you to win by hacking the test. You can't do great work by doing that. You can't trick God. So stop looking for that kind of shortcut. The way to beat the system is to focus on problems and solutions that others have overlooked, not to skimp on the work itself. 310 | 311 | Don't think of yourself as dependent on some gatekeeper giving you a "big break." Even if this were true, the best way to get it would be to focus on doing good work rather than chasing influential people. 312 | 313 | And don't take rejection by committees to heart. The qualities that impress admissions officers and prize committees are quite different from those required to do great work. The decisions of selection committees are only meaningful to the extent that they're part of a feedback loop, and very few are. 314 | 315 | People new to a field will often copy existing work. There's nothing inherently bad about that. There's no better way to learn how something works than by trying to reproduce it. Nor does copying necessarily make your work unoriginal. Originality is the presence of new ideas, not the absence of old ones. 316 | 317 | There's a good way to copy and a bad way. If you're going to copy something, do it openly instead of furtively, or worse still, unconsciously. This is what's meant by the famously misattributed phrase "Great artists steal." The really dangerous kind of copying, the kind that gives copying a bad name, is the kind that's done without realizing it, because you're nothing more than a train running on tracks laid down by someone else. But at the other extreme, copying can be a sign of superiority rather than subordination. 318 | 319 | In many fields it's almost inevitable that your early work will be in some sense based on other people's. Projects rarely arise in a vacuum. They're usually a reaction to previous work. When you're first starting out, you don't have any previous work; if you're going to react to something, it has to be someone else's. Once you're established, you can react to your own. But while the former gets called derivative and the latter doesn't, structurally the two cases are more similar than they seem. 320 | 321 | Oddly enough, the very novelty of the most novel ideas sometimes makes them seem at first to be more derivative than they are. New discoveries often have to be conceived initially as variations of existing things, even by their discoverers, because there isn't yet the conceptual vocabulary to express them. 322 | 323 | There are definitely some dangers to copying, though. One is that you'll tend to copy old things — things that were in their day at the frontier of knowledge, but no longer are. 324 | 325 | And when you do copy something, don't copy every feature of it. Some will make you ridiculous if you do. Don't copy the manner of an eminent 50 year old professor if you're 18, for example, or the idiom of a Renaissance poem hundreds of years later. 326 | 327 | Some of the features of things you admire are flaws they succeeded despite. Indeed, the features that are easiest to imitate are the most likely to be the flaws. 328 | 329 | This is particularly true for behavior. Some talented people are jerks, and this sometimes makes it seem to the inexperienced that being a jerk is part of being talented. It isn't; being talented is merely how they get away with it. 330 | 331 | One of the most powerful kinds of copying is to copy something from one field into another. History is so full of chance discoveries of this type that it's probably worth giving chance a hand by deliberately learning about other kinds of work. You can take ideas from quite distant fields if you let them be metaphors. 332 | 333 | Negative examples can be as inspiring as positive ones. In fact you can sometimes learn more from things done badly than from things done well; sometimes it only becomes clear what's needed when it's missing. 334 | 335 | If a lot of the best people in your field are collected in one place, it's usually a good idea to visit for a while. It will increase your ambition, and also, by showing you that these people are human, increase your self-confidence. 336 | 337 | If you're earnest you'll probably get a warmer welcome than you might expect. Most people who are very good at something are happy to talk about it with anyone who's genuinely interested. If they're really good at their work, then they probably have a hobbyist's interest in it, and hobbyists always want to talk about their hobbies. 338 | 339 | It may take some effort to find the people who are really good, though. Doing great work has such prestige that in some places, particularly universities, there's a polite fiction that everyone is engaged in it. And that is far from true. People within universities can't say so openly, but the quality of the work being done in different departments varies immensely. Some departments have people doing great work; others have in the past; others never have. 340 | 341 | Seek out the best colleagues. There are a lot of projects that can't be done alone, and even if you're working on one that can be, it's good to have other people to encourage you and to bounce ideas off. 342 | 343 | Colleagues don't just affect your work, though; they also affect you. So work with people you want to become like, because you will. 344 | 345 | Quality is more important than quantity in colleagues. It's better to have one or two great ones than a building full of pretty good ones. In fact it's not merely better, but necessary, judging from history: the degree to which great work happens in clusters suggests that one's colleagues often make the difference between doing great work and not. 346 | 347 | How do you know when you have sufficiently good colleagues? In my experience, when you do, you know. Which means if you're unsure, you probably don't. But it may be possible to give a more concrete answer than that. Here's an attempt: sufficiently good colleagues offer surprising insights. They can see and do things that you can't. So if you have a handful of colleagues good enough to keep you on your toes in this sense, you're probably over the threshold. 348 | 349 | Most of us can benefit from collaborating with colleagues, but some projects require people on a larger scale, and starting one of those is not for everyone. If you want to run a project like that, you'll have to become a manager, and managing well takes aptitude and interest like any other kind of work. If you don't have them, there is no middle path: you must either force yourself to learn management as a second language, or avoid such projects. 350 | 351 | Husband your morale. It's the basis of everything when you're working on ambitious projects. You have to nurture and protect it like a living organism. 352 | 353 | Morale starts with your view of life. You're more likely to do great work if you're an optimist, and more likely to if you think of yourself as lucky than if you think of yourself as a victim. 354 | 355 | Indeed, work can to some extent protect you from your problems. If you choose work that's pure, its very difficulties will serve as a refuge from the difficulties of everyday life. If this is escapism, it's a very productive form of it, and one that has been used by some of the greatest minds in history. 356 | 357 | Morale compounds via work: high morale helps you do good work, which increases your morale and helps you do even better work. But this cycle also operates in the other direction: if you're not doing good work, that can demoralize you and make it even harder to. Since it matters so much for this cycle to be running in the right direction, it can be a good idea to switch to easier work when you're stuck, just so you start to get something done. 358 | 359 | One of the biggest mistakes ambitious people make is to allow setbacks to destroy their morale all at once, like a balloon bursting. You can inoculate yourself against this by explicitly considering setbacks a part of your process. Solving hard problems always involves some backtracking. 360 | 361 | Doing great work is a depth-first search whose root node is the desire to. So "If at first you don't succeed, try, try again" isn't quite right. It should be: If at first you don't succeed, either try again, or backtrack and then try again. 362 | 363 | "Never give up" is also not quite right. Obviously there are times when it's the right choice to eject. A more precise version would be: Never let setbacks panic you into backtracking more than you need to. Corollary: Never abandon the root node. 364 | 365 | It's not necessarily a bad sign if work is a struggle, any more than it's a bad sign to be out of breath while running. It depends how fast you're running. So learn to distinguish good pain from bad. Good pain is a sign of effort; bad pain is a sign of damage. 366 | 367 | An audience is a critical component of morale. If you're a scholar, your audience may be your peers; in the arts, it may be an audience in the traditional sense. Either way it doesn't need to be big. The value of an audience doesn't grow anything like linearly with its size. Which is bad news if you're famous, but good news if you're just starting out, because it means a small but dedicated audience can be enough to sustain you. If a handful of people genuinely love what you're doing, that's enough. 368 | 369 | To the extent you can, avoid letting intermediaries come between you and your audience. In some types of work this is inevitable, but it's so liberating to escape it that you might be better off switching to an adjacent type if that will let you go direct. 370 | 371 | The people you spend time with will also have a big effect on your morale. You'll find there are some who increase your energy and others who decrease it, and the effect someone has is not always what you'd expect. Seek out the people who increase your energy and avoid those who decrease it. Though of course if there's someone you need to take care of, that takes precedence. 372 | 373 | Don't marry someone who doesn't understand that you need to work, or sees your work as competition for your attention. If you're ambitious, you need to work; it's almost like a medical condition; so someone who won't let you work either doesn't understand you, or does and doesn't care. 374 | 375 | Ultimately morale is physical. You think with your body, so it's important to take care of it. That means exercising regularly, eating and sleeping well, and avoiding the more dangerous kinds of drugs. Running and walking are particularly good forms of exercise because they're good for thinking. 376 | 377 | People who do great work are not necessarily happier than everyone else, but they're happier than they'd be if they didn't. In fact, if you're smart and ambitious, it's dangerous not to be productive. People who are smart and ambitious but don't achieve much tend to become bitter. 378 | 379 | It's ok to want to impress other people, but choose the right people. The opinion of people you respect is signal. Fame, which is the opinion of a much larger group you might or might not respect, just adds noise. 380 | 381 | The prestige of a type of work is at best a trailing indicator and sometimes completely mistaken. If you do anything well enough, you'll make it prestigious. So the question to ask about a type of work is not how much prestige it has, but how well it could be done. 382 | 383 | Competition can be an effective motivator, but don't let it choose the problem for you; don't let yourself get drawn into chasing something just because others are. In fact, don't let competitors make you do anything much more specific than work harder. 384 | 385 | Curiosity is the best guide. Your curiosity never lies, and it knows more than you do about what's worth paying attention to. 386 | 387 | Notice how often that word has come up. If you asked an oracle the secret to doing great work and the oracle replied with a single word, my bet would be on "curiosity." 388 | 389 | That doesn't translate directly to advice. It's not enough just to be curious, and you can't command curiosity anyway. But you can nurture it and let it drive you. 390 | 391 | Curiosity is the key to all four steps in doing great work: it will choose the field for you, get you to the frontier, cause you to notice the gaps in it, and drive you to explore them. The whole process is a kind of dance with curiosity. 392 | 393 | Believe it or not, I tried to make this essay as short as I could. But its length at least means it acts as a filter. If you made it this far, you must be interested in doing great work. And if so you're already further along than you might realize, because the set of people willing to want to is small. 394 | 395 | The factors in doing great work are factors in the literal, mathematical sense, and they are: ability, interest, effort, and luck. Luck by definition you can't do anything about, so we can ignore that. And we can assume effort, if you do in fact want to do great work. So the problem boils down to ability and interest. Can you find a kind of work where your ability and interest will combine to yield an explosion of new ideas? 396 | 397 | Here there are grounds for optimism. There are so many different ways to do great work, and even more that are still undiscovered. Out of all those different types of work, the one you're most suited for is probably a pretty close match. Probably a comically close match. It's just a question of finding it, and how far into it your ability and interest can take you. And you can only answer that by trying. 398 | 399 | Many more people could try to do great work than do. What holds them back is a combination of modesty and fear. It seems presumptuous to try to be Newton or Shakespeare. It also seems hard; surely if you tried something like that, you'd fail. Presumably the calculation is rarely explicit. Few people consciously decide not to try to do great work. But that's what's going on subconsciously; they shy away from the question. 400 | 401 | So I'm going to pull a sneaky trick on you. Do you want to do great work, or not? Now you have to decide consciously. Sorry about that. I wouldn't have done it to a general audience. But we already know you're interested. 402 | 403 | Don't worry about being presumptuous. You don't have to tell anyone. And if it's too hard and you fail, so what? Lots of people have worse problems than that. In fact you'll be lucky if it's the worst problem you have. 404 | 405 | Yes, you'll have to work hard. But again, lots of people have to work hard. And if you're working on something you find very interesting, which you necessarily will if you're on the right path, the work will probably feel less burdensome than a lot of your peers'. 406 | 407 | The discoveries are out there, waiting to be made. Why not by you? 408 | -------------------------------------------------------------------------------- /data/how-to-start-google.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How To Start Google 3 | date: 2024-03 4 | url: https://www.paulgraham.com/google.html 5 | --- 6 | 7 | (This is a talk I gave to 14 and 15 year olds about what to do now if they might want to start a startup later. Lots of schools think they should tell students something about startups. This is what I think they should tell them.) 8 | 9 | Most of you probably think that when you're released into the so-called real world you'll eventually have to get some kind of job. That's not true, and today I'm going to talk about a trick you can use to avoid ever having to get a job. 10 | 11 | The trick is to start your own company. So it's not a trick for avoiding work, because if you start your own company you'll work harder than you would if you had an ordinary job. But you will avoid many of the annoying things that come with a job, including a boss telling you what to do. 12 | 13 | It's more exciting to work on your own project than someone else's. And you can also get a lot richer. In fact, this is the standard way to get really rich. If you look at the lists of the richest people that occasionally get published in the press, nearly all of them did it by starting their own companies. 14 | 15 | Starting your own company can mean anything from starting a barber shop to starting Google. I'm here to talk about one extreme end of that continuum. I'm going to tell you how to start Google. 16 | 17 | The companies at the Google end of the continuum are called startups when they're young. The reason I know about them is that my wife Jessica and I started something called Y Combinator that is basically a startup factory. Since 2005, Y Combinator has funded over 4000 startups. So we know exactly what you need to start a startup, because we've helped people do it for the last 19 years. 18 | 19 | You might have thought I was joking when I said I was going to tell you how to start Google. You might be thinking "How could we start Google?" But that's effectively what the people who did start Google were thinking before they started it. If you'd told Larry Page and Sergey Brin, the founders of Google, that the company they were about to start would one day be worth over a trillion dollars, their heads would have exploded. 20 | 21 | All you can know when you start working on a startup is that it seems worth pursuing. You can't know whether it will turn into a company worth billions or one that goes out of business. So when I say I'm going to tell you how to start Google, I mean I'm going to tell you how to get to the point where you can start a company that has as much chance of being Google as Google had of being Google. 22 | 23 | How do you get from where you are now to the point where you can start a successful startup? You need three things. You need to be good at some kind of technology, you need an idea for what you're going to build, and you need cofounders to start the company with. 24 | 25 | How do you get good at technology? And how do you choose which technology to get good at? Both of those questions turn out to have the same answer: work on your own projects. Don't try to guess whether gene editing or LLMs or rockets will turn out to be the most valuable technology to know about. No one can predict that. Just work on whatever interests you the most. You'll work much harder on something you're interested in than something you're doing because you think you're supposed to. 26 | 27 | If you're not sure what technology to get good at, get good at programming. That has been the source of the median startup for the last 30 years, and this is probably not going to change in the next 10. 28 | 29 | Those of you who are taking computer science classes in school may at this point be thinking, ok, we've got this sorted. We're already being taught all about programming. But sorry, this is not enough. You have to be working on your own projects, not just learning stuff in classes. You can do well in computer science classes without ever really learning to program. In fact you can graduate with a degree in computer science from a top university and still not be any good at programming. That's why tech companies all make you take a coding test before they'll hire you, regardless of where you went to university or how well you did there. They know grades and exam results prove nothing. 30 | 31 | If you really want to learn to program, you have to work on your own projects. You learn so much faster that way. Imagine you're writing a game and there's something you want to do in it, and you don't know how. You're going to figure out how a lot faster than you'd learn anything in a class. 32 | 33 | You don't have to learn programming, though. If you're wondering what counts as technology, it includes practically everything you could describe using the words "make" or "build." So welding would count, or making clothes, or making videos. Whatever you're most interested in. The critical distinction is whether you're producing or just consuming. Are you writing computer games, or just playing them? That's the cutoff. 34 | 35 | Steve Jobs, the founder of Apple, spent time when he was a teenager studying calligraphy — the sort of beautiful writing that you see in medieval manuscripts. No one, including him, thought that this would help him in his career. He was just doing it because he was interested in it. But it turned out to help him a lot. The computer that made Apple really big, the Macintosh, came out at just the moment when computers got powerful enough to make letters like the ones in printed books instead of the computery-looking letters you see in 8 bit games. Apple destroyed everyone else at this, and one reason was that Steve was one of the few people in the computer business who really got graphic design. 36 | 37 | Don't feel like your projects have to be serious. They can be as frivolous as you like, so long as you're building things you're excited about. Probably 90% of programmers start out building games. They and their friends like to play games. So they build the kind of things they and their friends want. And that's exactly what you should be doing at 15 if you want to start a startup one day. 38 | 39 | You don't have to do just one project. In fact it's good to learn about multiple things. Steve Jobs didn't just learn calligraphy. He also learned about electronics, which was even more valuable. Whatever you're interested in. (Do you notice a theme here?) 40 | 41 | So that's the first of the three things you need, to get good at some kind or kinds of technology. You do it the same way you get good at the violin or football: practice. If you start a startup at 22, and you start writing your own programs now, then by the time you start the company you'll have spent at least 7 years practicing writing code, and you can get pretty good at anything after practicing it for 7 years. 42 | 43 | Let's suppose you're 22 and you've succeeded: You're now really good at some technology. How do you get startup ideas? It might seem like that's the hard part. Even if you are a good programmer, how do you get the idea to start Google? 44 | 45 | Actually it's easy to get startup ideas once you're good at technology. Once you're good at some technology, when you look at the world you see dotted outlines around the things that are missing. You start to be able to see both the things that are missing from the technology itself, and all the broken things that could be fixed using it, and each one of these is a potential startup. 46 | 47 | In the town near our house there's a shop with a sign warning that the door is hard to close. The sign has been there for several years. To the people in the shop it must seem like this mysterious natural phenomenon that the door sticks, and all they can do is put up a sign warning customers about it. But any carpenter looking at this situation would think "why don't you just plane off the part that sticks?" 48 | 49 | Once you're good at programming, all the missing software in the world starts to become as obvious as a sticking door to a carpenter. I'll give you a real world example. Back in the 20th century, American universities used to publish printed directories with all the students' names and contact info. When I tell you what these directories were called, you'll know which startup I'm talking about. They were called facebooks, because they usually had a picture of each student next to their name. 50 | 51 | So Mark Zuckerberg shows up at Harvard in 2002, and the university still hasn't gotten the facebook online. Each individual house has an online facebook, but there isn't one for the whole university. The university administration has been diligently having meetings about this, and will probably have solved the problem in another decade or so. Most of the students don't consciously notice that anything is wrong. But Mark is a programmer. He looks at this situation and thinks "Well, this is stupid. I could write a program to fix this in one night. Just let people upload their own photos and then combine the data into a new site for the whole university." So he does. And almost literally overnight he has thousands of users. 52 | 53 | Of course Facebook was not a startup yet. It was just a... project. There's that word again. Projects aren't just the best way to learn about technology. They're also the best source of startup ideas. 54 | 55 | Facebook was not unusual in this respect. Apple and Google also began as projects. Apple wasn't meant to be a company. Steve Wozniak just wanted to build his own computer. It only turned into a company when Steve Jobs said "Hey, I wonder if we could sell plans for this computer to other people." That's how Apple started. They weren't even selling computers, just plans for computers. Can you imagine how lame this company seemed? 56 | 57 | Ditto for Google. Larry and Sergey weren't trying to start a company at first. They were just trying to make search better. Before Google, most search engines didn't try to sort the results they gave you in order of importance. If you searched for "rugby" they just gave you every web page that contained the word "rugby." And the web was so small in 1997 that this actually worked! Kind of. There might only be 20 or 30 pages with the word "rugby," but the web was growing exponentially, which meant this way of doing search was becoming exponentially more broken. Most users just thought, "Wow, I sure have to look through a lot of search results to find what I want." Door sticks. But like Mark, Larry and Sergey were programmers. Like Mark, they looked at this situation and thought "Well, this is stupid. Some pages about rugby matter more than others. Let's figure out which those are and show them first." 58 | 59 | It's obvious in retrospect that this was a great idea for a startup. It wasn't obvious at the time. It's never obvious. If it was obviously a good idea to start Apple or Google or Facebook, someone else would have already done it. That's why the best startups grow out of projects that aren't meant to be startups. You're not trying to start a company. You're just following your instincts about what's interesting. And if you're young and good at technology, then your unconscious instincts about what's interesting are better than your conscious ideas about what would be a good company. 60 | 61 | So it's critical, if you're a young founder, to build things for yourself and your friends to use. The biggest mistake young founders make is to build something for some mysterious group of other people. But if you can make something that you and your friends truly want to use — something your friends aren't just using out of loyalty to you, but would be really sad to lose if you shut it down — then you almost certainly have the germ of a good startup idea. It may not seem like a startup to you. It may not be obvious how to make money from it. But trust me, there's a way. 62 | 63 | What you need in a startup idea, and all you need, is something your friends actually want. And those ideas aren't hard to see once you're good at technology. There are sticking doors everywhere. 64 | 65 | Now for the third and final thing you need: a cofounder, or cofounders. The optimal startup has two or three founders, so you need one or two cofounders. How do you find them? Can you predict what I'm going to say next? It's the same thing: projects. You find cofounders by working on projects with them. What you need in a cofounder is someone who's good at what they do and that you work well with, and the only way to judge this is to work with them on things. 66 | 67 | At this point I'm going to tell you something you might not want to hear. It really matters to do well in your classes, even the ones that are just memorization or blathering about literature, because you need to do well in your classes to get into a good university. And if you want to start a startup you should try to get into the best university you can, because that's where the best cofounders are. It's also where the best employees are. When Larry and Sergey started Google, they began by just hiring all the smartest people they knew out of Stanford, and this was a real advantage for them. 68 | 69 | The empirical evidence is clear on this. If you look at where the largest numbers of successful startups come from, it's pretty much the same as the list of the most selective universities. 70 | 71 | I don't think it's the prestigious names of these universities that cause more good startups to come out of them. Nor do I think it's because the quality of the teaching is better. What's driving this is simply the difficulty of getting in. You have to be pretty smart and determined to get into MIT or Cambridge, so if you do manage to get in, you'll find the other students include a lot of smart and determined people. 72 | 73 | You don't have to start a startup with someone you meet at university. The founders of Twitch met when they were seven. The founders of Stripe, Patrick and John Collison, met when John was born. But universities are the main source of cofounders. And because they're where the cofounders are, they're also where the ideas are, because the best ideas grow out of projects you do with the people who become your cofounders. 74 | 75 | So the list of what you need to do to get from here to starting a startup is quite short. You need to get good at technology, and the way to do that is to work on your own projects. And you need to do as well in school as you can, so you can get into a good university, because that's where the cofounders and the ideas are. 76 | 77 | That's it, just two things, build stuff and do well in school. -------------------------------------------------------------------------------- /data/how-to-work-hard.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How to work hard 3 | date: 2021-06 4 | url: https://www.paulgraham.com/hwh.html 5 | --- 6 | It might not seem there's much to learn about how to work hard. Anyone who's been to school knows what it entails, even if they chose not to do it. There are 12 year olds who work amazingly hard. And yet when I ask if I know more about working hard now than when I was in school, the answer is definitely yes. 7 | 8 | One thing I know is that if you want to do great things, you'll have to work very hard. I wasn't sure of that as a kid. Schoolwork varied in difficulty; one didn't always have to work super hard to do well. And some of the things famous adults did, they seemed to do almost effortlessly. Was there, perhaps, some way to evade hard work through sheer brilliance? Now I know the answer to that question. There isn't. 9 | 10 | The reason some subjects seemed easy was that my school had low standards. And the reason famous adults seemed to do things effortlessly was years of practice; they made it look easy. 11 | 12 | Of course, those famous adults usually had a lot of natural ability too. There are three ingredients in great work: natural ability, practice, and effort. You can do pretty well with just two, but to do the best work you need all three: you need great natural ability and to have practiced a lot and to be trying very hard. 13 | 14 | Bill Gates, for example, was among the smartest people in business in his era, but he was also among the hardest working. "I never took a day off in my twenties," he said. "Not one." It was similar with Lionel Messi. He had great natural ability, but when his youth coaches talk about him, what they remember is not his talent but his dedication and his desire to win. P. G. Wodehouse would probably get my vote for best English writer of the 20th century, if I had to choose. Certainly no one ever made it look easier. But no one ever worked harder. At 74, he wrote 15 | with each new book of mine I have, as I say, the feeling that this time I have picked a lemon in the garden of literature. A good thing, really, I suppose. Keeps one up on one's toes and makes one rewrite every sentence ten times. Or in many cases twenty times. 16 | Sounds a bit extreme, you think. And yet Bill Gates sounds even more extreme. Not one day off in ten years? These two had about as much natural ability as anyone could have, and yet they also worked about as hard as anyone could work. You need both. 17 | 18 | That seems so obvious, and yet in practice we find it slightly hard to grasp. There's a faint xor between talent and hard work. It comes partly from popular culture, where it seems to run very deep, and partly from the fact that the outliers are so rare. If great talent and great drive are both rare, then people with both are rare squared. Most people you meet who have a lot of one will have less of the other. But you'll need both if you want to be an outlier yourself. And since you can't really change how much natural talent you have, in practice doing great work, insofar as you can, reduces to working very hard. 19 | 20 | It's straightforward to work hard if you have clearly defined, externally imposed goals, as you do in school. There is some technique to it: you have to learn not to lie to yourself, not to procrastinate (which is a form of lying to yourself), not to get distracted, and not to give up when things go wrong. But this level of discipline seems to be within the reach of quite young children, if they want it. 21 | 22 | What I've learned since I was a kid is how to work toward goals that are neither clearly defined nor externally imposed. You'll probably have to learn both if you want to do really great things. 23 | 24 | The most basic level of which is simply to feel you should be working without anyone telling you to. Now, when I'm not working hard, alarm bells go off. I can't be sure I'm getting anywhere when I'm working hard, but I can be sure I'm getting nowhere when I'm not, and it feels awful. 25 | 26 | There wasn't a single point when I learned this. Like most little kids, I enjoyed the feeling of achievement when I learned or did something new. As I grew older, this morphed into a feeling of disgust when I wasn't achieving anything. The one precisely dateable landmark I have is when I stopped watching TV, at age 13. 27 | 28 | Several people I've talked to remember getting serious about work around this age. When I asked Patrick Collison when he started to find idleness distasteful, he said 29 | I think around age 13 or 14. I have a clear memory from around then of sitting in the sitting room, staring outside, and wondering why I was wasting my summer holiday. 30 | Perhaps something changes at adolescence. That would make sense. 31 | 32 | Strangely enough, the biggest obstacle to getting serious about work was probably school, which made work (what they called work) seem boring and pointless. I had to learn what real work was before I could wholeheartedly desire to do it. That took a while, because even in college a lot of the work is pointless; there are entire departments that are pointless. But as I learned the shape of real work, I found that my desire to do it slotted into it as if they'd been made for each other. 33 | 34 | I suspect most people have to learn what work is before they can love it. Hardy wrote eloquently about this in A Mathematician's Apology: 35 | I do not remember having felt, as a boy, any passion for mathematics, and such notions as I may have had of the career of a mathematician were far from noble. I thought of mathematics in terms of examinations and scholarships: I wanted to beat other boys, and this seemed to be the way in which I could do so most decisively. 36 | He didn't learn what math was really about till part way through college, when he read Jordan's Cours d'analyse. 37 | I shall never forget the astonishment with which I read that remarkable work, the first inspiration for so many mathematicians of my generation, and learnt for the first time as I read it what mathematics really meant. 38 | There are two separate kinds of fakeness you need to learn to discount in order to understand what real work is. One is the kind Hardy encountered in school. Subjects get distorted when they're adapted to be taught to kids — often so distorted that they're nothing like the work done by actual practitioners. The other kind of fakeness is intrinsic to certain types of work. Some types of work are inherently bogus, or at best mere busywork. 39 | 40 | There's a kind of solidity to real work. It's not all writing the Principia, but it all feels necessary. That's a vague criterion, but it's deliberately vague, because it has to cover a lot of different types. 41 | 42 | Once you know the shape of real work, you have to learn how many hours a day to spend on it. You can't solve this problem by simply working every waking hour, because in many kinds of work there's a point beyond which the quality of the result will start to decline. 43 | 44 | That limit varies depending on the type of work and the person. I've done several different kinds of work, and the limits were different for each. My limit for the harder types of writing or programming is about five hours a day. Whereas when I was running a startup, I could work all the time. At least for the three years I did it; if I'd kept going much longer, I'd probably have needed to take occasional vacations. 45 | 46 | The only way to find the limit is by crossing it. Cultivate a sensitivity to the quality of the work you're doing, and then you'll notice if it decreases because you're working too hard. Honesty is critical here, in both directions: you have to notice when you're being lazy, but also when you're working too hard. And if you think there's something admirable about working too hard, get that idea out of your head. You're not merely getting worse results, but getting them because you're showing off — if not to other people, then to yourself. 47 | 48 | Finding the limit of working hard is a constant, ongoing process, not something you do just once. Both the difficulty of the work and your ability to do it can vary hour to hour, so you need to be constantly judging both how hard you're trying and how well you're doing. 49 | 50 | Trying hard doesn't mean constantly pushing yourself to work, though. There may be some people who do, but I think my experience is fairly typical, and I only have to push myself occasionally when I'm starting a project or when I encounter some sort of check. That's when I'm in danger of procrastinating. But once I get rolling, I tend to keep going. 51 | 52 | What keeps me going depends on the type of work. When I was working on Viaweb, I was driven by fear of failure. I barely procrastinated at all then, because there was always something that needed doing, and if I could put more distance between me and the pursuing beast by doing it, why wait? Whereas what drives me now, writing essays, is the flaws in them. Between essays I fuss for a few days, like a dog circling while it decides exactly where to lie down. But once I get started on one, I don't have to push myself to work, because there's always some error or omission already pushing me. 53 | 54 | I do make some amount of effort to focus on important topics. Many problems have a hard core at the center, surrounded by easier stuff at the edges. Working hard means aiming toward the center to the extent you can. Some days you may not be able to; some days you'll only be able to work on the easier, peripheral stuff. But you should always be aiming as close to the center as you can without stalling. 55 | 56 | The bigger question of what to do with your life is one of these problems with a hard core. There are important problems at the center, which tend to be hard, and less important, easier ones at the edges. So as well as the small, daily adjustments involved in working on a specific problem, you'll occasionally have to make big, lifetime-scale adjustments about which type of work to do. And the rule is the same: working hard means aiming toward the center — toward the most ambitious problems. 57 | 58 | By center, though, I mean the actual center, not merely the current consensus about the center. The consensus about which problems are most important is often mistaken, both in general and within specific fields. If you disagree with it, and you're right, that could represent a valuable opportunity to do something new. 59 | 60 | The more ambitious types of work will usually be harder, but although you should not be in denial about this, neither should you treat difficulty as an infallible guide in deciding what to do. If you discover some ambitious type of work that's a bargain in the sense of being easier for you than other people, either because of the abilities you happen to have, or because of some new way you've found to approach it, or simply because you're more excited about it, by all means work on that. Some of the best work is done by people who find an easy way to do something hard. 61 | 62 | As well as learning the shape of real work, you need to figure out which kind you're suited for. And that doesn't just mean figuring out which kind your natural abilities match the best; it doesn't mean that if you're 7 feet tall, you have to play basketball. What you're suited for depends not just on your talents but perhaps even more on your interests. A deep interest in a topic makes people work harder than any amount of discipline can. 63 | 64 | It can be harder to discover your interests than your talents. There are fewer types of talent than interest, and they start to be judged early in childhood, whereas interest in a topic is a subtle thing that may not mature till your twenties, or even later. The topic may not even exist earlier. Plus there are some powerful sources of error you need to learn to discount. Are you really interested in x, or do you want to work on it because you'll make a lot of money, or because other people will be impressed with you, or because your parents want you to? 65 | 66 | The difficulty of figuring out what to work on varies enormously from one person to another. That's one of the most important things I've learned about work since I was a kid. As a kid, you get the impression that everyone has a calling, and all they have to do is figure out what it is. That's how it works in movies, and in the streamlined biographies fed to kids. Sometimes it works that way in real life. Some people figure out what to do as children and just do it, like Mozart. But others, like Newton, turn restlessly from one kind of work to another. Maybe in retrospect we can identify one as their calling — we can wish Newton spent more time on math and physics and less on alchemy and theology — but this is an illusion induced by hindsight bias. There was no voice calling to him that he could have heard. 67 | 68 | So while some people's lives converge fast, there will be others whose lives never converge. And for these people, figuring out what to work on is not so much a prelude to working hard as an ongoing part of it, like one of a set of simultaneous equations. For these people, the process I described earlier has a third component: along with measuring both how hard you're working and how well you're doing, you have to think about whether you should keep working in this field or switch to another. If you're working hard but not getting good enough results, you should switch. It sounds simple expressed that way, but in practice it's very difficult. You shouldn't give up on the first day just because you work hard and don't get anywhere. You need to give yourself time to get going. But how much time? And what should you do if work that was going well stops going well? How much time do you give yourself then? 69 | 70 | What even counts as good results? That can be really hard to decide. If you're exploring an area few others have worked in, you may not even know what good results look like. History is full of examples of people who misjudged the importance of what they were working on. 71 | 72 | The best test of whether it's worthwhile to work on something is whether you find it interesting. That may sound like a dangerously subjective measure, but it's probably the most accurate one you're going to get. You're the one working on the stuff. Who's in a better position than you to judge whether it's important, and what's a better predictor of its importance than whether it's interesting? 73 | 74 | For this test to work, though, you have to be honest with yourself. Indeed, that's the most striking thing about the whole question of working hard: how at each point it depends on being honest with yourself. 75 | 76 | Working hard is not just a dial you turn up to 11. It's a complicated, dynamic system that has to be tuned just right at each point. You have to understand the shape of real work, see clearly what kind you're best suited for, aim as close to the true core of it as you can, accurately judge at each moment both what you're capable of and how you're doing, and put in as many hours each day as you can without harming the quality of the result. This network is too complicated to trick. But if you're consistently honest and clear-sighted, it will automatically assume an optimal shape, and you'll be productive in a way few people are. -------------------------------------------------------------------------------- /data/superlinear-returns.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Superlinear Returns 3 | date: 2023-10 4 | url: https://paulgraham.com/superlinear.html 5 | --- 6 | 7 | One of the most important things I didn't understand about the world when I was a child is the degree to which the returns for performance are superlinear. 8 | 9 | Teachers and coaches implicitly told us the returns were linear. "You get out," I heard a thousand times, "what you put in." They meant well, but this is rarely true. If your product is only half as good as your competitor's, you don't get half as many customers. You get no customers, and you go out of business. 10 | 11 | It's obviously true that the returns for performance are superlinear in business. Some think this is a flaw of capitalism, and that if we changed the rules it would stop being true. But superlinear returns for performance are a feature of the world, not an artifact of rules we've invented. We see the same pattern in fame, power, military victories, knowledge, and even benefit to humanity. In all of these, the rich get richer. 12 | 13 | You can't understand the world without understanding the concept of superlinear returns. And if you're ambitious you definitely should, because this will be the wave you surf on. 14 | 15 | It may seem as if there are a lot of different situations with superlinear returns, but as far as I can tell they reduce to two fundamental causes: exponential growth and thresholds. 16 | 17 | The most obvious case of superlinear returns is when you're working on something that grows exponentially. For example, growing bacterial cultures. When they grow at all, they grow exponentially. But they're tricky to grow. Which means the difference in outcome between someone who's adept at it and someone who's not is very great. 18 | 19 | Startups can also grow exponentially, and we see the same pattern there. Some manage to achieve high growth rates. Most don't. And as a result you get qualitatively different outcomes: the companies with high growth rates tend to become immensely valuable, while the ones with lower growth rates may not even survive. 20 | 21 | Y Combinator encourages founders to focus on growth rate rather than absolute numbers. It prevents them from being discouraged early on, when the absolute numbers are still low. It also helps them decide what to focus on: you can use growth rate as a compass to tell you how to evolve the company. But the main advantage is that by focusing on growth rate you tend to get something that grows exponentially. 22 | 23 | YC doesn't explicitly tell founders that with growth rate "you get out what you put in," but it's not far from the truth. And if growth rate were proportional to performance, then the reward for performance p over time t would be proportional to pt. 24 | 25 | Even after decades of thinking about this, I find that sentence startling. 26 | 27 | Whenever how well you do depends on how well you've done, you'll get exponential growth. But neither our DNA nor our customs prepare us for it. No one finds exponential growth natural; every child is surprised, the first time they hear it, by the story of the man who asks the king for a single grain of rice the first day and double the amount each successive day. 28 | 29 | What we don't understand naturally we develop customs to deal with, but we don't have many customs about exponential growth either, because there have been so few instances of it in human history. In principle herding should have been one: the more animals you had, the more offspring they'd have. But in practice grazing land was the limiting factor, and there was no plan for growing that exponentially. 30 | 31 | Or more precisely, no generally applicable plan. There was a way to grow one's territory exponentially: by conquest. The more territory you control, the more powerful your army becomes, and the easier it is to conquer new territory. This is why history is full of empires. But so few people created or ran empires that their experiences didn't affect customs very much. The emperor was a remote and terrifying figure, not a source of lessons one could use in one's own life. 32 | 33 | The most common case of exponential growth in preindustrial times was probably scholarship. The more you know, the easier it is to learn new things. The result, then as now, was that some people were startlingly more knowledgeable than the rest about certain topics. But this didn't affect customs much either. Although empires of ideas can overlap and there can thus be far more emperors, in preindustrial times this type of empire had little practical effect. 34 | 35 | That has changed in the last few centuries. Now the emperors of ideas can design bombs that defeat the emperors of territory. But this phenomenon is still so new that we haven't fully assimilated it. Few even of the participants realize they're benefitting from exponential growth or ask what they can learn from other instances of it. 36 | 37 | The other source of superlinear returns is embodied in the expression "winner take all." In a sports match the relationship between performance and return is a step function: the winning team gets one win whether they do much better or just slightly better. 38 | 39 | The source of the step function is not competition per se, however. It's that there are thresholds in the outcome. You don't need competition to get those. There can be thresholds in situations where you're the only participant, like proving a theorem or hitting a target. 40 | 41 | It's remarkable how often a situation with one source of superlinear returns also has the other. Crossing thresholds leads to exponential growth: the winning side in a battle usually suffers less damage, which makes them more likely to win in the future. And exponential growth helps you cross thresholds: in a market with network effects, a company that grows fast enough can shut out potential competitors. 42 | 43 | Fame is an interesting example of a phenomenon that combines both sources of superlinear returns. Fame grows exponentially because existing fans bring you new ones. But the fundamental reason it's so concentrated is thresholds: there's only so much room on the A-list in the average person's head. 44 | 45 | The most important case combining both sources of superlinear returns may be learning. Knowledge grows exponentially, but there are also thresholds in it. Learning to ride a bicycle, for example. Some of these thresholds are akin to machine tools: once you learn to read, you're able to learn anything else much faster. But the most important thresholds of all are those representing new discoveries. Knowledge seems to be fractal in the sense that if you push hard at the boundary of one area of knowledge, you sometimes discover a whole new field. And if you do, you get first crack at all the new discoveries to be made in it. Newton did this, and so did Durer and Darwin. 46 | 47 | Are there general rules for finding situations with superlinear returns? The most obvious one is to seek work that compounds. 48 | 49 | There are two ways work can compound. It can compound directly, in the sense that doing well in one cycle causes you to do better in the next. That happens for example when you're building infrastructure, or growing an audience or brand. Or work can compound by teaching you, since learning compounds. This second case is an interesting one because you may feel you're doing badly as it's happening. You may be failing to achieve your immediate goal. But if you're learning a lot, then you're getting exponential growth nonetheless. 50 | 51 | This is one reason Silicon Valley is so tolerant of failure. People in Silicon Valley aren't blindly tolerant of failure. They'll only continue to bet on you if you're learning from your failures. But if you are, you are in fact a good bet: maybe your company didn't grow the way you wanted, but you yourself have, and that should yield results eventually. 52 | 53 | Indeed, the forms of exponential growth that don't consist of learning are so often intermixed with it that we should probably treat this as the rule rather than the exception. Which yields another heuristic: always be learning. If you're not learning, you're probably not on a path that leads to superlinear returns. 54 | 55 | But don't overoptimize what you're learning. Don't limit yourself to learning things that are already known to be valuable. You're learning; you don't know for sure yet what's going to be valuable, and if you're too strict you'll lop off the outliers. 56 | 57 | What about step functions? Are there also useful heuristics of the form "seek thresholds" or "seek competition?" Here the situation is trickier. The existence of a threshold doesn't guarantee the game will be worth playing. If you play a round of Russian roulette, you'll be in a situation with a threshold, certainly, but in the best case you're no better off. "Seek competition" is similarly useless; what if the prize isn't worth competing for? Sufficiently fast exponential growth guarantees both the shape and magnitude of the return curve — because something that grows fast enough will grow big even if it's trivially small at first — but thresholds only guarantee the shape. 58 | 59 | A principle for taking advantage of thresholds has to include a test to ensure the game is worth playing. Here's one that does: if you come across something that's mediocre yet still popular, it could be a good idea to replace it. For example, if a company makes a product that people dislike yet still buy, then presumably they'd buy a better alternative if you made one. 60 | 61 | It would be great if there were a way to find promising intellectual thresholds. Is there a way to tell which questions have whole new fields beyond them? I doubt we could ever predict this with certainty, but the prize is so valuable that it would be useful to have predictors that were even a little better than random, and there's hope of finding those. We can to some degree predict when a research problem isn't likely to lead to new discoveries: when it seems legit but boring. Whereas the kind that do lead to new discoveries tend to seem very mystifying, but perhaps unimportant. (If they were mystifying and obviously important, they'd be famous open questions with lots of people already working on them.) So one heuristic here is to be driven by curiosity rather than careerism — to give free rein to your curiosity instead of working on what you're supposed to. 62 | 63 | The prospect of superlinear returns for performance is an exciting one for the ambitious. And there's good news in this department: this territory is expanding in both directions. There are more types of work in which you can get superlinear returns, and the returns themselves are growing. 64 | 65 | There are two reasons for this, though they're so closely intertwined that they're more like one and a half: progress in technology, and the decreasing importance of organizations. 66 | 67 | Fifty years ago it used to be much more necessary to be part of an organization to work on ambitious projects. It was the only way to get the resources you needed, the only way to have colleagues, and the only way to get distribution. So in 1970 your prestige was in most cases the prestige of the organization you belonged to. And prestige was an accurate predictor, because if you weren't part of an organization, you weren't likely to achieve much. There were a handful of exceptions, most notably artists and writers, who worked alone using inexpensive tools and had their own brands. But even they were at the mercy of organizations for reaching audiences. 68 | 69 | A world dominated by organizations damped variation in the returns for performance. But this world has eroded significantly just in my lifetime. Now a lot more people can have the freedom that artists and writers had in the 20th century. There are lots of ambitious projects that don't require much initial funding, and lots of new ways to learn, make money, find colleagues, and reach audiences. 70 | 71 | There's still plenty of the old world left, but the rate of change has been dramatic by historical standards. Especially considering what's at stake. It's hard to imagine a more fundamental change than one in the returns for performance. 72 | 73 | Without the damping effect of institutions, there will be more variation in outcomes. Which doesn't imply everyone will be better off: people who do well will do even better, but those who do badly will do worse. That's an important point to bear in mind. Exposing oneself to superlinear returns is not for everyone. Most people will be better off as part of the pool. So who should shoot for superlinear returns? Ambitious people of two types: those who know they're so good that they'll be net ahead in a world with higher variation, and those, particularly the young, who can afford to risk trying it to find out. 74 | 75 | The switch away from institutions won't simply be an exodus of their current inhabitants. Many of the new winners will be people they'd never have let in. So the resulting democratization of opportunity will be both greater and more authentic than any tame intramural version the institutions themselves might have cooked up. 76 | 77 | Not everyone is happy about this great unlocking of ambition. It threatens some vested interests and contradicts some ideologies. But if you're an ambitious individual it's good news for you. How should you take advantage of it? 78 | 79 | The most obvious way to take advantage of superlinear returns for performance is by doing exceptionally good work. At the far end of the curve, incremental effort is a bargain. All the more so because there's less competition at the far end — and not just for the obvious reason that it's hard to do something exceptionally well, but also because people find the prospect so intimidating that few even try. Which means it's not just a bargain to do exceptional work, but a bargain even to try to. 80 | 81 | There are many variables that affect how good your work is, and if you want to be an outlier you need to get nearly all of them right. For example, to do something exceptionally well, you have to be interested in it. Mere diligence is not enough. So in a world with superlinear returns, it's even more valuable to know what you're interested in, and to find ways to work on it. It will also be important to choose work that suits your circumstances. For example, if there's a kind of work that inherently requires a huge expenditure of time and energy, it will be increasingly valuable to do it when you're young and don't yet have children. 82 | 83 | There's a surprising amount of technique to doing great work. It's not just a matter of trying hard. I'm going to take a shot giving a recipe in one paragraph. 84 | 85 | Choose work you have a natural aptitude for and a deep interest in. Develop a habit of working on your own projects; it doesn't matter what they are so long as you find them excitingly ambitious. Work as hard as you can without burning out, and this will eventually bring you to one of the frontiers of knowledge. These look smooth from a distance, but up close they're full of gaps. Notice and explore such gaps, and if you're lucky one will expand into a whole new field. Take as much risk as you can afford; if you're not failing occasionally you're probably being too conservative. Seek out the best colleagues. Develop good taste and learn from the best examples. Be honest, especially with yourself. Exercise and eat and sleep well and avoid the more dangerous drugs. When in doubt, follow your curiosity. It never lies, and it knows more than you do about what's worth paying attention to. 86 | 87 | And there is of course one other thing you need: to be lucky. Luck is always a factor, but it's even more of a factor when you're working on your own rather than as part of an organization. And though there are some valid aphorisms about luck being where preparedness meets opportunity and so on, there's also a component of true chance that you can't do anything about. The solution is to take multiple shots. Which is another reason to start taking risks early. 88 | 89 | The best example of a field with superlinear returns is probably science. It has exponential growth, in the form of learning, combined with thresholds at the extreme edge of performance — literally at the limits of knowledge. 90 | 91 | The result has been a level of inequality in scientific discovery that makes the wealth inequality of even the most stratified societies seem mild by comparison. Newton's discoveries were arguably greater than all his contemporaries' combined. 92 | 93 | This point may seem obvious, but it might be just as well to spell it out. Superlinear returns imply inequality. The steeper the return curve, the greater the variation in outcomes. 94 | 95 | In fact, the correlation between superlinear returns and inequality is so strong that it yields another heuristic for finding work of this type: look for fields where a few big winners outperform everyone else. A kind of work where everyone does about the same is unlikely to be one with superlinear returns. 96 | 97 | What are fields where a few big winners outperform everyone else? Here are some obvious ones: sports, politics, art, music, acting, directing, writing, math, science, starting companies, and investing. In sports the phenomenon is due to externally imposed thresholds; you only need to be a few percent faster to win every race. In politics, power grows much as it did in the days of emperors. And in some of the other fields (including politics) success is driven largely by fame, which has its own source of superlinear growth. But when we exclude sports and politics and the effects of fame, a remarkable pattern emerges: the remaining list is exactly the same as the list of fields where you have to be independent-minded to succeed — where your ideas have to be not just correct, but novel as well. 98 | 99 | This is obviously the case in science. You can't publish papers saying things that other people have already said. But it's just as true in investing, for example. It's only useful to believe that a company will do well if most other investors don't; if everyone else thinks the company will do well, then its stock price will already reflect that, and there's no room to make money. 100 | 101 | What else can we learn from these fields? In all of them you have to put in the initial effort. Superlinear returns seem small at first. At this rate, you find yourself thinking, I'll never get anywhere. But because the reward curve rises so steeply at the far end, it's worth taking extraordinary measures to get there. 102 | 103 | In the startup world, the name for this principle is "do things that don't scale." If you pay a ridiculous amount of attention to your tiny initial set of customers, ideally you'll kick off exponential growth by word of mouth. But this same principle applies to anything that grows exponentially. Learning, for example. When you first start learning something, you feel lost. But it's worth making the initial effort to get a toehold, because the more you learn, the easier it will get. 104 | 105 | There's another more subtle lesson in the list of fields with superlinear returns: not to equate work with a job. For most of the 20th century the two were identical for nearly everyone, and as a result we've inherited a custom that equates productivity with having a job. Even now to most people the phrase "your work" means their job. But to a writer or artist or scientist it means whatever they're currently studying or creating. For someone like that, their work is something they carry with them from job to job, if they have jobs at all. It may be done for an employer, but it's part of their portfolio. 106 | 107 | It's an intimidating prospect to enter a field where a few big winners outperform everyone else. Some people do this deliberately, but you don't need to. If you have sufficient natural ability and you follow your curiosity sufficiently far, you'll end up in one. Your curiosity won't let you be interested in boring questions, and interesting questions tend to create fields with superlinear returns if they're not already part of one. 108 | 109 | The territory of superlinear returns is by no means static. Indeed, the most extreme returns come from expanding it. So while both ambition and curiosity can get you into this territory, curiosity may be the more powerful of the two. Ambition tends to make you climb existing peaks, but if you stick close enough to an interesting enough question, it may grow into a mountain beneath you. 110 | -------------------------------------------------------------------------------- /data/weird-languages.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How To Start Google 3 | date: 2021-08 4 | url: https://www.paulgraham.com/weird.html 5 | --- 6 | When people say that in their experience all programming languages are basically equivalent, they're making a statement not about languages but about the kind of programming they've done. 7 | 8 | 99.5% of programming consists of gluing together calls to library functions. All popular languages are equally good at this. So one can easily spend one's whole career operating in the intersection of popular programming languages. 9 | 10 | But the other .5% of programming is disproportionately interesting. If you want to learn what it consists of, the weirdness of weird languages is a good clue to follow. 11 | 12 | Weird languages aren't weird by accident. Not the good ones, at least. The weirdness of the good ones usually implies the existence of some form of programming that's not just the usual gluing together of library calls. 13 | 14 | A concrete example: Lisp macros. Lisp macros seem weird even to many Lisp programmers. They're not only not in the intersection of popular languages, but by their nature would be hard to implement properly in a language without turning it into a dialect of Lisp. And macros are definitely evidence of techniques that go beyond glue programming. For example, solving problems by first writing a language for problems of that type, and then writing your specific application in it. Nor is this all you can do with macros; it's just one region in a space of program-manipulating techniques that even now is far from fully explored. 15 | 16 | So if you want to expand your concept of what programming can be, one way to do it is by learning weird languages. Pick a language that most programmers consider weird but whose median user is smart, and then focus on the differences between this language and the intersection of popular languages. What can you say in this language that would be impossibly inconvenient to say in others? In the process of learning how to say things you couldn't previously say, you'll probably be learning how to think things you couldn't previously think. 17 | -------------------------------------------------------------------------------- /embed.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | from tqdm import tqdm 4 | from lancedb import connect 5 | from pydantic import BaseModel 6 | from lancedb.pydantic import LanceModel, Vector 7 | from lancedb.embeddings import get_registry 8 | from typing import Iterable 9 | 10 | 11 | DB_PATH = Path(os.getcwd(), "db") 12 | DATA_PATH = Path(os.getcwd(), "data") 13 | DB_TABLE = "paul_graham" 14 | 15 | 16 | class Document(BaseModel): 17 | id: int 18 | text: str 19 | filename: str 20 | 21 | 22 | openai = get_registry().get("openai").create(name="text-embedding-3-large", dim=256) 23 | 24 | 25 | class TextChunk(LanceModel): 26 | id: int 27 | doc_id: int 28 | chunk_num: int 29 | start_pos: int 30 | end_pos: int 31 | text: str = openai.SourceField() 32 | # For some reason if we call openai.ndim(), it returns 1536 instead of 256 like we want 33 | vector: Vector(openai.ndims()) = openai.VectorField(default=None) 34 | 35 | 36 | def chunk_text( 37 | documents: Iterable[Document], window_size: int = 1024, overlap: int = 0 38 | ): 39 | id = 0 40 | for doc in documents: 41 | for chunk_num, start_pos in enumerate( 42 | range(0, len(doc.text), window_size - overlap) 43 | ): 44 | # TODO: Fix up this and use a Lance Model instead - have reached out to the team to ask for some help 45 | yield { 46 | "id": id, 47 | "doc_id": doc.id, 48 | "chunk_num": chunk_num, 49 | "start_pos": start_pos, 50 | "end_pos": start_pos + window_size, 51 | "text": doc.text[start_pos : start_pos + window_size], 52 | } 53 | id += 1 54 | 55 | 56 | def read_file_content(path: Path, file_suffix: str) -> Iterable[Document]: 57 | for i, file in enumerate(path.iterdir()): 58 | if file.suffix != file_suffix: 59 | continue 60 | yield Document(id=i, text=file.read_text(), filename=file.name) 61 | 62 | 63 | def batch_chunks(chunks, batch_size=10): 64 | batch = [] 65 | for item in chunks: 66 | batch.append(item) 67 | if len(batch) == batch_size: 68 | yield batch 69 | batch = [] 70 | 71 | if batch: 72 | yield batch 73 | 74 | 75 | def main(): 76 | assert "OPENAI_API_KEY" in os.environ, "OPENAI_API_KEY is not set" 77 | db = connect(DB_PATH) 78 | table = db.create_table(DB_TABLE, schema=TextChunk, mode="overwrite") 79 | 80 | documents = read_file_content(DATA_PATH, file_suffix=".md") 81 | chunks = chunk_text(documents) 82 | batched_chunks = batch_chunks(chunks, 20) 83 | 84 | for chunk_batch in tqdm(batched_chunks): 85 | table.add(chunk_batch) 86 | 87 | 88 | if __name__ == "__main__": 89 | main() 90 | -------------------------------------------------------------------------------- /output-50.jsonl: -------------------------------------------------------------------------------- 1 | {"question":"What are the two fundamental causes for situations with superlinear returns according to the text?","answer":"exponential growth and thresholds","chunk":"It may seem as if there are a lot of different situations with superlinear returns, but as far as I can tell they reduce to two fundamental causes: exponential growth and thresholds.","chunk_id":"db968e1fa1109241432e0d983423a9ac"} 2 | {"question":"What is the extreme example used in the text to illustrate starting a company?","answer":"Starting Google","chunk":"Starting your own company can mean anything from starting a barber shop to starting Google. I'm here to talk about one extreme end of that continuum. I'm going to tell you how to start Google.","chunk_id":"afc4b95c2537f788fe5711c9835b58bf"} 3 | {"question":"What trick can you use to avoid ever having to get a job according to the text?","answer":"The trick mentioned in the text is to avoid ever having to get a job.","chunk":"Most of you probably think that when you're released into the so-called real world you'll eventually have to get some kind of job. That's not true, and today I'm going to talk about a trick you can use to avoid ever having to get a job.","chunk_id":"b6bb3ecf14a22bc3144b3c8bb101a1e8"} 4 | {"question":"According to the text, what skill has been crucial for the success of startups for the last 30 years?","answer":"Programming","chunk":"If you're not sure what technology to get good at, get good at programming. That has been the source of the median startup for the last 30 years, and this is probably not going to change in the next 10.","chunk_id":"8501afc7dc4205533b048c117662c89a"} 5 | {"question":"What enables individuals to easily generate startup ideas according to the text?","answer":"Being skilled in technology allows individuals to identify missing elements in technology and opportunities to address broken aspects using technology, leading to the generation of startup ideas.","chunk":"Actually it's easy to get startup ideas once you're good at technology. Once you're good at some technology, when you look at the world you see dotted outlines around the things that are missing. You start to be able to see both the things that are missing from the technology itself, and all the broken things that could be fixed using it, and each one of these is a potential startup.","chunk_id":"64135aeb363911fea8510bbea00a38fe"} 6 | {"question":"Who are the founders of Google mentioned in the text?","answer":"Larry Page and Sergey Brin","chunk":"You might have thought I was joking when I said I was going to tell you how to start Google. You might be thinking \"How could we start Google?\" But that's effectively what the people who did start Google were thinking before they started it. If you'd told Larry Page and Sergey Brin, the founders of Google, that the company they were about to start would one day be worth over a trillion dollars, their heads would have exploded.","chunk_id":"65d0dc53f68d4e5a9b95e6268673cc09"} 7 | {"question":"According to the text, what would the reward for performance p over time t be if growth rate were proportional to performance?","answer":"The reward for performance p over time t would be proportional to pt.","chunk":"YC doesn't explicitly tell founders that with growth rate \"you get out what you put in,\" but it's not far from the truth. And if growth rate were proportional to performance, then the reward for performance p over time t would be proportional to pt.","chunk_id":"2ac49f59717bb978aec9ae1e37ef44eb"} 8 | {"question":"What advice does the text give to 14 and 15 year olds who might want to start a startup later?","answer":"The text advises 14 and 15 year olds to be patient and learn as much as they can in school, especially math and writing.","chunk":"(This is a talk I gave to 14 and 15 year olds about what to do now if they might want to start a startup later. Lots of schools think they should tell students something about startups. This is what I think they should tell them.)","chunk_id":"f8270d09dace076cb5dbd79309d08fa4"} 9 | {"question":"What are the two key things mentioned in the text for starting Google?","answer":"Building stuff and doing well in school.","chunk":"That's it, just two things, build stuff and do well in school.","chunk_id":"1b2bbfb940a23231fb26efac04374b22"} 10 | {"question":"What is the significance of learning about multiple things according to the text?","answer":"The text suggests that learning about multiple things is valuable because it allows individuals to broaden their knowledge and gain diverse skills, as demonstrated by Steve Jobs learning calligraphy and electronics.","chunk":"You don't have to do just one project. In fact it's good to learn about multiple things. Steve Jobs didn't just learn calligraphy. He also learned about electronics, which was even more valuable. Whatever you're interested in. (Do you notice a theme here?)","chunk_id":"110ccf8b32d476c82e371b8f0edd89c2"} 11 | {"question":"What examples are mentioned in the text where superlinear returns for performance are observed?","answer":"Fame, power, military victories, knowledge, and benefit to humanity.","chunk":"It's obviously true that the returns for performance are superlinear in business. Some think this is a flaw of capitalism, and that if we changed the rules it would stop being true. But superlinear returns for performance are a feature of the world, not an artifact of rules we've invented. We see the same pattern in fame, power, military victories, knowledge, and even benefit to humanity. In all of these, the rich get richer.","chunk_id":"7328e00298d2c1298c2e73bc1bf27c65"} 12 | {"question":"Why is it important to understand the concept of superlinear returns according to the text?","answer":"It is essential to understand the concept of superlinear returns to comprehend the world and succeed ambitiously, as highlighted in the text.","chunk":"You can't understand the world without understanding the concept of superlinear returns. And if you're ambitious you definitely should, because this will be the wave you surf on.","chunk_id":"dbbec66cbef84b1763a53134c8e782da"} 13 | {"question":"What is emphasized as a key factor in getting good at technology according to the text?","answer":"Practice is emphasized as a key factor in getting good at technology.","chunk":"So that's the first of the three things you need, to get good at some kind or kinds of technology. You do it the same way you get good at the violin or football: practice. If you start a startup at 22, and you start writing your own programs now, then by the time you start the company you'll have spent at least 7 years practicing writing code, and you can get pretty good at anything after practicing it for 7 years.","chunk_id":"7f3083273890cc5b640214085104b40d"} 14 | {"question":"What are the three essential components needed to start a successful startup, according to the text?","answer":"To start a successful startup, you need to be good at some kind of technology, have an idea for what you're going to build, and have cofounders to start the company with.","chunk":"How do you get from where you are now to the point where you can start a successful startup? You need three things. You need to be good at some kind of technology, you need an idea for what you're going to build, and you need cofounders to start the company with.","chunk_id":"09369eb77f4c743034d01d13744faf6d"} 15 | {"question":"According to the text, what is the significance of projects in relation to startup ideas?","answer":"Projects are the best source of startup ideas according to the text.","chunk":"Of course Facebook was not a startup yet. It was just a... project. There's that word again. Projects aren't just the best way to learn about technology. They're also the best source of startup ideas.","chunk_id":"2740f0b1e17186cacba76d013d7d3179"} 16 | {"question":"What does the text suggest about the origin of successful startups like Apple, Google, and Facebook?","answer":"The text suggests that successful startups like Apple, Google, and Facebook often grow out of projects that were not initially meant to be startups, emphasizing the importance of following one's instincts and exploring what is interesting rather than intentionally trying to start a company.","chunk":"It's obvious in retrospect that this was a great idea for a startup. It wasn't obvious at the time. It's never obvious. If it was obviously a good idea to start Apple or Google or Facebook, someone else would have already done it. That's why the best startups grow out of projects that aren't meant to be startups. You're not trying to start a company. You're just following your instincts about what's interesting. And if you're young and good at technology, then your unconscious instincts about what's interesting are better than your conscious ideas about what would be a good company.","chunk_id":"67cc7dca1c73fd07fc61ef1bca564baf"} 17 | {"question":"What is an example of superlinear returns mentioned in the text?","answer":"Working on something that grows exponentially, like growing bacterial cultures.","chunk":"The most obvious case of superlinear returns is when you're working on something that grows exponentially. For example, growing bacterial cultures. When they grow at all, they grow exponentially. But they're tricky to grow. Which means the difference in outcome between someone who's adept at it and someone who's not is very great.","chunk_id":"03c9013702d3b30ffe3d48073bf40d99"} 18 | {"question":"What is the recommended approach to getting good at technology according to the text?","answer":"The recommended approach is to work on your own projects based on your interests.","chunk":"How do you get good at technology? And how do you choose which technology to get good at? Both of those questions turn out to have the same answer: work on your own projects. Don't try to guess whether gene editing or LLMs or rockets will turn out to be the most valuable technology to know about. No one can predict that. Just work on whatever interests you the most. You'll work much harder on something you're interested in than something you're doing because you think you're supposed to.","chunk_id":"3bb3e61a043cd396acc7669d021ab532"} 19 | {"question":"According to the text, how can you find cofounders for a startup?","answer":"You can find cofounders by working on projects with them to assess their skills and compatibility.","chunk":"Now for the third and final thing you need: a cofounder, or cofounders. The optimal startup has two or three founders, so you need one or two cofounders. How do you find them? Can you predict what I'm going to say next? It's the same thing: projects. You find cofounders by working on projects with them. What you need in a cofounder is someone who's good at what they do and that you work well with, and the only way to judge this is to work with them on things.","chunk_id":"305778c59d3ef94fa4db9f28c43f55fd"} 20 | {"question":"How does the text suggest one can grow their territory exponentially?","answer":"The text suggests that one can grow their territory exponentially through conquest, as controlling more territory strengthens the army and makes it easier to conquer new land.","chunk":"Or more precisely, no generally applicable plan. There was a way to grow one's territory exponentially: by conquest. The more territory you control, the more powerful your army becomes, and the easier it is to conquer new territory. This is why history is full of empires. But so few people created or ran empires that their experiences didn't affect customs very much. The emperor was a remote and terrifying figure, not a source of lessons one could use in one's own life.","chunk_id":"83fe7dd856a75e38a461563e6c71b5ab"} 21 | {"question":"What concept about the returns for performance did the author not understand when they were a child?","answer":"The author did not understand the degree to which the returns for performance are superlinear.","chunk":"One of the most important things I didn't understand about the world when I was a child is the degree to which the returns for performance are superlinear.","chunk_id":"3bbef20e2e9694dcf8e12a7c45426ff0"} 22 | {"question":"What is the suggested approach in the text for avoiding the drawbacks of a job while still working hard?","answer":"The text suggests that starting your own company is the approach to avoid the drawbacks of a job while still working hard.","chunk":"The trick is to start your own company. So it's not a trick for avoiding work, because if you start your own company you'll work harder than you would if you had an ordinary job. But you will avoid many of the annoying things that come with a job, including a boss telling you what to do.","chunk_id":"ddd2c319f6b2bae1f9583b497bc615e4"} 23 | {"question":"How did Apple transition from being a project to a company?","answer":"Apple transitioned from being a project to a company when Steve Jobs proposed selling plans for the computer that Steve Wozniak had built, turning their venture into a business.","chunk":"Facebook was not unusual in this respect. Apple and Google also began as projects. Apple wasn't meant to be a company. Steve Wozniak just wanted to build his own computer. It only turned into a company when Steve Jobs said \"Hey, I wonder if we could sell plans for this computer to other people.\" That's how Apple started. They weren't even selling computers, just plans for computers. Can you imagine how lame this company seemed?","chunk_id":"eb3d178c5b1595e38d04de39d1a1de54"} 24 | {"question":"What is the reason for the lack of customs related to exponential growth according to the text?","answer":"The lack of customs related to exponential growth is attributed to the limited instances of it in human history.","chunk":"What we don't understand naturally we develop customs to deal with, but we don't have many customs about exponential growth either, because there have been so few instances of it in human history. In principle herding should have been one: the more animals you had, the more offspring they'd have. But in practice grazing land was the limiting factor, and there was no plan for growing that exponentially.","chunk_id":"df602c06b072a9ee919034b1903bad01"} 25 | {"question":"What is suggested as the driving force behind more good startups coming out of prestigious universities like MIT or Cambridge?","answer":"The difficulty of getting in, which attracts smart and determined individuals to these institutions.","chunk":"I don't think it's the prestigious names of these universities that cause more good startups to come out of them. Nor do I think it's because the quality of the teaching is better. What's driving this is simply the difficulty of getting in. You have to be pretty smart and determined to get into MIT or Cambridge, so if you do manage to get in, you'll find the other students include a lot of smart and determined people.","chunk_id":"b81f04934b6ca5bfd56be4f44fcaee9a"} 26 | {"question":"What misconception do teachers and coaches often convey about the relationship between effort and success?","answer":"Teachers and coaches often convey the misconception that the returns are linear, meaning you get out what you put in. However, this is rarely true in reality.","chunk":"Teachers and coaches implicitly told us the returns were linear. \"You get out,\" I heard a thousand times, \"what you put in.\" They meant well, but this is rarely true. If your product is only half as good as your competitor's, you don't get half as many customers. You get no customers, and you go out of business.","chunk_id":"a9e8cdfb42e07e3fafbce6b69e79ed53"} 27 | {"question":"What activities are considered technology according to the text?","answer":"Practically everything described using the words \"make\" or \"build,\" including welding, making clothes, and making videos.","chunk":"You don't have to learn programming, though. If you're wondering what counts as technology, it includes practically everything you could describe using the words \"make\" or \"build.\" So welding would count, or making clothes, or making videos. Whatever you're most interested in. The critical distinction is whether you're producing or just consuming. Are you writing computer games, or just playing them? That's the cutoff.","chunk_id":"7158d40157d16f1c2c96f0d7fe8104f4"} 28 | {"question":"What are the key steps to take in order to start a startup according to the text?","answer":"To start a startup, one needs to get good at technology by working on their own projects and do well in school to get into a good university where cofounders and ideas are found.","chunk":"So the list of what you need to do to get from here to starting a startup is quite short. You need to get good at technology, and the way to do that is to work on your own projects. And you need to do as well in school as you can, so you can get into a good university, because that's where the cofounders and the ideas are.","chunk_id":"f07ccedb98fa171dde6f29af7346f261"} 29 | {"question":"Why does the text suggest doing well in classes and aiming for admission to a top university if you want to start a startup?","answer":"The text suggests doing well in classes and aiming for admission to a top university if you want to start a startup because it is essential to get into a good university to access the best cofounders and employees. Larry and Sergey hired smart individuals from Stanford when they started Google, leading to a significant advantage for the company.","chunk":"At this point I'm going to tell you something you might not want to hear. It really matters to do well in your classes, even the ones that are just memorization or blathering about literature, because you need to do well in your classes to get into a good university. And if you want to start a startup you should try to get into the best university you can, because that's where the best cofounders are. It's also where the best employees are. When Larry and Sergey started Google, they began by just hiring all the smartest people they knew out of Stanford, and this was a real advantage for them.","chunk_id":"bd9da83d7d9b078854442d208ac2de3d"} 30 | {"question":"What solution does a carpenter suggest for the shop's door that is hard to close?","answer":"Plane off the part of the door that sticks.","chunk":"In the town near our house there's a shop with a sign warning that the door is hard to close. The sign has been there for several years. To the people in the shop it must seem like this mysterious natural phenomenon that the door sticks, and all they can do is put up a sign warning customers about it. But any carpenter looking at this situation would think \"why don't you just plane off the part that sticks?\"","chunk_id":"cb780355c90edb1fd4f57990a995d7cf"} 31 | {"question":"What was the most common case of exponential growth in preindustrial times according to the text?","answer":"The most common case of exponential growth in preindustrial times was likely scholarship, where the more knowledge one had, the easier it was to learn new things.","chunk":"The most common case of exponential growth in preindustrial times was probably scholarship. The more you know, the easier it is to learn new things. The result, then as now, was that some people were startlingly more knowledgeable than the rest about certain topics. But this didn't affect customs much either. Although empires of ideas can overlap and there can thus be far more emperors, in preindustrial times this type of empire had little practical effect.","chunk_id":"7a4ce1b65d7b83161dc40313b80ad9e2"} 32 | {"question":"What advice does the text provide regarding the type of projects to work on, especially at a young age?","answer":"The text advises building projects that are exciting and can be as frivolous as desired, focusing on creating things that the builder and their friends enjoy, particularly during the teenage years to prepare for starting a startup in the future.","chunk":"Don't feel like your projects have to be serious. They can be as frivolous as you like, so long as you're building things you're excited about. Probably 90% of programmers start out building games. They and their friends like to play games. So they build the kind of things they and their friends want. And that's exactly what you should be doing at 15 if you want to start a startup one day.","chunk_id":"34d05a90a811fafb4ad85ba988d56e93"} 33 | {"question":"What does the text suggest about the relationship between successful startups and the universities their founders attended?","answer":"The text suggests that there is a strong correlation between successful startups and the most selective universities where the founders studied.","chunk":"The empirical evidence is clear on this. If you look at where the largest numbers of successful startups come from, it's pretty much the same as the list of the most selective universities.","chunk_id":"7983241d508741e75f046023b7419e9c"} 34 | {"question":"What are the outcomes for companies with high growth rates compared to those with lower growth rates in startups?","answer":"Companies with high growth rates tend to become immensely valuable, while the ones with lower growth rates may not even survive.","chunk":"Startups can also grow exponentially, and we see the same pattern there. Some manage to achieve high growth rates. Most don't. And as a result you get qualitatively different outcomes: the companies with high growth rates tend to become immensely valuable, while the ones with lower growth rates may not even survive.","chunk_id":"26d677b5e3cd7db5e1971408839d0b65"} 35 | {"question":"What is the biggest mistake young founders make, according to the text?","answer":"The biggest mistake young founders make is to build something for some mysterious group of other people.","chunk":"So it's critical, if you're a young founder, to build things for yourself and your friends to use. The biggest mistake young founders make is to build something for some mysterious group of other people. But if you can make something that you and your friends truly want to use — something your friends aren't just using out of loyalty to you, but would be really sad to lose if you shut it down — then you almost certainly have the germ of a good startup idea. It may not seem like a startup to you. It may not be obvious how to make money from it. But trust me, there's a way.","chunk_id":"1411f3d714d808f16a1f58197234c18d"} 36 | {"question":"How can one generate startup ideas even if they excel in a specific technology like programming?","answer":"One can generate startup ideas by looking for problems to solve, exploring personal interests, observing inefficiencies in existing products or services, and seeking inspiration from everyday experiences. It is important to constantly be curious and open to new possibilities.","chunk":"Let's suppose you're 22 and you've succeeded: You're now really good at some technology. How do you get startup ideas? It might seem like that's the hard part. Even if you are a good programmer, how do you get the idea to start Google?","chunk_id":"be4b84b5f53cbd8c3763dce1700bef77"} 37 | {"question":"What startup is referred to in the text that is associated with the term 'facebooks'?","answer":"Facebook","chunk":"Once you're good at programming, all the missing software in the world starts to become as obvious as a sticking door to a carpenter. I'll give you a real world example. Back in the 20th century, American universities used to publish printed directories with all the students' names and contact info. When I tell you what these directories were called, you'll know which startup I'm talking about. They were called facebooks, because they usually had a picture of each student next to their name.","chunk_id":"534a02393cb614c450e125c66a896997"} 38 | {"question":"According to the text, how do most of the richest people become wealthy?","answer":"Most of the richest people become wealthy by starting their own companies.","chunk":"It's more exciting to work on your own project than someone else's. And you can also get a lot richer. In fact, this is the standard way to get really rich. If you look at the lists of the richest people that occasionally get published in the press, nearly all of them did it by starting their own companies.","chunk_id":"de494946e19340c348b991e5845cd8c4"} 39 | {"question":"According to the text, what is a more effective way to learn programming?","answer":"Working on personal projects and solving problems through practical application is a faster way to learn programming than traditional classroom learning.","chunk":"If you really want to learn to program, you have to work on your own projects. You learn so much faster that way. Imagine you're writing a game and there's something you want to do in it, and you don't know how. You're going to figure out how a lot faster than you'd learn anything in a class.","chunk_id":"51f84c57fa08bbd99b5004731274be79"} 40 | {"question":"Why do tech companies require coding tests from job applicants?","answer":"Tech companies require coding tests from job applicants because they believe grades and exam results do not necessarily prove one's programming skills. They want to assess the candidate's actual coding abilities, which may not be accurately reflected by academic performance alone.","chunk":"Those of you who are taking computer science classes in school may at this point be thinking, ok, we've got this sorted. We're already being taught all about programming. But sorry, this is not enough. You have to be working on your own projects, not just learning stuff in classes. You can do well in computer science classes without ever really learning to program. In fact you can graduate with a degree in computer science from a top university and still not be any good at programming. That's why tech companies all make you take a coding test before they'll hire you, regardless of where you went to university or how well you did there. They know grades and exam results prove nothing.","chunk_id":"1db1f51732c41c249f1e5297d0b66acb"} 41 | {"question":"What is the name of the startup factory started by Paul Graham and Jessica, and how many startups has it funded since 2005?","answer":"The startup factory started by Paul Graham and Jessica is called Y Combinator. Since 2005, Y Combinator has funded over 4000 startups.","chunk":"The companies at the Google end of the continuum are called startups when they're young. The reason I know about them is that my wife Jessica and I started something called Y Combinator that is basically a startup factory. Since 2005, Y Combinator has funded over 4000 startups. So we know exactly what you need to start a startup, because we've helped people do it for the last 19 years.","chunk_id":"66288d8fe7e8f36b7f4c2bf4d5af7b18"} 42 | {"question":"Why does the author state that even after decades of thinking about it, a specific sentence remains startling?","answer":"The author finds a particular sentence startling even after years of contemplation because it holds a deep or unexpected insight that continues to resonate with the author despite prolonged reflection.","chunk":"Even after decades of thinking about this, I find that sentence startling.","chunk_id":"de18acba58bcd5ec9995871f0f69005d"} 43 | {"question":"What story is mentioned in the text to illustrate the concept of exponential growth?","answer":"The story of the man who asks the king for a single grain of rice the first day and double the amount each successive day.","chunk":"Whenever how well you do depends on how well you've done, you'll get exponential growth. But neither our DNA nor our customs prepare us for it. No one finds exponential growth natural; every child is surprised, the first time they hear it, by the story of the man who asks the king for a single grain of rice the first day and double the amount each successive day.","chunk_id":"62d8a7d19a43298b6cda2a2349adc1ae"} 44 | {"question":"According to the text, what is the key factor needed in a startup idea?","answer":"The key factor needed in a startup idea, according to the text, is something your friends actually want.","chunk":"What you need in a startup idea, and all you need, is something your friends actually want. And those ideas aren't hard to see once you're good at technology. There are sticking doors everywhere.","chunk_id":"bd3afdf8933edef19ce4d1a61b547a93"} 45 | {"question":"How did Steve Jobs' teenage interest in calligraphy influence Apple's success in graphic design?","answer":"Steve Jobs' teenage interest in calligraphy influenced Apple's success in graphic design by shaping the design of the Macintosh computer, which featured beautiful, printed-book-like letters that stood out in the computer industry.","chunk":"Steve Jobs, the founder of Apple, spent time when he was a teenager studying calligraphy — the sort of beautiful writing that you see in medieval manuscripts. No one, including him, thought that this would help him in his career. He was just doing it because he was interested in it. But it turned out to help him a lot. The computer that made Apple really big, the Macintosh, came out at just the moment when computers got powerful enough to make letters like the ones in printed books instead of the computery-looking letters you see in 8 bit games. Apple destroyed everyone else at this, and one reason was that Steve was one of the few people in the computer business who really got graphic design.","chunk_id":"980edb7442a1c0b4847d9c01d2d087a6"} 46 | {"question":"What is the main message conveyed in the text about starting a startup like Google?","answer":"The text conveys that when starting a startup, the focus should be on pursuing an idea worth exploring, even if the outcome is uncertain. It suggests that by taking the necessary steps, a company can have as much chance of success as Google did.","chunk":"All you can know when you start working on a startup is that it seems worth pursuing. You can't know whether it will turn into a company worth billions or one that goes out of business. So when I say I'm going to tell you how to start Google, I mean I'm going to tell you how to get to the point where you can start a company that has as much chance of being Google as Google had of being Google.","chunk_id":"965e5abc2b1409f099518c51d13d7a5a"} 47 | {"question":"According to Y Combinator, why do they encourage founders to focus on growth rate rather than absolute numbers?","answer":"Y Combinator encourages founders to focus on growth rate rather than absolute numbers to prevent early discouragement, guide their focus, and achieve exponential growth.","chunk":"Y Combinator encourages founders to focus on growth rate rather than absolute numbers. It prevents them from being discouraged early on, when the absolute numbers are still low. It also helps them decide what to focus on: you can use growth rate as a compass to tell you how to evolve the company. But the main advantage is that by focusing on growth rate you tend to get something that grows exponentially.","chunk_id":"1a7ffb66eee2c0776a951b980216c488"} 48 | {"question":"Where did the founders of Twitch and Stripe meet their cofounders, and why are universities important for finding cofounders?","answer":"The founders of Twitch met when they were seven years old, and the founders of Stripe, Patrick and John Collison, met when John was born. Universities are crucial for finding cofounders because they are where ideas are generated from collaborations with future cofounders.","chunk":"You don't have to start a startup with someone you meet at university. The founders of Twitch met when they were seven. The founders of Stripe, Patrick and John Collison, met when John was born. But universities are the main source of cofounders. And because they're where the cofounders are, they're also where the ideas are, because the best ideas grow out of projects you do with the people who become your cofounders.","chunk_id":"708d5f638252ac76efe52388ffb7df2e"} 49 | {"question":"What motivated Larry and Sergey to work on improving search results before founding Google?","answer":"Larry and Sergey were motivated to work on improving search results because they noticed that most search engines at that time didn't prioritize search results by importance.","chunk":"Ditto for Google. Larry and Sergey weren't trying to start a company at first. They were just trying to make search better. Before Google, most search engines didn't try to sort the results they gave you in order of importance. If you searched for \"rugby\" they just gave you every web page that contained the word \"rugby.\" And the web was so small in 1997 that this actually worked! Kind of. There might only be 20 or 30 pages with the word \"rugby,\" but the web was growing exponentially, which meant this way of doing search was becoming exponentially more broken. Most users just thought, \"Wow, I sure have to look through a lot of search results to find what I want.\" Door sticks. But like Mark, Larry and Sergey were programmers. Like Mark, they looked at this situation and thought \"Well, this is stupid. Some pages about rugby matter more than others. Let's figure out which those are and show them first.\"","chunk_id":"170c43d3e296cbbd3f6fe0d1279e9e10"} 50 | {"question":"What situation at Harvard University prompted Mark Zuckerberg to create Facebook?","answer":"The lack of a centralized online platform for the whole university, despite individual house Facebooks being present, prompted Mark Zuckerberg to create Facebook.","chunk":"So Mark Zuckerberg shows up at Harvard in 2002, and the university still hasn't gotten the facebook online. Each individual house has an online facebook, but there isn't one for the whole university. The university administration has been diligently having meetings about this, and will probably have solved the problem in another decade or so. Most of the students don't consciously notice that anything is wrong. But Mark is a programmer. He looks at this situation and thinks \"Well, this is stupid. I could write a program to fix this in one night. Just let people upload their own photos and then combine the data into a new site for the whole university.\" So he does. And almost literally overnight he has thousands of users.","chunk_id":"89e2387b62ad20b76b1af5196c84160d"} 51 | -------------------------------------------------------------------------------- /output.jsonl: -------------------------------------------------------------------------------- 1 | {"question":"What are the three things needed to start a successful startup according to the text?","answer":"Being good at technology, having an idea for what to build, and having cofounders to start the company with.","chunk":"How do you get from where you are now to the point where you can start a successful startup? You need three things. You need to be good at some kind of technology, you need an idea for what you're going to build, and you need cofounders to start the company with.","chunk_id":"09369eb77f4c743034d01d13744faf6d"} 2 | {"question":"What is the trick mentioned in the text to avoid having to get a job in the real world?","answer":"The trick mentioned in the text is to find ways to navigate the employment landscape without the traditional concept of a job.","chunk":"Most of you probably think that when you're released into the so-called real world you'll eventually have to get some kind of job. That's not true, and today I'm going to talk about a trick you can use to avoid ever having to get a job.","chunk_id":"b6bb3ecf14a22bc3144b3c8bb101a1e8"} 3 | {"question":"What would have happened if you had told Larry Page and Sergey Brin that the company they were about to start would one day be worth over a trillion dollars?","answer":"Their heads would have exploded.","chunk":"You might have thought I was joking when I said I was going to tell you how to start Google. You might be thinking \"How could we start Google?\" But that's effectively what the people who did start Google were thinking before they started it. If you'd told Larry Page and Sergey Brin, the founders of Google, that the company they were about to start would one day be worth over a trillion dollars, their heads would have exploded.","chunk_id":"65d0dc53f68d4e5a9b95e6268673cc09"} 4 | {"question":"What advice does the text provide for 14 and 15 year olds who may want to start a startup in the future?","answer":"The text suggests that schools should share information about startups with students to prepare them for future entrepreneurial endeavors.","chunk":"(This is a talk I gave to 14 and 15 year olds about what to do now if they might want to start a startup later. Lots of schools think they should tell students something about startups. This is what I think they should tell them.)","chunk_id":"f8270d09dace076cb5dbd79309d08fa4"} 5 | {"question":"According to the text, what is the standard way to get really rich?","answer":"Starting your own company.","chunk":"It's more exciting to work on your own project than someone else's. And you can also get a lot richer. In fact, this is the standard way to get really rich. If you look at the lists of the richest people that occasionally get published in the press, nearly all of them did it by starting their own companies.","chunk_id":"de494946e19340c348b991e5845cd8c4"} 6 | {"question":"What does the text consider as an extreme example when discussing starting a company?","answer":"Starting Google is considered an extreme example when discussing starting a company in the text.","chunk":"Starting your own company can mean anything from starting a barber shop to starting Google. I'm here to talk about one extreme end of that continuum. I'm going to tell you how to start Google.","chunk_id":"afc4b95c2537f788fe5711c9835b58bf"} 7 | {"question":"What is the name of the startup factory mentioned in the text, founded by the author and his wife?","answer":"Y Combinator","chunk":"The companies at the Google end of the continuum are called startups when they're young. The reason I know about them is that my wife Jessica and I started something called Y Combinator that is basically a startup factory. Since 2005, Y Combinator has funded over 4000 startups. So we know exactly what you need to start a startup, because we've helped people do it for the last 19 years.","chunk_id":"66288d8fe7e8f36b7f4c2bf4d5af7b18"} 8 | {"question":"According to the text, what is the key to getting good at technology and choosing which technology to focus on?","answer":"Working on your own projects based on your interests.","chunk":"How do you get good at technology? And how do you choose which technology to get good at? Both of those questions turn out to have the same answer: work on your own projects. Don't try to guess whether gene editing or LLMs or rockets will turn out to be the most valuable technology to know about. No one can predict that. Just work on whatever interests you the most. You'll work much harder on something you're interested in than something you're doing because you think you're supposed to.","chunk_id":"3bb3e61a043cd396acc7669d021ab532"} 9 | {"question":"What is the key takeaway regarding starting your own company from the text chunk?","answer":"The key takeaway is that starting your own company involves hard work but can provide freedom from traditional job constraints like having a boss.","chunk":"The trick is to start your own company. So it's not a trick for avoiding work, because if you start your own company you'll work harder than you would if you had an ordinary job. But you will avoid many of the annoying things that come with a job, including a boss telling you what to do.","chunk_id":"ddd2c319f6b2bae1f9583b497bc615e4"} 10 | {"question":"What is the main point regarding starting a startup according to the text?","answer":"The main point is that when starting a startup, you can't predict its future success or failure, but you should pursue ideas that seem worth pursuing.","chunk":"All you can know when you start working on a startup is that it seems worth pursuing. You can't know whether it will turn into a company worth billions or one that goes out of business. So when I say I'm going to tell you how to start Google, I mean I'm going to tell you how to get to the point where you can start a company that has as much chance of being Google as Google had of being Google.","chunk_id":"965e5abc2b1409f099518c51d13d7a5a"} 11 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "rag-app" 3 | version = "0.1.0" 4 | description = "Your description here" 5 | authors = ["Jason Liu ", "Ivan Leo "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.10" 9 | openai = "^1.1.0" 10 | pydantic = "^2.0.2" 11 | typer = "^0.9.0" 12 | lancedb = "^0.6.1" 13 | tqdm = "^4.66.2" 14 | rich = "^13.6.0" 15 | python-frontmatter = "^1.1.0" 16 | duckdb="0.10.0" 17 | unstructured="^0.12.6" 18 | pandas="2.2.1" 19 | instructor="^0.6.4" 20 | tenacity="^8.2.3" 21 | scikit-learn = "^1.3.2" 22 | 23 | 24 | [tool.poetry.dev-dependencies] 25 | pytest = "^7.1.2" 26 | lancedb = "^0.6.1" 27 | 28 | [build-system] 29 | requires = ["poetry-core>=1.0.0"] 30 | build-backend = "poetry.core.masonry.api" 31 | 32 | [tool.poetry.scripts] 33 | rag-app = 'rag_app.cli:app' 34 | -------------------------------------------------------------------------------- /rag_app/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | This is a README detailing how to use `rag-app`, a simple cli which can be used to interact with a local lancedb database. You can install this by running the command 4 | 5 | ``` 6 | pip3 install -e . 7 | ``` 8 | 9 | 10 | # Commands 11 | 12 | ## Querying the Database 13 | 14 | We can use the command `rag-app query db ` in order to query our database. We use the `text-embedding-3` model with a dimensionality of 256 by default. 15 | 16 | ``` 17 | >> rag-app query db ./db paul_graham "What's the biggest challenge facing any startup" 18 | ┏━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ 19 | ┃ ┃ ┃ 20 | ┃ Chunk ID ┃ Result ┃ 21 | ┃ ┃ ┃ 22 | ┣━━━━━━━━━━━━╋━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ 23 | ┃ ┃ ┃ 24 | ┃ 57 ┃ o ideas fit together like two puzzle pieces. How do you get from starting small to doing something great? By making ┃ 25 | ┃ ┃ successive versions. Great things are almost always made in successive versions. You start with something small and ┃ 26 | ┃ ┃ evolve it, and the final version is both cleverer and more ambitious than anything you could have planned. It's ┃ 27 | ┃ ┃ particularly useful to make successive versions when you're making something for people — to get an initial version in ┃ 28 | ┃ ┃ front of them quickly, and then evolve it based on their response. Begin by trying the simplest thing that could ┃ 29 | ┃ ┃ possibly work. Surprisingly often, it does. If it doesn't, this will at least get you started. Don't try to cram too ┃ 30 | ┃ ┃ much new stuff into any one version. There are names for doing this with the first version (taking too long to ship) and ┃ 31 | ┃ ┃ the second (the second system effect), but these are both merely instances of a more general principle. An early ┃ 32 | ┃ ┃ version of a new project will sometimes be dismissed as a toy. It's a good sign when people ┃ 33 | ┃ ┃ ┃ 34 | ┣━━━━━━━━━━━━╋━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ 35 | ┃ ┃ ┃ 36 | ┃ 17 ┃ fields where you have to be independent-minded to succeed — where your ideas have to be not just correct, but novel as ┃ 37 | ┃ ┃ well. This is obviously the case in science. You can't publish papers saying things that other people have already ┃ 38 | ┃ ┃ said. But it's just as true in investing, for example. It's only useful to believe that a company will do well if most ┃ 39 | ┃ ┃ other investors don't; if everyone else thinks the company will do well, then its stock price will already reflect that, ┃ 40 | ┃ ┃ and there's no room to make money. What else can we learn from these fields? In all of them you have to put in the ┃ 41 | ┃ ┃ initial effort. Superlinear returns seem small at first. At this rate, you find yourself thinking, I'll never get ┃ 42 | ┃ ┃ anywhere. But because the reward curve rises so steeply at the far end, it's worth taking extraordinary measures to get ┃ 43 | ┃ ┃ there. In the startup world, the name for this principle is "do things that don't scale." If you pay a ridiculous ┃ 44 | ┃ ┃ amount of attention to your tiny initial set of customers, ideally you'll kick off e ┃ 45 | ┃ ┃ ┃ 46 | ┣━━━━━━━━━━━━╋━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ 47 | ``` 48 | -------------------------------------------------------------------------------- /rag_app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jxnl/n-levels-of-rag/2ce1106b0163d11fcf7959187b42f0d3a67fbe05/rag_app/__init__.py -------------------------------------------------------------------------------- /rag_app/cli.py: -------------------------------------------------------------------------------- 1 | import typer 2 | import rag_app.query as QueryApp 3 | import rag_app.ingest as IngestApp 4 | import rag_app.generate_synthetic_question as GenerateApp 5 | import rag_app.evaluate as EvaluateApp 6 | 7 | app = typer.Typer( 8 | name="Rag-App", 9 | help="A CLI for querying a local RAG application backed by LanceDB", 10 | ) 11 | 12 | app.add_typer( 13 | QueryApp.app, 14 | name="query", 15 | help="Commands to help query your local lancedb instance", 16 | ) 17 | app.add_typer( 18 | IngestApp.app, 19 | name="ingest", 20 | help="Commands to help ingest data into your local lancedb instance", 21 | ) 22 | app.add_typer( 23 | GenerateApp.app, 24 | name="generate", 25 | help="Commands to help generate synthethic data from your documents", 26 | ) 27 | app.add_typer( 28 | EvaluateApp.app, 29 | name="evaluate", 30 | help="Commands to help evaluate the quality of your rag application", 31 | ) 32 | -------------------------------------------------------------------------------- /rag_app/evaluate.py: -------------------------------------------------------------------------------- 1 | import typer 2 | from pathlib import Path 3 | import json 4 | import duckdb 5 | from rag_app.models import EvaluationDataItem, KeywordExtractionResponse 6 | from lancedb import connect 7 | from typing import List, Union 8 | from openai import AsyncOpenAI 9 | import pandas as pd 10 | from pydantic import BaseModel 11 | from tqdm.asyncio import tqdm_asyncio as asyncio 12 | from rag_app.src.chunking import batch_items 13 | from tenacity import retry, stop_after_attempt, wait_fixed 14 | from asyncio import run 15 | from rag_app.models import TextChunk 16 | from rag_app.src.metrics import ( 17 | calculate_mrr, 18 | calculate_ndcg, 19 | slice_predictions_decorator, 20 | ) 21 | from rich.console import Console 22 | from collections import OrderedDict 23 | import instructor 24 | from rich.table import Table 25 | 26 | app = typer.Typer() 27 | evals = OrderedDict() 28 | SIZES = [3, 5, 10, 20] 29 | for size in SIZES: 30 | evals[f"MRR@{size}"] = slice_predictions_decorator(size)(calculate_mrr) 31 | 32 | for size in SIZES: 33 | evals[f"NDCG@{size}"] = slice_predictions_decorator(size)(calculate_ndcg) 34 | 35 | 36 | class EmbeddedEvaluationItem(BaseModel): 37 | question: str 38 | embedding: List[float] 39 | chunk_id: str 40 | 41 | 42 | class FullTextSearchEvaluationItem(BaseModel): 43 | question: str 44 | keywords: List[str] 45 | chunk_id: str 46 | 47 | 48 | class BM25SearchEvaluationItem(BaseModel): 49 | question: str 50 | chunk_id: str 51 | 52 | 53 | class QueryResult(BaseModel): 54 | source: Union[ 55 | EmbeddedEvaluationItem, FullTextSearchEvaluationItem, BM25SearchEvaluationItem 56 | ] 57 | results: List[TextChunk] 58 | 59 | 60 | @retry(stop=stop_after_attempt(5), wait=wait_fixed(30)) 61 | async def embed_query(queries: List[EvaluationDataItem], client: AsyncOpenAI): 62 | query_strings = [query.question for query in queries] 63 | embeddings = await client.embeddings.create( 64 | input=query_strings, model="text-embedding-3-large", dimensions=256 65 | ) 66 | embeddings = [embedding_object.embedding for embedding_object in embeddings.data] 67 | 68 | return [ 69 | EmbeddedEvaluationItem( 70 | question=query.question, embedding=embedding, chunk_id=query.chunk_id 71 | ) 72 | for query, embedding in zip(queries, embeddings) 73 | ] 74 | 75 | 76 | async def embed_test_queries( 77 | queries: List[EvaluationDataItem], 78 | ) -> List[EmbeddedEvaluationItem]: 79 | client = AsyncOpenAI() 80 | batched_queries = batch_items(queries) 81 | coros = [embed_query(query_batch, client) for query_batch in batched_queries] 82 | result = await asyncio.gather(*coros) 83 | return [item for sublist in result for item in sublist] 84 | 85 | 86 | async def fetch_relevant_results( 87 | queries: List[EmbeddedEvaluationItem], 88 | db_path: str, 89 | table_name: str, 90 | ) -> List[QueryResult]: 91 | db = connect(db_path) 92 | table = db.open_table(table_name) 93 | 94 | async def query_table(query: EmbeddedEvaluationItem): 95 | results = table.search(query.embedding).limit(25).to_pydantic(TextChunk) 96 | return QueryResult(results=results, source=query) 97 | 98 | coros = [query_table(query) for query in queries] 99 | return await asyncio.gather(*coros) 100 | 101 | 102 | async def generate_keywords_for_questions( 103 | queries: List[EvaluationDataItem], 104 | ) -> List[str]: 105 | async def generate_query_keywords(query: EvaluationDataItem, client: AsyncOpenAI): 106 | response: KeywordExtractionResponse = await client.chat.completions.create( 107 | model="gpt-4-0613", 108 | response_model=KeywordExtractionResponse, 109 | messages=[ 110 | { 111 | "role": "system", 112 | "content": "You are a world class search engine. You are about to be given a question by a user. Make sure to generate as many possible keywords that are relevant to the question at hand which can help to identify relevant chunks of information to the user's query.", 113 | }, 114 | { 115 | "role": "assistant", 116 | "content": "Make sure to extract all possible keywords within the question itself first before generating new ones. Also expand all accronyms, identify synonyms and related topics.", 117 | }, 118 | {"role": "user", "content": f"The question is {query.question}."}, 119 | ], 120 | max_retries=5, 121 | ) 122 | return FullTextSearchEvaluationItem( 123 | question=query.question, 124 | keywords=response.keywords, 125 | chunk_id=query.chunk_id, 126 | ) 127 | 128 | client = instructor.patch(AsyncOpenAI()) 129 | coros = [generate_query_keywords(query, client) for query in queries] 130 | return await asyncio.gather(*coros) 131 | 132 | 133 | async def match_chunks_with_keywords( 134 | queries: List[FullTextSearchEvaluationItem], db_path: str, table_name: str 135 | ): 136 | async def query_table(query: FullTextSearchEvaluationItem) -> pd.DataFrame: 137 | keywords = query.keywords 138 | keywords = [keyword.replace("'", "''") for keyword in keywords] 139 | keyword_conditions = " +\n".join( 140 | [ 141 | f"CASE WHEN regexp_matches(text, '(?i){keyword}') THEN 1 ELSE 0 END" 142 | for keyword in keywords 143 | ] 144 | ) 145 | keyword_search = "|".join(keywords) 146 | duckdb_query = f""" 147 | SELECT 148 | *, 149 | ({keyword_conditions}) AS num_keywords_matched 150 | FROM 151 | chunks 152 | WHERE 153 | regexp_matches(text, '(?i)({keyword_search})') 154 | ORDER BY 155 | num_keywords_matched DESC 156 | LIMIT 25 157 | """ 158 | 159 | db = connect(db_path) 160 | chunk_table = db.open_table(table_name) 161 | chunks = chunk_table.to_lance() 162 | result = duckdb.query(duckdb_query).to_df() 163 | return QueryResult( 164 | source=query, 165 | results=[ 166 | TextChunk( 167 | **{ 168 | key: value 169 | for key, value in row.items() 170 | if key != "num_keywords_matched" 171 | } 172 | ) 173 | for index, row in result.iterrows() 174 | ], 175 | ) 176 | 177 | coros = [query_table(query) for query in queries] 178 | return await asyncio.gather(*coros) 179 | 180 | 181 | def match_chunks_with_bm25( 182 | db_path: str, table_name: str, queries: List[EvaluationDataItem] 183 | ): 184 | db = connect(db_path) 185 | chunk_table = db.open_table(table_name) 186 | try: 187 | chunk_table.create_fts_index("text", replace=False) 188 | except ValueError as e: 189 | print("Index on the column 'text' has already been created.") 190 | 191 | def query_table(query: EvaluationDataItem): 192 | db = connect(db_path) 193 | chunk_table = db.open_table(table_name) 194 | 195 | retrieved_queries = ( 196 | chunk_table.search(query.question).limit(25).to_pydantic(TextChunk) 197 | ) 198 | return QueryResult( 199 | source=BM25SearchEvaluationItem( 200 | question=query.question, chunk_id=query.chunk_id 201 | ), 202 | results=retrieved_queries, 203 | ) 204 | 205 | return [query_table(query) for query in queries] 206 | 207 | 208 | def score(query: QueryResult) -> dict[str, float]: 209 | y_true = query.source.chunk_id 210 | y_pred = [x.chunk_id for x in query.results] 211 | 212 | metrics = {label: metric_fn(y_true, y_pred) for label, metric_fn in evals.items()} 213 | metrics = { 214 | label: round(value, 2) if value != "N/A" else value 215 | for label, value in metrics.items() 216 | } 217 | return { 218 | **metrics, 219 | "chunk_id": query.source.chunk_id, 220 | "retrieved_size": len(query.results), 221 | } 222 | 223 | 224 | @app.command(help="Evaluate document retrieval") 225 | def from_jsonl( 226 | input_file_path: str = typer.Option( 227 | help="Jsonl file to read in labels from", 228 | ), 229 | db_path: str = typer.Option(help="Your LanceDB path"), 230 | table_name: str = typer.Option(help="Table to read data from"), 231 | eval_mode=typer.Option(help="Query Method ( semantic or fts )", default="semantic"), 232 | ): 233 | assert Path( 234 | input_file_path 235 | ).parent.exists(), f"The directory {Path(input_file_path).parent} does not exist." 236 | assert ( 237 | Path(input_file_path).suffix == ".jsonl" 238 | ), "The output file must have a .jsonl extension." 239 | assert Path(db_path).exists(), f"Database path {db_path} does not exist" 240 | 241 | with open(input_file_path, "r") as file: 242 | data = file.readlines() 243 | evaluation_data = [EvaluationDataItem(**json.loads(item)) for item in data] 244 | 245 | if eval_mode == "semantic": 246 | embedded_queries = run(embed_test_queries(evaluation_data)) 247 | query_results = run( 248 | fetch_relevant_results(embedded_queries, db_path, table_name) 249 | ) 250 | elif eval_mode == "fts": 251 | fts_queries = run(generate_keywords_for_questions(evaluation_data)) 252 | query_results = run( 253 | match_chunks_with_keywords(fts_queries, db_path, table_name) 254 | ) 255 | elif eval_mode == "bm25": 256 | query_results = match_chunks_with_bm25(db_path, table_name, evaluation_data) 257 | else: 258 | raise ValueError( 259 | "Invalid eval mode. Only semantic, fts or bm25 is supported at the moment" 260 | ) 261 | 262 | evals = [score(result) for result in query_results] 263 | 264 | df = pd.DataFrame(evals) 265 | df = df.set_index("chunk_id") 266 | console = Console() 267 | console.print(df) 268 | print("") 269 | numeric_df = df.apply(pd.to_numeric, errors="coerce") 270 | mean_values_table = Table(title="Mean Values") 271 | mean_values_table.add_column("Metric", style="cyan") 272 | mean_values_table.add_column("Value", style="magenta") 273 | for metric, value in numeric_df.mean().items(): 274 | mean_values_table.add_row(metric, str(round(value, 2))) 275 | console.print(mean_values_table) 276 | -------------------------------------------------------------------------------- /rag_app/generate_synthetic_question.py: -------------------------------------------------------------------------------- 1 | import typer 2 | from pathlib import Path 3 | from rag_app.src.chunking import read_files, chunk_text 4 | from instructor import patch 5 | from openai import AsyncOpenAI 6 | from tqdm.asyncio import tqdm_asyncio as asyncio 7 | from asyncio import run 8 | from rag_app.models import TextChunk, EvaluationDataItem, QuestionAnswerPair 9 | from typing import List 10 | from tenacity import retry, stop_after_attempt, wait_fixed 11 | 12 | 13 | app = typer.Typer() 14 | 15 | 16 | @retry(stop=stop_after_attempt(5), wait=wait_fixed(30)) 17 | async def generate_question_answer_pair( 18 | client: AsyncOpenAI, 19 | chunk: TextChunk, 20 | ) -> tuple[QuestionAnswerPair, TextChunk]: 21 | res = await client.chat.completions.create( 22 | model="gpt-3.5-turbo", 23 | messages=[ 24 | { 25 | "role": "system", 26 | "content": "You are a world class algorithm that excels at generating great questions that can be only answered by a specific text that will soon be passed to you. ", 27 | }, 28 | { 29 | "role": "assistant", 30 | "content": f"Generate a question and answer pair that uses information and content that is specific to the following text chunk, including a chain of thought:\n\n{chunk}", 31 | }, 32 | ], 33 | response_model=QuestionAnswerPair, 34 | ) 35 | return (res, chunk) 36 | 37 | 38 | async def gather_questions(chunks: TextChunk) -> List[EvaluationDataItem]: 39 | client = patch(AsyncOpenAI()) 40 | coros = [generate_question_answer_pair(client, chunk) for chunk in chunks] 41 | output = [] 42 | for response in asyncio.as_completed(coros): 43 | questionAnswer, chunkData = await response 44 | assert isinstance(chunkData, TextChunk) 45 | assert isinstance(questionAnswer, QuestionAnswerPair) 46 | output.append( 47 | EvaluationDataItem( 48 | **{ 49 | "question": questionAnswer.question, 50 | "answer": questionAnswer.answer, 51 | "chunk": chunkData.text, 52 | "chunk_id": chunkData.chunk_id, 53 | } 54 | ) 55 | ) 56 | return output 57 | 58 | 59 | @app.command(help="Generate questions for each chunk in a given file") 60 | def synthethic_questions( 61 | folder_path: str = typer.Option(help="Folder to read data from"), 62 | max_questions: int = typer.Option( 63 | help="max number of question/answer pairs to generate", default=-1 64 | ), 65 | output_path: str = typer.Option( 66 | help="Json file to write output to", default="output.jsonl" 67 | ), 68 | ): 69 | assert Path( 70 | output_path 71 | ).parent.exists(), f"The directory {Path(output_path).parent} does not exist." 72 | assert ( 73 | Path(output_path).suffix == ".jsonl" 74 | ), "The output file must have a .jsonl extension." 75 | 76 | file = read_files(Path(folder_path), file_suffix=".md") 77 | chunks = chunk_text(file) 78 | chunks = [TextChunk(**chunk) for chunk in chunks] 79 | if max_questions > 0: 80 | chunks = chunks[:max_questions] 81 | 82 | questions = run(gather_questions(chunks)) 83 | with open(output_path, "w") as f: 84 | for question in questions: 85 | f.write(question.model_dump_json() + "\n") 86 | -------------------------------------------------------------------------------- /rag_app/ingest.py: -------------------------------------------------------------------------------- 1 | import typer 2 | from lancedb import connect 3 | from rag_app.models import TextChunk, Document 4 | from pathlib import Path 5 | from tqdm import tqdm 6 | from rich import print 7 | from rag_app.src.chunking import read_files, batch_items, chunk_text 8 | 9 | app = typer.Typer() 10 | 11 | 12 | @app.command(help="Ingest data into a given lancedb") 13 | def from_folder( 14 | db_path: str = typer.Option(help="Your LanceDB path"), 15 | table_name: str = typer.Option(help="Table to ingest data into"), 16 | folder_path: str = typer.Option(help="Folder to read data from"), 17 | file_suffix: str = typer.Option(default=".md", help="File suffix to filter by"), 18 | ): 19 | db = connect(db_path) 20 | 21 | if table_name not in db.table_names(): 22 | db.create_table(table_name, schema=TextChunk, mode="overwrite") 23 | 24 | if "document" not in db.table_names(): 25 | db.create_table("document", schema=Document, mode="overwrite") 26 | 27 | table = db.open_table(table_name) 28 | document_table = db.open_table("document") 29 | path = Path(folder_path) 30 | 31 | if not path.exists(): 32 | raise ValueError(f"Ingestion folder of {folder_path} does not exist") 33 | 34 | files = read_files(path, file_suffix) 35 | document_table.add(list(files)) 36 | 37 | files = read_files(path, file_suffix=file_suffix) 38 | chunks = chunk_text(files) 39 | batched_chunks = batch_items(chunks) 40 | 41 | ttl = 0 42 | for chunk_batch in tqdm(batched_chunks): 43 | table.add(chunk_batch) 44 | ttl += len(chunk_batch) 45 | 46 | print(f"Added {ttl} chunks to {table_name}") 47 | -------------------------------------------------------------------------------- /rag_app/models.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import List 3 | from pydantic import field_validator 4 | from lancedb.embeddings import get_registry 5 | from lancedb.pydantic import LanceModel, Vector 6 | from pydantic import BaseModel, Field 7 | 8 | openai = get_registry().get("openai").create(name="text-embedding-3-large", dim=256) 9 | 10 | 11 | class TextChunk(LanceModel): 12 | chunk_id: str 13 | doc_id: str 14 | text: str = openai.SourceField() 15 | vector: Vector(openai.ndims()) = openai.VectorField(default=None) 16 | post_title: str 17 | publish_date: datetime 18 | chunk_number: int 19 | source: str 20 | 21 | 22 | class DocumentMetadata(LanceModel): 23 | date: str 24 | url: str 25 | title: str 26 | 27 | @field_validator("date") 28 | @classmethod 29 | def metadata_must_contain_a_valid_date_string(cls, v: str): 30 | try: 31 | datetime.strptime(v, "%Y-%m") 32 | except Exception as e: 33 | raise ValueError( 34 | f"Date format must be YYYY-MM (Eg. 2024-10). Unable to parse provided date of {v} " 35 | ) 36 | return v 37 | 38 | 39 | class Document(LanceModel): 40 | id: str 41 | content: str 42 | filename: str 43 | metadata: DocumentMetadata 44 | 45 | 46 | class QuestionAnswerPair(BaseModel): 47 | """ 48 | This model represents a pair of a question generated from a text chunk, its corresponding answer, 49 | and the chain of thought leading to the answer. The chain of thought provides insight into how the answer 50 | was derived from the question. 51 | """ 52 | 53 | chain_of_thought: str = Field( 54 | ..., description="The reasoning process leading to the answer." 55 | ) 56 | question: str = Field( 57 | ..., description="The generated question from the text chunk." 58 | ) 59 | answer: str = Field(..., description="The answer to the generated question.") 60 | 61 | 62 | class EvaluationDataItem(BaseModel): 63 | question: str 64 | answer: str 65 | chunk: str 66 | chunk_id: str 67 | 68 | 69 | class KeywordExtractionResponse(BaseModel): 70 | """ 71 | This is a response model which represents a list of potential short keywords that can be used to identify/find relevant sources of information related to the question. 72 | 73 | These should come both from the question itself AND generated by the llm 74 | """ 75 | 76 | keywords: List[str] = Field( 77 | ..., 78 | title="Search Terms", 79 | description="A list of search terms, expanded acronyms, and synonyms, each item limited to a maximum of two words.", 80 | min_length=8, 81 | json_schema_extra={ 82 | "examples": [ 83 | "climate change", 84 | "global warming", 85 | "CO2 emissions", 86 | ] 87 | }, 88 | ) 89 | 90 | @field_validator("keywords") 91 | @classmethod 92 | def validate_search_terms(cls, v): 93 | invalid_terms = [term for term in v if len(term.split()) > 2] 94 | if invalid_terms: 95 | raise ValueError( 96 | f"The following keywords contain more than two words and are not allowed: {', '.join(invalid_terms)}. Consider breaking up these terms into smaller phrases. Existing list of words is {', '.join(v)}" 97 | ) 98 | return v 99 | -------------------------------------------------------------------------------- /rag_app/query.py: -------------------------------------------------------------------------------- 1 | import openai 2 | from rag_app.models import TextChunk 3 | from lancedb import connect 4 | from typing import List 5 | from pathlib import Path 6 | from rich.console import Console 7 | from rich.table import Table 8 | from rich import box 9 | import duckdb 10 | import typer 11 | 12 | app = typer.Typer() 13 | 14 | 15 | @app.command(help="Query LanceDB for some results") 16 | def db( 17 | db_path: str = typer.Option(help="Your LanceDB path"), 18 | table_name: str = typer.Option(help="Table to ingest data into"), 19 | query: str = typer.Option(help="Text to query against existing vector db chunks"), 20 | n: int = typer.Option(default=3, help="Maximum number of chunks to return"), 21 | ): 22 | if not Path(db_path).exists(): 23 | raise ValueError(f"Database path {db_path} does not exist.") 24 | db = connect(db_path) 25 | db_table = db.open_table(table_name) 26 | 27 | client = openai.OpenAI() 28 | query_vector = ( 29 | client.embeddings.create( 30 | input=query, model="text-embedding-3-large", dimensions=256 31 | ) 32 | .data[0] 33 | .embedding 34 | ) 35 | 36 | results: List[TextChunk] = ( 37 | db_table.search(query_vector).limit(n).to_pydantic(TextChunk) 38 | ) 39 | 40 | sql_table = db_table.to_lance() 41 | df = duckdb.query( 42 | "SELECT doc_id, count(chunk_id) as count FROM sql_table GROUP BY doc_id" 43 | ).to_df() 44 | 45 | doc_id_to_count = df.set_index("doc_id")["count"].to_dict() 46 | 47 | table = Table(title="Results", box=box.HEAVY, padding=(1, 2), show_lines=True) 48 | table.add_column("Chunk Id", style="magenta") 49 | table.add_column("Content", style="magenta", max_width=120) 50 | table.add_column("Post Title", style="green", max_width=30) 51 | table.add_column("Chunk Number", style="yellow") 52 | table.add_column("Publish Date", style="blue") 53 | 54 | for result in results: 55 | chunk_number = f"{result.chunk_number}/{doc_id_to_count[result.doc_id]}" 56 | table.add_row( 57 | result.chunk_id, 58 | f"{result.post_title}({result.source})", 59 | result.text, 60 | chunk_number, 61 | result.publish_date.strftime("%Y-%m"), 62 | ) 63 | Console().print(table) 64 | -------------------------------------------------------------------------------- /rag_app/src/chunking.py: -------------------------------------------------------------------------------- 1 | import frontmatter 2 | import hashlib 3 | from datetime import datetime 4 | from unstructured.partition.text import partition_text 5 | from rag_app.models import Document 6 | from typing import Iterable 7 | from pathlib import Path 8 | from typing import List, TypeVar, Iterable 9 | 10 | T = TypeVar("T") 11 | 12 | 13 | def generate_string_hash(s: str): 14 | return hashlib.md5(s.encode("utf-8")).hexdigest() 15 | 16 | 17 | def read_files(path: Path, file_suffix: str) -> Iterable[Document]: 18 | for file in path.iterdir(): 19 | if file.suffix != file_suffix: 20 | continue 21 | post = frontmatter.load(file) 22 | yield Document( 23 | id=generate_string_hash(post.content), 24 | content=post.content, 25 | filename=file.name, 26 | metadata=post.metadata, 27 | ) 28 | 29 | 30 | def batch_items(items: Iterable[T], batch_size: int = 20) -> Iterable[List[T]]: 31 | batch = [] 32 | for item in items: 33 | batch.append(item) 34 | if len(batch) == batch_size: 35 | yield batch 36 | batch = [] 37 | 38 | if batch: 39 | yield batch 40 | 41 | 42 | def chunk_text( 43 | documents: Iterable[Document], window_size: int = 1024, overlap: int = 0 44 | ): 45 | for doc in documents: 46 | for chunk_num, chunk in enumerate(partition_text(text=doc.content)): 47 | yield { 48 | "chunk_id": generate_string_hash(chunk.text), 49 | "chunk_number": chunk_num + 1, 50 | "doc_id": doc.id, 51 | "text": chunk.text, 52 | "post_title": doc.metadata.title, 53 | "publish_date": datetime.strptime(doc.metadata.date, "%Y-%m"), 54 | "source": doc.metadata.url, 55 | } 56 | -------------------------------------------------------------------------------- /rag_app/src/metrics.py: -------------------------------------------------------------------------------- 1 | from sklearn.metrics import ndcg_score 2 | import numpy as np 3 | 4 | 5 | def calculate_mrr(chunk_id, predictions): 6 | return 0 if chunk_id not in predictions else 1 / (predictions.index(chunk_id) + 1) 7 | 8 | 9 | def calculate_ndcg(chunk_id, predictions): 10 | if len(predictions) == 0: 11 | return "N/A" 12 | if len(predictions) == 1: 13 | return 1 if chunk_id in predictions else 0 14 | 15 | y_pred = np.linspace(1, 0, len(predictions)).tolist() 16 | y_true = [0 if item != chunk_id else 1 for item in predictions] 17 | 18 | return ndcg_score([y_true], [y_pred]) 19 | 20 | 21 | def slice_predictions_decorator(num_elements: int): 22 | def decorator(func): 23 | def wrapper(chunk_id, predictions): 24 | sliced_predictions = predictions[:num_elements] 25 | return func(chunk_id, sliced_predictions) 26 | 27 | return wrapper 28 | 29 | return decorator 30 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | lancedb 2 | tqdm 3 | openai -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | # Duck DB test 2 | from lancedb import connect 3 | import duckdb 4 | 5 | db = connect("./db") 6 | 7 | chunk_table = db.open_table("pg") 8 | doc_table = db.open_table("document") 9 | ## Test that we can do simple select queries 10 | chunks = chunk_table.to_lance() 11 | docs = doc_table.to_lance() 12 | 13 | queries = [ 14 | "SELECT * FROM docs", 15 | "SELECT doc_id, count(chunk_id) as count FROM chunks GROUP BY doc_id", 16 | "SELECT * FROM docs WHERE metadata->>'date' LIKE '2021%';", 17 | "SELECT * FROM chunks INNER JOIN docs on chunks.doc_id = docs.id LIMIT 10", 18 | "SELECT * FROM chunks INNER JOIN docs on chunks.doc_id = docs.id WHERE doc_id = '897b17269004b437f58b5bf1f883e2dc' LIMIT 10 ", 19 | ] 20 | 21 | for query in queries: 22 | print(f"EXECUTING >> {query}") 23 | print(duckdb.query(query)) 24 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jxnl/n-levels-of-rag/2ce1106b0163d11fcf7959187b42f0d3a67fbe05/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | from rag_app.models import DocumentMetadata, Document 4 | 5 | 6 | def test_document_metadata_valid_date(): 7 | valid_metadata = { 8 | "date": "2024-10", 9 | "url": "https://example.com", 10 | "title": "Test Title", 11 | } 12 | 13 | try: 14 | DocumentMetadata(**valid_metadata) 15 | except ValidationError as e: 16 | pytest.fail(f"Unexpected ValidationError: {e}") 17 | 18 | 19 | def test_document_metadata_invalid_date(): 20 | invalid_metadata = { 21 | "date": "2024-13", # Invalid month 22 | "url": "https://example.com", 23 | "title": "Test Title", 24 | } 25 | with pytest.raises(ValueError) as excinfo: 26 | DocumentMetadata(**invalid_metadata) 27 | assert "Date format must be YYYY-MM" in str(excinfo.value) 28 | 29 | 30 | def test_document_creation(): 31 | document_data = { 32 | "id": "doc123", 33 | "content": "This is a test document.", 34 | "filename": "test_document.txt", 35 | "metadata": { 36 | "date": "2024-10", 37 | "url": "https://example.com", 38 | "title": "Test Title", 39 | }, 40 | } 41 | try: 42 | Document(**document_data) 43 | except ValidationError as e: 44 | pytest.fail(f"Unexpected ValidationError: {e}") 45 | 46 | 47 | def test_document_creation_with_invalid_metadata(): 48 | document_data = { 49 | "id": "doc123", 50 | "content": "This is a test document.", 51 | "filename": "test_document.txt", 52 | "metadata": { 53 | "date": "2024-13", # Invalid month 54 | "url": "https://example.com", 55 | "title": "Test Title", 56 | }, 57 | } 58 | with pytest.raises(ValueError) as excinfo: 59 | Document(**document_data) 60 | assert "Date format must be YYYY-MM" in str(excinfo.value) 61 | --------------------------------------------------------------------------------