├── .env ├── .eslintrc.cjs ├── .gitignore ├── LICENSE ├── README.md ├── convex ├── _generated │ ├── api.d.ts │ ├── api.js │ ├── dataModel.d.ts │ ├── server.d.ts │ └── server.js ├── ai.ts ├── crons.ts ├── files.ts ├── log.ts ├── repo.ts ├── schema.ts ├── search.ts ├── settings.ts ├── syncState.ts └── tsconfig.json ├── dryad_settings.png ├── dryad_ss.png ├── env_ss.png ├── index.html ├── package-lock.json ├── package.json ├── postcss.config.js ├── public └── favicon.png ├── src ├── App.css ├── App.tsx ├── assets │ ├── 3-dots-bounce.svg │ ├── 90-ring-with-bg.svg │ ├── Github_white.svg │ ├── convex_logo.svg │ ├── dryad_logo.png │ └── question.svg ├── components │ ├── CodeDisplay.tsx │ ├── EventLog.tsx │ ├── Info.tsx │ ├── SearchBox.tsx │ └── SearchResults.tsx ├── index.css ├── main.tsx └── vite-env.d.ts ├── tailwind.config.js ├── tsconfig.json ├── tsconfig.node.json └── vite.config.ts /.env: -------------------------------------------------------------------------------- 1 | VITE_CONVEX_URL="https://polished-lynx-960.convex.cloud" 2 | -------------------------------------------------------------------------------- /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { browser: true, es2020: true }, 4 | extends: [ 5 | "eslint:recommended", 6 | "plugin:@typescript-eslint/recommended", 7 | "plugin:react-hooks/recommended", 8 | ], 9 | ignorePatterns: ["dist", ".eslintrc.cjs"], 10 | parser: "@typescript-eslint/parser", 11 | plugins: ["react-refresh"], 12 | rules: { 13 | "react-refresh/only-export-components": [ 14 | "warn", 15 | { allowConstantExport: true }, 16 | ], 17 | }, 18 | }; 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | .vercel 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Convex 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dryad - talk to your tree 2 | 3 | Easy semantic code search on any GitHub repository in ~1000 SLOC. 4 | 5 | [Check out the running demo](http://convex.dev/dryad) 6 | 7 | ![dryad](dryad_ss.png) 8 | 9 | Dryad is intended to be a useful demo project and starter template for building more sophisticated 10 | semantic search web apps. 11 | 12 | Features: 13 | 14 | - Automatically tracks changes in the target repo and keeps the search index in sync with `HEAD` 15 | - Built with [Convex](https://convex.dev), [OpenAI](https://openai.com), 16 | [Vite](https://vitejs.dev/) + [React](https://react.dev/). 17 | - Easy to read, fork, and modify. 18 | - Reconfigurable on the fly using the Convex dashboard 19 | 20 | # Running your own dryad (on your favorite codebase) 21 | 22 | First, clone the repository and start it up: 23 | 24 | $ git clone https://github.com/get-convex/dryad.git 25 | $ npm i 26 | $ npm run dev 27 | 28 | This will create your Convex backend deployment, which will 29 | attempt to start indexing the default repository (https://github.com/get-convex/convex-helpers). 30 | Then, the frontend will start up, running on vite's usual port 5173. 31 | 32 | In another terminal in this same repository, launch the Convex dashboard and watch the logs to 33 | follow along with backend indexing: 34 | 35 | $ npx convex dashboard 36 | 37 | In the `Logs` panel, you'll see errors about missing environment variables. 38 | We have a little more set up to do! 39 | 40 | ## 1. Set deployment environment variables for OpenAI and GitHub 41 | 42 | ### OpenAI 43 | 44 | Dryad uses OpenAI for summarization and embedding. You'll need an OpenAI platform account 45 | and an API key. Visit [platform.openai.com](https://platform.openai.com) to 46 | take care of that. 47 | 48 | > :warning: Summarizing and indexing even a moderate codebase consumes a fair amount of OpenAI 49 | > credits. You will almost certainly need a paid account! 50 | 51 | ### GitHub 52 | 53 | Anonymous uses of the GitHub API get rate limited very easily. So dryad require that you 54 | generate a personal access token using your GitHub account. Visit 55 | [https://github.com/settings/tokens](https://github.com/settings/tokens) to generate 56 | a token for dryad. 57 | 58 | ### Setting these environment variables in your Convex deployment 59 | 60 | With your OpenAI API key and GitHub access token in hand, go back to your 61 | Convex deployment's dashboard. In the left navigation panel, click "Settings", 62 | and then "Environment Variables". 63 | 64 | Name the two secret environment variables `OPENAI_API_KEY` and `GITHUB_ACCESS_TOKEN`, like so: 65 | 66 | ![dashboard environment variables](env_ss.png) 67 | 68 | ## 2. Customize your dryad settings in the `settings` table 69 | 70 | If you check the `Logs` view in your Convex dashboard, dryad now should 71 | be running successfully! But it's indexing the default repository, 72 | `get-convex/convex-helpers`. You probably want it indexing your own 73 | code instead. 74 | 75 | Good news! It's easy to customize dryad's behavior. Dryad keeps all 76 | its configuration in a `settings` table in your Convex database 77 | itself. Click on the `Data` view in the dashboard, and then choose 78 | the `settings` table: 79 | 80 | ![settings table](dryad_settings.png) 81 | 82 | Double click any value in the settings document to edit it, or click the blue "EDIT" button to add missing fields to the document. Normally, you shouldn't need to do anything for your changes to take effect. But if you want to reindex anyway click the `Fn` function runner in the lower right panel 83 | of the dashboard, and then choose to run `syncState:reset` from the dropdown. No arguments are required. 84 | 85 | The schema of this table can be found in `convex/schema.ts` in this repository. Here's what it looks like: 86 | 87 | ```tsx 88 | // Various project settings you can tweak in the dashboard as we go. 89 | settings: defineTable({ 90 | org: v.string(), 91 | repo: v.string(), 92 | branch: v.string(), 93 | extensions: v.array(v.string()), 94 | exclusions: v.optional(v.array(v.string())), // defaults to no exclusions 95 | byteLimit: v.optional(v.number()), // defaults to 24,000 bytes 96 | chatModel: v.optional(v.string()), // defaults to gpt-4 97 | }), 98 | ``` 99 | 100 | ### Settings fields 101 | 102 | - **org** - The organization owner of the target GitHub repo to index. For React (https://github.com/facebook/react), this is `facebook`. 103 | - **repo** - The repository name of the target GitHub repo to index. For React (https://github.com/facebook/react), this is `react`. 104 | - **branch** - The the branch name in the repository to index. This is usually 'main', or 'master'. 105 | - **extensions** - An array of file extensions (like '.ts') that should be considered code and therefore dryad should attempt to index. 106 | - **exclusions** - An array of relative file paths with the repository you wish to explicitly skip indexing. 107 | - **byteLimit** - Do not index files larger than this byte count. Large files will produce more tokens 108 | that the OpenAI model is able to process in one pass. 109 | - **chatModel** - Which OpenAI chat model to use for summarizing the purposes of source files. Typical choices are `gpt-3.5-turbo`, `gpt-4`. 110 | 111 | # How dryad works 112 | 113 | Three main things to cover: 114 | 115 | 1. Keeping up to date with repository changes 116 | 2. Indexing source files 117 | 3. Searching for semantic matches 118 | 119 | ## 1. Keeping up to date with repository changes 120 | 121 | Every minute, dryad calls a job named `repo:sync`. This 122 | is a Convex action which uses a table called `syncState` to 123 | loop between two states: 124 | 125 | 1. Polling for a new commit on HEAD. 126 | 2. Indexing that commit 127 | 128 | While polling for a new commit, dryad uses the GitHub API (via Octokit) 129 | to check the sha of the target repo + branch. As long as the value coming back from GitHub 130 | remains the same as the last indexed sha in `syncState.commit`, `repo:sync` exits until the next poll. 131 | 132 | But when a new commit is discovered, the `syncState.commit` field is set to 133 | that is set to the new sha, and tha `commitDone` field is set to false. This puts 134 | dryad into "Indexing that commit" mode. 135 | 136 | When indexing a commit, `repo:sync` first uses the GitHub "trees" API to fetch the entire 137 | file tree of that commit, including the file checksums associated with every file. 138 | 139 | Dryad then walks this whole tree, looking for source code files (according to the `settings`` 140 | table's extension specification). For every source file, it determines if the checksum 141 | has changed since the last time the file was indexed. If the file is new or has changed, 142 | it is downloaded from the repo and re-indexed. 143 | 144 | Otherwise the file is marked current–still valid in new commit. 145 | 146 | Finally, after all files in the tree are properly indexed, any files that no longer part of this new commit tree are removed from the index. 147 | 148 | And with that, `commitDone` is set to true and dryad goes back to polling for a new commit. 149 | 150 | ## 2. Indexing source files 151 | 152 | Indexing source files involves three steps: 153 | 154 | 1. Ask ChatGPT to summarize the "primary goals" of the source file in JSON format. 155 | 1. Take each of those goals and independently ask OpenAI to generate a vector embedding 156 | for it. [Learn more about embeddings here.](https://youtu.be/m6eWdnRhBpA) 157 | 1. Store each goal and associated vector into Convex's `fileGoals` table, with a reference to the parent source file record in `files`. The goal's vector field is using Convex's vector indexing to support fast searching from the web app. 158 | 159 | ## 3. Searching for semantic matches 160 | 161 | When someone submits a query in the web app, dryad uses the same OpenAI embeddings API to generate 162 | a vector, and then uses Convex's vector index to find source files with a semantically-similar goal 163 | to the search term. 164 | 165 | Searching only returns each source file one time, returning the highest-ranked goal as the primary 166 | reason for that file's inclusion in the result set. 167 | 168 | # Exercises – Next improvements for dryad 169 | 170 | Dryad is quite basic at this point! There are a lot of directions you could take the project in. 171 | 172 | The project's issues have been seeded with [a collection of potential extensions and improvements to dryad](https://github.com/get-convex/dryad/labels/good%20first%20issue) to get the wheels turning 173 | about more sophisticated things that could be built from dryad. 174 | 175 | Happy hacking! 176 | 177 | # Community 178 | 179 | [Join our discord to talk about dryad.](https://convex.dev/community) 180 | 181 | # What is Convex? 182 | 183 | [Convex](https://convex.dev) is a hosted backend platform with a 184 | built-in database that lets you write your 185 | [database schema](https://docs.convex.dev/database/schemas) and 186 | [server functions](https://docs.convex.dev/functions) in 187 | [TypeScript](https://docs.convex.dev/typescript). Server-side database 188 | [queries](https://docs.convex.dev/functions/query-functions) automatically 189 | [cache](https://docs.convex.dev/functions/query-functions#caching--reactivity) and 190 | [subscribe](https://docs.convex.dev/client/react#reactivity) to data, powering a 191 | [realtime `useQuery` hook](https://docs.convex.dev/client/react#fetching-data) in our 192 | [React client](https://docs.convex.dev/client/react). There are also clients for 193 | [Python](https://docs.convex.dev/client/python), 194 | [Rust](https://docs.convex.dev/client/rust), 195 | [ReactNative](https://docs.convex.dev/client/react-native), and 196 | [Node](https://docs.convex.dev/client/javascript), as well as a straightforward 197 | [HTTP API](https://github.com/get-convex/convex-js/blob/main/src/browser/http_client.ts#L40). 198 | 199 | The database supports 200 | [NoSQL-style documents](https://docs.convex.dev/database/document-storage) with 201 | [relationships](https://docs.convex.dev/database/document-ids) and 202 | [custom indexes](https://docs.convex.dev/database/indexes/) 203 | (including on fields in nested objects). 204 | 205 | The 206 | [`query`](https://docs.convex.dev/functions/query-functions) and 207 | [`mutation`](https://docs.convex.dev/functions/mutation-functions) server functions have transactional, 208 | low latency access to the database and leverage our 209 | [`v8` runtime](https://docs.convex.dev/functions/runtimes) with 210 | [determinism guardrails](https://docs.convex.dev/functions/runtimes#using-randomness-and-time-in-queries-and-mutations) 211 | to provide the strongest ACID guarantees on the market: 212 | immediate consistency, 213 | serializable isolation, and 214 | automatic conflict resolution via 215 | [optimistic multi-version concurrency control](https://docs.convex.dev/database/advanced/occ) (OCC / MVCC). 216 | 217 | The [`action` server functions](https://docs.convex.dev/functions/actions) have 218 | access to external APIs and enable other side-effects and non-determinism in 219 | either our 220 | [optimized `v8` runtime](https://docs.convex.dev/functions/runtimes) or a more 221 | [flexible `node` runtime](https://docs.convex.dev/functions/runtimes#nodejs-runtime). 222 | 223 | Functions can run in the background via 224 | [scheduling](https://docs.convex.dev/scheduling/scheduled-functions) and 225 | [cron jobs](https://docs.convex.dev/scheduling/cron-jobs). 226 | 227 | Development is cloud-first, with 228 | [hot reloads for server function](https://docs.convex.dev/cli#run-the-convex-dev-server) editing via the 229 | [CLI](https://docs.convex.dev/cli). There is a 230 | [dashboard UI](https://docs.convex.dev/dashboard) to 231 | [browse and edit data](https://docs.convex.dev/dashboard/deployments/data), 232 | [edit environment variables](https://docs.convex.dev/production/environment-variables), 233 | [view logs](https://docs.convex.dev/dashboard/deployments/logs), 234 | [run server functions](https://docs.convex.dev/dashboard/deployments/functions), and more. 235 | 236 | There are built-in features for 237 | [reactive pagination](https://docs.convex.dev/database/pagination), 238 | [file storage](https://docs.convex.dev/file-storage), 239 | [reactive search](https://docs.convex.dev/text-search), 240 | [https endpoints](https://docs.convex.dev/functions/http-actions) (for webhooks), 241 | [streaming import/export](https://docs.convex.dev/database/import-export/), and 242 | [runtime data validation](https://docs.convex.dev/database/schemas#validators) for 243 | [function arguments](https://docs.convex.dev/functions/args-validation) and 244 | [database data](https://docs.convex.dev/database/schemas#schema-validation). 245 | 246 | Everything scales automatically, and it’s [free to start](https://www.convex.dev/plans). -------------------------------------------------------------------------------- /convex/_generated/api.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated `api` utility. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * Generated by convex@1.2.1-alpha.1. 8 | * To regenerate, run `npx convex dev`. 9 | * @module 10 | */ 11 | 12 | import type { 13 | ApiFromModules, 14 | FilterApi, 15 | FunctionReference, 16 | } from "convex/server"; 17 | import type * as ai from "../ai"; 18 | import type * as crons from "../crons"; 19 | import type * as files from "../files"; 20 | import type * as log from "../log"; 21 | import type * as repo from "../repo"; 22 | import type * as search from "../search"; 23 | import type * as settings from "../settings"; 24 | import type * as syncState from "../syncState"; 25 | 26 | /** 27 | * A utility for referencing Convex functions in your app's API. 28 | * 29 | * Usage: 30 | * ```js 31 | * const myFunctionReference = api.myModule.myFunction; 32 | * ``` 33 | */ 34 | declare const fullApi: ApiFromModules<{ 35 | ai: typeof ai; 36 | crons: typeof crons; 37 | files: typeof files; 38 | log: typeof log; 39 | repo: typeof repo; 40 | search: typeof search; 41 | settings: typeof settings; 42 | syncState: typeof syncState; 43 | }>; 44 | export declare const api: FilterApi< 45 | typeof fullApi, 46 | FunctionReference 47 | >; 48 | export declare const internal: FilterApi< 49 | typeof fullApi, 50 | FunctionReference 51 | >; 52 | -------------------------------------------------------------------------------- /convex/_generated/api.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated `api` utility. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * Generated by convex@1.2.1-alpha.1. 8 | * To regenerate, run `npx convex dev`. 9 | * @module 10 | */ 11 | 12 | import { anyApi } from "convex/server"; 13 | 14 | /** 15 | * A utility for referencing Convex functions in your app's API. 16 | * 17 | * Usage: 18 | * ```js 19 | * const myFunctionReference = api.myModule.myFunction; 20 | * ``` 21 | */ 22 | export const api = anyApi; 23 | export const internal = anyApi; 24 | -------------------------------------------------------------------------------- /convex/_generated/dataModel.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated data model types. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * Generated by convex@1.2.1-alpha.1. 8 | * To regenerate, run `npx convex dev`. 9 | * @module 10 | */ 11 | 12 | import type { DataModelFromSchemaDefinition } from "convex/server"; 13 | import type { DocumentByName, TableNamesInDataModel } from "convex/server"; 14 | import type { GenericId } from "convex/values"; 15 | import schema from "../schema"; 16 | 17 | /** 18 | * The names of all of your Convex tables. 19 | */ 20 | export type TableNames = TableNamesInDataModel; 21 | 22 | /** 23 | * The type of a document stored in Convex. 24 | * 25 | * @typeParam TableName - A string literal type of the table name (like "users"). 26 | */ 27 | export type Doc = DocumentByName< 28 | DataModel, 29 | TableName 30 | >; 31 | 32 | /** 33 | * An identifier for a document in Convex. 34 | * 35 | * Convex documents are uniquely identified by their `Id`, which is accessible 36 | * on the `_id` field. To learn more, see [Document IDs](https://docs.convex.dev/using/document-ids). 37 | * 38 | * Documents can be loaded using `db.get(id)` in query and mutation functions. 39 | * 40 | * IDs are just strings at runtime, but this type can be used to distinguish them from other 41 | * strings when type checking. 42 | * 43 | * @typeParam TableName - A string literal type of the table name (like "users"). 44 | */ 45 | export type Id = GenericId; 46 | 47 | /** 48 | * A type describing your Convex data model. 49 | * 50 | * This type includes information about what tables you have, the type of 51 | * documents stored in those tables, and the indexes defined on them. 52 | * 53 | * This type is used to parameterize methods like `queryGeneric` and 54 | * `mutationGeneric` to make them type-safe. 55 | */ 56 | export type DataModel = DataModelFromSchemaDefinition; 57 | -------------------------------------------------------------------------------- /convex/_generated/server.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated utilities for implementing server-side Convex query and mutation functions. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * Generated by convex@1.2.1-alpha.1. 8 | * To regenerate, run `npx convex dev`. 9 | * @module 10 | */ 11 | 12 | import { 13 | ActionBuilder, 14 | HttpActionBuilder, 15 | MutationBuilder, 16 | QueryBuilder, 17 | GenericActionCtx, 18 | GenericMutationCtx, 19 | GenericQueryCtx, 20 | GenericDatabaseReader, 21 | GenericDatabaseWriter, 22 | } from "convex/server"; 23 | import type { DataModel } from "./dataModel.js"; 24 | 25 | /** 26 | * Define a query in this Convex app's public API. 27 | * 28 | * This function will be allowed to read your Convex database and will be accessible from the client. 29 | * 30 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 31 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 32 | */ 33 | export declare const query: QueryBuilder; 34 | 35 | /** 36 | * Define a query that is only accessible from other Convex functions (but not from the client). 37 | * 38 | * This function will be allowed to read from your Convex database. It will not be accessible from the client. 39 | * 40 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 41 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 42 | */ 43 | export declare const internalQuery: QueryBuilder; 44 | 45 | /** 46 | * Define a mutation in this Convex app's public API. 47 | * 48 | * This function will be allowed to modify your Convex database and will be accessible from the client. 49 | * 50 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 51 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 52 | */ 53 | export declare const mutation: MutationBuilder; 54 | 55 | /** 56 | * Define a mutation that is only accessible from other Convex functions (but not from the client). 57 | * 58 | * This function will be allowed to modify your Convex database. It will not be accessible from the client. 59 | * 60 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 61 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 62 | */ 63 | export declare const internalMutation: MutationBuilder; 64 | 65 | /** 66 | * Define an action in this Convex app's public API. 67 | * 68 | * An action is a function which can execute any JavaScript code, including non-deterministic 69 | * code and code with side-effects, like calling third-party services. 70 | * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. 71 | * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. 72 | * 73 | * @param func - The action. It receives an {@link ActionCtx} as its first argument. 74 | * @returns The wrapped action. Include this as an `export` to name it and make it accessible. 75 | */ 76 | export declare const action: ActionBuilder; 77 | 78 | /** 79 | * Define an action that is only accessible from other Convex functions (but not from the client). 80 | * 81 | * @param func - The function. It receives an {@link ActionCtx} as its first argument. 82 | * @returns The wrapped function. Include this as an `export` to name it and make it accessible. 83 | */ 84 | export declare const internalAction: ActionBuilder; 85 | 86 | /** 87 | * Define an HTTP action. 88 | * 89 | * This function will be used to respond to HTTP requests received by a Convex 90 | * deployment if the requests matches the path and method where this action 91 | * is routed. Be sure to route your action in `convex/http.js`. 92 | * 93 | * @param func - The function. It receives an {@link ActionCtx} as its first argument. 94 | * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. 95 | */ 96 | export declare const httpAction: HttpActionBuilder; 97 | 98 | /** 99 | * A set of services for use within Convex query functions. 100 | * 101 | * The query context is passed as the first argument to any Convex query 102 | * function run on the server. 103 | * 104 | * This differs from the {@link MutationCtx} because all of the services are 105 | * read-only. 106 | */ 107 | export type QueryCtx = GenericQueryCtx; 108 | 109 | /** 110 | * A set of services for use within Convex mutation functions. 111 | * 112 | * The mutation context is passed as the first argument to any Convex mutation 113 | * function run on the server. 114 | */ 115 | export type MutationCtx = GenericMutationCtx; 116 | 117 | /** 118 | * A set of services for use within Convex action functions. 119 | * 120 | * The action context is passed as the first argument to any Convex action 121 | * function run on the server. 122 | */ 123 | export type ActionCtx = GenericActionCtx; 124 | 125 | /** 126 | * An interface to read from the database within Convex query functions. 127 | * 128 | * The two entry points are {@link DatabaseReader.get}, which fetches a single 129 | * document by its {@link Id}, or {@link DatabaseReader.query}, which starts 130 | * building a query. 131 | */ 132 | export type DatabaseReader = GenericDatabaseReader; 133 | 134 | /** 135 | * An interface to read from and write to the database within Convex mutation 136 | * functions. 137 | * 138 | * Convex guarantees that all writes within a single mutation are 139 | * executed atomically, so you never have to worry about partial writes leaving 140 | * your data in an inconsistent state. See [the Convex Guide](https://docs.convex.dev/understanding/convex-fundamentals/functions#atomicity-and-optimistic-concurrency-control) 141 | * for the guarantees Convex provides your functions. 142 | */ 143 | export type DatabaseWriter = GenericDatabaseWriter; 144 | -------------------------------------------------------------------------------- /convex/_generated/server.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated utilities for implementing server-side Convex query and mutation functions. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * Generated by convex@1.2.1-alpha.1. 8 | * To regenerate, run `npx convex dev`. 9 | * @module 10 | */ 11 | 12 | import { 13 | actionGeneric, 14 | httpActionGeneric, 15 | queryGeneric, 16 | mutationGeneric, 17 | internalActionGeneric, 18 | internalMutationGeneric, 19 | internalQueryGeneric, 20 | } from "convex/server"; 21 | 22 | /** 23 | * Define a query in this Convex app's public API. 24 | * 25 | * This function will be allowed to read your Convex database and will be accessible from the client. 26 | * 27 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 28 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 29 | */ 30 | export const query = queryGeneric; 31 | 32 | /** 33 | * Define a query that is only accessible from other Convex functions (but not from the client). 34 | * 35 | * This function will be allowed to read from your Convex database. It will not be accessible from the client. 36 | * 37 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 38 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 39 | */ 40 | export const internalQuery = internalQueryGeneric; 41 | 42 | /** 43 | * Define a mutation in this Convex app's public API. 44 | * 45 | * This function will be allowed to modify your Convex database and will be accessible from the client. 46 | * 47 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 48 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 49 | */ 50 | export const mutation = mutationGeneric; 51 | 52 | /** 53 | * Define a mutation that is only accessible from other Convex functions (but not from the client). 54 | * 55 | * This function will be allowed to modify your Convex database. It will not be accessible from the client. 56 | * 57 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 58 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 59 | */ 60 | export const internalMutation = internalMutationGeneric; 61 | 62 | /** 63 | * Define an action in this Convex app's public API. 64 | * 65 | * An action is a function which can execute any JavaScript code, including non-deterministic 66 | * code and code with side-effects, like calling third-party services. 67 | * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. 68 | * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. 69 | * 70 | * @param func - The action. It receives an {@link ActionCtx} as its first argument. 71 | * @returns The wrapped action. Include this as an `export` to name it and make it accessible. 72 | */ 73 | export const action = actionGeneric; 74 | 75 | /** 76 | * Define an action that is only accessible from other Convex functions (but not from the client). 77 | * 78 | * @param func - The function. It receives an {@link ActionCtx} as its first argument. 79 | * @returns The wrapped function. Include this as an `export` to name it and make it accessible. 80 | */ 81 | export const internalAction = internalActionGeneric; 82 | 83 | /** 84 | * Define a Convex HTTP action. 85 | * 86 | * @param func - The function. It receives an {@link ActionCtx} as its first argument, and a `Request` object 87 | * as its second. 88 | * @returns The wrapped endpoint function. Route a URL path to this function in `convex/http.js`. 89 | */ 90 | export const httpAction = httpActionGeneric; 91 | -------------------------------------------------------------------------------- /convex/ai.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Calls to OpenAI for summarization and embedding. 3 | */ 4 | "use node"; 5 | import OpenAI from "openai"; 6 | import { Doc } from "./_generated/dataModel"; 7 | 8 | const DEFAULT_MODEL = "gpt-4"; 9 | 10 | /** Given a file at `path` and a file contents of `body`, provide a summary of 11 | * the high-level goals of the source code file. 12 | */ 13 | export async function summarize( 14 | settings: Doc<"settings">, 15 | path: string, 16 | body: string, 17 | ): Promise { 18 | // This prompt could use a lot of iteration. In general, this version of asking 19 | // for the response in JSON format is pretty reliable. 20 | 21 | // We ask for the "main goals" of the file so that we get thematic extractions 22 | // that are likely to be semantically adjacent to questions a user might ask about 23 | // where to find particular code that does X, Y, or Z. 24 | const prompt = ` 25 | Please provide a list of the main goals 26 | of the following computer source code file. This code comes from a file 27 | called '${path}'. 28 | 29 | Without any comment, return your answer in the following ECMA-404 compliant JSON format: 30 | {"programming_language":"Rust","goals":["Authenticate users","Validate JWT payloads","Ensure strong passwords"]} 31 | 32 | The body of the file follows the delimeter "---". 33 | 34 | --- 35 | ${body} 36 | `; 37 | const openai = new OpenAI({ 38 | apiKey: process.env.OPENAI_API_KEY, 39 | }); 40 | const completion = await openai.chat.completions.create({ 41 | messages: [{ role: "user", content: prompt }], 42 | model: settings.chatModel ?? DEFAULT_MODEL, 43 | }); 44 | return completion.choices[0].message.content!; 45 | } 46 | 47 | /** Use OpenAI to generate N vector embeddings for the given array of N strings. 48 | * 49 | * We'll put these into a vector index so we can quickly calculate cosine similarity 50 | * with embeddings generated from user-provided prompts. 51 | */ 52 | export async function generateEmbeddings( 53 | fragments: string[], 54 | ): Promise { 55 | const openai = new OpenAI({ 56 | apiKey: process.env.OPENAI_API_KEY, 57 | }); 58 | const embedding = await openai.embeddings.create({ 59 | input: fragments, 60 | model: "text-embedding-ada-002", 61 | }); 62 | const vectors = embedding.data.map((e) => e.embedding); 63 | return vectors; 64 | } 65 | 66 | /** Helper function to generate just a single embedding. 67 | * 68 | * Wraps the more full featured function `generateEmbeddings`. 69 | */ 70 | export async function generateEmbedding(fragment: string): Promise { 71 | return (await generateEmbeddings([fragment]))[0]; 72 | } 73 | -------------------------------------------------------------------------------- /convex/crons.ts: -------------------------------------------------------------------------------- 1 | /** Cron jobs our Convex project uses. */ 2 | import { cronJobs } from "convex/server"; 3 | import { internal } from "./_generated/api"; 4 | 5 | const crons = cronJobs(); 6 | 7 | /** Every minute check the GitHub repo for new commits. 8 | * 9 | * In an ideal world, we could use a web hook instead. Exercise left to future 10 | * developers. 11 | */ 12 | crons.interval( 13 | "poll for new commits", 14 | { minutes: 1 }, // every minute 15 | internal.repo.sync, 16 | ); 17 | 18 | export default crons; 19 | -------------------------------------------------------------------------------- /convex/files.ts: -------------------------------------------------------------------------------- 1 | /** Model management for file and file goal state. 2 | * 3 | * These functions abstract the transactional model for how we 4 | * update and delete file contents as the repository changes, and how 5 | * we relationally associate goals (and their embeddings) with the 6 | * file that generated them. 7 | */ 8 | import { Id } from "./_generated/dataModel"; 9 | import { internalMutation, internalQuery } from "./_generated/server"; 10 | import { v } from "convex/values"; 11 | import { MutationCtx } from "./_generated/server"; 12 | import { writeLog } from "./log"; 13 | 14 | /** Given a source code file at `path`, check to see if we've 15 | * already indexed it with sha `fileSha`. 16 | * 17 | * If so, we'll return `false` and update the file's `treeSha` 18 | * to the provided one to mark it still in the current snapshot. 19 | * 20 | * If not, we'll return `true`, telling the crawler to re-download 21 | * and re-index the newer file contents. 22 | */ 23 | export const checkPending = internalMutation({ 24 | args: { 25 | path: v.string(), 26 | fileSha: v.string(), 27 | treeSha: v.string(), 28 | }, 29 | handler: async (ctx, args): Promise => { 30 | const fileData = await ctx.db 31 | .query("files") 32 | .withIndex("path", (q) => q.eq("path", args.path)) 33 | .first(); 34 | if (fileData === null) { 35 | return true; // No file yet? go ahead and run it. 36 | } 37 | if (args.fileSha !== fileData!.fileSha) { 38 | return true; 39 | } 40 | // Otherwise... same file. If this is a new commit, just mark it current. 41 | if (args.treeSha !== fileData.treeSha) { 42 | await ctx.db.patch(fileData._id, { 43 | treeSha: args.treeSha, 44 | }); 45 | } 46 | 47 | return false; 48 | }, 49 | }); 50 | 51 | /** Delete the file with id = `id` and also delete all associated goals and 52 | * embeddings. 53 | */ 54 | async function recursiveDeleteFile(ctx: MutationCtx, id: Id<"files">) { 55 | const oldFile = await ctx.db.get(id); 56 | await ctx.db.delete(id); 57 | const goals = await ctx.db 58 | .query("fileGoals") 59 | .withIndex("fileId", (q) => q.eq("fileId", id)) 60 | .collect(); 61 | for (const goal of goals) { 62 | await ctx.db.delete(goal._id); 63 | } 64 | await writeLog(ctx, "cleanup", oldFile!.fileSha, oldFile!.path); 65 | } 66 | 67 | /** Add the given file at `path` with summarized `goals` and embeddings 68 | * in `vectors` to our database. The file was fetched as part of the 69 | * tree snapshot at `treeSha`, and was inferred to be in programming 70 | * language `language`. 71 | */ 72 | export const index = internalMutation({ 73 | args: { 74 | path: v.string(), 75 | fileSha: v.string(), 76 | treeSha: v.string(), 77 | goals: v.array(v.string()), 78 | vectors: v.array(v.array(v.number())), 79 | language: v.string(), 80 | }, 81 | handler: async (ctx, args) => { 82 | // 1. Delete existing record if it exists. 83 | const existing = await ctx.db 84 | .query("files") 85 | .withIndex("path", (q) => q.eq("path", args.path)) 86 | .first(); 87 | if (existing !== null) { 88 | await recursiveDeleteFile(ctx, existing._id); 89 | } 90 | 91 | // 2. Insert new file record. 92 | const newFile = await ctx.db.insert("files", { 93 | path: args.path, 94 | fileSha: args.fileSha, 95 | treeSha: args.treeSha, 96 | language: args.language, 97 | }); 98 | for (let i = 0; i < args.vectors.length; i++) { 99 | await ctx.db.insert("fileGoals", { 100 | fileId: newFile, 101 | goal: args.goals[i], 102 | vector: args.vectors[i], 103 | }); 104 | } 105 | 106 | // 3. Log 107 | await writeLog(ctx, "add", args.fileSha, args.path); 108 | }, 109 | }); 110 | 111 | /** Grab a batch of any files in the database at any commit not 112 | * equal to the given `commit`. Recursively delete them so they're 113 | * no longer searchable. 114 | */ 115 | export const clearDeadFiles = internalMutation({ 116 | args: { 117 | commit: v.string(), 118 | }, 119 | handler: async (ctx, args) => { 120 | const syncState = await ctx.db.query("sync").unique(); 121 | if (syncState!.commitDone || syncState!.commit !== args.commit) { 122 | return; 123 | } 124 | 125 | // A little ugly two pass thing. Convex doesn't have a great way 126 | // to do "not equals" on an index, so we just fetch records less than 127 | // and then greater than the specific valid value (the current commit 128 | // hash. 129 | let batch = await ctx.db 130 | .query("files") 131 | .withIndex("treeSha", (q) => q.lt("treeSha", args.commit)) 132 | .take(10); 133 | if (batch.length === 0) { 134 | batch = await ctx.db 135 | .query("files") 136 | .withIndex("treeSha", (q) => q.gt("treeSha", args.commit)) 137 | .take(10); 138 | } 139 | // If we *still* don't have anything to clean up, we're done 140 | // and only valid files are left in the search index for the 141 | // given commit. 142 | if (batch.length === 0) { 143 | // We're done cleaning up other entries! 144 | await ctx.db.patch(syncState!._id, { 145 | commitDone: true, 146 | }); 147 | await writeLog(ctx, "finish", args.commit); 148 | return false; // no more work to do. 149 | } else { 150 | for (const f of batch) { 151 | await recursiveDeleteFile(ctx, f._id); 152 | } 153 | return true; // possibly more work to do. 154 | } 155 | }, 156 | }); 157 | 158 | /** Grab the goal provided by `id` and information about its 159 | * associated file. 160 | * 161 | * This is used to fetch a search result, where a query has 162 | * had a match with a particular embedding associated with a goal. 163 | * */ 164 | export const getGoalAndFile = internalQuery({ 165 | args: { 166 | id: v.id("fileGoals"), 167 | }, 168 | handler: async (ctx, args) => { 169 | const fg = await ctx.db.get(args.id); 170 | if (!fg) { 171 | return null; 172 | } 173 | const f = await ctx.db.get(fg!.fileId); 174 | return { 175 | path: f!.path, 176 | language: f!.language, 177 | goal: fg.goal, 178 | treeSha: f!.treeSha, 179 | }; 180 | }, 181 | }); 182 | -------------------------------------------------------------------------------- /convex/log.ts: -------------------------------------------------------------------------------- 1 | /** Represent the simple history of operation logs. 2 | * 3 | * These are used to display sync progress in the app. 4 | */ 5 | import { MutationCtx, query } from "./_generated/server"; 6 | 7 | /** Get the last 30 log entries. */ 8 | export const get = query({ 9 | handler: async (ctx) => { 10 | const entries = await ctx.db 11 | .query("log") 12 | .withIndex("cursor") 13 | .order("desc") 14 | .take(30); 15 | return entries; 16 | }, 17 | }); 18 | 19 | /** Write a new log entry out. "add" and "cleanup" have an associated path 20 | * and file checksum. "start" and "finish" are only repo-level, and so the 21 | * sha is for the whole tree. */ 22 | export const writeLog = async ( 23 | ctx: MutationCtx, 24 | operator: "add" | "cleanup" | "finish" | "start", 25 | sha: string, 26 | path?: string, 27 | ) => { 28 | // Cursor is monotonically increasing, so we need to grab the maximum value 29 | // and we'll increment by one. 30 | const lastEntry = 31 | (await ctx.db.query("log").withIndex("cursor").order("desc").first()) 32 | ?.cursor ?? 0; 33 | await ctx.db.insert("log", { 34 | cursor: lastEntry + 1, 35 | sha: sha, 36 | operator, 37 | path, 38 | }); 39 | }; 40 | -------------------------------------------------------------------------------- /convex/repo.ts: -------------------------------------------------------------------------------- 1 | /** Primary "crawling" logic that keeps the index in sync 2 | * with the content in a specific GitHub repository. 3 | * 4 | * The entry point here, `sync`, is called by a cron job. 5 | */ 6 | "use node"; 7 | import { Octokit } from "@octokit/rest"; 8 | import { internalAction } from "./_generated/server"; 9 | import { api, internal } from "./_generated/api"; 10 | import { Doc } from "./_generated/dataModel"; 11 | import { ActionCtx } from "./_generated/server"; 12 | import { generateEmbeddings, summarize } from "./ai"; 13 | 14 | // Don't attempt to process single files larger than this until 15 | // access to gpt-4-32k is commonly available. 16 | // A more precise way to do this would be to use tiktoken and calculate 17 | // the true number of tokens for the specific model, but ¯\_(ツ)_/¯ 18 | // If we don't skip files biger than this, we'll get OpenAI API call 19 | // errors about too many tokens. 20 | const DEFAULT_BYTE_LIMIT = 24000; 21 | 22 | /** This is the main "step forward" function which ensures steady progress 23 | * on converging our stored index toward the current `HEAD` of the target 24 | * repositories given branch. 25 | */ 26 | export const sync = internalAction({ 27 | args: {}, 28 | handler: async (ctx) => { 29 | // 0. Grab settings. 30 | const settings = await ctx.runQuery(api.settings.get); 31 | if (!settings) { 32 | console.log( 33 | "Please establish the 'settings' table. Call syncState:init to get initial working values", 34 | ); 35 | throw "no settings"; 36 | } 37 | // 1. Load sync state 38 | let syncState = await ctx.runQuery(internal.syncState.get); 39 | // We're not currently in the middle of processing a commit? 40 | if (syncState.commitDone) { 41 | // Grab the current HEAD of the target branch. 42 | const commit = await getHeadCommit(settings); 43 | // NOOP if origin's HEAD is already our processed commit. 44 | if (syncState.commit === commit) { 45 | console.log("No new commits. Skipping pass."); 46 | return; 47 | } 48 | // Otherwise, time to enter into the next commit. 49 | await ctx.runMutation(internal.syncState.startCommit, { 50 | commit, 51 | }); 52 | syncState = await ctx.runQuery(internal.syncState.get); 53 | } 54 | 55 | // 2. Let's make progress on current file contents and indexing 56 | // one batch at a time until we're done. 57 | console.log("Still work to do in current batch"); 58 | const done = await indexCurrentFiles(settings, ctx, syncState); 59 | if (!done) { 60 | // Why do we do this? 61 | // Just to keep logs flowing, release resources, etc. Shorter-running 62 | // actions are easier to keep manageable. So we'll just immediately 63 | // reschedule ourselves to run again in a new runtime instance. 64 | console.log("Batch done; ending action and running again momentarily."); 65 | await ctx.scheduler.runAfter(0, internal.repo.sync); 66 | return; 67 | } else { 68 | // We have now ensured all *existing* files in this new commit tree 69 | // are accurately indexed. However there are possibly some "dead" 70 | // files that were deleted. These are no longer in the repo but 71 | // are still in our search index. Let's keep going. 72 | console.log("Batch finished. Let's clean up dead files."); 73 | } 74 | 75 | // 3. All current file contents are marked and indexed. 76 | // Now, clean up files that are no longer in current tree 77 | // and mark the commit done. 78 | await clearOutdatedFiles(ctx, syncState.commit!); 79 | }, 80 | }); 81 | 82 | /** Grab the sha256 of the HEAD of our target repo and branch, as designated by 83 | * the settings table. 84 | */ 85 | async function getHeadCommit(settings: Doc<"settings">): Promise { 86 | const github = createOctokit(); 87 | const latestCommit = await github.rest.repos.listCommits({ 88 | repo: settings.repo, 89 | owner: settings.org, 90 | per_page: 1, 91 | sha: settings.branch, 92 | }); 93 | const commitSha: string = latestCommit.data[0].sha; 94 | return commitSha; 95 | } 96 | 97 | /** Is the given path likely to be code as defined by the project's set 98 | * of target file extensions? 99 | * 100 | * We want to limit what files we send to OpenAI since it's expensive to 101 | * stream low value, large file contents to ChatGPT just to get confusing 102 | * messages back. 103 | */ 104 | function isPathCode(extensions: Set, path?: string): boolean { 105 | if (!path) { 106 | return false; 107 | } 108 | const idx = path.lastIndexOf("."); 109 | if (idx === -1) { 110 | return false; 111 | } 112 | return extensions.has(path.substring(idx)); 113 | } 114 | 115 | /** Do a pass on indexing a chunk of up to 10 new or changed 116 | * files in the current commit we're working on. 117 | */ 118 | async function indexCurrentFiles( 119 | settings: Doc<"settings">, 120 | ctx: ActionCtx, 121 | syncState: Doc<"sync">, 122 | ): Promise { 123 | // Grab a few things from the settings table. 124 | const extensions: Set = new Set(); 125 | for (const e of settings.extensions!) { 126 | extensions.add(e); 127 | } 128 | const exclusions: Set = new Set(); 129 | for (const e of settings.exclusions ?? []) { 130 | exclusions.add(e); 131 | } 132 | const byteLimit = settings.byteLimit ?? DEFAULT_BYTE_LIMIT; 133 | 134 | // Create our github client instance. 135 | const github = createOctokit(); 136 | 137 | // Let's grab the whole tree state at the current commit. 138 | const response = await github.rest.git.getTree({ 139 | owner: settings.org, 140 | repo: settings.repo, 141 | tree_sha: syncState.commit!, 142 | recursive: "true", 143 | }); 144 | const tree = response.data.tree; 145 | // Shuffle the tree to process random files. 146 | // This is useful to minimize collisions if we have multiple jobs 147 | // that end up running in parallel. 148 | shuffleArray(tree); 149 | 150 | // We'll increment this whenever we actually encounter a new or changed file 151 | // to know how many we've processed with OpenAI etc. 152 | let processedFiles = 0; 153 | for (const treeItem of tree) { 154 | // If the file is code, isn't too big, isn't explicitly excluded by our settings, 155 | // and has newer content than we've indexed (this is what `checkPending` determines)... 156 | // It's time to download and get to work summarizing and indexing this source code 157 | // file. 158 | if ( 159 | isPathCode(extensions, treeItem.path) && 160 | (treeItem.size ?? byteLimit) < byteLimit && 161 | !exclusions.has(treeItem.path!) && 162 | (await ctx.runMutation(internal.files.checkPending, { 163 | path: treeItem.path!, 164 | fileSha: treeItem.sha!, 165 | treeSha: syncState.commit!, 166 | })) 167 | ) { 168 | console.log(`Processing ${treeItem.path}`); 169 | // 1. Get the actual contnents of the source file. 170 | const contents = await fetchRepoFileContents( 171 | settings, 172 | github, 173 | treeItem.sha!, 174 | ); 175 | // 2. Get the goals summary from ChatGPT 176 | const summary = await summarize(settings, treeItem.path!, contents); 177 | console.log(summary); 178 | // Note: If this throws an exception, it's likely b/c GPT did something wonky and the 179 | // JSON wasn't quite formatted correctly. The next pass will almost definitely 180 | // unblock the job when this file is retried. 181 | const summaryObject = JSON.parse(summary); 182 | const goals = summaryObject["goals"] as string[]; 183 | 184 | // 3. Have OpenAI generate embeddings for all the goals 185 | const vectors = await generateEmbeddings(goals); 186 | 187 | // 4. Store the file, goals, and vectors into Convex for search. 188 | await ctx.runMutation(internal.files.index, { 189 | fileSha: treeItem.sha!, 190 | treeSha: syncState.commit!, 191 | path: treeItem.path!, 192 | language: summaryObject["programming_language"] as string, 193 | goals, 194 | vectors, 195 | }); 196 | // We'll do up to 10 files in one pass to keep the runtime of an individual 197 | // action a few minutes or less. 198 | processedFiles += 1; 199 | if (processedFiles === 10) { 200 | return false; 201 | } 202 | } 203 | } 204 | return true; // All files done. 205 | } 206 | 207 | /** Create Octokit instance using our GitHub access token. */ 208 | function createOctokit() { 209 | return new Octokit({ 210 | auth: process.env.GITHUB_ACCESS_TOKEN, 211 | }); 212 | } 213 | 214 | /** Grab the contents of a specific file (keyed by a blob's sha256) from 215 | * the repository on GitHub. 216 | */ 217 | async function fetchRepoFileContents( 218 | settings: Doc<"settings">, 219 | github: Octokit, 220 | sha: string, 221 | ): Promise { 222 | const response = await github.rest.git.getBlob({ 223 | owner: settings.org, 224 | repo: settings.repo, 225 | file_sha: sha, 226 | }); 227 | // The file contents are in base64, since the file can contain 228 | // non-ascii characters. 229 | function decodeBase64(base64: string) { 230 | const text = atob(base64); 231 | const length = text.length; 232 | const bytes = new Uint8Array(length); 233 | for (let i = 0; i < length; i++) { 234 | bytes[i] = text.charCodeAt(i); 235 | } 236 | const decoder = new TextDecoder(); // default is utf-8 237 | return decoder.decode(bytes); 238 | } 239 | const fileContents = decodeBase64(response.data.content); 240 | return fileContents; 241 | } 242 | 243 | /** Run passes of the clearDeadFiles mutation to get rid of files 244 | * no longer in the repository's tree. 245 | */ 246 | async function clearOutdatedFiles(ctx: ActionCtx, commit: string) { 247 | while (await ctx.runMutation(internal.files.clearDeadFiles, { commit })) { 248 | // Keep clearing batches of outdated files until they're all gone. 249 | } 250 | } 251 | 252 | /* eslint-disable @typescript-eslint/no-explicit-any */ 253 | function shuffleArray(array: any[]) { 254 | for (let i = array.length - 1; i > 0; i--) { 255 | const j = Math.floor(Math.random() * (i + 1)); 256 | [array[i], array[j]] = [array[j], array[i]]; 257 | } 258 | } 259 | -------------------------------------------------------------------------------- /convex/schema.ts: -------------------------------------------------------------------------------- 1 | /** Our Convex schema definitions. */ 2 | import { defineSchema, defineTable } from "convex/server"; 3 | import { v } from "convex/values"; 4 | 5 | export default defineSchema({ 6 | // Sync state -- what commit are we work on, or are we in sync with? 7 | sync: defineTable({ 8 | commit: v.union(v.null(), v.string()), 9 | commitDone: v.boolean(), 10 | }), 11 | // All files in the tree. 12 | files: defineTable({ 13 | path: v.string(), 14 | language: v.string(), 15 | fileSha: v.string(), 16 | treeSha: v.string(), 17 | }) 18 | .index("path", ["path"]) 19 | .index("treeSha", ["treeSha"]), 20 | // Goals associated with the files as determined by ChatGPT 21 | fileGoals: defineTable({ 22 | fileId: v.id("files"), 23 | vector: v.array(v.float64()), 24 | goal: v.string(), 25 | }) 26 | .index("fileId", ["fileId"]) 27 | .vectorIndex("by_embedding", { 28 | vectorField: "vector", 29 | dimensions: 1536, 30 | }), 31 | // Various project settings you can tweak in the dashboard as we go. 32 | settings: defineTable({ 33 | org: v.string(), 34 | repo: v.string(), 35 | branch: v.string(), 36 | extensions: v.array(v.string()), 37 | exclusions: v.optional(v.array(v.string())), 38 | byteLimit: v.optional(v.number()), 39 | chatModel: v.optional(v.string()), 40 | }), 41 | // Log of all sync and indexing operations. 42 | log: defineTable({ 43 | cursor: v.number(), 44 | operator: v.union( 45 | v.literal("add"), 46 | v.literal("cleanup"), 47 | v.literal("start"), 48 | v.literal("finish"), 49 | ), 50 | path: v.optional(v.string()), 51 | sha: v.string(), 52 | }).index("cursor", ["cursor"]), 53 | }); 54 | -------------------------------------------------------------------------------- /convex/search.ts: -------------------------------------------------------------------------------- 1 | /** Search API for use by the web app. */ 2 | import { action } from "./_generated/server"; 3 | import { v } from "convex/values"; 4 | import { generateEmbedding } from "./ai"; 5 | import { internal } from "./_generated/api"; 6 | 7 | export type SearchResult = { 8 | path: string; 9 | language: string; 10 | goal: string; 11 | score: number; 12 | treeSha: string; 13 | }; 14 | 15 | /** Conduct a search. 16 | * 17 | * Generate an embedding from `query` and then use the vector index 18 | * on our goals to find source files that are likely to help. 19 | */ 20 | export const search = action({ 21 | args: { 22 | query: v.string(), 23 | }, 24 | handler: async (ctx, args): Promise => { 25 | const embedding = await generateEmbedding(args.query); 26 | // We're going to grab 30 goals even though we're only returning 27 | // up to 10 results. 28 | // Why? 29 | // Well, it's possible (even likely!) for the same file to be matched 30 | // several times for a given query string if the file has similar goals. 31 | // We'll just return the file once with the "top" goal as the rationale. 32 | // Second, and more subtly: for performance reasons, vector search is not 33 | // transactionally consistent with the database's document. So it's possible 34 | // for background indexing to delete a file right after we get the id 35 | // in a search result. So not all files will exist when we go fetch them. 36 | const results = await ctx.vectorSearch("fileGoals", "by_embedding", { 37 | vector: embedding, 38 | limit: 30, 39 | }); 40 | 41 | // De-duplicate the same document. 42 | const seenDocuments = new Set(); 43 | const docs = []; 44 | 45 | for (const r of results) { 46 | // Skip this document, we're already including it in the result. 47 | if (seenDocuments.has(r._id)) { 48 | continue; 49 | } 50 | 51 | // Go get the file info and goal text, etc, for the given goal id. 52 | const searchInfo = await ctx.runQuery(internal.files.getGoalAndFile, { 53 | id: r._id, 54 | }); 55 | 56 | // Timing / race issue between vector search and backing database? 57 | // The document no longer exists. Oh well, we'll just keep moving. 58 | if (searchInfo === null) { 59 | continue; 60 | } 61 | // Add the score to the result coming back from our query. 62 | const fullSearchInfo = Object.assign(searchInfo, { 63 | score: r._score, 64 | }) as SearchResult; 65 | 66 | // Add this document to the result set and bail if we have enough. 67 | seenDocuments.add(r._id); 68 | docs.push(fullSearchInfo); 69 | if (seenDocuments.size === 10) { 70 | break; 71 | } 72 | } 73 | return docs; 74 | }, 75 | }); 76 | -------------------------------------------------------------------------------- /convex/settings.ts: -------------------------------------------------------------------------------- 1 | /** Project settings. Mostly hand-edited in the dashboard. */ 2 | import { Doc } from "./_generated/dataModel"; 3 | import { query } from "./_generated/server"; 4 | 5 | /** Return the project settings to whoever cares (the browser, for example). */ 6 | export const get = query({ 7 | handler: async ({ db }): Promise | null> => { 8 | return await db.query("settings").first(); 9 | }, 10 | }); 11 | -------------------------------------------------------------------------------- /convex/syncState.ts: -------------------------------------------------------------------------------- 1 | /** The sync state model is all about keeping track of this cycle: 2 | * 3 | * 1. No commit yet 4 | * 2. Working on first commit. 5 | * 3. Done with first commit. 6 | * Forever: 7 | * 4. Poll for next commit. 8 | * 5. Work on next commit. 9 | * 6. Done with next commit. 10 | */ 11 | import { Doc } from "./_generated/dataModel"; 12 | import { 13 | MutationCtx, 14 | internalMutation, 15 | internalQuery, 16 | } from "./_generated/server"; 17 | import { v } from "convex/values"; 18 | import { writeLog } from "./log"; 19 | 20 | /** Return the current sync state. */ 21 | export const get = internalQuery({ 22 | args: {}, 23 | handler: async (ctx): Promise> => { 24 | return (await ctx.db.query("sync").first())!; 25 | }, 26 | }); 27 | 28 | /** We've discovered a new commit. Let's start working on it. 29 | * 30 | * This means commit = and commitDone = false. 31 | */ 32 | export const startCommit = internalMutation({ 33 | args: { 34 | commit: v.string(), 35 | }, 36 | handler: async (ctx, args) => { 37 | const id = (await ctx.db.query("sync").first())!._id; 38 | await ctx.db.patch(id, { 39 | commit: args.commit, 40 | commitDone: false, 41 | }); 42 | await writeLog(ctx, "start", args.commit); 43 | }, 44 | }); 45 | 46 | /** Reset the sync state back to an "initial sync" one. */ 47 | export const reset = internalMutation({ 48 | handler: async (ctx) => { 49 | const old = await ctx.db.query("sync").first(); 50 | if (old !== null) { 51 | await ctx.db.delete(old._id); 52 | } 53 | await runInit(ctx); 54 | }, 55 | }); 56 | 57 | async function runInit(ctx: MutationCtx) { 58 | const existingSync = await ctx.db.query("sync").first(); 59 | if (existingSync === null) { 60 | await ctx.db.insert("sync", { 61 | commit: null, 62 | commitDone: true, 63 | }); 64 | } 65 | const settings = await ctx.db.query("settings").first(); 66 | if (settings === null) { 67 | await ctx.db.insert("settings", { 68 | org: "get-convex", 69 | repo: "convex-helpers", 70 | branch: "main", 71 | extensions: [".js", ".html", ".jsx", ".ts", ".tsx", ".css"], 72 | }); 73 | } 74 | } 75 | 76 | /** Initialize the sync state and settings if they're missing */ 77 | export const init = internalMutation({ 78 | handler: runInit, 79 | }); 80 | -------------------------------------------------------------------------------- /convex/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | /* This TypeScript project config describes the environment that 3 | * Convex functions run in and is used to typecheck them. 4 | * You can modify it, but some settings required to use Convex. 5 | */ 6 | "compilerOptions": { 7 | /* These settings are not required by Convex and can be modified. */ 8 | "allowJs": true, 9 | "strict": true, 10 | 11 | /* These compiler options are required by Convex */ 12 | "target": "ESNext", 13 | "lib": ["ES2021", "dom"], 14 | "forceConsistentCasingInFileNames": true, 15 | "allowSyntheticDefaultImports": true, 16 | "module": "ESNext", 17 | "moduleResolution": "Node", 18 | "isolatedModules": true, 19 | "noEmit": true 20 | }, 21 | "include": ["./**/*"], 22 | "exclude": ["./_generated"] 23 | } 24 | -------------------------------------------------------------------------------- /dryad_settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/get-convex/dryad/230f39530642ecc224bf9bb72d032a5fa54a9d20/dryad_settings.png -------------------------------------------------------------------------------- /dryad_ss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/get-convex/dryad/230f39530642ecc224bf9bb72d032a5fa54a9d20/dryad_ss.png -------------------------------------------------------------------------------- /env_ss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/get-convex/dryad/230f39530642ecc224bf9bb72d032a5fa54a9d20/env_ss.png -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 16 | 17 | Dryad | Talk to your (code) tree 18 | 19 | 20 |
21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dryad", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "npm-run-all dev:init --parallel dev:frontend dev:backend", 8 | "build": "tsc && vite build", 9 | "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", 10 | "preview": "vite preview", 11 | "format": "npx prettier -w .", 12 | "dev:backend": "convex dev", 13 | "dev:frontend": "vite", 14 | "dev:init": "convex dev --run syncState:init --until-success" 15 | }, 16 | "dependencies": { 17 | "@headlessui/react": "^1.7.17", 18 | "@heroicons/react": "^2.0.18", 19 | "@octokit/rest": "^20.0.1", 20 | "@tailwindcss/forms": "^0.5.6", 21 | "convex": "=1.2.1-alpha.1", 22 | "openai": "^4.3.1", 23 | "react": "^18.2.0", 24 | "react-dom": "^18.2.0", 25 | "react-syntax-highlighter": "^15.5.0", 26 | "rooks": "^7.14.1", 27 | "spinners-react": "^1.0.7" 28 | }, 29 | "devDependencies": { 30 | "@types/react": "^18.2.15", 31 | "@types/react-dom": "^18.2.7", 32 | "@types/react-syntax-highlighter": "^15.5.7", 33 | "@typescript-eslint/eslint-plugin": "^6.0.0", 34 | "@typescript-eslint/parser": "^6.0.0", 35 | "@vitejs/plugin-react": "^4.0.3", 36 | "autoprefixer": "^10.4.15", 37 | "eslint": "^8.45.0", 38 | "eslint-plugin-react-hooks": "^4.6.0", 39 | "eslint-plugin-react-refresh": "^0.4.3", 40 | "npm-run-all": "^4.1.5", 41 | "postcss": "^8.4.29", 42 | "prettier": "^3.0.3", 43 | "tailwindcss": "^3.3.3", 44 | "typescript": "^5.0.2", 45 | "vite": "^4.4.5" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /public/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/get-convex/dryad/230f39530642ecc224bf9bb72d032a5fa54a9d20/public/favicon.png -------------------------------------------------------------------------------- /src/App.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/get-convex/dryad/230f39530642ecc224bf9bb72d032a5fa54a9d20/src/App.css -------------------------------------------------------------------------------- /src/App.tsx: -------------------------------------------------------------------------------- 1 | import dryadLogo from "./assets/dryad_logo.png"; 2 | import githubLogo from "./assets/Github_white.svg"; 3 | import convexLogo from "./assets/convex_logo.svg"; 4 | import "./App.css"; 5 | import SearchBox from "./components/SearchBox"; 6 | import { useRef, useState } from "react"; 7 | import { SearchResult } from "../convex/search"; 8 | import SearchResults from "./components/SearchResults"; 9 | import CodeDisplay from "./components/CodeDisplay"; 10 | import { useQuery } from "convex/react"; 11 | import { api } from "../convex/_generated/api"; 12 | import Info from "./components/Info"; 13 | import EventLog from "./components/EventLog"; 14 | 15 | function App() { 16 | const settings = useQuery(api.settings.get); 17 | const [results, setResults] = useState([]); 18 | const [loadedResult, setLoadedResult] = useState(null); 19 | const [infoOpen, setInfoOpen] = useState(false); 20 | const codeDisplayRef = useRef(null); 21 | return ( 22 |
23 |
24 |
25 |
26 | Dryad logo 27 |
28 |
29 |
dryad
30 |
31 | talk to your tree 32 |
33 |
34 |
35 |
36 | Indexing events{" "} 37 | live 38 |
39 |
40 | 41 |
42 |
43 |
44 |
45 |
46 | { 49 | setResults([]); 50 | setLoadedResult(null); 51 | }} 52 | /> 53 | { 56 | if (codeDisplayRef.current !== null) { 57 | (codeDisplayRef.current as HTMLElement).scrollIntoView(true); 58 | } 59 | setLoadedResult(result); 60 | }} 61 | /> 62 |
63 |
67 | {loadedResult ? ( 68 | 69 | ) : results.length === 0 ? ( 70 |
Search for something!
71 | ) : ( 72 |
Pick a file to dive into the code
73 | )} 74 |
75 |
76 | 77 |
78 | 111 |
112 | ); 113 | } 114 | 115 | export default App; 116 | -------------------------------------------------------------------------------- /src/assets/3-dots-bounce.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/assets/90-ring-with-bg.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | 12 | Unicorn! · GitHub 13 | 56 | 57 | 58 | 59 |
60 |

61 | 63 |

64 | 65 |

We had issues producing the response to your request.

66 |

Sorry about that. Please try refreshing and contact us if the problem persists.

67 |
68 | Contact Support — 69 | GitHub Status — 70 | @githubstatus 71 |
72 | 73 | 76 | 77 | 80 |
81 | 82 | 83 | -------------------------------------------------------------------------------- /src/assets/Github_white.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /src/assets/convex_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 8 | 9 | 10 | 11 | 12 | 16 | 21 | 24 | 25 | 30 | 32 | 34 | 36 | 37 | -------------------------------------------------------------------------------- /src/assets/dryad_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/get-convex/dryad/230f39530642ecc224bf9bb72d032a5fa54a9d20/src/assets/dryad_logo.png -------------------------------------------------------------------------------- /src/assets/question.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/components/CodeDisplay.tsx: -------------------------------------------------------------------------------- 1 | import { FC, useEffect, useState } from "react"; 2 | import SyntaxHighlighter from "react-syntax-highlighter"; 3 | import { dracula } from "react-syntax-highlighter/dist/esm/styles/hljs"; 4 | import { SearchResult } from "../../convex/search"; 5 | import { useQuery } from "convex/react"; 6 | import { api } from "../../convex/_generated/api"; 7 | import spinnerImage from "../assets/3-dots-bounce.svg"; 8 | import { ArrowTopRightOnSquareIcon } from "@heroicons/react/20/solid"; 9 | 10 | export type CodeDisplayProps = { 11 | result: SearchResult; 12 | }; 13 | 14 | const CodeDisplay: FC = ({ result }) => { 15 | const settings = useQuery(api.settings.get); 16 | const [code, setCode] = useState(null as null | string); 17 | 18 | useEffect(() => { 19 | setCode(null); 20 | if (!settings) { 21 | // Wait for settings to load. 22 | return; 23 | } 24 | const getCode = async () => { 25 | const url = `https://raw.githubusercontent.com/${settings.org}/${settings.repo}/${result.treeSha}/${result.path}`; 26 | const body = await (await fetch(url)).text(); 27 | setCode(body); 28 | }; 29 | getCode(); 30 | }, [settings, result]); 31 | 32 | if (code === null) { 33 | return ( 34 |
35 | Loading... 36 |
37 | ); 38 | } else { 39 | return ( 40 | <> 41 |
42 | 45 | {result.path} 46 |
47 | 59 | 66 | {code} 67 | 68 | 69 | ); 70 | } 71 | }; 72 | 73 | export default CodeDisplay; 74 | -------------------------------------------------------------------------------- /src/components/EventLog.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | ArrowPathIcon, 3 | DocumentMinusIcon, 4 | DocumentPlusIcon, 5 | HandThumbUpIcon, 6 | } from "@heroicons/react/20/solid"; 7 | import { FC } from "react"; 8 | import { useQuery } from "convex/react"; 9 | import { api } from "../../convex/_generated/api"; 10 | import { Doc } from "../../convex/_generated/dataModel"; 11 | 12 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 13 | function classNames(...classes: any[]) { 14 | return classes.filter(Boolean).join(" "); 15 | } 16 | 17 | function entryToVisual(entry: Doc<"log">) { 18 | const shortSha = `${entry.sha.substring(0, 8)}...`; 19 | if (entry.operator === "start") { 20 | const content = ( 21 | 22 | Starting sync of repoistory commit {shortSha} 23 | 24 | ); 25 | return { iconClass: "bg-orange-400", icon: ArrowPathIcon, content }; 26 | } else if (entry.operator === "add") { 27 | const content = ( 28 | 29 | Indexing new/changed file {entry.path} with SHA{" "} 30 | {shortSha} 31 | 32 | ); 33 | return { iconClass: "bg-green-400", icon: DocumentPlusIcon, content }; 34 | } else if (entry.operator === "cleanup") { 35 | const content = ( 36 | 37 | Removing missing file {entry.path} with SHA{" "} 38 | {shortSha} 39 | 40 | ); 41 | return { iconClass: "bg-red-400", icon: DocumentMinusIcon, content }; 42 | } else if (entry.operator === "finish") { 43 | const content = ( 44 | 45 | Successfully indexed all content at repository commit{" "} 46 | {shortSha} 47 | 48 | ); 49 | return { iconClass: "bg-blue-400", icon: HandThumbUpIcon, content }; 50 | } 51 | throw "no such value"; 52 | } 53 | 54 | const EventLog: FC = () => { 55 | const entries = useQuery(api.log.get) ?? []; 56 | return ( 57 |
58 |
    59 | {entries.map((entry, idx) => { 60 | const ui = entryToVisual(entry); 61 | return ( 62 |
  • 63 |
    64 | {idx !== entries.length - 1 ? ( 65 |
    104 |
    • 105 | ); 106 | })} 107 |
108 |
109 | ); 110 | }; 111 | export default EventLog; 112 | -------------------------------------------------------------------------------- /src/components/Info.tsx: -------------------------------------------------------------------------------- 1 | import { FC, Fragment } from "react"; 2 | import { Dialog, Transition } from "@headlessui/react"; 3 | import dyradLogo from "../assets/dryad_logo.png"; 4 | 5 | export type InfoProps = { 6 | open: boolean; 7 | setOpen: (makeOpen: boolean) => void; 8 | }; 9 | 10 | const Info: FC = ({ open, setOpen }) => { 11 | return ( 12 | 13 | 14 | 23 |
24 | 25 | 26 |
27 |
28 | 37 | 38 |
39 |
40 | 45 |
46 |
47 | 51 | Dryad: Semantic Code Search Demo 52 | 53 |
54 |

55 | Dryad is a demo app and template for building semantic 56 | search projects using generative AI, embeddings, and 57 | Convex. 58 |

59 |

60 | Dryad polls a configured GitHub repository for new 61 | commits. On each new commit, any changed files are 62 | downloaded and indexed to maintain a "snapshot" of the 63 | semantics of all the repository's code. 64 |

65 |

66 | The indexing is comprised of three steps: First, ChatGPT 67 | is given the source file and asked to generate a list of 68 | the top responsibilities / goals for that particular 69 | file. Next, each of these goals is passed into OpenAI's 70 | embeddings API to generate a vector. Finally, these 71 | vectors are indexed using Convex's vector indexing. 72 | Later, the same embedding API is used at search time to 73 | create a vector from the query string. Vector search 74 | uses cosine similarity to find all pages that have a 75 | semantically similar goal to the given query string. 76 |

77 |

78 | Dryad is MIT-licensed, open source, and only ~1,000 79 | lines of code. So check out the GitHub repo, run your 80 | own instance to index any repository you like, and 81 | develop improvements to the project. 82 |

83 |
84 |
85 |
86 |
87 | 94 |
95 | 96 | 97 |
98 |
99 |
100 | 101 | ); 102 | }; 103 | 104 | export default Info; 105 | -------------------------------------------------------------------------------- /src/components/SearchBox.tsx: -------------------------------------------------------------------------------- 1 | import { useAction, useQuery } from "convex/react"; 2 | import { api } from "../../convex/_generated/api"; 3 | import { useRef, FC } from "react"; 4 | import { useKeys } from "rooks"; 5 | import { SearchResult } from "../../convex/search"; 6 | 7 | export type SearchProps = { 8 | update: (results: SearchResult[]) => void; 9 | newSearch: () => void; 10 | }; 11 | 12 | const SearchBox: FC = ({ update, newSearch }) => { 13 | const inputRef = useRef(null); 14 | const containerRef = useRef(document); 15 | const search = useAction(api.search.search); 16 | const settings = useQuery(api.settings.get); 17 | 18 | useKeys( 19 | ["Meta", "k"], 20 | (e) => { 21 | if (inputRef.current) { 22 | const input = inputRef.current as HTMLInputElement; 23 | input.focus(); 24 | } 25 | e.preventDefault(); 26 | }, 27 | { 28 | target: containerRef, 29 | preventLostKeyup: true, 30 | }, 31 | ); 32 | useKeys( 33 | ["Enter"], 34 | (e) => { 35 | console.log("submit!"); 36 | if (inputRef.current) { 37 | const input = inputRef.current as HTMLInputElement; 38 | const query = input.value; 39 | const executeAction = async () => { 40 | const results = await search({ query }); 41 | update(results); 42 | }; 43 | newSearch(); 44 | executeAction(); 45 | } 46 | e.preventDefault(); 47 | }, 48 | { 49 | target: inputRef, 50 | preventLostKeyup: true, 51 | }, 52 | ); 53 | return ( 54 |
55 | 73 |
74 | 83 |
84 | 85 | ⌘K 86 | 87 |
88 |
89 |
90 | ); 91 | }; 92 | 93 | export default SearchBox; 94 | -------------------------------------------------------------------------------- /src/components/SearchResults.tsx: -------------------------------------------------------------------------------- 1 | import { FC } from "react"; 2 | import { SearchResult } from "../../convex/search"; 3 | 4 | export type SearchResultsProps = { 5 | results: SearchResult[]; 6 | loadResult: (result: SearchResult) => void; 7 | }; 8 | 9 | const SearchResults: FC = ({ results, loadResult }) => { 10 | return ( 11 |
    12 | {results.map((r, i) => { 13 | return ( 14 |
  • { 18 | loadResult(r); 19 | e.preventDefault(); 20 | }} 21 | > 22 |
    23 |
    24 | #{i + 1} ( 25 | 26 | {(r.score * 100.0).toPrecision(3)}% match) 27 | 28 |
    29 |
    30 | 33 | 34 | {r.path} 35 | 36 |
    37 |
    {r.goal}
    38 |
    39 |
    • 40 | ); 41 | })} 42 |
43 | ); 44 | }; 45 | 46 | export default SearchResults; 47 | -------------------------------------------------------------------------------- /src/index.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | body { 6 | background: rgb(28, 53, 58); 7 | } 8 | 9 | .title_font { 10 | font-family: "Raleway", sans-serif; 11 | } 12 | 13 | .texture-footer { 14 | background-color: #52280a; 15 | background-image: repeating-linear-gradient( 16 | 120deg, 17 | rgba(255, 255, 255, 0.1), 18 | rgba(255, 255, 255, 0.1) 1px, 19 | transparent 1px, 20 | transparent 60px 21 | ), 22 | repeating-linear-gradient( 23 | 60deg, 24 | rgba(255, 255, 255, 0.1), 25 | rgba(255, 255, 255, 0.1) 1px, 26 | transparent 1px, 27 | transparent 60px 28 | ), 29 | linear-gradient( 30 | 60deg, 31 | rgba(0, 0, 0, 0.1) 25%, 32 | transparent 25%, 33 | transparent 75%, 34 | rgba(0, 0, 0, 0.1) 75%, 35 | rgba(0, 0, 0, 0.1) 36 | ), 37 | linear-gradient( 38 | 120deg, 39 | rgba(0, 0, 0, 0.1) 25%, 40 | transparent 25%, 41 | transparent 75%, 42 | rgba(0, 0, 0, 0.1) 75%, 43 | rgba(0, 0, 0, 0.1) 44 | ); 45 | background-size: 70px 120px; 46 | } 47 | -------------------------------------------------------------------------------- /src/main.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom/client"; 3 | import App from "./App.tsx"; 4 | import "./index.css"; 5 | import { ConvexProvider, ConvexReactClient } from "convex/react"; 6 | 7 | const convex = new ConvexReactClient(import.meta.env.VITE_CONVEX_URL as string); 8 | 9 | ReactDOM.createRoot(document.getElementById("root")!).render( 10 | 11 | 12 | 13 | 14 | , 15 | ); 16 | -------------------------------------------------------------------------------- /src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | export default { 3 | content: ["./index.html", "./src/**/*.{js,ts,jsx,tsx}"], 4 | theme: { 5 | extend: { 6 | colors: { 7 | dryad: "#1C353A", 8 | }, 9 | }, 10 | }, 11 | plugins: [ 12 | // eslint-disable-next-line no-undef 13 | require("@tailwindcss/forms"), 14 | ], 15 | }; 16 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "useDefineForClassFields": true, 5 | "lib": ["ES2020", "DOM", "DOM.Iterable"], 6 | "module": "ESNext", 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "resolveJsonModule": true, 13 | "isolatedModules": true, 14 | "noEmit": true, 15 | "jsx": "react-jsx", 16 | 17 | /* Linting */ 18 | "strict": true, 19 | "noUnusedLocals": true, 20 | "noUnusedParameters": true, 21 | "noFallthroughCasesInSwitch": true 22 | }, 23 | "include": ["src"], 24 | "references": [{ "path": "./tsconfig.node.json" }] 25 | } 26 | -------------------------------------------------------------------------------- /tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "skipLibCheck": true, 5 | "module": "ESNext", 6 | "moduleResolution": "bundler", 7 | "allowSyntheticDefaultImports": true 8 | }, 9 | "include": ["vite.config.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vite"; 2 | import react from "@vitejs/plugin-react"; 3 | 4 | // https://vitejs.dev/config/ 5 | export default defineConfig({ 6 | plugins: [react()], 7 | }); 8 | --------------------------------------------------------------------------------