├── .github └── workflows │ └── node.js.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── commonjs.json ├── dashboard_screenshot.png ├── eslint.config.js ├── esm.json ├── example ├── .gitignore ├── README.md ├── convex │ ├── _generated │ │ ├── api.d.ts │ │ ├── api.js │ │ ├── dataModel.d.ts │ │ ├── server.d.ts │ │ └── server.js │ ├── convex.config.ts │ ├── example.ts │ ├── schema.ts │ └── tsconfig.json ├── eslint.config.js ├── index.html ├── package-lock.json ├── package.json ├── src │ ├── App.css │ ├── App.tsx │ ├── index.css │ ├── main.tsx │ └── vite-env.d.ts ├── tsconfig.json └── vite.config.ts ├── package-lock.json ├── package.json ├── src ├── client │ ├── index.test.ts │ └── index.ts ├── component │ ├── _generated │ │ ├── api.d.ts │ │ ├── api.js │ │ ├── dataModel.d.ts │ │ ├── server.d.ts │ │ └── server.js │ ├── convex.config.ts │ ├── lib.test.ts │ ├── lib.ts │ ├── schema.ts │ └── setup.test.ts └── shared.ts ├── tsconfig.json ├── tsconfig.test.json └── vitest.config.ts /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | on: 3 | push: 4 | branches: ["main"] 5 | pull_request: 6 | branches: ["main"] 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - name: Use Node.js 13 | uses: actions/setup-node@v4 14 | with: 15 | node-version: "18.x" 16 | cache: "npm" 17 | - run: npm i 18 | - run: npm ci 19 | - run: cd example && npm i && cd .. 20 | - run: npm test 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | *.local 4 | *.log 5 | /.vscode/ 6 | /docs/.vitepress/cache 7 | dist 8 | dist-ssr 9 | explorations 10 | node_modules 11 | .eslintcache 12 | 13 | # this is a package-json-redirect stub dir, see https://github.com/andrewbranch/example-subpath-exports-ts-compat?tab=readme-ov-file 14 | frontend/package.json 15 | # npm pack output 16 | *.tgz 17 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Developing guide 2 | 3 | ## Running locally 4 | 5 | Setup: 6 | 7 | ```sh 8 | npm i 9 | cd example 10 | npm i 11 | npm run dev:convex -- --once 12 | ``` 13 | 14 | Run the frontend and backend: 15 | 16 | ```sh 17 | npm run dev 18 | ``` 19 | 20 | ## Testing 21 | 22 | ```sh 23 | rm -rf dist/ && npm run build 24 | npm run typecheck 25 | cd example 26 | npm run lint 27 | cd .. 28 | ``` 29 | 30 | ## Deploying 31 | 32 | ### Building a one-off package 33 | 34 | ```sh 35 | rm -rf dist/ && npm run build 36 | npm pack 37 | ``` 38 | 39 | ### Deploying a new version 40 | 41 | ```sh 42 | # this will change the version and commit it (if you run it in the root directory) 43 | npm version patch 44 | npm publish --dry-run 45 | # sanity check files being included 46 | npm publish 47 | git push --tags 48 | git push 49 | ``` 50 | 51 | #### Alpha release 52 | 53 | The same as above, but it requires extra flags so the release is only installed with `@alpha`: 54 | 55 | ```sh 56 | npm version prerelease --preid alpha 57 | npm publish --tag alpha 58 | git push --tags 59 | ``` 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Convex Stateful Migrations Component 2 | 3 | [![npm version](https://badge.fury.io/js/@convex-dev%2Fmigrations.svg)](https://badge.fury.io/js/@convex-dev%2Fmigrations) 4 | 5 | 6 | 7 | Define and run migrations, like this one setting a default value for users: 8 | 9 | ```ts 10 | export const setDefaultValue = migrations.define({ 11 | table: "users", 12 | migrateOne: async (ctx, user) => { 13 | if (user.optionalField === undefined) { 14 | await ctx.db.patch(user._id, { optionalField: "default" }); 15 | } 16 | }, 17 | }); 18 | ``` 19 | 20 | You can then run it programmatically or from the CLI. See [below](#running-migrations-one-at-a-time). 21 | 22 | Migrations allow you to define functions that run on all documents in a table 23 | (or a specified subset). They run in batches asynchronously (online migration). 24 | 25 | The component tracks the migrations state so it can avoid running twice, 26 | pick up where it left off (in the case of a bug or failure along the way), 27 | and expose the migration state in realtime via Convex queries. 28 | 29 | See the [migration primer post](https://stack.convex.dev/intro-to-migrations) 30 | for a conceptual overview of online vs. offline migrations. 31 | If your migration is trivial and you're moving fast, also check out 32 | [lightweight migrations in the dashboard](https://stack.convex.dev/lightweight-zero-downtime-migrations). 33 | 34 | Typical steps for doing a migration: 35 | 36 | 1. Modify your schema to allow old and new values. Typically this is adding a 37 | new optional field or marking a field as optional so it can be deleted. 38 | As part of this, update your code to handle both versions. 39 | 2. Define a migration to change the data to the new schema. 40 | 3. Push the migration and schema changes. 41 | 4. Run the migration(s) to completion. 42 | 5. Modify your schema and code to assume the new value. 43 | Pushing this change will only succeed once all the data matches the new schema. 44 | This is the default behavior for Convex, unless you disable schema validation. 45 | 46 | See [this Stack post](https://stack.convex.dev/migrating-data-with-mutations) 47 | for walkthroughs of common use cases. 48 | 49 | ## Pre-requisite: Convex 50 | 51 | You'll need an existing Convex project to use the component. 52 | Convex is a hosted backend platform, including a database, serverless functions, 53 | and a ton more you can learn about [here](https://docs.convex.dev/get-started). 54 | 55 | Run `npm create convex` or follow any of the [quickstarts](https://docs.convex.dev/home) to set one up. 56 | 57 | ## Installation 58 | 59 | Install the component package: 60 | 61 | ```ts 62 | npm install @convex-dev/migrations 63 | ``` 64 | 65 | Create a `convex.config.ts` file in your app's `convex/` folder and install the component by calling `use`: 66 | 67 | ```ts 68 | // convex/convex.config.ts 69 | import { defineApp } from "convex/server"; 70 | import migrations from "@convex-dev/migrations/convex.config"; 71 | 72 | const app = defineApp(); 73 | app.use(migrations); 74 | 75 | export default app; 76 | ``` 77 | 78 | ## Initialization 79 | 80 | Examples below are assuming the code is in `convex/migrations.ts`. 81 | This is not a requirement. 82 | If you want to use a different file, make sure to change the examples below from 83 | `internal.migrations.*` to your new file name, like `internal.myFolder.myMigrationsFile.*` 84 | or CLI arguments like `migrations:*` to `myFolder/myMigrationsFile:*`. 85 | 86 | ```ts 87 | import { Migrations } from "@convex-dev/migrations"; 88 | import { components } from "./_generated/api.js"; 89 | import { DataModel } from "./_generated/dataModel.js"; 90 | 91 | export const migrations = new Migrations(components.migrations); 92 | export const run = migrations.runner(); 93 | ``` 94 | 95 | The type parameter `DataModel` is optional. It provides type safety for migration definitions. 96 | As always, database operations in migrations will abide by your schema definition at runtime. 97 | **Note**: if you use [custom functions](https://stack.convex.dev/custom-functions) 98 | to override `internalMutation`, see [below](#override-the-internalmutation-to-apply-custom-db-behavior). 99 | 100 | ## Define migrations 101 | 102 | Within the `migrateOne` function, you can write code to modify a single document 103 | in the specified table. Making changes is optional, and you can also read and 104 | write to other tables from this function. 105 | 106 | ```ts 107 | export const setDefaultValue = migrations.define({ 108 | table: "myTable", 109 | migrateOne: async (ctx, doc) => { 110 | if (doc.optionalField === undefined) { 111 | await ctx.db.patch(doc._id, { optionalField: "default" }); 112 | } 113 | }, 114 | }); 115 | ``` 116 | 117 | ### Shorthand syntax 118 | 119 | Since the most common migration involves patching each document, 120 | if you return an object, it will be applied as a patch automatically. 121 | 122 | ```ts 123 | export const clearField = migrations.define({ 124 | table: "myTable", 125 | migrateOne: () => ({ optionalField: undefined }), 126 | }); 127 | // is equivalent to `await ctx.db.patch(doc._id, { optionalField: undefined })` 128 | ``` 129 | 130 | ### Migrating a subset of a table using an index 131 | 132 | If you only want to migrate a range of documents, you can avoid processing the 133 | whole table by specifying a `customRange`. You can use any existing index you 134 | have on the table, or the built-in `by_creation_time` index. 135 | 136 | ```ts 137 | export const validateRequiredField = migrations.define({ 138 | table: "myTable", 139 | customRange: (query) => 140 | query.withIndex("by_requiredField", (q) => q.eq("requiredField", "")), 141 | migrateOne: async (_ctx, doc) => { 142 | console.log("Needs fixup: " + doc._id); 143 | // Shorthand for patching 144 | return { requiredField: "" }; 145 | }, 146 | }); 147 | ``` 148 | 149 | ## Running migrations one at a time 150 | 151 | ### Using the Dashboard or CLI 152 | 153 | To define a one-off function to run a single migration, pass a reference to it: 154 | 155 | ```ts 156 | export const runIt = migrations.runner(internal.migrations.setDefaultValue); 157 | ``` 158 | 159 | To run it from the CLI: 160 | 161 | ```sh 162 | npx convex run convex/migrations.ts:runIt # or shorthand: migrations:runIt 163 | ``` 164 | 165 | **Note**: pass the `--prod` argument to run this and below commands in production 166 | 167 | Running it from the dashboard: 168 | 169 | ![Dashboard screenshot](./dashboard_screenshot.png) 170 | 171 | You can also expose a general-purpose function to run your migrations. 172 | For example, in `convex/migrations.ts` add: 173 | 174 | ```ts 175 | export const run = migrations.runner(); 176 | ``` 177 | 178 | Then run it with the [function name](https://docs.convex.dev/functions/query-functions#query-names): 179 | 180 | ```sh 181 | npx convex run migrations:run '{fn: "migrations:setDefaultValue"}' 182 | ``` 183 | 184 | See [below](#shorthand-running-syntax) for a way to just pass `setDefaultValue`. 185 | 186 | ### Programmatically 187 | 188 | You can also run migrations from other Convex mutations or actions: 189 | 190 | ```ts 191 | await migrations.runOne(ctx, internal.example.setDefaultValue); 192 | ``` 193 | 194 | ### Behavior 195 | 196 | - If it is already running it will refuse to start another duplicate worker. 197 | - If it had previously failed on some batch, it will continue from that batch 198 | unless you manually specify `cursor`. 199 | - If you provide an explicit `cursor` (`null` means to start at the beginning), 200 | it will start from there. 201 | - If you pass `true` for `dryRun` then it will run one batch and then throw, 202 | so no changes are committed, and you can see what it would have done. 203 | See [below](#test-a-migration-with-dryrun) 204 | This is good for validating it does what you expect. 205 | 206 | ## Running migrations serially 207 | 208 | You can run a series of migrations in order. This is useful if some migrations 209 | depend on previous ones, or if you keep a running list of all migrations that 210 | should run on the next deployment. 211 | 212 | ### Using the Dashboard or CLI 213 | 214 | You can also pass a list of migrations to `runner` to have it run a series of 215 | migrations instead of just one: 216 | 217 | ```ts 218 | export const runAll = migrations.runner([ 219 | internal.migrations.setDefaultValue, 220 | internal.migrations.validateRequiredField, 221 | internal.migrations.convertUnionField, 222 | ]); 223 | ``` 224 | 225 | Then just run: 226 | 227 | ```sh 228 | npx convex run migrations:runAll # migrations:runAll is equivalent to convex/migrations.ts:runAll on the CLI 229 | ``` 230 | 231 | With the `runner` functions, you can pass a "next" argument to run 232 | a series of migrations after the first: 233 | 234 | ```sh 235 | npx convex run migrations:runIt '{next:["migrations:clearField"]}' 236 | # OR 237 | npx convex run migrations:run '{fn: "migrations:setDefaultValue", next:["migrations:clearField"]}' 238 | ``` 239 | 240 | ### Programmatically 241 | 242 | ```ts 243 | await migrations.runSerially(ctx, [ 244 | internal.migrations.setDefaultValue, 245 | internal.migrations.validateRequiredField, 246 | internal.migrations.convertUnionField, 247 | ]); 248 | ``` 249 | 250 | ### Behavior 251 | 252 | - If a migration is already in progress when attempted, it will no-op. 253 | - If a migration had already completed, it will skip it. 254 | - If a migration had partial progress, it will resume from where it left off. 255 | - If a migration fails or is canceled, it will not continue on, 256 | in case you had some dependencies between the migrations. 257 | Call the series again to retry. 258 | 259 | Note: if you start multiple serial migrations, the behavior is: 260 | 261 | - If they don't overlap on functions, they will happily run in parallel. 262 | - If they have a function in common and one completes before the other 263 | attempts it, the second will just skip it. 264 | - If they have a function in common and one is in progress, the second will 265 | no-op and not run any further migrations in its series. 266 | 267 | ## Operations 268 | 269 | ### Test a migration with dryRun 270 | 271 | Before running a migration that may irreversibly change data, you can validate 272 | a batch by passing `dryRun` to any `runner` or `runOne` command: 273 | 274 | ```sh 275 | npx convex run migrations:runIt '{dryRun: true}' 276 | ``` 277 | 278 | ### Restart a migration 279 | 280 | Pass `null` for the `cursor` to force a migration to start over. 281 | 282 | ```sh 283 | npx convex run migrations:runIt '{cursor: null}' 284 | ``` 285 | 286 | You can also pass in any valid cursor to start from. You can find valid cursors 287 | in the response of calls to `getStatus`. This can allow retrying a migration 288 | from a known good point as you iterate on the code. 289 | 290 | ### Stop a migration 291 | 292 | You can stop a migration from the CLI or dashboard, calling the component API directly: 293 | 294 | ```sh 295 | npx convex run --component migrations lib:cancel '{name: "migrations:myMigration"}' 296 | ``` 297 | 298 | Or via `migrations.cancel` programatically. 299 | 300 | ```ts 301 | await migrations.cancel(ctx, internal.migrations.myMigration); 302 | ``` 303 | 304 | ### Get the status of migrations 305 | 306 | To see the live status of migrations as they progress, you can query it via the CLI: 307 | 308 | ```sh 309 | npx convex run --component migrations lib:getStatus --watch 310 | ``` 311 | 312 | The `--watch` will live-update the status as it changes. 313 | Or programmatically: 314 | 315 | ```ts 316 | const status: MigrationStatus[] = await migrations.getStatus(ctx, { 317 | limit: 10, 318 | }); 319 | // or 320 | const status: MigrationStatus[] = await migrations.getStatus(ctx, { 321 | migrations: [ 322 | internal.migrations.setDefaultValue, 323 | internal.migrations.validateRequiredField, 324 | internal.migrations.convertUnionField, 325 | ], 326 | }); 327 | ``` 328 | 329 | The type is annotated to avoid circular type dependencies, for instance if you 330 | are returning the result from a query that is defined in the same file as the 331 | referenced migrations. 332 | 333 | ### Running migrations as part of a production deploy 334 | 335 | As part of your build and deploy command, you can chain the corresponding 336 | `npx convex run` command, such as: 337 | 338 | ```sh 339 | npx convex deploy --cmd 'npm run build' && npx convex run convex/migrations.ts:runAll --prod 340 | ``` 341 | 342 | ## Configuration options 343 | 344 | ### Override the internalMutation to apply custom DB behavior 345 | 346 | You can customize which `internalMutation` implementation the underly migration should use. 347 | 348 | This might be important if you use [custom functions](https://stack.convex.dev/custom-functions) 349 | to intercept database writes to apply validation or 350 | [trigger operations on changes](https://stack.convex.dev/triggers). 351 | 352 | Assuming you define your own `internalMutation` in `convex/functions.ts`: 353 | 354 | ```ts 355 | import { internalMutation } from "./functions"; 356 | import { Migrations } from "@convex-dev/migrations"; 357 | import { components } from "./_generated/api"; 358 | 359 | export const migrations = new Migrations(components.migrations, { 360 | internalMutation, 361 | }); 362 | ``` 363 | 364 | See [this article](https://stack.convex.dev/migrating-data-with-mutations) 365 | for more information on usage and advanced patterns. 366 | 367 | ### Custom batch size 368 | 369 | The component will fetch your data in batches of 100, and call your function on 370 | each document in a batch. 371 | If you want to change the batch size, you can specify it. 372 | This can be useful if your documents are large, to avoid running over the 373 | [transaction limit](https://docs.convex.dev/production/state/limits#transactions), 374 | or if your documents are updating frequently and you are seeing OCC conflicts 375 | while migrating. 376 | 377 | ```ts 378 | export const clearField = migrations.define({ 379 | table: "myTable", 380 | batchSize: 10, 381 | migrateOne: () => ({ optionalField: undefined }), 382 | }); 383 | ``` 384 | 385 | You can also override this batch size for an individual invocation: 386 | 387 | ```ts 388 | await migrations.runOne(ctx, internal.migrations.clearField, { 389 | batchSize: 1, 390 | }); 391 | ``` 392 | 393 | ### Parallelizing batches 394 | 395 | Each batch is processed serially, but within a batch you can have each 396 | `migrateOne` call run in parallel if you pass `parallelize: true`. 397 | If you do so, ensure your callback doesn't assume that each call is isolated. 398 | For instance, if each call reads then updates the same counter, then multiple 399 | functions in the same batch could read the same counter value, and get off by 400 | one. 401 | As a result, migrations are run serially by default. 402 | 403 | ```ts 404 | export const clearField = migrations.define({ 405 | table: "myTable", 406 | parallelize: true, 407 | migrateOne: () => ({ optionalField: undefined }), 408 | }); 409 | ``` 410 | 411 | ### Shorthand running syntax: 412 | 413 | For those that don't want to type out `migrations:myNewMigration` every time 414 | they run a migration from the CLI or dashboard, especially if you define your migrations 415 | elsewhere like `ops/db/migrations:myNewMigration`, you can configure a prefix: 416 | 417 | ```ts 418 | export const migrations = new Migrations(components.migrations, { 419 | internalMigration, 420 | migrationsLocationPrefix: "migrations:", 421 | }); 422 | ``` 423 | 424 | And then just call: 425 | 426 | ```sh 427 | npx convex run migrations:run '{fn: "myNewMutation", next: ["myNextMutation"]}' 428 | ``` 429 | 430 | Or in code: 431 | 432 | ```ts 433 | await migrations.getStatus(ctx, { migrations: ["myNewMutation"] }); 434 | await migrations.cancel(ctx, "myNewMutation"); 435 | ``` 436 | 437 | 438 | -------------------------------------------------------------------------------- /commonjs.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "include": ["src/**/*"], 4 | "exclude": ["src/**/*.test.*", "../src/package.json"], 5 | "compilerOptions": { 6 | "outDir": "./dist/commonjs" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /dashboard_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/get-convex/migrations/fdcba16e94aea41b13f5b0a0a1b36ee8705b400a/dashboard_screenshot.png -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | import globals from "globals"; 2 | import pluginJs from "@eslint/js"; 3 | import tseslint from "typescript-eslint"; 4 | 5 | export default [ 6 | { files: ["src/**/*.{js,mjs,cjs,ts,tsx}"] }, 7 | { 8 | ignores: [ 9 | "dist/**", 10 | "eslint.config.js", 11 | "**/_generated/", 12 | "node10stubs.mjs", 13 | ], 14 | }, 15 | { 16 | languageOptions: { 17 | globals: globals.worker, 18 | parser: tseslint.parser, 19 | 20 | parserOptions: { 21 | project: true, 22 | tsconfigRootDir: ".", 23 | }, 24 | }, 25 | }, 26 | pluginJs.configs.recommended, 27 | ...tseslint.configs.recommended, 28 | { 29 | rules: { 30 | "@typescript-eslint/no-floating-promises": "error", 31 | "eslint-comments/no-unused-disable": "off", 32 | 33 | // allow (_arg: number) => {} and const _foo = 1; 34 | "no-unused-vars": "off", 35 | "@typescript-eslint/no-unused-vars": [ 36 | "warn", 37 | { 38 | argsIgnorePattern: "^_", 39 | varsIgnorePattern: "^_", 40 | }, 41 | ], 42 | }, 43 | }, 44 | ]; 45 | -------------------------------------------------------------------------------- /esm.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "include": ["src/**/*"], 4 | "exclude": ["src/**/*.test.*", "../src/package.json"], 5 | "compilerOptions": { 6 | "outDir": "./dist/esm" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /example/.gitignore: -------------------------------------------------------------------------------- 1 | !**/glob-import/dir/node_modules 2 | .DS_Store 3 | .idea 4 | *.cpuprofile 5 | *.local 6 | *.log 7 | /.vscode/ 8 | /docs/.vitepress/cache 9 | dist 10 | dist-ssr 11 | explorations 12 | node_modules 13 | playground-temp 14 | temp 15 | TODOs.md 16 | .eslintcache 17 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | # Example app 2 | 3 | Components need an app that uses them in order to run codegen. An example app is also useful 4 | for testing and documentation. 5 | 6 | -------------------------------------------------------------------------------- /example/convex/_generated/api.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated `api` utility. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * To regenerate, run `npx convex dev`. 8 | * @module 9 | */ 10 | 11 | import type * as example from "../example.js"; 12 | 13 | import type { 14 | ApiFromModules, 15 | FilterApi, 16 | FunctionReference, 17 | } from "convex/server"; 18 | /** 19 | * A utility for referencing Convex functions in your app's API. 20 | * 21 | * Usage: 22 | * ```js 23 | * const myFunctionReference = api.myModule.myFunction; 24 | * ``` 25 | */ 26 | declare const fullApi: ApiFromModules<{ 27 | example: typeof example; 28 | }>; 29 | declare const fullApiWithMounts: typeof fullApi; 30 | 31 | export declare const api: FilterApi< 32 | typeof fullApiWithMounts, 33 | FunctionReference 34 | >; 35 | export declare const internal: FilterApi< 36 | typeof fullApiWithMounts, 37 | FunctionReference 38 | >; 39 | 40 | export declare const components: { 41 | migrations: { 42 | lib: { 43 | cancel: FunctionReference< 44 | "mutation", 45 | "internal", 46 | { name: string }, 47 | { 48 | batchSize?: number; 49 | cursor?: string | null; 50 | error?: string; 51 | isDone: boolean; 52 | latestEnd?: number; 53 | latestStart: number; 54 | name: string; 55 | next?: Array; 56 | processed: number; 57 | state: "inProgress" | "success" | "failed" | "canceled" | "unknown"; 58 | } 59 | >; 60 | cancelAll: FunctionReference< 61 | "mutation", 62 | "internal", 63 | { sinceTs?: number }, 64 | Array<{ 65 | batchSize?: number; 66 | cursor?: string | null; 67 | error?: string; 68 | isDone: boolean; 69 | latestEnd?: number; 70 | latestStart: number; 71 | name: string; 72 | next?: Array; 73 | processed: number; 74 | state: "inProgress" | "success" | "failed" | "canceled" | "unknown"; 75 | }> 76 | >; 77 | clearAll: FunctionReference< 78 | "mutation", 79 | "internal", 80 | { before?: number }, 81 | null 82 | >; 83 | getStatus: FunctionReference< 84 | "query", 85 | "internal", 86 | { limit?: number; names?: Array }, 87 | Array<{ 88 | batchSize?: number; 89 | cursor?: string | null; 90 | error?: string; 91 | isDone: boolean; 92 | latestEnd?: number; 93 | latestStart: number; 94 | name: string; 95 | next?: Array; 96 | processed: number; 97 | state: "inProgress" | "success" | "failed" | "canceled" | "unknown"; 98 | }> 99 | >; 100 | migrate: FunctionReference< 101 | "mutation", 102 | "internal", 103 | { 104 | batchSize?: number; 105 | cursor?: string | null; 106 | dryRun: boolean; 107 | fnHandle: string; 108 | name: string; 109 | next?: Array<{ fnHandle: string; name: string }>; 110 | }, 111 | { 112 | batchSize?: number; 113 | cursor?: string | null; 114 | error?: string; 115 | isDone: boolean; 116 | latestEnd?: number; 117 | latestStart: number; 118 | name: string; 119 | next?: Array; 120 | processed: number; 121 | state: "inProgress" | "success" | "failed" | "canceled" | "unknown"; 122 | } 123 | >; 124 | }; 125 | }; 126 | }; 127 | -------------------------------------------------------------------------------- /example/convex/_generated/api.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated `api` utility. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * To regenerate, run `npx convex dev`. 8 | * @module 9 | */ 10 | 11 | import { anyApi, componentsGeneric } from "convex/server"; 12 | 13 | /** 14 | * A utility for referencing Convex functions in your app's API. 15 | * 16 | * Usage: 17 | * ```js 18 | * const myFunctionReference = api.myModule.myFunction; 19 | * ``` 20 | */ 21 | export const api = anyApi; 22 | export const internal = anyApi; 23 | export const components = componentsGeneric(); 24 | -------------------------------------------------------------------------------- /example/convex/_generated/dataModel.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated data model types. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * To regenerate, run `npx convex dev`. 8 | * @module 9 | */ 10 | 11 | import type { 12 | DataModelFromSchemaDefinition, 13 | DocumentByName, 14 | TableNamesInDataModel, 15 | SystemTableNames, 16 | } from "convex/server"; 17 | import type { GenericId } from "convex/values"; 18 | import schema from "../schema.js"; 19 | 20 | /** 21 | * The names of all of your Convex tables. 22 | */ 23 | export type TableNames = TableNamesInDataModel; 24 | 25 | /** 26 | * The type of a document stored in Convex. 27 | * 28 | * @typeParam TableName - A string literal type of the table name (like "users"). 29 | */ 30 | export type Doc = DocumentByName< 31 | DataModel, 32 | TableName 33 | >; 34 | 35 | /** 36 | * An identifier for a document in Convex. 37 | * 38 | * Convex documents are uniquely identified by their `Id`, which is accessible 39 | * on the `_id` field. To learn more, see [Document IDs](https://docs.convex.dev/using/document-ids). 40 | * 41 | * Documents can be loaded using `db.get(id)` in query and mutation functions. 42 | * 43 | * IDs are just strings at runtime, but this type can be used to distinguish them from other 44 | * strings when type checking. 45 | * 46 | * @typeParam TableName - A string literal type of the table name (like "users"). 47 | */ 48 | export type Id = 49 | GenericId; 50 | 51 | /** 52 | * A type describing your Convex data model. 53 | * 54 | * This type includes information about what tables you have, the type of 55 | * documents stored in those tables, and the indexes defined on them. 56 | * 57 | * This type is used to parameterize methods like `queryGeneric` and 58 | * `mutationGeneric` to make them type-safe. 59 | */ 60 | export type DataModel = DataModelFromSchemaDefinition; 61 | -------------------------------------------------------------------------------- /example/convex/_generated/server.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated utilities for implementing server-side Convex query and mutation functions. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * To regenerate, run `npx convex dev`. 8 | * @module 9 | */ 10 | 11 | import { 12 | ActionBuilder, 13 | AnyComponents, 14 | HttpActionBuilder, 15 | MutationBuilder, 16 | QueryBuilder, 17 | GenericActionCtx, 18 | GenericMutationCtx, 19 | GenericQueryCtx, 20 | GenericDatabaseReader, 21 | GenericDatabaseWriter, 22 | FunctionReference, 23 | } from "convex/server"; 24 | import type { DataModel } from "./dataModel.js"; 25 | 26 | type GenericCtx = 27 | | GenericActionCtx 28 | | GenericMutationCtx 29 | | GenericQueryCtx; 30 | 31 | /** 32 | * Define a query in this Convex app's public API. 33 | * 34 | * This function will be allowed to read your Convex database and will be accessible from the client. 35 | * 36 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 37 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 38 | */ 39 | export declare const query: QueryBuilder; 40 | 41 | /** 42 | * Define a query that is only accessible from other Convex functions (but not from the client). 43 | * 44 | * This function will be allowed to read from your Convex database. It will not be accessible from the client. 45 | * 46 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 47 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 48 | */ 49 | export declare const internalQuery: QueryBuilder; 50 | 51 | /** 52 | * Define a mutation in this Convex app's public API. 53 | * 54 | * This function will be allowed to modify your Convex database and will be accessible from the client. 55 | * 56 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 57 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 58 | */ 59 | export declare const mutation: MutationBuilder; 60 | 61 | /** 62 | * Define a mutation that is only accessible from other Convex functions (but not from the client). 63 | * 64 | * This function will be allowed to modify your Convex database. It will not be accessible from the client. 65 | * 66 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 67 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 68 | */ 69 | export declare const internalMutation: MutationBuilder; 70 | 71 | /** 72 | * Define an action in this Convex app's public API. 73 | * 74 | * An action is a function which can execute any JavaScript code, including non-deterministic 75 | * code and code with side-effects, like calling third-party services. 76 | * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. 77 | * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. 78 | * 79 | * @param func - The action. It receives an {@link ActionCtx} as its first argument. 80 | * @returns The wrapped action. Include this as an `export` to name it and make it accessible. 81 | */ 82 | export declare const action: ActionBuilder; 83 | 84 | /** 85 | * Define an action that is only accessible from other Convex functions (but not from the client). 86 | * 87 | * @param func - The function. It receives an {@link ActionCtx} as its first argument. 88 | * @returns The wrapped function. Include this as an `export` to name it and make it accessible. 89 | */ 90 | export declare const internalAction: ActionBuilder; 91 | 92 | /** 93 | * Define an HTTP action. 94 | * 95 | * This function will be used to respond to HTTP requests received by a Convex 96 | * deployment if the requests matches the path and method where this action 97 | * is routed. Be sure to route your action in `convex/http.js`. 98 | * 99 | * @param func - The function. It receives an {@link ActionCtx} as its first argument. 100 | * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. 101 | */ 102 | export declare const httpAction: HttpActionBuilder; 103 | 104 | /** 105 | * A set of services for use within Convex query functions. 106 | * 107 | * The query context is passed as the first argument to any Convex query 108 | * function run on the server. 109 | * 110 | * This differs from the {@link MutationCtx} because all of the services are 111 | * read-only. 112 | */ 113 | export type QueryCtx = GenericQueryCtx; 114 | 115 | /** 116 | * A set of services for use within Convex mutation functions. 117 | * 118 | * The mutation context is passed as the first argument to any Convex mutation 119 | * function run on the server. 120 | */ 121 | export type MutationCtx = GenericMutationCtx; 122 | 123 | /** 124 | * A set of services for use within Convex action functions. 125 | * 126 | * The action context is passed as the first argument to any Convex action 127 | * function run on the server. 128 | */ 129 | export type ActionCtx = GenericActionCtx; 130 | 131 | /** 132 | * An interface to read from the database within Convex query functions. 133 | * 134 | * The two entry points are {@link DatabaseReader.get}, which fetches a single 135 | * document by its {@link Id}, or {@link DatabaseReader.query}, which starts 136 | * building a query. 137 | */ 138 | export type DatabaseReader = GenericDatabaseReader; 139 | 140 | /** 141 | * An interface to read from and write to the database within Convex mutation 142 | * functions. 143 | * 144 | * Convex guarantees that all writes within a single mutation are 145 | * executed atomically, so you never have to worry about partial writes leaving 146 | * your data in an inconsistent state. See [the Convex Guide](https://docs.convex.dev/understanding/convex-fundamentals/functions#atomicity-and-optimistic-concurrency-control) 147 | * for the guarantees Convex provides your functions. 148 | */ 149 | export type DatabaseWriter = GenericDatabaseWriter; 150 | -------------------------------------------------------------------------------- /example/convex/_generated/server.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated utilities for implementing server-side Convex query and mutation functions. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * To regenerate, run `npx convex dev`. 8 | * @module 9 | */ 10 | 11 | import { 12 | actionGeneric, 13 | httpActionGeneric, 14 | queryGeneric, 15 | mutationGeneric, 16 | internalActionGeneric, 17 | internalMutationGeneric, 18 | internalQueryGeneric, 19 | componentsGeneric, 20 | } from "convex/server"; 21 | 22 | /** 23 | * Define a query in this Convex app's public API. 24 | * 25 | * This function will be allowed to read your Convex database and will be accessible from the client. 26 | * 27 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 28 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 29 | */ 30 | export const query = queryGeneric; 31 | 32 | /** 33 | * Define a query that is only accessible from other Convex functions (but not from the client). 34 | * 35 | * This function will be allowed to read from your Convex database. It will not be accessible from the client. 36 | * 37 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 38 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 39 | */ 40 | export const internalQuery = internalQueryGeneric; 41 | 42 | /** 43 | * Define a mutation in this Convex app's public API. 44 | * 45 | * This function will be allowed to modify your Convex database and will be accessible from the client. 46 | * 47 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 48 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 49 | */ 50 | export const mutation = mutationGeneric; 51 | 52 | /** 53 | * Define a mutation that is only accessible from other Convex functions (but not from the client). 54 | * 55 | * This function will be allowed to modify your Convex database. It will not be accessible from the client. 56 | * 57 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 58 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 59 | */ 60 | export const internalMutation = internalMutationGeneric; 61 | 62 | /** 63 | * Define an action in this Convex app's public API. 64 | * 65 | * An action is a function which can execute any JavaScript code, including non-deterministic 66 | * code and code with side-effects, like calling third-party services. 67 | * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. 68 | * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. 69 | * 70 | * @param func - The action. It receives an {@link ActionCtx} as its first argument. 71 | * @returns The wrapped action. Include this as an `export` to name it and make it accessible. 72 | */ 73 | export const action = actionGeneric; 74 | 75 | /** 76 | * Define an action that is only accessible from other Convex functions (but not from the client). 77 | * 78 | * @param func - The function. It receives an {@link ActionCtx} as its first argument. 79 | * @returns The wrapped function. Include this as an `export` to name it and make it accessible. 80 | */ 81 | export const internalAction = internalActionGeneric; 82 | 83 | /** 84 | * Define a Convex HTTP action. 85 | * 86 | * @param func - The function. It receives an {@link ActionCtx} as its first argument, and a `Request` object 87 | * as its second. 88 | * @returns The wrapped endpoint function. Route a URL path to this function in `convex/http.js`. 89 | */ 90 | export const httpAction = httpActionGeneric; 91 | -------------------------------------------------------------------------------- /example/convex/convex.config.ts: -------------------------------------------------------------------------------- 1 | import { defineApp } from "convex/server"; 2 | import migrations from "@convex-dev/migrations/convex.config"; 3 | 4 | const app = defineApp(); 5 | app.use(migrations); 6 | 7 | export default app; 8 | -------------------------------------------------------------------------------- /example/convex/example.ts: -------------------------------------------------------------------------------- 1 | import { Migrations, MigrationStatus } from "@convex-dev/migrations"; 2 | import { v } from "convex/values"; 3 | import { components, internal } from "./_generated/api.js"; 4 | import { DataModel } from "./_generated/dataModel.js"; 5 | import { internalMutation, internalQuery } from "./_generated/server.js"; 6 | 7 | export const migrations = new Migrations(components.migrations); 8 | 9 | // Allows you to run `npx convex run example:run '{"fn":"example:setDefaultValue"}'` 10 | export const run = migrations.runner(); 11 | 12 | // This allows you to just run `npx convex run example:runIt` 13 | export const runIt = migrations.runner(internal.example.setDefaultValue); 14 | 15 | export const setDefaultValue = migrations.define({ 16 | table: "myTable", 17 | batchSize: 2, 18 | migrateOne: async (_ctx, doc) => { 19 | if (doc.optionalField === undefined) { 20 | return { optionalField: "default" }; 21 | } 22 | }, 23 | parallelize: true, 24 | }); 25 | 26 | export const clearField = migrations.define({ 27 | table: "myTable", 28 | migrateOne: () => ({ optionalField: undefined }), 29 | }); 30 | 31 | export const validateRequiredField = migrations.define({ 32 | table: "myTable", 33 | // Specify a custom range to only include documents that need to change. 34 | // This is useful if you have a large dataset and only a small percentage of 35 | // documents need to be migrated. 36 | customRange: (query) => 37 | query.withIndex("by_requiredField", (q) => q.eq("requiredField", "")), 38 | migrateOne: async (_ctx, doc) => { 39 | console.log("Needs fixup: " + doc._id); 40 | // Shorthand for patching 41 | return { requiredField: "" }; 42 | }, 43 | }); 44 | 45 | // If you prefer the old-style migration definition, you can define `migration`: 46 | const migration = migrations.define.bind(migrations); 47 | // Then use it like this: 48 | export const convertUnionField = migration({ 49 | table: "myTable", 50 | migrateOne: async (ctx, doc) => { 51 | if (typeof doc.unionField === "number") { 52 | await ctx.db.patch(doc._id, { unionField: doc.unionField.toString() }); 53 | } 54 | }, 55 | }); 56 | 57 | export const failingMigration = migrations.define({ 58 | table: "myTable", 59 | batchSize: 1, 60 | migrateOne: async (ctx, doc) => { 61 | if (doc._id !== (await ctx.db.query("myTable").first())?._id) { 62 | throw new Error("This migration fails after the first"); 63 | } 64 | }, 65 | }); 66 | 67 | export const runOneAtATime = internalMutation({ 68 | args: {}, 69 | handler: async (ctx) => { 70 | await migrations.runOne(ctx, internal.example.failingMigration, { 71 | batchSize: 1, 72 | }); 73 | }, 74 | }); 75 | 76 | // It's handy to have a list of all migrations that folks should run in order. 77 | const allMigrations = [ 78 | internal.example.setDefaultValue, 79 | internal.example.validateRequiredField, 80 | internal.example.convertUnionField, 81 | internal.example.failingMigration, 82 | ]; 83 | 84 | export const runAll = migrations.runner(allMigrations); 85 | 86 | // Call this from a deploy script to run them after pushing code. 87 | export const postDeploy = internalMutation({ 88 | args: {}, 89 | handler: async (ctx) => { 90 | // Do other post-deploy things... 91 | await migrations.runSerially(ctx, allMigrations); 92 | }, 93 | }); 94 | 95 | // Handy for checking the status from the CLI / dashboard. 96 | export const getStatus = internalQuery({ 97 | args: {}, 98 | handler: async (ctx): Promise => { 99 | return migrations.getStatus(ctx, { 100 | migrations: allMigrations, 101 | }); 102 | }, 103 | }); 104 | 105 | export const cancel = internalMutation({ 106 | args: { name: v.optional(v.string()) }, 107 | handler: async (ctx, args) => { 108 | if (args.name) { 109 | await migrations.cancel(ctx, args.name); 110 | } else { 111 | await migrations.cancelAll(ctx); 112 | } 113 | }, 114 | }); 115 | 116 | export const seed = internalMutation({ 117 | args: { count: v.optional(v.number()) }, 118 | handler: async (ctx, args) => { 119 | for (let i = 0; i < (args.count ?? 10); i++) { 120 | await ctx.db.insert("myTable", { 121 | requiredField: "seed " + i, 122 | optionalField: i % 2 ? "optionalValue" : undefined, 123 | unionField: i % 2 ? "1" : 1, 124 | }); 125 | } 126 | }, 127 | }); 128 | 129 | // Alternatively, you can specify a prefix. 130 | export const migrationsWithPrefix = new Migrations(components.migrations, { 131 | // Specifying the internalMutation means you don't need the type parameter. 132 | // Also, if you have a custom internalMutation, you can specify it here. 133 | internalMutation, 134 | migrationsLocationPrefix: "example:", 135 | }); 136 | 137 | // Allows you to run `npx convex run example:runWithPrefix '{"fn":"setDefaultValue"}'` 138 | export const runWithPrefix = migrationsWithPrefix.runner(); 139 | -------------------------------------------------------------------------------- /example/convex/schema.ts: -------------------------------------------------------------------------------- 1 | import { v } from "convex/values"; 2 | import { defineSchema, defineTable } from "convex/server"; 3 | 4 | export default defineSchema({ 5 | // Any tables used by the example app go here. 6 | myTable: defineTable({ 7 | requiredField: v.string(), 8 | optionalField: v.optional(v.string()), 9 | unionField: v.union(v.string(), v.number()), 10 | }).index("by_requiredField", ["requiredField"]), 11 | }); 12 | -------------------------------------------------------------------------------- /example/convex/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | /* This TypeScript project config describes the environment that 3 | * Convex functions run in and is used to typecheck them. 4 | * You can modify it, but some settings required to use Convex. 5 | */ 6 | "compilerOptions": { 7 | /* These settings are not required by Convex and can be modified. */ 8 | "allowJs": true, 9 | "strict": true, 10 | "skipLibCheck": true, 11 | 12 | /* These compiler options are required by Convex */ 13 | "target": "ESNext", 14 | "lib": ["ES2021", "dom", "ESNext.Array"], 15 | "forceConsistentCasingInFileNames": true, 16 | "allowSyntheticDefaultImports": true, 17 | "module": "ESNext", 18 | "moduleResolution": "Bundler", 19 | "isolatedModules": true, 20 | "noEmit": true, 21 | 22 | /* This should only be used in this example. Real apps should not attempt 23 | * to compile TypeScript because differences between tsconfig.json files can 24 | * cause the code to be compiled differently. 25 | */ 26 | "customConditions": ["@convex-dev/component-source"] 27 | }, 28 | "include": ["./**/*"], 29 | "exclude": ["./_generated"] 30 | } 31 | -------------------------------------------------------------------------------- /example/eslint.config.js: -------------------------------------------------------------------------------- 1 | import js from "@eslint/js"; 2 | import globals from "globals"; 3 | import reactHooks from "eslint-plugin-react-hooks"; 4 | import reactRefresh from "eslint-plugin-react-refresh"; 5 | import tseslint from "typescript-eslint"; 6 | 7 | export default tseslint.config( 8 | { ignores: ["dist"] }, 9 | { 10 | extends: [js.configs.recommended, ...tseslint.configs.recommended], 11 | files: ["**/*.{ts,tsx}"], 12 | ignores: ["convex"], 13 | languageOptions: { 14 | ecmaVersion: 2020, 15 | globals: globals.browser, 16 | }, 17 | plugins: { 18 | "react-hooks": reactHooks, 19 | "react-refresh": reactRefresh, 20 | }, 21 | rules: { 22 | ...reactHooks.configs.recommended.rules, 23 | "react-refresh/only-export-components": [ 24 | "warn", 25 | { allowConstantExport: true }, 26 | ], 27 | 28 | "no-unused-vars": "off", 29 | "@typescript-eslint/no-unused-vars": [ 30 | "warn", 31 | { 32 | argsIgnorePattern: "^_", 33 | varsIgnorePattern: "^_", 34 | }, 35 | ], 36 | }, 37 | } 38 | ); 39 | -------------------------------------------------------------------------------- /example/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Vite + React + TS 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "uses-component", 3 | "private": true, 4 | "type": "module", 5 | "version": "0.0.0", 6 | "scripts": { 7 | "dev": "convex dev --live-component-sources --typecheck-components", 8 | "dev:frontend": "vite", 9 | "logs": "convex logs", 10 | "lint": "tsc -p convex && eslint convex" 11 | }, 12 | "dependencies": { 13 | "@convex-dev/migrations": "file:..", 14 | "convex": "file:../node_modules/convex", 15 | "convex-test": "file:../node_modules/convex-test", 16 | "react": "^18.3.1", 17 | "react-dom": "^18.3.1" 18 | }, 19 | "devDependencies": { 20 | "@eslint/eslintrc": "^3.1.0", 21 | "@eslint/js": "^9.9.0", 22 | "@types/react": "^18.3.3", 23 | "@types/react-dom": "^18.3.0", 24 | "@vitejs/plugin-react": "^4.3.1", 25 | "eslint": "^9.9.0", 26 | "eslint-plugin-react-hooks": "^5.1.0-rc.0", 27 | "eslint-plugin-react-refresh": "^0.4.9", 28 | "globals": "^15.9.0", 29 | "typescript": "^5.5.0", 30 | "typescript-eslint": "^8.0.1", 31 | "vite": "^5.4.1", 32 | "vitest": "file:../node_modules/vitest" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /example/src/App.css: -------------------------------------------------------------------------------- 1 | #root { 2 | max-width: 1280px; 3 | margin: 0 auto; 4 | padding: 2rem; 5 | text-align: center; 6 | } 7 | 8 | .logo { 9 | height: 6em; 10 | padding: 1.5em; 11 | will-change: filter; 12 | transition: filter 300ms; 13 | } 14 | .logo:hover { 15 | filter: drop-shadow(0 0 2em #646cffaa); 16 | } 17 | .logo.react:hover { 18 | filter: drop-shadow(0 0 2em #61dafbaa); 19 | } 20 | 21 | @keyframes logo-spin { 22 | from { 23 | transform: rotate(0deg); 24 | } 25 | to { 26 | transform: rotate(360deg); 27 | } 28 | } 29 | 30 | @media (prefers-reduced-motion: no-preference) { 31 | a:nth-of-type(2) .logo { 32 | animation: logo-spin infinite 20s linear; 33 | } 34 | } 35 | 36 | .card { 37 | padding: 2em; 38 | } 39 | 40 | .read-the-docs { 41 | color: #888; 42 | } 43 | -------------------------------------------------------------------------------- /example/src/App.tsx: -------------------------------------------------------------------------------- 1 | import "./App.css"; 2 | import { useMutation, useQuery } from "convex/react"; 3 | import { api } from "../convex/_generated/api"; 4 | 5 | function App() { 6 | const count = useQuery(api.app.getCount); 7 | const addOne = useMutation(api.app.addOne); 8 | 9 | return ( 10 | <> 11 |

Convex migrations Component Example

12 |
13 | 14 |

15 | See example/convex/example.ts for all the ways to use 16 | this component 17 |

18 |
19 | 20 | ); 21 | } 22 | 23 | export default App; 24 | -------------------------------------------------------------------------------- /example/src/index.css: -------------------------------------------------------------------------------- 1 | :root { 2 | font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif; 3 | line-height: 1.5; 4 | font-weight: 400; 5 | 6 | color-scheme: light dark; 7 | color: rgba(255, 255, 255, 0.87); 8 | background-color: #242424; 9 | 10 | font-synthesis: none; 11 | text-rendering: optimizeLegibility; 12 | -webkit-font-smoothing: antialiased; 13 | -moz-osx-font-smoothing: grayscale; 14 | } 15 | 16 | a { 17 | font-weight: 500; 18 | color: #646cff; 19 | text-decoration: inherit; 20 | } 21 | a:hover { 22 | color: #535bf2; 23 | } 24 | 25 | body { 26 | margin: 0; 27 | display: flex; 28 | place-items: center; 29 | min-width: 320px; 30 | min-height: 100vh; 31 | } 32 | 33 | h1 { 34 | font-size: 3.2em; 35 | line-height: 1.1; 36 | } 37 | 38 | button { 39 | border-radius: 8px; 40 | border: 1px solid transparent; 41 | padding: 0.6em 1.2em; 42 | font-size: 1em; 43 | font-weight: 500; 44 | font-family: inherit; 45 | background-color: #1a1a1a; 46 | cursor: pointer; 47 | transition: border-color 0.25s; 48 | } 49 | button:hover { 50 | border-color: #646cff; 51 | } 52 | button:focus, 53 | button:focus-visible { 54 | outline: 4px auto -webkit-focus-ring-color; 55 | } 56 | 57 | @media (prefers-color-scheme: light) { 58 | :root { 59 | color: #213547; 60 | background-color: #ffffff; 61 | } 62 | a:hover { 63 | color: #747bff; 64 | } 65 | button { 66 | background-color: #f9f9f9; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /example/src/main.tsx: -------------------------------------------------------------------------------- 1 | import { StrictMode } from "react"; 2 | import { createRoot } from "react-dom/client"; 3 | import { ConvexProvider, ConvexReactClient } from "convex/react"; 4 | import App from "./App.tsx"; 5 | import "./index.css"; 6 | 7 | const address = import.meta.env.VITE_CONVEX_URL; 8 | 9 | const convex = new ConvexReactClient(address); 10 | 11 | createRoot(document.getElementById("root")!).render( 12 | 13 | 14 | 15 | 16 | 17 | ); 18 | -------------------------------------------------------------------------------- /example/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /example/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "lib": ["DOM", "DOM.Iterable", "ESNext"], 5 | "skipLibCheck": true, 6 | "allowSyntheticDefaultImports": true, 7 | "strict": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "module": "ESNext", 10 | "moduleResolution": "Bundler", 11 | "resolveJsonModule": true, 12 | "isolatedModules": true, 13 | "allowImportingTsExtensions": true, 14 | "noEmit": true, 15 | "jsx": "react-jsx", 16 | 17 | /* This should only be used in this example. Real apps should not attempt 18 | * to compile TypeScript because differences between tsconfig.json files can 19 | * cause the code to be compiled differently. 20 | */ 21 | "customConditions": ["@convex-dev/component-source"] 22 | }, 23 | "include": ["./src", "vite.config.ts"] 24 | } 25 | -------------------------------------------------------------------------------- /example/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vite"; 2 | import react from "@vitejs/plugin-react"; 3 | 4 | // https://vitejs.dev/config/ 5 | export default defineConfig({ 6 | plugins: [react()], 7 | resolve: { 8 | conditions: ["@convex-dev/component-source"], 9 | }, 10 | }); 11 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@convex-dev/migrations", 3 | "description": "A migrations component for Convex. Define, run, and track your database migrations. Run from a CLI or Convex server function.", 4 | "repository": "github:get-convex/migrations", 5 | "homepage": "https://github.com/get-convex/migrations#readme", 6 | "bugs": { 7 | "email": "support@convex.dev", 8 | "url": "https://github.com/get-convex/migrations/issues" 9 | }, 10 | "version": "0.2.8", 11 | "license": "Apache-2.0", 12 | "keywords": [ 13 | "convex", 14 | "component" 15 | ], 16 | "type": "module", 17 | "scripts": { 18 | "build": "npm run build:esm && npm run build:cjs", 19 | "build:esm": "tsc --project ./esm.json && echo '{\\n \"type\": \"module\"\\n}' > dist/esm/package.json", 20 | "build:cjs": "tsc --project ./commonjs.json && echo '{\\n \"type\": \"commonjs\"\\n}' > dist/commonjs/package.json", 21 | "dev": "cd example; npm run dev", 22 | "typecheck": "tsc --noEmit", 23 | "alpha": "rm -rf dist && npm run build && npm run test && npm run typecheck && npm version prerelease --preid alpha && npm publish --tag alpha && git push --tags", 24 | "release": "rm -rf dist && npm run build && npm run test && npm run typecheck && npm version patch && npm publish && git push --tags", 25 | "prepare": "npm run build", 26 | "test": "vitest run", 27 | "test:watch": "vitest" 28 | }, 29 | "files": [ 30 | "dist", 31 | "src" 32 | ], 33 | "exports": { 34 | "./package.json": "./package.json", 35 | ".": { 36 | "import": { 37 | "@convex-dev/component-source": "./src/client/index.ts", 38 | "types": "./dist/esm/client/index.d.ts", 39 | "default": "./dist/esm/client/index.js" 40 | }, 41 | "require": { 42 | "@convex-dev/component-source": "./src/client/index.ts", 43 | "types": "./dist/commonjs/client/index.d.ts", 44 | "default": "./dist/commonjs/client/index.js" 45 | } 46 | }, 47 | "./convex.config": { 48 | "import": { 49 | "@convex-dev/component-source": "./src/component/convex.config.ts", 50 | "types": "./dist/esm/component/convex.config.d.ts", 51 | "default": "./dist/esm/component/convex.config.js" 52 | } 53 | } 54 | }, 55 | "peerDependencies": { 56 | "convex": "~1.16.5 || >=1.17.0 <1.35.0" 57 | }, 58 | "devDependencies": { 59 | "@eslint/js": "^9.9.1", 60 | "@types/node": "^18.17.0", 61 | "convex-test": "^0.0.35", 62 | "eslint": "^9.9.1", 63 | "globals": "^15.9.0", 64 | "prettier": "3.2.5", 65 | "typescript": "~5.0.3", 66 | "typescript-eslint": "^8.4.0", 67 | "vitest": "^3.0.5" 68 | }, 69 | "main": "./dist/commonjs/client/index.js", 70 | "types": "./dist/commonjs/client/index.d.ts", 71 | "module": "./dist/esm/client/index.js" 72 | } 73 | -------------------------------------------------------------------------------- /src/client/index.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, test, expect } from 'vitest'; 2 | import { Migrations, DEFAULT_BATCH_SIZE } from './index.js'; 3 | 4 | describe('Migrations class', () => { 5 | test('can instantiate without error', () => { 6 | const dummyComponent: any = {}; 7 | expect(() => new Migrations(dummyComponent)).not.toThrow(); 8 | }); 9 | }); 10 | 11 | describe('DEFAULT_BATCH_SIZE', () => { 12 | test('should equal 100', () => { 13 | expect(DEFAULT_BATCH_SIZE).toBe(100); 14 | }); 15 | }); -------------------------------------------------------------------------------- /src/client/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | createFunctionHandle, 3 | DocumentByName, 4 | Expand, 5 | FunctionReference, 6 | GenericDataModel, 7 | GenericMutationCtx, 8 | GenericQueryCtx, 9 | getFunctionName, 10 | internalMutationGeneric, 11 | makeFunctionReference, 12 | MutationBuilder, 13 | NamedTableInfo, 14 | OrderedQuery, 15 | QueryInitializer, 16 | RegisteredMutation, 17 | TableNamesInDataModel, 18 | } from "convex/server"; 19 | import { 20 | MigrationArgs, 21 | migrationArgs, 22 | MigrationResult, 23 | MigrationStatus, 24 | } from "../shared.js"; 25 | export type { MigrationArgs, MigrationResult, MigrationStatus }; 26 | import { api } from "../component/_generated/api.js"; // the component's public api 27 | 28 | import { ConvexError, GenericId } from "convex/values"; 29 | 30 | // Note: this value is hard-coded in the docstring below. Please keep in sync. 31 | export const DEFAULT_BATCH_SIZE = 100; 32 | 33 | export class Migrations { 34 | /** 35 | * Makes the migration wrapper, with types for your own tables. 36 | * 37 | * It will keep track of migration state. 38 | * Add in convex/migrations.ts for example: 39 | * ```ts 40 | * import { Migrations } from "@convex-dev/migrations"; 41 | * import { components } from "./_generated/api.js"; 42 | * import { internalMutation } from "./_generated/server"; 43 | * 44 | * export const migrations = new Migrations(components.migrations, { internalMutation }); 45 | * // the private mutation to run migrations. 46 | * export const run = migrations.runner(); 47 | * 48 | * export const myMigration = migrations.define({ 49 | * table: "users", 50 | * migrateOne: async (ctx, doc) => { 51 | * await ctx.db.patch(doc._id, { someField: "value" }); 52 | * } 53 | * }); 54 | * ``` 55 | * You can then run it from the CLI or dashboard: 56 | * ```sh 57 | * npx convex run migrations:run '{"fn": "migrations:myMigration"}' 58 | * ``` 59 | * For starting a migration from code, see {@link runOne}/{@link runSerially}. 60 | * @param component - The migrations component. It will be on components.migrations 61 | * after being configured in in convex.config.js. 62 | * @param options - Configure options and set the internalMutation to use. 63 | */ 64 | constructor( 65 | public component: UseApi, 66 | public options?: { 67 | /** 68 | * Uses the internal mutation to run the migration. 69 | * This also provides the types for your tables. 70 | * ```ts 71 | * import { internalMutation } from "./_generated/server.js"; 72 | * ``` 73 | */ 74 | internalMutation?: MutationBuilder; 75 | /** 76 | * How many documents to process in a batch. 77 | * Your migrateOne function will be called for each document in a batch in 78 | * a single transaction. 79 | */ 80 | defaultBatchSize?: number; 81 | /** 82 | * Prefix to add to the function name when running migrations. 83 | * For example, if you have a function named "foo" in a file 84 | * "convex/bar/baz.ts", you can set {migrationsLocationPrefix: "bar/baz:"} 85 | * and then run: 86 | * ```sh 87 | * npx convex run migrations:run '{"fn": "foo"}' 88 | * ``` 89 | */ 90 | migrationsLocationPrefix?: string; 91 | } 92 | ) {} 93 | 94 | /** 95 | * Creates a migration runner that can be called from the CLI or dashboard. 96 | * 97 | * For starting a migration from code, see {@link runOne}/{@link runSerially}. 98 | * 99 | * It can be created for a specific migration: 100 | * ```ts 101 | * export const runMyMigration = runner(internal.migrations.myMigration); 102 | * ``` 103 | * CLI: `npx convex run migrations:runMyMigration` 104 | * 105 | * Or for any migration: 106 | * ```ts 107 | * export const run = runner(); 108 | * ``` 109 | * CLI: `npx convex run migrations:run '{"fn": "migrations:myMigration"}'` 110 | * 111 | * Where `myMigration` is the name of the migration function, defined in 112 | * "convex/migrations.ts" along with the run function. 113 | * 114 | * @param specificMigration If you want a migration runner for one migration, 115 | * pass in the migration function reference like `internal.migrations.foo`. 116 | * Otherwise it will be a generic runner that requires the migration name. 117 | * @returns An internal mutation, 118 | */ 119 | runner( 120 | specificMigrationOrSeries?: 121 | | MigrationFunctionReference 122 | | MigrationFunctionReference[] 123 | ) { 124 | return internalMutationGeneric({ 125 | args: migrationArgs, 126 | handler: async (ctx, args) => { 127 | const [specificMigration, next] = Array.isArray( 128 | specificMigrationOrSeries 129 | ) 130 | ? [ 131 | specificMigrationOrSeries[0], 132 | await Promise.all( 133 | specificMigrationOrSeries.slice(1).map(async (fnRef) => ({ 134 | name: getFunctionName(fnRef), 135 | fnHandle: await createFunctionHandle(fnRef), 136 | })) 137 | ), 138 | ] 139 | : [specificMigrationOrSeries, undefined]; 140 | if (args.fn && specificMigration) { 141 | throw new Error("Specify only one of fn or specificMigration"); 142 | } 143 | if (!args.fn && !specificMigration) { 144 | throw new Error( 145 | `Specify the migration: '{"fn": "migrations:foo"}'\n` + 146 | "Or initialize a `runner` runner specific to the migration like\n" + 147 | "`export const runMyMigration = runner(internal.migrations.myMigration)`" 148 | ); 149 | } 150 | return await this._runInteractive(ctx, args, specificMigration, next); 151 | }, 152 | }); 153 | } 154 | 155 | private async _runInteractive( 156 | ctx: RunMutationCtx, 157 | args: MigrationArgs, 158 | fnRef?: MigrationFunctionReference, 159 | next?: { name: string; fnHandle: string }[] 160 | ) { 161 | const name = args.fn ? this.prefixedName(args.fn) : getFunctionName(fnRef!); 162 | async function makeFn(fn: string) { 163 | try { 164 | return await createFunctionHandle( 165 | makeFunctionReference<"mutation">(fn) 166 | ); 167 | } catch { 168 | throw new Error( 169 | `Can't find function ${fn}\n` + 170 | "The name should match the folder/file:method\n" + 171 | "See https://docs.convex.dev/functions/query-functions#query-names" 172 | ); 173 | } 174 | } 175 | const fnHandle = args.fn 176 | ? await makeFn(name) 177 | : await createFunctionHandle(fnRef!); 178 | if (args.next) { 179 | next = (next ?? []).concat( 180 | await Promise.all( 181 | args.next.map(async (nextFn) => ({ 182 | name: this.prefixedName(nextFn), 183 | fnHandle: await makeFn(this.prefixedName(nextFn)), 184 | })) 185 | ) 186 | ); 187 | } 188 | let status: MigrationStatus; 189 | try { 190 | status = await ctx.runMutation(this.component.lib.migrate, { 191 | name, 192 | fnHandle, 193 | cursor: args.cursor, 194 | batchSize: args.batchSize, 195 | next, 196 | dryRun: args.dryRun ?? false, 197 | }); 198 | } catch (e) { 199 | if ( 200 | args.dryRun && 201 | e instanceof ConvexError && 202 | e.data.kind === "DRY RUN" 203 | ) { 204 | status = e.data.status; 205 | } else { 206 | throw e; 207 | } 208 | } 209 | 210 | return logStatusAndInstructions(name, status, args); 211 | } 212 | 213 | /** 214 | * Use this to wrap a mutation that will be run over all documents in a table. 215 | * Your mutation only needs to handle changing one document at a time, 216 | * passed into migrateOne. 217 | * Optionally specify a custom batch size to override the default (100). 218 | * 219 | * In convex/migrations.ts for example: 220 | * ```ts 221 | * export const foo = migrations.define({ 222 | * table: "users", 223 | * migrateOne: async (ctx, doc) => { 224 | * await ctx.db.patch(doc._id, { someField: "value" }); 225 | * }, 226 | * }); 227 | * ``` 228 | * 229 | * You can run this manually from the CLI or dashboard: 230 | * ```sh 231 | * # Start or resume a migration. No-ops if it's already done: 232 | * npx convex run migrations:run '{"fn": "migrations:foo"}' 233 | * 234 | * # Restart a migration from a cursor (null is from the beginning): 235 | * npx convex run migrations:run '{"fn": "migrations:foo", "cursor": null }' 236 | * 237 | * # Dry run - runs one batch but doesn't schedule or commit changes. 238 | * # so you can see what it would do without committing the transaction. 239 | * npx convex run migrations:run '{"fn": "migrations:foo", "dryRun": true}' 240 | * # or 241 | * npx convex run migrations:myMigration '{"dryRun": true}' 242 | * 243 | * # Run many migrations serially: 244 | * npx convex run migrations:run '{"fn": "migrations:foo", "next": ["migrations:bar", "migrations:baz"] }' 245 | * ``` 246 | * 247 | * The fn is the string form of the function reference. See: 248 | * https://docs.convex.dev/functions/query-functions#query-names 249 | * 250 | * See {@link runOne} and {@link runSerially} for programmatic use. 251 | * 252 | * @param table - The table to run the migration over. 253 | * @param migrateOne - The function to run on each document. 254 | * @param batchSize - The number of documents to process in a batch. 255 | * If not set, defaults to the value passed to makeMigration, 256 | * or {@link DEFAULT_BATCH_SIZE}. Overriden by arg at runtime if supplied. 257 | * @param parallelize - If true, each migration batch will be run in parallel. 258 | * @returns An internal mutation that runs the migration. 259 | */ 260 | define>({ 261 | table, 262 | migrateOne, 263 | customRange, 264 | batchSize: functionDefaultBatchSize, 265 | parallelize, 266 | }: { 267 | table: TableName; 268 | migrateOne: ( 269 | ctx: GenericMutationCtx, 270 | doc: DocumentByName & { _id: GenericId } 271 | ) => 272 | | void 273 | | Partial> 274 | | Promise> | void>; 275 | customRange?: ( 276 | q: QueryInitializer> 277 | ) => OrderedQuery>; 278 | batchSize?: number; 279 | parallelize?: boolean; 280 | }) { 281 | const defaultBatchSize = 282 | functionDefaultBatchSize ?? 283 | this.options?.defaultBatchSize ?? 284 | DEFAULT_BATCH_SIZE; 285 | // Under the hood it's an internal mutation that calls the migrateOne 286 | // function for every document in a page, recursively scheduling batches. 287 | return ( 288 | (this.options?.internalMutation as MutationBuilder< 289 | DataModel, 290 | "internal" 291 | >) ?? (internalMutationGeneric as MutationBuilder) 292 | )({ 293 | args: migrationArgs, 294 | handler: async (ctx, args) => { 295 | if (args.fn) { 296 | // This is a one-off execution from the CLI or dashboard. 297 | // While not the recommended appproach, it's helpful for one-offs and 298 | // compatibility with the old way of running migrations. 299 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 300 | return (await this._runInteractive(ctx, args)) as any; 301 | } else if (args.next?.length) { 302 | throw new Error("You can only pass next if you also provide fn"); 303 | } else if ( 304 | args.cursor === undefined || 305 | args.cursor === "" || 306 | args.dryRun === undefined || 307 | args.batchSize === 0 308 | ) { 309 | console.warn( 310 | "Running this from the CLI or dashboard? Here's some args to use:" 311 | ); 312 | console.warn({ 313 | "Dry run": '{ "dryRun": true, "cursor": null }', 314 | "For real": '{ "fn": "path/to/migrations:yourFnName" }', 315 | }); 316 | } 317 | 318 | const numItems = args.batchSize || defaultBatchSize; 319 | if (args.cursor === undefined || args.cursor === "") { 320 | if (args.dryRun === undefined) { 321 | console.warn( 322 | "No cursor or dryRun specified - doing a dry run on the next batch." 323 | ); 324 | args.cursor = null; 325 | args.dryRun = true; 326 | } else if (args.dryRun) { 327 | console.warn("Setting cursor to null for dry run"); 328 | args.cursor = null; 329 | } else { 330 | throw new Error(`Cursor must be specified for a one-off execution. 331 | Use null to start from the beginning. 332 | Use the value in the migrations database to pick up from where it left off.`); 333 | } 334 | } 335 | 336 | const q = ctx.db.query(table); 337 | const range = customRange ? customRange(q) : q; 338 | let continueCursor: string; 339 | let page: DocumentByName[]; 340 | let isDone: boolean; 341 | try { 342 | ({ continueCursor, page, isDone } = await range.paginate({ 343 | cursor: args.cursor, 344 | numItems, 345 | })); 346 | } catch (e) { 347 | console.error( 348 | "Error paginating. This can happen if the migration " + 349 | "was initially run on a different table, different custom range, " + 350 | "or you upgraded convex-helpers with in-progress migrations. " + 351 | "This creates an invalid pagination cursor. " + 352 | "Run all migrations to completion on the old cursor, or re-run " + 353 | "them explicitly with the cursor set to null. " + 354 | "If the problem persists, contact support@convex.dev" 355 | ); 356 | throw e; 357 | } 358 | async function doOne(doc: DocumentByName) { 359 | try { 360 | const next = await migrateOne( 361 | ctx, 362 | doc as { _id: GenericId } 363 | ); 364 | if (next && Object.keys(next).length > 0) { 365 | await ctx.db.patch(doc._id as GenericId, next); 366 | } 367 | } catch (error) { 368 | console.error(`Document failed: ${doc._id}`); 369 | throw error; 370 | } 371 | } 372 | if (parallelize) { 373 | await Promise.all(page.map(doOne)); 374 | } else { 375 | for (const doc of page) { 376 | await doOne(doc); 377 | } 378 | } 379 | const result = { 380 | continueCursor, 381 | isDone, 382 | processed: page.length, 383 | }; 384 | if (args.dryRun) { 385 | // Throwing an error rolls back the transaction 386 | let anyChanges = false; 387 | for (const before of page) { 388 | const after = await ctx.db.get(before._id as GenericId); 389 | if (JSON.stringify(after) !== JSON.stringify(before)) { 390 | console.debug("DRY RUN: Example change", { 391 | before, 392 | after, 393 | }); 394 | anyChanges = true; 395 | break; 396 | } 397 | } 398 | if (!anyChanges) { 399 | console.debug( 400 | "DRY RUN: No changes were found in the first page. " + 401 | `Try {"dryRun": true, "cursor": "${continueCursor}"}` 402 | ); 403 | } 404 | throw new ConvexError({ 405 | kind: "DRY RUN", 406 | result, 407 | }); 408 | } 409 | if (args.dryRun === undefined) { 410 | // We are running it in a one-off mode. 411 | // The component will always provide dryRun. 412 | // A bit of a hack / implicit, but non-critical logging. 413 | console.debug(`Next cursor: ${continueCursor}`); 414 | } 415 | return result; 416 | }, 417 | }) satisfies RegisteredMutation< 418 | "internal", 419 | MigrationArgs, 420 | Promise 421 | >; 422 | } 423 | 424 | /** 425 | * Start a migration from a server function via a function reference. 426 | * 427 | * ```ts 428 | * const migrations = new Migrations(components.migrations, { internalMutation }); 429 | * 430 | * // in a mutation or action: 431 | * await migrations.runOne(ctx, internal.migrations.myMigration, { 432 | * cursor: null, // optional override 433 | * batchSize: 10, // optional override 434 | * }); 435 | * ``` 436 | * 437 | * Overrides any options you passed in, such as resetting the cursor. 438 | * If it's already in progress, it will no-op. 439 | * If you run a migration that had previously failed which was part of a series, 440 | * it will not resume the series. 441 | * To resume a series, call the series again: {@link Migrations.runSerially}. 442 | * 443 | * Note: It's up to you to determine if it's safe to run a migration while 444 | * others are in progress. It won't run multiple instance of the same migration 445 | * but it currently allows running multiple migrations on the same table. 446 | * 447 | * @param ctx Context from a mutation or action. Needs `runMutation`. 448 | * @param fnRef The migration function to run. Like `internal.migrations.foo`. 449 | * @param opts Options to start the migration. 450 | * @param opts.cursor The cursor to start from. 451 | * null: start from the beginning. 452 | * undefined: start or resume from where it failed. If done, it won't restart. 453 | * @param opts.batchSize The number of documents to process in a batch. 454 | * @param opts.dryRun If true, it will run a batch and then throw an error. 455 | * It's helpful to see what it would do without committing the transaction. 456 | */ 457 | async runOne( 458 | ctx: RunMutationCtx, 459 | fnRef: MigrationFunctionReference, 460 | opts?: { 461 | cursor?: string | null; 462 | batchSize?: number; 463 | dryRun?: boolean; 464 | } 465 | ) { 466 | return ctx.runMutation(this.component.lib.migrate, { 467 | name: getFunctionName(fnRef), 468 | fnHandle: await createFunctionHandle(fnRef), 469 | cursor: opts?.cursor, 470 | batchSize: opts?.batchSize, 471 | dryRun: opts?.dryRun ?? false, 472 | }); 473 | } 474 | 475 | /** 476 | * Start a series of migrations, running one a time. Each call starts a series. 477 | * 478 | * ```ts 479 | * const migrations = new Migrations(components.migrations, { internalMutation }); 480 | * 481 | * // in a mutation or action: 482 | * await migrations.runSerially(ctx, [ 483 | * internal.migrations.myMigration, 484 | * internal.migrations.myOtherMigration, 485 | * ]); 486 | * ``` 487 | * 488 | * It runs one batch at a time currently. 489 | * If a migration has previously completed it will skip it. 490 | * If a migration had partial progress, it will resume from where it left off. 491 | * If a migration is already in progress when attempted, it will no-op. 492 | * If a migration fails or is canceled, it will stop executing and NOT execute 493 | * any subsequent migrations in the series. Call the series again to retry. 494 | * 495 | * This is useful to run as an post-deploy script where you specify all the 496 | * live migrations that should be run. 497 | * 498 | * Note: if you start multiple serial migrations, the behavior is: 499 | * - If they don't overlap on functions, they will happily run in parallel. 500 | * - If they have a function in common and one completes before the other 501 | * attempts it, the second will just skip it. 502 | * - If they have a function in common and one is in progress, the second will 503 | * no-op and not run any further migrations in its series. 504 | * 505 | * To stop a migration in progress, see {@link cancelMigration}. 506 | * 507 | * @param ctx Context from a mutation or action. Needs `runMutation`. 508 | * @param fnRefs The migrations to run in order. Like [internal.migrations.foo]. 509 | */ 510 | async runSerially(ctx: RunMutationCtx, fnRefs: MigrationFunctionReference[]) { 511 | if (fnRefs.length === 0) return; 512 | const [fnRef, ...rest] = fnRefs; 513 | const next = await Promise.all( 514 | rest.map(async (fnRef) => ({ 515 | name: getFunctionName(fnRef), 516 | fnHandle: await createFunctionHandle(fnRef), 517 | })) 518 | ); 519 | return ctx.runMutation(this.component.lib.migrate, { 520 | name: getFunctionName(fnRef), 521 | fnHandle: await createFunctionHandle(fnRef), 522 | next, 523 | dryRun: false, 524 | }); 525 | } 526 | 527 | /** 528 | * Get the status of a migration or all migrations. 529 | * @param ctx Context from a query, mutation or action. Needs `runQuery`. 530 | * @param migrations The migrations to get the status of. Defaults to all. 531 | * @param limit How many migrations to fetch, if not specified by name. 532 | * @returns The status of the migrations, in the order of the input. 533 | */ 534 | async getStatus( 535 | ctx: RunQueryCtx, 536 | { 537 | migrations, 538 | limit, 539 | }: { 540 | migrations?: (string | MigrationFunctionReference)[]; 541 | limit?: number; 542 | } 543 | ): Promise { 544 | const names = migrations?.map((m) => 545 | typeof m === "string" ? this.prefixedName(m) : getFunctionName(m) 546 | ); 547 | return ctx.runQuery(this.component.lib.getStatus, { 548 | names, 549 | limit, 550 | }); 551 | } 552 | 553 | /** 554 | * Cancels a migration if it's in progress. 555 | * You can resume it later by calling the migration without an explicit cursor. 556 | * If the migration had "next" migrations, e.g. from {@link runSerially}, 557 | * they will not run. To resume, call the series again or manually pass "next". 558 | * @param ctx Context from a mutation or action. Needs `runMutation`. 559 | * @param migration Migration to cancel. Either the name like "migrations:foo" 560 | * or the function reference like `internal.migrations.foo`. 561 | * @returns The status of the migration after attempting to cancel it. 562 | */ 563 | async cancel( 564 | ctx: RunMutationCtx, 565 | migration: MigrationFunctionReference | string 566 | ): Promise { 567 | const name = 568 | typeof migration === "string" 569 | ? this.prefixedName(migration) 570 | : getFunctionName(migration); 571 | return ctx.runMutation(this.component.lib.cancel, { name }); 572 | } 573 | 574 | /** 575 | * Cancels all migrations that are in progress. 576 | * You can resume it later by calling the migration without an explicit cursor. 577 | * If the migration had "next" migrations, e.g. from {@link runSerially}, 578 | * they will not run. To resume, call the series again or manually pass "next". 579 | * @param ctx Context from a mutation or action. Needs `runMutation`. 580 | * @returns The status of up to 100 of the canceled migrations. 581 | */ 582 | async cancelAll(ctx: RunMutationCtx) { 583 | return ctx.runMutation(this.component.lib.cancelAll, {}); 584 | } 585 | 586 | // Helper to prefix the name with the location. 587 | // migrationsLocationPrefix of "bar/baz:" and name "foo" => "bar/baz:foo" 588 | private prefixedName(name: string) { 589 | return this.options?.migrationsLocationPrefix && !name.includes(":") 590 | ? `${this.options.migrationsLocationPrefix}${name}` 591 | : name; 592 | } 593 | } 594 | 595 | export type MigrationFunctionReference = FunctionReference< 596 | "mutation", 597 | "internal", 598 | MigrationArgs 599 | >; 600 | 601 | /* Type utils follow */ 602 | 603 | type RunQueryCtx = { 604 | runQuery: GenericQueryCtx["runQuery"]; 605 | }; 606 | type RunMutationCtx = { 607 | runMutation: GenericMutationCtx["runMutation"]; 608 | }; 609 | 610 | export type OpaqueIds = 611 | T extends GenericId 612 | ? string 613 | : T extends (infer U)[] 614 | ? OpaqueIds[] 615 | : T extends object 616 | ? { [K in keyof T]: OpaqueIds } 617 | : T; 618 | 619 | export type UseApi = Expand<{ 620 | [mod in keyof API]: API[mod] extends FunctionReference< 621 | infer FType, 622 | "public", 623 | infer FArgs, 624 | infer FReturnType, 625 | infer FComponentPath 626 | > 627 | ? FunctionReference< 628 | FType, 629 | "internal", 630 | OpaqueIds, 631 | OpaqueIds, 632 | FComponentPath 633 | > 634 | : UseApi; 635 | }>; 636 | 637 | function logStatusAndInstructions( 638 | name: string, 639 | status: MigrationStatus, 640 | args: { 641 | fn?: string; 642 | cursor?: string | null; 643 | batchSize?: number; 644 | dryRun?: boolean; 645 | } 646 | ) { 647 | const output: Record = {}; 648 | if (status.isDone) { 649 | if (status.latestEnd! < Date.now()) { 650 | output["Status"] = "Migration already done."; 651 | } else if (status.latestStart === status.latestEnd) { 652 | output["Status"] = "Migration was started and finished in one batch."; 653 | } else { 654 | output["Status"] = "Migration completed with this batch."; 655 | } 656 | } else { 657 | if (status.state === "failed") { 658 | output["Status"] = `Migration failed: ${status.error}`; 659 | } else if (status.state === "canceled") { 660 | output["Status"] = "Migration canceled."; 661 | } else if (status.latestStart >= Date.now()) { 662 | output["Status"] = "Migration started."; 663 | } else { 664 | output["Status"] = "Migration running."; 665 | } 666 | } 667 | if (args.dryRun) { 668 | output["DryRun"] = "No changes were committed."; 669 | output["Status"] = "DRY RUN: " + output["Status"]; 670 | } 671 | output["Name"] = name; 672 | output["lastStarted"] = new Date(status.latestStart).toISOString(); 673 | if (status.latestEnd) { 674 | output["lastFinished"] = new Date(status.latestEnd).toISOString(); 675 | } 676 | output["processed"] = status.processed; 677 | if (status.next?.length) { 678 | if (status.isDone) { 679 | output["nowUp"] = status.next; 680 | } else { 681 | output["nextUp"] = status.next; 682 | } 683 | } 684 | const nextArgs = (status.next || []).map((n) => `"${n}"`).join(", "); 685 | const run = `npx convex run --component migrations`; 686 | if (!args.dryRun) { 687 | if (status.state === "inProgress") { 688 | output["toCancel"] = { 689 | cmd: `${run} lib:cancel`, 690 | args: `{"name": "${name}"}`, 691 | prod: `--prod`, 692 | }; 693 | output["toMonitorStatus"] = { 694 | cmd: `${run} --watch lib:getStatus`, 695 | args: `{"names": ["${name}"${status.next?.length ? ", " + nextArgs : ""}]}`, 696 | prod: `--prod`, 697 | }; 698 | } else { 699 | output["toStartOver"] = JSON.stringify({ ...args, cursor: null }); 700 | if (status.next?.length) { 701 | output["toMonitorStatus"] = { 702 | cmd: `${run} --watch lib:getStatus`, 703 | args: `{"names": [${nextArgs}]}`, 704 | prod: `--prod`, 705 | }; 706 | } 707 | } 708 | } 709 | return output; 710 | } 711 | -------------------------------------------------------------------------------- /src/component/_generated/api.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated `api` utility. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * To regenerate, run `npx convex dev`. 8 | * @module 9 | */ 10 | 11 | import type * as lib from "../lib.js"; 12 | 13 | import type { 14 | ApiFromModules, 15 | FilterApi, 16 | FunctionReference, 17 | } from "convex/server"; 18 | /** 19 | * A utility for referencing Convex functions in your app's API. 20 | * 21 | * Usage: 22 | * ```js 23 | * const myFunctionReference = api.myModule.myFunction; 24 | * ``` 25 | */ 26 | declare const fullApi: ApiFromModules<{ 27 | lib: typeof lib; 28 | }>; 29 | export type Mounts = { 30 | lib: { 31 | cancel: FunctionReference< 32 | "mutation", 33 | "public", 34 | { name: string }, 35 | { 36 | batchSize?: number; 37 | cursor?: string | null; 38 | error?: string; 39 | isDone: boolean; 40 | latestEnd?: number; 41 | latestStart: number; 42 | name: string; 43 | next?: Array; 44 | processed: number; 45 | state: "inProgress" | "success" | "failed" | "canceled" | "unknown"; 46 | } 47 | >; 48 | cancelAll: FunctionReference< 49 | "mutation", 50 | "public", 51 | { sinceTs?: number }, 52 | Array<{ 53 | batchSize?: number; 54 | cursor?: string | null; 55 | error?: string; 56 | isDone: boolean; 57 | latestEnd?: number; 58 | latestStart: number; 59 | name: string; 60 | next?: Array; 61 | processed: number; 62 | state: "inProgress" | "success" | "failed" | "canceled" | "unknown"; 63 | }> 64 | >; 65 | clearAll: FunctionReference< 66 | "mutation", 67 | "public", 68 | { before?: number }, 69 | null 70 | >; 71 | getStatus: FunctionReference< 72 | "query", 73 | "public", 74 | { limit?: number; names?: Array }, 75 | Array<{ 76 | batchSize?: number; 77 | cursor?: string | null; 78 | error?: string; 79 | isDone: boolean; 80 | latestEnd?: number; 81 | latestStart: number; 82 | name: string; 83 | next?: Array; 84 | processed: number; 85 | state: "inProgress" | "success" | "failed" | "canceled" | "unknown"; 86 | }> 87 | >; 88 | migrate: FunctionReference< 89 | "mutation", 90 | "public", 91 | { 92 | batchSize?: number; 93 | cursor?: string | null; 94 | dryRun: boolean; 95 | fnHandle: string; 96 | name: string; 97 | next?: Array<{ fnHandle: string; name: string }>; 98 | }, 99 | { 100 | batchSize?: number; 101 | cursor?: string | null; 102 | error?: string; 103 | isDone: boolean; 104 | latestEnd?: number; 105 | latestStart: number; 106 | name: string; 107 | next?: Array; 108 | processed: number; 109 | state: "inProgress" | "success" | "failed" | "canceled" | "unknown"; 110 | } 111 | >; 112 | }; 113 | }; 114 | // For now fullApiWithMounts is only fullApi which provides 115 | // jump-to-definition in component client code. 116 | // Use Mounts for the same type without the inference. 117 | declare const fullApiWithMounts: typeof fullApi; 118 | 119 | export declare const api: FilterApi< 120 | typeof fullApiWithMounts, 121 | FunctionReference 122 | >; 123 | export declare const internal: FilterApi< 124 | typeof fullApiWithMounts, 125 | FunctionReference 126 | >; 127 | 128 | export declare const components: {}; 129 | -------------------------------------------------------------------------------- /src/component/_generated/api.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated `api` utility. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * To regenerate, run `npx convex dev`. 8 | * @module 9 | */ 10 | 11 | import { anyApi, componentsGeneric } from "convex/server"; 12 | 13 | /** 14 | * A utility for referencing Convex functions in your app's API. 15 | * 16 | * Usage: 17 | * ```js 18 | * const myFunctionReference = api.myModule.myFunction; 19 | * ``` 20 | */ 21 | export const api = anyApi; 22 | export const internal = anyApi; 23 | export const components = componentsGeneric(); 24 | -------------------------------------------------------------------------------- /src/component/_generated/dataModel.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated data model types. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * To regenerate, run `npx convex dev`. 8 | * @module 9 | */ 10 | 11 | import type { 12 | DataModelFromSchemaDefinition, 13 | DocumentByName, 14 | TableNamesInDataModel, 15 | SystemTableNames, 16 | } from "convex/server"; 17 | import type { GenericId } from "convex/values"; 18 | import schema from "../schema.js"; 19 | 20 | /** 21 | * The names of all of your Convex tables. 22 | */ 23 | export type TableNames = TableNamesInDataModel; 24 | 25 | /** 26 | * The type of a document stored in Convex. 27 | * 28 | * @typeParam TableName - A string literal type of the table name (like "users"). 29 | */ 30 | export type Doc = DocumentByName< 31 | DataModel, 32 | TableName 33 | >; 34 | 35 | /** 36 | * An identifier for a document in Convex. 37 | * 38 | * Convex documents are uniquely identified by their `Id`, which is accessible 39 | * on the `_id` field. To learn more, see [Document IDs](https://docs.convex.dev/using/document-ids). 40 | * 41 | * Documents can be loaded using `db.get(id)` in query and mutation functions. 42 | * 43 | * IDs are just strings at runtime, but this type can be used to distinguish them from other 44 | * strings when type checking. 45 | * 46 | * @typeParam TableName - A string literal type of the table name (like "users"). 47 | */ 48 | export type Id = 49 | GenericId; 50 | 51 | /** 52 | * A type describing your Convex data model. 53 | * 54 | * This type includes information about what tables you have, the type of 55 | * documents stored in those tables, and the indexes defined on them. 56 | * 57 | * This type is used to parameterize methods like `queryGeneric` and 58 | * `mutationGeneric` to make them type-safe. 59 | */ 60 | export type DataModel = DataModelFromSchemaDefinition; 61 | -------------------------------------------------------------------------------- /src/component/_generated/server.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated utilities for implementing server-side Convex query and mutation functions. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * To regenerate, run `npx convex dev`. 8 | * @module 9 | */ 10 | 11 | import { 12 | ActionBuilder, 13 | AnyComponents, 14 | HttpActionBuilder, 15 | MutationBuilder, 16 | QueryBuilder, 17 | GenericActionCtx, 18 | GenericMutationCtx, 19 | GenericQueryCtx, 20 | GenericDatabaseReader, 21 | GenericDatabaseWriter, 22 | FunctionReference, 23 | } from "convex/server"; 24 | import type { DataModel } from "./dataModel.js"; 25 | 26 | type GenericCtx = 27 | | GenericActionCtx 28 | | GenericMutationCtx 29 | | GenericQueryCtx; 30 | 31 | /** 32 | * Define a query in this Convex app's public API. 33 | * 34 | * This function will be allowed to read your Convex database and will be accessible from the client. 35 | * 36 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 37 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 38 | */ 39 | export declare const query: QueryBuilder; 40 | 41 | /** 42 | * Define a query that is only accessible from other Convex functions (but not from the client). 43 | * 44 | * This function will be allowed to read from your Convex database. It will not be accessible from the client. 45 | * 46 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 47 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 48 | */ 49 | export declare const internalQuery: QueryBuilder; 50 | 51 | /** 52 | * Define a mutation in this Convex app's public API. 53 | * 54 | * This function will be allowed to modify your Convex database and will be accessible from the client. 55 | * 56 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 57 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 58 | */ 59 | export declare const mutation: MutationBuilder; 60 | 61 | /** 62 | * Define a mutation that is only accessible from other Convex functions (but not from the client). 63 | * 64 | * This function will be allowed to modify your Convex database. It will not be accessible from the client. 65 | * 66 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 67 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 68 | */ 69 | export declare const internalMutation: MutationBuilder; 70 | 71 | /** 72 | * Define an action in this Convex app's public API. 73 | * 74 | * An action is a function which can execute any JavaScript code, including non-deterministic 75 | * code and code with side-effects, like calling third-party services. 76 | * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. 77 | * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. 78 | * 79 | * @param func - The action. It receives an {@link ActionCtx} as its first argument. 80 | * @returns The wrapped action. Include this as an `export` to name it and make it accessible. 81 | */ 82 | export declare const action: ActionBuilder; 83 | 84 | /** 85 | * Define an action that is only accessible from other Convex functions (but not from the client). 86 | * 87 | * @param func - The function. It receives an {@link ActionCtx} as its first argument. 88 | * @returns The wrapped function. Include this as an `export` to name it and make it accessible. 89 | */ 90 | export declare const internalAction: ActionBuilder; 91 | 92 | /** 93 | * Define an HTTP action. 94 | * 95 | * This function will be used to respond to HTTP requests received by a Convex 96 | * deployment if the requests matches the path and method where this action 97 | * is routed. Be sure to route your action in `convex/http.js`. 98 | * 99 | * @param func - The function. It receives an {@link ActionCtx} as its first argument. 100 | * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. 101 | */ 102 | export declare const httpAction: HttpActionBuilder; 103 | 104 | /** 105 | * A set of services for use within Convex query functions. 106 | * 107 | * The query context is passed as the first argument to any Convex query 108 | * function run on the server. 109 | * 110 | * This differs from the {@link MutationCtx} because all of the services are 111 | * read-only. 112 | */ 113 | export type QueryCtx = GenericQueryCtx; 114 | 115 | /** 116 | * A set of services for use within Convex mutation functions. 117 | * 118 | * The mutation context is passed as the first argument to any Convex mutation 119 | * function run on the server. 120 | */ 121 | export type MutationCtx = GenericMutationCtx; 122 | 123 | /** 124 | * A set of services for use within Convex action functions. 125 | * 126 | * The action context is passed as the first argument to any Convex action 127 | * function run on the server. 128 | */ 129 | export type ActionCtx = GenericActionCtx; 130 | 131 | /** 132 | * An interface to read from the database within Convex query functions. 133 | * 134 | * The two entry points are {@link DatabaseReader.get}, which fetches a single 135 | * document by its {@link Id}, or {@link DatabaseReader.query}, which starts 136 | * building a query. 137 | */ 138 | export type DatabaseReader = GenericDatabaseReader; 139 | 140 | /** 141 | * An interface to read from and write to the database within Convex mutation 142 | * functions. 143 | * 144 | * Convex guarantees that all writes within a single mutation are 145 | * executed atomically, so you never have to worry about partial writes leaving 146 | * your data in an inconsistent state. See [the Convex Guide](https://docs.convex.dev/understanding/convex-fundamentals/functions#atomicity-and-optimistic-concurrency-control) 147 | * for the guarantees Convex provides your functions. 148 | */ 149 | export type DatabaseWriter = GenericDatabaseWriter; 150 | -------------------------------------------------------------------------------- /src/component/_generated/server.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * Generated utilities for implementing server-side Convex query and mutation functions. 4 | * 5 | * THIS CODE IS AUTOMATICALLY GENERATED. 6 | * 7 | * To regenerate, run `npx convex dev`. 8 | * @module 9 | */ 10 | 11 | import { 12 | actionGeneric, 13 | httpActionGeneric, 14 | queryGeneric, 15 | mutationGeneric, 16 | internalActionGeneric, 17 | internalMutationGeneric, 18 | internalQueryGeneric, 19 | componentsGeneric, 20 | } from "convex/server"; 21 | 22 | /** 23 | * Define a query in this Convex app's public API. 24 | * 25 | * This function will be allowed to read your Convex database and will be accessible from the client. 26 | * 27 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 28 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 29 | */ 30 | export const query = queryGeneric; 31 | 32 | /** 33 | * Define a query that is only accessible from other Convex functions (but not from the client). 34 | * 35 | * This function will be allowed to read from your Convex database. It will not be accessible from the client. 36 | * 37 | * @param func - The query function. It receives a {@link QueryCtx} as its first argument. 38 | * @returns The wrapped query. Include this as an `export` to name it and make it accessible. 39 | */ 40 | export const internalQuery = internalQueryGeneric; 41 | 42 | /** 43 | * Define a mutation in this Convex app's public API. 44 | * 45 | * This function will be allowed to modify your Convex database and will be accessible from the client. 46 | * 47 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 48 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 49 | */ 50 | export const mutation = mutationGeneric; 51 | 52 | /** 53 | * Define a mutation that is only accessible from other Convex functions (but not from the client). 54 | * 55 | * This function will be allowed to modify your Convex database. It will not be accessible from the client. 56 | * 57 | * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. 58 | * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. 59 | */ 60 | export const internalMutation = internalMutationGeneric; 61 | 62 | /** 63 | * Define an action in this Convex app's public API. 64 | * 65 | * An action is a function which can execute any JavaScript code, including non-deterministic 66 | * code and code with side-effects, like calling third-party services. 67 | * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. 68 | * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. 69 | * 70 | * @param func - The action. It receives an {@link ActionCtx} as its first argument. 71 | * @returns The wrapped action. Include this as an `export` to name it and make it accessible. 72 | */ 73 | export const action = actionGeneric; 74 | 75 | /** 76 | * Define an action that is only accessible from other Convex functions (but not from the client). 77 | * 78 | * @param func - The function. It receives an {@link ActionCtx} as its first argument. 79 | * @returns The wrapped function. Include this as an `export` to name it and make it accessible. 80 | */ 81 | export const internalAction = internalActionGeneric; 82 | 83 | /** 84 | * Define a Convex HTTP action. 85 | * 86 | * @param func - The function. It receives an {@link ActionCtx} as its first argument, and a `Request` object 87 | * as its second. 88 | * @returns The wrapped endpoint function. Route a URL path to this function in `convex/http.js`. 89 | */ 90 | export const httpAction = httpActionGeneric; 91 | -------------------------------------------------------------------------------- /src/component/convex.config.ts: -------------------------------------------------------------------------------- 1 | import { defineComponent } from "convex/server"; 2 | 3 | export default defineComponent("migrations"); 4 | -------------------------------------------------------------------------------- /src/component/lib.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, test, expect } from "vitest"; 2 | import { ApiFromModules, anyApi, createFunctionHandle } from "convex/server"; 3 | import { convexTest } from "convex-test"; 4 | import { modules } from "./setup.test.js"; 5 | import { api } from "./_generated/api.js"; 6 | import { MigrationArgs, MigrationResult } from "../client/index.js"; 7 | import { mutation } from "./_generated/server.js"; 8 | import schema from "./schema.js"; 9 | 10 | export const doneMigration = mutation({ 11 | handler: async (_, _args: MigrationArgs): Promise => { 12 | return { 13 | isDone: true, 14 | continueCursor: "foo", 15 | processed: 1, 16 | }; 17 | }, 18 | }); 19 | 20 | const testApi: ApiFromModules<{ 21 | fns: { doneMigration: typeof doneMigration }; 22 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 23 | }>["fns"] = anyApi["lib.test"] as any; 24 | 25 | describe("migrate", () => { 26 | test("runs a simple migration in one go", async () => { 27 | const t = convexTest(schema, modules); 28 | const fnHandle = await createFunctionHandle(testApi.doneMigration); 29 | const result = await t.mutation(api.lib.migrate, { 30 | name: "testMigration", 31 | fnHandle: fnHandle, 32 | dryRun: false, 33 | }); 34 | expect(result.isDone).toBe(true); 35 | expect(result.cursor).toBe("foo"); 36 | expect(result.processed).toBe(1); 37 | expect(result.error).toBeUndefined(); 38 | expect(result.batchSize).toBeUndefined(); 39 | expect(result.next).toBeUndefined(); 40 | expect(result.latestEnd).toBeTypeOf("number"); 41 | expect(result.state).toBe("success"); 42 | }); 43 | 44 | test("throws error for batchSize <= 0", async () => { 45 | const args = { 46 | name: "testMigration", 47 | fnHandle: "function://dummy", 48 | cursor: null, 49 | batchSize: 0, 50 | next: [], 51 | dryRun: false, 52 | }; 53 | const t = convexTest(schema, modules); 54 | // Assumes testApi has shape matching api.lib – adjust per actual ConvexTest usage 55 | await expect(t.mutation(api.lib.migrate, args)).rejects.toThrow( 56 | "Batch size must be greater than 0" 57 | ); 58 | }); 59 | 60 | test("throws error for invalid fnHandle", async () => { 61 | const args = { 62 | name: "testMigration", 63 | fnHandle: "invalid_handle", 64 | cursor: null, 65 | batchSize: 10, 66 | next: [], 67 | dryRun: false, 68 | }; 69 | const t = convexTest(schema, modules); 70 | await expect(t.mutation(api.lib.migrate, args)).rejects.toThrow( 71 | "Invalid fnHandle" 72 | ); 73 | }); 74 | }); 75 | 76 | describe("cancel", () => { 77 | test("throws error if migration not found", async () => { 78 | // For cancel, ConvexTest-like patterns would be similar – this code demonstrates minimal direct call 79 | const t = convexTest(schema, modules); 80 | await expect( 81 | t.mutation(api.lib.cancel, { name: "nonexistent" }) 82 | ).rejects.toThrow(); 83 | }); 84 | }); 85 | 86 | describe("It doesn't attempt a migration if it's already done", () => { 87 | test("runs a simple migration in one go", async () => { 88 | const t = convexTest(schema, modules); 89 | const fnHandle = "function://invalid"; 90 | await t.run((ctx) => 91 | ctx.db.insert("migrations", { 92 | name: "testMigration", 93 | latestStart: Date.now(), 94 | isDone: true, 95 | cursor: "foo", 96 | processed: 1, 97 | }) 98 | ); 99 | // It'd throw if it tried to run the migration. 100 | const result = await t.mutation(api.lib.migrate, { 101 | name: "testMigration", 102 | fnHandle: fnHandle, 103 | dryRun: false, 104 | }); 105 | expect(result.isDone).toBe(true); 106 | }); 107 | }); 108 | -------------------------------------------------------------------------------- /src/component/lib.ts: -------------------------------------------------------------------------------- 1 | import { ConvexError, ObjectType, v } from "convex/values"; 2 | import { mutation, MutationCtx, query, QueryCtx } from "./_generated/server.js"; 3 | import { FunctionHandle, WithoutSystemFields } from "convex/server"; 4 | import { 5 | MigrationArgs, 6 | MigrationResult, 7 | MigrationStatus, 8 | migrationStatus, 9 | } from "../shared.js"; 10 | import { api } from "./_generated/api.js"; 11 | import { Doc } from "./_generated/dataModel.js"; 12 | 13 | export type MigrationFunctionHandle = FunctionHandle< 14 | "mutation", 15 | MigrationArgs, 16 | MigrationResult 17 | >; 18 | 19 | const runMigrationArgs = { 20 | name: v.string(), 21 | fnHandle: v.string(), 22 | cursor: v.optional(v.union(v.string(), v.null())), 23 | 24 | batchSize: v.optional(v.number()), 25 | next: v.optional( 26 | v.array( 27 | v.object({ 28 | name: v.string(), 29 | fnHandle: v.string(), 30 | }) 31 | ) 32 | ), 33 | dryRun: v.boolean(), 34 | }; 35 | 36 | export const migrate = mutation({ 37 | args: runMigrationArgs, 38 | returns: migrationStatus, 39 | handler: async (ctx, args) => { 40 | // Step 1: Get or create the state. 41 | const { fnHandle, batchSize, next: next_, dryRun, ...initialState } = args; 42 | if (batchSize !== undefined && batchSize <= 0) { 43 | throw new Error("Batch size must be greater than 0"); 44 | } 45 | if (!fnHandle.startsWith("function://")) { 46 | throw new Error( 47 | "Invalid fnHandle.\n" + 48 | "Do not call this from the CLI or dashboard directly.\n" + 49 | "Instead use the `migrations.runner` function to run migrations." + 50 | "See https://www.convex.dev/components/migrations" 51 | ); 52 | } 53 | const state = 54 | (await ctx.db 55 | .query("migrations") 56 | .withIndex("name", (q) => q.eq("name", args.name)) 57 | .unique()) ?? 58 | (await ctx.db.get( 59 | await ctx.db.insert("migrations", { 60 | ...initialState, 61 | cursor: args.cursor ?? null, 62 | isDone: false, 63 | processed: 0, 64 | latestStart: Date.now(), 65 | }) 66 | ))!; 67 | 68 | // Update the state if the cursor arg differs. 69 | if (state.cursor !== args.cursor) { 70 | // This happens if: 71 | // 1. The migration is being started/resumed (args.cursor unset). 72 | // 2. The migration is being resumed at a different cursor. 73 | // 3. There are two instances of the same migration racing. 74 | const worker = 75 | state.workerId && (await ctx.db.system.get(state.workerId)); 76 | if ( 77 | worker && 78 | (worker.state.kind === "pending" || worker.state.kind === "inProgress") 79 | ) { 80 | // Case 3. The migration is already in progress. 81 | console.debug({ state, worker }); 82 | return getMigrationState(ctx, state); 83 | } 84 | // Case 2. Update the cursor. 85 | if (args.cursor !== undefined) { 86 | state.cursor = args.cursor; 87 | state.isDone = false; 88 | state.latestStart = Date.now(); 89 | state.latestEnd = undefined; 90 | state.processed = 0; 91 | } 92 | // For Case 1, Step 2 will take the right action. 93 | } 94 | 95 | function updateState(result: MigrationResult) { 96 | state.cursor = result.continueCursor; 97 | state.isDone = result.isDone; 98 | state.processed += result.processed; 99 | if (result.isDone && state.latestEnd === undefined) { 100 | state.latestEnd = Date.now(); 101 | } 102 | } 103 | 104 | try { 105 | // Step 2: Run the migration. 106 | if (!state.isDone) { 107 | const result = await ctx.runMutation( 108 | fnHandle as MigrationFunctionHandle, 109 | { 110 | cursor: state.cursor, 111 | batchSize, 112 | dryRun, 113 | } 114 | ); 115 | updateState(result); 116 | state.error = undefined; 117 | } 118 | 119 | // Step 3: Schedule the next batch or next migration. 120 | if (!state.isDone) { 121 | // Recursively schedule the next batch. 122 | state.workerId = await ctx.scheduler.runAfter(0, api.lib.migrate, { 123 | ...args, 124 | cursor: state.cursor, 125 | }); 126 | } else { 127 | state.workerId = undefined; 128 | // Schedule the next migration in the series. 129 | const next = next_ ?? []; 130 | // Find the next migration that hasn't been done. 131 | let i = 0; 132 | for (; i < next.length; i++) { 133 | const doc = await ctx.db 134 | .query("migrations") 135 | .withIndex("name", (q) => q.eq("name", next[i]!.name)) 136 | .unique(); 137 | if (!doc || !doc.isDone) { 138 | const [nextFn, ...rest] = next.slice(i); 139 | if (nextFn) { 140 | await ctx.scheduler.runAfter(0, api.lib.migrate, { 141 | name: nextFn.name, 142 | fnHandle: nextFn.fnHandle, 143 | next: rest, 144 | batchSize, 145 | dryRun, 146 | }); 147 | } 148 | break; 149 | } 150 | } 151 | if (args.cursor === undefined) { 152 | if (next.length && i === next.length) { 153 | console.info(`Migration${i > 0 ? "s" : ""} up next already done.`); 154 | } 155 | } else { 156 | console.info( 157 | `Migration ${args.name} is done.` + 158 | (i < next.length ? ` Next: ${next[i]!.name}` : "") 159 | ); 160 | } 161 | } 162 | } catch (e) { 163 | state.workerId = undefined; 164 | if (dryRun && e instanceof ConvexError && e.data.kind === "DRY RUN") { 165 | // Add the state to the error to bubble up. 166 | updateState(e.data.result); 167 | } else { 168 | state.error = e instanceof Error ? e.message : String(e); 169 | console.error(`Migration ${args.name} failed: ${state.error}`); 170 | } 171 | if (dryRun) { 172 | const status = await getMigrationState(ctx, state); 173 | status.batchSize = batchSize; 174 | status.next = next_?.map((n) => n.name); 175 | throw new ConvexError({ 176 | kind: "DRY RUN", 177 | status, 178 | }); 179 | } 180 | } 181 | 182 | // Step 4: Update the state 183 | await ctx.db.patch(state._id, state); 184 | if (args.dryRun) { 185 | // By throwing an error, the transaction will be rolled back and nothing 186 | // will be scheduled. 187 | console.debug({ args, state }); 188 | throw new Error( 189 | "Error: Dry run attempted to update state - rolling back transaction." 190 | ); 191 | } 192 | return getMigrationState(ctx, state); 193 | }, 194 | }); 195 | 196 | export const getStatus = query({ 197 | args: { 198 | names: v.optional(v.array(v.string())), 199 | limit: v.optional(v.number()), 200 | }, 201 | returns: v.array(migrationStatus), 202 | handler: async (ctx, args) => { 203 | const docs = args.names 204 | ? await Promise.all( 205 | args.names.map( 206 | async (m) => 207 | (await ctx.db 208 | .query("migrations") 209 | .withIndex("name", (q) => q.eq("name", m)) 210 | .unique()) ?? { 211 | name: m, 212 | processed: 0, 213 | cursor: null, 214 | latestStart: 0, 215 | workerId: undefined, 216 | isDone: false as const, 217 | } 218 | ) 219 | ) 220 | : await ctx.db 221 | .query("migrations") 222 | .order("desc") 223 | .take(args.limit ?? 10); 224 | 225 | return Promise.all( 226 | docs.reverse().map(async (migration) => getMigrationState(ctx, migration)) 227 | ); 228 | }, 229 | }); 230 | 231 | async function getMigrationState( 232 | ctx: QueryCtx, 233 | migration: WithoutSystemFields> 234 | ): Promise { 235 | const worker = 236 | migration.workerId && (await ctx.db.system.get(migration.workerId)); 237 | const args = worker?.args[0] as 238 | | ObjectType 239 | | undefined; 240 | const state = migration.isDone 241 | ? "success" 242 | : migration.error || worker?.state.kind === "failed" 243 | ? "failed" 244 | : worker?.state.kind === "canceled" 245 | ? "canceled" 246 | : worker?.state.kind === "inProgress" || 247 | worker?.state.kind === "pending" 248 | ? "inProgress" 249 | : "unknown"; 250 | return { 251 | name: migration.name, 252 | cursor: migration.cursor, 253 | processed: migration.processed, 254 | isDone: migration.isDone, 255 | latestStart: migration.latestStart, 256 | latestEnd: migration.latestEnd, 257 | error: migration.error, 258 | state, 259 | batchSize: args?.batchSize, 260 | next: args?.next?.map((n: { name: string }) => n.name), 261 | }; 262 | } 263 | 264 | export const cancel = mutation({ 265 | args: { name: v.string() }, 266 | returns: migrationStatus, 267 | handler: async (ctx, args) => { 268 | const migration = await ctx.db 269 | .query("migrations") 270 | .withIndex("name", (q) => q.eq("name", args.name)) 271 | .unique(); 272 | 273 | if (!migration) { 274 | throw new Error(`Migration ${args.name} not found`); 275 | } 276 | const state = await cancelMigration(ctx, migration); 277 | if (state.state !== "canceled") { 278 | console.log( 279 | `Did not cancel migration ${migration.name}. Status was ${state.state}` 280 | ); 281 | } 282 | return state; 283 | }, 284 | }); 285 | 286 | async function cancelMigration(ctx: MutationCtx, migration: Doc<"migrations">) { 287 | const state = await getMigrationState(ctx, migration); 288 | if (state.isDone) { 289 | return state; 290 | } 291 | if (state.state === "inProgress") { 292 | await ctx.scheduler.cancel(migration.workerId!); 293 | console.log(`Canceled migration ${migration.name}`); 294 | return { ...state, state: "canceled" as const }; 295 | } 296 | return state; 297 | } 298 | 299 | export const cancelAll = mutation({ 300 | // Paginating with creation time for now 301 | args: { sinceTs: v.optional(v.number()) }, 302 | returns: v.array(migrationStatus), 303 | handler: async (ctx, args) => { 304 | const results = await ctx.db 305 | .query("migrations") 306 | .withIndex("isDone", (q) => 307 | args.sinceTs 308 | ? q.eq("isDone", false).gte("_creationTime", args.sinceTs) 309 | : q.eq("isDone", false) 310 | ) 311 | .take(100); 312 | if (results.length === 100) { 313 | await ctx.scheduler.runAfter(0, api.lib.cancelAll, { 314 | sinceTs: results[results.length - 1]!._creationTime, 315 | }); 316 | } 317 | return Promise.all(results.map((m) => cancelMigration(ctx, m))); 318 | }, 319 | }); 320 | 321 | export const clearAll = mutation({ 322 | args: { before: v.optional(v.number()) }, 323 | returns: v.null(), 324 | handler: async (ctx, args) => { 325 | const results = await ctx.db 326 | .query("migrations") 327 | .withIndex("by_creation_time", (q) => 328 | q.lte("_creationTime", args.before ?? Date.now()) 329 | ) 330 | .order("desc") 331 | .take(100); 332 | for (const m of results) { 333 | await ctx.db.delete(m._id); 334 | } 335 | if (results.length === 100) { 336 | await ctx.scheduler.runAfter(0, api.lib.clearAll, { 337 | before: results[99]._creationTime, 338 | }); 339 | } 340 | }, 341 | }); 342 | -------------------------------------------------------------------------------- /src/component/schema.ts: -------------------------------------------------------------------------------- 1 | import { defineSchema, defineTable } from "convex/server"; 2 | import { v } from "convex/values"; 3 | 4 | export default defineSchema({ 5 | migrations: defineTable({ 6 | name: v.string(), // Defaults to the function name. 7 | cursor: v.union(v.string(), v.null()), 8 | isDone: v.boolean(), 9 | workerId: v.optional(v.id("_scheduled_functions")), 10 | error: v.optional(v.string()), 11 | // The number of documents processed so far. 12 | processed: v.number(), 13 | latestStart: v.number(), 14 | latestEnd: v.optional(v.number()), 15 | }) 16 | .index("name", ["name"]) 17 | .index("isDone", ["isDone"]), 18 | }); 19 | -------------------------------------------------------------------------------- /src/component/setup.test.ts: -------------------------------------------------------------------------------- 1 | /// 2 | import { test } from "vitest"; 3 | export const modules = import.meta.glob("./**/*.*s"); 4 | 5 | test("setup", () => {}); 6 | -------------------------------------------------------------------------------- /src/shared.ts: -------------------------------------------------------------------------------- 1 | import { Infer, ObjectType, v } from "convex/values"; 2 | 3 | export const migrationArgs = { 4 | fn: v.optional(v.string()), 5 | cursor: v.optional(v.union(v.string(), v.null())), 6 | batchSize: v.optional(v.number()), 7 | dryRun: v.optional(v.boolean()), 8 | next: v.optional(v.array(v.string())), 9 | }; 10 | export type MigrationArgs = ObjectType; 11 | 12 | export type MigrationResult = { 13 | continueCursor: string; 14 | isDone: boolean; 15 | processed: number; 16 | }; 17 | 18 | export const migrationStatus = v.object({ 19 | name: v.string(), 20 | cursor: v.optional(v.union(v.string(), v.null())), 21 | processed: v.number(), 22 | isDone: v.boolean(), 23 | error: v.optional(v.string()), 24 | state: v.union( 25 | v.literal("inProgress"), 26 | v.literal("success"), 27 | v.literal("failed"), 28 | v.literal("canceled"), 29 | v.literal("unknown") 30 | ), 31 | latestStart: v.number(), 32 | latestEnd: v.optional(v.number()), 33 | batchSize: v.optional(v.number()), 34 | next: v.optional(v.array(v.string())), 35 | }); 36 | export type MigrationStatus = Infer; 37 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowJs": true, 4 | "checkJs": true, 5 | "strict": true, 6 | 7 | "target": "ESNext", 8 | "lib": ["ES2021", "dom"], 9 | "forceConsistentCasingInFileNames": true, 10 | "allowSyntheticDefaultImports": true, 11 | "module": "NodeNext", 12 | "moduleResolution": "NodeNext", 13 | 14 | "isolatedModules": true, 15 | "declaration": true, 16 | "declarationMap": true, 17 | "sourceMap": true, 18 | "outDir": "./dist", 19 | "skipLibCheck": true 20 | }, 21 | "exclude": ["vitest.config.ts"], 22 | "include": ["./src/**/*"] 23 | } 24 | -------------------------------------------------------------------------------- /tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "types": ["vitest/globals"], 5 | "module": "NodeNext", 6 | "moduleResolution": "NodeNext" 7 | }, 8 | "include": ["src/**/*", "example/**/*"] 9 | } 10 | -------------------------------------------------------------------------------- /vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vitest/config"; 2 | 3 | export default defineConfig({ 4 | test: { 5 | globals: true, 6 | environment: "node", 7 | typecheck: { 8 | tsconfig: "./tsconfig.test.json", 9 | }, 10 | }, 11 | }); 12 | --------------------------------------------------------------------------------