├── .eslintrc.js ├── .github └── workflows │ ├── ci.yml │ ├── publish-docs.yml │ └── publish-package.yml ├── .gitignore ├── .prettierrc.json ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── __tests__ ├── app.ts ├── types │ ├── DefaultMigrator.test-d.ts │ ├── Traverser.test-d.ts │ ├── _helpers.ts │ ├── createBatchMigrator.test-d.ts │ ├── createMigrator.test-d.ts │ └── createTraverser.test-d.ts └── utils │ ├── collectionPopulator.ts │ ├── getLexicographicallyNextString.test.ts │ ├── getLexicographicallyNextString.ts │ ├── index.ts │ └── writeFileAndMkdirSync.ts ├── babel.config.js ├── examples ├── addNewField.ts ├── addNewFieldFromPrevFields.ts ├── changeTraversalConfig.ts ├── exitEarlyPredicate.ts ├── fastMigrator.ts ├── fastTraverser.ts ├── migrationPredicate.ts ├── quickStart.ts └── renameField.ts ├── jest.config.global.ts ├── jest.config.ts ├── package-lock.json ├── package.json ├── scripts └── generate-docs.sh ├── src ├── api │ ├── createBatchMigrator.ts │ ├── createMigrator.ts │ ├── createTraverser.ts │ ├── index.ts │ └── interfaces │ │ ├── BatchCallback.ts │ │ ├── BatchMigrator.ts │ │ ├── DefaultMigrator.ts │ │ ├── ExitEarlyPredicate.ts │ │ ├── MigrationPredicate.ts │ │ ├── MigrationResult.ts │ │ ├── Migrator.ts │ │ ├── SetDataGetter.ts │ │ ├── SetOptions.ts │ │ ├── SetPartialDataGetter.ts │ │ ├── Traversable.ts │ │ ├── TraversalConfig.ts │ │ ├── TraversalResult.ts │ │ ├── TraverseEachCallback.ts │ │ ├── TraverseEachConfig.ts │ │ ├── Traverser.ts │ │ ├── UpdateDataGetter.ts │ │ ├── UpdateFieldValueGetter.ts │ │ └── index.ts ├── errors │ ├── ImplementationError.ts │ ├── InvalidConfigError.ts │ └── index.ts ├── index.ts └── internal │ ├── ds │ ├── PromiseQueue.ts │ ├── SLLQueueExtended.ts │ ├── __tests__ │ │ ├── PromiseQueue.test.ts │ │ └── SLLQueueExtended.test.ts │ └── index.ts │ ├── errors │ ├── IllegalArgumentError.ts │ └── index.ts │ ├── implementations │ ├── BasicBatchMigratorImpl.ts │ ├── BasicDefaultMigratorImpl.ts │ ├── PromiseQueueBasedTraverserImpl.ts │ ├── __tests__ │ │ ├── Migrator │ │ │ ├── BasicBatchMigratorImpl │ │ │ │ ├── deleteField.test.ts │ │ │ │ ├── deleteFields.test.ts │ │ │ │ ├── helpers.ts │ │ │ │ ├── renameField.test.ts │ │ │ │ ├── renameFields.test.ts │ │ │ │ ├── update.test.ts │ │ │ │ └── updateWithDerivedData.test.ts │ │ │ ├── BasicDefaultMigratorImpl │ │ │ │ ├── deleteField.test.ts │ │ │ │ ├── deleteFields.test.ts │ │ │ │ ├── helpers.ts │ │ │ │ ├── renameField.test.ts │ │ │ │ ├── renameFields.test.ts │ │ │ │ ├── update.test.ts │ │ │ │ └── updateWithDerivedData.test.ts │ │ │ ├── config.ts │ │ │ ├── helpers.ts │ │ │ └── shared │ │ │ │ ├── deleteField.ts │ │ │ │ ├── deleteFields.ts │ │ │ │ ├── renameField.ts │ │ │ │ ├── renameFields.ts │ │ │ │ ├── update.ts │ │ │ │ └── updateWithDerivedData.ts │ │ └── Traverser │ │ │ ├── PromiseQueueBasedTraverserImpl │ │ │ ├── helpers.ts │ │ │ ├── traverse.test.ts │ │ │ └── withExitEarlyPredicate.test.ts │ │ │ ├── config.ts │ │ │ ├── helpers.ts │ │ │ └── shared │ │ │ ├── traverse.ts │ │ │ └── withExitEarlyPredicate.ts │ ├── abstract │ │ ├── AbstractMigrator.ts │ │ ├── AbstractTraverser.ts │ │ └── index.ts │ └── index.ts │ └── utils │ ├── __tests__ │ └── isPositiveInteger.test.ts │ ├── index.ts │ ├── isTraverser.ts │ ├── math.ts │ └── object.ts ├── tsconfig.build.json ├── tsconfig.json ├── tsconfig.test.json └── typedoc.json /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | es2021: true, 4 | node: true, 5 | }, 6 | extends: [ 7 | 'eslint:recommended', 8 | 'plugin:@typescript-eslint/recommended', 9 | 'plugin:import/errors', 10 | 'plugin:import/warnings', 11 | 'plugin:prettier/recommended', 12 | 'prettier', 13 | ], 14 | parser: '@typescript-eslint/parser', 15 | parserOptions: { 16 | ecmaVersion: 12, 17 | sourceType: 'module', 18 | }, 19 | plugins: ['@typescript-eslint', 'import', 'prettier'], 20 | rules: { 21 | '@typescript-eslint/explicit-function-return-type': [ 22 | 2, 23 | { 24 | allowExpressions: true, 25 | }, 26 | ], 27 | '@typescript-eslint/explicit-module-boundary-types': 0, 28 | '@typescript-eslint/no-empty-function': 0, 29 | '@typescript-eslint/no-explicit-any': 0, 30 | '@typescript-eslint/no-non-null-assertion': 0, 31 | 'no-constant-condition': 0, 32 | }, 33 | settings: { 34 | 'import/resolver': { 35 | node: { 36 | extensions: ['.json', '.js', '.ts'], 37 | }, 38 | }, 39 | }, 40 | }; 41 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [pull_request] 4 | 5 | concurrency: 6 | group: ${{ github.ref }} 7 | cancel-in-progress: true 8 | 9 | jobs: 10 | validate: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Use Node.js 15 | uses: actions/setup-node@v4 16 | with: 17 | node-version: '22.x' 18 | cache: npm 19 | - name: Install project dependencies 20 | run: npm ci 21 | - name: Build the source 22 | run: npm run build 23 | - name: Run ESLint checks 24 | run: npm run lint 25 | - name: Run Prettier checks 26 | run: npm run format-check 27 | - name: Run tests 28 | run: npm run test 29 | -------------------------------------------------------------------------------- /.github/workflows/publish-docs.yml: -------------------------------------------------------------------------------- 1 | name: Publish docs 2 | 3 | on: 4 | # Runs on pushes targeting the main branch 5 | push: 6 | branches: ['main'] 7 | 8 | # Allows you to run this workflow manually from the Actions tab 9 | workflow_dispatch: 10 | 11 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 12 | permissions: 13 | contents: write 14 | pages: write 15 | id-token: write 16 | 17 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 18 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 19 | concurrency: 20 | group: 'pages' 21 | cancel-in-progress: false 22 | 23 | jobs: 24 | generate_and_commit_docs: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - name: Checkout docs branch 28 | uses: actions/checkout@v4 29 | with: 30 | ref: 'docs' 31 | 32 | - name: Set account identity 33 | run: | 34 | git config --global user.email "kafkas@users.noreply.github.com" 35 | git config --global user.name "Documentation Bot" 36 | 37 | - name: Rebase with main branch 38 | run: | 39 | git fetch 40 | git rebase -Xours origin/main 41 | 42 | - name: Install dependencies 43 | run: npm install 44 | 45 | - name: Generate docs for current version 46 | run: npm run docs-generate 47 | 48 | - name: Commit and Push Changes 49 | run: | 50 | git add docs/ 51 | git commit -m "Update docs (bot)" || echo "No changes (bot)" 52 | git push -f 53 | 54 | deploy_to_github_pages: 55 | needs: generate_and_commit_docs 56 | environment: 57 | name: github-pages 58 | url: ${{ steps.deployment.outputs.page_url }} 59 | runs-on: ubuntu-latest 60 | steps: 61 | - name: Checkout docs branch 62 | uses: actions/checkout@v4 63 | with: 64 | ref: 'docs' 65 | 66 | - name: Setup Pages 67 | uses: actions/configure-pages@v4 68 | 69 | - name: Upload artifact 70 | uses: actions/upload-pages-artifact@v3 71 | with: 72 | path: './docs' 73 | 74 | - name: Deploy to GitHub Pages 75 | id: deployment 76 | uses: actions/deploy-pages@v4 77 | -------------------------------------------------------------------------------- /.github/workflows/publish-package.yml: -------------------------------------------------------------------------------- 1 | name: Publish package 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | version_increment: 7 | type: choice 8 | required: true 9 | description: 'Select version increment type' 10 | options: 11 | - patch 12 | - minor 13 | - major 14 | 15 | permissions: 16 | contents: write 17 | 18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 20 | concurrency: 21 | group: 'version' 22 | cancel-in-progress: true 23 | 24 | jobs: 25 | bump_version_and_publish: 26 | runs-on: ubuntu-latest 27 | permissions: 28 | contents: write 29 | packages: write 30 | pull-requests: write 31 | steps: 32 | - uses: actions/checkout@v4 33 | 34 | - name: Use Node.js 35 | uses: actions/setup-node@v4 36 | with: 37 | node-version: '22.x' 38 | cache: 'npm' 39 | registry-url: 'https://registry.npmjs.org' 40 | 41 | - name: Install project dependencies 42 | run: npm ci 43 | 44 | - name: Build the source 45 | run: npm run build 46 | 47 | - name: Set git identity 48 | run: | 49 | git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" 50 | git config --global user.name "github-actions[bot]" 51 | 52 | - name: Bump version 53 | run: npm version ${{ inputs.version_increment }} 54 | 55 | - name: Push commit and create tag and release 56 | env: 57 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 58 | run: | 59 | version="$(jq -r .version package.json)" 60 | git push origin 61 | git push origin "v$version" 62 | gh release create "v$version" --title "v$version" 63 | 64 | - name: Publish the package to NPM 65 | run: npm publish 66 | env: 67 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | 3 | coverage/ 4 | 5 | dist/ 6 | 7 | .DS_Store 8 | firestore-debug.log 9 | firebase-debug.log 10 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 100, 3 | "tabWidth": 2, 4 | "singleQuote": true, 5 | "trailingComma": "es5", 6 | "endOfLine": "auto" 7 | } 8 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.fontLigatures": "'ss01', 'ss02', 'ss03', 'ss04', 'ss05', 'ss06', 'zero', 'onum'" 3 | } 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Anar Kafkas 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | Firewalk 4 | 5 |

6 | 7 |

8 | A light, fast, and memory-efficient collection traversal library for Firestore and Node.js. 9 |

10 | 11 | --- 12 | 13 |

14 | 15 | Firewalk is released under the MIT license. 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | PRs welcome! 26 |

27 | 28 | Firewalk is a Node.js library that _walks_ you through Firestore collections. 29 | 30 | When you have millions of documents in a collection and you need to make changes to them or just read them, you can't just retrieve all of them at once as your program's memory usage will explode. Firewalk's configurable traverser objects let you do this in a simple, intuitive and memory-efficient way using batch processing with concurrency control. 31 | 32 | Firewalk is an extremely light and well-typed library that is useful in a variety of scenarios. You can use it in database migration scripts (e.g. when you need to add a new field to all docs) or a scheduled Cloud Function that needs to check every doc in a collection periodically or even a locally run script that retrieves some data from a collection. 33 | 34 | **Note**: This library was previously known as Firecode. We're currently in the process of porting over the documentation from the 35 | previous site. 36 | 37 | [**Read the introductory blog post ▸**](https://anarkafkas.medium.com/traversing-firestore-collections-efficiently-6e43cea1eefd) 38 | 39 | [**View the full documentation (docs) ▸**](https://kafkas.github.io/firewalk) 40 | 41 | ## Overview 42 | 43 | 1. [Prerequisites](#Prerequisites) 44 | 1. [Compatibility Map](#Compatibility-Map) 45 | 1. [Installation](#Installation) 46 | 1. [Core Concepts](#Core-Concepts) 47 | 1. [Quick Start](#Quick-Start) 48 | 1. [More Examples](#More-Examples) 49 | 1. [API](#API) 50 | 1. [Upgrading](#Upgrading) 51 | 1. [License](#License) 52 | 53 | ## Prerequisites 54 | 55 | Firewalk is designed to work with the [Firebase Admin SDK](https://github.com/firebase/firebase-admin-node) so if you haven't already installed it, you'll need add it as a dependency to your project. 56 | 57 | ```bash 58 | npm install firebase-admin 59 | ``` 60 | 61 | ### Compatibility Map 62 | 63 | Make sure to install the right version of Firewalk depending on the `firebase-admin` version your project is on. 64 | 65 | | firewalk | firebase-admin | 66 | | -------- | -------------- | 67 | | v1 | v9-10 | 68 | | v2 | v11-13 | 69 | 70 | ## Installation 71 | 72 | You can add Firewalk to your project with npm or yarn. 73 | 74 | ```bash 75 | npm install firewalk 76 | ``` 77 | 78 | ## Core Concepts 79 | 80 | There are only 2 kinds of objects you need to be familiar with when using this library: 81 | 82 | 1. **Traverser**: An object that walks you through a collection of documents (or more generally a [Traversable](https://kafkas.github.io/firewalk/types/Traversable.html)). 83 | 84 | 2. **Migrator**: A convenience object used for database migrations. It lets you easily write to the documents within a given traversable and uses a traverser to do that. You can easily write your own migration logic in the traverser callback if you don't want to use a migrator. 85 | 86 | ## Quick Start 87 | 88 | Suppose we have a `users` collection and we want to send an email to each user. This is how easy it is to do that efficiently with a Firewalk traverser: 89 | 90 | ```ts 91 | import { firestore } from 'firebase-admin'; 92 | import { createTraverser } from 'firewalk'; 93 | 94 | const usersCollection = firestore().collection('users'); 95 | const traverser = createTraverser(usersCollection); 96 | 97 | const { batchCount, docCount } = await traverser.traverse(async (batchDocs, batchIndex) => { 98 | const batchSize = batchDocs.length; 99 | await Promise.all( 100 | batchDocs.map(async (doc) => { 101 | const { email, firstName } = doc.data(); 102 | await sendEmail({ to: email, content: `Hello ${firstName}!` }); 103 | }) 104 | ); 105 | console.log(`Batch ${batchIndex} done! We emailed ${batchSize} users in this batch.`); 106 | }); 107 | 108 | console.log(`Traversal done! We emailed ${docCount} users in ${batchCount} batches!`); 109 | ``` 110 | 111 | We are doing 3 things here: 112 | 113 | 1. Create a reference to the `users` collection 114 | 2. Pass that reference to the `createTraverser()` function 115 | 3. Invoke `.traverse()` with an async callback that is called for each batch of document snapshots 116 | 117 | This pretty much sums up the core functionality of this library! The `.traverse()` method returns a Promise that resolves when the entire traversal finishes, which can take a while if you have millions of docs. The Promise resolves with an object containing the traversal details e.g. the number of docs you touched. 118 | 119 | ## More Examples 120 | 121 | ### Traverse faster by increasing concurrency 122 | 123 | ```ts 124 | const projectsColRef = firestore().collection('projects'); 125 | const traverser = createTraverser(projectsColRef, { 126 | batchSize: 500, 127 | // This means we are prepared to hold 500 * 20 = 10,000 docs in memory. 128 | // We sacrifice some memory to traverse faster. 129 | maxConcurrentBatchCount: 20, 130 | }); 131 | const { docCount } = await traverser.traverse(async (_, batchIndex) => { 132 | console.log(`Gonna process batch ${batchIndex} now!`); 133 | // ... 134 | }); 135 | console.log(`Traversed ${docCount} projects super-fast!`); 136 | ``` 137 | 138 | ### Add a new field using a migrator 139 | 140 | ```ts 141 | const projectsColRef = firestore().collection('projects'); 142 | const migrator = createMigrator(projectsColRef); 143 | const { migratedDocCount } = await migrator.update('isCompleted', false); 144 | console.log(`Updated ${migratedDocCount} projects!`); 145 | ``` 146 | 147 | ### Add a new field derived from the previous fields 148 | 149 | ```ts 150 | type UserDoc = { 151 | firstName: string; 152 | lastName: string; 153 | }; 154 | const usersColRef = firestore().collection('users') as firestore.CollectionReference; 155 | const migrator = createMigrator(usersColRef); 156 | const { migratedDocCount } = await migrator.updateWithDerivedData((snap) => { 157 | const { firstName, lastName } = snap.data(); 158 | return { 159 | fullName: `${firstName} ${lastName}`, 160 | }; 161 | }); 162 | console.log(`Updated ${migratedDocCount} users!`); 163 | ``` 164 | 165 | ### Migrate faster by increasing concurrency 166 | 167 | ```ts 168 | const projectsColRef = firestore().collection('projects'); 169 | const migrator = createMigrator(projectsColRef, { maxConcurrentBatchCount: 25 }); 170 | const { migratedDocCount } = await migrator.update('isCompleted', false); 171 | console.log(`Updated ${migratedDocCount} projects super-fast!`); 172 | ``` 173 | 174 | ### Change traversal config 175 | 176 | ```ts 177 | const walletsWithNegativeBalance = firestore().collection('wallets').where('money', '<', 0); 178 | const migrator = createMigrator(walletsWithNegativeBalance, { 179 | // We want each batch to have 500 docs. The size of the very last batch may be less than 500 180 | batchSize: 500, 181 | // We want to wait 500ms before moving to the next batch 182 | sleepTimeBetweenBatches: 500, 183 | }); 184 | // Wipe out their debts! 185 | const { migratedDocCount } = await migrator.set({ money: 0 }); 186 | console.log(`Set ${migratedDocCount} wallets!`); 187 | ``` 188 | 189 | ### Rename a field 190 | 191 | ```ts 192 | const postsColGroup = firestore().collectionGroup('posts'); 193 | const migrator = createMigrator(postsColGroup); 194 | const { migratedDocCount } = await migrator.renameField('postedAt', 'publishedAt'); 195 | console.log(`Updated ${migratedDocCount} posts!`); 196 | ``` 197 | 198 | ## [API](https://kafkas.github.io/firewalk) 199 | 200 | You can find the full API reference for Firewalk [here](https://kafkas.github.io/firewalk). We maintain detailed docs for each major version. Here are some of the core functions that this library provides. 201 | 202 | ### [createTraverser](https://kafkas.github.io/firewalk/functions/createTraverser.html) 203 | 204 | Creates an object which can be used to traverse a Firestore collection or, more generally, a [Traversable](https://kafkas.github.io/firewalk/types/Traversable.html). 205 | 206 | For each batch of document snapshots in the traversable, the traverser invokes a specified async callback and immediately moves to the next batch. It does not wait for the callback Promise to resolve before moving to the next batch. That is, when `maxConcurrentBatchCount` > 1, there is no guarantee that any given batch will finish processing before a later batch. 207 | 208 | The traverser becomes faster as you increase `maxConcurrentBatchCount`, but this will consume more memory. You should increase concurrency when you want to trade some memory for speed. 209 | 210 | #### Complexity: 211 | 212 | - Time complexity: _O_((_N_ / `batchSize`) \* (_Q_(`batchSize`) + _C_(`batchSize`) / `maxConcurrentBatchCount`)) 213 | - Space complexity: _O_(`maxConcurrentBatchCount` \* (`batchSize` \* _D_ + _S_)) 214 | - Billing: _max_(1, _N_) reads 215 | 216 | where: 217 | 218 | - _N_: number of docs in the traversable 219 | - _Q_(`batchSize`): average batch query time 220 | - _C_(`batchSize`): average callback processing time 221 | - _D_: average document size 222 | - _S_: average extra space used by the callback 223 | 224 | ### [createMigrator](https://kafkas.github.io/firewalk/functions/createMigrator.html) 225 | 226 | Creates a migrator that facilitates database migrations. The migrator accepts a custom traverser to traverse the collection. Otherwise it will create a default traverser with your desired traversal config. This migrator does not use atomic batch writes so it is possible that when a write fails other writes go through. 227 | 228 | #### Complexity: 229 | 230 | - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 231 | - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 232 | - Billing: _max_(1, _N_) reads, _K_ writes 233 | 234 | where: 235 | 236 | - _N_: number of docs in the traversable 237 | - _K_: number of docs that passed the migration predicate (_K_<=_N_) 238 | - _W_(`batchSize`): average batch write time 239 | - _TC_(`traverser`): time complexity of the underlying traverser 240 | - _SC_(`traverser`): space complexity of the underlying traverser 241 | 242 | ### [createBatchMigrator](https://kafkas.github.io/firewalk/functions/createBatchMigrator.html) 243 | 244 | Creates a migrator that facilitates database migrations. The migrator accepts a custom traverser to traverse the collection. Otherwise it will create a default traverser with your desired traversal config. This migrator uses atomic batch writes so the entire operation will fail if a single write isn't successful. 245 | 246 | #### Complexity: 247 | 248 | - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 249 | - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 250 | - Billing: _max_(1, _N_) reads, _K_ writes 251 | 252 | where: 253 | 254 | - _N_: number of docs in the traversable 255 | - _K_: number of docs that passed the migration predicate (_K_<=_N_) 256 | - _W_(`batchSize`): average batch write time 257 | - _TC_(`traverser`): time complexity of the underlying traverser 258 | - _SC_(`traverser`): space complexity of the underlying traverser 259 | 260 | ## Upgrading 261 | 262 | This project adheres to [SemVer](https://semver.org). Before upgrading to a new major version, make sure to check out the [Releases](https://github.com/kafkas/firewalk/releases) page to view all the breaking changes. 263 | 264 | ## License 265 | 266 | This project is made available under the MIT License. 267 | -------------------------------------------------------------------------------- /__tests__/app.ts: -------------------------------------------------------------------------------- 1 | import * as admin from 'firebase-admin'; 2 | 3 | class Application { 4 | public get firestore(): admin.firestore.Firestore { 5 | return this.firebaseApp.firestore(); 6 | } 7 | 8 | public constructor(private readonly firebaseApp: admin.app.App) {} 9 | } 10 | 11 | let _app: Application | undefined; 12 | 13 | export function app(): Application { 14 | const firebaseApp = admin.initializeApp(); 15 | return _app ?? (_app = new Application(firebaseApp)); 16 | } 17 | -------------------------------------------------------------------------------- /__tests__/types/DefaultMigrator.test-d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | 3 | import { firestore } from 'firebase-admin'; 4 | import { expectError, expectType } from 'tsd'; 5 | import { DefaultMigrator, Traverser, createMigrator, createTraverser } from '../../src'; 6 | import { TestAppModelType, TestDbModelType, collectionRef } from './_helpers'; 7 | 8 | const defaultMigrator = createMigrator(collectionRef); 9 | 10 | expectType>(defaultMigrator.traverser); 11 | 12 | (() => { 13 | const modifiedMigrator = defaultMigrator.withPredicate((doc) => { 14 | expectType>(doc); 15 | return false; 16 | }); 17 | expectType>(modifiedMigrator); 18 | })(); 19 | 20 | (() => { 21 | const traverser = createTraverser(collectionRef); 22 | const modifiedMigrator = defaultMigrator.withTraverser(traverser); 23 | expectType>(modifiedMigrator); 24 | })(); 25 | 26 | defaultMigrator.onBeforeBatchStart((batchDocs, batchIndex) => { 27 | expectType[]>(batchDocs); 28 | expectType(batchIndex); 29 | }); 30 | 31 | defaultMigrator.onAfterBatchComplete((batchDocs, batchIndex) => { 32 | expectType[]>(batchDocs); 33 | expectType(batchIndex); 34 | }); 35 | 36 | defaultMigrator.deleteField('oldField'); 37 | defaultMigrator.deleteField(new firestore.FieldPath('nested', 'field')); 38 | 39 | defaultMigrator.deleteFields(new firestore.FieldPath('nested', 'field'), 'field2', 'field3'); 40 | 41 | defaultMigrator.renameField('oldField', new firestore.FieldPath('new', 'field')); 42 | defaultMigrator.renameField(new firestore.FieldPath('old', 'field'), 'newField'); 43 | 44 | defaultMigrator.renameFields( 45 | ['field1', new firestore.FieldPath('nested', 'field2')], 46 | ['field1', 'field2'] 47 | ); 48 | 49 | expectError(defaultMigrator.set({ num: 0 })); 50 | defaultMigrator.set({ num: 0, text: '' }); 51 | defaultMigrator.set({ num: 0 }, { merge: true }); 52 | 53 | expectError( 54 | defaultMigrator.setWithDerivedData((doc) => { 55 | expectType>(doc); 56 | return { num: 0 }; 57 | }) 58 | ); 59 | defaultMigrator.setWithDerivedData((doc) => { 60 | expectType>(doc); 61 | return { num: 0, text: '' }; 62 | }); 63 | defaultMigrator.setWithDerivedData( 64 | (doc) => { 65 | expectType>(doc); 66 | return { num: 0 }; 67 | }, 68 | { merge: true } 69 | ); 70 | 71 | expectError( 72 | defaultMigrator.update({ 73 | anyField: '', 74 | }) 75 | ); 76 | 77 | defaultMigrator.update('anyField', 'anyValue'); 78 | 79 | expectError( 80 | defaultMigrator.updateWithDerivedData((doc) => { 81 | expectType>(doc); 82 | return { anyField: '' }; 83 | }) 84 | ); 85 | 86 | expectError( 87 | defaultMigrator.updateWithDerivedData((doc) => { 88 | expectType>(doc); 89 | return ['anyField', 'anyValue']; 90 | }) 91 | ); 92 | 93 | // TODO: We need to expect an error here if the return type of the callback is not a plain object or an array 94 | // expectError( 95 | // defaultMigrator.updateWithDerivedData((doc) => { 96 | // expectType>(doc); 97 | // return new Map([]); 98 | // }) 99 | // ); 100 | -------------------------------------------------------------------------------- /__tests__/types/Traverser.test-d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | 3 | import type { firestore } from 'firebase-admin'; 4 | import { expectType } from 'tsd'; 5 | import { 6 | createTraverser, 7 | Traversable, 8 | TraversalConfig, 9 | TraversalResult, 10 | Traverser, 11 | } from '../../src'; 12 | import { collectionRef, TestAppModelType, TestDbModelType } from './_helpers'; 13 | 14 | const traverser = createTraverser(collectionRef); 15 | 16 | // TODO: Ideally we want to expect a firestore.CollectionReference here because 17 | // we initialized the traverser with a collection reference. 18 | expectType>(traverser.traversable); 19 | 20 | expectType(traverser.traversalConfig); 21 | 22 | (() => { 23 | // TODO: See if there is a better way to check inferred parameters 24 | const modifiedTraverser = traverser.withConfig({ 25 | batchSize: 0, 26 | sleepTimeBetweenBatches: 0, 27 | maxDocCount: 0, 28 | maxConcurrentBatchCount: 0, 29 | }); 30 | expectType>(modifiedTraverser); 31 | })(); 32 | 33 | (() => { 34 | const modifiedTraverser = traverser.withExitEarlyPredicate((batchDocs, batchIndex) => { 35 | expectType[]>(batchDocs); 36 | expectType(batchIndex); 37 | return false; 38 | }); 39 | expectType>(modifiedTraverser); 40 | })(); 41 | 42 | (async () => { 43 | const traversalResult1 = await traverser.traverseEach(async (doc, docIndex, batchIndex) => { 44 | expectType>(doc); 45 | expectType(docIndex); 46 | expectType(batchIndex); 47 | }); 48 | expectType(traversalResult1); 49 | })(); 50 | 51 | (async () => { 52 | const traversalResult2 = await traverser.traverse(async (batchDocs, batchIndex) => { 53 | expectType[]>(batchDocs); 54 | expectType(batchIndex); 55 | }); 56 | expectType(traversalResult2); 57 | })(); 58 | -------------------------------------------------------------------------------- /__tests__/types/_helpers.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | 3 | export type TestAppModelType = { 4 | text: string; 5 | num: number; 6 | }; 7 | 8 | export type TestDbModelType = { 9 | text: string; 10 | num: number; 11 | }; 12 | 13 | export const collectionRef = firestore().collection('projects') as firestore.CollectionReference< 14 | TestAppModelType, 15 | TestDbModelType 16 | >; 17 | -------------------------------------------------------------------------------- /__tests__/types/createBatchMigrator.test-d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | 3 | import { expectType } from 'tsd'; 4 | import { createBatchMigrator, createTraverser, BatchMigrator } from '../../src'; 5 | import { collectionRef, TestAppModelType, TestDbModelType } from './_helpers'; 6 | 7 | // Signature 1 8 | 9 | const traverser = createTraverser(collectionRef); 10 | const batchMigrator = createBatchMigrator(traverser); 11 | expectType>(batchMigrator); 12 | 13 | // Signature 2 14 | 15 | const batchMigrator2 = createBatchMigrator(collectionRef, { 16 | batchSize: 0, 17 | sleepTimeBetweenBatches: 0, 18 | maxDocCount: 0, 19 | }); 20 | expectType>(batchMigrator2); 21 | -------------------------------------------------------------------------------- /__tests__/types/createMigrator.test-d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | 3 | import { expectType } from 'tsd'; 4 | import { createMigrator, createTraverser, DefaultMigrator } from '../../src'; 5 | import { collectionRef, TestAppModelType, TestDbModelType } from './_helpers'; 6 | 7 | // Signature 1 8 | 9 | const traverser = createTraverser(collectionRef); 10 | const defaultMigrator = createMigrator(traverser); 11 | expectType>(defaultMigrator); 12 | 13 | // Signature 2 14 | 15 | const defaultMigrator2 = createMigrator(collectionRef, { 16 | batchSize: 0, 17 | sleepTimeBetweenBatches: 0, 18 | maxDocCount: 0, 19 | maxConcurrentBatchCount: 0, 20 | }); 21 | expectType>(defaultMigrator2); 22 | -------------------------------------------------------------------------------- /__tests__/types/createTraverser.test-d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | 3 | import { expectType } from 'tsd'; 4 | import { createTraverser, Traverser } from '../../src'; 5 | import { collectionRef, TestAppModelType, TestDbModelType } from './_helpers'; 6 | 7 | const traverser = createTraverser(collectionRef, { 8 | batchSize: 0, 9 | sleepTimeBetweenBatches: 0, 10 | maxDocCount: 0, 11 | maxConcurrentBatchCount: 0, 12 | }); 13 | 14 | expectType>(traverser); 15 | -------------------------------------------------------------------------------- /__tests__/utils/collectionPopulator.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | 3 | interface CollectionPopulatorBuilder< 4 | AppModelType = firestore.DocumentData, 5 | DbModelType extends firestore.DocumentData = firestore.DocumentData 6 | > { 7 | withData( 8 | dataOrGetData: AppModelType | (() => AppModelType) 9 | ): CollectionPopulator; 10 | } 11 | 12 | interface CollectionPopulator< 13 | AppModelType = firestore.DocumentData, 14 | DbModelType extends firestore.DocumentData = firestore.DocumentData 15 | > { 16 | populate(opts: { 17 | count: number; 18 | }): Promise[]>; 19 | } 20 | 21 | export function collectionPopulator< 22 | AppModelType = firestore.DocumentData, 23 | DbModelType extends firestore.DocumentData = firestore.DocumentData 24 | >( 25 | collectionRef: firestore.CollectionReference 26 | ): CollectionPopulatorBuilder { 27 | return { 28 | withData: (dataOrGetData) => { 29 | return { 30 | populate: async ({ count: docCount }) => { 31 | const promises = new Array(docCount).fill(null).map(async () => { 32 | const data = 33 | typeof dataOrGetData === 'function' 34 | ? (dataOrGetData as () => AppModelType)() 35 | : dataOrGetData; 36 | return await collectionRef.add(data); 37 | }); 38 | 39 | return await Promise.all(promises); 40 | }, 41 | }; 42 | }, 43 | }; 44 | } 45 | -------------------------------------------------------------------------------- /__tests__/utils/getLexicographicallyNextString.test.ts: -------------------------------------------------------------------------------- 1 | import { getLexicographicallyNextString } from './getLexicographicallyNextString'; 2 | 3 | test('correctly computes lexicographically next string', () => { 4 | expect(getLexicographicallyNextString('')).toStrictEqual('a'); 5 | expect(getLexicographicallyNextString('a')).toStrictEqual('b'); 6 | expect(getLexicographicallyNextString('abc')).toStrictEqual('abd'); 7 | expect(getLexicographicallyNextString('abc123')).toStrictEqual('abc124'); 8 | expect(getLexicographicallyNextString('abc123xyz')).toStrictEqual('abc123xzz'); 9 | expect(getLexicographicallyNextString('z')).toStrictEqual('za'); 10 | expect(getLexicographicallyNextString('zz')).toStrictEqual('zza'); 11 | }); 12 | -------------------------------------------------------------------------------- /__tests__/utils/getLexicographicallyNextString.ts: -------------------------------------------------------------------------------- 1 | export function getLexicographicallyNextString(str: string): string { 2 | if (str == '') return 'a'; 3 | 4 | let i = str.length - 1; 5 | 6 | while (str[i] == 'z' && i >= 0) { 7 | i--; 8 | } 9 | 10 | if (i === -1) { 11 | str = str + 'a'; 12 | } else { 13 | str = str.substring(0, i) + String.fromCharCode(str.charCodeAt(i) + 1) + str.substring(i + 1); 14 | } 15 | 16 | return str; 17 | } 18 | -------------------------------------------------------------------------------- /__tests__/utils/index.ts: -------------------------------------------------------------------------------- 1 | export { collectionPopulator } from './collectionPopulator'; 2 | export { writeFileAndMkdirSync } from './writeFileAndMkdirSync'; 3 | -------------------------------------------------------------------------------- /__tests__/utils/writeFileAndMkdirSync.ts: -------------------------------------------------------------------------------- 1 | import { existsSync, mkdirSync, writeFileSync } from 'fs'; 2 | import { dirname } from 'path'; 3 | 4 | export function writeFileAndMkdirSync( 5 | path: string, 6 | data: string | Record | unknown[], 7 | opts?: Parameters[2] 8 | ): void { 9 | const d = typeof data === 'string' ? data : JSON.stringify(data); 10 | ensureDirectoryExists(path); 11 | writeFileSync(path, d, opts); 12 | } 13 | 14 | function ensureDirectoryExists(path: string): void { 15 | const dirName = dirname(path); 16 | if (existsSync(dirName)) { 17 | return; 18 | } 19 | ensureDirectoryExists(dirName); 20 | mkdirSync(dirName); 21 | } 22 | -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | // babel.config.js 2 | module.exports = { 3 | presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'], 4 | }; 5 | -------------------------------------------------------------------------------- /examples/addNewField.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { createMigrator } from 'firewalk'; 3 | 4 | const projectsColRef = firestore().collection('projects'); 5 | const migrator = createMigrator(projectsColRef); 6 | const { migratedDocCount } = await migrator.update('isCompleted', false); 7 | console.log(`Updated ${migratedDocCount} projects!`); 8 | -------------------------------------------------------------------------------- /examples/addNewFieldFromPrevFields.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { createMigrator } from 'firewalk'; 3 | 4 | type UserDoc = { 5 | firstName: string; 6 | lastName: string; 7 | }; 8 | const usersColRef = firestore().collection('users') as firestore.CollectionReference; 9 | const migrator = createMigrator(usersColRef); 10 | const { migratedDocCount } = await migrator.updateWithDerivedData((snap) => { 11 | const { firstName, lastName } = snap.data(); 12 | return { 13 | fullName: `${firstName} ${lastName}`, 14 | }; 15 | }); 16 | console.log(`Updated ${migratedDocCount} users!`); 17 | -------------------------------------------------------------------------------- /examples/changeTraversalConfig.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { createMigrator } from 'firewalk'; 3 | 4 | const walletsWithNegativeBalance = firestore().collection('wallets').where('money', '<', 0); 5 | const migrator = createMigrator(walletsWithNegativeBalance, { 6 | // We want each batch to have 500 docs. The size of the very last batch may be less than 500 7 | batchSize: 500, 8 | // We want to wait 500ms before moving to the next batch 9 | sleepTimeBetweenBatches: 500, 10 | }); 11 | // Wipe out their debts! 12 | const { migratedDocCount } = await migrator.set({ money: 0 }); 13 | console.log(`Updated ${migratedDocCount} wallets!`); 14 | -------------------------------------------------------------------------------- /examples/exitEarlyPredicate.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { createTraverser } from 'firewalk'; 3 | 4 | const projectsColRef = firestore().collection('projects'); 5 | const traverser = createTraverser(projectsColRef); 6 | const newTraverser = traverser 7 | .withExitEarlyPredicate((batchDocs) => batchDocs.some((d) => d.get('name') === undefined)) 8 | .withExitEarlyPredicate((_, batchIndex) => batchIndex === 100); 9 | -------------------------------------------------------------------------------- /examples/fastMigrator.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { createMigrator } from 'firewalk'; 3 | 4 | const projectsColRef = firestore().collection('projects'); 5 | const migrator = createMigrator(projectsColRef, { maxConcurrentBatchCount: 25 }); 6 | const { migratedDocCount } = await migrator.update('isCompleted', false); 7 | console.log(`Updated ${migratedDocCount} projects super-fast!`); 8 | -------------------------------------------------------------------------------- /examples/fastTraverser.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { createTraverser } from 'firewalk'; 3 | 4 | const projectsColRef = firestore().collection('projects'); 5 | const traverser = createTraverser(projectsColRef, { 6 | batchSize: 500, 7 | // This means we are prepared to hold 500 * 20 = 10,000 docs in memory. 8 | // We sacrifice some memory to traverse faster. 9 | maxConcurrentBatchCount: 20, 10 | }); 11 | const { docCount } = await traverser.traverse(async (_, batchIndex) => { 12 | console.log(`Gonna process batch ${batchIndex} now!`); 13 | // ... 14 | }); 15 | console.log(`Traversed ${docCount} projects super-fast!`); 16 | -------------------------------------------------------------------------------- /examples/migrationPredicate.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { createMigrator } from 'firewalk'; 3 | 4 | const projectsColRef = firestore().collectionGroup('projects'); 5 | const migrator = createMigrator(projectsColRef); 6 | const newMigrator = migrator 7 | .withPredicate((doc) => doc.get('name') !== undefined) 8 | .withPredicate((doc) => doc.ref.path.startsWith('users/')); 9 | -------------------------------------------------------------------------------- /examples/quickStart.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { createTraverser } from 'firewalk'; 3 | 4 | const usersCollection = firestore().collection('users'); 5 | const traverser = createTraverser(usersCollection); 6 | 7 | const { batchCount, docCount } = await traverser.traverse(async (batchDocs, batchIndex) => { 8 | const batchSize = batchDocs.length; 9 | await Promise.all( 10 | batchDocs.map(async (doc) => { 11 | const { email, firstName } = doc.data(); 12 | await sendEmail({ to: email, content: `Hello ${firstName}!` }); 13 | }) 14 | ); 15 | console.log(`Batch ${batchIndex} done! We emailed ${batchSize} users in this batch.`); 16 | }); 17 | 18 | console.log(`Traversal done! We emailed ${docCount} users in ${batchCount} batches!`); 19 | -------------------------------------------------------------------------------- /examples/renameField.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { createMigrator } from 'firewalk'; 3 | 4 | const postsColGroup = firestore().collectionGroup('posts'); 5 | const migrator = createMigrator(postsColGroup); 6 | const { migratedDocCount } = await migrator.renameField('postedAt', 'publishedAt'); 7 | console.log(`Updated ${migratedDocCount} posts!`); 8 | -------------------------------------------------------------------------------- /jest.config.global.ts: -------------------------------------------------------------------------------- 1 | export default function globalJestSetup(): void { 2 | process.env.FIRESTORE_EMULATOR_HOST = `127.0.0.1:8080`; 3 | } 4 | -------------------------------------------------------------------------------- /jest.config.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * For a detailed explanation regarding each configuration property and type check, visit: 3 | * https://jestjs.io/docs/configuration 4 | */ 5 | 6 | export default { 7 | // All imported modules in your tests should be mocked automatically 8 | // automock: false, 9 | 10 | // Stop running tests after `n` failures 11 | // bail: 0, 12 | 13 | // The directory where Jest should store its cached dependency information 14 | // cacheDirectory: "/private/var/folders/6d/p1n8g20s74q1b14xk0_54gt80000gn/T/jest_dx", 15 | 16 | // Automatically clear mock calls and instances between every test 17 | clearMocks: true, 18 | 19 | // Indicates whether the coverage information should be collected while executing the test 20 | collectCoverage: true, 21 | 22 | // An array of glob patterns indicating a set of files for which coverage information should be collected 23 | // collectCoverageFrom: undefined, 24 | 25 | // The directory where Jest should output its coverage files 26 | coverageDirectory: 'coverage', 27 | 28 | // An array of regexp pattern strings used to skip coverage collection 29 | // coveragePathIgnorePatterns: [ 30 | // "/node_modules/" 31 | // ], 32 | 33 | // Indicates which provider should be used to instrument code for coverage 34 | // coverageProvider: "babel", 35 | 36 | // A list of reporter names that Jest uses when writing coverage reports 37 | // coverageReporters: [ 38 | // "json", 39 | // "text", 40 | // "lcov", 41 | // "clover" 42 | // ], 43 | 44 | // An object that configures minimum threshold enforcement for coverage results 45 | // coverageThreshold: undefined, 46 | 47 | // A path to a custom dependency extractor 48 | // dependencyExtractor: undefined, 49 | 50 | // Make calling deprecated APIs throw helpful error messages 51 | // errorOnDeprecated: false, 52 | 53 | // Force coverage collection from ignored files using an array of glob patterns 54 | // forceCoverageMatch: [], 55 | 56 | // A path to a module which exports an async function that is triggered once before all test suites 57 | globalSetup: './jest.config.global.ts', 58 | 59 | // A path to a module which exports an async function that is triggered once after all test suites 60 | // globalTeardown: undefined, 61 | 62 | // A set of global variables that need to be available in all test environments 63 | // globals: {}, 64 | 65 | // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers. 66 | // maxWorkers: "50%", 67 | 68 | // An array of directory names to be searched recursively up from the requiring module's location 69 | // moduleDirectories: [ 70 | // "node_modules" 71 | // ], 72 | 73 | // An array of file extensions your modules use 74 | // moduleFileExtensions: [ 75 | // "js", 76 | // "jsx", 77 | // "ts", 78 | // "tsx", 79 | // "json", 80 | // "node" 81 | // ], 82 | 83 | // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module 84 | // moduleNameMapper: {}, 85 | 86 | // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader 87 | // modulePathIgnorePatterns: [], 88 | 89 | // Activates notifications for test results 90 | // notify: false, 91 | 92 | // An enum that specifies notification mode. Requires { notify: true } 93 | // notifyMode: "failure-change", 94 | 95 | // A preset that is used as a base for Jest's configuration 96 | // preset: undefined, 97 | 98 | // Run tests from one or more projects 99 | // projects: undefined, 100 | 101 | // Use this configuration option to add custom reporters to Jest 102 | // reporters: undefined, 103 | 104 | // Automatically reset mock state between every test 105 | // resetMocks: false, 106 | 107 | // Reset the module registry before running each individual test 108 | // resetModules: false, 109 | 110 | // A path to a custom resolver 111 | // resolver: undefined, 112 | 113 | // Automatically restore mock state between every test 114 | // restoreMocks: false, 115 | 116 | // The root directory that Jest should scan for tests and modules within 117 | // rootDir: undefined, 118 | 119 | // A list of paths to directories that Jest should use to search for files in 120 | // roots: [ 121 | // "" 122 | // ], 123 | 124 | // Allows you to use a custom runner instead of Jest's default test runner 125 | // runner: "jest-runner", 126 | 127 | // The paths to modules that run some code to configure or set up the testing environment before each test 128 | // setupFiles: [], 129 | 130 | // A list of paths to modules that run some code to configure or set up the testing framework before each test 131 | // setupFilesAfterEnv: [], 132 | 133 | // The number of seconds after which a test is considered as slow and reported as such in the results. 134 | // slowTestThreshold: 5, 135 | 136 | // A list of paths to snapshot serializer modules Jest should use for snapshot testing 137 | // snapshotSerializers: [], 138 | 139 | // The test environment that will be used for testing 140 | // testEnvironment: "jest-environment-node", 141 | 142 | // Options that will be passed to the testEnvironment 143 | // testEnvironmentOptions: {}, 144 | 145 | // Adds a location field to test results 146 | // testLocationInResults: false, 147 | 148 | // The glob patterns Jest uses to detect test files 149 | testMatch: ['**/?(*.)+(spec|test).[tj]s?(x)'], 150 | 151 | // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped 152 | testPathIgnorePatterns: ['/node_modules/'], 153 | 154 | // The regexp pattern or array of patterns that Jest uses to detect test files 155 | // testRegex: [], 156 | 157 | // This option allows the use of a custom results processor 158 | // testResultsProcessor: undefined, 159 | 160 | // This option allows use of a custom test runner 161 | // testRunner: "jest-circus/runner", 162 | 163 | // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href 164 | // testURL: "http://localhost", 165 | 166 | // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout" 167 | // timers: "real", 168 | 169 | // A map from regular expressions to paths to transformers 170 | // transform: undefined, 171 | 172 | // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation 173 | // transformIgnorePatterns: [ 174 | // "/node_modules/", 175 | // "\\.pnp\\.[^\\/]+$" 176 | // ], 177 | 178 | // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them 179 | // unmockedModulePathPatterns: undefined, 180 | 181 | // Indicates whether each individual test should be reported during the run 182 | // verbose: undefined, 183 | 184 | // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode 185 | // watchPathIgnorePatterns: [], 186 | 187 | // Whether to use watchman for file crawling 188 | // watchman: true, 189 | }; 190 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firewalk", 3 | "version": "2.3.0", 4 | "description": "A collection traversal library for Firestore", 5 | "keywords": [ 6 | "Firewalk", 7 | "firewalk", 8 | "Firebase", 9 | "firebase", 10 | "Firestore", 11 | "firestore", 12 | "Cloud Firestore", 13 | "cloud firestore", 14 | "TypeScript", 15 | "typescript" 16 | ], 17 | "author": "Anar Kafkas ", 18 | "homepage": "https://kafkas.github.io/firewalk", 19 | "license": "MIT", 20 | "main": "dist/index.js", 21 | "types": "dist/index.d.ts", 22 | "directories": { 23 | "dist": "dist", 24 | "test": "__tests__" 25 | }, 26 | "files": [ 27 | "dist" 28 | ], 29 | "repository": { 30 | "type": "git", 31 | "url": "git+https://github.com/kafkas/firewalk.git" 32 | }, 33 | "devDependencies": { 34 | "@babel/core": "^7.17.0", 35 | "@babel/preset-env": "^7.16.11", 36 | "@babel/preset-typescript": "^7.16.7", 37 | "@types/compare-versions": "^3.3.0", 38 | "@types/jest": "^29.5.10", 39 | "@types/lodash": "^4.14.178", 40 | "@types/node": "^22.10.2", 41 | "@typescript-eslint/eslint-plugin": "^5.10.2", 42 | "@typescript-eslint/parser": "^5.10.2", 43 | "babel-jest": "^27.5.1", 44 | "compare-versions": "^4.1.3", 45 | "eslint": "^8.8.0", 46 | "eslint-config-prettier": "^8.3.0", 47 | "eslint-plugin-import": "^2.25.4", 48 | "eslint-plugin-prettier": "^4.0.0", 49 | "firebase-admin": "^13.0.1", 50 | "firebase-tools": "^12.9.1", 51 | "jest": "^29.7.0", 52 | "lodash": "^4.17.21", 53 | "prettier": "2.5.1", 54 | "ts-node": "^10.4.0", 55 | "tsd": "0.31.2", 56 | "typedoc": "0.27.5", 57 | "typescript": "5.7.2" 58 | }, 59 | "scripts": { 60 | "build": "rm -rf dist && tsc -p tsconfig.build.json", 61 | "docs-generate": "sh scripts/generate-docs.sh", 62 | "format": "prettier --write \"{src,test}/**/*.{ts,tsx,json}\"", 63 | "format-check": "prettier --check \"{src,test}/**/*.{ts,tsx,json}\"", 64 | "lint": "eslint \"{src,__tests__}/**/*/*.{js,ts}\"", 65 | "test": "npm run test:compile && npm run test:types && npm run test:jest", 66 | "test:compile": "tsc -p tsconfig.test.json", 67 | "test:jest": "firebase emulators:exec --project demo-firewalk --only firestore 'jest'", 68 | "test:types": "tsd" 69 | }, 70 | "bugs": { 71 | "url": "https://github.com/kafkas/firewalk/issues" 72 | }, 73 | "peerDependencies": { 74 | "firebase-admin": "11 - 13" 75 | }, 76 | "publishConfig": { 77 | "access": "public" 78 | }, 79 | "tsd": { 80 | "directory": "__tests__/types" 81 | }, 82 | "dependencies": { 83 | "@proficient/ds": "0.3.2", 84 | "@proficient/util": "0.2.2" 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /scripts/generate-docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Run from project root 3 | 4 | # Parse version 5 | complete_version=$(npm pkg get version | tr -d '"') 6 | 7 | # Extract the major version 8 | major_version=$(echo $complete_version | awk -F '.' '{print $1}') 9 | 10 | typedoc --out docs/v$major_version 11 | -------------------------------------------------------------------------------- /src/api/createBatchMigrator.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import { isTraverser } from '../internal/utils'; 3 | import { BasicBatchMigratorImpl } from '../internal/implementations'; 4 | import type { InvalidConfigError } from '../errors'; /* eslint-disable-line */ 5 | import type { BatchMigrator, Traversable, TraversalConfig, Traverser } from './interfaces'; 6 | import { createTraverser } from './createTraverser'; 7 | 8 | /** 9 | * Creates a migrator that facilitates database migrations. Accepts a custom traverser object as argument which the 10 | * migrator will use when traversing the collection and writing to documents. This migrator uses atomic batch writes 11 | * when writing to docs so the entire operation will fail if a single write isn't successful. 12 | * 13 | * @remarks 14 | * 15 | * Note that the {@link TraversalConfig.batchSize} config value must not exceed 500 for a traverser used in a {@link BatchMigrator}. 16 | * This is because in Firestore, each write batch can write to a maximum of 500 documents. 17 | * 18 | * @param traverser - The traverser object that this migrator will use when traversing the collection and writing to documents. 19 | * @returns A new {@link BatchMigrator} object. 20 | * @throws {@link InvalidConfigError} Thrown if the traversal config of the specified traverser is not compatible with this migrator. 21 | */ 22 | export function createBatchMigrator< 23 | AppModelType = firestore.DocumentData, 24 | DbModelType extends firestore.DocumentData = firestore.DocumentData 25 | >(traverser: Traverser): BatchMigrator; 26 | 27 | /** 28 | * Creates a migrator that facilitates database migrations. The migrator creates a default traverser that 29 | * it uses when traversing the collection and writing to documents. This migrator uses atomic batch writes when writing 30 | * to docs so the entire operation will fail if a single write isn't successful. 31 | * 32 | * @remarks 33 | * 34 | * Note that the {@link TraversalConfig.batchSize} config value must not exceed 500 for a traverser used in a {@link BatchMigrator}. 35 | * This is because in Firestore, each write batch can write to a maximum of 500 documents. 36 | * 37 | * @param traversable - A collection-like traversable group of documents. 38 | * @param traversalConfig - Optional. The traversal configuration with which the default traverser will be created. 39 | * @returns A new {@link BatchMigrator} object. 40 | * @throws {@link InvalidConfigError} Thrown if the specified `traversalConfig` is invalid or incompatible with this migrator. 41 | */ 42 | export function createBatchMigrator< 43 | AppModelType = firestore.DocumentData, 44 | DbModelType extends firestore.DocumentData = firestore.DocumentData 45 | >( 46 | traversable: Traversable, 47 | traversalConfig?: Partial 48 | ): BatchMigrator; 49 | 50 | export function createBatchMigrator< 51 | AppModelType = firestore.DocumentData, 52 | DbModelType extends firestore.DocumentData = firestore.DocumentData 53 | >( 54 | traversableOrTraverser: 55 | | Traverser 56 | | Traversable, 57 | traversalConfig?: Partial 58 | ): BatchMigrator { 59 | const traverser = isTraverser(traversableOrTraverser) 60 | ? (traversableOrTraverser as Traverser) 61 | : createTraverser( 62 | traversableOrTraverser as Traversable, 63 | traversalConfig 64 | ); 65 | return new BasicBatchMigratorImpl(traverser); 66 | } 67 | -------------------------------------------------------------------------------- /src/api/createMigrator.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import { isTraverser } from '../internal/utils'; 3 | import { BasicDefaultMigratorImpl } from '../internal/implementations'; 4 | import type { InvalidConfigError } from '../errors'; /* eslint-disable-line */ 5 | import type { DefaultMigrator, Traversable, TraversalConfig, Traverser } from './interfaces'; 6 | import { createTraverser } from './createTraverser'; 7 | 8 | /** 9 | * Creates a migrator that facilitates database migrations. Accepts a custom traverser object as argument which the 10 | * migrator will use when traversing the collection and writing to documents. This migrator does not use atomic batch 11 | * writes so it is possible that when a write fails other writes go through. 12 | * 13 | * @param traverser - The traverser object that this migrator will use when traversing the collection and writing to documents. 14 | * @returns A new {@link DefaultMigrator} object. 15 | */ 16 | export function createMigrator< 17 | AppModelType = firestore.DocumentData, 18 | DbModelType extends firestore.DocumentData = firestore.DocumentData 19 | >(traverser: Traverser): DefaultMigrator; 20 | 21 | /** 22 | * Creates a migrator that facilitates database migrations. The migrator creates a default traverser that 23 | * it uses when traversing the collection and writing to documents. This migrator does not use atomic batch writes 24 | * so it is possible that when a write fails other writes go through. 25 | * 26 | * @param traversable - A collection-like traversable group of documents to migrate. 27 | * @param traversalConfig - Optional. The traversal configuration with which the default traverser will be created. 28 | * @returns A new {@link DefaultMigrator} object. 29 | * @throws {@link InvalidConfigError} Thrown if the specified `traversalConfig` is invalid. 30 | */ 31 | export function createMigrator< 32 | AppModelType = firestore.DocumentData, 33 | DbModelType extends firestore.DocumentData = firestore.DocumentData 34 | >( 35 | traversable: Traversable, 36 | traversalConfig?: Partial 37 | ): DefaultMigrator; 38 | 39 | export function createMigrator< 40 | AppModelType = firestore.DocumentData, 41 | DbModelType extends firestore.DocumentData = firestore.DocumentData 42 | >( 43 | traversableOrTraverser: 44 | | Traverser 45 | | Traversable, 46 | traversalConfig?: Partial 47 | ): DefaultMigrator { 48 | const traverser = isTraverser(traversableOrTraverser) 49 | ? (traversableOrTraverser as Traverser) 50 | : createTraverser( 51 | traversableOrTraverser as Traversable, 52 | traversalConfig 53 | ); 54 | return new BasicDefaultMigratorImpl(traverser); 55 | } 56 | -------------------------------------------------------------------------------- /src/api/createTraverser.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import { PromiseQueueBasedTraverserImpl } from '../internal/implementations'; 3 | import type { InvalidConfigError } from '../errors'; /* eslint-disable-line */ 4 | import type { Traversable, TraversalConfig, Traverser } from './interfaces'; 5 | 6 | /** 7 | * Creates an object which can be used to traverse a Firestore collection or, more generally, 8 | * a {@link Traversable}. 9 | * 10 | * @remarks 11 | * 12 | * For each batch of document snapshots in the traversable, this traverser invokes a specified 13 | * async callback and immediately moves to the next batch. It does not wait for the callback 14 | * Promise to resolve before moving to the next batch. That is, when `maxConcurrentBatchCount` > 1, 15 | * there is no guarantee that any given batch will finish processing before a later batch. 16 | * 17 | * The traverser becomes faster as you increase `maxConcurrentBatchCount`, but this will consume 18 | * more memory. You should increase concurrency when you want to trade some memory for speed. 19 | * 20 | * @param traversable - A collection-like group of documents. Can be one of [CollectionReference](https://googleapis.dev/nodejs/firestore/latest/CollectionReference.html), [CollectionGroup](https://googleapis.dev/nodejs/firestore/latest/CollectionGroup.html) and [Query](https://googleapis.dev/nodejs/firestore/latest/Query.html). 21 | * @param config - Optional. The traversal configuration with which the traverser will be created. 22 | * @returns A new {@link Traverser} object. 23 | * @throws {@link InvalidConfigError} Thrown if the specified `config` is invalid. 24 | */ 25 | export function createTraverser< 26 | AppModelType = firestore.DocumentData, 27 | DbModelType extends firestore.DocumentData = firestore.DocumentData 28 | >( 29 | traversable: Traversable, 30 | config?: Partial 31 | ): Traverser { 32 | return new PromiseQueueBasedTraverserImpl(traversable, [], config); 33 | } 34 | -------------------------------------------------------------------------------- /src/api/index.ts: -------------------------------------------------------------------------------- 1 | export { createBatchMigrator } from './createBatchMigrator'; 2 | export { createMigrator } from './createMigrator'; 3 | export { createTraverser } from './createTraverser'; 4 | 5 | export * from './interfaces'; 6 | -------------------------------------------------------------------------------- /src/api/interfaces/BatchCallback.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | 3 | /** 4 | * A function that takes batch doc snapshots and the 0-based batch index as its arguments. 5 | */ 6 | export type BatchCallback< 7 | AppModelType = firestore.DocumentData, 8 | DbModelType extends firestore.DocumentData = firestore.DocumentData 9 | > = ( 10 | batchDocs: firestore.QueryDocumentSnapshot[], 11 | batchIndex: number 12 | ) => void | Promise; 13 | -------------------------------------------------------------------------------- /src/api/interfaces/BatchMigrator.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { MigrationPredicate, Migrator, Traverser } from '.'; 3 | 4 | /** 5 | * A batch migrator object that uses atomic batch writes. 6 | */ 7 | export interface BatchMigrator< 8 | AppModelType = firestore.DocumentData, 9 | DbModelType extends firestore.DocumentData = firestore.DocumentData 10 | > extends Migrator { 11 | /** 12 | * Applies a migration predicate that indicates whether to migrate the current document or not. By default, all 13 | * documents are migrated. 14 | * 15 | * @remarks 16 | * 17 | * If you have already applied other migration predicates to this migrator, this and all the other predicates will be 18 | * evaluated and the resulting booleans will be AND'd to get the boolean that indicates whether to migrate the document 19 | * or not. This is consistent with the intuitive default behavior that all documents are migrated. 20 | * 21 | * @example 22 | * 23 | * ```ts 24 | * const newMigrator = migrator 25 | * .withPredicate((doc) => doc.get('name') !== undefined) 26 | * .withPredicate((doc) => doc.ref.path.startsWith('users/')); 27 | * ``` 28 | * 29 | * In the above case `newMigrator` will migrate only the documents whose `name` field is not missing AND whose path 30 | * starts with `"users/"`. 31 | * 32 | * @param predicate - A function that takes a document snapshot and returns a boolean indicating whether to migrate it. 33 | * @returns A new {@link BatchMigrator} object. 34 | */ 35 | withPredicate( 36 | predicate: MigrationPredicate 37 | ): BatchMigrator; 38 | 39 | /** 40 | * Applies a new traverser that will be used by the migrator. 41 | * 42 | * @param traverser - The new traverser that the migrator will use. 43 | * @returns A new {@link BatchMigrator} object. 44 | */ 45 | withTraverser( 46 | traverser: Traverser 47 | ): BatchMigrator; 48 | } 49 | -------------------------------------------------------------------------------- /src/api/interfaces/DefaultMigrator.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { MigrationPredicate, Migrator, Traverser } from '.'; 3 | 4 | /** 5 | * A batch migrator object that does not use atomic batch writes. 6 | */ 7 | export interface DefaultMigrator< 8 | AppModelType = firestore.DocumentData, 9 | DbModelType extends firestore.DocumentData = firestore.DocumentData 10 | > extends Migrator { 11 | /** 12 | * Applies a migration predicate that indicates whether to migrate the current document or not. By default, all 13 | * documents are migrated. 14 | * 15 | * @remarks 16 | * 17 | * If you have already applied other migration predicates to this migrator, this and all the other predicates will be 18 | * evaluated and the resulting booleans will be AND'd to get the boolean that indicates whether to migrate the document 19 | * or not. This is consistent with the intuitive default behavior that all documents are migrated. 20 | * 21 | * @example 22 | * 23 | * ```ts 24 | * const newMigrator = migrator 25 | * .withPredicate((doc) => doc.get('name') !== undefined) 26 | * .withPredicate((doc) => doc.ref.path.startsWith('users/')); 27 | * ``` 28 | * 29 | * In the above case `newMigrator` will migrate only the documents whose `name` field is not missing AND whose path 30 | * starts with `"users/"`. 31 | * 32 | * @param predicate - A function that takes a document snapshot and returns a boolean indicating whether to migrate it. 33 | * @returns A new {@link DefaultMigrator} object. 34 | */ 35 | withPredicate( 36 | predicate: MigrationPredicate 37 | ): DefaultMigrator; 38 | 39 | /** 40 | * Applies a new traverser that will be used by the migrator. 41 | * 42 | * @param traverser - The new traverser that the migrator will use. 43 | * @returns A new {@link DefaultMigrator} object. 44 | */ 45 | withTraverser( 46 | traverser: Traverser 47 | ): DefaultMigrator; 48 | } 49 | -------------------------------------------------------------------------------- /src/api/interfaces/ExitEarlyPredicate.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { BatchCallback } from '.'; 3 | 4 | /** 5 | * A function that takes batch doc snapshots and the 0-based batch index and returns a boolean 6 | * indicating whether to exit traversal early. 7 | */ 8 | export type ExitEarlyPredicate< 9 | AppModelType = firestore.DocumentData, 10 | DbModelType extends firestore.DocumentData = firestore.DocumentData 11 | > = (...args: Parameters>) => boolean; 12 | -------------------------------------------------------------------------------- /src/api/interfaces/MigrationPredicate.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | 3 | /** 4 | * A function that takes a document snapshot and returns a boolean indicating whether to migrate it. 5 | */ 6 | export type MigrationPredicate< 7 | AppModelType = firestore.DocumentData, 8 | DbModelType extends firestore.DocumentData = firestore.DocumentData 9 | > = (doc: firestore.QueryDocumentSnapshot) => boolean; 10 | -------------------------------------------------------------------------------- /src/api/interfaces/MigrationResult.ts: -------------------------------------------------------------------------------- 1 | import type { TraversalResult } from '.'; 2 | 3 | /** 4 | * Represents an object that contains the details of a migration. 5 | */ 6 | export interface MigrationResult { 7 | /** 8 | * The traversal result of the underlying traverser. 9 | */ 10 | readonly traversalResult: TraversalResult; 11 | 12 | /** 13 | * The number of documents that have been migrated. 14 | */ 15 | readonly migratedDocCount: number; 16 | } 17 | -------------------------------------------------------------------------------- /src/api/interfaces/Migrator.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { 3 | BatchCallback, 4 | MigrationPredicate, 5 | MigrationResult, 6 | SetDataGetter, 7 | SetOptions, 8 | SetPartialDataGetter, 9 | Traverser, 10 | UpdateDataGetter, 11 | UpdateFieldValueGetter, 12 | } from '.'; 13 | 14 | /** 15 | * Represents the general interface of a migrator. 16 | */ 17 | export interface Migrator< 18 | AppModelType = firestore.DocumentData, 19 | DbModelType extends firestore.DocumentData = firestore.DocumentData 20 | > { 21 | /** 22 | * The underlying traverser. 23 | */ 24 | readonly traverser: Traverser; 25 | 26 | /** 27 | * Applies a migration predicate that indicates whether to migrate the current document or not. By default, all 28 | * documents are migrated. 29 | * 30 | * @remarks 31 | * 32 | * If you have already applied other migration predicates to this migrator, this and all the other predicates will be 33 | * evaluated and the resulting booleans will be AND'd to get the boolean that indicates whether to migrate the document 34 | * or not. This is consistent with the intuitive default behavior that all documents are migrated. 35 | * 36 | * @example 37 | * 38 | * ```ts 39 | * const newMigrator = migrator 40 | * .withPredicate((doc) => doc.get('name') !== undefined) 41 | * .withPredicate((doc) => doc.ref.path.startsWith('users/')); 42 | * ``` 43 | * 44 | * In the above case `newMigrator` will migrate only the documents whose `name` field is not missing AND whose path 45 | * starts with `"users/"`. 46 | * 47 | * @param predicate - A function that takes a document snapshot and returns a boolean indicating whether to migrate it. 48 | * @returns A new {@link Migrator} object. 49 | */ 50 | withPredicate( 51 | predicate: MigrationPredicate 52 | ): Migrator; 53 | 54 | /** 55 | * Applies a new traverser that will be used by the migrator. 56 | * 57 | * @param traverser - The new traverser that the migrator will use. 58 | * @returns A new {@link Migrator} object. 59 | */ 60 | withTraverser( 61 | traverser: Traverser 62 | ): Migrator; 63 | 64 | /** 65 | * Registers a callback function that fires right before a batch starts processing. You can register at most 1 66 | * callback. If you call this function multiple times, only the last callback will be registered. 67 | * 68 | * @param callback - A synchronous callback that takes batch doc snapshots and the 0-based batch index as its arguments. 69 | */ 70 | onBeforeBatchStart(callback: BatchCallback): void; 71 | 72 | /** 73 | * Registers a callback function that fires after a batch is processed. You can register at most 1 callback. If you call 74 | * this function multiple times, only the last callback will be registered. 75 | * 76 | * @param callback - A synchronous callback that takes batch doc snapshots and the 0-based batch index as its arguments. 77 | */ 78 | onAfterBatchComplete(callback: BatchCallback): void; 79 | 80 | /** 81 | * Deletes the specified field from all documents in this collection. 82 | * 83 | * @remarks 84 | * 85 | * **Complexity:** 86 | * 87 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 88 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 89 | * - Billing: _max_(1, _N_) reads, _K_ writes 90 | * 91 | * where: 92 | * 93 | * - _N_: number of docs in the traversable 94 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 95 | * - _W_(`batchSize`): average batch write time 96 | * - _TC_(`traverser`): time complexity of the underlying traverser 97 | * - _SC_(`traverser`): space complexity of the underlying traverser 98 | * 99 | * @param field - The field to delete. 100 | * @returns A Promise resolving to an object representing the details of the migration. 101 | */ 102 | deleteField(field: string | firestore.FieldPath): Promise; 103 | 104 | /** 105 | * Deletes the specified fields from all documents in this collection. 106 | * 107 | * @remarks 108 | * 109 | * **Complexity:** 110 | * 111 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 112 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 113 | * - Billing: _max_(1, _N_) reads, _K_ writes 114 | * 115 | * where: 116 | * 117 | * - _N_: number of docs in the traversable 118 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 119 | * - _W_(`batchSize`): average batch write time 120 | * - _TC_(`traverser`): time complexity of the underlying traverser 121 | * - _SC_(`traverser`): space complexity of the underlying traverser 122 | * 123 | * @example 124 | * 125 | * ```ts 126 | * const field1 = new firestore.FieldPath('name', 'last'); 127 | * const field2 = 'lastModifiedAt'; 128 | * await migrator.deleteFields(field1, field2); 129 | * ``` 130 | * 131 | * @param fields - A list of fields to delete. 132 | * @returns A Promise resolving to an object representing the details of the migration. 133 | */ 134 | deleteFields(...fields: (string | firestore.FieldPath)[]): Promise; 135 | 136 | /** 137 | * Renames the specified field in all documents in this collection. Ignores the fields that are missing. 138 | * 139 | * @remarks 140 | * 141 | * **Complexity:** 142 | * 143 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 144 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 145 | * - Billing: _max_(1, _N_) reads, _K_ writes 146 | * 147 | * where: 148 | * 149 | * - _N_: number of docs in the traversable 150 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 151 | * - _W_(`batchSize`): average batch write time 152 | * - _TC_(`traverser`): time complexity of the underlying traverser 153 | * - _SC_(`traverser`): space complexity of the underlying traverser 154 | * 155 | * @param oldField - The old field. 156 | * @param newField - The new field. 157 | * @returns A Promise resolving to an object representing the details of the migration. 158 | */ 159 | renameField( 160 | oldField: string | firestore.FieldPath, 161 | newField: string | firestore.FieldPath 162 | ): Promise; 163 | 164 | /** 165 | * Renames the specified fields in all documents in this collection. Ignores the fields that are missing. 166 | * 167 | * @remarks 168 | * 169 | * **Complexity:** 170 | * 171 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 172 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 173 | * - Billing: _max_(1, _N_) reads, _K_ writes 174 | * 175 | * where: 176 | * 177 | * - _N_: number of docs in the traversable 178 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 179 | * - _W_(`batchSize`): average batch write time 180 | * - _TC_(`traverser`): time complexity of the underlying traverser 181 | * - _SC_(`traverser`): space complexity of the underlying traverser 182 | * 183 | * @example 184 | * 185 | * ```ts 186 | * const change1 = [new firestore.FieldPath('name', 'last'), 'lastName']; 187 | * const change2 = ['lastModifiedAt', 'lastUpdatedAt']; 188 | * await migrator.renameFields(change1, change2); 189 | * ``` 190 | * 191 | * @param changes - A list of `[oldField, newField]` tuples. Each tuple is an array of 2 elements: 192 | * the old field to rename and the new field. 193 | * @returns A Promise resolving to an object representing the details of the migration. 194 | */ 195 | renameFields( 196 | ...changes: [oldField: string | firestore.FieldPath, newField: string | firestore.FieldPath][] 197 | ): Promise; 198 | 199 | /** 200 | * Sets all documents in this collection with the provided data. 201 | * 202 | * @remarks 203 | * 204 | * **Complexity:** 205 | * 206 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 207 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 208 | * - Billing: _max_(1, _N_) reads, _K_ writes 209 | * 210 | * where: 211 | * 212 | * - _N_: number of docs in the traversable 213 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 214 | * - _W_(`batchSize`): average batch write time 215 | * - _TC_(`traverser`): time complexity of the underlying traverser 216 | * - _SC_(`traverser`): space complexity of the underlying traverser 217 | * 218 | * @param data - A data object with which to set each document. 219 | * @param options - An object to configure the set behavior. 220 | * @returns A Promise resolving to an object representing the details of the migration. 221 | */ 222 | set( 223 | data: firestore.PartialWithFieldValue, 224 | options: SetOptions 225 | ): Promise; 226 | 227 | /** 228 | * Sets all documents in this collection with the provided data. 229 | * 230 | * @remarks 231 | * 232 | * **Complexity:** 233 | * 234 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 235 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 236 | * - Billing: _max_(1, _N_) reads, _K_ writes 237 | * 238 | * where: 239 | * 240 | * - _N_: number of docs in the traversable 241 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 242 | * - _W_(`batchSize`): average batch write time 243 | * - _TC_(`traverser`): time complexity of the underlying traverser 244 | * - _SC_(`traverser`): space complexity of the underlying traverser 245 | * 246 | * @param data - A data object with which to set each document. 247 | * @returns A Promise resolving to an object representing the details of the migration. 248 | */ 249 | set(data: firestore.WithFieldValue): Promise; 250 | 251 | /** 252 | * Sets all documents in this collection with the provided data. 253 | * 254 | * @remarks 255 | * 256 | * **Complexity:** 257 | * 258 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 259 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 260 | * - Billing: _max_(1, _N_) reads, _K_ writes 261 | * 262 | * where: 263 | * 264 | * - _N_: number of docs in the traversable 265 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 266 | * - _W_(`batchSize`): average batch write time 267 | * - _TC_(`traverser`): time complexity of the underlying traverser 268 | * - _SC_(`traverser`): space complexity of the underlying traverser 269 | * 270 | * @param getData - A function that takes a document snapshot and returns a data object with 271 | * which to set each document. 272 | * @param options - An object to configure the set behavior. 273 | * @returns A Promise resolving to an object representing the details of the migration. 274 | */ 275 | setWithDerivedData( 276 | getData: SetPartialDataGetter, 277 | options: SetOptions 278 | ): Promise; 279 | 280 | /** 281 | * Sets all documents in this collection with the provided data. 282 | * 283 | * @remarks 284 | * 285 | * **Complexity:** 286 | * 287 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 288 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 289 | * - Billing: _max_(1, _N_) reads, _K_ writes 290 | * 291 | * where: 292 | * 293 | * - _N_: number of docs in the traversable 294 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 295 | * - _W_(`batchSize`): average batch write time 296 | * - _TC_(`traverser`): time complexity of the underlying traverser 297 | * - _SC_(`traverser`): space complexity of the underlying traverser 298 | * 299 | * @param getData - A function that takes a document snapshot and returns a data object with 300 | * which to set each document. 301 | * @returns A Promise resolving to an object representing the details of the migration. 302 | */ 303 | setWithDerivedData(getData: SetDataGetter): Promise; 304 | 305 | /** 306 | * Updates all documents in this collection with the provided data. 307 | * 308 | * @remarks 309 | * 310 | * **Complexity:** 311 | * 312 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 313 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 314 | * - Billing: _max_(1, _N_) reads, _K_ writes 315 | * 316 | * where: 317 | * 318 | * - _N_: number of docs in the traversable 319 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 320 | * - _W_(`batchSize`): average batch write time 321 | * - _TC_(`traverser`): time complexity of the underlying traverser 322 | * - _SC_(`traverser`): space complexity of the underlying traverser 323 | * 324 | * @param data - A non-empty data object with which to update each document. 325 | * @param precondition - A Precondition to enforce on this update. 326 | * @returns A Promise resolving to an object representing the details of the migration. 327 | */ 328 | update( 329 | data: firestore.UpdateData, 330 | precondition?: firestore.Precondition 331 | ): Promise; 332 | 333 | /** 334 | * Updates all documents in this collection with the provided field-value pair. 335 | * 336 | * @remarks 337 | * 338 | * **Complexity:** 339 | * 340 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 341 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 342 | * - Billing: _max_(1, _N_) reads, _K_ writes 343 | * 344 | * where: 345 | * 346 | * - _N_: number of docs in the traversable 347 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 348 | * - _W_(`batchSize`): average batch write time 349 | * - _TC_(`traverser`): time complexity of the underlying traverser 350 | * - _SC_(`traverser`): space complexity of the underlying traverser 351 | * 352 | * @param field - The first field to update in each document. 353 | * @param value - The first value corresponding to the first field. Must not be `undefined`. 354 | * @param moreFieldsOrPrecondition - An alternating list of field paths and values to update, 355 | * optionally followed by a Precondition to enforce on this update. 356 | * 357 | * @returns A Promise resolving to an object representing the details of the migration. 358 | */ 359 | update( 360 | field: string | firestore.FieldPath, 361 | value: any, 362 | ...moreFieldsOrPrecondition: any[] 363 | ): Promise; 364 | 365 | /** 366 | * Updates all documents in this collection with the provided data. 367 | * 368 | * @remarks 369 | * 370 | * **Complexity:** 371 | * 372 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 373 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 374 | * - Billing: _max_(1, _N_) reads, _K_ writes 375 | * 376 | * where: 377 | * 378 | * - _N_: number of docs in the traversable 379 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 380 | * - _W_(`batchSize`): average batch write time 381 | * - _TC_(`traverser`): time complexity of the underlying traverser 382 | * - _SC_(`traverser`): space complexity of the underlying traverser 383 | * 384 | * @param getData - A function that takes a document snapshot and returns a non-empty data object with 385 | * which to update each document. 386 | * @returns A Promise resolving to an object representing the details of the migration. 387 | */ 388 | updateWithDerivedData( 389 | getData: UpdateDataGetter, 390 | precondition?: firestore.Precondition 391 | ): Promise; 392 | 393 | /** 394 | * Updates all documents in this collection with the provided data. 395 | * 396 | * @remarks 397 | * 398 | * **Complexity:** 399 | * 400 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`) 401 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`) 402 | * - Billing: _max_(1, _N_) reads, _K_ writes 403 | * 404 | * where: 405 | * 406 | * - _N_: number of docs in the traversable 407 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_) 408 | * - _W_(`batchSize`): average batch write time 409 | * - _TC_(`traverser`): time complexity of the underlying traverser 410 | * - _SC_(`traverser`): space complexity of the underlying traverser 411 | * 412 | * @param getData - A function that takes a document snapshot and returns an alternating list of field 413 | * paths and values to update, optionally followed by a Precondition to enforce on this update. 414 | * @returns A Promise resolving to an object representing the details of the migration. 415 | */ 416 | updateWithDerivedData( 417 | getData: UpdateFieldValueGetter 418 | ): Promise; 419 | } 420 | -------------------------------------------------------------------------------- /src/api/interfaces/SetDataGetter.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | 3 | /** 4 | * A function that takes a document snapshot and derives the data with which to set that document. 5 | */ 6 | export type SetDataGetter< 7 | AppModelType = firestore.DocumentData, 8 | DbModelType extends firestore.DocumentData = firestore.DocumentData 9 | > = ( 10 | doc: firestore.QueryDocumentSnapshot 11 | ) => firestore.WithFieldValue; 12 | -------------------------------------------------------------------------------- /src/api/interfaces/SetOptions.ts: -------------------------------------------------------------------------------- 1 | export type { SetOptions } from '@google-cloud/firestore'; 2 | -------------------------------------------------------------------------------- /src/api/interfaces/SetPartialDataGetter.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | 3 | /** 4 | * A function that takes a document snapshot and derives the data with which to partially set that document. 5 | */ 6 | export type SetPartialDataGetter< 7 | AppModelType = firestore.DocumentData, 8 | DbModelType extends firestore.DocumentData = firestore.DocumentData 9 | > = ( 10 | doc: firestore.QueryDocumentSnapshot 11 | ) => firestore.PartialWithFieldValue; 12 | -------------------------------------------------------------------------------- /src/api/interfaces/Traversable.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | 3 | /** 4 | * A collection-like group of documents. Can be one of [CollectionReference](https://googleapis.dev/nodejs/firestore/latest/CollectionReference.html), 5 | * [CollectionGroup](https://googleapis.dev/nodejs/firestore/latest/CollectionGroup.html) and [Query](https://googleapis.dev/nodejs/firestore/latest/Query.html). 6 | */ 7 | export type Traversable< 8 | AppModelType = firestore.DocumentData, 9 | DbModelType extends firestore.DocumentData = firestore.DocumentData 10 | > = 11 | | firestore.CollectionReference 12 | | firestore.CollectionGroup 13 | | firestore.Query; 14 | -------------------------------------------------------------------------------- /src/api/interfaces/TraversalConfig.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The configuration with which a traverser is created. 3 | */ 4 | export interface TraversalConfig { 5 | /** 6 | * The number of documents that will be traversed in each batch. 7 | * 8 | * @defaultValue 250 9 | */ 10 | readonly batchSize: number; 11 | 12 | /** 13 | * The amount of time (in ms) to "sleep" before moving on to the next batch. 14 | * 15 | * @defaultValue 0 16 | */ 17 | readonly sleepTimeBetweenBatches: number; 18 | 19 | /** 20 | * The maximum number of documents that will be traversed. 21 | * 22 | * @defaultValue `Infinity` 23 | */ 24 | readonly maxDocCount: number; 25 | 26 | /** 27 | * The maximum number of batches that can be held in memory and processed concurrently. 28 | * 29 | * @remarks 30 | * 31 | * This field must be a positive integer representing the maximum number of batches that you wish 32 | * the traverser to process concurrently at any given time. The traverser will pause when the number 33 | * of batches being processed concurrently reaches this number and continue when a batch has been 34 | * processed. This means that the higher the value of `maxConcurrentBatchCount`, the more memory the 35 | * traverser will consume but also the faster it will finish the traversal. 36 | * 37 | * @example 38 | * 39 | * ```typescript 40 | * const projectsColRef = firestore().collection('projects'); 41 | * const traverser = createTraverser(projectsColRef, { 42 | * batchSize: 500, 43 | * maxConcurrentBatchCount: 20, 44 | * }); 45 | * ``` 46 | * 47 | * By providing this config we have indicated that we want each batch to contain 500 documents and the 48 | * traverser to process 20 batches concurrently at any given time. This means we have ensured that our 49 | * machine can handle 500 * 20 = 10,000 documents in memory at any given time. 50 | * 51 | * @defaultValue 1 52 | */ 53 | readonly maxConcurrentBatchCount: number; 54 | 55 | /** 56 | * The maximum number of times the traverser will retry processing a given batch, in case of an error. 57 | * By default, batches are not retried. 58 | * 59 | * @remarks 60 | * 61 | * This field must be a non-negative integer representing the maximum number of times the traverser will 62 | * retry processing a given batch. By default, the traverser invokes the batch callback only once i.e. 63 | * with 0 retries but, if `maxBatchRetryCount` > 0, it will keep invoking the callback until it succeeds 64 | * or the total number of retries reaches `maxBatchRetryCount`. 65 | * 66 | * @defaultValue 0 67 | */ 68 | readonly maxBatchRetryCount: number; 69 | 70 | /** 71 | * A non-negative integer or a function that takes the 0-based index of the last trial and returns a 72 | * non-negative integer indicating the amount of time (in ms) to "sleep" before retrying processing 73 | * the current batch. This is useful if you want to implement something like exponential backoff. 74 | * 75 | * @defaultValue 1000 76 | */ 77 | readonly sleepTimeBetweenTrials: number | ((lastTrialIndex: number) => number); 78 | } 79 | -------------------------------------------------------------------------------- /src/api/interfaces/TraversalResult.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Represents an object that contains the details of a traversal. 3 | */ 4 | export interface TraversalResult { 5 | /** 6 | * The number of batches that have been retrieved in this traversal. 7 | */ 8 | readonly batchCount: number; 9 | 10 | /** 11 | * The number of documents that have been retrieved in this traversal. 12 | */ 13 | readonly docCount: number; 14 | } 15 | -------------------------------------------------------------------------------- /src/api/interfaces/TraverseEachCallback.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | 3 | /** 4 | * An asynchronous function that takes a document snapshot, its 0-based index within the batch, and 5 | * the 0-based index of the batch as arguments. 6 | */ 7 | export type TraverseEachCallback< 8 | AppModelType = firestore.DocumentData, 9 | DbModelType extends firestore.DocumentData = firestore.DocumentData 10 | > = ( 11 | doc: firestore.QueryDocumentSnapshot, 12 | docIndex: number, 13 | batchIndex: number 14 | ) => void | Promise; 15 | -------------------------------------------------------------------------------- /src/api/interfaces/TraverseEachConfig.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The configuration that a given traverser uses in sequential traversals. 3 | */ 4 | export interface TraverseEachConfig { 5 | /** 6 | * The amount of time (in ms) to "sleep" before moving to the next doc. 7 | * 8 | * @defaultValue 0 9 | */ 10 | readonly sleepTimeBetweenDocs: number; 11 | } 12 | -------------------------------------------------------------------------------- /src/api/interfaces/Traverser.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { 3 | BatchCallback, 4 | ExitEarlyPredicate, 5 | Traversable, 6 | TraversalConfig, 7 | TraversalResult, 8 | TraverseEachCallback, 9 | TraverseEachConfig, 10 | } from '.'; 11 | 12 | /** 13 | * A traverser object that facilitates Firestore collection traversals. 14 | */ 15 | export interface Traverser< 16 | AppModelType = firestore.DocumentData, 17 | DbModelType extends firestore.DocumentData = firestore.DocumentData 18 | > { 19 | /** 20 | * The underlying traversable. 21 | */ 22 | readonly traversable: Traversable; 23 | 24 | /** 25 | * Existing traversal config. 26 | */ 27 | readonly traversalConfig: TraversalConfig; 28 | 29 | /** 30 | * Applies the specified config values to the traverser. 31 | * 32 | * @param config - Partial traversal configuration. 33 | * @returns A new {@link Traverser} object. 34 | */ 35 | withConfig(config: Partial): Traverser; 36 | 37 | /** 38 | * Applies the specified exit-early predicate to the traverser. After retrieving each batch, the traverser 39 | * will evaluate the predicate with the current batch doc snapshots and batch index and decide whether to 40 | * continue the traversal or exit early. 41 | * 42 | * @remarks 43 | * 44 | * If you have already applied other exit-early predicates to this traverser, this and all the other predicates 45 | * will be evaluated and the resulting booleans will be OR'd to get the boolean that indicates whether to exit 46 | * early or not. This is consistent with the intuitive default behavior that the traverser doesn't exit early. 47 | * 48 | * @example 49 | * 50 | * ```ts 51 | * const newTraverser = traverser 52 | * .withExitEarlyPredicate((batchDocs) => batchDocs.some((d) => d.get('name') === undefined)) 53 | * .withExitEarlyPredicate((_, batchIndex) => batchIndex === 99); 54 | * ``` 55 | * 56 | * In the above case `newTraverser` will exit early if the `name` field of one of the docs in the batch is 57 | * missing OR if the batch index is 99. That is, it will never reach batch 100 and depending on the documents 58 | * in the database it may exit earlier than 99. 59 | * 60 | * @param predicate - A synchronous function that takes batch doc snapshots and the 0-based batch index and 61 | * returns a boolean indicating whether to exit traversal early. 62 | * @returns A new {@link Traverser} object. 63 | */ 64 | withExitEarlyPredicate( 65 | predicate: ExitEarlyPredicate 66 | ): Traverser; 67 | 68 | /** 69 | * Traverses the entire collection in batches of the size specified in traversal config. Invokes the specified 70 | * callback sequentially for each document snapshot in each batch. 71 | * 72 | * @param callback - An asynchronous callback function to invoke for each document snapshot in each batch. 73 | * @param config - The sequential traversal configuration. 74 | * @returns A Promise resolving to an object representing the details of the traversal. The Promise resolves 75 | * when the entire traversal ends. 76 | */ 77 | traverseEach( 78 | callback: TraverseEachCallback, 79 | config?: Partial 80 | ): Promise; 81 | 82 | /** 83 | * Traverses the entire collection in batches of the size specified in traversal config. Invokes the specified 84 | * async callback for each batch of document snapshots and immediately moves to the next batch. Does not wait 85 | * for the callback Promise to resolve before moving to the next batch so there is no guarantee that any given 86 | * batch will finish processing before a later batch. 87 | * 88 | * @remarks 89 | * 90 | * This method throws only when the batch callback throws and the number of retries equals `maxBatchRetryCount`. 91 | * The last error thrown by the batch callback propagates up to this method. 92 | * 93 | * **Complexity:** 94 | * 95 | * - Time complexity: _O_((_N_ / `batchSize`) \* (_Q_(`batchSize`) + _C_(`batchSize`) / `maxConcurrentBatchCount`)) 96 | * - Space complexity: _O_(`maxConcurrentBatchCount` * (`batchSize` * _D_ + _S_)) 97 | * - Billing: _max_(1, _N_) reads 98 | * 99 | * where: 100 | * 101 | * - _N_: number of docs in the traversable 102 | * - _Q_(`batchSize`): average batch query time 103 | * - _C_(`batchSize`): average callback processing time 104 | * - _D_: average document size 105 | * - _S_: average extra space used by the callback 106 | * 107 | * @param callback - An asynchronous callback function to invoke for each batch of document snapshots. 108 | * @returns A Promise resolving to an object representing the details of the traversal. 109 | */ 110 | traverse(callback: BatchCallback): Promise; 111 | } 112 | -------------------------------------------------------------------------------- /src/api/interfaces/UpdateDataGetter.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | 3 | /** 4 | * A function that takes a document snapshot and derives the data with which to update that document. 5 | */ 6 | export type UpdateDataGetter< 7 | AppModelType = firestore.DocumentData, 8 | DbModelType extends firestore.DocumentData = firestore.DocumentData 9 | > = ( 10 | doc: firestore.QueryDocumentSnapshot 11 | ) => firestore.UpdateData; 12 | -------------------------------------------------------------------------------- /src/api/interfaces/UpdateFieldValueGetter.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | 3 | /** 4 | * A function that takes a document snapshot and returns an alternating list of field paths and 5 | * values to update within that document, optionally followed by a Precondition to enforce on 6 | * the update. 7 | */ 8 | export type UpdateFieldValueGetter< 9 | AppModelType = firestore.DocumentData, 10 | DbModelType extends firestore.DocumentData = firestore.DocumentData 11 | > = ( 12 | doc: firestore.QueryDocumentSnapshot 13 | ) => [string | firestore.FieldPath, any, ...any[]]; 14 | -------------------------------------------------------------------------------- /src/api/interfaces/index.ts: -------------------------------------------------------------------------------- 1 | export type { BatchCallback } from './BatchCallback'; 2 | export type { BatchMigrator } from './BatchMigrator'; 3 | export type { DefaultMigrator } from './DefaultMigrator'; 4 | export type { ExitEarlyPredicate } from './ExitEarlyPredicate'; 5 | export type { MigrationPredicate } from './MigrationPredicate'; 6 | export type { MigrationResult } from './MigrationResult'; 7 | export type { Migrator } from './Migrator'; 8 | export type { SetDataGetter } from './SetDataGetter'; 9 | export type { SetOptions } from './SetOptions'; 10 | export type { SetPartialDataGetter } from './SetPartialDataGetter'; 11 | export type { Traversable } from './Traversable'; 12 | export type { TraversalConfig } from './TraversalConfig'; 13 | export type { TraversalResult } from './TraversalResult'; 14 | export type { TraverseEachCallback } from './TraverseEachCallback'; 15 | export type { TraverseEachConfig } from './TraverseEachConfig'; 16 | export type { Traverser } from './Traverser'; 17 | export type { UpdateDataGetter } from './UpdateDataGetter'; 18 | export type { UpdateFieldValueGetter } from './UpdateFieldValueGetter'; 19 | -------------------------------------------------------------------------------- /src/errors/ImplementationError.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * An error thrown when a piece of code is found to be incorrectly implemented. If the maintainers of the 3 | * library have done everything right, this error should never be encountered. 4 | */ 5 | export class ImplementationError extends Error { 6 | constructor(message: string) { 7 | super(`Implementation Error: ${message}`); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/errors/InvalidConfigError.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * An error thrown when an invalid configuration is provided. 3 | */ 4 | export class InvalidConfigError extends Error { 5 | constructor(message: string) { 6 | super(`Invalid config: ${message}`); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/errors/index.ts: -------------------------------------------------------------------------------- 1 | export { ImplementationError } from './ImplementationError'; 2 | export { InvalidConfigError } from './InvalidConfigError'; 3 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './api'; 2 | export * from './errors'; 3 | -------------------------------------------------------------------------------- /src/internal/ds/PromiseQueue.ts: -------------------------------------------------------------------------------- 1 | import type { IllegalArgumentError } from '../errors'; /* eslint-disable-line */ 2 | import { SLLQueueExtended } from './SLLQueueExtended'; 3 | 4 | export class PromiseQueue { 5 | readonly #queue: SLLQueueExtended; 6 | readonly #map: Map>; 7 | #lastPromiseId: number; 8 | #isProcessing: boolean; 9 | 10 | public constructor() { 11 | this.#queue = new SLLQueueExtended(); 12 | this.#map = new Map(); 13 | this.#lastPromiseId = 0; 14 | this.#isProcessing = false; 15 | } 16 | 17 | public get size(): number { 18 | return this.#map.size; 19 | } 20 | 21 | public get isProcessing(): boolean { 22 | return this.#isProcessing; 23 | } 24 | 25 | public enqueue(promise: Promise): void { 26 | const promiseId = this.#getIdForNewPromise(); 27 | this.#queue.enqueue(promiseId); 28 | this.#map.set(promiseId, promise); 29 | } 30 | 31 | #getIdForNewPromise(): number { 32 | return ++this.#lastPromiseId; 33 | } 34 | 35 | /** 36 | * Processes all Promises in the queue. 37 | */ 38 | public processAll(): Promise { 39 | return this.processFirst(this.#queue.count); 40 | } 41 | 42 | /** 43 | * Processes the first `promiseCount` Promises in the queue. 44 | * @throws {@link IllegalArgumentError} Thrown if `promiseCount` is invalid. 45 | */ 46 | public async processFirst(promiseCount: number): Promise { 47 | this.#isProcessing = true; 48 | const promiseIds = this.#queue.dequeueFirst(promiseCount); 49 | const results = await Promise.all( 50 | promiseIds.map(async (id) => { 51 | const promise = this.#map.get(id)!; 52 | const result = await promise; 53 | this.#map.delete(id); 54 | return result; 55 | }) 56 | ); 57 | this.#isProcessing = false; 58 | return results; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/internal/ds/SLLQueueExtended.ts: -------------------------------------------------------------------------------- 1 | import { SLLQueue } from '@proficient/ds'; 2 | import { IllegalArgumentError } from '../errors'; 3 | 4 | export class SLLQueueExtended extends SLLQueue { 5 | /** 6 | * Dequeues the first `itemCount` items into an array. 7 | * 8 | * @param itemCount - The number of items to dequeue. 9 | * @returns The array of dequeued items. 10 | * @throws {@link IllegalArgumentError} Thrown if `itemCount` is invalid. 11 | */ 12 | public dequeueFirst(itemCount: number): E[] { 13 | if (!Number.isInteger(itemCount) || itemCount < 0 || itemCount > this.count) { 14 | throw new IllegalArgumentError( 15 | 'The `itemCount` argument must be a non-negative integer less than or equal to the size of the queue.' 16 | ); 17 | } 18 | 19 | const items = new Array(itemCount); 20 | 21 | for (let i = 0; i < itemCount; i++) { 22 | items[i] = this.dequeue(); 23 | } 24 | 25 | return items; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/internal/ds/__tests__/PromiseQueue.test.ts: -------------------------------------------------------------------------------- 1 | import { PromiseQueue } from '..'; 2 | import { sleep } from '../../utils'; 3 | 4 | describe('PromiseQueue', () => { 5 | const runTask = (): Promise => sleep(0); 6 | 7 | test('correctly processes all items', async () => { 8 | const q = new PromiseQueue(); 9 | 10 | expect(q.size).toBe(0); 11 | 12 | q.enqueue(runTask()); 13 | q.enqueue(runTask()); 14 | q.enqueue(runTask()); 15 | 16 | expect(q.size).toBe(3); 17 | 18 | await q.processAll(); 19 | 20 | expect(q.size).toBe(0); 21 | }); 22 | 23 | test('correctly processes the first `n` items', async () => { 24 | const q = new PromiseQueue(); 25 | 26 | q.enqueue(runTask()); 27 | q.enqueue(runTask()); 28 | q.enqueue(runTask()); 29 | 30 | await q.processFirst(0); 31 | 32 | expect(q.size).toBe(3); 33 | 34 | await q.processFirst(1); 35 | 36 | expect(q.size).toBe(2); 37 | 38 | await q.processFirst(2); 39 | 40 | expect(q.size).toBe(0); 41 | }); 42 | }); 43 | -------------------------------------------------------------------------------- /src/internal/ds/__tests__/SLLQueueExtended.test.ts: -------------------------------------------------------------------------------- 1 | import { SLLQueueExtended } from '..'; 2 | 3 | describe('SLLQueueExtended', () => { 4 | describe('dequeueFirst(n)', () => { 5 | test('returns empty array if empty and n=0', () => { 6 | const q = new SLLQueueExtended(); 7 | expect(q.dequeueFirst(0)).toEqual([]); 8 | }); 9 | 10 | test('returns empty array if n=0', () => { 11 | const q = new SLLQueueExtended(); 12 | q.enqueue(1); 13 | expect(q.dequeueFirst(0)).toEqual([]); 14 | }); 15 | 16 | describe('throws if', () => { 17 | test('n is negative', () => { 18 | const q = new SLLQueueExtended(); 19 | expect(() => q.dequeueFirst(-1)).toThrow(); 20 | }); 21 | 22 | test('n is not an integer', () => { 23 | const q = new SLLQueueExtended(); 24 | q.enqueue(1); 25 | q.enqueue(2); 26 | expect(() => q.dequeueFirst(1.5)).toThrow(); 27 | }); 28 | 29 | test('n is greater than queue size', () => { 30 | const q = new SLLQueueExtended(); 31 | q.enqueue(1); 32 | q.enqueue(2); 33 | expect(() => q.dequeueFirst(3)).toThrow(); 34 | }); 35 | }); 36 | 37 | test('returns correct number of items in FIFO order', () => { 38 | const q = new SLLQueueExtended(); 39 | q.enqueue(1); 40 | q.enqueue(2); 41 | q.enqueue(3); 42 | q.enqueue(4); 43 | expect(q.dequeueFirst(3)).toEqual([1, 2, 3]); 44 | }); 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /src/internal/ds/index.ts: -------------------------------------------------------------------------------- 1 | export { PromiseQueue } from './PromiseQueue'; 2 | export { SLLQueueExtended } from './SLLQueueExtended'; 3 | -------------------------------------------------------------------------------- /src/internal/errors/IllegalArgumentError.ts: -------------------------------------------------------------------------------- 1 | export class IllegalArgumentError extends Error { 2 | public constructor(message: string) { 3 | super(`Encountered an illegal argument: ${message}`); 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/internal/errors/index.ts: -------------------------------------------------------------------------------- 1 | export { IllegalArgumentError } from './IllegalArgumentError'; 2 | -------------------------------------------------------------------------------- /src/internal/implementations/BasicBatchMigratorImpl.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { 3 | BatchMigrator, 4 | MigrationPredicate, 5 | MigrationResult, 6 | SetDataGetter, 7 | SetOptions, 8 | SetPartialDataGetter, 9 | Traverser, 10 | UpdateDataGetter, 11 | UpdateFieldValueGetter, 12 | } from '../../api'; 13 | import { InvalidConfigError } from '../../errors'; 14 | import { isPositiveInteger } from '../utils'; 15 | import { AbstractMigrator, RegisteredCallbacks } from './abstract'; 16 | import { IllegalArgumentError } from '../errors'; 17 | 18 | export class BasicBatchMigratorImpl< 19 | AppModelType = firestore.DocumentData, 20 | DbModelType extends firestore.DocumentData = firestore.DocumentData 21 | > 22 | extends AbstractMigrator 23 | implements BatchMigrator 24 | { 25 | static readonly #MAX_BATCH_WRITE_DOC_COUNT = 500; 26 | 27 | public constructor( 28 | public readonly traverser: Traverser, 29 | registeredCallbacks?: RegisteredCallbacks, 30 | migrationPredicates?: MigrationPredicate[] 31 | ) { 32 | super(registeredCallbacks, migrationPredicates); 33 | this.#validateTraverserCompatibility(); 34 | } 35 | 36 | #validateTraverserCompatibility(): void { 37 | const { batchSize } = this.traverser.traversalConfig; 38 | const maxBatchWriteDocCount = BasicBatchMigratorImpl.#MAX_BATCH_WRITE_DOC_COUNT; 39 | if ( 40 | typeof batchSize === 'number' && 41 | (!isPositiveInteger(batchSize) || batchSize > maxBatchWriteDocCount) 42 | ) { 43 | throw new InvalidConfigError( 44 | `The 'batchSize' field in the traversal config of a BatchMigrator's traverser must be a positive integer less than or equal to ${maxBatchWriteDocCount}. In Firestore, each write batch can write to a maximum of ${maxBatchWriteDocCount} documents.` 45 | ); 46 | } 47 | } 48 | 49 | public withPredicate( 50 | predicate: MigrationPredicate 51 | ): BatchMigrator { 52 | return new BasicBatchMigratorImpl(this.traverser, this.registeredCallbacks, [ 53 | ...this.migrationPredicates, 54 | predicate, 55 | ]); 56 | } 57 | 58 | public withTraverser( 59 | traverser: Traverser 60 | ): BatchMigrator { 61 | return new BasicBatchMigratorImpl( 62 | traverser, 63 | this.registeredCallbacks, 64 | this.migrationPredicates 65 | ); 66 | } 67 | 68 | public set( 69 | data: firestore.PartialWithFieldValue, 70 | options: SetOptions 71 | ): Promise; 72 | 73 | public set(data: firestore.WithFieldValue): Promise; 74 | 75 | public async set( 76 | data: firestore.PartialWithFieldValue | firestore.WithFieldValue, 77 | options?: SetOptions 78 | ): Promise { 79 | return this.#migrate((writeBatch, doc) => { 80 | if (options === undefined) { 81 | // Signature 2 82 | writeBatch.set(doc.ref, data as firestore.WithFieldValue); 83 | } else { 84 | // Signature 1 85 | writeBatch.set(doc.ref, data as firestore.PartialWithFieldValue, options); 86 | } 87 | }); 88 | } 89 | 90 | public setWithDerivedData( 91 | getData: SetPartialDataGetter, 92 | options: SetOptions 93 | ): Promise; 94 | 95 | public setWithDerivedData( 96 | getData: SetDataGetter 97 | ): Promise; 98 | 99 | public async setWithDerivedData( 100 | getData: 101 | | SetPartialDataGetter 102 | | SetDataGetter, 103 | options?: SetOptions 104 | ): Promise { 105 | return this.#migrate((writeBatch, doc) => { 106 | if (options === undefined) { 107 | // Signature 2 108 | const data = (getData as SetDataGetter)(doc); 109 | writeBatch.set(doc.ref, data); 110 | } else { 111 | // Signature 1 112 | const data = (getData as SetPartialDataGetter)(doc); 113 | writeBatch.set(doc.ref, data, options); 114 | } 115 | }); 116 | } 117 | 118 | public update( 119 | data: firestore.UpdateData, 120 | precondition?: firestore.Precondition 121 | ): Promise; 122 | 123 | public update( 124 | field: string | firestore.FieldPath, 125 | value: any, 126 | ...moreFieldsOrPrecondition: any[] 127 | ): Promise; 128 | 129 | public update( 130 | dataOrField: firestore.UpdateData | string | firestore.FieldPath, 131 | preconditionOrValue?: any, 132 | ...moreFieldsOrPrecondition: any[] 133 | ): Promise { 134 | return this.#migrate((writeBatch, doc) => { 135 | if ( 136 | typeof dataOrField === 'string' || 137 | dataOrField instanceof this.firestoreConstructor.FieldPath 138 | ) { 139 | // Signature 2 140 | const field = dataOrField; 141 | const value = preconditionOrValue; 142 | writeBatch.update(doc.ref, field, value, ...moreFieldsOrPrecondition); 143 | } else if (dataOrField !== undefined) { 144 | // Signature 1 145 | const data = dataOrField; 146 | const precondition = preconditionOrValue as firestore.Precondition | undefined; 147 | if (precondition === undefined) { 148 | writeBatch.update(doc.ref, data); 149 | } else { 150 | writeBatch.update(doc.ref, data, precondition); 151 | } 152 | } else { 153 | throw new IllegalArgumentError( 154 | `Unsupported signature detected. The 'dataOrField' argument cannot be undefined. The 'dataOrField' argument must be a string, a FieldPath, or an object.` 155 | ); 156 | } 157 | }); 158 | } 159 | 160 | public updateWithDerivedData( 161 | getData: UpdateDataGetter, 162 | precondition?: firestore.Precondition 163 | ): Promise; 164 | 165 | public updateWithDerivedData( 166 | getData: UpdateFieldValueGetter 167 | ): Promise; 168 | 169 | public updateWithDerivedData( 170 | getData: ( 171 | doc: firestore.QueryDocumentSnapshot 172 | ) => 173 | | ReturnType> 174 | | ReturnType>, 175 | precondition?: firestore.Precondition 176 | ): Promise { 177 | return this.#migrate((writeBatch, doc) => { 178 | const data = getData(doc); 179 | if (Array.isArray(data)) { 180 | // Signature 2 181 | writeBatch.update(doc.ref, ...(data as [string | firestore.FieldPath, any, ...any[]])); 182 | } else if (data !== undefined) { 183 | // Signature 1 184 | if (precondition === undefined) { 185 | writeBatch.update(doc.ref, data); 186 | } else { 187 | writeBatch.update(doc.ref, data, precondition); 188 | } 189 | } else { 190 | throw new IllegalArgumentError( 191 | `Unsupported signature detected. The 'data' argument cannot be undefined. The 'data' argument must be an array, an object, or a valid Firestore update signature.` 192 | ); 193 | } 194 | }); 195 | } 196 | 197 | async #migrate( 198 | migrateDoc: ( 199 | writeBatch: firestore.WriteBatch, 200 | doc: firestore.QueryDocumentSnapshot 201 | ) => void 202 | ): Promise { 203 | return this.migrateWithTraverser(async (batchDocs) => { 204 | let migratedDocCount = 0; 205 | const writeBatch = this.firestoreInstance.batch(); 206 | batchDocs.forEach((doc) => { 207 | if (this.shouldMigrateDoc(doc)) { 208 | migrateDoc(writeBatch, doc); 209 | migratedDocCount++; 210 | } 211 | }); 212 | await writeBatch.commit(); 213 | return migratedDocCount; 214 | }); 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /src/internal/implementations/BasicDefaultMigratorImpl.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { 3 | DefaultMigrator, 4 | MigrationPredicate, 5 | MigrationResult, 6 | SetDataGetter, 7 | SetOptions, 8 | SetPartialDataGetter, 9 | Traverser, 10 | UpdateDataGetter, 11 | UpdateFieldValueGetter, 12 | } from '../../api'; 13 | import { AbstractMigrator, RegisteredCallbacks } from './abstract'; 14 | import { IllegalArgumentError } from '../errors'; 15 | 16 | export class BasicDefaultMigratorImpl< 17 | AppModelType = firestore.DocumentData, 18 | DbModelType extends firestore.DocumentData = firestore.DocumentData 19 | > 20 | extends AbstractMigrator 21 | implements DefaultMigrator 22 | { 23 | public constructor( 24 | public readonly traverser: Traverser, 25 | registeredCallbacks?: RegisteredCallbacks, 26 | migrationPredicates?: MigrationPredicate[] 27 | ) { 28 | super(registeredCallbacks, migrationPredicates); 29 | } 30 | 31 | public withPredicate( 32 | predicate: MigrationPredicate 33 | ): DefaultMigrator { 34 | return new BasicDefaultMigratorImpl(this.traverser, this.registeredCallbacks, [ 35 | ...this.migrationPredicates, 36 | predicate, 37 | ]); 38 | } 39 | 40 | public withTraverser( 41 | traverser: Traverser 42 | ): DefaultMigrator { 43 | return new BasicDefaultMigratorImpl( 44 | traverser, 45 | this.registeredCallbacks, 46 | this.migrationPredicates 47 | ); 48 | } 49 | 50 | public set( 51 | data: firestore.PartialWithFieldValue, 52 | options: SetOptions 53 | ): Promise; 54 | 55 | public set(data: firestore.WithFieldValue): Promise; 56 | 57 | public async set( 58 | data: firestore.PartialWithFieldValue | firestore.WithFieldValue, 59 | options?: SetOptions 60 | ): Promise { 61 | return this.#migrate(async (doc) => { 62 | if (options === undefined) { 63 | // Signature 2 64 | await doc.ref.set(data as firestore.WithFieldValue); 65 | } else { 66 | // Signature 1 67 | await doc.ref.set(data as firestore.PartialWithFieldValue, options); 68 | } 69 | }); 70 | } 71 | 72 | public setWithDerivedData( 73 | getData: SetPartialDataGetter, 74 | options: SetOptions 75 | ): Promise; 76 | 77 | public setWithDerivedData( 78 | getData: SetDataGetter 79 | ): Promise; 80 | 81 | public async setWithDerivedData( 82 | getData: 83 | | SetPartialDataGetter 84 | | SetDataGetter, 85 | options?: SetOptions 86 | ): Promise { 87 | return this.#migrate(async (doc) => { 88 | if (options === undefined) { 89 | // Signature 2 90 | const data = (getData as SetDataGetter)(doc); 91 | await doc.ref.set(data); 92 | } else { 93 | // Signature 1 94 | const data = (getData as SetPartialDataGetter)(doc); 95 | await doc.ref.set(data, options); 96 | } 97 | }); 98 | } 99 | 100 | public update( 101 | data: firestore.UpdateData, 102 | precondition?: firestore.Precondition 103 | ): Promise; 104 | 105 | public update( 106 | field: string | firestore.FieldPath, 107 | value: any, 108 | ...moreFieldsOrPrecondition: any[] 109 | ): Promise; 110 | 111 | public update( 112 | dataOrField: firestore.UpdateData | string | firestore.FieldPath, 113 | preconditionOrValue?: any, 114 | ...moreFieldsOrPrecondition: any[] 115 | ): Promise { 116 | return this.#migrate(async (doc) => { 117 | if ( 118 | typeof dataOrField === 'string' || 119 | dataOrField instanceof this.firestoreConstructor.FieldPath 120 | ) { 121 | // Signature 2 122 | const field = dataOrField; 123 | const value = preconditionOrValue; 124 | await doc.ref.update(field, value, ...moreFieldsOrPrecondition); 125 | } else if (typeof dataOrField === 'object' && dataOrField !== null) { 126 | // Signature 1 127 | const data = dataOrField; 128 | const precondition = preconditionOrValue as firestore.Precondition | undefined; 129 | if (precondition === undefined) { 130 | await doc.ref.update(data); 131 | } else { 132 | await doc.ref.update(data, precondition); 133 | } 134 | } else { 135 | throw new IllegalArgumentError( 136 | `Unsupported signature detected. The 'dataOrField' argument cannot be undefined. The 'dataOrField' argument must be a string, a FieldPath, or an object.` 137 | ); 138 | } 139 | }); 140 | } 141 | 142 | public updateWithDerivedData( 143 | getData: UpdateDataGetter, 144 | precondition?: firestore.Precondition 145 | ): Promise; 146 | 147 | public updateWithDerivedData( 148 | getData: UpdateFieldValueGetter 149 | ): Promise; 150 | 151 | public updateWithDerivedData( 152 | getData: ( 153 | doc: firestore.QueryDocumentSnapshot 154 | ) => 155 | | ReturnType> 156 | | ReturnType>, 157 | precondition?: firestore.Precondition 158 | ): Promise { 159 | return this.#migrate(async (doc) => { 160 | const data = getData(doc); 161 | 162 | if (Array.isArray(data)) { 163 | // Signature 2 164 | await doc.ref.update(...(data as [string | firestore.FieldPath, any, ...any[]])); 165 | } else if (typeof data === 'object' && data !== null) { 166 | // Signature 1 167 | if (precondition === undefined) { 168 | await doc.ref.update(data); 169 | } else { 170 | await doc.ref.update(data, precondition); 171 | } 172 | } else { 173 | throw new IllegalArgumentError( 174 | `Unsupported signature detected. The 'data' argument must be an array, an object, or a valid Firestore update signature.` 175 | ); 176 | } 177 | }); 178 | } 179 | 180 | async #migrate( 181 | migrateDoc: (doc: firestore.QueryDocumentSnapshot) => Promise 182 | ): Promise { 183 | return this.migrateWithTraverser(async (batchDocs) => { 184 | let migratedDocCount = 0; 185 | const promises = batchDocs.map(async (doc) => { 186 | if (this.shouldMigrateDoc(doc)) { 187 | await migrateDoc(doc); 188 | migratedDocCount++; 189 | } 190 | }); 191 | await Promise.all(promises); 192 | return migratedDocCount; 193 | }); 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /src/internal/implementations/PromiseQueueBasedTraverserImpl.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { 3 | BatchCallback, 4 | ExitEarlyPredicate, 5 | Traversable, 6 | TraversalConfig, 7 | TraversalResult, 8 | Traverser, 9 | } from '../../api'; 10 | import { ImplementationError } from '../../errors'; 11 | import { IllegalArgumentError } from '../errors'; 12 | import { PromiseQueue } from '../ds'; 13 | import { makeRetriable, registerInterval, sleep } from '../utils'; 14 | import { AbstractTraverser } from './abstract'; 15 | 16 | export class PromiseQueueBasedTraverserImpl< 17 | AppModelType = firestore.DocumentData, 18 | DbModelType extends firestore.DocumentData = firestore.DocumentData 19 | > 20 | extends AbstractTraverser 21 | implements Traverser 22 | { 23 | static readonly #defaultConfig: TraversalConfig = { 24 | ...AbstractTraverser.baseConfig, 25 | }; 26 | 27 | public constructor( 28 | public readonly traversable: Traversable, 29 | exitEarlyPredicates: ExitEarlyPredicate[] = [], 30 | config?: Partial 31 | ) { 32 | super({ ...PromiseQueueBasedTraverserImpl.#defaultConfig, ...config }, exitEarlyPredicates); 33 | } 34 | 35 | public withConfig(config: Partial): Traverser { 36 | return new PromiseQueueBasedTraverserImpl(this.traversable, this.exitEarlyPredicates, { 37 | ...this.traversalConfig, 38 | ...config, 39 | }); 40 | } 41 | 42 | public withExitEarlyPredicate( 43 | predicate: ExitEarlyPredicate 44 | ): Traverser { 45 | return new PromiseQueueBasedTraverserImpl( 46 | this.traversable, 47 | [...this.exitEarlyPredicates, predicate], 48 | this.traversalConfig 49 | ); 50 | } 51 | 52 | public async traverse( 53 | callback: BatchCallback 54 | ): Promise { 55 | const { traversalConfig } = this; 56 | const { maxConcurrentBatchCount } = traversalConfig; 57 | callback = this.#makeRetriableAccordingToConfig(callback); 58 | if (maxConcurrentBatchCount === 1) { 59 | return this.runTraversal(async (batchDocs, batchIndex) => { 60 | await callback(batchDocs, batchIndex); 61 | }); 62 | } 63 | const callbackPromiseQueue = new PromiseQueue(); 64 | const unregisterQueueProcessor = registerInterval( 65 | async () => { 66 | if (!callbackPromiseQueue.isProcessing) { 67 | const processableItemCount = this.#getProcessableItemCount(callbackPromiseQueue.size); 68 | try { 69 | await callbackPromiseQueue.processFirst(processableItemCount); 70 | } catch (err) { 71 | throw err instanceof IllegalArgumentError 72 | ? new ImplementationError( 73 | `Encountered an expected error originating from an incorrectly implemented PromiseQueue data structure.` 74 | ) 75 | : err; 76 | } 77 | } 78 | }, 79 | () => this.#getProcessQueueInterval(callbackPromiseQueue.size) 80 | ); 81 | const traversalResult = await this.runTraversal((batchDocs, batchIndex) => { 82 | callbackPromiseQueue.enqueue(callback(batchDocs, batchIndex) ?? Promise.resolve()); 83 | return async () => { 84 | while (callbackPromiseQueue.size >= maxConcurrentBatchCount) { 85 | // TODO: There probably is a better way to compute sleep duration 86 | const processQueueInterval = this.#getProcessQueueInterval(callbackPromiseQueue.size); 87 | await sleep(processQueueInterval); 88 | } 89 | }; 90 | }); 91 | await unregisterQueueProcessor(); 92 | // There may still be some Promises left in the queue but there won't be any new ones coming in. 93 | // Wait for the existing ones to resolve and exit. 94 | await callbackPromiseQueue.processAll(); 95 | return traversalResult; 96 | } 97 | 98 | #makeRetriableAccordingToConfig( 99 | callback: BatchCallback 100 | ): BatchCallback { 101 | const { maxBatchRetryCount, sleepTimeBetweenTrials } = this.traversalConfig; 102 | let cb = callback; 103 | if (maxBatchRetryCount > 0) { 104 | const retriableCallback = makeRetriable(callback, { 105 | maxTrialCount: 1 + maxBatchRetryCount, 106 | sleepTimeBetweenTrials, 107 | returnErrors: true, 108 | }); 109 | cb = async (...args) => { 110 | const result = await retriableCallback(...args); 111 | if (!result.hasSucceeded) { 112 | const { errors } = result; 113 | const lastError = errors[errors.length - 1]; 114 | throw lastError; 115 | } 116 | }; 117 | } 118 | return cb; 119 | } 120 | 121 | /** 122 | * Computes the number of queue items to process based on the traversal configuration and queue size. 123 | * 124 | * @param queueSize - The current size of the queue. 125 | * @returns An integer within the range [0, `queueSize`]. 126 | */ 127 | #getProcessableItemCount(queueSize: number): number { 128 | // TODO: Implement using traversal config and queue size 129 | return queueSize; 130 | } 131 | 132 | /** 133 | * Computes the duration (in ms) for which to sleep before re-running the queue processing logic. 134 | * 135 | * @param queueSize - The current size of the queue. 136 | * @returns A non-negative integer. 137 | */ 138 | #getProcessQueueInterval(queueSize: number): number { 139 | // TODO: Implement using traversal config and queue size 140 | return 250; 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/deleteField.test.ts: -------------------------------------------------------------------------------- 1 | import { testDeleteField } from '../shared/deleteField'; 2 | import { describeBasicBatchMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicBatchMigratorMethodTest('deleteField', testDeleteField); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/deleteFields.test.ts: -------------------------------------------------------------------------------- 1 | import { testDeleteFields } from '../shared/deleteFields'; 2 | import { describeBasicBatchMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicBatchMigratorMethodTest('deleteFields', testDeleteFields); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/helpers.ts: -------------------------------------------------------------------------------- 1 | import { BasicBatchMigratorImpl } from '../../../BasicBatchMigratorImpl'; 2 | import { describeMigratorMethodTest } from '../helpers'; 3 | import type { MigratorMethodTester } from '../config'; 4 | 5 | export function describeBasicBatchMigratorMethodTest( 6 | methodName: string, 7 | methodTester: MigratorMethodTester 8 | ): void { 9 | describeMigratorMethodTest(BasicBatchMigratorImpl, methodName, methodTester); 10 | } 11 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/renameField.test.ts: -------------------------------------------------------------------------------- 1 | import { testRenameField } from '../shared/renameField'; 2 | import { describeBasicBatchMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicBatchMigratorMethodTest('renameField', testRenameField); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/renameFields.test.ts: -------------------------------------------------------------------------------- 1 | import { testRenameFields } from '../shared/renameFields'; 2 | import { describeBasicBatchMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicBatchMigratorMethodTest('renameFields', testRenameFields); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/update.test.ts: -------------------------------------------------------------------------------- 1 | import { testUpdate } from '../shared/update'; 2 | import { describeBasicBatchMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicBatchMigratorMethodTest('update', testUpdate); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/updateWithDerivedData.test.ts: -------------------------------------------------------------------------------- 1 | import { testUpdateWithDerivedData } from '../shared/updateWithDerivedData'; 2 | import { describeBasicBatchMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicBatchMigratorMethodTest('updateWithDerivedData', testUpdateWithDerivedData); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/deleteField.test.ts: -------------------------------------------------------------------------------- 1 | import { testDeleteField } from '../shared/deleteField'; 2 | import { describeBasicDefaultMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicDefaultMigratorMethodTest('deleteField', testDeleteField); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/deleteFields.test.ts: -------------------------------------------------------------------------------- 1 | import { testDeleteFields } from '../shared/deleteFields'; 2 | import { describeBasicDefaultMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicDefaultMigratorMethodTest('deleteFields', testDeleteFields); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/helpers.ts: -------------------------------------------------------------------------------- 1 | import { BasicDefaultMigratorImpl } from '../../../BasicDefaultMigratorImpl'; 2 | import { describeMigratorMethodTest } from '../helpers'; 3 | import type { MigratorMethodTester } from '../config'; 4 | 5 | export function describeBasicDefaultMigratorMethodTest( 6 | methodName: string, 7 | methodTester: MigratorMethodTester 8 | ): void { 9 | describeMigratorMethodTest(BasicDefaultMigratorImpl, methodName, methodTester); 10 | } 11 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/renameField.test.ts: -------------------------------------------------------------------------------- 1 | import { testRenameField } from '../shared/renameField'; 2 | import { describeBasicDefaultMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicDefaultMigratorMethodTest('renameField', testRenameField); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/renameFields.test.ts: -------------------------------------------------------------------------------- 1 | import { testRenameFields } from '../shared/renameFields'; 2 | import { describeBasicDefaultMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicDefaultMigratorMethodTest('renameFields', testRenameFields); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/update.test.ts: -------------------------------------------------------------------------------- 1 | import { testUpdate } from '../shared/update'; 2 | import { describeBasicDefaultMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicDefaultMigratorMethodTest('update', testUpdate); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/updateWithDerivedData.test.ts: -------------------------------------------------------------------------------- 1 | import { testUpdateWithDerivedData } from '../shared/updateWithDerivedData'; 2 | import { describeBasicDefaultMigratorMethodTest } from './helpers'; 3 | 4 | describeBasicDefaultMigratorMethodTest('updateWithDerivedData', testUpdateWithDerivedData); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/config.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import type { Migrator, TraversalConfig } from '../../../../../src'; 3 | 4 | export type MigratorMethodTester = ( 5 | migrator: Migrator, 6 | colRef: firestore.CollectionReference 7 | ) => void; 8 | 9 | export interface TestItemDoc { 10 | docId?: string; 11 | map1: { 12 | num1?: number; 13 | num2?: number; 14 | string1?: string; 15 | }; 16 | num2: number; 17 | string2: string; 18 | string3: string; 19 | timestamp1?: firestore.Timestamp; 20 | timestamp2?: firestore.Timestamp; 21 | } 22 | 23 | export const DEFAULT_TIMEOUT = 60_000; 24 | 25 | export const DEFAULT_TRAVERSABLE_SIZE = 40; 26 | 27 | export const TRAVERSAL_CONFIG: Partial = { 28 | batchSize: 10, 29 | }; 30 | 31 | export const INITIAL_DATA = Object.freeze({ 32 | map1: { 33 | num1: 1, 34 | string1: 'abc', 35 | }, 36 | num2: 2, 37 | string2: 'abc', 38 | string3: 'abc', 39 | timestamp1: firestore.Timestamp.fromDate(new Date()), 40 | }); 41 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/helpers.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import { app } from '../../../../../__tests__/app'; 3 | import { collectionPopulator } from '../../../../../__tests__/utils'; 4 | import { createTraverser, Migrator, Traverser } from '../../../../api'; 5 | import { 6 | TestItemDoc, 7 | MigratorMethodTester, 8 | TRAVERSAL_CONFIG, 9 | DEFAULT_TIMEOUT, 10 | INITIAL_DATA, 11 | DEFAULT_TRAVERSABLE_SIZE, 12 | } from './config'; 13 | 14 | export type MigratorImplClass = { 15 | new < 16 | AppModelType = firestore.DocumentData, 17 | DbModelType extends firestore.DocumentData = firestore.DocumentData 18 | >( 19 | traverser: Traverser 20 | ): Migrator; 21 | }; 22 | 23 | export function describeMigratorMethodTest( 24 | migratorImplClass: MigratorImplClass, 25 | methodName: string, 26 | methodTester: MigratorMethodTester 27 | ): void { 28 | const description = `${migratorImplClass.name}.${methodName}`; 29 | const colRef = app().firestore.collection( 30 | description 31 | ) as firestore.CollectionReference; 32 | const traverser = createTraverser(colRef, TRAVERSAL_CONFIG); 33 | const migrator = new migratorImplClass(traverser); 34 | describe(description, () => { 35 | methodTester(migrator, colRef); 36 | }); 37 | } 38 | 39 | interface MigrationTester { 40 | test( 41 | name: string, 42 | testFn: ( 43 | initialData: TestItemDoc, 44 | docRefs: firestore.DocumentReference[] 45 | ) => Promise, 46 | timeout?: number 47 | ): void; 48 | } 49 | 50 | export function migrationTester( 51 | colRef: firestore.CollectionReference 52 | ): MigrationTester { 53 | return { 54 | test: (name, testFn, timeout = DEFAULT_TIMEOUT) => { 55 | test( 56 | name, 57 | async () => { 58 | const docRefs = await collectionPopulator(colRef) 59 | .withData(INITIAL_DATA) 60 | .populate({ count: DEFAULT_TRAVERSABLE_SIZE }); 61 | await testFn(INITIAL_DATA, docRefs); 62 | await Promise.all(docRefs.map((docRef) => docRef.delete())); 63 | }, 64 | timeout 65 | ); 66 | }, 67 | }; 68 | } 69 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/shared/deleteField.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { cloneDeep } from 'lodash'; 3 | import type { Migrator } from '../../../../../api'; 4 | import type { TestItemDoc } from '../config'; 5 | import { migrationTester } from '../helpers'; 6 | 7 | /** 8 | * Assumes that the collection is initially empty. 9 | */ 10 | export function testDeleteField( 11 | migrator: Migrator, 12 | colRef: firestore.CollectionReference 13 | ): void { 14 | migrationTester(colRef).test( 15 | 'correctly deletes a single field from each doc', 16 | async (initialData) => { 17 | await migrator.deleteField(new firestore.FieldPath('map1', 'string1')); 18 | const { docs } = await colRef.get(); 19 | docs.forEach((snap) => { 20 | const data = snap.data(); 21 | const expected = (() => { 22 | const copy = { ...cloneDeep(initialData) }; 23 | delete copy.map1.string1; 24 | return copy; 25 | })(); 26 | expect(data).toEqual(expected); 27 | }); 28 | } 29 | ); 30 | } 31 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/shared/deleteFields.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { cloneDeep } from 'lodash'; 3 | import type { Migrator } from '../../../../../api'; 4 | import type { TestItemDoc } from '../config'; 5 | import { migrationTester } from '../helpers'; 6 | 7 | /** 8 | * Assumes that the collection is initially empty. 9 | */ 10 | export function testDeleteFields( 11 | migrator: Migrator, 12 | colRef: firestore.CollectionReference 13 | ): void { 14 | migrationTester(colRef).test( 15 | 'correctly deletes a single field from each doc', 16 | async (initialData) => { 17 | await migrator.deleteFields( 18 | new firestore.FieldPath('map1', 'num1'), 19 | new firestore.FieldPath('map1', 'string1'), 20 | 'timestamp1' 21 | ); 22 | const { docs } = await colRef.get(); 23 | docs.forEach((snap) => { 24 | const data = snap.data(); 25 | const expected = (() => { 26 | const copy = { ...cloneDeep(initialData) }; 27 | delete copy.map1.num1; 28 | delete copy.map1.string1; 29 | delete copy.timestamp1; 30 | return copy; 31 | })(); 32 | expect(data).toEqual(expected); 33 | }); 34 | } 35 | ); 36 | } 37 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/shared/renameField.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { cloneDeep } from 'lodash'; 3 | import type { Migrator } from '../../../../../api'; 4 | import type { TestItemDoc } from '../config'; 5 | import { migrationTester } from '../helpers'; 6 | 7 | /** 8 | * Assumes that the collection is initially empty. 9 | */ 10 | export function testRenameField( 11 | migrator: Migrator, 12 | colRef: firestore.CollectionReference 13 | ): void { 14 | migrationTester(colRef).test( 15 | 'correctly renames a single field in each doc', 16 | async (initialData) => { 17 | await migrator.renameField('timestamp1', 'timestamp2'); 18 | const { docs } = await colRef.get(); 19 | docs.forEach((snap) => { 20 | const data = snap.data(); 21 | const expected = (() => { 22 | const copy = { ...cloneDeep(initialData) }; 23 | delete copy.timestamp1; 24 | copy['timestamp2'] = initialData.timestamp1; 25 | return copy; 26 | })(); 27 | expect(data).toEqual(expected); 28 | }); 29 | } 30 | ); 31 | } 32 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/shared/renameFields.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { cloneDeep } from 'lodash'; 3 | import type { Migrator } from '../../../../../api'; 4 | import type { TestItemDoc } from '../config'; 5 | import { migrationTester } from '../helpers'; 6 | 7 | /** 8 | * Assumes that the collection is initially empty. 9 | */ 10 | export function testRenameFields( 11 | migrator: Migrator, 12 | colRef: firestore.CollectionReference 13 | ): void { 14 | migrationTester(colRef).test( 15 | 'correctly renames multiple fields in each doc', 16 | async (initialData) => { 17 | await migrator.renameFields( 18 | [new firestore.FieldPath('map1', 'num1'), new firestore.FieldPath('map1', 'num2')], 19 | ['timestamp1', 'timestamp2'] 20 | ); 21 | const { docs } = await migrator.traverser.traversable.get(); 22 | docs.forEach((snap) => { 23 | const data = snap.data(); 24 | const expected = (() => { 25 | const copy = { ...cloneDeep(initialData) }; 26 | delete copy.map1.num1; 27 | delete copy.timestamp1; 28 | copy['map1']['num2'] = initialData.map1.num1; 29 | copy['timestamp2'] = initialData.timestamp1; 30 | return copy; 31 | })(); 32 | expect(data).toEqual(expected); 33 | }); 34 | } 35 | ); 36 | } 37 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/shared/update.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { cloneDeep } from 'lodash'; 3 | import type { Migrator } from '../../../../../api'; 4 | import type { TestItemDoc } from '../config'; 5 | import { migrationTester } from '../helpers'; 6 | 7 | /** 8 | * Assumes that the collection is initially empty. 9 | */ 10 | export function testUpdate( 11 | migrator: Migrator, 12 | colRef: firestore.CollectionReference 13 | ): void { 14 | migrationTester(colRef).test( 15 | 'correctly updates each doc with the provided data', 16 | async (initialData) => { 17 | const updateData = { string2: 'a' }; 18 | await migrator.update(updateData); 19 | const { docs } = await migrator.traverser.traversable.get(); 20 | docs.forEach((snap) => { 21 | const data = snap.data(); 22 | expect(data).toEqual({ 23 | ...cloneDeep(initialData), 24 | ...updateData, 25 | }); 26 | }); 27 | } 28 | ); 29 | } 30 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Migrator/shared/updateWithDerivedData.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import { cloneDeep } from 'lodash'; 3 | import type { Migrator } from '../../../../../api'; 4 | import type { TestItemDoc } from '../config'; 5 | import { migrationTester } from '../helpers'; 6 | 7 | /** 8 | * Assumes that the collection is initially empty. 9 | */ 10 | export function testUpdateWithDerivedData( 11 | migrator: Migrator, 12 | colRef: firestore.CollectionReference 13 | ): void { 14 | migrationTester(colRef).test( 15 | 'correctly updates each doc with the provided data getter', 16 | async (initialData) => { 17 | await migrator.updateWithDerivedData((snap) => ({ docId: snap.id })); 18 | const { docs } = await migrator.traverser.traversable.get(); 19 | docs.forEach((snap) => { 20 | const data = snap.data(); 21 | expect(data).toEqual({ 22 | ...cloneDeep(initialData), 23 | docId: snap.id, 24 | }); 25 | }); 26 | } 27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Traverser/PromiseQueueBasedTraverserImpl/helpers.ts: -------------------------------------------------------------------------------- 1 | import { PromiseQueueBasedTraverserImpl } from '../../../PromiseQueueBasedTraverserImpl'; 2 | import { describeTraverserMethodTest } from '../helpers'; 3 | import type { TraverserMethodTester } from '../config'; 4 | 5 | export function describePromiseQueueBasedTraverserMethodTest( 6 | methodName: string, 7 | methodTester: TraverserMethodTester 8 | ): void { 9 | describeTraverserMethodTest(PromiseQueueBasedTraverserImpl, methodName, methodTester); 10 | } 11 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Traverser/PromiseQueueBasedTraverserImpl/traverse.test.ts: -------------------------------------------------------------------------------- 1 | import { testTraverse } from '../shared/traverse'; 2 | import { describePromiseQueueBasedTraverserMethodTest } from './helpers'; 3 | 4 | describePromiseQueueBasedTraverserMethodTest('traverse', testTraverse); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Traverser/PromiseQueueBasedTraverserImpl/withExitEarlyPredicate.test.ts: -------------------------------------------------------------------------------- 1 | import { testWithExitEarlyPredicate } from '../shared/withExitEarlyPredicate'; 2 | import { describePromiseQueueBasedTraverserMethodTest } from './helpers'; 3 | 4 | describePromiseQueueBasedTraverserMethodTest('withExitEarlyPredicate', testWithExitEarlyPredicate); 5 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Traverser/config.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { TraversalConfig, Traverser } from '../../../../../src'; 3 | 4 | export type TraverserMethodTester = ( 5 | traverser: Traverser, 6 | colRef: firestore.CollectionReference 7 | ) => void; 8 | 9 | export interface TestItemDoc { 10 | number: number; 11 | } 12 | 13 | export const DEFAULT_TIMEOUT = 60_000; 14 | 15 | export const DEFAULT_TRAVERSABLE_SIZE = 100; 16 | 17 | export const TRAVERSAL_CONFIG: Partial = { 18 | batchSize: 10, 19 | }; 20 | 21 | export const INITIAL_DATA = Object.freeze({ 22 | number: 1, 23 | }); 24 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Traverser/helpers.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import { app } from '../../../../../__tests__/app'; 3 | import { collectionPopulator } from '../../../../../__tests__/utils'; 4 | import type { Traversable, Traverser } from '../../../../api'; 5 | import { 6 | TestItemDoc, 7 | TraverserMethodTester, 8 | DEFAULT_TIMEOUT, 9 | INITIAL_DATA, 10 | DEFAULT_TRAVERSABLE_SIZE, 11 | } from './config'; 12 | 13 | export type TraverserImplClass = { 14 | new < 15 | AppModelType = firestore.DocumentData, 16 | DbModelType extends firestore.DocumentData = firestore.DocumentData 17 | >( 18 | traversable: Traversable 19 | ): Traverser; 20 | }; 21 | 22 | export function describeTraverserMethodTest( 23 | traverserImplClass: TraverserImplClass, 24 | methodName: string, 25 | methodTester: TraverserMethodTester 26 | ): void { 27 | const description = `${traverserImplClass.name}.${methodName}`; 28 | const colRef = app().firestore.collection( 29 | description 30 | ) as firestore.CollectionReference; 31 | const traverser = new traverserImplClass(colRef); 32 | describe(description, () => { 33 | methodTester(traverser, colRef); 34 | }); 35 | } 36 | 37 | interface TraversalTester { 38 | test( 39 | name: string, 40 | testFn: ( 41 | initialData: TestItemDoc, 42 | docRefs: firestore.DocumentReference[] 43 | ) => Promise, 44 | timeout?: number 45 | ): void; 46 | } 47 | 48 | export function traversalTester( 49 | colRef: firestore.CollectionReference 50 | ): TraversalTester { 51 | return { 52 | test: (name, testFn, timeout = DEFAULT_TIMEOUT) => { 53 | test( 54 | name, 55 | async () => { 56 | const docRefs = await collectionPopulator(colRef) 57 | .withData(INITIAL_DATA) 58 | .populate({ count: DEFAULT_TRAVERSABLE_SIZE }); 59 | await testFn(INITIAL_DATA, docRefs); 60 | await Promise.all(docRefs.map((docRef) => docRef.delete())); 61 | }, 62 | timeout 63 | ); 64 | }, 65 | }; 66 | } 67 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Traverser/shared/traverse.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import type { Traverser } from '../../../../../api'; 3 | import type { TestItemDoc } from '../config'; 4 | import { traversalTester } from '../helpers'; 5 | 6 | export function testTraverse( 7 | traverser: Traverser, 8 | colRef: firestore.CollectionReference 9 | ): void { 10 | traversalTester(colRef).test( 11 | 'processes each document exactly once w/o external interference', 12 | async (_, docRefs) => { 13 | const collectionDocIds = docRefs.map((doc) => doc.id); 14 | const expectedProcessCountMap = new Map(collectionDocIds.map((id) => [id, 1])); 15 | const processCountMap = new Map(); 16 | 17 | await traverser.traverse(async (batchDocs) => { 18 | batchDocs.forEach((doc) => { 19 | const id = doc.id; 20 | processCountMap.set(id, (processCountMap.get(id) ?? 0) + 1); 21 | }); 22 | }); 23 | 24 | expect(processCountMap).toEqual(expectedProcessCountMap); 25 | } 26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /src/internal/implementations/__tests__/Traverser/shared/withExitEarlyPredicate.ts: -------------------------------------------------------------------------------- 1 | import { firestore } from 'firebase-admin'; 2 | import type { TraversalConfig, Traverser } from '../../../../../api'; 3 | import type { TestItemDoc } from '../config'; 4 | import { traversalTester } from '../helpers'; 5 | 6 | export function testWithExitEarlyPredicate( 7 | traverser: Traverser, 8 | colRef: firestore.CollectionReference 9 | ): void { 10 | traversalTester(colRef).test('exits early when instructed as such', async () => { 11 | const t = traverser 12 | .withConfig({ batchSize: 10 } as Partial) 13 | .withExitEarlyPredicate((_, batchIndex) => batchIndex === 5); 14 | 15 | let processedBatchIndices: number[] = []; 16 | 17 | await t.traverse(async (_, batchIndex) => { 18 | processedBatchIndices.push(batchIndex); 19 | }); 20 | 21 | expect(processedBatchIndices).toEqual([0, 1, 2, 3, 4, 5]); 22 | 23 | processedBatchIndices = []; 24 | 25 | await t 26 | .withExitEarlyPredicate((_, batchIndex) => batchIndex === 3) 27 | .traverse(async (_, batchIndex) => { 28 | processedBatchIndices.push(batchIndex); 29 | }); 30 | 31 | expect(processedBatchIndices).toEqual([0, 1, 2, 3]); 32 | }); 33 | } 34 | -------------------------------------------------------------------------------- /src/internal/implementations/abstract/AbstractMigrator.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { 3 | BatchCallback, 4 | MigrationPredicate, 5 | MigrationResult, 6 | Migrator, 7 | SetDataGetter, 8 | SetOptions, 9 | SetPartialDataGetter, 10 | Traverser, 11 | UpdateDataGetter, 12 | UpdateFieldValueGetter, 13 | } from '../../../api'; 14 | 15 | export type RegisteredCallbacks< 16 | AppModelType = firestore.DocumentData, 17 | DbModelType extends firestore.DocumentData = firestore.DocumentData 18 | > = { 19 | onBeforeBatchStart?: BatchCallback; 20 | onAfterBatchComplete?: BatchCallback; 21 | }; 22 | 23 | type UpdateFieldValueArgs = [ 24 | field: string | firestore.FieldPath, 25 | value: any, 26 | ...moreFieldsOrPrecondition: any[] 27 | ]; 28 | 29 | export abstract class AbstractMigrator< 30 | AppModelType = firestore.DocumentData, 31 | DbModelType extends firestore.DocumentData = firestore.DocumentData 32 | > implements Migrator 33 | { 34 | protected constructor( 35 | protected readonly registeredCallbacks: RegisteredCallbacks = {}, 36 | protected readonly migrationPredicates: MigrationPredicate[] = [] 37 | ) {} 38 | 39 | protected get firestoreInstance(): firestore.Firestore { 40 | return this.traverser.traversable.firestore; 41 | } 42 | 43 | protected get firestoreConstructor(): typeof firestore { 44 | return this.firestoreInstance.constructor as typeof firestore; 45 | } 46 | 47 | public onBeforeBatchStart(callback: BatchCallback): void { 48 | this.registeredCallbacks.onBeforeBatchStart = callback; 49 | } 50 | 51 | public onAfterBatchComplete(callback: BatchCallback): void { 52 | this.registeredCallbacks.onAfterBatchComplete = callback; 53 | } 54 | 55 | public deleteField(field: string | firestore.FieldPath): Promise { 56 | return this.deleteFields(field); 57 | } 58 | 59 | public deleteFields(...fields: (string | firestore.FieldPath)[]): Promise { 60 | const updateFieldValuePairs = fields.reduce((acc, field) => { 61 | acc.push(field, this.firestoreConstructor.FieldValue.delete()); 62 | return acc; 63 | }, [] as unknown as UpdateFieldValueArgs); 64 | return this.update(...updateFieldValuePairs); 65 | } 66 | 67 | public renameField( 68 | oldField: string | firestore.FieldPath, 69 | newField: string | firestore.FieldPath 70 | ): Promise { 71 | return this.renameFields([oldField, newField]); 72 | } 73 | 74 | public renameFields( 75 | ...changes: [oldField: string | firestore.FieldPath, newField: string | firestore.FieldPath][] 76 | ): Promise { 77 | return this.withPredicate((snap) => 78 | changes.some(([oldField]) => snap.get(oldField) !== undefined) 79 | ).updateWithDerivedData((snap) => { 80 | const updateFieldValuePairs: unknown[] = []; 81 | changes.forEach((change) => { 82 | const [oldField, newField] = change; 83 | const value = snap.get(oldField); 84 | if (value !== undefined) { 85 | updateFieldValuePairs.push( 86 | oldField, 87 | this.firestoreConstructor.FieldValue.delete(), 88 | newField, 89 | value 90 | ); 91 | } 92 | }); 93 | return updateFieldValuePairs as [string | firestore.FieldPath, any, ...any[]]; 94 | }); 95 | } 96 | 97 | protected async migrateWithTraverser( 98 | migrateBatch: ( 99 | batchDocs: firestore.QueryDocumentSnapshot[] 100 | ) => Promise 101 | ): Promise { 102 | let migratedDocCount = 0; 103 | const traversalResult = await this.traverser.traverse(async (batchDocs, batchIndex) => { 104 | await this.registeredCallbacks.onBeforeBatchStart?.(batchDocs, batchIndex); 105 | const migratedBatchDocCount = await migrateBatch(batchDocs); 106 | migratedDocCount += migratedBatchDocCount; 107 | await this.registeredCallbacks.onAfterBatchComplete?.(batchDocs, batchIndex); 108 | }); 109 | return { traversalResult, migratedDocCount }; 110 | } 111 | 112 | protected shouldMigrateDoc( 113 | doc: firestore.QueryDocumentSnapshot 114 | ): boolean { 115 | return this.migrationPredicates.every((predicate) => predicate(doc)); 116 | } 117 | 118 | public abstract readonly traverser: Traverser; 119 | 120 | public abstract withPredicate( 121 | predicate: MigrationPredicate 122 | ): Migrator; 123 | 124 | public abstract withTraverser( 125 | traverser: Traverser 126 | ): Migrator; 127 | 128 | public abstract set( 129 | data: firestore.PartialWithFieldValue, 130 | options: SetOptions 131 | ): Promise; 132 | 133 | public abstract set(data: firestore.WithFieldValue): Promise; 134 | 135 | public abstract setWithDerivedData( 136 | getData: SetPartialDataGetter, 137 | options: SetOptions 138 | ): Promise; 139 | 140 | public abstract setWithDerivedData( 141 | getData: SetDataGetter 142 | ): Promise; 143 | 144 | public abstract update( 145 | data: firestore.UpdateData, 146 | precondition?: firestore.Precondition 147 | ): Promise; 148 | 149 | public abstract update( 150 | field: string | firestore.FieldPath, 151 | value: any, 152 | ...moreFieldsOrPrecondition: any[] 153 | ): Promise; 154 | 155 | public abstract updateWithDerivedData( 156 | getData: UpdateDataGetter, 157 | precondition?: firestore.Precondition 158 | ): Promise; 159 | 160 | public abstract updateWithDerivedData( 161 | getData: UpdateFieldValueGetter 162 | ): Promise; 163 | } 164 | -------------------------------------------------------------------------------- /src/internal/implementations/abstract/AbstractTraverser.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { 3 | BatchCallback, 4 | ExitEarlyPredicate, 5 | Traversable, 6 | TraversalConfig, 7 | TraversalResult, 8 | TraverseEachCallback, 9 | TraverseEachConfig, 10 | Traverser, 11 | } from '../../../api'; 12 | import { InvalidConfigError } from '../../../errors'; 13 | import { 14 | extractKeys, 15 | isNonNegativeInteger, 16 | isPositiveInteger, 17 | isUnboundedPositiveInteger, 18 | sleep, 19 | } from '../../utils'; 20 | 21 | export type OnAfterBatchProcess = () => void | Promise; 22 | 23 | export type BatchProcessor< 24 | AppModelType = firestore.DocumentData, 25 | DbModelType extends firestore.DocumentData = firestore.DocumentData 26 | > = ( 27 | ...args: Parameters> 28 | ) => void | Promise | OnAfterBatchProcess | Promise; 29 | 30 | type TraversalConfigRules = { 31 | [K in keyof TraversalConfig]: { 32 | isValid: (val: unknown) => boolean; 33 | valDescription: string; 34 | }; 35 | }; 36 | 37 | export abstract class AbstractTraverser< 38 | AppModelType = firestore.DocumentData, 39 | DbModelType extends firestore.DocumentData = firestore.DocumentData 40 | > implements Traverser 41 | { 42 | protected static readonly baseConfig: TraversalConfig = { 43 | batchSize: 250, 44 | sleepTimeBetweenBatches: 0, 45 | maxDocCount: Infinity, 46 | maxConcurrentBatchCount: 1, 47 | maxBatchRetryCount: 0, 48 | sleepTimeBetweenTrials: 1_000, 49 | }; 50 | 51 | static readonly #configRules: TraversalConfigRules = { 52 | batchSize: { 53 | isValid: isPositiveInteger, 54 | valDescription: 'a positive integer', 55 | }, 56 | sleepTimeBetweenBatches: { 57 | isValid: isNonNegativeInteger, 58 | valDescription: 'a non-negative integer', 59 | }, 60 | maxDocCount: { 61 | isValid: isUnboundedPositiveInteger, 62 | valDescription: 'a positive integer or infinity', 63 | }, 64 | maxConcurrentBatchCount: { 65 | isValid: isPositiveInteger, 66 | valDescription: 'a positive integer', 67 | }, 68 | maxBatchRetryCount: { 69 | isValid: isNonNegativeInteger, 70 | valDescription: 'a non-negative integer', 71 | }, 72 | sleepTimeBetweenTrials: { 73 | isValid: (val) => typeof val === 'function' || isNonNegativeInteger(val), 74 | valDescription: 'a non-negative integer or a function that returns a non-negative integer', 75 | }, 76 | }; 77 | 78 | protected static readonly baseTraverseEachConfig: TraverseEachConfig = { 79 | sleepTimeBetweenDocs: 0, 80 | }; 81 | 82 | protected constructor( 83 | public readonly traversalConfig: TraversalConfig, 84 | protected readonly exitEarlyPredicates: ExitEarlyPredicate[] 85 | ) { 86 | this.#validateConfig(); 87 | } 88 | 89 | #validateConfig(): void { 90 | extractKeys(AbstractTraverser.#configRules).forEach((key) => { 91 | const val = this.traversalConfig[key]; 92 | const { isValid, valDescription } = AbstractTraverser.#configRules[key]; 93 | 94 | if (!isValid(val)) { 95 | throw new InvalidConfigError( 96 | `The '${key}' field in traversal config must be ${valDescription}.` 97 | ); 98 | } 99 | }); 100 | } 101 | 102 | public async traverseEach( 103 | callback: TraverseEachCallback, 104 | config: Partial = {} 105 | ): Promise { 106 | const { sleepTimeBetweenDocs } = { 107 | ...AbstractTraverser.baseTraverseEachConfig, 108 | ...config, 109 | }; 110 | 111 | const { batchCount, docCount } = await this.traverse(async (batchDocs, batchIndex) => { 112 | for (let i = 0; i < batchDocs.length; i++) { 113 | await callback(batchDocs[i], i, batchIndex); 114 | if (sleepTimeBetweenDocs > 0) { 115 | await sleep(sleepTimeBetweenDocs); 116 | } 117 | } 118 | }); 119 | 120 | return { batchCount, docCount }; 121 | } 122 | 123 | protected async runTraversal( 124 | processBatch: BatchProcessor 125 | ): Promise { 126 | const { batchSize, sleepTimeBetweenBatches, maxDocCount } = this.traversalConfig; 127 | 128 | let curBatchIndex = 0; 129 | let docCount = 0; 130 | let query = this.traversable.limit(Math.min(batchSize, maxDocCount)); 131 | 132 | while (true) { 133 | const { docs: batchDocs } = await query.get(); 134 | const batchDocCount = batchDocs.length; 135 | 136 | if (batchDocCount === 0) { 137 | break; 138 | } 139 | 140 | const lastDocInBatch = batchDocs[batchDocCount - 1]; 141 | 142 | docCount += batchDocCount; 143 | 144 | const onAfterBatchProcess = await processBatch(batchDocs, curBatchIndex); 145 | 146 | if (this.shouldExitEarly(batchDocs, curBatchIndex) || docCount === maxDocCount) { 147 | break; 148 | } 149 | 150 | await onAfterBatchProcess?.(); 151 | 152 | if (sleepTimeBetweenBatches > 0) { 153 | await sleep(sleepTimeBetweenBatches); 154 | } 155 | 156 | query = query.startAfter(lastDocInBatch).limit(Math.min(batchSize, maxDocCount - docCount)); 157 | curBatchIndex++; 158 | } 159 | 160 | return { batchCount: curBatchIndex, docCount }; 161 | } 162 | 163 | protected shouldExitEarly( 164 | batchDocs: firestore.QueryDocumentSnapshot[], 165 | batchIndex: number 166 | ): boolean { 167 | return this.exitEarlyPredicates.some((predicate) => predicate(batchDocs, batchIndex)); 168 | } 169 | 170 | public abstract readonly traversable: Traversable; 171 | 172 | public abstract withConfig( 173 | config: Partial 174 | ): Traverser; 175 | 176 | public abstract withExitEarlyPredicate( 177 | predicate: ExitEarlyPredicate 178 | ): Traverser; 179 | 180 | public abstract traverse( 181 | callback: BatchCallback 182 | ): Promise; 183 | } 184 | -------------------------------------------------------------------------------- /src/internal/implementations/abstract/index.ts: -------------------------------------------------------------------------------- 1 | export { AbstractMigrator, RegisteredCallbacks } from './AbstractMigrator'; 2 | export { AbstractTraverser } from './AbstractTraverser'; 3 | -------------------------------------------------------------------------------- /src/internal/implementations/index.ts: -------------------------------------------------------------------------------- 1 | export { BasicBatchMigratorImpl } from './BasicBatchMigratorImpl'; 2 | export { BasicDefaultMigratorImpl } from './BasicDefaultMigratorImpl'; 3 | export { PromiseQueueBasedTraverserImpl } from './PromiseQueueBasedTraverserImpl'; 4 | -------------------------------------------------------------------------------- /src/internal/utils/__tests__/isPositiveInteger.test.ts: -------------------------------------------------------------------------------- 1 | import { isPositiveInteger } from '..'; 2 | 3 | describe('isPositiveInteger', () => { 4 | test('respects math', () => { 5 | expect(isPositiveInteger(-Infinity)).toBe(false); 6 | expect(isPositiveInteger(-1.2)).toBe(false); 7 | expect(isPositiveInteger(-1)).toBe(false); 8 | expect(isPositiveInteger(0)).toBe(false); 9 | expect(isPositiveInteger(0.5)).toBe(false); 10 | expect(isPositiveInteger(1)).toBe(true); 11 | expect(isPositiveInteger(2)).toBe(true); 12 | expect(isPositiveInteger(Infinity)).toBe(false); 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /src/internal/utils/index.ts: -------------------------------------------------------------------------------- 1 | export { makeRetriable, registerInterval, sleep } from '@proficient/util'; 2 | export { isTraverser } from './isTraverser'; 3 | export * from './math'; 4 | export * from './object'; 5 | -------------------------------------------------------------------------------- /src/internal/utils/isTraverser.ts: -------------------------------------------------------------------------------- 1 | import type { firestore } from 'firebase-admin'; 2 | import type { Traverser } from '../../api'; 3 | 4 | export function isTraverser< 5 | AppModelType = firestore.DocumentData, 6 | DbModelType extends firestore.DocumentData = firestore.DocumentData 7 | >(candidate: unknown): candidate is Traverser { 8 | const t = candidate as Traverser; 9 | return ( 10 | !!t && 11 | typeof t === 'object' && 12 | t.traversable !== null && 13 | typeof t.traversable === 'object' && 14 | t.traversalConfig !== null && 15 | typeof t.traversalConfig === 'object' && 16 | typeof t.withConfig === 'function' && 17 | typeof t.withExitEarlyPredicate === 'function' && 18 | typeof t.traverseEach === 'function' && 19 | typeof t.traverse === 'function' 20 | ); 21 | } 22 | -------------------------------------------------------------------------------- /src/internal/utils/math.ts: -------------------------------------------------------------------------------- 1 | export function isNonNegativeInteger(num: unknown): boolean { 2 | return isPositiveInteger(num) || num === 0; 3 | } 4 | 5 | export function isUnboundedPositiveInteger(num: unknown): boolean { 6 | return isPositiveInteger(num) || num === Infinity; 7 | } 8 | 9 | export function isPositiveInteger(num: unknown): boolean { 10 | return typeof num === 'number' && Number.isInteger(num) && num > 0; 11 | } 12 | -------------------------------------------------------------------------------- /src/internal/utils/object.ts: -------------------------------------------------------------------------------- 1 | export const extractKeys = Object.keys as (o: T) => Extract[]; 2 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "dist", 4 | "rootDir": "src" 5 | }, 6 | "include": ["src"], 7 | "exclude": ["src/**/__tests__/**/*"], 8 | "extends": "./tsconfig.json" 9 | } 10 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "declaration": true, 4 | "module": "CommonJS", 5 | "moduleResolution": "node", 6 | "skipLibCheck": true, 7 | "sourceMap": true, 8 | "strict": true, 9 | "target": "ES6" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "noEmit": true, 4 | "module": "CommonJS", 5 | "moduleResolution": "node", 6 | "strict": true, 7 | "target": "ES6" 8 | }, 9 | "extends": "./tsconfig.json", 10 | "include": ["jest.config.global.ts", "jest.config.ts", "**/__tests__/**/*"], 11 | "exclude": ["__tests__/types/**/*"] 12 | } 13 | -------------------------------------------------------------------------------- /typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "entryPoints": ["src/index.ts"], 3 | "tsconfig": "src/tsconfig.json", 4 | "searchInComments": true, 5 | "navigation": { 6 | "includeCategories": true, 7 | "includeGroups": true 8 | }, 9 | "categorizeByGroup": true, 10 | "githubPages": false 11 | } 12 | --------------------------------------------------------------------------------