├── .eslintrc.js
├── .github
└── workflows
│ ├── ci.yml
│ ├── publish-docs.yml
│ └── publish-package.yml
├── .gitignore
├── .prettierrc.json
├── .vscode
└── settings.json
├── LICENSE
├── README.md
├── __tests__
├── app.ts
├── types
│ ├── DefaultMigrator.test-d.ts
│ ├── Traverser.test-d.ts
│ ├── _helpers.ts
│ ├── createBatchMigrator.test-d.ts
│ ├── createMigrator.test-d.ts
│ └── createTraverser.test-d.ts
└── utils
│ ├── collectionPopulator.ts
│ ├── getLexicographicallyNextString.test.ts
│ ├── getLexicographicallyNextString.ts
│ ├── index.ts
│ └── writeFileAndMkdirSync.ts
├── babel.config.js
├── examples
├── addNewField.ts
├── addNewFieldFromPrevFields.ts
├── changeTraversalConfig.ts
├── exitEarlyPredicate.ts
├── fastMigrator.ts
├── fastTraverser.ts
├── migrationPredicate.ts
├── quickStart.ts
└── renameField.ts
├── jest.config.global.ts
├── jest.config.ts
├── package-lock.json
├── package.json
├── scripts
└── generate-docs.sh
├── src
├── api
│ ├── createBatchMigrator.ts
│ ├── createMigrator.ts
│ ├── createTraverser.ts
│ ├── index.ts
│ └── interfaces
│ │ ├── BatchCallback.ts
│ │ ├── BatchMigrator.ts
│ │ ├── DefaultMigrator.ts
│ │ ├── ExitEarlyPredicate.ts
│ │ ├── MigrationPredicate.ts
│ │ ├── MigrationResult.ts
│ │ ├── Migrator.ts
│ │ ├── SetDataGetter.ts
│ │ ├── SetOptions.ts
│ │ ├── SetPartialDataGetter.ts
│ │ ├── Traversable.ts
│ │ ├── TraversalConfig.ts
│ │ ├── TraversalResult.ts
│ │ ├── TraverseEachCallback.ts
│ │ ├── TraverseEachConfig.ts
│ │ ├── Traverser.ts
│ │ ├── UpdateDataGetter.ts
│ │ ├── UpdateFieldValueGetter.ts
│ │ └── index.ts
├── errors
│ ├── ImplementationError.ts
│ ├── InvalidConfigError.ts
│ └── index.ts
├── index.ts
└── internal
│ ├── ds
│ ├── PromiseQueue.ts
│ ├── SLLQueueExtended.ts
│ ├── __tests__
│ │ ├── PromiseQueue.test.ts
│ │ └── SLLQueueExtended.test.ts
│ └── index.ts
│ ├── errors
│ ├── IllegalArgumentError.ts
│ └── index.ts
│ ├── implementations
│ ├── BasicBatchMigratorImpl.ts
│ ├── BasicDefaultMigratorImpl.ts
│ ├── PromiseQueueBasedTraverserImpl.ts
│ ├── __tests__
│ │ ├── Migrator
│ │ │ ├── BasicBatchMigratorImpl
│ │ │ │ ├── deleteField.test.ts
│ │ │ │ ├── deleteFields.test.ts
│ │ │ │ ├── helpers.ts
│ │ │ │ ├── renameField.test.ts
│ │ │ │ ├── renameFields.test.ts
│ │ │ │ ├── update.test.ts
│ │ │ │ └── updateWithDerivedData.test.ts
│ │ │ ├── BasicDefaultMigratorImpl
│ │ │ │ ├── deleteField.test.ts
│ │ │ │ ├── deleteFields.test.ts
│ │ │ │ ├── helpers.ts
│ │ │ │ ├── renameField.test.ts
│ │ │ │ ├── renameFields.test.ts
│ │ │ │ ├── update.test.ts
│ │ │ │ └── updateWithDerivedData.test.ts
│ │ │ ├── config.ts
│ │ │ ├── helpers.ts
│ │ │ └── shared
│ │ │ │ ├── deleteField.ts
│ │ │ │ ├── deleteFields.ts
│ │ │ │ ├── renameField.ts
│ │ │ │ ├── renameFields.ts
│ │ │ │ ├── update.ts
│ │ │ │ └── updateWithDerivedData.ts
│ │ └── Traverser
│ │ │ ├── PromiseQueueBasedTraverserImpl
│ │ │ ├── helpers.ts
│ │ │ ├── traverse.test.ts
│ │ │ └── withExitEarlyPredicate.test.ts
│ │ │ ├── config.ts
│ │ │ ├── helpers.ts
│ │ │ └── shared
│ │ │ ├── traverse.ts
│ │ │ └── withExitEarlyPredicate.ts
│ ├── abstract
│ │ ├── AbstractMigrator.ts
│ │ ├── AbstractTraverser.ts
│ │ └── index.ts
│ └── index.ts
│ └── utils
│ ├── __tests__
│ └── isPositiveInteger.test.ts
│ ├── index.ts
│ ├── isTraverser.ts
│ ├── math.ts
│ └── object.ts
├── tsconfig.build.json
├── tsconfig.json
├── tsconfig.test.json
└── typedoc.json
/.eslintrc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | env: {
3 | es2021: true,
4 | node: true,
5 | },
6 | extends: [
7 | 'eslint:recommended',
8 | 'plugin:@typescript-eslint/recommended',
9 | 'plugin:import/errors',
10 | 'plugin:import/warnings',
11 | 'plugin:prettier/recommended',
12 | 'prettier',
13 | ],
14 | parser: '@typescript-eslint/parser',
15 | parserOptions: {
16 | ecmaVersion: 12,
17 | sourceType: 'module',
18 | },
19 | plugins: ['@typescript-eslint', 'import', 'prettier'],
20 | rules: {
21 | '@typescript-eslint/explicit-function-return-type': [
22 | 2,
23 | {
24 | allowExpressions: true,
25 | },
26 | ],
27 | '@typescript-eslint/explicit-module-boundary-types': 0,
28 | '@typescript-eslint/no-empty-function': 0,
29 | '@typescript-eslint/no-explicit-any': 0,
30 | '@typescript-eslint/no-non-null-assertion': 0,
31 | 'no-constant-condition': 0,
32 | },
33 | settings: {
34 | 'import/resolver': {
35 | node: {
36 | extensions: ['.json', '.js', '.ts'],
37 | },
38 | },
39 | },
40 | };
41 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: CI
2 |
3 | on: [pull_request]
4 |
5 | concurrency:
6 | group: ${{ github.ref }}
7 | cancel-in-progress: true
8 |
9 | jobs:
10 | validate:
11 | runs-on: ubuntu-latest
12 | steps:
13 | - uses: actions/checkout@v4
14 | - name: Use Node.js
15 | uses: actions/setup-node@v4
16 | with:
17 | node-version: '22.x'
18 | cache: npm
19 | - name: Install project dependencies
20 | run: npm ci
21 | - name: Build the source
22 | run: npm run build
23 | - name: Run ESLint checks
24 | run: npm run lint
25 | - name: Run Prettier checks
26 | run: npm run format-check
27 | - name: Run tests
28 | run: npm run test
29 |
--------------------------------------------------------------------------------
/.github/workflows/publish-docs.yml:
--------------------------------------------------------------------------------
1 | name: Publish docs
2 |
3 | on:
4 | # Runs on pushes targeting the main branch
5 | push:
6 | branches: ['main']
7 |
8 | # Allows you to run this workflow manually from the Actions tab
9 | workflow_dispatch:
10 |
11 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
12 | permissions:
13 | contents: write
14 | pages: write
15 | id-token: write
16 |
17 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
18 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
19 | concurrency:
20 | group: 'pages'
21 | cancel-in-progress: false
22 |
23 | jobs:
24 | generate_and_commit_docs:
25 | runs-on: ubuntu-latest
26 | steps:
27 | - name: Checkout docs branch
28 | uses: actions/checkout@v4
29 | with:
30 | ref: 'docs'
31 |
32 | - name: Set account identity
33 | run: |
34 | git config --global user.email "kafkas@users.noreply.github.com"
35 | git config --global user.name "Documentation Bot"
36 |
37 | - name: Rebase with main branch
38 | run: |
39 | git fetch
40 | git rebase -Xours origin/main
41 |
42 | - name: Install dependencies
43 | run: npm install
44 |
45 | - name: Generate docs for current version
46 | run: npm run docs-generate
47 |
48 | - name: Commit and Push Changes
49 | run: |
50 | git add docs/
51 | git commit -m "Update docs (bot)" || echo "No changes (bot)"
52 | git push -f
53 |
54 | deploy_to_github_pages:
55 | needs: generate_and_commit_docs
56 | environment:
57 | name: github-pages
58 | url: ${{ steps.deployment.outputs.page_url }}
59 | runs-on: ubuntu-latest
60 | steps:
61 | - name: Checkout docs branch
62 | uses: actions/checkout@v4
63 | with:
64 | ref: 'docs'
65 |
66 | - name: Setup Pages
67 | uses: actions/configure-pages@v4
68 |
69 | - name: Upload artifact
70 | uses: actions/upload-pages-artifact@v3
71 | with:
72 | path: './docs'
73 |
74 | - name: Deploy to GitHub Pages
75 | id: deployment
76 | uses: actions/deploy-pages@v4
77 |
--------------------------------------------------------------------------------
/.github/workflows/publish-package.yml:
--------------------------------------------------------------------------------
1 | name: Publish package
2 |
3 | on:
4 | workflow_dispatch:
5 | inputs:
6 | version_increment:
7 | type: choice
8 | required: true
9 | description: 'Select version increment type'
10 | options:
11 | - patch
12 | - minor
13 | - major
14 |
15 | permissions:
16 | contents: write
17 |
18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
20 | concurrency:
21 | group: 'version'
22 | cancel-in-progress: true
23 |
24 | jobs:
25 | bump_version_and_publish:
26 | runs-on: ubuntu-latest
27 | permissions:
28 | contents: write
29 | packages: write
30 | pull-requests: write
31 | steps:
32 | - uses: actions/checkout@v4
33 |
34 | - name: Use Node.js
35 | uses: actions/setup-node@v4
36 | with:
37 | node-version: '22.x'
38 | cache: 'npm'
39 | registry-url: 'https://registry.npmjs.org'
40 |
41 | - name: Install project dependencies
42 | run: npm ci
43 |
44 | - name: Build the source
45 | run: npm run build
46 |
47 | - name: Set git identity
48 | run: |
49 | git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
50 | git config --global user.name "github-actions[bot]"
51 |
52 | - name: Bump version
53 | run: npm version ${{ inputs.version_increment }}
54 |
55 | - name: Push commit and create tag and release
56 | env:
57 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
58 | run: |
59 | version="$(jq -r .version package.json)"
60 | git push origin
61 | git push origin "v$version"
62 | gh release create "v$version" --title "v$version"
63 |
64 | - name: Publish the package to NPM
65 | run: npm publish
66 | env:
67 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
68 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 |
3 | coverage/
4 |
5 | dist/
6 |
7 | .DS_Store
8 | firestore-debug.log
9 | firebase-debug.log
10 |
--------------------------------------------------------------------------------
/.prettierrc.json:
--------------------------------------------------------------------------------
1 | {
2 | "printWidth": 100,
3 | "tabWidth": 2,
4 | "singleQuote": true,
5 | "trailingComma": "es5",
6 | "endOfLine": "auto"
7 | }
8 |
--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
1 | {
2 | "editor.fontLigatures": "'ss01', 'ss02', 'ss03', 'ss04', 'ss05', 'ss06', 'zero', 'onum'"
3 | }
4 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2023 Anar Kafkas
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
6 |
7 |
8 | A light, fast, and memory-efficient collection traversal library for Firestore and Node.js.
9 |
10 |
11 | ---
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 | Firewalk is a Node.js library that _walks_ you through Firestore collections.
29 |
30 | When you have millions of documents in a collection and you need to make changes to them or just read them, you can't just retrieve all of them at once as your program's memory usage will explode. Firewalk's configurable traverser objects let you do this in a simple, intuitive and memory-efficient way using batch processing with concurrency control.
31 |
32 | Firewalk is an extremely light and well-typed library that is useful in a variety of scenarios. You can use it in database migration scripts (e.g. when you need to add a new field to all docs) or a scheduled Cloud Function that needs to check every doc in a collection periodically or even a locally run script that retrieves some data from a collection.
33 |
34 | **Note**: This library was previously known as Firecode. We're currently in the process of porting over the documentation from the
35 | previous site.
36 |
37 | [**Read the introductory blog post ▸**](https://anarkafkas.medium.com/traversing-firestore-collections-efficiently-6e43cea1eefd)
38 |
39 | [**View the full documentation (docs) ▸**](https://kafkas.github.io/firewalk)
40 |
41 | ## Overview
42 |
43 | 1. [Prerequisites](#Prerequisites)
44 | 1. [Compatibility Map](#Compatibility-Map)
45 | 1. [Installation](#Installation)
46 | 1. [Core Concepts](#Core-Concepts)
47 | 1. [Quick Start](#Quick-Start)
48 | 1. [More Examples](#More-Examples)
49 | 1. [API](#API)
50 | 1. [Upgrading](#Upgrading)
51 | 1. [License](#License)
52 |
53 | ## Prerequisites
54 |
55 | Firewalk is designed to work with the [Firebase Admin SDK](https://github.com/firebase/firebase-admin-node) so if you haven't already installed it, you'll need add it as a dependency to your project.
56 |
57 | ```bash
58 | npm install firebase-admin
59 | ```
60 |
61 | ### Compatibility Map
62 |
63 | Make sure to install the right version of Firewalk depending on the `firebase-admin` version your project is on.
64 |
65 | | firewalk | firebase-admin |
66 | | -------- | -------------- |
67 | | v1 | v9-10 |
68 | | v2 | v11-13 |
69 |
70 | ## Installation
71 |
72 | You can add Firewalk to your project with npm or yarn.
73 |
74 | ```bash
75 | npm install firewalk
76 | ```
77 |
78 | ## Core Concepts
79 |
80 | There are only 2 kinds of objects you need to be familiar with when using this library:
81 |
82 | 1. **Traverser**: An object that walks you through a collection of documents (or more generally a [Traversable](https://kafkas.github.io/firewalk/types/Traversable.html)).
83 |
84 | 2. **Migrator**: A convenience object used for database migrations. It lets you easily write to the documents within a given traversable and uses a traverser to do that. You can easily write your own migration logic in the traverser callback if you don't want to use a migrator.
85 |
86 | ## Quick Start
87 |
88 | Suppose we have a `users` collection and we want to send an email to each user. This is how easy it is to do that efficiently with a Firewalk traverser:
89 |
90 | ```ts
91 | import { firestore } from 'firebase-admin';
92 | import { createTraverser } from 'firewalk';
93 |
94 | const usersCollection = firestore().collection('users');
95 | const traverser = createTraverser(usersCollection);
96 |
97 | const { batchCount, docCount } = await traverser.traverse(async (batchDocs, batchIndex) => {
98 | const batchSize = batchDocs.length;
99 | await Promise.all(
100 | batchDocs.map(async (doc) => {
101 | const { email, firstName } = doc.data();
102 | await sendEmail({ to: email, content: `Hello ${firstName}!` });
103 | })
104 | );
105 | console.log(`Batch ${batchIndex} done! We emailed ${batchSize} users in this batch.`);
106 | });
107 |
108 | console.log(`Traversal done! We emailed ${docCount} users in ${batchCount} batches!`);
109 | ```
110 |
111 | We are doing 3 things here:
112 |
113 | 1. Create a reference to the `users` collection
114 | 2. Pass that reference to the `createTraverser()` function
115 | 3. Invoke `.traverse()` with an async callback that is called for each batch of document snapshots
116 |
117 | This pretty much sums up the core functionality of this library! The `.traverse()` method returns a Promise that resolves when the entire traversal finishes, which can take a while if you have millions of docs. The Promise resolves with an object containing the traversal details e.g. the number of docs you touched.
118 |
119 | ## More Examples
120 |
121 | ### Traverse faster by increasing concurrency
122 |
123 | ```ts
124 | const projectsColRef = firestore().collection('projects');
125 | const traverser = createTraverser(projectsColRef, {
126 | batchSize: 500,
127 | // This means we are prepared to hold 500 * 20 = 10,000 docs in memory.
128 | // We sacrifice some memory to traverse faster.
129 | maxConcurrentBatchCount: 20,
130 | });
131 | const { docCount } = await traverser.traverse(async (_, batchIndex) => {
132 | console.log(`Gonna process batch ${batchIndex} now!`);
133 | // ...
134 | });
135 | console.log(`Traversed ${docCount} projects super-fast!`);
136 | ```
137 |
138 | ### Add a new field using a migrator
139 |
140 | ```ts
141 | const projectsColRef = firestore().collection('projects');
142 | const migrator = createMigrator(projectsColRef);
143 | const { migratedDocCount } = await migrator.update('isCompleted', false);
144 | console.log(`Updated ${migratedDocCount} projects!`);
145 | ```
146 |
147 | ### Add a new field derived from the previous fields
148 |
149 | ```ts
150 | type UserDoc = {
151 | firstName: string;
152 | lastName: string;
153 | };
154 | const usersColRef = firestore().collection('users') as firestore.CollectionReference;
155 | const migrator = createMigrator(usersColRef);
156 | const { migratedDocCount } = await migrator.updateWithDerivedData((snap) => {
157 | const { firstName, lastName } = snap.data();
158 | return {
159 | fullName: `${firstName} ${lastName}`,
160 | };
161 | });
162 | console.log(`Updated ${migratedDocCount} users!`);
163 | ```
164 |
165 | ### Migrate faster by increasing concurrency
166 |
167 | ```ts
168 | const projectsColRef = firestore().collection('projects');
169 | const migrator = createMigrator(projectsColRef, { maxConcurrentBatchCount: 25 });
170 | const { migratedDocCount } = await migrator.update('isCompleted', false);
171 | console.log(`Updated ${migratedDocCount} projects super-fast!`);
172 | ```
173 |
174 | ### Change traversal config
175 |
176 | ```ts
177 | const walletsWithNegativeBalance = firestore().collection('wallets').where('money', '<', 0);
178 | const migrator = createMigrator(walletsWithNegativeBalance, {
179 | // We want each batch to have 500 docs. The size of the very last batch may be less than 500
180 | batchSize: 500,
181 | // We want to wait 500ms before moving to the next batch
182 | sleepTimeBetweenBatches: 500,
183 | });
184 | // Wipe out their debts!
185 | const { migratedDocCount } = await migrator.set({ money: 0 });
186 | console.log(`Set ${migratedDocCount} wallets!`);
187 | ```
188 |
189 | ### Rename a field
190 |
191 | ```ts
192 | const postsColGroup = firestore().collectionGroup('posts');
193 | const migrator = createMigrator(postsColGroup);
194 | const { migratedDocCount } = await migrator.renameField('postedAt', 'publishedAt');
195 | console.log(`Updated ${migratedDocCount} posts!`);
196 | ```
197 |
198 | ## [API](https://kafkas.github.io/firewalk)
199 |
200 | You can find the full API reference for Firewalk [here](https://kafkas.github.io/firewalk). We maintain detailed docs for each major version. Here are some of the core functions that this library provides.
201 |
202 | ### [createTraverser](https://kafkas.github.io/firewalk/functions/createTraverser.html)
203 |
204 | Creates an object which can be used to traverse a Firestore collection or, more generally, a [Traversable](https://kafkas.github.io/firewalk/types/Traversable.html).
205 |
206 | For each batch of document snapshots in the traversable, the traverser invokes a specified async callback and immediately moves to the next batch. It does not wait for the callback Promise to resolve before moving to the next batch. That is, when `maxConcurrentBatchCount` > 1, there is no guarantee that any given batch will finish processing before a later batch.
207 |
208 | The traverser becomes faster as you increase `maxConcurrentBatchCount`, but this will consume more memory. You should increase concurrency when you want to trade some memory for speed.
209 |
210 | #### Complexity:
211 |
212 | - Time complexity: _O_((_N_ / `batchSize`) \* (_Q_(`batchSize`) + _C_(`batchSize`) / `maxConcurrentBatchCount`))
213 | - Space complexity: _O_(`maxConcurrentBatchCount` \* (`batchSize` \* _D_ + _S_))
214 | - Billing: _max_(1, _N_) reads
215 |
216 | where:
217 |
218 | - _N_: number of docs in the traversable
219 | - _Q_(`batchSize`): average batch query time
220 | - _C_(`batchSize`): average callback processing time
221 | - _D_: average document size
222 | - _S_: average extra space used by the callback
223 |
224 | ### [createMigrator](https://kafkas.github.io/firewalk/functions/createMigrator.html)
225 |
226 | Creates a migrator that facilitates database migrations. The migrator accepts a custom traverser to traverse the collection. Otherwise it will create a default traverser with your desired traversal config. This migrator does not use atomic batch writes so it is possible that when a write fails other writes go through.
227 |
228 | #### Complexity:
229 |
230 | - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
231 | - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
232 | - Billing: _max_(1, _N_) reads, _K_ writes
233 |
234 | where:
235 |
236 | - _N_: number of docs in the traversable
237 | - _K_: number of docs that passed the migration predicate (_K_<=_N_)
238 | - _W_(`batchSize`): average batch write time
239 | - _TC_(`traverser`): time complexity of the underlying traverser
240 | - _SC_(`traverser`): space complexity of the underlying traverser
241 |
242 | ### [createBatchMigrator](https://kafkas.github.io/firewalk/functions/createBatchMigrator.html)
243 |
244 | Creates a migrator that facilitates database migrations. The migrator accepts a custom traverser to traverse the collection. Otherwise it will create a default traverser with your desired traversal config. This migrator uses atomic batch writes so the entire operation will fail if a single write isn't successful.
245 |
246 | #### Complexity:
247 |
248 | - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
249 | - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
250 | - Billing: _max_(1, _N_) reads, _K_ writes
251 |
252 | where:
253 |
254 | - _N_: number of docs in the traversable
255 | - _K_: number of docs that passed the migration predicate (_K_<=_N_)
256 | - _W_(`batchSize`): average batch write time
257 | - _TC_(`traverser`): time complexity of the underlying traverser
258 | - _SC_(`traverser`): space complexity of the underlying traverser
259 |
260 | ## Upgrading
261 |
262 | This project adheres to [SemVer](https://semver.org). Before upgrading to a new major version, make sure to check out the [Releases](https://github.com/kafkas/firewalk/releases) page to view all the breaking changes.
263 |
264 | ## License
265 |
266 | This project is made available under the MIT License.
267 |
--------------------------------------------------------------------------------
/__tests__/app.ts:
--------------------------------------------------------------------------------
1 | import * as admin from 'firebase-admin';
2 |
3 | class Application {
4 | public get firestore(): admin.firestore.Firestore {
5 | return this.firebaseApp.firestore();
6 | }
7 |
8 | public constructor(private readonly firebaseApp: admin.app.App) {}
9 | }
10 |
11 | let _app: Application | undefined;
12 |
13 | export function app(): Application {
14 | const firebaseApp = admin.initializeApp();
15 | return _app ?? (_app = new Application(firebaseApp));
16 | }
17 |
--------------------------------------------------------------------------------
/__tests__/types/DefaultMigrator.test-d.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable import/no-unresolved */
2 |
3 | import { firestore } from 'firebase-admin';
4 | import { expectError, expectType } from 'tsd';
5 | import { DefaultMigrator, Traverser, createMigrator, createTraverser } from '../../src';
6 | import { TestAppModelType, TestDbModelType, collectionRef } from './_helpers';
7 |
8 | const defaultMigrator = createMigrator(collectionRef);
9 |
10 | expectType>(defaultMigrator.traverser);
11 |
12 | (() => {
13 | const modifiedMigrator = defaultMigrator.withPredicate((doc) => {
14 | expectType>(doc);
15 | return false;
16 | });
17 | expectType>(modifiedMigrator);
18 | })();
19 |
20 | (() => {
21 | const traverser = createTraverser(collectionRef);
22 | const modifiedMigrator = defaultMigrator.withTraverser(traverser);
23 | expectType>(modifiedMigrator);
24 | })();
25 |
26 | defaultMigrator.onBeforeBatchStart((batchDocs, batchIndex) => {
27 | expectType[]>(batchDocs);
28 | expectType(batchIndex);
29 | });
30 |
31 | defaultMigrator.onAfterBatchComplete((batchDocs, batchIndex) => {
32 | expectType[]>(batchDocs);
33 | expectType(batchIndex);
34 | });
35 |
36 | defaultMigrator.deleteField('oldField');
37 | defaultMigrator.deleteField(new firestore.FieldPath('nested', 'field'));
38 |
39 | defaultMigrator.deleteFields(new firestore.FieldPath('nested', 'field'), 'field2', 'field3');
40 |
41 | defaultMigrator.renameField('oldField', new firestore.FieldPath('new', 'field'));
42 | defaultMigrator.renameField(new firestore.FieldPath('old', 'field'), 'newField');
43 |
44 | defaultMigrator.renameFields(
45 | ['field1', new firestore.FieldPath('nested', 'field2')],
46 | ['field1', 'field2']
47 | );
48 |
49 | expectError(defaultMigrator.set({ num: 0 }));
50 | defaultMigrator.set({ num: 0, text: '' });
51 | defaultMigrator.set({ num: 0 }, { merge: true });
52 |
53 | expectError(
54 | defaultMigrator.setWithDerivedData((doc) => {
55 | expectType>(doc);
56 | return { num: 0 };
57 | })
58 | );
59 | defaultMigrator.setWithDerivedData((doc) => {
60 | expectType>(doc);
61 | return { num: 0, text: '' };
62 | });
63 | defaultMigrator.setWithDerivedData(
64 | (doc) => {
65 | expectType>(doc);
66 | return { num: 0 };
67 | },
68 | { merge: true }
69 | );
70 |
71 | expectError(
72 | defaultMigrator.update({
73 | anyField: '',
74 | })
75 | );
76 |
77 | defaultMigrator.update('anyField', 'anyValue');
78 |
79 | expectError(
80 | defaultMigrator.updateWithDerivedData((doc) => {
81 | expectType>(doc);
82 | return { anyField: '' };
83 | })
84 | );
85 |
86 | expectError(
87 | defaultMigrator.updateWithDerivedData((doc) => {
88 | expectType>(doc);
89 | return ['anyField', 'anyValue'];
90 | })
91 | );
92 |
93 | // TODO: We need to expect an error here if the return type of the callback is not a plain object or an array
94 | // expectError(
95 | // defaultMigrator.updateWithDerivedData((doc) => {
96 | // expectType>(doc);
97 | // return new Map([]);
98 | // })
99 | // );
100 |
--------------------------------------------------------------------------------
/__tests__/types/Traverser.test-d.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable import/no-unresolved */
2 |
3 | import type { firestore } from 'firebase-admin';
4 | import { expectType } from 'tsd';
5 | import {
6 | createTraverser,
7 | Traversable,
8 | TraversalConfig,
9 | TraversalResult,
10 | Traverser,
11 | } from '../../src';
12 | import { collectionRef, TestAppModelType, TestDbModelType } from './_helpers';
13 |
14 | const traverser = createTraverser(collectionRef);
15 |
16 | // TODO: Ideally we want to expect a firestore.CollectionReference here because
17 | // we initialized the traverser with a collection reference.
18 | expectType>(traverser.traversable);
19 |
20 | expectType(traverser.traversalConfig);
21 |
22 | (() => {
23 | // TODO: See if there is a better way to check inferred parameters
24 | const modifiedTraverser = traverser.withConfig({
25 | batchSize: 0,
26 | sleepTimeBetweenBatches: 0,
27 | maxDocCount: 0,
28 | maxConcurrentBatchCount: 0,
29 | });
30 | expectType>(modifiedTraverser);
31 | })();
32 |
33 | (() => {
34 | const modifiedTraverser = traverser.withExitEarlyPredicate((batchDocs, batchIndex) => {
35 | expectType[]>(batchDocs);
36 | expectType(batchIndex);
37 | return false;
38 | });
39 | expectType>(modifiedTraverser);
40 | })();
41 |
42 | (async () => {
43 | const traversalResult1 = await traverser.traverseEach(async (doc, docIndex, batchIndex) => {
44 | expectType>(doc);
45 | expectType(docIndex);
46 | expectType(batchIndex);
47 | });
48 | expectType(traversalResult1);
49 | })();
50 |
51 | (async () => {
52 | const traversalResult2 = await traverser.traverse(async (batchDocs, batchIndex) => {
53 | expectType[]>(batchDocs);
54 | expectType(batchIndex);
55 | });
56 | expectType(traversalResult2);
57 | })();
58 |
--------------------------------------------------------------------------------
/__tests__/types/_helpers.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 |
3 | export type TestAppModelType = {
4 | text: string;
5 | num: number;
6 | };
7 |
8 | export type TestDbModelType = {
9 | text: string;
10 | num: number;
11 | };
12 |
13 | export const collectionRef = firestore().collection('projects') as firestore.CollectionReference<
14 | TestAppModelType,
15 | TestDbModelType
16 | >;
17 |
--------------------------------------------------------------------------------
/__tests__/types/createBatchMigrator.test-d.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable import/no-unresolved */
2 |
3 | import { expectType } from 'tsd';
4 | import { createBatchMigrator, createTraverser, BatchMigrator } from '../../src';
5 | import { collectionRef, TestAppModelType, TestDbModelType } from './_helpers';
6 |
7 | // Signature 1
8 |
9 | const traverser = createTraverser(collectionRef);
10 | const batchMigrator = createBatchMigrator(traverser);
11 | expectType>(batchMigrator);
12 |
13 | // Signature 2
14 |
15 | const batchMigrator2 = createBatchMigrator(collectionRef, {
16 | batchSize: 0,
17 | sleepTimeBetweenBatches: 0,
18 | maxDocCount: 0,
19 | });
20 | expectType>(batchMigrator2);
21 |
--------------------------------------------------------------------------------
/__tests__/types/createMigrator.test-d.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable import/no-unresolved */
2 |
3 | import { expectType } from 'tsd';
4 | import { createMigrator, createTraverser, DefaultMigrator } from '../../src';
5 | import { collectionRef, TestAppModelType, TestDbModelType } from './_helpers';
6 |
7 | // Signature 1
8 |
9 | const traverser = createTraverser(collectionRef);
10 | const defaultMigrator = createMigrator(traverser);
11 | expectType>(defaultMigrator);
12 |
13 | // Signature 2
14 |
15 | const defaultMigrator2 = createMigrator(collectionRef, {
16 | batchSize: 0,
17 | sleepTimeBetweenBatches: 0,
18 | maxDocCount: 0,
19 | maxConcurrentBatchCount: 0,
20 | });
21 | expectType>(defaultMigrator2);
22 |
--------------------------------------------------------------------------------
/__tests__/types/createTraverser.test-d.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable import/no-unresolved */
2 |
3 | import { expectType } from 'tsd';
4 | import { createTraverser, Traverser } from '../../src';
5 | import { collectionRef, TestAppModelType, TestDbModelType } from './_helpers';
6 |
7 | const traverser = createTraverser(collectionRef, {
8 | batchSize: 0,
9 | sleepTimeBetweenBatches: 0,
10 | maxDocCount: 0,
11 | maxConcurrentBatchCount: 0,
12 | });
13 |
14 | expectType>(traverser);
15 |
--------------------------------------------------------------------------------
/__tests__/utils/collectionPopulator.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 |
3 | interface CollectionPopulatorBuilder<
4 | AppModelType = firestore.DocumentData,
5 | DbModelType extends firestore.DocumentData = firestore.DocumentData
6 | > {
7 | withData(
8 | dataOrGetData: AppModelType | (() => AppModelType)
9 | ): CollectionPopulator;
10 | }
11 |
12 | interface CollectionPopulator<
13 | AppModelType = firestore.DocumentData,
14 | DbModelType extends firestore.DocumentData = firestore.DocumentData
15 | > {
16 | populate(opts: {
17 | count: number;
18 | }): Promise[]>;
19 | }
20 |
21 | export function collectionPopulator<
22 | AppModelType = firestore.DocumentData,
23 | DbModelType extends firestore.DocumentData = firestore.DocumentData
24 | >(
25 | collectionRef: firestore.CollectionReference
26 | ): CollectionPopulatorBuilder {
27 | return {
28 | withData: (dataOrGetData) => {
29 | return {
30 | populate: async ({ count: docCount }) => {
31 | const promises = new Array(docCount).fill(null).map(async () => {
32 | const data =
33 | typeof dataOrGetData === 'function'
34 | ? (dataOrGetData as () => AppModelType)()
35 | : dataOrGetData;
36 | return await collectionRef.add(data);
37 | });
38 |
39 | return await Promise.all(promises);
40 | },
41 | };
42 | },
43 | };
44 | }
45 |
--------------------------------------------------------------------------------
/__tests__/utils/getLexicographicallyNextString.test.ts:
--------------------------------------------------------------------------------
1 | import { getLexicographicallyNextString } from './getLexicographicallyNextString';
2 |
3 | test('correctly computes lexicographically next string', () => {
4 | expect(getLexicographicallyNextString('')).toStrictEqual('a');
5 | expect(getLexicographicallyNextString('a')).toStrictEqual('b');
6 | expect(getLexicographicallyNextString('abc')).toStrictEqual('abd');
7 | expect(getLexicographicallyNextString('abc123')).toStrictEqual('abc124');
8 | expect(getLexicographicallyNextString('abc123xyz')).toStrictEqual('abc123xzz');
9 | expect(getLexicographicallyNextString('z')).toStrictEqual('za');
10 | expect(getLexicographicallyNextString('zz')).toStrictEqual('zza');
11 | });
12 |
--------------------------------------------------------------------------------
/__tests__/utils/getLexicographicallyNextString.ts:
--------------------------------------------------------------------------------
1 | export function getLexicographicallyNextString(str: string): string {
2 | if (str == '') return 'a';
3 |
4 | let i = str.length - 1;
5 |
6 | while (str[i] == 'z' && i >= 0) {
7 | i--;
8 | }
9 |
10 | if (i === -1) {
11 | str = str + 'a';
12 | } else {
13 | str = str.substring(0, i) + String.fromCharCode(str.charCodeAt(i) + 1) + str.substring(i + 1);
14 | }
15 |
16 | return str;
17 | }
18 |
--------------------------------------------------------------------------------
/__tests__/utils/index.ts:
--------------------------------------------------------------------------------
1 | export { collectionPopulator } from './collectionPopulator';
2 | export { writeFileAndMkdirSync } from './writeFileAndMkdirSync';
3 |
--------------------------------------------------------------------------------
/__tests__/utils/writeFileAndMkdirSync.ts:
--------------------------------------------------------------------------------
1 | import { existsSync, mkdirSync, writeFileSync } from 'fs';
2 | import { dirname } from 'path';
3 |
4 | export function writeFileAndMkdirSync(
5 | path: string,
6 | data: string | Record | unknown[],
7 | opts?: Parameters[2]
8 | ): void {
9 | const d = typeof data === 'string' ? data : JSON.stringify(data);
10 | ensureDirectoryExists(path);
11 | writeFileSync(path, d, opts);
12 | }
13 |
14 | function ensureDirectoryExists(path: string): void {
15 | const dirName = dirname(path);
16 | if (existsSync(dirName)) {
17 | return;
18 | }
19 | ensureDirectoryExists(dirName);
20 | mkdirSync(dirName);
21 | }
22 |
--------------------------------------------------------------------------------
/babel.config.js:
--------------------------------------------------------------------------------
1 | // babel.config.js
2 | module.exports = {
3 | presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'],
4 | };
5 |
--------------------------------------------------------------------------------
/examples/addNewField.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 | import { createMigrator } from 'firewalk';
3 |
4 | const projectsColRef = firestore().collection('projects');
5 | const migrator = createMigrator(projectsColRef);
6 | const { migratedDocCount } = await migrator.update('isCompleted', false);
7 | console.log(`Updated ${migratedDocCount} projects!`);
8 |
--------------------------------------------------------------------------------
/examples/addNewFieldFromPrevFields.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 | import { createMigrator } from 'firewalk';
3 |
4 | type UserDoc = {
5 | firstName: string;
6 | lastName: string;
7 | };
8 | const usersColRef = firestore().collection('users') as firestore.CollectionReference;
9 | const migrator = createMigrator(usersColRef);
10 | const { migratedDocCount } = await migrator.updateWithDerivedData((snap) => {
11 | const { firstName, lastName } = snap.data();
12 | return {
13 | fullName: `${firstName} ${lastName}`,
14 | };
15 | });
16 | console.log(`Updated ${migratedDocCount} users!`);
17 |
--------------------------------------------------------------------------------
/examples/changeTraversalConfig.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 | import { createMigrator } from 'firewalk';
3 |
4 | const walletsWithNegativeBalance = firestore().collection('wallets').where('money', '<', 0);
5 | const migrator = createMigrator(walletsWithNegativeBalance, {
6 | // We want each batch to have 500 docs. The size of the very last batch may be less than 500
7 | batchSize: 500,
8 | // We want to wait 500ms before moving to the next batch
9 | sleepTimeBetweenBatches: 500,
10 | });
11 | // Wipe out their debts!
12 | const { migratedDocCount } = await migrator.set({ money: 0 });
13 | console.log(`Updated ${migratedDocCount} wallets!`);
14 |
--------------------------------------------------------------------------------
/examples/exitEarlyPredicate.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 | import { createTraverser } from 'firewalk';
3 |
4 | const projectsColRef = firestore().collection('projects');
5 | const traverser = createTraverser(projectsColRef);
6 | const newTraverser = traverser
7 | .withExitEarlyPredicate((batchDocs) => batchDocs.some((d) => d.get('name') === undefined))
8 | .withExitEarlyPredicate((_, batchIndex) => batchIndex === 100);
9 |
--------------------------------------------------------------------------------
/examples/fastMigrator.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 | import { createMigrator } from 'firewalk';
3 |
4 | const projectsColRef = firestore().collection('projects');
5 | const migrator = createMigrator(projectsColRef, { maxConcurrentBatchCount: 25 });
6 | const { migratedDocCount } = await migrator.update('isCompleted', false);
7 | console.log(`Updated ${migratedDocCount} projects super-fast!`);
8 |
--------------------------------------------------------------------------------
/examples/fastTraverser.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 | import { createTraverser } from 'firewalk';
3 |
4 | const projectsColRef = firestore().collection('projects');
5 | const traverser = createTraverser(projectsColRef, {
6 | batchSize: 500,
7 | // This means we are prepared to hold 500 * 20 = 10,000 docs in memory.
8 | // We sacrifice some memory to traverse faster.
9 | maxConcurrentBatchCount: 20,
10 | });
11 | const { docCount } = await traverser.traverse(async (_, batchIndex) => {
12 | console.log(`Gonna process batch ${batchIndex} now!`);
13 | // ...
14 | });
15 | console.log(`Traversed ${docCount} projects super-fast!`);
16 |
--------------------------------------------------------------------------------
/examples/migrationPredicate.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 | import { createMigrator } from 'firewalk';
3 |
4 | const projectsColRef = firestore().collectionGroup('projects');
5 | const migrator = createMigrator(projectsColRef);
6 | const newMigrator = migrator
7 | .withPredicate((doc) => doc.get('name') !== undefined)
8 | .withPredicate((doc) => doc.ref.path.startsWith('users/'));
9 |
--------------------------------------------------------------------------------
/examples/quickStart.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 | import { createTraverser } from 'firewalk';
3 |
4 | const usersCollection = firestore().collection('users');
5 | const traverser = createTraverser(usersCollection);
6 |
7 | const { batchCount, docCount } = await traverser.traverse(async (batchDocs, batchIndex) => {
8 | const batchSize = batchDocs.length;
9 | await Promise.all(
10 | batchDocs.map(async (doc) => {
11 | const { email, firstName } = doc.data();
12 | await sendEmail({ to: email, content: `Hello ${firstName}!` });
13 | })
14 | );
15 | console.log(`Batch ${batchIndex} done! We emailed ${batchSize} users in this batch.`);
16 | });
17 |
18 | console.log(`Traversal done! We emailed ${docCount} users in ${batchCount} batches!`);
19 |
--------------------------------------------------------------------------------
/examples/renameField.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 | import { createMigrator } from 'firewalk';
3 |
4 | const postsColGroup = firestore().collectionGroup('posts');
5 | const migrator = createMigrator(postsColGroup);
6 | const { migratedDocCount } = await migrator.renameField('postedAt', 'publishedAt');
7 | console.log(`Updated ${migratedDocCount} posts!`);
8 |
--------------------------------------------------------------------------------
/jest.config.global.ts:
--------------------------------------------------------------------------------
1 | export default function globalJestSetup(): void {
2 | process.env.FIRESTORE_EMULATOR_HOST = `127.0.0.1:8080`;
3 | }
4 |
--------------------------------------------------------------------------------
/jest.config.ts:
--------------------------------------------------------------------------------
1 | /*
2 | * For a detailed explanation regarding each configuration property and type check, visit:
3 | * https://jestjs.io/docs/configuration
4 | */
5 |
6 | export default {
7 | // All imported modules in your tests should be mocked automatically
8 | // automock: false,
9 |
10 | // Stop running tests after `n` failures
11 | // bail: 0,
12 |
13 | // The directory where Jest should store its cached dependency information
14 | // cacheDirectory: "/private/var/folders/6d/p1n8g20s74q1b14xk0_54gt80000gn/T/jest_dx",
15 |
16 | // Automatically clear mock calls and instances between every test
17 | clearMocks: true,
18 |
19 | // Indicates whether the coverage information should be collected while executing the test
20 | collectCoverage: true,
21 |
22 | // An array of glob patterns indicating a set of files for which coverage information should be collected
23 | // collectCoverageFrom: undefined,
24 |
25 | // The directory where Jest should output its coverage files
26 | coverageDirectory: 'coverage',
27 |
28 | // An array of regexp pattern strings used to skip coverage collection
29 | // coveragePathIgnorePatterns: [
30 | // "/node_modules/"
31 | // ],
32 |
33 | // Indicates which provider should be used to instrument code for coverage
34 | // coverageProvider: "babel",
35 |
36 | // A list of reporter names that Jest uses when writing coverage reports
37 | // coverageReporters: [
38 | // "json",
39 | // "text",
40 | // "lcov",
41 | // "clover"
42 | // ],
43 |
44 | // An object that configures minimum threshold enforcement for coverage results
45 | // coverageThreshold: undefined,
46 |
47 | // A path to a custom dependency extractor
48 | // dependencyExtractor: undefined,
49 |
50 | // Make calling deprecated APIs throw helpful error messages
51 | // errorOnDeprecated: false,
52 |
53 | // Force coverage collection from ignored files using an array of glob patterns
54 | // forceCoverageMatch: [],
55 |
56 | // A path to a module which exports an async function that is triggered once before all test suites
57 | globalSetup: './jest.config.global.ts',
58 |
59 | // A path to a module which exports an async function that is triggered once after all test suites
60 | // globalTeardown: undefined,
61 |
62 | // A set of global variables that need to be available in all test environments
63 | // globals: {},
64 |
65 | // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
66 | // maxWorkers: "50%",
67 |
68 | // An array of directory names to be searched recursively up from the requiring module's location
69 | // moduleDirectories: [
70 | // "node_modules"
71 | // ],
72 |
73 | // An array of file extensions your modules use
74 | // moduleFileExtensions: [
75 | // "js",
76 | // "jsx",
77 | // "ts",
78 | // "tsx",
79 | // "json",
80 | // "node"
81 | // ],
82 |
83 | // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
84 | // moduleNameMapper: {},
85 |
86 | // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
87 | // modulePathIgnorePatterns: [],
88 |
89 | // Activates notifications for test results
90 | // notify: false,
91 |
92 | // An enum that specifies notification mode. Requires { notify: true }
93 | // notifyMode: "failure-change",
94 |
95 | // A preset that is used as a base for Jest's configuration
96 | // preset: undefined,
97 |
98 | // Run tests from one or more projects
99 | // projects: undefined,
100 |
101 | // Use this configuration option to add custom reporters to Jest
102 | // reporters: undefined,
103 |
104 | // Automatically reset mock state between every test
105 | // resetMocks: false,
106 |
107 | // Reset the module registry before running each individual test
108 | // resetModules: false,
109 |
110 | // A path to a custom resolver
111 | // resolver: undefined,
112 |
113 | // Automatically restore mock state between every test
114 | // restoreMocks: false,
115 |
116 | // The root directory that Jest should scan for tests and modules within
117 | // rootDir: undefined,
118 |
119 | // A list of paths to directories that Jest should use to search for files in
120 | // roots: [
121 | // ""
122 | // ],
123 |
124 | // Allows you to use a custom runner instead of Jest's default test runner
125 | // runner: "jest-runner",
126 |
127 | // The paths to modules that run some code to configure or set up the testing environment before each test
128 | // setupFiles: [],
129 |
130 | // A list of paths to modules that run some code to configure or set up the testing framework before each test
131 | // setupFilesAfterEnv: [],
132 |
133 | // The number of seconds after which a test is considered as slow and reported as such in the results.
134 | // slowTestThreshold: 5,
135 |
136 | // A list of paths to snapshot serializer modules Jest should use for snapshot testing
137 | // snapshotSerializers: [],
138 |
139 | // The test environment that will be used for testing
140 | // testEnvironment: "jest-environment-node",
141 |
142 | // Options that will be passed to the testEnvironment
143 | // testEnvironmentOptions: {},
144 |
145 | // Adds a location field to test results
146 | // testLocationInResults: false,
147 |
148 | // The glob patterns Jest uses to detect test files
149 | testMatch: ['**/?(*.)+(spec|test).[tj]s?(x)'],
150 |
151 | // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
152 | testPathIgnorePatterns: ['/node_modules/'],
153 |
154 | // The regexp pattern or array of patterns that Jest uses to detect test files
155 | // testRegex: [],
156 |
157 | // This option allows the use of a custom results processor
158 | // testResultsProcessor: undefined,
159 |
160 | // This option allows use of a custom test runner
161 | // testRunner: "jest-circus/runner",
162 |
163 | // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href
164 | // testURL: "http://localhost",
165 |
166 | // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout"
167 | // timers: "real",
168 |
169 | // A map from regular expressions to paths to transformers
170 | // transform: undefined,
171 |
172 | // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
173 | // transformIgnorePatterns: [
174 | // "/node_modules/",
175 | // "\\.pnp\\.[^\\/]+$"
176 | // ],
177 |
178 | // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
179 | // unmockedModulePathPatterns: undefined,
180 |
181 | // Indicates whether each individual test should be reported during the run
182 | // verbose: undefined,
183 |
184 | // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
185 | // watchPathIgnorePatterns: [],
186 |
187 | // Whether to use watchman for file crawling
188 | // watchman: true,
189 | };
190 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "firewalk",
3 | "version": "2.3.0",
4 | "description": "A collection traversal library for Firestore",
5 | "keywords": [
6 | "Firewalk",
7 | "firewalk",
8 | "Firebase",
9 | "firebase",
10 | "Firestore",
11 | "firestore",
12 | "Cloud Firestore",
13 | "cloud firestore",
14 | "TypeScript",
15 | "typescript"
16 | ],
17 | "author": "Anar Kafkas ",
18 | "homepage": "https://kafkas.github.io/firewalk",
19 | "license": "MIT",
20 | "main": "dist/index.js",
21 | "types": "dist/index.d.ts",
22 | "directories": {
23 | "dist": "dist",
24 | "test": "__tests__"
25 | },
26 | "files": [
27 | "dist"
28 | ],
29 | "repository": {
30 | "type": "git",
31 | "url": "git+https://github.com/kafkas/firewalk.git"
32 | },
33 | "devDependencies": {
34 | "@babel/core": "^7.17.0",
35 | "@babel/preset-env": "^7.16.11",
36 | "@babel/preset-typescript": "^7.16.7",
37 | "@types/compare-versions": "^3.3.0",
38 | "@types/jest": "^29.5.10",
39 | "@types/lodash": "^4.14.178",
40 | "@types/node": "^22.10.2",
41 | "@typescript-eslint/eslint-plugin": "^5.10.2",
42 | "@typescript-eslint/parser": "^5.10.2",
43 | "babel-jest": "^27.5.1",
44 | "compare-versions": "^4.1.3",
45 | "eslint": "^8.8.0",
46 | "eslint-config-prettier": "^8.3.0",
47 | "eslint-plugin-import": "^2.25.4",
48 | "eslint-plugin-prettier": "^4.0.0",
49 | "firebase-admin": "^13.0.1",
50 | "firebase-tools": "^12.9.1",
51 | "jest": "^29.7.0",
52 | "lodash": "^4.17.21",
53 | "prettier": "2.5.1",
54 | "ts-node": "^10.4.0",
55 | "tsd": "0.31.2",
56 | "typedoc": "0.27.5",
57 | "typescript": "5.7.2"
58 | },
59 | "scripts": {
60 | "build": "rm -rf dist && tsc -p tsconfig.build.json",
61 | "docs-generate": "sh scripts/generate-docs.sh",
62 | "format": "prettier --write \"{src,test}/**/*.{ts,tsx,json}\"",
63 | "format-check": "prettier --check \"{src,test}/**/*.{ts,tsx,json}\"",
64 | "lint": "eslint \"{src,__tests__}/**/*/*.{js,ts}\"",
65 | "test": "npm run test:compile && npm run test:types && npm run test:jest",
66 | "test:compile": "tsc -p tsconfig.test.json",
67 | "test:jest": "firebase emulators:exec --project demo-firewalk --only firestore 'jest'",
68 | "test:types": "tsd"
69 | },
70 | "bugs": {
71 | "url": "https://github.com/kafkas/firewalk/issues"
72 | },
73 | "peerDependencies": {
74 | "firebase-admin": "11 - 13"
75 | },
76 | "publishConfig": {
77 | "access": "public"
78 | },
79 | "tsd": {
80 | "directory": "__tests__/types"
81 | },
82 | "dependencies": {
83 | "@proficient/ds": "0.3.2",
84 | "@proficient/util": "0.2.2"
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/scripts/generate-docs.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | # Run from project root
3 |
4 | # Parse version
5 | complete_version=$(npm pkg get version | tr -d '"')
6 |
7 | # Extract the major version
8 | major_version=$(echo $complete_version | awk -F '.' '{print $1}')
9 |
10 | typedoc --out docs/v$major_version
11 |
--------------------------------------------------------------------------------
/src/api/createBatchMigrator.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import { isTraverser } from '../internal/utils';
3 | import { BasicBatchMigratorImpl } from '../internal/implementations';
4 | import type { InvalidConfigError } from '../errors'; /* eslint-disable-line */
5 | import type { BatchMigrator, Traversable, TraversalConfig, Traverser } from './interfaces';
6 | import { createTraverser } from './createTraverser';
7 |
8 | /**
9 | * Creates a migrator that facilitates database migrations. Accepts a custom traverser object as argument which the
10 | * migrator will use when traversing the collection and writing to documents. This migrator uses atomic batch writes
11 | * when writing to docs so the entire operation will fail if a single write isn't successful.
12 | *
13 | * @remarks
14 | *
15 | * Note that the {@link TraversalConfig.batchSize} config value must not exceed 500 for a traverser used in a {@link BatchMigrator}.
16 | * This is because in Firestore, each write batch can write to a maximum of 500 documents.
17 | *
18 | * @param traverser - The traverser object that this migrator will use when traversing the collection and writing to documents.
19 | * @returns A new {@link BatchMigrator} object.
20 | * @throws {@link InvalidConfigError} Thrown if the traversal config of the specified traverser is not compatible with this migrator.
21 | */
22 | export function createBatchMigrator<
23 | AppModelType = firestore.DocumentData,
24 | DbModelType extends firestore.DocumentData = firestore.DocumentData
25 | >(traverser: Traverser): BatchMigrator;
26 |
27 | /**
28 | * Creates a migrator that facilitates database migrations. The migrator creates a default traverser that
29 | * it uses when traversing the collection and writing to documents. This migrator uses atomic batch writes when writing
30 | * to docs so the entire operation will fail if a single write isn't successful.
31 | *
32 | * @remarks
33 | *
34 | * Note that the {@link TraversalConfig.batchSize} config value must not exceed 500 for a traverser used in a {@link BatchMigrator}.
35 | * This is because in Firestore, each write batch can write to a maximum of 500 documents.
36 | *
37 | * @param traversable - A collection-like traversable group of documents.
38 | * @param traversalConfig - Optional. The traversal configuration with which the default traverser will be created.
39 | * @returns A new {@link BatchMigrator} object.
40 | * @throws {@link InvalidConfigError} Thrown if the specified `traversalConfig` is invalid or incompatible with this migrator.
41 | */
42 | export function createBatchMigrator<
43 | AppModelType = firestore.DocumentData,
44 | DbModelType extends firestore.DocumentData = firestore.DocumentData
45 | >(
46 | traversable: Traversable,
47 | traversalConfig?: Partial
48 | ): BatchMigrator;
49 |
50 | export function createBatchMigrator<
51 | AppModelType = firestore.DocumentData,
52 | DbModelType extends firestore.DocumentData = firestore.DocumentData
53 | >(
54 | traversableOrTraverser:
55 | | Traverser
56 | | Traversable,
57 | traversalConfig?: Partial
58 | ): BatchMigrator {
59 | const traverser = isTraverser(traversableOrTraverser)
60 | ? (traversableOrTraverser as Traverser)
61 | : createTraverser(
62 | traversableOrTraverser as Traversable,
63 | traversalConfig
64 | );
65 | return new BasicBatchMigratorImpl(traverser);
66 | }
67 |
--------------------------------------------------------------------------------
/src/api/createMigrator.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import { isTraverser } from '../internal/utils';
3 | import { BasicDefaultMigratorImpl } from '../internal/implementations';
4 | import type { InvalidConfigError } from '../errors'; /* eslint-disable-line */
5 | import type { DefaultMigrator, Traversable, TraversalConfig, Traverser } from './interfaces';
6 | import { createTraverser } from './createTraverser';
7 |
8 | /**
9 | * Creates a migrator that facilitates database migrations. Accepts a custom traverser object as argument which the
10 | * migrator will use when traversing the collection and writing to documents. This migrator does not use atomic batch
11 | * writes so it is possible that when a write fails other writes go through.
12 | *
13 | * @param traverser - The traverser object that this migrator will use when traversing the collection and writing to documents.
14 | * @returns A new {@link DefaultMigrator} object.
15 | */
16 | export function createMigrator<
17 | AppModelType = firestore.DocumentData,
18 | DbModelType extends firestore.DocumentData = firestore.DocumentData
19 | >(traverser: Traverser): DefaultMigrator;
20 |
21 | /**
22 | * Creates a migrator that facilitates database migrations. The migrator creates a default traverser that
23 | * it uses when traversing the collection and writing to documents. This migrator does not use atomic batch writes
24 | * so it is possible that when a write fails other writes go through.
25 | *
26 | * @param traversable - A collection-like traversable group of documents to migrate.
27 | * @param traversalConfig - Optional. The traversal configuration with which the default traverser will be created.
28 | * @returns A new {@link DefaultMigrator} object.
29 | * @throws {@link InvalidConfigError} Thrown if the specified `traversalConfig` is invalid.
30 | */
31 | export function createMigrator<
32 | AppModelType = firestore.DocumentData,
33 | DbModelType extends firestore.DocumentData = firestore.DocumentData
34 | >(
35 | traversable: Traversable,
36 | traversalConfig?: Partial
37 | ): DefaultMigrator;
38 |
39 | export function createMigrator<
40 | AppModelType = firestore.DocumentData,
41 | DbModelType extends firestore.DocumentData = firestore.DocumentData
42 | >(
43 | traversableOrTraverser:
44 | | Traverser
45 | | Traversable,
46 | traversalConfig?: Partial
47 | ): DefaultMigrator {
48 | const traverser = isTraverser(traversableOrTraverser)
49 | ? (traversableOrTraverser as Traverser)
50 | : createTraverser(
51 | traversableOrTraverser as Traversable,
52 | traversalConfig
53 | );
54 | return new BasicDefaultMigratorImpl(traverser);
55 | }
56 |
--------------------------------------------------------------------------------
/src/api/createTraverser.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import { PromiseQueueBasedTraverserImpl } from '../internal/implementations';
3 | import type { InvalidConfigError } from '../errors'; /* eslint-disable-line */
4 | import type { Traversable, TraversalConfig, Traverser } from './interfaces';
5 |
6 | /**
7 | * Creates an object which can be used to traverse a Firestore collection or, more generally,
8 | * a {@link Traversable}.
9 | *
10 | * @remarks
11 | *
12 | * For each batch of document snapshots in the traversable, this traverser invokes a specified
13 | * async callback and immediately moves to the next batch. It does not wait for the callback
14 | * Promise to resolve before moving to the next batch. That is, when `maxConcurrentBatchCount` > 1,
15 | * there is no guarantee that any given batch will finish processing before a later batch.
16 | *
17 | * The traverser becomes faster as you increase `maxConcurrentBatchCount`, but this will consume
18 | * more memory. You should increase concurrency when you want to trade some memory for speed.
19 | *
20 | * @param traversable - A collection-like group of documents. Can be one of [CollectionReference](https://googleapis.dev/nodejs/firestore/latest/CollectionReference.html), [CollectionGroup](https://googleapis.dev/nodejs/firestore/latest/CollectionGroup.html) and [Query](https://googleapis.dev/nodejs/firestore/latest/Query.html).
21 | * @param config - Optional. The traversal configuration with which the traverser will be created.
22 | * @returns A new {@link Traverser} object.
23 | * @throws {@link InvalidConfigError} Thrown if the specified `config` is invalid.
24 | */
25 | export function createTraverser<
26 | AppModelType = firestore.DocumentData,
27 | DbModelType extends firestore.DocumentData = firestore.DocumentData
28 | >(
29 | traversable: Traversable,
30 | config?: Partial
31 | ): Traverser {
32 | return new PromiseQueueBasedTraverserImpl(traversable, [], config);
33 | }
34 |
--------------------------------------------------------------------------------
/src/api/index.ts:
--------------------------------------------------------------------------------
1 | export { createBatchMigrator } from './createBatchMigrator';
2 | export { createMigrator } from './createMigrator';
3 | export { createTraverser } from './createTraverser';
4 |
5 | export * from './interfaces';
6 |
--------------------------------------------------------------------------------
/src/api/interfaces/BatchCallback.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 |
3 | /**
4 | * A function that takes batch doc snapshots and the 0-based batch index as its arguments.
5 | */
6 | export type BatchCallback<
7 | AppModelType = firestore.DocumentData,
8 | DbModelType extends firestore.DocumentData = firestore.DocumentData
9 | > = (
10 | batchDocs: firestore.QueryDocumentSnapshot[],
11 | batchIndex: number
12 | ) => void | Promise;
13 |
--------------------------------------------------------------------------------
/src/api/interfaces/BatchMigrator.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import type { MigrationPredicate, Migrator, Traverser } from '.';
3 |
4 | /**
5 | * A batch migrator object that uses atomic batch writes.
6 | */
7 | export interface BatchMigrator<
8 | AppModelType = firestore.DocumentData,
9 | DbModelType extends firestore.DocumentData = firestore.DocumentData
10 | > extends Migrator {
11 | /**
12 | * Applies a migration predicate that indicates whether to migrate the current document or not. By default, all
13 | * documents are migrated.
14 | *
15 | * @remarks
16 | *
17 | * If you have already applied other migration predicates to this migrator, this and all the other predicates will be
18 | * evaluated and the resulting booleans will be AND'd to get the boolean that indicates whether to migrate the document
19 | * or not. This is consistent with the intuitive default behavior that all documents are migrated.
20 | *
21 | * @example
22 | *
23 | * ```ts
24 | * const newMigrator = migrator
25 | * .withPredicate((doc) => doc.get('name') !== undefined)
26 | * .withPredicate((doc) => doc.ref.path.startsWith('users/'));
27 | * ```
28 | *
29 | * In the above case `newMigrator` will migrate only the documents whose `name` field is not missing AND whose path
30 | * starts with `"users/"`.
31 | *
32 | * @param predicate - A function that takes a document snapshot and returns a boolean indicating whether to migrate it.
33 | * @returns A new {@link BatchMigrator} object.
34 | */
35 | withPredicate(
36 | predicate: MigrationPredicate
37 | ): BatchMigrator;
38 |
39 | /**
40 | * Applies a new traverser that will be used by the migrator.
41 | *
42 | * @param traverser - The new traverser that the migrator will use.
43 | * @returns A new {@link BatchMigrator} object.
44 | */
45 | withTraverser(
46 | traverser: Traverser
47 | ): BatchMigrator;
48 | }
49 |
--------------------------------------------------------------------------------
/src/api/interfaces/DefaultMigrator.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import type { MigrationPredicate, Migrator, Traverser } from '.';
3 |
4 | /**
5 | * A batch migrator object that does not use atomic batch writes.
6 | */
7 | export interface DefaultMigrator<
8 | AppModelType = firestore.DocumentData,
9 | DbModelType extends firestore.DocumentData = firestore.DocumentData
10 | > extends Migrator {
11 | /**
12 | * Applies a migration predicate that indicates whether to migrate the current document or not. By default, all
13 | * documents are migrated.
14 | *
15 | * @remarks
16 | *
17 | * If you have already applied other migration predicates to this migrator, this and all the other predicates will be
18 | * evaluated and the resulting booleans will be AND'd to get the boolean that indicates whether to migrate the document
19 | * or not. This is consistent with the intuitive default behavior that all documents are migrated.
20 | *
21 | * @example
22 | *
23 | * ```ts
24 | * const newMigrator = migrator
25 | * .withPredicate((doc) => doc.get('name') !== undefined)
26 | * .withPredicate((doc) => doc.ref.path.startsWith('users/'));
27 | * ```
28 | *
29 | * In the above case `newMigrator` will migrate only the documents whose `name` field is not missing AND whose path
30 | * starts with `"users/"`.
31 | *
32 | * @param predicate - A function that takes a document snapshot and returns a boolean indicating whether to migrate it.
33 | * @returns A new {@link DefaultMigrator} object.
34 | */
35 | withPredicate(
36 | predicate: MigrationPredicate
37 | ): DefaultMigrator;
38 |
39 | /**
40 | * Applies a new traverser that will be used by the migrator.
41 | *
42 | * @param traverser - The new traverser that the migrator will use.
43 | * @returns A new {@link DefaultMigrator} object.
44 | */
45 | withTraverser(
46 | traverser: Traverser
47 | ): DefaultMigrator;
48 | }
49 |
--------------------------------------------------------------------------------
/src/api/interfaces/ExitEarlyPredicate.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import type { BatchCallback } from '.';
3 |
4 | /**
5 | * A function that takes batch doc snapshots and the 0-based batch index and returns a boolean
6 | * indicating whether to exit traversal early.
7 | */
8 | export type ExitEarlyPredicate<
9 | AppModelType = firestore.DocumentData,
10 | DbModelType extends firestore.DocumentData = firestore.DocumentData
11 | > = (...args: Parameters>) => boolean;
12 |
--------------------------------------------------------------------------------
/src/api/interfaces/MigrationPredicate.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 |
3 | /**
4 | * A function that takes a document snapshot and returns a boolean indicating whether to migrate it.
5 | */
6 | export type MigrationPredicate<
7 | AppModelType = firestore.DocumentData,
8 | DbModelType extends firestore.DocumentData = firestore.DocumentData
9 | > = (doc: firestore.QueryDocumentSnapshot) => boolean;
10 |
--------------------------------------------------------------------------------
/src/api/interfaces/MigrationResult.ts:
--------------------------------------------------------------------------------
1 | import type { TraversalResult } from '.';
2 |
3 | /**
4 | * Represents an object that contains the details of a migration.
5 | */
6 | export interface MigrationResult {
7 | /**
8 | * The traversal result of the underlying traverser.
9 | */
10 | readonly traversalResult: TraversalResult;
11 |
12 | /**
13 | * The number of documents that have been migrated.
14 | */
15 | readonly migratedDocCount: number;
16 | }
17 |
--------------------------------------------------------------------------------
/src/api/interfaces/Migrator.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import type {
3 | BatchCallback,
4 | MigrationPredicate,
5 | MigrationResult,
6 | SetDataGetter,
7 | SetOptions,
8 | SetPartialDataGetter,
9 | Traverser,
10 | UpdateDataGetter,
11 | UpdateFieldValueGetter,
12 | } from '.';
13 |
14 | /**
15 | * Represents the general interface of a migrator.
16 | */
17 | export interface Migrator<
18 | AppModelType = firestore.DocumentData,
19 | DbModelType extends firestore.DocumentData = firestore.DocumentData
20 | > {
21 | /**
22 | * The underlying traverser.
23 | */
24 | readonly traverser: Traverser;
25 |
26 | /**
27 | * Applies a migration predicate that indicates whether to migrate the current document or not. By default, all
28 | * documents are migrated.
29 | *
30 | * @remarks
31 | *
32 | * If you have already applied other migration predicates to this migrator, this and all the other predicates will be
33 | * evaluated and the resulting booleans will be AND'd to get the boolean that indicates whether to migrate the document
34 | * or not. This is consistent with the intuitive default behavior that all documents are migrated.
35 | *
36 | * @example
37 | *
38 | * ```ts
39 | * const newMigrator = migrator
40 | * .withPredicate((doc) => doc.get('name') !== undefined)
41 | * .withPredicate((doc) => doc.ref.path.startsWith('users/'));
42 | * ```
43 | *
44 | * In the above case `newMigrator` will migrate only the documents whose `name` field is not missing AND whose path
45 | * starts with `"users/"`.
46 | *
47 | * @param predicate - A function that takes a document snapshot and returns a boolean indicating whether to migrate it.
48 | * @returns A new {@link Migrator} object.
49 | */
50 | withPredicate(
51 | predicate: MigrationPredicate
52 | ): Migrator;
53 |
54 | /**
55 | * Applies a new traverser that will be used by the migrator.
56 | *
57 | * @param traverser - The new traverser that the migrator will use.
58 | * @returns A new {@link Migrator} object.
59 | */
60 | withTraverser(
61 | traverser: Traverser
62 | ): Migrator;
63 |
64 | /**
65 | * Registers a callback function that fires right before a batch starts processing. You can register at most 1
66 | * callback. If you call this function multiple times, only the last callback will be registered.
67 | *
68 | * @param callback - A synchronous callback that takes batch doc snapshots and the 0-based batch index as its arguments.
69 | */
70 | onBeforeBatchStart(callback: BatchCallback): void;
71 |
72 | /**
73 | * Registers a callback function that fires after a batch is processed. You can register at most 1 callback. If you call
74 | * this function multiple times, only the last callback will be registered.
75 | *
76 | * @param callback - A synchronous callback that takes batch doc snapshots and the 0-based batch index as its arguments.
77 | */
78 | onAfterBatchComplete(callback: BatchCallback): void;
79 |
80 | /**
81 | * Deletes the specified field from all documents in this collection.
82 | *
83 | * @remarks
84 | *
85 | * **Complexity:**
86 | *
87 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
88 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
89 | * - Billing: _max_(1, _N_) reads, _K_ writes
90 | *
91 | * where:
92 | *
93 | * - _N_: number of docs in the traversable
94 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
95 | * - _W_(`batchSize`): average batch write time
96 | * - _TC_(`traverser`): time complexity of the underlying traverser
97 | * - _SC_(`traverser`): space complexity of the underlying traverser
98 | *
99 | * @param field - The field to delete.
100 | * @returns A Promise resolving to an object representing the details of the migration.
101 | */
102 | deleteField(field: string | firestore.FieldPath): Promise;
103 |
104 | /**
105 | * Deletes the specified fields from all documents in this collection.
106 | *
107 | * @remarks
108 | *
109 | * **Complexity:**
110 | *
111 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
112 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
113 | * - Billing: _max_(1, _N_) reads, _K_ writes
114 | *
115 | * where:
116 | *
117 | * - _N_: number of docs in the traversable
118 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
119 | * - _W_(`batchSize`): average batch write time
120 | * - _TC_(`traverser`): time complexity of the underlying traverser
121 | * - _SC_(`traverser`): space complexity of the underlying traverser
122 | *
123 | * @example
124 | *
125 | * ```ts
126 | * const field1 = new firestore.FieldPath('name', 'last');
127 | * const field2 = 'lastModifiedAt';
128 | * await migrator.deleteFields(field1, field2);
129 | * ```
130 | *
131 | * @param fields - A list of fields to delete.
132 | * @returns A Promise resolving to an object representing the details of the migration.
133 | */
134 | deleteFields(...fields: (string | firestore.FieldPath)[]): Promise;
135 |
136 | /**
137 | * Renames the specified field in all documents in this collection. Ignores the fields that are missing.
138 | *
139 | * @remarks
140 | *
141 | * **Complexity:**
142 | *
143 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
144 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
145 | * - Billing: _max_(1, _N_) reads, _K_ writes
146 | *
147 | * where:
148 | *
149 | * - _N_: number of docs in the traversable
150 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
151 | * - _W_(`batchSize`): average batch write time
152 | * - _TC_(`traverser`): time complexity of the underlying traverser
153 | * - _SC_(`traverser`): space complexity of the underlying traverser
154 | *
155 | * @param oldField - The old field.
156 | * @param newField - The new field.
157 | * @returns A Promise resolving to an object representing the details of the migration.
158 | */
159 | renameField(
160 | oldField: string | firestore.FieldPath,
161 | newField: string | firestore.FieldPath
162 | ): Promise;
163 |
164 | /**
165 | * Renames the specified fields in all documents in this collection. Ignores the fields that are missing.
166 | *
167 | * @remarks
168 | *
169 | * **Complexity:**
170 | *
171 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
172 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
173 | * - Billing: _max_(1, _N_) reads, _K_ writes
174 | *
175 | * where:
176 | *
177 | * - _N_: number of docs in the traversable
178 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
179 | * - _W_(`batchSize`): average batch write time
180 | * - _TC_(`traverser`): time complexity of the underlying traverser
181 | * - _SC_(`traverser`): space complexity of the underlying traverser
182 | *
183 | * @example
184 | *
185 | * ```ts
186 | * const change1 = [new firestore.FieldPath('name', 'last'), 'lastName'];
187 | * const change2 = ['lastModifiedAt', 'lastUpdatedAt'];
188 | * await migrator.renameFields(change1, change2);
189 | * ```
190 | *
191 | * @param changes - A list of `[oldField, newField]` tuples. Each tuple is an array of 2 elements:
192 | * the old field to rename and the new field.
193 | * @returns A Promise resolving to an object representing the details of the migration.
194 | */
195 | renameFields(
196 | ...changes: [oldField: string | firestore.FieldPath, newField: string | firestore.FieldPath][]
197 | ): Promise;
198 |
199 | /**
200 | * Sets all documents in this collection with the provided data.
201 | *
202 | * @remarks
203 | *
204 | * **Complexity:**
205 | *
206 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
207 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
208 | * - Billing: _max_(1, _N_) reads, _K_ writes
209 | *
210 | * where:
211 | *
212 | * - _N_: number of docs in the traversable
213 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
214 | * - _W_(`batchSize`): average batch write time
215 | * - _TC_(`traverser`): time complexity of the underlying traverser
216 | * - _SC_(`traverser`): space complexity of the underlying traverser
217 | *
218 | * @param data - A data object with which to set each document.
219 | * @param options - An object to configure the set behavior.
220 | * @returns A Promise resolving to an object representing the details of the migration.
221 | */
222 | set(
223 | data: firestore.PartialWithFieldValue,
224 | options: SetOptions
225 | ): Promise;
226 |
227 | /**
228 | * Sets all documents in this collection with the provided data.
229 | *
230 | * @remarks
231 | *
232 | * **Complexity:**
233 | *
234 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
235 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
236 | * - Billing: _max_(1, _N_) reads, _K_ writes
237 | *
238 | * where:
239 | *
240 | * - _N_: number of docs in the traversable
241 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
242 | * - _W_(`batchSize`): average batch write time
243 | * - _TC_(`traverser`): time complexity of the underlying traverser
244 | * - _SC_(`traverser`): space complexity of the underlying traverser
245 | *
246 | * @param data - A data object with which to set each document.
247 | * @returns A Promise resolving to an object representing the details of the migration.
248 | */
249 | set(data: firestore.WithFieldValue): Promise;
250 |
251 | /**
252 | * Sets all documents in this collection with the provided data.
253 | *
254 | * @remarks
255 | *
256 | * **Complexity:**
257 | *
258 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
259 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
260 | * - Billing: _max_(1, _N_) reads, _K_ writes
261 | *
262 | * where:
263 | *
264 | * - _N_: number of docs in the traversable
265 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
266 | * - _W_(`batchSize`): average batch write time
267 | * - _TC_(`traverser`): time complexity of the underlying traverser
268 | * - _SC_(`traverser`): space complexity of the underlying traverser
269 | *
270 | * @param getData - A function that takes a document snapshot and returns a data object with
271 | * which to set each document.
272 | * @param options - An object to configure the set behavior.
273 | * @returns A Promise resolving to an object representing the details of the migration.
274 | */
275 | setWithDerivedData(
276 | getData: SetPartialDataGetter,
277 | options: SetOptions
278 | ): Promise;
279 |
280 | /**
281 | * Sets all documents in this collection with the provided data.
282 | *
283 | * @remarks
284 | *
285 | * **Complexity:**
286 | *
287 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
288 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
289 | * - Billing: _max_(1, _N_) reads, _K_ writes
290 | *
291 | * where:
292 | *
293 | * - _N_: number of docs in the traversable
294 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
295 | * - _W_(`batchSize`): average batch write time
296 | * - _TC_(`traverser`): time complexity of the underlying traverser
297 | * - _SC_(`traverser`): space complexity of the underlying traverser
298 | *
299 | * @param getData - A function that takes a document snapshot and returns a data object with
300 | * which to set each document.
301 | * @returns A Promise resolving to an object representing the details of the migration.
302 | */
303 | setWithDerivedData(getData: SetDataGetter): Promise;
304 |
305 | /**
306 | * Updates all documents in this collection with the provided data.
307 | *
308 | * @remarks
309 | *
310 | * **Complexity:**
311 | *
312 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
313 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
314 | * - Billing: _max_(1, _N_) reads, _K_ writes
315 | *
316 | * where:
317 | *
318 | * - _N_: number of docs in the traversable
319 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
320 | * - _W_(`batchSize`): average batch write time
321 | * - _TC_(`traverser`): time complexity of the underlying traverser
322 | * - _SC_(`traverser`): space complexity of the underlying traverser
323 | *
324 | * @param data - A non-empty data object with which to update each document.
325 | * @param precondition - A Precondition to enforce on this update.
326 | * @returns A Promise resolving to an object representing the details of the migration.
327 | */
328 | update(
329 | data: firestore.UpdateData,
330 | precondition?: firestore.Precondition
331 | ): Promise;
332 |
333 | /**
334 | * Updates all documents in this collection with the provided field-value pair.
335 | *
336 | * @remarks
337 | *
338 | * **Complexity:**
339 | *
340 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
341 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
342 | * - Billing: _max_(1, _N_) reads, _K_ writes
343 | *
344 | * where:
345 | *
346 | * - _N_: number of docs in the traversable
347 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
348 | * - _W_(`batchSize`): average batch write time
349 | * - _TC_(`traverser`): time complexity of the underlying traverser
350 | * - _SC_(`traverser`): space complexity of the underlying traverser
351 | *
352 | * @param field - The first field to update in each document.
353 | * @param value - The first value corresponding to the first field. Must not be `undefined`.
354 | * @param moreFieldsOrPrecondition - An alternating list of field paths and values to update,
355 | * optionally followed by a Precondition to enforce on this update.
356 | *
357 | * @returns A Promise resolving to an object representing the details of the migration.
358 | */
359 | update(
360 | field: string | firestore.FieldPath,
361 | value: any,
362 | ...moreFieldsOrPrecondition: any[]
363 | ): Promise;
364 |
365 | /**
366 | * Updates all documents in this collection with the provided data.
367 | *
368 | * @remarks
369 | *
370 | * **Complexity:**
371 | *
372 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
373 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
374 | * - Billing: _max_(1, _N_) reads, _K_ writes
375 | *
376 | * where:
377 | *
378 | * - _N_: number of docs in the traversable
379 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
380 | * - _W_(`batchSize`): average batch write time
381 | * - _TC_(`traverser`): time complexity of the underlying traverser
382 | * - _SC_(`traverser`): space complexity of the underlying traverser
383 | *
384 | * @param getData - A function that takes a document snapshot and returns a non-empty data object with
385 | * which to update each document.
386 | * @returns A Promise resolving to an object representing the details of the migration.
387 | */
388 | updateWithDerivedData(
389 | getData: UpdateDataGetter,
390 | precondition?: firestore.Precondition
391 | ): Promise;
392 |
393 | /**
394 | * Updates all documents in this collection with the provided data.
395 | *
396 | * @remarks
397 | *
398 | * **Complexity:**
399 | *
400 | * - Time complexity: _TC_(`traverser`) where _C_(`batchSize`) = _W_(`batchSize`)
401 | * - Space complexity: _SC_(`traverser`) where _S_ = _O_(`batchSize`)
402 | * - Billing: _max_(1, _N_) reads, _K_ writes
403 | *
404 | * where:
405 | *
406 | * - _N_: number of docs in the traversable
407 | * - _K_: number of docs that passed the migration predicate (_K_<=_N_)
408 | * - _W_(`batchSize`): average batch write time
409 | * - _TC_(`traverser`): time complexity of the underlying traverser
410 | * - _SC_(`traverser`): space complexity of the underlying traverser
411 | *
412 | * @param getData - A function that takes a document snapshot and returns an alternating list of field
413 | * paths and values to update, optionally followed by a Precondition to enforce on this update.
414 | * @returns A Promise resolving to an object representing the details of the migration.
415 | */
416 | updateWithDerivedData(
417 | getData: UpdateFieldValueGetter
418 | ): Promise;
419 | }
420 |
--------------------------------------------------------------------------------
/src/api/interfaces/SetDataGetter.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 |
3 | /**
4 | * A function that takes a document snapshot and derives the data with which to set that document.
5 | */
6 | export type SetDataGetter<
7 | AppModelType = firestore.DocumentData,
8 | DbModelType extends firestore.DocumentData = firestore.DocumentData
9 | > = (
10 | doc: firestore.QueryDocumentSnapshot
11 | ) => firestore.WithFieldValue;
12 |
--------------------------------------------------------------------------------
/src/api/interfaces/SetOptions.ts:
--------------------------------------------------------------------------------
1 | export type { SetOptions } from '@google-cloud/firestore';
2 |
--------------------------------------------------------------------------------
/src/api/interfaces/SetPartialDataGetter.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 |
3 | /**
4 | * A function that takes a document snapshot and derives the data with which to partially set that document.
5 | */
6 | export type SetPartialDataGetter<
7 | AppModelType = firestore.DocumentData,
8 | DbModelType extends firestore.DocumentData = firestore.DocumentData
9 | > = (
10 | doc: firestore.QueryDocumentSnapshot
11 | ) => firestore.PartialWithFieldValue;
12 |
--------------------------------------------------------------------------------
/src/api/interfaces/Traversable.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 |
3 | /**
4 | * A collection-like group of documents. Can be one of [CollectionReference](https://googleapis.dev/nodejs/firestore/latest/CollectionReference.html),
5 | * [CollectionGroup](https://googleapis.dev/nodejs/firestore/latest/CollectionGroup.html) and [Query](https://googleapis.dev/nodejs/firestore/latest/Query.html).
6 | */
7 | export type Traversable<
8 | AppModelType = firestore.DocumentData,
9 | DbModelType extends firestore.DocumentData = firestore.DocumentData
10 | > =
11 | | firestore.CollectionReference
12 | | firestore.CollectionGroup
13 | | firestore.Query;
14 |
--------------------------------------------------------------------------------
/src/api/interfaces/TraversalConfig.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * The configuration with which a traverser is created.
3 | */
4 | export interface TraversalConfig {
5 | /**
6 | * The number of documents that will be traversed in each batch.
7 | *
8 | * @defaultValue 250
9 | */
10 | readonly batchSize: number;
11 |
12 | /**
13 | * The amount of time (in ms) to "sleep" before moving on to the next batch.
14 | *
15 | * @defaultValue 0
16 | */
17 | readonly sleepTimeBetweenBatches: number;
18 |
19 | /**
20 | * The maximum number of documents that will be traversed.
21 | *
22 | * @defaultValue `Infinity`
23 | */
24 | readonly maxDocCount: number;
25 |
26 | /**
27 | * The maximum number of batches that can be held in memory and processed concurrently.
28 | *
29 | * @remarks
30 | *
31 | * This field must be a positive integer representing the maximum number of batches that you wish
32 | * the traverser to process concurrently at any given time. The traverser will pause when the number
33 | * of batches being processed concurrently reaches this number and continue when a batch has been
34 | * processed. This means that the higher the value of `maxConcurrentBatchCount`, the more memory the
35 | * traverser will consume but also the faster it will finish the traversal.
36 | *
37 | * @example
38 | *
39 | * ```typescript
40 | * const projectsColRef = firestore().collection('projects');
41 | * const traverser = createTraverser(projectsColRef, {
42 | * batchSize: 500,
43 | * maxConcurrentBatchCount: 20,
44 | * });
45 | * ```
46 | *
47 | * By providing this config we have indicated that we want each batch to contain 500 documents and the
48 | * traverser to process 20 batches concurrently at any given time. This means we have ensured that our
49 | * machine can handle 500 * 20 = 10,000 documents in memory at any given time.
50 | *
51 | * @defaultValue 1
52 | */
53 | readonly maxConcurrentBatchCount: number;
54 |
55 | /**
56 | * The maximum number of times the traverser will retry processing a given batch, in case of an error.
57 | * By default, batches are not retried.
58 | *
59 | * @remarks
60 | *
61 | * This field must be a non-negative integer representing the maximum number of times the traverser will
62 | * retry processing a given batch. By default, the traverser invokes the batch callback only once i.e.
63 | * with 0 retries but, if `maxBatchRetryCount` > 0, it will keep invoking the callback until it succeeds
64 | * or the total number of retries reaches `maxBatchRetryCount`.
65 | *
66 | * @defaultValue 0
67 | */
68 | readonly maxBatchRetryCount: number;
69 |
70 | /**
71 | * A non-negative integer or a function that takes the 0-based index of the last trial and returns a
72 | * non-negative integer indicating the amount of time (in ms) to "sleep" before retrying processing
73 | * the current batch. This is useful if you want to implement something like exponential backoff.
74 | *
75 | * @defaultValue 1000
76 | */
77 | readonly sleepTimeBetweenTrials: number | ((lastTrialIndex: number) => number);
78 | }
79 |
--------------------------------------------------------------------------------
/src/api/interfaces/TraversalResult.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Represents an object that contains the details of a traversal.
3 | */
4 | export interface TraversalResult {
5 | /**
6 | * The number of batches that have been retrieved in this traversal.
7 | */
8 | readonly batchCount: number;
9 |
10 | /**
11 | * The number of documents that have been retrieved in this traversal.
12 | */
13 | readonly docCount: number;
14 | }
15 |
--------------------------------------------------------------------------------
/src/api/interfaces/TraverseEachCallback.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 |
3 | /**
4 | * An asynchronous function that takes a document snapshot, its 0-based index within the batch, and
5 | * the 0-based index of the batch as arguments.
6 | */
7 | export type TraverseEachCallback<
8 | AppModelType = firestore.DocumentData,
9 | DbModelType extends firestore.DocumentData = firestore.DocumentData
10 | > = (
11 | doc: firestore.QueryDocumentSnapshot,
12 | docIndex: number,
13 | batchIndex: number
14 | ) => void | Promise;
15 |
--------------------------------------------------------------------------------
/src/api/interfaces/TraverseEachConfig.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * The configuration that a given traverser uses in sequential traversals.
3 | */
4 | export interface TraverseEachConfig {
5 | /**
6 | * The amount of time (in ms) to "sleep" before moving to the next doc.
7 | *
8 | * @defaultValue 0
9 | */
10 | readonly sleepTimeBetweenDocs: number;
11 | }
12 |
--------------------------------------------------------------------------------
/src/api/interfaces/Traverser.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import type {
3 | BatchCallback,
4 | ExitEarlyPredicate,
5 | Traversable,
6 | TraversalConfig,
7 | TraversalResult,
8 | TraverseEachCallback,
9 | TraverseEachConfig,
10 | } from '.';
11 |
12 | /**
13 | * A traverser object that facilitates Firestore collection traversals.
14 | */
15 | export interface Traverser<
16 | AppModelType = firestore.DocumentData,
17 | DbModelType extends firestore.DocumentData = firestore.DocumentData
18 | > {
19 | /**
20 | * The underlying traversable.
21 | */
22 | readonly traversable: Traversable;
23 |
24 | /**
25 | * Existing traversal config.
26 | */
27 | readonly traversalConfig: TraversalConfig;
28 |
29 | /**
30 | * Applies the specified config values to the traverser.
31 | *
32 | * @param config - Partial traversal configuration.
33 | * @returns A new {@link Traverser} object.
34 | */
35 | withConfig(config: Partial): Traverser;
36 |
37 | /**
38 | * Applies the specified exit-early predicate to the traverser. After retrieving each batch, the traverser
39 | * will evaluate the predicate with the current batch doc snapshots and batch index and decide whether to
40 | * continue the traversal or exit early.
41 | *
42 | * @remarks
43 | *
44 | * If you have already applied other exit-early predicates to this traverser, this and all the other predicates
45 | * will be evaluated and the resulting booleans will be OR'd to get the boolean that indicates whether to exit
46 | * early or not. This is consistent with the intuitive default behavior that the traverser doesn't exit early.
47 | *
48 | * @example
49 | *
50 | * ```ts
51 | * const newTraverser = traverser
52 | * .withExitEarlyPredicate((batchDocs) => batchDocs.some((d) => d.get('name') === undefined))
53 | * .withExitEarlyPredicate((_, batchIndex) => batchIndex === 99);
54 | * ```
55 | *
56 | * In the above case `newTraverser` will exit early if the `name` field of one of the docs in the batch is
57 | * missing OR if the batch index is 99. That is, it will never reach batch 100 and depending on the documents
58 | * in the database it may exit earlier than 99.
59 | *
60 | * @param predicate - A synchronous function that takes batch doc snapshots and the 0-based batch index and
61 | * returns a boolean indicating whether to exit traversal early.
62 | * @returns A new {@link Traverser} object.
63 | */
64 | withExitEarlyPredicate(
65 | predicate: ExitEarlyPredicate
66 | ): Traverser;
67 |
68 | /**
69 | * Traverses the entire collection in batches of the size specified in traversal config. Invokes the specified
70 | * callback sequentially for each document snapshot in each batch.
71 | *
72 | * @param callback - An asynchronous callback function to invoke for each document snapshot in each batch.
73 | * @param config - The sequential traversal configuration.
74 | * @returns A Promise resolving to an object representing the details of the traversal. The Promise resolves
75 | * when the entire traversal ends.
76 | */
77 | traverseEach(
78 | callback: TraverseEachCallback,
79 | config?: Partial
80 | ): Promise;
81 |
82 | /**
83 | * Traverses the entire collection in batches of the size specified in traversal config. Invokes the specified
84 | * async callback for each batch of document snapshots and immediately moves to the next batch. Does not wait
85 | * for the callback Promise to resolve before moving to the next batch so there is no guarantee that any given
86 | * batch will finish processing before a later batch.
87 | *
88 | * @remarks
89 | *
90 | * This method throws only when the batch callback throws and the number of retries equals `maxBatchRetryCount`.
91 | * The last error thrown by the batch callback propagates up to this method.
92 | *
93 | * **Complexity:**
94 | *
95 | * - Time complexity: _O_((_N_ / `batchSize`) \* (_Q_(`batchSize`) + _C_(`batchSize`) / `maxConcurrentBatchCount`))
96 | * - Space complexity: _O_(`maxConcurrentBatchCount` * (`batchSize` * _D_ + _S_))
97 | * - Billing: _max_(1, _N_) reads
98 | *
99 | * where:
100 | *
101 | * - _N_: number of docs in the traversable
102 | * - _Q_(`batchSize`): average batch query time
103 | * - _C_(`batchSize`): average callback processing time
104 | * - _D_: average document size
105 | * - _S_: average extra space used by the callback
106 | *
107 | * @param callback - An asynchronous callback function to invoke for each batch of document snapshots.
108 | * @returns A Promise resolving to an object representing the details of the traversal.
109 | */
110 | traverse(callback: BatchCallback): Promise;
111 | }
112 |
--------------------------------------------------------------------------------
/src/api/interfaces/UpdateDataGetter.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 |
3 | /**
4 | * A function that takes a document snapshot and derives the data with which to update that document.
5 | */
6 | export type UpdateDataGetter<
7 | AppModelType = firestore.DocumentData,
8 | DbModelType extends firestore.DocumentData = firestore.DocumentData
9 | > = (
10 | doc: firestore.QueryDocumentSnapshot
11 | ) => firestore.UpdateData;
12 |
--------------------------------------------------------------------------------
/src/api/interfaces/UpdateFieldValueGetter.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 |
3 | /**
4 | * A function that takes a document snapshot and returns an alternating list of field paths and
5 | * values to update within that document, optionally followed by a Precondition to enforce on
6 | * the update.
7 | */
8 | export type UpdateFieldValueGetter<
9 | AppModelType = firestore.DocumentData,
10 | DbModelType extends firestore.DocumentData = firestore.DocumentData
11 | > = (
12 | doc: firestore.QueryDocumentSnapshot
13 | ) => [string | firestore.FieldPath, any, ...any[]];
14 |
--------------------------------------------------------------------------------
/src/api/interfaces/index.ts:
--------------------------------------------------------------------------------
1 | export type { BatchCallback } from './BatchCallback';
2 | export type { BatchMigrator } from './BatchMigrator';
3 | export type { DefaultMigrator } from './DefaultMigrator';
4 | export type { ExitEarlyPredicate } from './ExitEarlyPredicate';
5 | export type { MigrationPredicate } from './MigrationPredicate';
6 | export type { MigrationResult } from './MigrationResult';
7 | export type { Migrator } from './Migrator';
8 | export type { SetDataGetter } from './SetDataGetter';
9 | export type { SetOptions } from './SetOptions';
10 | export type { SetPartialDataGetter } from './SetPartialDataGetter';
11 | export type { Traversable } from './Traversable';
12 | export type { TraversalConfig } from './TraversalConfig';
13 | export type { TraversalResult } from './TraversalResult';
14 | export type { TraverseEachCallback } from './TraverseEachCallback';
15 | export type { TraverseEachConfig } from './TraverseEachConfig';
16 | export type { Traverser } from './Traverser';
17 | export type { UpdateDataGetter } from './UpdateDataGetter';
18 | export type { UpdateFieldValueGetter } from './UpdateFieldValueGetter';
19 |
--------------------------------------------------------------------------------
/src/errors/ImplementationError.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * An error thrown when a piece of code is found to be incorrectly implemented. If the maintainers of the
3 | * library have done everything right, this error should never be encountered.
4 | */
5 | export class ImplementationError extends Error {
6 | constructor(message: string) {
7 | super(`Implementation Error: ${message}`);
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/src/errors/InvalidConfigError.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * An error thrown when an invalid configuration is provided.
3 | */
4 | export class InvalidConfigError extends Error {
5 | constructor(message: string) {
6 | super(`Invalid config: ${message}`);
7 | }
8 | }
9 |
--------------------------------------------------------------------------------
/src/errors/index.ts:
--------------------------------------------------------------------------------
1 | export { ImplementationError } from './ImplementationError';
2 | export { InvalidConfigError } from './InvalidConfigError';
3 |
--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
1 | export * from './api';
2 | export * from './errors';
3 |
--------------------------------------------------------------------------------
/src/internal/ds/PromiseQueue.ts:
--------------------------------------------------------------------------------
1 | import type { IllegalArgumentError } from '../errors'; /* eslint-disable-line */
2 | import { SLLQueueExtended } from './SLLQueueExtended';
3 |
4 | export class PromiseQueue {
5 | readonly #queue: SLLQueueExtended;
6 | readonly #map: Map>;
7 | #lastPromiseId: number;
8 | #isProcessing: boolean;
9 |
10 | public constructor() {
11 | this.#queue = new SLLQueueExtended();
12 | this.#map = new Map();
13 | this.#lastPromiseId = 0;
14 | this.#isProcessing = false;
15 | }
16 |
17 | public get size(): number {
18 | return this.#map.size;
19 | }
20 |
21 | public get isProcessing(): boolean {
22 | return this.#isProcessing;
23 | }
24 |
25 | public enqueue(promise: Promise): void {
26 | const promiseId = this.#getIdForNewPromise();
27 | this.#queue.enqueue(promiseId);
28 | this.#map.set(promiseId, promise);
29 | }
30 |
31 | #getIdForNewPromise(): number {
32 | return ++this.#lastPromiseId;
33 | }
34 |
35 | /**
36 | * Processes all Promises in the queue.
37 | */
38 | public processAll(): Promise {
39 | return this.processFirst(this.#queue.count);
40 | }
41 |
42 | /**
43 | * Processes the first `promiseCount` Promises in the queue.
44 | * @throws {@link IllegalArgumentError} Thrown if `promiseCount` is invalid.
45 | */
46 | public async processFirst(promiseCount: number): Promise {
47 | this.#isProcessing = true;
48 | const promiseIds = this.#queue.dequeueFirst(promiseCount);
49 | const results = await Promise.all(
50 | promiseIds.map(async (id) => {
51 | const promise = this.#map.get(id)!;
52 | const result = await promise;
53 | this.#map.delete(id);
54 | return result;
55 | })
56 | );
57 | this.#isProcessing = false;
58 | return results;
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/internal/ds/SLLQueueExtended.ts:
--------------------------------------------------------------------------------
1 | import { SLLQueue } from '@proficient/ds';
2 | import { IllegalArgumentError } from '../errors';
3 |
4 | export class SLLQueueExtended extends SLLQueue {
5 | /**
6 | * Dequeues the first `itemCount` items into an array.
7 | *
8 | * @param itemCount - The number of items to dequeue.
9 | * @returns The array of dequeued items.
10 | * @throws {@link IllegalArgumentError} Thrown if `itemCount` is invalid.
11 | */
12 | public dequeueFirst(itemCount: number): E[] {
13 | if (!Number.isInteger(itemCount) || itemCount < 0 || itemCount > this.count) {
14 | throw new IllegalArgumentError(
15 | 'The `itemCount` argument must be a non-negative integer less than or equal to the size of the queue.'
16 | );
17 | }
18 |
19 | const items = new Array(itemCount);
20 |
21 | for (let i = 0; i < itemCount; i++) {
22 | items[i] = this.dequeue();
23 | }
24 |
25 | return items;
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/internal/ds/__tests__/PromiseQueue.test.ts:
--------------------------------------------------------------------------------
1 | import { PromiseQueue } from '..';
2 | import { sleep } from '../../utils';
3 |
4 | describe('PromiseQueue', () => {
5 | const runTask = (): Promise => sleep(0);
6 |
7 | test('correctly processes all items', async () => {
8 | const q = new PromiseQueue();
9 |
10 | expect(q.size).toBe(0);
11 |
12 | q.enqueue(runTask());
13 | q.enqueue(runTask());
14 | q.enqueue(runTask());
15 |
16 | expect(q.size).toBe(3);
17 |
18 | await q.processAll();
19 |
20 | expect(q.size).toBe(0);
21 | });
22 |
23 | test('correctly processes the first `n` items', async () => {
24 | const q = new PromiseQueue();
25 |
26 | q.enqueue(runTask());
27 | q.enqueue(runTask());
28 | q.enqueue(runTask());
29 |
30 | await q.processFirst(0);
31 |
32 | expect(q.size).toBe(3);
33 |
34 | await q.processFirst(1);
35 |
36 | expect(q.size).toBe(2);
37 |
38 | await q.processFirst(2);
39 |
40 | expect(q.size).toBe(0);
41 | });
42 | });
43 |
--------------------------------------------------------------------------------
/src/internal/ds/__tests__/SLLQueueExtended.test.ts:
--------------------------------------------------------------------------------
1 | import { SLLQueueExtended } from '..';
2 |
3 | describe('SLLQueueExtended', () => {
4 | describe('dequeueFirst(n)', () => {
5 | test('returns empty array if empty and n=0', () => {
6 | const q = new SLLQueueExtended();
7 | expect(q.dequeueFirst(0)).toEqual([]);
8 | });
9 |
10 | test('returns empty array if n=0', () => {
11 | const q = new SLLQueueExtended();
12 | q.enqueue(1);
13 | expect(q.dequeueFirst(0)).toEqual([]);
14 | });
15 |
16 | describe('throws if', () => {
17 | test('n is negative', () => {
18 | const q = new SLLQueueExtended();
19 | expect(() => q.dequeueFirst(-1)).toThrow();
20 | });
21 |
22 | test('n is not an integer', () => {
23 | const q = new SLLQueueExtended();
24 | q.enqueue(1);
25 | q.enqueue(2);
26 | expect(() => q.dequeueFirst(1.5)).toThrow();
27 | });
28 |
29 | test('n is greater than queue size', () => {
30 | const q = new SLLQueueExtended();
31 | q.enqueue(1);
32 | q.enqueue(2);
33 | expect(() => q.dequeueFirst(3)).toThrow();
34 | });
35 | });
36 |
37 | test('returns correct number of items in FIFO order', () => {
38 | const q = new SLLQueueExtended();
39 | q.enqueue(1);
40 | q.enqueue(2);
41 | q.enqueue(3);
42 | q.enqueue(4);
43 | expect(q.dequeueFirst(3)).toEqual([1, 2, 3]);
44 | });
45 | });
46 | });
47 |
--------------------------------------------------------------------------------
/src/internal/ds/index.ts:
--------------------------------------------------------------------------------
1 | export { PromiseQueue } from './PromiseQueue';
2 | export { SLLQueueExtended } from './SLLQueueExtended';
3 |
--------------------------------------------------------------------------------
/src/internal/errors/IllegalArgumentError.ts:
--------------------------------------------------------------------------------
1 | export class IllegalArgumentError extends Error {
2 | public constructor(message: string) {
3 | super(`Encountered an illegal argument: ${message}`);
4 | }
5 | }
6 |
--------------------------------------------------------------------------------
/src/internal/errors/index.ts:
--------------------------------------------------------------------------------
1 | export { IllegalArgumentError } from './IllegalArgumentError';
2 |
--------------------------------------------------------------------------------
/src/internal/implementations/BasicBatchMigratorImpl.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import type {
3 | BatchMigrator,
4 | MigrationPredicate,
5 | MigrationResult,
6 | SetDataGetter,
7 | SetOptions,
8 | SetPartialDataGetter,
9 | Traverser,
10 | UpdateDataGetter,
11 | UpdateFieldValueGetter,
12 | } from '../../api';
13 | import { InvalidConfigError } from '../../errors';
14 | import { isPositiveInteger } from '../utils';
15 | import { AbstractMigrator, RegisteredCallbacks } from './abstract';
16 | import { IllegalArgumentError } from '../errors';
17 |
18 | export class BasicBatchMigratorImpl<
19 | AppModelType = firestore.DocumentData,
20 | DbModelType extends firestore.DocumentData = firestore.DocumentData
21 | >
22 | extends AbstractMigrator
23 | implements BatchMigrator
24 | {
25 | static readonly #MAX_BATCH_WRITE_DOC_COUNT = 500;
26 |
27 | public constructor(
28 | public readonly traverser: Traverser,
29 | registeredCallbacks?: RegisteredCallbacks,
30 | migrationPredicates?: MigrationPredicate[]
31 | ) {
32 | super(registeredCallbacks, migrationPredicates);
33 | this.#validateTraverserCompatibility();
34 | }
35 |
36 | #validateTraverserCompatibility(): void {
37 | const { batchSize } = this.traverser.traversalConfig;
38 | const maxBatchWriteDocCount = BasicBatchMigratorImpl.#MAX_BATCH_WRITE_DOC_COUNT;
39 | if (
40 | typeof batchSize === 'number' &&
41 | (!isPositiveInteger(batchSize) || batchSize > maxBatchWriteDocCount)
42 | ) {
43 | throw new InvalidConfigError(
44 | `The 'batchSize' field in the traversal config of a BatchMigrator's traverser must be a positive integer less than or equal to ${maxBatchWriteDocCount}. In Firestore, each write batch can write to a maximum of ${maxBatchWriteDocCount} documents.`
45 | );
46 | }
47 | }
48 |
49 | public withPredicate(
50 | predicate: MigrationPredicate
51 | ): BatchMigrator {
52 | return new BasicBatchMigratorImpl(this.traverser, this.registeredCallbacks, [
53 | ...this.migrationPredicates,
54 | predicate,
55 | ]);
56 | }
57 |
58 | public withTraverser(
59 | traverser: Traverser
60 | ): BatchMigrator {
61 | return new BasicBatchMigratorImpl(
62 | traverser,
63 | this.registeredCallbacks,
64 | this.migrationPredicates
65 | );
66 | }
67 |
68 | public set(
69 | data: firestore.PartialWithFieldValue,
70 | options: SetOptions
71 | ): Promise;
72 |
73 | public set(data: firestore.WithFieldValue): Promise;
74 |
75 | public async set(
76 | data: firestore.PartialWithFieldValue | firestore.WithFieldValue,
77 | options?: SetOptions
78 | ): Promise {
79 | return this.#migrate((writeBatch, doc) => {
80 | if (options === undefined) {
81 | // Signature 2
82 | writeBatch.set(doc.ref, data as firestore.WithFieldValue);
83 | } else {
84 | // Signature 1
85 | writeBatch.set(doc.ref, data as firestore.PartialWithFieldValue, options);
86 | }
87 | });
88 | }
89 |
90 | public setWithDerivedData(
91 | getData: SetPartialDataGetter,
92 | options: SetOptions
93 | ): Promise;
94 |
95 | public setWithDerivedData(
96 | getData: SetDataGetter
97 | ): Promise;
98 |
99 | public async setWithDerivedData(
100 | getData:
101 | | SetPartialDataGetter
102 | | SetDataGetter,
103 | options?: SetOptions
104 | ): Promise {
105 | return this.#migrate((writeBatch, doc) => {
106 | if (options === undefined) {
107 | // Signature 2
108 | const data = (getData as SetDataGetter)(doc);
109 | writeBatch.set(doc.ref, data);
110 | } else {
111 | // Signature 1
112 | const data = (getData as SetPartialDataGetter)(doc);
113 | writeBatch.set(doc.ref, data, options);
114 | }
115 | });
116 | }
117 |
118 | public update(
119 | data: firestore.UpdateData,
120 | precondition?: firestore.Precondition
121 | ): Promise;
122 |
123 | public update(
124 | field: string | firestore.FieldPath,
125 | value: any,
126 | ...moreFieldsOrPrecondition: any[]
127 | ): Promise;
128 |
129 | public update(
130 | dataOrField: firestore.UpdateData | string | firestore.FieldPath,
131 | preconditionOrValue?: any,
132 | ...moreFieldsOrPrecondition: any[]
133 | ): Promise {
134 | return this.#migrate((writeBatch, doc) => {
135 | if (
136 | typeof dataOrField === 'string' ||
137 | dataOrField instanceof this.firestoreConstructor.FieldPath
138 | ) {
139 | // Signature 2
140 | const field = dataOrField;
141 | const value = preconditionOrValue;
142 | writeBatch.update(doc.ref, field, value, ...moreFieldsOrPrecondition);
143 | } else if (dataOrField !== undefined) {
144 | // Signature 1
145 | const data = dataOrField;
146 | const precondition = preconditionOrValue as firestore.Precondition | undefined;
147 | if (precondition === undefined) {
148 | writeBatch.update(doc.ref, data);
149 | } else {
150 | writeBatch.update(doc.ref, data, precondition);
151 | }
152 | } else {
153 | throw new IllegalArgumentError(
154 | `Unsupported signature detected. The 'dataOrField' argument cannot be undefined. The 'dataOrField' argument must be a string, a FieldPath, or an object.`
155 | );
156 | }
157 | });
158 | }
159 |
160 | public updateWithDerivedData(
161 | getData: UpdateDataGetter,
162 | precondition?: firestore.Precondition
163 | ): Promise;
164 |
165 | public updateWithDerivedData(
166 | getData: UpdateFieldValueGetter
167 | ): Promise;
168 |
169 | public updateWithDerivedData(
170 | getData: (
171 | doc: firestore.QueryDocumentSnapshot
172 | ) =>
173 | | ReturnType>
174 | | ReturnType>,
175 | precondition?: firestore.Precondition
176 | ): Promise {
177 | return this.#migrate((writeBatch, doc) => {
178 | const data = getData(doc);
179 | if (Array.isArray(data)) {
180 | // Signature 2
181 | writeBatch.update(doc.ref, ...(data as [string | firestore.FieldPath, any, ...any[]]));
182 | } else if (data !== undefined) {
183 | // Signature 1
184 | if (precondition === undefined) {
185 | writeBatch.update(doc.ref, data);
186 | } else {
187 | writeBatch.update(doc.ref, data, precondition);
188 | }
189 | } else {
190 | throw new IllegalArgumentError(
191 | `Unsupported signature detected. The 'data' argument cannot be undefined. The 'data' argument must be an array, an object, or a valid Firestore update signature.`
192 | );
193 | }
194 | });
195 | }
196 |
197 | async #migrate(
198 | migrateDoc: (
199 | writeBatch: firestore.WriteBatch,
200 | doc: firestore.QueryDocumentSnapshot
201 | ) => void
202 | ): Promise {
203 | return this.migrateWithTraverser(async (batchDocs) => {
204 | let migratedDocCount = 0;
205 | const writeBatch = this.firestoreInstance.batch();
206 | batchDocs.forEach((doc) => {
207 | if (this.shouldMigrateDoc(doc)) {
208 | migrateDoc(writeBatch, doc);
209 | migratedDocCount++;
210 | }
211 | });
212 | await writeBatch.commit();
213 | return migratedDocCount;
214 | });
215 | }
216 | }
217 |
--------------------------------------------------------------------------------
/src/internal/implementations/BasicDefaultMigratorImpl.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import type {
3 | DefaultMigrator,
4 | MigrationPredicate,
5 | MigrationResult,
6 | SetDataGetter,
7 | SetOptions,
8 | SetPartialDataGetter,
9 | Traverser,
10 | UpdateDataGetter,
11 | UpdateFieldValueGetter,
12 | } from '../../api';
13 | import { AbstractMigrator, RegisteredCallbacks } from './abstract';
14 | import { IllegalArgumentError } from '../errors';
15 |
16 | export class BasicDefaultMigratorImpl<
17 | AppModelType = firestore.DocumentData,
18 | DbModelType extends firestore.DocumentData = firestore.DocumentData
19 | >
20 | extends AbstractMigrator
21 | implements DefaultMigrator
22 | {
23 | public constructor(
24 | public readonly traverser: Traverser,
25 | registeredCallbacks?: RegisteredCallbacks,
26 | migrationPredicates?: MigrationPredicate[]
27 | ) {
28 | super(registeredCallbacks, migrationPredicates);
29 | }
30 |
31 | public withPredicate(
32 | predicate: MigrationPredicate
33 | ): DefaultMigrator {
34 | return new BasicDefaultMigratorImpl(this.traverser, this.registeredCallbacks, [
35 | ...this.migrationPredicates,
36 | predicate,
37 | ]);
38 | }
39 |
40 | public withTraverser(
41 | traverser: Traverser
42 | ): DefaultMigrator {
43 | return new BasicDefaultMigratorImpl(
44 | traverser,
45 | this.registeredCallbacks,
46 | this.migrationPredicates
47 | );
48 | }
49 |
50 | public set(
51 | data: firestore.PartialWithFieldValue,
52 | options: SetOptions
53 | ): Promise;
54 |
55 | public set(data: firestore.WithFieldValue): Promise;
56 |
57 | public async set(
58 | data: firestore.PartialWithFieldValue | firestore.WithFieldValue,
59 | options?: SetOptions
60 | ): Promise {
61 | return this.#migrate(async (doc) => {
62 | if (options === undefined) {
63 | // Signature 2
64 | await doc.ref.set(data as firestore.WithFieldValue);
65 | } else {
66 | // Signature 1
67 | await doc.ref.set(data as firestore.PartialWithFieldValue, options);
68 | }
69 | });
70 | }
71 |
72 | public setWithDerivedData(
73 | getData: SetPartialDataGetter,
74 | options: SetOptions
75 | ): Promise;
76 |
77 | public setWithDerivedData(
78 | getData: SetDataGetter
79 | ): Promise;
80 |
81 | public async setWithDerivedData(
82 | getData:
83 | | SetPartialDataGetter
84 | | SetDataGetter,
85 | options?: SetOptions
86 | ): Promise {
87 | return this.#migrate(async (doc) => {
88 | if (options === undefined) {
89 | // Signature 2
90 | const data = (getData as SetDataGetter)(doc);
91 | await doc.ref.set(data);
92 | } else {
93 | // Signature 1
94 | const data = (getData as SetPartialDataGetter)(doc);
95 | await doc.ref.set(data, options);
96 | }
97 | });
98 | }
99 |
100 | public update(
101 | data: firestore.UpdateData,
102 | precondition?: firestore.Precondition
103 | ): Promise;
104 |
105 | public update(
106 | field: string | firestore.FieldPath,
107 | value: any,
108 | ...moreFieldsOrPrecondition: any[]
109 | ): Promise;
110 |
111 | public update(
112 | dataOrField: firestore.UpdateData | string | firestore.FieldPath,
113 | preconditionOrValue?: any,
114 | ...moreFieldsOrPrecondition: any[]
115 | ): Promise {
116 | return this.#migrate(async (doc) => {
117 | if (
118 | typeof dataOrField === 'string' ||
119 | dataOrField instanceof this.firestoreConstructor.FieldPath
120 | ) {
121 | // Signature 2
122 | const field = dataOrField;
123 | const value = preconditionOrValue;
124 | await doc.ref.update(field, value, ...moreFieldsOrPrecondition);
125 | } else if (typeof dataOrField === 'object' && dataOrField !== null) {
126 | // Signature 1
127 | const data = dataOrField;
128 | const precondition = preconditionOrValue as firestore.Precondition | undefined;
129 | if (precondition === undefined) {
130 | await doc.ref.update(data);
131 | } else {
132 | await doc.ref.update(data, precondition);
133 | }
134 | } else {
135 | throw new IllegalArgumentError(
136 | `Unsupported signature detected. The 'dataOrField' argument cannot be undefined. The 'dataOrField' argument must be a string, a FieldPath, or an object.`
137 | );
138 | }
139 | });
140 | }
141 |
142 | public updateWithDerivedData(
143 | getData: UpdateDataGetter,
144 | precondition?: firestore.Precondition
145 | ): Promise;
146 |
147 | public updateWithDerivedData(
148 | getData: UpdateFieldValueGetter
149 | ): Promise;
150 |
151 | public updateWithDerivedData(
152 | getData: (
153 | doc: firestore.QueryDocumentSnapshot
154 | ) =>
155 | | ReturnType>
156 | | ReturnType>,
157 | precondition?: firestore.Precondition
158 | ): Promise {
159 | return this.#migrate(async (doc) => {
160 | const data = getData(doc);
161 |
162 | if (Array.isArray(data)) {
163 | // Signature 2
164 | await doc.ref.update(...(data as [string | firestore.FieldPath, any, ...any[]]));
165 | } else if (typeof data === 'object' && data !== null) {
166 | // Signature 1
167 | if (precondition === undefined) {
168 | await doc.ref.update(data);
169 | } else {
170 | await doc.ref.update(data, precondition);
171 | }
172 | } else {
173 | throw new IllegalArgumentError(
174 | `Unsupported signature detected. The 'data' argument must be an array, an object, or a valid Firestore update signature.`
175 | );
176 | }
177 | });
178 | }
179 |
180 | async #migrate(
181 | migrateDoc: (doc: firestore.QueryDocumentSnapshot) => Promise
182 | ): Promise {
183 | return this.migrateWithTraverser(async (batchDocs) => {
184 | let migratedDocCount = 0;
185 | const promises = batchDocs.map(async (doc) => {
186 | if (this.shouldMigrateDoc(doc)) {
187 | await migrateDoc(doc);
188 | migratedDocCount++;
189 | }
190 | });
191 | await Promise.all(promises);
192 | return migratedDocCount;
193 | });
194 | }
195 | }
196 |
--------------------------------------------------------------------------------
/src/internal/implementations/PromiseQueueBasedTraverserImpl.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import type {
3 | BatchCallback,
4 | ExitEarlyPredicate,
5 | Traversable,
6 | TraversalConfig,
7 | TraversalResult,
8 | Traverser,
9 | } from '../../api';
10 | import { ImplementationError } from '../../errors';
11 | import { IllegalArgumentError } from '../errors';
12 | import { PromiseQueue } from '../ds';
13 | import { makeRetriable, registerInterval, sleep } from '../utils';
14 | import { AbstractTraverser } from './abstract';
15 |
16 | export class PromiseQueueBasedTraverserImpl<
17 | AppModelType = firestore.DocumentData,
18 | DbModelType extends firestore.DocumentData = firestore.DocumentData
19 | >
20 | extends AbstractTraverser
21 | implements Traverser
22 | {
23 | static readonly #defaultConfig: TraversalConfig = {
24 | ...AbstractTraverser.baseConfig,
25 | };
26 |
27 | public constructor(
28 | public readonly traversable: Traversable,
29 | exitEarlyPredicates: ExitEarlyPredicate[] = [],
30 | config?: Partial
31 | ) {
32 | super({ ...PromiseQueueBasedTraverserImpl.#defaultConfig, ...config }, exitEarlyPredicates);
33 | }
34 |
35 | public withConfig(config: Partial): Traverser {
36 | return new PromiseQueueBasedTraverserImpl(this.traversable, this.exitEarlyPredicates, {
37 | ...this.traversalConfig,
38 | ...config,
39 | });
40 | }
41 |
42 | public withExitEarlyPredicate(
43 | predicate: ExitEarlyPredicate
44 | ): Traverser {
45 | return new PromiseQueueBasedTraverserImpl(
46 | this.traversable,
47 | [...this.exitEarlyPredicates, predicate],
48 | this.traversalConfig
49 | );
50 | }
51 |
52 | public async traverse(
53 | callback: BatchCallback
54 | ): Promise {
55 | const { traversalConfig } = this;
56 | const { maxConcurrentBatchCount } = traversalConfig;
57 | callback = this.#makeRetriableAccordingToConfig(callback);
58 | if (maxConcurrentBatchCount === 1) {
59 | return this.runTraversal(async (batchDocs, batchIndex) => {
60 | await callback(batchDocs, batchIndex);
61 | });
62 | }
63 | const callbackPromiseQueue = new PromiseQueue();
64 | const unregisterQueueProcessor = registerInterval(
65 | async () => {
66 | if (!callbackPromiseQueue.isProcessing) {
67 | const processableItemCount = this.#getProcessableItemCount(callbackPromiseQueue.size);
68 | try {
69 | await callbackPromiseQueue.processFirst(processableItemCount);
70 | } catch (err) {
71 | throw err instanceof IllegalArgumentError
72 | ? new ImplementationError(
73 | `Encountered an expected error originating from an incorrectly implemented PromiseQueue data structure.`
74 | )
75 | : err;
76 | }
77 | }
78 | },
79 | () => this.#getProcessQueueInterval(callbackPromiseQueue.size)
80 | );
81 | const traversalResult = await this.runTraversal((batchDocs, batchIndex) => {
82 | callbackPromiseQueue.enqueue(callback(batchDocs, batchIndex) ?? Promise.resolve());
83 | return async () => {
84 | while (callbackPromiseQueue.size >= maxConcurrentBatchCount) {
85 | // TODO: There probably is a better way to compute sleep duration
86 | const processQueueInterval = this.#getProcessQueueInterval(callbackPromiseQueue.size);
87 | await sleep(processQueueInterval);
88 | }
89 | };
90 | });
91 | await unregisterQueueProcessor();
92 | // There may still be some Promises left in the queue but there won't be any new ones coming in.
93 | // Wait for the existing ones to resolve and exit.
94 | await callbackPromiseQueue.processAll();
95 | return traversalResult;
96 | }
97 |
98 | #makeRetriableAccordingToConfig(
99 | callback: BatchCallback
100 | ): BatchCallback {
101 | const { maxBatchRetryCount, sleepTimeBetweenTrials } = this.traversalConfig;
102 | let cb = callback;
103 | if (maxBatchRetryCount > 0) {
104 | const retriableCallback = makeRetriable(callback, {
105 | maxTrialCount: 1 + maxBatchRetryCount,
106 | sleepTimeBetweenTrials,
107 | returnErrors: true,
108 | });
109 | cb = async (...args) => {
110 | const result = await retriableCallback(...args);
111 | if (!result.hasSucceeded) {
112 | const { errors } = result;
113 | const lastError = errors[errors.length - 1];
114 | throw lastError;
115 | }
116 | };
117 | }
118 | return cb;
119 | }
120 |
121 | /**
122 | * Computes the number of queue items to process based on the traversal configuration and queue size.
123 | *
124 | * @param queueSize - The current size of the queue.
125 | * @returns An integer within the range [0, `queueSize`].
126 | */
127 | #getProcessableItemCount(queueSize: number): number {
128 | // TODO: Implement using traversal config and queue size
129 | return queueSize;
130 | }
131 |
132 | /**
133 | * Computes the duration (in ms) for which to sleep before re-running the queue processing logic.
134 | *
135 | * @param queueSize - The current size of the queue.
136 | * @returns A non-negative integer.
137 | */
138 | #getProcessQueueInterval(queueSize: number): number {
139 | // TODO: Implement using traversal config and queue size
140 | return 250;
141 | }
142 | }
143 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/deleteField.test.ts:
--------------------------------------------------------------------------------
1 | import { testDeleteField } from '../shared/deleteField';
2 | import { describeBasicBatchMigratorMethodTest } from './helpers';
3 |
4 | describeBasicBatchMigratorMethodTest('deleteField', testDeleteField);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/deleteFields.test.ts:
--------------------------------------------------------------------------------
1 | import { testDeleteFields } from '../shared/deleteFields';
2 | import { describeBasicBatchMigratorMethodTest } from './helpers';
3 |
4 | describeBasicBatchMigratorMethodTest('deleteFields', testDeleteFields);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/helpers.ts:
--------------------------------------------------------------------------------
1 | import { BasicBatchMigratorImpl } from '../../../BasicBatchMigratorImpl';
2 | import { describeMigratorMethodTest } from '../helpers';
3 | import type { MigratorMethodTester } from '../config';
4 |
5 | export function describeBasicBatchMigratorMethodTest(
6 | methodName: string,
7 | methodTester: MigratorMethodTester
8 | ): void {
9 | describeMigratorMethodTest(BasicBatchMigratorImpl, methodName, methodTester);
10 | }
11 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/renameField.test.ts:
--------------------------------------------------------------------------------
1 | import { testRenameField } from '../shared/renameField';
2 | import { describeBasicBatchMigratorMethodTest } from './helpers';
3 |
4 | describeBasicBatchMigratorMethodTest('renameField', testRenameField);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/renameFields.test.ts:
--------------------------------------------------------------------------------
1 | import { testRenameFields } from '../shared/renameFields';
2 | import { describeBasicBatchMigratorMethodTest } from './helpers';
3 |
4 | describeBasicBatchMigratorMethodTest('renameFields', testRenameFields);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/update.test.ts:
--------------------------------------------------------------------------------
1 | import { testUpdate } from '../shared/update';
2 | import { describeBasicBatchMigratorMethodTest } from './helpers';
3 |
4 | describeBasicBatchMigratorMethodTest('update', testUpdate);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicBatchMigratorImpl/updateWithDerivedData.test.ts:
--------------------------------------------------------------------------------
1 | import { testUpdateWithDerivedData } from '../shared/updateWithDerivedData';
2 | import { describeBasicBatchMigratorMethodTest } from './helpers';
3 |
4 | describeBasicBatchMigratorMethodTest('updateWithDerivedData', testUpdateWithDerivedData);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/deleteField.test.ts:
--------------------------------------------------------------------------------
1 | import { testDeleteField } from '../shared/deleteField';
2 | import { describeBasicDefaultMigratorMethodTest } from './helpers';
3 |
4 | describeBasicDefaultMigratorMethodTest('deleteField', testDeleteField);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/deleteFields.test.ts:
--------------------------------------------------------------------------------
1 | import { testDeleteFields } from '../shared/deleteFields';
2 | import { describeBasicDefaultMigratorMethodTest } from './helpers';
3 |
4 | describeBasicDefaultMigratorMethodTest('deleteFields', testDeleteFields);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/helpers.ts:
--------------------------------------------------------------------------------
1 | import { BasicDefaultMigratorImpl } from '../../../BasicDefaultMigratorImpl';
2 | import { describeMigratorMethodTest } from '../helpers';
3 | import type { MigratorMethodTester } from '../config';
4 |
5 | export function describeBasicDefaultMigratorMethodTest(
6 | methodName: string,
7 | methodTester: MigratorMethodTester
8 | ): void {
9 | describeMigratorMethodTest(BasicDefaultMigratorImpl, methodName, methodTester);
10 | }
11 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/renameField.test.ts:
--------------------------------------------------------------------------------
1 | import { testRenameField } from '../shared/renameField';
2 | import { describeBasicDefaultMigratorMethodTest } from './helpers';
3 |
4 | describeBasicDefaultMigratorMethodTest('renameField', testRenameField);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/renameFields.test.ts:
--------------------------------------------------------------------------------
1 | import { testRenameFields } from '../shared/renameFields';
2 | import { describeBasicDefaultMigratorMethodTest } from './helpers';
3 |
4 | describeBasicDefaultMigratorMethodTest('renameFields', testRenameFields);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/update.test.ts:
--------------------------------------------------------------------------------
1 | import { testUpdate } from '../shared/update';
2 | import { describeBasicDefaultMigratorMethodTest } from './helpers';
3 |
4 | describeBasicDefaultMigratorMethodTest('update', testUpdate);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/BasicDefaultMigratorImpl/updateWithDerivedData.test.ts:
--------------------------------------------------------------------------------
1 | import { testUpdateWithDerivedData } from '../shared/updateWithDerivedData';
2 | import { describeBasicDefaultMigratorMethodTest } from './helpers';
3 |
4 | describeBasicDefaultMigratorMethodTest('updateWithDerivedData', testUpdateWithDerivedData);
5 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/config.ts:
--------------------------------------------------------------------------------
1 | import { firestore } from 'firebase-admin';
2 | import type { Migrator, TraversalConfig } from '../../../../../src';
3 |
4 | export type MigratorMethodTester = (
5 | migrator: Migrator,
6 | colRef: firestore.CollectionReference
7 | ) => void;
8 |
9 | export interface TestItemDoc {
10 | docId?: string;
11 | map1: {
12 | num1?: number;
13 | num2?: number;
14 | string1?: string;
15 | };
16 | num2: number;
17 | string2: string;
18 | string3: string;
19 | timestamp1?: firestore.Timestamp;
20 | timestamp2?: firestore.Timestamp;
21 | }
22 |
23 | export const DEFAULT_TIMEOUT = 60_000;
24 |
25 | export const DEFAULT_TRAVERSABLE_SIZE = 40;
26 |
27 | export const TRAVERSAL_CONFIG: Partial = {
28 | batchSize: 10,
29 | };
30 |
31 | export const INITIAL_DATA = Object.freeze({
32 | map1: {
33 | num1: 1,
34 | string1: 'abc',
35 | },
36 | num2: 2,
37 | string2: 'abc',
38 | string3: 'abc',
39 | timestamp1: firestore.Timestamp.fromDate(new Date()),
40 | });
41 |
--------------------------------------------------------------------------------
/src/internal/implementations/__tests__/Migrator/helpers.ts:
--------------------------------------------------------------------------------
1 | import type { firestore } from 'firebase-admin';
2 | import { app } from '../../../../../__tests__/app';
3 | import { collectionPopulator } from '../../../../../__tests__/utils';
4 | import { createTraverser, Migrator, Traverser } from '../../../../api';
5 | import {
6 | TestItemDoc,
7 | MigratorMethodTester,
8 | TRAVERSAL_CONFIG,
9 | DEFAULT_TIMEOUT,
10 | INITIAL_DATA,
11 | DEFAULT_TRAVERSABLE_SIZE,
12 | } from './config';
13 |
14 | export type MigratorImplClass = {
15 | new <
16 | AppModelType = firestore.DocumentData,
17 | DbModelType extends firestore.DocumentData = firestore.DocumentData
18 | >(
19 | traverser: Traverser
20 | ): Migrator;
21 | };
22 |
23 | export function describeMigratorMethodTest(
24 | migratorImplClass: MigratorImplClass,
25 | methodName: string,
26 | methodTester: MigratorMethodTester
27 | ): void {
28 | const description = `${migratorImplClass.name}.${methodName}`;
29 | const colRef = app().firestore.collection(
30 | description
31 | ) as firestore.CollectionReference