├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug.md │ └── fr.md └── workflows │ ├── e2e_test.old │ ├── npm_publish_bq_scripts.yml │ ├── readmes-updated.yml │ ├── release.yml │ ├── scripts │ ├── npm_publish.sh │ └── release.sh │ ├── test.yml │ └── validate.yml ├── .gitignore ├── .husky ├── .gitignore └── pre-commit ├── .prettierignore ├── .prettierrc.yaml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── _emulator ├── .firebaserc ├── .gitignore ├── extensions │ ├── delete-user-data.env.local │ ├── firestore-bigquery-export.env.local │ ├── firestore-counter.env.local │ ├── firestore-send-email-sendgrid.env.local │ ├── firestore-send-email-sendgrid.secret.local │ ├── firestore-send-email.env.local │ ├── firestore-send-email.secret.local │ ├── firestore-translate-text.env.local │ └── storage-resize-images.env.local ├── firebase.json ├── firestore.indexes.json ├── firestore.rules ├── functions │ ├── .gitignore │ ├── package-lock.json │ ├── package.json │ ├── src │ │ └── index.ts │ └── tsconfig.json └── storage.rules ├── codecov.yml ├── commitlint.config.js ├── delete-user-data ├── CHANGELOG.md ├── POSTINSTALL.md ├── PREINSTALL.md ├── README.md ├── extension.yaml ├── functions │ ├── .gitignore │ ├── __tests__ │ │ ├── handleDelete.test.ts │ │ ├── helpers.test.ts │ │ ├── helpers │ │ │ ├── index.ts │ │ │ └── setupEnvironment.ts │ │ ├── recursiveDelete.test.ts │ │ ├── runBatchPubSubDeletions.test.ts │ │ ├── search.test.ts │ │ ├── searchFunction.test.ts │ │ ├── setupTests.ts │ │ └── tsconfig.json │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── config.ts │ │ ├── helpers.ts │ │ ├── index.ts │ │ ├── logs.ts │ │ ├── recursiveDelete.ts │ │ ├── runBatchPubSubDeletions.ts │ │ ├── runCustomSearchFunction.ts │ │ ├── search.ts │ │ └── types.ts │ └── tsconfig.json └── test-data │ ├── README.md │ ├── images │ └── pic.png │ ├── index.ts │ ├── package-lock.json │ ├── package.json │ └── tsconfig.json ├── docs ├── delete-user-data │ └── get-started.md ├── firestore-bigquery-export │ ├── Clustering.md │ ├── Partitioning.md │ ├── Wildcards.md │ ├── cross-project-support.md │ ├── generating-schemas.md │ ├── get-started.md │ ├── importing-data.md │ ├── media │ │ ├── clustering.png │ │ └── wildcards.png │ └── transforming-data.md ├── firestore-bundle-builder │ ├── api-reference.md │ ├── get-started.md │ └── media │ │ └── admin-ui.png ├── firestore-counter │ └── get-started.md ├── firestore-send-email │ ├── get-started.md │ ├── manage-delivery-service.md │ ├── smtp-connection-url.md │ └── use-handlebars-template.md ├── firestore-shorten-urls-bitly │ └── get-started.md ├── firestore-translate-text │ └── get-started.md ├── rtdb-limit-child-nodes │ └── get-started.md └── storage-resize-images │ ├── customize-output-options.md │ ├── get-started.md │ └── handle-resize-image-extension-events.md ├── firestore-bigquery-export ├── CHANGELOG.md ├── CONTRIBUTING.md ├── POSTINSTALL.md ├── PREINSTALL.md ├── README.md ├── docs │ └── images │ │ └── firestore-bigquery-export-dep-diagram.svg ├── extension.yaml ├── firestore-bigquery-change-tracker │ ├── .gitignore │ ├── README.md │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── __tests__ │ │ │ ├── bigquery │ │ │ │ ├── alternativeProject.test.ts │ │ │ │ ├── checkUpdates.test.ts │ │ │ │ ├── clustering.test.ts │ │ │ │ ├── e2e.test.ts │ │ │ │ ├── failedTransaction.test.ts │ │ │ │ ├── materializedViews │ │ │ │ │ ├── initializeLatestMaterializedView.test.ts │ │ │ │ │ ├── initializeLatestView.test.ts │ │ │ │ │ ├── integration.test.ts │ │ │ │ │ └── shouldRecreateMaterializedView.test.ts │ │ │ │ ├── msql │ │ │ │ │ ├── incremental │ │ │ │ │ │ └── standard.sql │ │ │ │ │ └── nonIncremental │ │ │ │ │ │ └── standard.sql │ │ │ │ ├── partitioning.test.ts │ │ │ │ ├── snapshot.test.ts │ │ │ │ ├── stresstest.test.ts │ │ │ │ └── wildcardDocument.test.ts │ │ │ ├── emulator-params.env │ │ │ ├── firebase.json │ │ │ ├── fixtures │ │ │ │ ├── changeTracker.ts │ │ │ │ ├── clearTables.ts │ │ │ │ ├── queries.ts │ │ │ │ └── sql │ │ │ │ │ ├── generateSnapshotStresstestTable.sql │ │ │ │ │ ├── latestConsistentSnapshot.sql │ │ │ │ │ └── latestConsistentSnapshotNoGroupBy.sql │ │ │ └── logger.test.ts │ │ ├── bigquery │ │ │ ├── checkUpdates.ts │ │ │ ├── clustering.ts │ │ │ ├── handleFailedTransactions.ts │ │ │ ├── index.ts │ │ │ ├── initializeLatestMaterializedView.ts │ │ │ ├── initializeLatestView.ts │ │ │ ├── partitioning.ts │ │ │ ├── schema.ts │ │ │ ├── snapshot.ts │ │ │ ├── utils.test.ts │ │ │ ├── utils.ts │ │ │ └── validateProject.ts │ │ ├── errors.ts │ │ ├── index.ts │ │ ├── logger.ts │ │ ├── logs.ts │ │ ├── tracker.ts │ │ └── types.ts │ ├── tsconfig.json │ └── tsconfig.test.json ├── functions │ ├── .gitignore │ ├── __tests__ │ │ ├── __mocks__ │ │ │ ├── console.ts │ │ │ └── firestore.ts │ │ ├── __snapshots__ │ │ │ └── config.test.ts.snap │ │ ├── config.test.ts │ │ ├── e2e.test.ts │ │ ├── fixtures │ │ │ └── documentData.ts │ │ ├── functions.test.ts │ │ ├── jest.setup.ts │ │ ├── test.types.d.ts │ │ └── util.test.ts │ ├── big-query-table-test │ │ ├── install-params.env │ │ ├── install-script.js │ │ └── install-test.sh │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── config.ts │ │ ├── events.ts │ │ ├── index.ts │ │ ├── logs.ts │ │ └── util.ts │ ├── stress_test │ │ ├── count.js │ │ ├── main.js │ │ └── worker.js │ ├── tsconfig.json │ └── tsconfig.test.json ├── guides │ ├── EXAMPLE_QUERIES.md │ ├── GENERATE_SCHEMA_VIEWS.md │ ├── IMPORT_EXISTING_DOCUMENTS.md │ └── OBSERVABILITY.md └── scripts │ ├── gen-schema-view │ ├── .gitignore │ ├── README.md │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── schemas │ │ ├── test.json │ │ ├── user-schema-version-login.json │ │ ├── user_array.json │ │ ├── user_complex.json │ │ └── user_full.json │ ├── src │ │ ├── __tests__ │ │ │ ├── bigquery │ │ │ │ ├── schema.test.ts │ │ │ │ ├── schema │ │ │ │ │ ├── extractors.test.ts │ │ │ │ │ └── processLeafField.test.ts │ │ │ │ └── snapshot.test.ts │ │ │ ├── config │ │ │ │ ├── index.test.ts │ │ │ │ ├── interactive.test.ts │ │ │ │ └── non-interactive.test.ts │ │ │ ├── e2e │ │ │ │ ├── e2e.test.ts │ │ │ │ ├── e2e_mocking.test.ts │ │ │ │ ├── helpers │ │ │ │ │ ├── deleteDatasets.js │ │ │ │ │ └── setup.ts │ │ │ │ └── schemas │ │ │ │ │ ├── arraysNestedInMapsSchema │ │ │ │ │ └── arraysNestedInMapsSchema.json │ │ │ │ │ ├── basic │ │ │ │ │ └── basic.json │ │ │ │ │ ├── mappedArray │ │ │ │ │ └── mappedArray.json │ │ │ │ │ └── nestedMapSchema │ │ │ │ │ └── nestedMapSchema.json │ │ │ ├── fixtures │ │ │ │ ├── schema-files │ │ │ │ │ ├── deep-directory │ │ │ │ │ │ └── 1 │ │ │ │ │ │ │ ├── 2 │ │ │ │ │ │ │ ├── leaf-schemas-1.json │ │ │ │ │ │ │ ├── leaf-schemas-2.json │ │ │ │ │ │ │ ├── leaf-schemas-3.json │ │ │ │ │ │ │ ├── leaf-schemas-4.json │ │ │ │ │ │ │ └── leaf-schemas-5.json │ │ │ │ │ │ │ ├── other-schemas-1.json │ │ │ │ │ │ │ ├── other-schemas-2.json │ │ │ │ │ │ │ ├── other-schemas-3.json │ │ │ │ │ │ │ ├── other-schemas-4.json │ │ │ │ │ │ │ └── other-schemas-5.json │ │ │ │ │ └── full-directory │ │ │ │ │ │ ├── schema-1.json │ │ │ │ │ │ ├── schema-1.txt │ │ │ │ │ │ ├── schema-2.json │ │ │ │ │ │ ├── schema-2.txt │ │ │ │ │ │ ├── schema-3.json │ │ │ │ │ │ ├── schema-3.txt │ │ │ │ │ │ ├── schema-4.json │ │ │ │ │ │ ├── schema-4.txt │ │ │ │ │ │ ├── schema-5.json │ │ │ │ │ │ └── schema-5.txt │ │ │ │ ├── schemas │ │ │ │ │ ├── arraysNestedInMapsSchema.json │ │ │ │ │ ├── columnRename.json │ │ │ │ │ ├── complexSchema.json │ │ │ │ │ ├── emptyMapSchema.json │ │ │ │ │ ├── emptySchema.json │ │ │ │ │ ├── fullSchema.json │ │ │ │ │ ├── fullSchemaSquared.json │ │ │ │ │ ├── jsonSchema.json │ │ │ │ │ ├── nestedMapSchema.json │ │ │ │ │ └── referenceSchema.json │ │ │ │ └── sql │ │ │ │ │ ├── arraysNestedInMapsSchema.sql │ │ │ │ │ ├── changelogColumnRenameSchema.sql │ │ │ │ │ ├── emptyMapSchema.sql │ │ │ │ │ ├── emptySchemaChangeLog.sql │ │ │ │ │ ├── emptySchemaLatest.sql │ │ │ │ │ ├── emptySchemaLatestFromView.sql │ │ │ │ │ ├── fullSchemaChangeLog.sql │ │ │ │ │ ├── fullSchemaLatest.sql │ │ │ │ │ ├── fullSchemaLatestFromView.sql │ │ │ │ │ ├── fullSchemaLatestLatestFromView.sql │ │ │ │ │ ├── fullSchemaSquared.sql │ │ │ │ │ ├── jsonColumn.sql │ │ │ │ │ ├── latestConsistentSnapshot.sql │ │ │ │ │ ├── latestConsistentSnapshotNoGroupBy.sql │ │ │ │ │ ├── nestedMapSchemaChangeLog.sql │ │ │ │ │ ├── referenceSchema.sql │ │ │ │ │ └── viewColumnRenameSchema.sql │ │ │ ├── genkit │ │ │ │ ├── sampleFirestoreDocuments.test.ts │ │ │ │ └── serializeDocument.test.ts │ │ │ └── schema-loader-utils │ │ │ │ ├── readSchemas.test.ts │ │ │ │ └── schema-loader-utils.test.ts │ │ ├── config │ │ │ ├── index.ts │ │ │ ├── interactive.ts │ │ │ └── non-interactive.ts │ │ ├── index.ts │ │ ├── logs.ts │ │ ├── schema-loader-utils.ts │ │ ├── schema │ │ │ ├── extractors.ts │ │ │ ├── genkit.ts │ │ │ ├── index.ts │ │ │ └── processLeafField.ts │ │ ├── snapshot.ts │ │ └── udf.ts │ └── tsconfig.json │ ├── grant-crossproject-access.ps1 │ ├── grant-crossproject-access.sh │ └── import │ ├── .gitignore │ ├── README.md │ ├── __tests__ │ ├── e2e.test.ts │ ├── firebase.json │ ├── getRowsFromDocs.test.ts │ ├── helpers │ │ └── waitFor.ts │ ├── runMultiThread.test.ts │ ├── runMultiThreadMock.test.ts │ ├── test.types.d.ts │ └── tsconfig.json │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── config.ts │ ├── helper.ts │ ├── index.ts │ ├── logs.ts │ ├── program.ts │ ├── run-multi-thread.ts │ ├── run-single-thread.ts │ ├── types.ts │ └── worker.ts │ └── tsconfig.json ├── firestore-counter ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── POSTINSTALL.md ├── PREINSTALL.md ├── README.md ├── clients │ ├── android │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── src │ │ │ └── main │ │ │ ├── AndroidManifest.xml │ │ │ └── java │ │ │ └── com │ │ │ └── firebase │ │ │ └── firestore │ │ │ └── counter │ │ │ └── FirestoreShardedCounter.java │ ├── dart │ │ ├── README.md │ │ ├── analysis_options.yaml │ │ ├── integration_test │ │ │ └── e2e_distributed_counter_test.dart │ │ ├── lib │ │ │ └── distributed_counter.dart │ │ ├── pubspec.lock │ │ ├── pubspec.yaml │ │ ├── test_driver │ │ │ └── integration_test.dart │ │ └── web │ │ │ ├── favicon.png │ │ │ ├── icons │ │ │ ├── Icon-192.png │ │ │ ├── Icon-512.png │ │ │ ├── Icon-maskable-192.png │ │ │ └── Icon-maskable-512.png │ │ │ ├── index.html │ │ │ └── manifest.json │ ├── ios │ │ ├── Package.swift │ │ └── Sources │ │ │ └── FirestoreCounter │ │ │ └── FirestoreCounter.swift │ ├── node │ │ ├── index.js │ │ └── package.json │ └── web │ │ ├── dist │ │ └── sharded-counter.js │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── src │ │ └── index.ts │ │ ├── tsconfig.json │ │ └── webpack.config.js ├── extension.yaml ├── functions │ ├── .gitignore │ ├── __tests__ │ │ ├── aggregator.test.ts │ │ ├── controller.emulator.test.ts │ │ ├── e2e.test.ts │ │ ├── planner.test.ts │ │ ├── test-client.ts │ │ ├── test.types.d.ts │ │ ├── tsconfig.json │ │ └── worker.emulator.test.ts │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── aggregator.ts │ │ ├── common.ts │ │ ├── controller.ts │ │ ├── events.ts │ │ ├── index.ts │ │ ├── planner.ts │ │ └── worker.ts │ └── tsconfig.json └── stress_test │ ├── bin │ ├── driver.ts │ ├── package.json │ └── tsconfig.json │ ├── firebase.json │ └── functions │ ├── package.json │ ├── src │ └── index.ts │ ├── tsconfig.json │ └── tslint.json ├── firestore-send-email ├── .gitignore ├── CHANGELOG.md ├── POSTINSTALL.md ├── PREINSTALL.md ├── README.md ├── extension.yaml ├── functions │ ├── .gitignore │ ├── __tests__ │ │ ├── config.test.ts │ │ ├── createSMTPServer.ts │ │ ├── e2e.test.ts │ │ ├── e2e │ │ │ └── sendgrid.test.ts │ │ ├── functions.test.ts │ │ ├── helpers.test.ts │ │ ├── jest.setup.ts │ │ ├── nodemailer-sendgrid │ │ │ └── index.test.ts │ │ ├── prepare-payload.test.ts │ │ ├── test.types.d.ts │ │ ├── tsconfig.json │ │ └── validation.test.ts │ ├── jest.config.js │ ├── jest.teardown.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── config.ts │ │ ├── events.ts │ │ ├── helpers.ts │ │ ├── index.ts │ │ ├── logs.ts │ │ ├── nodemailer-sendgrid │ │ │ ├── index.ts │ │ │ └── types.ts │ │ ├── prepare-payload.ts │ │ ├── templates.ts │ │ ├── types.ts │ │ └── validation.ts │ └── tsconfig.json └── scripts │ └── oauth2-refresh-token-helper.js ├── firestore-shorten-urls-bitly ├── .gitignore ├── CHANGELOG.md ├── POSTINSTALL.md ├── PREINSTALL.md ├── README.md ├── extension.yaml └── functions │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── abstract-shortener.ts │ ├── config.ts │ ├── events.ts │ ├── index.ts │ └── logs.ts │ └── tsconfig.json ├── firestore-translate-text ├── CHANGELOG.md ├── POSTINSTALL.md ├── PREINSTALL.md ├── README.md ├── extension.yaml └── functions │ ├── .gitignore │ ├── __tests__ │ ├── __snapshots__ │ │ └── config.test.ts.snap │ ├── config.test.ts │ ├── functions.test.ts │ ├── jest.setup.ts │ ├── mocks │ │ ├── firestore.ts │ │ └── translate.ts │ ├── test.types.d.ts │ ├── tsconfig.json │ └── unit │ │ └── translateMultipleBackfill.test.ts │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── config.ts │ ├── events.ts │ ├── index.ts │ ├── logs │ │ ├── index.ts │ │ └── messages.ts │ ├── translate │ │ ├── common.ts │ │ ├── index.ts │ │ ├── translateDocument.ts │ │ ├── translateMultiple.ts │ │ └── translateSingle.ts │ └── validators.ts │ └── tsconfig.json ├── jest.config.js ├── lerna.json ├── package-lock.json ├── package.json ├── rtdb-limit-child-nodes ├── CHANGELOG.md ├── POSTINSTALL.md ├── PREINSTALL.md ├── README.md ├── extension.yaml └── functions │ ├── .gitignore │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── config.ts │ ├── index.ts │ └── logs.ts │ └── tsconfig.json ├── samples └── rtdb-uppercase-messages │ ├── CHANGELOG.md │ ├── POSTINSTALL.md │ ├── PREINSTALL.md │ ├── extension.yaml │ ├── functions │ ├── .gitignore │ ├── index.js │ ├── integration-tests │ │ ├── .firebaserc │ │ ├── .gitignore │ │ ├── extensions │ │ │ ├── .gitignore │ │ │ └── rtdb-uppercase-messages.env │ │ ├── firebase.json │ │ └── functions │ │ │ ├── .gitignore │ │ │ ├── index.js │ │ │ ├── package-lock.json │ │ │ └── package.json │ ├── package-lock.json │ └── package.json │ └── rtdb-seed-data.json ├── scripts └── publish.sh ├── storage-resize-images ├── CHANGELOG.md ├── POSTINSTALL.md ├── PREINSTALL.md ├── README.md ├── extension.yaml └── functions │ ├── .gitignore │ ├── __tests__ │ ├── __mocks__ │ │ └── src │ │ │ ├── config.ts │ │ │ └── index.ts │ ├── __snapshots__ │ │ └── config.test.ts.snap │ ├── config.test.ts │ ├── content-filter.test.ts │ ├── convert-image.test.ts │ ├── e2e.test.ts │ ├── filters.test.ts │ ├── fixtures │ │ ├── config.json │ │ ├── config_hack.txt │ │ └── settings.json │ ├── function.test.ts │ ├── gun-image.png │ ├── not-an-image.jpeg │ ├── resize.test.ts │ ├── retry-queue.test.ts │ ├── storage.rules │ ├── test-image.gif │ ├── test-image.jpeg │ ├── test-image.png │ ├── test-img.jfif │ ├── test-jpg.jpg │ ├── test.types.d.ts │ ├── tsconfig.json │ ├── unit │ │ └── modifyImage.test.ts │ ├── util.test.ts │ ├── util.ts │ └── vulnerability.test.ts │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── config.ts │ ├── content-filter.ts │ ├── events.ts │ ├── file-operations.ts │ ├── filters.ts │ ├── global.ts │ ├── index.ts │ ├── logs.ts │ ├── placeholder.png │ ├── resize-image.ts │ └── util.ts │ └── tsconfig.json └── tsconfig.json /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Dependencies, package.json and mono-repo files 2 | package.json @firebase/invertase 3 | */package.json @firebase/invertase 4 | lerna.json @firebase/invertase 5 | 6 | # Tests 7 | jest.config.js @firebase/invertase 8 | */jest.config.js @firebase/invertase 9 | __tests__/* @firebase/invertase 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: ⚠️ Report a Bug 3 | about: Think you found a bug in an extension? Report it here. 4 | title: "\U0001F41B [EXTENSION_NAME_HERE] Your issue title here" 5 | labels: "type: bug" 6 | --- 7 | 8 | 12 | 13 | ### [READ] Step 1: Are you in the right place? 14 | 15 | Issues filed here should be about bugs for a **specific extension in this repository**. 16 | If you have a general question, need help debugging, or fall into some 17 | other category use one of these other channels: 18 | 19 | - For general technical questions, post a question on [StackOverflow](http://stackoverflow.com/) 20 | with the firebase tag. 21 | - For general Firebase discussion, use the [firebase-talk](https://groups.google.com/forum/#!forum/firebase-talk) 22 | google group. 23 | - To file a bug against the Firebase Extensions platform, or for an issue affecting multiple extensions, please reach out to 24 | [Firebase support](https://firebase.google.com/support/troubleshooter/contact/) directly. 25 | 26 | ### [REQUIRED] Step 2: Describe your configuration 27 | 28 | - Extension name: **\_** (`storage-resize-images`, `firestore-send-email`, etc) 29 | - Extension version: **\_** 30 | - Configuration values (redact info where appropriate): 31 | - **\_** 32 | - **\_** 33 | 34 | ### [REQUIRED] Step 3: Describe the problem 35 | 36 | #### Steps to reproduce: 37 | 38 | What happened? How can we make the problem occur? 39 | This could be a description, [log/console output](https://firebase.google.com/docs/extensions/manage-installed-extensions#view-logs), etc. 40 | 41 | ##### Expected result 42 | 43 | ##### Actual result 44 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/fr.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 💡 Feature Request 3 | about: Have a feature you'd like to see in an extension? Request it here. 4 | title: "\U0001F41B [EXTENSION_NAME_HERE] Your request title here" 5 | labels: "type: feature request" 6 | --- 7 | 8 | 12 | 13 | ### [READ] Step 1: Are you in the right place? 14 | 15 | Issues filed here should be about a feature request for a **specific extension in this repository**. To file a feature request that affects multiple extensions or the Firebase Extensions platform, please reach out to 16 | [Firebase support](https://firebase.google.com/support/troubleshooter/report/features/) directly. 17 | 18 | ### [REQUIRED] Step 2: Extension name 19 | 20 | This feature request is for extension: **\_** (`storage-resize-images`, `firestore-send-email`, etc) 21 | 22 | ### What feature would you like to see? 23 | 24 | Describe the feature you would like to add, or how you'd like to see the extension change. 25 | 26 | ### How would you use it? 27 | 28 | Tell us how you'd use this feature in your app. 29 | -------------------------------------------------------------------------------- /.github/workflows/npm_publish_bq_scripts.yml: -------------------------------------------------------------------------------- 1 | name: Publish npm package 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | package_name: 7 | description: "The package name to publish" 8 | required: true 9 | default: "@firebaseextensions/fs-bq-schema-views" 10 | jobs: 11 | publish_if_newer_version: 12 | runs-on: ubuntu-latest 13 | name: publish_if_newer_version 14 | steps: 15 | - uses: actions/checkout@v3 16 | - name: Setup node 17 | uses: actions/setup-node@v3 18 | with: 19 | node-version: 20 20 | - name: NPM install 21 | run: npm install 22 | - name: Publish BigQuery Schema Views 23 | if: 24 | ${{ github.event.inputs.package_name == 25 | '@firebaseextensions/fs-bq-schema-views'}} 26 | env: 27 | NPM_TOKEN: ${{ secrets.NPM_TOKEN_BQ_SCHEMA_VIEWS }} 28 | run: | 29 | cd firestore-bigquery-export/scripts/gen-schema-view 30 | ${{ github.workspace }}/.github/workflows/scripts/npm_publish.sh 31 | - name: Publish BigQuery Import Collection 32 | if: 33 | ${{ github.event.inputs.package_name == 34 | '@firebaseextensions/fs-bq-import-collection'}} 35 | env: 36 | NPM_TOKEN: ${{ secrets.NPM_TOKEN_BQ_IMPORT_COLLECTION }} 37 | run: | 38 | cd firestore-bigquery-export/scripts/import 39 | ${{ github.workspace }}/.github/workflows/scripts/npm_publish.sh 40 | - name: Publish BigQuery Change Tracker 41 | if: 42 | ${{ github.event.inputs.package_name == 43 | '@firebaseextensions/firestore-bigquery-change-tracker'}} 44 | env: 45 | NPM_TOKEN: ${{ secrets.NPM_TOKEN_BQ_CHANGE_TRACKER }} 46 | run: | 47 | cd firestore-bigquery-export/firestore-bigquery-change-tracker 48 | ${{ github.workspace }}/.github/workflows/scripts/npm_publish.sh 49 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | release: 10 | name: "Create Releases" 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Release Script 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | run: | 18 | ./.github/workflows/scripts/release.sh 19 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Testing 2 | 3 | on: 4 | push: 5 | branches: [next] 6 | pull_request: 7 | branches: ["**"] 8 | 9 | jobs: 10 | nodejs: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | node: ["20"] 15 | name: node.js_${{ matrix.node }}_test 16 | steps: 17 | - uses: actions/checkout@v3 18 | - name: Setup node 19 | uses: actions/setup-node@v3 20 | with: 21 | node-version: ${{ matrix.node }} 22 | cache: "npm" 23 | cache-dependency-path: "**/package-lock.json" 24 | - name: npm install 25 | run: npm i 26 | - name: Build emulator functions 27 | run: cd _emulator/functions && npm i && npm run build & cd ../.. 28 | - name: Install Firebase CLI 29 | uses: nick-invision/retry@v1 30 | with: 31 | timeout_minutes: 10 32 | retry_wait_seconds: 60 33 | max_attempts: 3 34 | command: npm i -g firebase-tools@14 35 | - name: Setup e2e secrets 36 | run: | 37 | echo SMTP_PASSWORD=${{ secrets.SENDGRID_API_KEY }} >> _emulator/extensions/firestore-send-email-sendgrid.secret.local 38 | - name: npm test 39 | run: npm run test:ci 40 | -------------------------------------------------------------------------------- /.github/workflows/validate.yml: -------------------------------------------------------------------------------- 1 | name: Validate 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - "**" 7 | 8 | jobs: 9 | formatting: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - name: Setup node 14 | uses: actions/setup-node@v3 15 | with: 16 | node-version: 20 17 | - name: NPM install 18 | run: SKIP_POSTINSTALL=yes npm i 19 | - name: Prettier Lint Check 20 | run: npm run lint 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | node_modules/ 3 | *.env 4 | !emulator-params.env 5 | !install-params.env 6 | firebase-debug.log 7 | database-debug.log 8 | ui-debug.log 9 | .DS_Store 10 | mods-test-data/key.json 11 | .gradle 12 | .idea 13 | build 14 | *.iml 15 | local.properties 16 | coverage 17 | yarn.lock 18 | _emulator/pubsub-debug.log 19 | firestore-debug.log 20 | pubsub-debug.log 21 | tmp/ 22 | fixtures/ -------------------------------------------------------------------------------- /.husky/.gitignore: -------------------------------------------------------------------------------- 1 | _ 2 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | npx lint-staged 5 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | package.json 2 | package-lock.json 3 | 4 | **/node_modules/** 5 | 6 | # generated files 7 | README.md 8 | **/functions/lib/** 9 | **/lib/** 10 | **/dist/** 11 | coverage 12 | 13 | # extension install md files 14 | # - excluded as prettier escapes variables e.g. `${PROJECT_ID}` becomes `\${PROJECT_ID}` 15 | POSTINSTALL.md 16 | PREINSTALL.md 17 | -------------------------------------------------------------------------------- /.prettierrc.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | trailingComma: es5 16 | arrowParens: always 17 | overrides: 18 | - files: "*.{yml,yaml}" 19 | options: 20 | proseWrap: always 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Firebase Extensions 2 | 3 | This repository contains the source for Firebase Extensions. Created and tested by Firebase, these official Firebase extensions are reliable and secure. 4 | 5 | To learn more about Firebase Extensions, including how to install them in your Firebase projects, visit the [Firebase documentation](https://firebase.google.com/docs/extensions). 6 | 7 | Each directory in this repo contains the source code for the extension and a README to explain how the extension works, including information about the APIs enabled, resources created, and the access granted to the extension. 8 | 9 | When you find an extension that solves a need for your app or project, all you do is install and configure the extension. With extensions, you don't spend time researching, writing, and debugging the code that implements functionality or automates a task for your app or project. 10 | 11 | You can also browse official Firebase extensions from the following sources: 12 | 13 | * [Firebase Extensions product page](https://firebase.google.com/products/extensions) 14 | * [Firebase Extensions dashboard](https://console.firebase.google.com/project/_/extensions/) in the Firebase console 15 | You can also browse official Firebase extensions on the [Extensions Marketplace](https://extensions.dev). 16 | 17 | ## Documentation 18 | 19 | Documentation for the [Extensions by Firebase](https://firebase.google.com/docs/extensions) section are now stored in this repository. 20 | 21 | They can be found under [Docs](https://github.com/firebase/extensions/tree/master/docs) 22 | -------------------------------------------------------------------------------- /_emulator/.firebaserc: -------------------------------------------------------------------------------- 1 | { 2 | "projects": { 3 | "default": "demo-test" 4 | }, 5 | "targets": {}, 6 | "etags": { 7 | "dev-extensions-testing": { 8 | "extensionInstances": { 9 | "firestore-bigquery-export": "02acbd8b443b9635716d52d65758a78db1e51140191caecaaf60d932d314a62a" 10 | } 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /_emulator/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | firebase-debug.log* 8 | firebase-debug.*.log* 9 | 10 | # Firebase cache 11 | .firebase/ 12 | 13 | # Firebase config 14 | 15 | # Uncomment this if you'd like others to create their own Firebase project. 16 | # For a team working on the same Firebase project(s), it is recommended to leave 17 | # it commented so all members can deploy to the same project(s) in .firebaserc. 18 | # .firebaserc 19 | 20 | # Runtime data 21 | pids 22 | *.pid 23 | *.seed 24 | *.pid.lock 25 | 26 | # Directory for instrumented libs generated by jscoverage/JSCover 27 | lib-cov 28 | 29 | # Coverage directory used by tools like istanbul 30 | coverage 31 | 32 | # nyc test coverage 33 | .nyc_output 34 | 35 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 36 | .grunt 37 | 38 | # Bower dependency directory (https://bower.io/) 39 | bower_components 40 | 41 | # node-waf configuration 42 | .lock-wscript 43 | 44 | # Compiled binary addons (http://nodejs.org/api/addons.html) 45 | build/Release 46 | 47 | # Dependency directories 48 | node_modules/ 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Optional REPL history 57 | .node_repl_history 58 | 59 | # Output of 'npm pack' 60 | *.tgz 61 | 62 | # Yarn Integrity file 63 | .yarn-integrity 64 | 65 | # dotenv environment variables file 66 | .env 67 | -------------------------------------------------------------------------------- /_emulator/extensions/delete-user-data.env.local: -------------------------------------------------------------------------------- 1 | LOCATION=europe-west2 2 | FIRESTORE_QUERY_COLLECTION=queries 3 | ENABLE_AUTO_DISCOVERY=true 4 | AUTO_DISCOVERY_SEARCH_FIELDS=field1,field2 5 | SEARCH_FUNCTION=http://127.0.0.1:5001/demo-test/us-central1/findDocumentReferences 6 | AUTO_DISCOVERY_TOPIC=discovery 7 | AUTO_DISCOVERY_SEARCH_DEPTH=4 8 | DELETION_TOPIC=deletions 9 | PROJECT_ID=demo-test 10 | EXT_INSTANCE_ID = "demo-ext" 11 | -------------------------------------------------------------------------------- /_emulator/extensions/firestore-bigquery-export.env.local: -------------------------------------------------------------------------------- 1 | BIGQUERY_PROJECT_ID=dev-extensions-testing 2 | DATABASE_ID=(default) 3 | COLLECTION_PATH=posts 4 | DATASET_ID=firestore_export 5 | DATASET_LOCATION=us-central1 6 | firebaseextensions.v1beta.function/location=us-central1 7 | TABLE_ID=bq_e2e_test 8 | TABLE_PARTITIONING=HOUR 9 | TIME_PARTITIONING_FIELD_TYPE=omit 10 | USE_NEW_SNAPSHOT_QUERY_SYNTAX=yes 11 | WILDCARD_IDS=false -------------------------------------------------------------------------------- /_emulator/extensions/firestore-counter.env.local: -------------------------------------------------------------------------------- 1 | LOCATION=europe-west2 2 | PROJECT_ID=demo-test 3 | INTERNAL_STATE_PATH=_firebase_ext_/sharded_counter 4 | SCHEDULE_FREQUENCY=1 5 | INTERNAL_STATE_PATH=_firebase_ext_/sharded_counter 6 | -------------------------------------------------------------------------------- /_emulator/extensions/firestore-send-email-sendgrid.env.local: -------------------------------------------------------------------------------- 1 | DEFAULT_FROM=test-assertion@email.com 2 | firebaseextensions.v1beta.function/location=us-central1 3 | MAIL_COLLECTION=mail-sg 4 | SMTP_CONNECTION_URI=smtps://apikey@smtp.sendgrid.net:465 5 | TTL_EXPIRE_TYPE=never 6 | TTL_EXPIRE_VALUE=1 -------------------------------------------------------------------------------- /_emulator/extensions/firestore-send-email-sendgrid.secret.local: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/_emulator/extensions/firestore-send-email-sendgrid.secret.local -------------------------------------------------------------------------------- /_emulator/extensions/firestore-send-email.env.local: -------------------------------------------------------------------------------- 1 | LOCATION=europe-west2 2 | TEMPLATES_COLLECTION=templates 3 | MAIL_COLLECTION=mail 4 | SMTP_CONNECTION_URI=smtps://fakeemail@gmail.com:secret-password@smtp.gmail.com:465 5 | SMTP_PASSWORD=secret-password 6 | DEFAULT_FROM=fakeemail@gmail.com 7 | DEFAULT_REPLY_TO=fakeemail@gmail.com 8 | TESTING=true 9 | TTL_EXPIRE_TYPE=day 10 | TTL_EXPIRE_VALUE=5 11 | TLS_OPTIONS={} -------------------------------------------------------------------------------- /_emulator/extensions/firestore-send-email.secret.local: -------------------------------------------------------------------------------- 1 | LOCATION=europe-west2 2 | TEMPLATES_COLLECTION=templates 3 | MAIL_COLLECTION=mail 4 | SMTP_CONNECTION_URI=smtps://fakeemail@gmail.com:secret-password@smtp.gmail.com:465 5 | SMTP_PASSWORD=secret-password 6 | DEFAULT_FROM=fakeemail@gmail.com 7 | DEFAULT_REPLY_TO=fakeemail@gmail.com 8 | TESTING=true 9 | FIRESTORE_EXPIRE_AT=day -------------------------------------------------------------------------------- /_emulator/extensions/firestore-translate-text.env.local: -------------------------------------------------------------------------------- 1 | LOCATION=europe-west2 2 | LANGUAGES=en,es 3 | INPUT_FIELD_NAME=input 4 | OUTPUT_FIELD_NAME=output 5 | DO_BACKFILL=false 6 | COLLECTION_PATH=translations -------------------------------------------------------------------------------- /_emulator/extensions/storage-resize-images.env.local: -------------------------------------------------------------------------------- 1 | LOCATION=europe-west2 2 | IMG_BUCKET=${STORAGE_BUCKET} 3 | IMG_SIZES=300x300 4 | DELETE_ORIGINAL_FILE=true 5 | MAKE_PUBLIC=true 6 | RESIZED_IMAGES_PATH=thumbnails 7 | FAILED_IMAGES_PATH=failed 8 | IMAGE_TYPE=webp 9 | IS_ANIMATED=true 10 | DO_BACKFILL=false 11 | REGENERATE_TOKEN=true 12 | SHARP_OPTIONS='{"fit":"cover", "position": "top", "animated": false}' -------------------------------------------------------------------------------- /_emulator/firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "extensions": { 3 | "firestore-send-email": "../firestore-send-email", 4 | "delete-user-data": "../delete-user-data", 5 | "storage-resize-images": "../storage-resize-images", 6 | "firestore-counter": "../firestore-counter", 7 | "firestore-bigquery-export": "../firestore-bigquery-export", 8 | "firestore-send-email-sendgrid": "../firestore-send-email" 9 | }, 10 | "storage": { 11 | "rules": "storage.rules" 12 | }, 13 | "emulators": { 14 | "hub": { 15 | "port": 4000 16 | }, 17 | "storage": { 18 | "port": 9199 19 | }, 20 | "auth": { 21 | "port": 9099 22 | }, 23 | "pubsub": { 24 | "port": 8085 25 | }, 26 | "functions": { 27 | "port": 5001 28 | }, 29 | "ui": { 30 | "enabled": true 31 | }, 32 | "firestore": { 33 | "host": "127.0.0.1", 34 | "port": 8080 35 | } 36 | }, 37 | "functions": { 38 | "port": 5002, 39 | "source": "functions" 40 | }, 41 | "firestore": { 42 | "rules": "firestore.rules", 43 | "indexes": "firestore.indexes.json" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /_emulator/firestore.indexes.json: -------------------------------------------------------------------------------- 1 | { 2 | "indexes": [], 3 | "fieldOverrides": [] 4 | } 5 | -------------------------------------------------------------------------------- /_emulator/firestore.rules: -------------------------------------------------------------------------------- 1 | rules_version = '2'; 2 | service cloud.firestore { 3 | match /databases/{database}/documents { 4 | match /{document=**} { 5 | // This rule allows anyone with your database reference to view, edit, 6 | // and delete all data in your database. It is useful for getting 7 | // started, but it is configured to expire after 30 days because it 8 | // leaves your app open to attackers. At that time, all client 9 | // requests to your database will be denied. 10 | // 11 | // Make sure to write security rules for your app before that time, or 12 | // else all client requests to your database will be denied until you 13 | // update your rules. 14 | allow read, write; 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /_emulator/functions/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled JavaScript files 2 | lib/**/*.js 3 | lib/**/*.js.map 4 | 5 | # TypeScript v1 declaration files 6 | typings/ 7 | 8 | # Node.js dependency directory 9 | node_modules/ 10 | -------------------------------------------------------------------------------- /_emulator/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "functions", 3 | "scripts": { 4 | "build": "tsc", 5 | "build:watch": "tsc --watch", 6 | "serve": "npm run build && firebase emulators:start --only functions", 7 | "shell": "npm run build && firebase functions:shell", 8 | "start": "npm run shell", 9 | "deploy": "firebase deploy --only functions", 10 | "logs": "firebase functions:log" 11 | }, 12 | "engines": { 13 | "node": "18" 14 | }, 15 | "main": "lib/index.js", 16 | "dependencies": { 17 | "@types/express-serve-static-core": "4.17.30", 18 | "firebase-admin": "^12.1.0", 19 | "firebase-functions": "^4.9.0" 20 | }, 21 | "devDependencies": { 22 | "typescript": "^4.6.4" 23 | }, 24 | "private": true 25 | } 26 | -------------------------------------------------------------------------------- /_emulator/functions/src/index.ts: -------------------------------------------------------------------------------- 1 | import * as functions from "firebase-functions"; 2 | 3 | export const findDocumentReferences = functions.https.onRequest( 4 | (request, response) => { 5 | /** Simply send back the data that we have posted */ 6 | response.send(["searchFunction/testing/functions-testing/example"]); 7 | } 8 | ); 9 | -------------------------------------------------------------------------------- /_emulator/functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "noImplicitReturns": true, 5 | "noUnusedLocals": true, 6 | "outDir": "lib", 7 | "sourceMap": true, 8 | "strict": true, 9 | "target": "es2017" 10 | }, 11 | "compileOnSave": true, 12 | "include": ["src"] 13 | } 14 | -------------------------------------------------------------------------------- /_emulator/storage.rules: -------------------------------------------------------------------------------- 1 | rules_version = '2'; 2 | service firebase.storage { 3 | match /b/{bucket}/o { 4 | match /{allPaths=**} { 5 | allow read, write: if true; 6 | } 7 | } 8 | } -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | precision: 2 3 | round: up 4 | range: "35...100" 5 | 6 | status: 7 | project: 8 | default: true 9 | patch: off 10 | 11 | comment: 12 | layout: "reach, diff, files" 13 | behavior: once 14 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { extends: ["@commitlint/config-conventional"] }; 2 | -------------------------------------------------------------------------------- /delete-user-data/POSTINSTALL.md: -------------------------------------------------------------------------------- 1 | ### See it in action 2 | 3 | You can test out this extension right away! 4 | 5 | 1. Go to your [Authentication dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/authentication/users) in the Firebase console. 6 | 7 | 1. Click **Add User** to add a test user, then copy the test user's UID to your clipboard. 8 | 9 | 1. Create a new Cloud Firestore document, a new Realtime Database entry, or upload a new file to Storage - incorporating the user's UID into the path according to the schema that you configured. 10 | 11 | 1. Go back to your [Authentication dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/authentication/users), then delete the test user. 12 | 13 | 1. In a few seconds, the new data you added above will be deleted from Cloud Firestore, Realtime Database, and/or Storage (depending on what you configured). 14 | 15 | ### Monitoring 16 | 17 | As a best practice, you can [monitor the activity](https://firebase.google.com/docs/extensions/manage-installed-extensions#monitor) of your installed extension, including checks on its health, usage, and logs. 18 | -------------------------------------------------------------------------------- /delete-user-data/functions/.gitignore: -------------------------------------------------------------------------------- 1 | lib -------------------------------------------------------------------------------- /delete-user-data/functions/__tests__/helpers/setupEnvironment.ts: -------------------------------------------------------------------------------- 1 | export default () => { 2 | process.env.FIRESTORE_EMULATOR_HOST = "127.0.0.1:8080"; 3 | process.env.FIREBASE_FIRESTORE_EMULATOR_ADDRESS = "127.0.0.1:8080"; 4 | process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; 5 | process.env.PUBSUB_EMULATOR_HOST = "127.0.0.1:8085"; 6 | process.env.GOOGLE_CLOUD_PROJECT = "demo-test"; 7 | }; 8 | -------------------------------------------------------------------------------- /delete-user-data/functions/__tests__/recursiveDelete.test.ts: -------------------------------------------------------------------------------- 1 | // Import your function and any necessary Firebase modules 2 | import { recursiveDelete } from "../src/recursiveDelete"; // Update with your actual file path 3 | import * as admin from "firebase-admin"; 4 | 5 | const bulkWriterMock = () => ({ 6 | onWriteError: jest.fn(), 7 | close: jest.fn(() => Promise.resolve()), 8 | }); 9 | // Mock admin and firestore 10 | 11 | admin.initializeApp(); 12 | 13 | describe("recursiveDelete", () => { 14 | // Common setup 15 | const db = admin.firestore(); 16 | 17 | test("successfully deletes a document reference", async () => { 18 | const ref = "documents/doc1"; 19 | db.doc(ref).create({ 20 | foo: "bar", 21 | }); 22 | 23 | await recursiveDelete(ref); 24 | 25 | const doc = db.doc(ref); 26 | await doc.get().then((doc) => { 27 | expect(doc.exists).toBe(false); 28 | }); 29 | }); 30 | 31 | test("successfully deletes a collection reference", async () => { 32 | const ref = "documents/doc1/collection1"; 33 | db.collection(ref).add({ 34 | foo: "bar", 35 | }); 36 | 37 | await recursiveDelete(ref); 38 | 39 | const collection = db.collection(ref); 40 | await collection.get().then((collection) => { 41 | expect(collection.docs.length).toBe(0); 42 | }); 43 | }); 44 | 45 | test("successfully deletes a document with a subcollection", async () => { 46 | const parentRef = "documents/doc1"; 47 | const ref = "documents/doc1/collection1/doc2/collection2"; 48 | db.collection(ref).add({ 49 | foo: "bar", 50 | }); 51 | 52 | await recursiveDelete(parentRef); 53 | 54 | const collection = db.collection(ref); 55 | await collection.get().then((collection) => { 56 | expect(collection.docs.length).toBe(0); 57 | }); 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /delete-user-data/functions/__tests__/runBatchPubSubDeletions.test.ts: -------------------------------------------------------------------------------- 1 | import * as admin from "firebase-admin"; 2 | import { runBatchPubSubDeletions } from "../src/runBatchPubSubDeletions"; 3 | import setupEnvironment from "./helpers/setupEnvironment"; 4 | 5 | import { Paths } from "../src/types"; 6 | 7 | admin.initializeApp(); 8 | setupEnvironment(); 9 | 10 | const db = admin.firestore(); 11 | 12 | const generateTopLevelUserCollection = async (name) => { 13 | const collection = db.collection(name); 14 | 15 | return collection; 16 | }; 17 | 18 | describe("runBatchPubSubDeletions", () => { 19 | let rootCollection: admin.firestore.CollectionReference; 20 | 21 | beforeEach(async () => { 22 | rootCollection = await generateTopLevelUserCollection( 23 | "runBatchPubSubDeletions" 24 | ); 25 | }); 26 | 27 | test("cannot delete paths with an invalid userId", async () => { 28 | /** Add a new document for testing */ 29 | const doc = await rootCollection.add({ testing: "testing" }); 30 | const invalidUserId = "invalidUserId"; 31 | 32 | const paths: Paths = { firestorePaths: [`${rootCollection.id}/${doc.id}`] }; 33 | 34 | /** Run deletion */ 35 | await runBatchPubSubDeletions(paths, invalidUserId); 36 | 37 | /** Wait 10 seconds */ 38 | await new Promise((resolve) => setTimeout(resolve, 10000)); 39 | 40 | /** Check document still exist */ 41 | const documentCheck = await doc.get(); 42 | expect(documentCheck.exists).toBe(true); 43 | }, 60000); 44 | }); 45 | -------------------------------------------------------------------------------- /delete-user-data/functions/__tests__/searchFunction.test.ts: -------------------------------------------------------------------------------- 1 | import * as admin from "firebase-admin"; 2 | import { UserRecord } from "firebase-functions/v1/auth"; 3 | import { createFirebaseUser, waitForDocumentDeletion } from "./helpers"; 4 | import setupEnvironment from "../__tests__/helpers/setupEnvironment"; 5 | 6 | setupEnvironment(); 7 | 8 | admin.initializeApp(); 9 | const auth = admin.auth(); 10 | const db = admin.firestore(); 11 | 12 | describe("search", () => { 13 | let user: UserRecord; 14 | 15 | beforeEach(async () => { 16 | user = await createFirebaseUser(); 17 | }); 18 | 19 | test("can delete a single document", async () => { 20 | await admin 21 | .firestore() 22 | .collection("searchFunction") 23 | .doc("testing") 24 | .collection("functions-testing") 25 | .doc("example") 26 | .set({ functions: "testing" }); 27 | 28 | const doc = db.doc("functions/functions-testing"); 29 | 30 | await auth.deleteUser(user.uid); 31 | 32 | await waitForDocumentDeletion(doc); 33 | }, 12000); 34 | }); 35 | -------------------------------------------------------------------------------- /delete-user-data/functions/__tests__/setupTests.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | 3 | (async function () { 4 | const $ = path.resolve( 5 | __dirname, 6 | `../../../_emulator/extensions/delete-user-data.env.local` 7 | ); 8 | 9 | require("dotenv").config({ 10 | path: path.resolve( 11 | __dirname, 12 | `../../../_emulator/extensions/delete-user-data.env.local` 13 | ), 14 | }); 15 | })(); 16 | -------------------------------------------------------------------------------- /delete-user-data/functions/__tests__/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "lib", 5 | "target": "ES2020" 6 | }, 7 | "include": ["**/*"] 8 | } 9 | -------------------------------------------------------------------------------- /delete-user-data/functions/jest.config.js: -------------------------------------------------------------------------------- 1 | const packageJson = require("./package.json"); 2 | 3 | module.exports = { 4 | name: packageJson.name, 5 | displayName: packageJson.name, 6 | testEnvironment: "node", 7 | preset: "ts-jest", 8 | testMatch: ["**/__tests__/*.test.ts"], 9 | setupFilesAfterEnv: ["/__tests__/setupTests.ts"], 10 | moduleNameMapper: { 11 | "firebase-admin/firestore": 12 | "/node_modules/firebase-admin/lib/firestore", 13 | "firebase-admin/auth": "/node_modules/firebase-admin/lib/auth", 14 | }, 15 | snapshotFormat: { 16 | escapeString: true, 17 | printBasicPrototype: true, 18 | }, 19 | }; 20 | -------------------------------------------------------------------------------- /delete-user-data/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "delete-user-data-functions", 3 | "description": "Automatically delete a user's data when they delete their account.", 4 | "main": "lib/index.js", 5 | "scripts": { 6 | "prepare": "npm run build", 7 | "build": "npm run clean && npm run compile", 8 | "build:watch": "npm run clean && tsc --watch", 9 | "clean": "rimraf lib", 10 | "compile": "tsc", 11 | "local:emulator": "cd ../../_emulator && firebase emulators:start -P demo-test", 12 | "test": "cd ../../_emulator && firebase emulators:exec jest -P demo-test", 13 | "test:local": "cd ../../_emulator && firebase emulators:exec \"CI_TEST=true jest --detectOpenHandles --verbose --forceExit --testMatch **/delete-user-data/**/*.test.ts\"", 14 | "test:watch": "concurrently \"npm run local:emulator\" \"jest --watch\"", 15 | "generate-readme": "firebase ext:info .. --markdown > ../README.md", 16 | "test:emulator-running": "jest" 17 | }, 18 | "author": "Lauren Long ", 19 | "license": "Apache-2.0", 20 | "dependencies": { 21 | "@google-cloud/pubsub": "^4.3.3", 22 | "@types/express-serve-static-core": "4.17.24", 23 | "@types/node": "^16.18.34", 24 | "concurrently": "^7.2.1", 25 | "firebase-admin": "^12.1.0", 26 | "firebase-functions": "^4.9.0", 27 | "lodash.chunk": "^4.2.0", 28 | "node-fetch": "^2.6.2", 29 | "rimraf": "^2.6.3", 30 | "typescript": "^4.9.4", 31 | "@types/jest": "29.5.0", 32 | "jest": "29.5.0", 33 | "ts-jest": "29.1.2" 34 | }, 35 | "engines": { 36 | "node": "18" 37 | }, 38 | "private": true, 39 | "devDependencies": { 40 | "@types/lodash.chunk": "^4.2.7", 41 | "@types/node-fetch": "^2.6.2", 42 | "concurrency": "^0.1.4", 43 | "dotenv": "^16.0.2", 44 | "firebase-functions-test": "^3.2.0", 45 | "wait-port": "^0.2.9" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /delete-user-data/functions/src/config.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export default { 18 | location: process.env.LOCATION, 19 | firestorePaths: process.env.FIRESTORE_PATHS, 20 | firestoreDeleteMode: process.env.FIRESTORE_DELETE_MODE, 21 | rtdbPaths: process.env.RTDB_PATHS, 22 | storagePaths: process.env.STORAGE_PATHS, 23 | enableSearch: process.env.ENABLE_AUTO_DISCOVERY === "yes", 24 | storageBucketDefault: 25 | process.env.CLOUD_STORAGE_BUCKET || process.env.STORAGE_BUCKET, 26 | selectedDatabaseInstance: process.env.SELECTED_DATABASE_INSTANCE, 27 | selectedDatabaseLocation: process.env.SELECTED_DATABASE_LOCATION, 28 | searchFields: process.env.AUTO_DISCOVERY_SEARCH_FIELDS || "", 29 | searchFunction: process.env.SEARCH_FUNCTION, 30 | discoveryTopic: `ext-${process.env.EXT_INSTANCE_ID}-discovery`, 31 | deletionTopic: `ext-${process.env.EXT_INSTANCE_ID}-deletion`, 32 | searchDepth: process.env.AUTO_DISCOVERY_SEARCH_DEPTH 33 | ? parseInt(process.env.AUTO_DISCOVERY_SEARCH_DEPTH) 34 | : 3, 35 | }; 36 | -------------------------------------------------------------------------------- /delete-user-data/functions/src/recursiveDelete.ts: -------------------------------------------------------------------------------- 1 | import * as admin from "firebase-admin"; 2 | 3 | const MAX_RETRY_ATTEMPTS = 3; 4 | 5 | export const recursiveDelete = async (path: string) => { 6 | const db = admin.firestore(); 7 | // Recursively delete a reference and log the references of failures. 8 | const bulkWriter = db.bulkWriter(); 9 | 10 | bulkWriter.onWriteError((error) => { 11 | if (error.failedAttempts < MAX_RETRY_ATTEMPTS) { 12 | return true; 13 | } else { 14 | console.warn("Failed to delete document: ", error.documentRef.path); 15 | return false; 16 | } 17 | }); 18 | 19 | const isDocument = path.split("/").length % 2 === 0; 20 | 21 | const reference = isDocument ? db.doc(path) : db.collection(path); 22 | 23 | await db.recursiveDelete(reference, bulkWriter); 24 | }; 25 | -------------------------------------------------------------------------------- /delete-user-data/functions/src/runBatchPubSubDeletions.ts: -------------------------------------------------------------------------------- 1 | import chunk from "lodash.chunk"; 2 | const { PubSub } = require("@google-cloud/pubsub"); 3 | 4 | import * as config from "./config"; 5 | 6 | type Paths = { 7 | firestorePaths: string[]; 8 | }; 9 | 10 | export async function runBatchPubSubDeletions(paths: Paths, uid: string) { 11 | /** Define pubsub */ 12 | const pubsub = new PubSub(); 13 | 14 | const { firestorePaths } = paths; 15 | 16 | if (!firestorePaths || !Array.isArray(firestorePaths)) { 17 | return; 18 | } 19 | 20 | if (firestorePaths.length === 0) { 21 | return; 22 | } 23 | 24 | /** Define batch array variables */ 25 | for await (const chunkedPaths of chunk(firestorePaths, 450)) { 26 | const topic = pubsub.topic( 27 | `projects/${ 28 | process.env.GOOGLE_CLOUD_PROJECT || process.env.PROJECT_ID 29 | }/topics/${config.default.deletionTopic}` 30 | ); 31 | await topic.publish( 32 | Buffer.from(JSON.stringify({ paths: chunkedPaths, uid })) 33 | ); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /delete-user-data/functions/src/runCustomSearchFunction.ts: -------------------------------------------------------------------------------- 1 | import fetch from "node-fetch"; 2 | import { runBatchPubSubDeletions } from "./runBatchPubSubDeletions"; 3 | import * as logs from "./logs"; 4 | import config from "./config"; 5 | 6 | export const runCustomSearchFunction = async (uid: string): Promise => { 7 | const response = await fetch(config.searchFunction, { 8 | method: "POST", 9 | body: JSON.stringify({ uid }), 10 | headers: { "Content-Type": "application/json" }, 11 | }); 12 | 13 | if (!response.ok) { 14 | const body = await response.text(); 15 | logs.customFunctionError(new Error(body)); 16 | return; 17 | } 18 | 19 | /** Get user resonse **/ 20 | const json = await response.json(); 21 | 22 | // Support returning an array directly 23 | if (Array.isArray(json)) { 24 | return runBatchPubSubDeletions({ firestorePaths: json }, uid); 25 | } 26 | 27 | return runBatchPubSubDeletions(json, uid); 28 | }; 29 | -------------------------------------------------------------------------------- /delete-user-data/functions/src/search.ts: -------------------------------------------------------------------------------- 1 | import * as admin from "firebase-admin"; 2 | import * as config from "./config"; 3 | const { PubSub } = require("@google-cloud/pubsub"); 4 | 5 | export const search = async ( 6 | uid: string, 7 | depth: number, 8 | document?: admin.firestore.DocumentReference 9 | ) => { 10 | const db = admin.firestore(); 11 | 12 | const pubsub = new PubSub(); 13 | 14 | const topic = pubsub.topic( 15 | `projects/${ 16 | process.env.GOOGLE_CLOUD_PROJECT || process.env.PROJECT_ID 17 | }/topics/${config.default.discoveryTopic}` 18 | ); 19 | 20 | const collections = !document 21 | ? await db.listCollections() 22 | : await document.listCollections(); 23 | 24 | for (const collection of collections) { 25 | topic.publish( 26 | Buffer.from(JSON.stringify({ path: collection.path, uid, depth })) 27 | ); 28 | } 29 | }; 30 | -------------------------------------------------------------------------------- /delete-user-data/functions/src/types.ts: -------------------------------------------------------------------------------- 1 | export type Paths = { 2 | firestorePaths: string[]; 3 | }; 4 | -------------------------------------------------------------------------------- /delete-user-data/functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "lib", 4 | "target": "ES2020", 5 | "module": "commonjs", 6 | "noImplicitReturns": true, 7 | "sourceMap": false, 8 | "resolveJsonModule": true, 9 | "esModuleInterop": true, 10 | "allowSyntheticDefaultImports": true, 11 | "types": ["node", "jest", "@types/node"] 12 | }, 13 | "compileOnSave": true, 14 | "include": ["src"] 15 | } 16 | -------------------------------------------------------------------------------- /delete-user-data/test-data/README.md: -------------------------------------------------------------------------------- 1 | # Extensions Test Data 2 | 3 | This package will populate some test data for the following extensions: 4 | 5 | - delete-user-data 6 | 7 | To run the package, you'll need: 8 | 9 | 1. Your Firebase project ID 10 | 2. A service account key for your project 11 | 12 | Simply run the command below and follow the prompts: 13 | 14 | ``` 15 | npm run populate 16 | ``` 17 | -------------------------------------------------------------------------------- /delete-user-data/test-data/images/pic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/delete-user-data/test-data/images/pic.png -------------------------------------------------------------------------------- /delete-user-data/test-data/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-data", 3 | "description": "Populate a Firebase project with users and data needed to test extensions", 4 | "main": "index.js", 5 | "scripts": { 6 | "populate": "ts-node index.ts" 7 | }, 8 | "author": "Chris Bianca ", 9 | "license": "Apache-2.0", 10 | "dependencies": { 11 | "firebase-admin": "^12.1.0" 12 | }, 13 | "devDependencies": { 14 | "inquirer": "^8.1.2", 15 | "ts-node": "^10.2.1", 16 | "typescript": "^4.3.5" 17 | }, 18 | "private": true 19 | } 20 | -------------------------------------------------------------------------------- /delete-user-data/test-data/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json" 3 | } 4 | -------------------------------------------------------------------------------- /docs/firestore-bigquery-export/Clustering.md: -------------------------------------------------------------------------------- 1 | # Clustering 2 | 3 | If you specified a "Clustering" parameter during configuration of the extension, you can include a [clustering](https://cloud.google.com/bigquery/docs/clustered-tables) order of fields in your BigQuery generated tables. 4 | 5 | Clustering is a feature that organises table data by sorting specified columns and is an ideal solution for reducing the number of reads made on a table through a query. 6 | 7 | When defining clustering, multiple columns can be defined allowing specific ordering of records to minimise the number of unnecessary reads on a query. 8 | 9 | ## Adding columns to cluster 10 | 11 | Through the extension, adding clustering is as simple as adding a comma-separated list of the columns you would like to cluster by - this is then applied to the BigQuery table. 12 | 13 | Clustering allows up to a maximum of four fields and can be configured similar to 14 | 15 | `document_id, document_name, timestamp, event_id, data` 16 | 17 | ![example](/docs/firestore-bigquery-export/media/clustering.png) 18 | 19 | ## Ignoring Columns 20 | 21 | Any columns that are added as part of the configuration, but do not exist on the table schema will be ignored. 22 | 23 | ## Pre-existing tables 24 | 25 | If a pre-existing table exists, the extension will overwrite the previous configuration. 26 | -------------------------------------------------------------------------------- /docs/firestore-bigquery-export/Wildcards.md: -------------------------------------------------------------------------------- 1 | # Wildcards 2 | 3 | The data is stored as a If you specified a "wildcard" parameter during configuration of the extension, the wildcard values are added as a `JSON string` as an additional column in your exported dataset. 4 | 5 | A [wildcard](https://firebase.google.com/docs/functions/firestore-events#wildcards-parameters) column contains the wildcard values extracted from a Firestore collection/sub-collection path. 6 | 7 | ## Parameters column 8 | 9 | A new `string` column called `path_params` will be generated along side the default schemas. 10 | 11 | All generated tables and views are updated with this new column: 12 | 13 | ![example](/docs/firestore-bigquery-export/media/wildcards.png) 14 | 15 | An example path value could be `regions/{regionId}/countries` resulting in an object similar to 16 | 17 | ```js 18 | { 19 | regionId: "Central America"; 20 | } 21 | ``` 22 | 23 | ## Querying the data 24 | 25 | As the data is defined as a `JSON string` a JSON_VALUE extractor will allow the value to be used as part of the query, for example: 26 | 27 | ```sql 28 | SELECT document_id FROM `dataset.countries_raw_changelog` c 29 | WHERE JSON_VALUE(c.path_params, "$.regionId") = "South America" 30 | ``` 31 | -------------------------------------------------------------------------------- /docs/firestore-bigquery-export/cross-project-support.md: -------------------------------------------------------------------------------- 1 | # Cross-Project Support 2 | 3 | If you specified an `BIGQUERY_PROJECT_ID` parameter during configuration of the extension, BigQuery will sync data to tables and views in the defined GCP project. 4 | 5 | ## Example Scenario 6 | 7 | A typical scenario for this would be to install multiple instances of the extension to share the data across multiple (separate) instances without having to support multiple Firestore instances. 8 | 9 | ## Additional Setup 10 | 11 | When defining a specific BigQuery project ID, a manual step to set up permissions is required: 12 | 13 | 1. Navigate to https://console.cloud.google.com/iam-admin/iam?project=${param:BIGQUERY_PROJECT_ID} 14 | 2. Add the **BigQuery Data Editor** role to the following service account: 15 | `ext-${param:EXT_INSTANCE_ID}@${param:PROJECT_ID}.iam.gserviceaccount.com`. 16 | -------------------------------------------------------------------------------- /docs/firestore-bigquery-export/media/clustering.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/docs/firestore-bigquery-export/media/clustering.png -------------------------------------------------------------------------------- /docs/firestore-bigquery-export/media/wildcards.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/docs/firestore-bigquery-export/media/wildcards.png -------------------------------------------------------------------------------- /docs/firestore-bigquery-export/transforming-data.md: -------------------------------------------------------------------------------- 1 | # Transforming Data 2 | 3 | If you specified a `transform function url` parameter during configuration of the extension, you can include a custom url to intercept and modify the `payload` before it is returned and saved into the BigQuery table. 4 | 5 | ## Payload Format 6 | 7 | When data from Firestore has been created or modified, the entire payload is copied to the BigQuery table under the data column in the following format. 8 | 9 | ```json 10 | { 11 | data: [{ 12 | insertId: int; 13 | json: { 14 | timestamp: int; 15 | event_id: int; 16 | document_name: string; 17 | document_id: int; 18 | operation: ChangeType; 19 | data: string; 20 | }, 21 | }] 22 | } 23 | ``` 24 | 25 | ## Syncing modified Data 26 | 27 | By providing a custom URL, users can receive the original payload and modify it as required. The returned data from the response is then saved into the BigQuery table. 28 | -------------------------------------------------------------------------------- /docs/firestore-bundle-builder/media/admin-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/docs/firestore-bundle-builder/media/admin-ui.png -------------------------------------------------------------------------------- /docs/storage-resize-images/customize-output-options.md: -------------------------------------------------------------------------------- 1 | # Customize output options 2 | 3 | ## Use the `OUTPUT_OPTIONS` parameter to customize generated images. 4 | 5 | You may need to customize more than just the size of the images generated by this extension (for example, changing the image quality). That is why you can specify the output options parameter. 6 | 7 | The `OUTPUT_OPTIONS` parameter allows you to customize the output options separately for each image format you wish to use. For example, if you want to reduce the quality of the png images uploaded to your bucket, you can enter an options map like this: 8 | 9 | ```json 10 | { "png": { "quality": 50 } } 11 | ``` 12 | 13 | > The `quality` option accepts a number 0-100 and defaults to 80. See a full list of [png image output options](https://sharp.pixelplumbing.com/api-output#png). 14 | 15 | ## The `OUTPUT_OPTIONS` parameter 16 | 17 | The output options parameter must be a valid JSON object that maps the preferred image formats to their option objects. For example: 18 | 19 | ```json 20 | { 21 | "jpeg": { "quality": 5, "chromaSubsampling": "4:4:4" }, 22 | "png": { "pallete": true } 23 | } 24 | ``` 25 | 26 | > The extension will use `JSON.parse()` method to parse the output options object. 27 | 28 | Image formats and Format options not included in the above object will fall back to the default. 29 | 30 | ## Output options API 31 | 32 | The extension uses the [Sharp](https://github.com/lovell/sharp) open-source library to process images uploaded to your bucket. See the official Sharp documentation for a [complete list of options for each image format](https://sharp.pixelplumbing.com/api-output#jpeg). The list also includes the default value for each output option. 33 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/.gitignore: -------------------------------------------------------------------------------- 1 | lib 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/README.md: -------------------------------------------------------------------------------- 1 | The `firestore-bigquery-change-tracker` package is a dependency for the official Firebase Extension [_Stream Firestore to BigQuery_](https://github.com/firebase/extensions/tree/master/firestore-bigquery-export), [_schema views script_](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md) & the [_import Firestore documents script_](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/IMPORT_EXISTING_DOCUMENTS.md). 2 | 3 | Its main purpose is to initialize & update the BigQuery table & view generated by using the `firestore-bigquery-export` extension. 4 | 5 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/jest.config.js: -------------------------------------------------------------------------------- 1 | const packageJson = require("./package.json"); 2 | 3 | module.exports = { 4 | name: packageJson.name, 5 | displayName: packageJson.name, 6 | rootDir: "./", 7 | types: ["jest", "node"], 8 | globals: { 9 | "ts-jest": { 10 | tsConfig: "/tsconfig.test.json", 11 | }, 12 | }, 13 | preset: "ts-jest", 14 | testMatch: ["**/src/__tests__/**/*.test.ts"], 15 | testEnvironment: "node", 16 | testTimeout: 180000, 17 | collectCoverage: true, 18 | moduleNameMapper: { 19 | "firebase-admin/firestore": 20 | "/node_modules/firebase-admin/lib/firestore", 21 | }, 22 | }; 23 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@firebaseextensions/firestore-bigquery-change-tracker", 3 | "repository": { 4 | "type": "git", 5 | "url": "github.com/firebase/extensions.git", 6 | "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker" 7 | }, 8 | "version": "1.1.42", 9 | "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports", 10 | "main": "./lib/index.js", 11 | "scripts": { 12 | "build": "npm run clean && npm run compile", 13 | "clean": "rimraf lib", 14 | "compile": "tsc", 15 | "test:local": "jest", 16 | "prepare": "npm run build", 17 | "generate-stresstest-table": "bq query --project_id=extensions-testing --use_legacy_sql=false < ./src/__tests__/fixtures/sql/generateSnapshotStresstestTable.sql" 18 | }, 19 | "files": [ 20 | "lib/*.js", 21 | "lib/bigquery/*.js", 22 | "lib/*.d.ts", 23 | "lib/bigquery/*.d.ts" 24 | ], 25 | "author": "Jan Wyszynski ", 26 | "license": "Apache-2.0", 27 | "dependencies": { 28 | "@google-cloud/bigquery": "^7.6.0", 29 | "@google-cloud/resource-manager": "^5.1.0", 30 | "firebase-admin": "^13.2.0", 31 | "firebase-functions": "^6.3.2", 32 | "generate-schema": "^2.6.0", 33 | "inquirer": "^6.4.0", 34 | "lodash": "^4.17.14", 35 | "node-fetch": "^2.6.1", 36 | "sql-formatter": "^2.3.3", 37 | "traverse": "^0.6.6" 38 | }, 39 | "devDependencies": { 40 | "@types/chai": "^4.1.6", 41 | "@types/jest": "^29.5.14", 42 | "@types/node": "14.18.34", 43 | "@types/traverse": "^0.6.32", 44 | "chai": "^4.2.0", 45 | "jest": "29.5.0", 46 | "jest-config": "29.5.0", 47 | "jest-environment-node": "29.5.0", 48 | "jest-summarizing-reporter": "^1.1.4", 49 | "mocked-env": "^1.3.2", 50 | "nyc": "^17.1.0", 51 | "rimraf": "^2.6.3", 52 | "ts-jest": "29.1.2", 53 | "typescript": "^4.9.4" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/failedTransaction.test.ts: -------------------------------------------------------------------------------- 1 | import * as admin from "firebase-admin"; 2 | import { FirestoreBigQueryEventHistoryTrackerConfig } from "../../bigquery"; 3 | 4 | import handleFailedTransactions from "../../bigquery/handleFailedTransactions"; 5 | 6 | // admin.initializeApp(); 7 | const db = admin.firestore(); 8 | 9 | describe("handleFailedTransactions", () => { 10 | it("should be defined", () => { 11 | expect(handleFailedTransactions).toBeDefined(); 12 | }); 13 | 14 | it("should handle more than 500 records", async () => { 15 | const collectionName = "testing"; 16 | const doc = db.collection("testing").doc("600"); 17 | 18 | const config: FirestoreBigQueryEventHistoryTrackerConfig = { 19 | backupTableId: collectionName, 20 | datasetId: "", 21 | tableId: "", 22 | datasetLocation: "", 23 | transformFunction: undefined, 24 | timePartitioning: undefined, 25 | timePartitioningField: undefined, 26 | timePartitioningFieldType: undefined, 27 | timePartitioningFirestoreField: undefined, 28 | clustering: undefined, 29 | bqProjectId: undefined, 30 | }; 31 | 32 | const rows = Array.from(Array(700).keys()).map((x) => { 33 | return { 34 | insertId: x.toString(), 35 | }; 36 | }); 37 | 38 | handleFailedTransactions(rows, config, Error("example_error")); 39 | 40 | return new Promise((resolve, reject) => { 41 | const unsubscribe = doc.onSnapshot((snapshot) => { 42 | const document = snapshot.data(); 43 | 44 | if (document && document.error_details) { 45 | expect(document.error_details).toEqual("example_error"); 46 | unsubscribe(); 47 | resolve(true); 48 | } 49 | }); 50 | }); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/msql/incremental/standard.sql: -------------------------------------------------------------------------------- 1 | CREATE MATERIALIZED VIEW `test.test_dataset.materialized_view_test` 2 | AS ( 3 | WITH latests AS ( 4 | SELECT 5 | document_name, 6 | MAX_BY(document_id, timestamp) AS document_id, 7 | MAX(timestamp) AS timestamp, 8 | MAX_BY(event_id, timestamp) AS event_id, 9 | MAX_BY(operation, timestamp) AS operation, 10 | MAX_BY(data, timestamp) AS data, 11 | MAX_BY(old_data, timestamp) AS old_data, 12 | MAX_BY(extra_field, timestamp) AS extra_field 13 | FROM `test.test_dataset.test_table` 14 | GROUP BY document_name 15 | ) 16 | SELECT * 17 | FROM latests 18 | ) -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/msql/nonIncremental/standard.sql: -------------------------------------------------------------------------------- 1 | CREATE MATERIALIZED VIEW `test.test_dataset.materialized_view_test` 2 | OPTIONS ( 3 | allow_non_incremental_definition = true, 4 | enable_refresh = true, 5 | refresh_interval_minutes = 60, 6 | max_staleness = INTERVAL "4:0:0" HOUR TO SECOND 7 | ) 8 | AS ( 9 | WITH latests AS ( 10 | SELECT 11 | document_name, 12 | MAX_BY(document_id, timestamp) AS document_id, 13 | MAX(timestamp) AS timestamp, 14 | MAX_BY(event_id, timestamp) AS event_id, 15 | MAX_BY(operation, timestamp) AS operation, 16 | MAX_BY(data, timestamp) AS data, 17 | MAX_BY(old_data, timestamp) AS old_data, 18 | MAX_BY(extra_field, timestamp) AS extra_field 19 | FROM `test.test_dataset.test_table` 20 | GROUP BY document_name 21 | ) 22 | SELECT * 23 | FROM latests 24 | WHERE operation != "DELETE" 25 | ) -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/emulator-params.env: -------------------------------------------------------------------------------- 1 | LOCATION=europe-west2 2 | PROJECT_ID=extensions-testing 3 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "emulators": { 3 | "auth": { 4 | "port": 9099 5 | }, 6 | "firestore": { 7 | "port": 8080 8 | }, 9 | "functions": { 10 | "source": "../../../functions", 11 | "port": 5001 12 | }, 13 | "ui": { 14 | "enabled": true 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/clearTables.ts: -------------------------------------------------------------------------------- 1 | import { BigQuery } from "@google-cloud/bigquery"; 2 | 3 | export const deleteTable = async ({ 4 | projectId = "dev-extensions-testing", 5 | datasetId = "", 6 | }) => { 7 | const bq = new BigQuery({ projectId }); 8 | return new Promise((resolve) => { 9 | const dataset = bq.dataset(datasetId); 10 | 11 | let handle = setInterval(async () => { 12 | const [datasetExists] = await dataset.exists(); 13 | 14 | if (!datasetExists) { 15 | clearInterval(handle); 16 | return resolve(dataset); 17 | } 18 | 19 | try { 20 | if (datasetExists) { 21 | await dataset.delete({ force: true }); 22 | } 23 | } catch (ex) {} 24 | }, 500); 25 | }); 26 | }; 27 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/queries.ts: -------------------------------------------------------------------------------- 1 | import { BigQuery } from "@google-cloud/bigquery"; 2 | 3 | export const defaultQuery = ( 4 | bqProjectId: string, 5 | datasetId: string, 6 | tableId: string 7 | ): string => `SELECT * 8 | FROM \`${bqProjectId}.${datasetId}.${tableId}\` 9 | LIMIT 1`; 10 | 11 | export const getBigQueryTableData = async (bqProjectId, datasetId, tableId) => { 12 | const bq = new BigQuery({ projectId: bqProjectId }); 13 | 14 | // Setup queries 15 | const [changeLogQuery] = await bq.createQueryJob({ 16 | query: defaultQuery(bqProjectId, datasetId, `${tableId}_raw_changelog`), 17 | }); 18 | 19 | const [latestViewQuery] = await bq.createQueryJob({ 20 | query: defaultQuery(bqProjectId, datasetId, `${tableId}_raw_latest`), 21 | }); 22 | 23 | // // Wait for the queries to finish 24 | const [changeLogRows] = await changeLogQuery.getQueryResults(); 25 | const [latestRows] = await latestViewQuery.getQueryResults(); 26 | 27 | return [changeLogRows, latestRows]; 28 | }; 29 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/sql/latestConsistentSnapshot.sql: -------------------------------------------------------------------------------- 1 | -- Retrieves the latest document change events for all live documents. 2 | -- timestamp: The Firestore timestamp at which the event took place. 3 | -- operation: One of INSERT, UPDATE, DELETE, IMPORT. 4 | -- event_id: The id of the event that triggered the cloud function mirrored the event. 5 | -- data: A raw JSON payload of the current state of the document. 6 | -- document_id: The document id as defined in the Firestore database 7 | WITH latest AS ( 8 | SELECT 9 | MAX(timestamp) AS latest_timestamp, 10 | document_name 11 | FROM 12 | `test.test_dataset.test_table` 13 | GROUP BY 14 | document_name 15 | ) 16 | SELECT 17 | t.document_name, 18 | document_id, 19 | timestamp AS timestamp, 20 | ANY_VALUE(event_id) AS event_id, 21 | operation AS operation, 22 | ANY_VALUE(data) AS data 23 | FROM 24 | `test.test_dataset.test_table` AS t 25 | JOIN latest ON ( 26 | t.document_name = latest.document_name 27 | AND IFNULL(t.timestamp, TIMESTAMP("1970-01-01 00:00:00+00")) = IFNULL( 28 | latest.latest_timestamp, 29 | TIMESTAMP("1970-01-01 00:00:00+00") 30 | ) 31 | ) 32 | WHERE 33 | operation != "DELETE" 34 | GROUP BY 35 | document_name, 36 | document_id, 37 | timestamp, 38 | operation -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/fixtures/sql/latestConsistentSnapshotNoGroupBy.sql: -------------------------------------------------------------------------------- 1 | -- Retrieves the latest document change events for all live documents. 2 | -- timestamp: The Firestore timestamp at which the event took place. 3 | -- operation: One of INSERT, UPDATE, DELETE, IMPORT. 4 | -- event_id: The id of the event that triggered the cloud function mirrored the event. 5 | -- data: A raw JSON payload of the current state of the document. 6 | -- document_id: The document id as defined in the Firestore database 7 | WITH latest AS ( 8 | SELECT 9 | MAX(timestamp) AS latest_timestamp, 10 | document_name 11 | FROM 12 | `test.test_dataset.test_table` 13 | GROUP BY 14 | document_name 15 | ) 16 | SELECT 17 | t.document_name, 18 | document_id 19 | FROM 20 | `test.test_dataset.test_table` AS t 21 | JOIN latest ON ( 22 | t.document_name = latest.document_name 23 | AND IFNULL(t.timestamp, TIMESTAMP("1970-01-01 00:00:00+00")) = IFNULL(latest.latest_timestamp, TIMESTAMP("1970-01-01 00:00:00+00")) 24 | ) 25 | WHERE 26 | operation != "DELETE" 27 | GROUP BY 28 | document_name, 29 | document_id 30 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/handleFailedTransactions.ts: -------------------------------------------------------------------------------- 1 | import * as admin from "firebase-admin"; 2 | import { initializeApp } from "firebase-admin/app"; 3 | import { getFirestore } from "firebase-admin/firestore"; 4 | import { FirestoreBigQueryEventHistoryTrackerConfig } from "."; 5 | 6 | if (!admin.apps.length) { 7 | initializeApp(); 8 | } 9 | 10 | export default async ( 11 | rows: any[], 12 | config: FirestoreBigQueryEventHistoryTrackerConfig, 13 | e: Error 14 | ): Promise => { 15 | const db = getFirestore(config.firestoreInstanceId!); 16 | db.settings({ 17 | ignoreUndefinedProperties: true, 18 | }); 19 | const batchArray = [db.batch()]; 20 | 21 | let operationCounter = 0; 22 | let batchIndex = 0; 23 | 24 | rows?.forEach((row) => { 25 | var ref = db.collection(config.backupTableId).doc(row.insertId); 26 | 27 | batchArray[batchIndex].set(ref, { 28 | ...row, 29 | error_details: e.message, 30 | }); 31 | 32 | operationCounter++; 33 | 34 | // Check if max limit for batch has been met. 35 | if (operationCounter === 499) { 36 | batchArray.push(db.batch()); 37 | batchIndex++; 38 | operationCounter = 0; 39 | } 40 | }); 41 | 42 | for (let batch of batchArray) { 43 | await batch.commit(); 44 | } 45 | 46 | return Promise.resolve(); 47 | }; 48 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/validateProject.ts: -------------------------------------------------------------------------------- 1 | const { ProjectsClient } = require("@google-cloud/resource-manager"); 2 | 3 | /* TODO: searchProjectsAsync sometimes returns {}. 4 | * Could be resource intensive, if checked on every records insert. 5 | */ 6 | export const validateProject = async (id: string): Promise => { 7 | let isValid = false; 8 | 9 | const client = new ProjectsClient(); 10 | const projects = client.searchProjectsAsync(); 11 | 12 | for await (const project of projects) { 13 | if (project.projectId === id) { 14 | isValid = true; 15 | } 16 | } 17 | 18 | return isValid; 19 | }; 20 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/errors.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const changedFieldMode = ( 18 | fieldName: string, 19 | bqMode: string, 20 | schemaMode: string 21 | ) => 22 | new Error( 23 | `Field ${fieldName} has different field mode. BigQuery mode: ${bqMode}; Schema mode: ${schemaMode}` 24 | ); 25 | 26 | export const changedFieldType = ( 27 | fieldName: string, 28 | bqType: string, 29 | schemaType: string 30 | ) => 31 | new Error( 32 | `Field: ${fieldName} has changed field type. BigQuery type: ${bqType}; Schema type: ${schemaType}` 33 | ); 34 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export { 18 | FirestoreBigQueryEventHistoryTracker, 19 | RawChangelogSchema, 20 | RawChangelogViewSchema, 21 | } from "./bigquery"; 22 | export { 23 | ChangeType, 24 | FirestoreDocumentChangeEvent, 25 | FirestoreEventHistoryTracker, 26 | } from "./tracker"; 27 | export { LogLevel, Logger } from "./logger"; 28 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/tracker.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export enum ChangeType { 18 | CREATE, 19 | DELETE, 20 | UPDATE, 21 | IMPORT, 22 | } 23 | 24 | export interface FirestoreDocumentChangeEvent { 25 | // The timestamp represented in ISO format. 26 | // Date is not appropriate because it only has millisecond precision. 27 | // Cloud Firestore timestamps have microsecond precision. 28 | timestamp: string; 29 | operation: ChangeType; 30 | documentName: string; 31 | eventId: string; 32 | documentId: string; 33 | pathParams?: { documentId: string; [key: string]: string } | null; 34 | data: Object; 35 | oldData?: Object | null; 36 | useNewSnapshotQuerySyntax?: boolean | null; 37 | } 38 | 39 | export interface FirestoreEventHistoryTracker { 40 | record(event: FirestoreDocumentChangeEvent[]); 41 | } 42 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/src/types.ts: -------------------------------------------------------------------------------- 1 | export enum PartitionFieldType { 2 | DATE = "DATE", 3 | DATETIME = "DATETIME", 4 | TIMESTAMP = "TIMESTAMP", 5 | } 6 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "lib", 5 | "types": ["node", "jest"], 6 | "target": "ES2020", 7 | "skipLibCheck": true, 8 | "declaration": true 9 | }, 10 | "include": ["src"] 11 | } 12 | -------------------------------------------------------------------------------- /firestore-bigquery-export/firestore-bigquery-change-tracker/tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "types": ["node", "jest"], 5 | "typeRoots": ["./node_modules/@types"] 6 | }, 7 | "include": ["src/**/*.ts", "src/**/*.test.ts", "src/__tests__/**/*.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/.gitignore: -------------------------------------------------------------------------------- 1 | lib -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/__tests__/__mocks__/console.ts: -------------------------------------------------------------------------------- 1 | import { logger } from "firebase-functions"; 2 | 3 | export const mockConsoleLog = jest.spyOn(logger, "log").mockImplementation(); 4 | 5 | export const mockConsoleError = jest 6 | .spyOn(logger, "error") 7 | .mockImplementation(); 8 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/__tests__/__mocks__/firestore.ts: -------------------------------------------------------------------------------- 1 | import * as functionsTestInit from "firebase-functions-test"; 2 | 3 | export const snapshot = ( 4 | input = { input: "hello" }, 5 | path = "translations/id1" 6 | ) => { 7 | let functionsTest = functionsTestInit(); 8 | return functionsTest.firestore.makeDocumentSnapshot(input, path); 9 | }; 10 | 11 | export const mockDocumentSnapshotFactory = (documentSnapshot) => { 12 | return jest.fn().mockImplementation(() => { 13 | return { 14 | exists: true, 15 | get: documentSnapshot.get.bind(documentSnapshot), 16 | ref: { path: documentSnapshot.ref.path }, 17 | }; 18 | })(); 19 | }; 20 | 21 | export const makeChange = (before, after) => { 22 | let functionsTest = functionsTestInit(); 23 | return functionsTest.makeChange(before, after); 24 | }; 25 | 26 | export const mockFirestoreTransaction = jest.fn().mockImplementation(() => { 27 | return (transactionHandler) => { 28 | transactionHandler({ 29 | update(ref, field, data) { 30 | mockFirestoreUpdate(field, data); 31 | }, 32 | }); 33 | }; 34 | }); 35 | 36 | export const mockFirestoreUpdate = jest.fn(); 37 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/__tests__/fixtures/documentData.ts: -------------------------------------------------------------------------------- 1 | import * as admin from "firebase-admin"; 2 | 3 | export const documentData = async () => { 4 | const firstReference = admin.firestore().doc("reference/reference1"); 5 | firstReference.set({ 6 | a_string: "a_string_value", 7 | an_integr: 25, 8 | a_boolean: false, 9 | a_list: ["a_string_value", "b_string_value", "c_string_value"], 10 | a_date: new Date(2023, 7, 19, 7, 12, 38), 11 | }); 12 | 13 | const secondReference = admin.firestore().doc("reference/reference2"); 14 | secondReference.set({ 15 | a_string: "a_string_value", 16 | an_integr: 30, 17 | a_boolean: true, 18 | a_list: ["a_string_value", "b_string_value", "c_string_value"], 19 | a_date: new Date(2023, 7, 19, 7, 12, 38), 20 | }); 21 | 22 | return { 23 | // String 24 | a_string: "a_string_value", 25 | 26 | // Number 27 | an_integer: 30, 28 | 29 | // Boolean 30 | a_boolean: true, 31 | 32 | // Array 33 | a_list: ["a_string_value", "b_string_value", "c_string_value"], 34 | 35 | // Object 36 | an_object_list: { 37 | street: "a_street_string_value", 38 | city: "a_city_string_value", 39 | state: "a_state_string_value", 40 | zip: "a_zip_string_value", 41 | }, 42 | 43 | // Timestamp 44 | a_date: new Date(2023, 7, 19, 7, 12, 38), 45 | 46 | // GeoPoint 47 | a_geo_object: { 48 | latitude: 36.7783, 49 | longitude: -119.4179, 50 | }, 51 | 52 | // Reference 53 | singleReference: firstReference, 54 | 55 | // Array of References 56 | reference_list: [firstReference, secondReference], 57 | }; 58 | }; 59 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/__tests__/jest.setup.ts: -------------------------------------------------------------------------------- 1 | global.config = () => require("../src/config").default; 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/__tests__/test.types.d.ts: -------------------------------------------------------------------------------- 1 | declare namespace NodeJS { 2 | interface Global { 3 | config: () => jest.ModuleMocker; 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/__tests__/util.test.ts: -------------------------------------------------------------------------------- 1 | import { getChangeType } from "../src/util"; 2 | import * as functionsTestInit from "firebase-functions-test"; 3 | import { ChangeType } from "@firebaseextensions/firestore-bigquery-change-tracker"; 4 | import { DocumentSnapshot } from "firebase-functions/lib/v1/providers/firestore"; 5 | 6 | const functionsTest = functionsTestInit(); 7 | const { 8 | firestore: { makeDocumentSnapshot }, 9 | } = functionsTest; 10 | 11 | export const makeChange = (before, after) => { 12 | return functionsTest.makeChange(before, after); 13 | }; 14 | 15 | describe("util.getChangeType", () => { 16 | test("return a delete change type", () => { 17 | const before: DocumentSnapshot = makeDocumentSnapshot( 18 | { foo: "bar" }, 19 | "docs/1" 20 | ); 21 | const after: DocumentSnapshot = makeDocumentSnapshot([], "docs/1"); 22 | const changeType: ChangeType = getChangeType(makeChange(before, after)); 23 | 24 | expect(changeType === ChangeType.DELETE).toBeTruthy(); 25 | }); 26 | 27 | test("return a create change type", () => { 28 | const before: DocumentSnapshot = makeDocumentSnapshot([], "docs/1"); 29 | const after: DocumentSnapshot = makeDocumentSnapshot( 30 | { foo: "bar" }, 31 | "docs/1" 32 | ); 33 | 34 | const changeType: ChangeType = getChangeType(makeChange(before, after)); 35 | 36 | expect(changeType === ChangeType.CREATE).toBeTruthy(); 37 | }); 38 | 39 | test("return a update change type", () => { 40 | const before: DocumentSnapshot = makeDocumentSnapshot( 41 | { foo: "bar" }, 42 | "docs/1" 43 | ); 44 | 45 | const after: DocumentSnapshot = makeDocumentSnapshot( 46 | { foo: "bars" }, 47 | "docs/1" 48 | ); 49 | const changeType: ChangeType = getChangeType(makeChange(before, after)); 50 | 51 | expect(changeType === ChangeType.UPDATE).toBeTruthy(); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/big-query-table-test/install-params.env: -------------------------------------------------------------------------------- 1 | COLLECTION_PATH=bigQueryProductionTests 2 | DATASET_ID=bigquery_production_tests 3 | TABLE_ID=production_test 4 | LOCATION=europe-west2 -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | 3 | const packageJson = require("./package.json"); 4 | 5 | module.exports = { 6 | preset: "ts-jest", 7 | testEnvironment: "node", 8 | name: packageJson.name, 9 | displayName: packageJson.name, 10 | rootDir: "./", 11 | transform: { 12 | "^.+\\.tsx?$": [ 13 | "ts-jest", 14 | { 15 | tsconfig: "/tsconfig.test.json", 16 | }, 17 | ], 18 | }, 19 | snapshotFormat: { 20 | escapeString: true, 21 | printBasicPrototype: true, 22 | }, 23 | preset: "ts-jest", 24 | testEnvironment: "node", 25 | testEnvironmentOptions: { 26 | NODE_ENV: "test", 27 | }, 28 | setupFiles: ["/__tests__/jest.setup.ts"], 29 | testMatch: ["**/__tests__/*.test.ts"], 30 | testPathIgnorePatterns: process.env.CI_TEST === "true" ? ["e2e"] : [], 31 | moduleNameMapper: { 32 | "firebase-admin/eventarc": 33 | "/node_modules/firebase-admin/lib/eventarc", 34 | "firebase-admin/firestore": 35 | "/node_modules/firebase-admin/lib/firestore", 36 | "firebase-functions/v2": "/node_modules/firebase-functions/lib/v2", 37 | "firebase-admin/auth": "/node_modules/firebase-admin/lib/auth", 38 | "firebase-admin/app": "/node_modules/firebase-admin/lib/app", 39 | "firebase-admin/database": 40 | "/node_modules/firebase-admin/lib/database", 41 | "firebase-admin/functions": 42 | "/node_modules/firebase-admin/lib/functions", 43 | "firebase-admin/extensions": 44 | "/node_modules/firebase-admin/lib/extensions", 45 | }, 46 | }; 47 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firestore-bigquery-export", 3 | "description": "Export a Cloud Firestore collection to BigQuery", 4 | "main": "lib/index.js", 5 | "scripts": { 6 | "build": "npm run clean && npm run compile", 7 | "prepare": "npm run build", 8 | "clean": "rimraf lib", 9 | "compile": "tsc", 10 | "test": "jest", 11 | "generate-readme": "firebase ext:info .. --markdown > ../README.md" 12 | }, 13 | "author": "Jan Wyszynski ", 14 | "license": "Apache-2.0", 15 | "dependencies": { 16 | "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.42", 17 | "@google-cloud/bigquery": "^7.6.0", 18 | "@types/chai": "^4.1.6", 19 | "@types/express-serve-static-core": "4.17.30", 20 | "@types/jest": "29.5.0", 21 | "@types/node": "^20.4.4", 22 | "chai": "^4.2.0", 23 | "firebase-admin": "^13.2.0", 24 | "firebase-functions": "^6.3.2", 25 | "firebase-functions-test": "^3.4.1", 26 | "generate-schema": "^2.6.0", 27 | "inquirer": "^6.4.0", 28 | "jest": "29.5.0", 29 | "jest-config": "29.5.0", 30 | "lodash": "^4.17.14", 31 | "nyc": "^17.1.0", 32 | "rimraf": "^2.6.3", 33 | "sql-formatter": "^2.3.3", 34 | "ts-jest": "29.1.2", 35 | "ts-node": "^9.0.0", 36 | "typescript": "^4.8.4" 37 | }, 38 | "private": true, 39 | "devDependencies": { 40 | "faker": "^5.1.0", 41 | "mocked-env": "^1.3.2" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/stress_test/count.js: -------------------------------------------------------------------------------- 1 | const admin = require("firebase-admin"); 2 | 3 | // Initialize Firebase Admin with your credentials 4 | // Make sure you've already set up your Firebase Admin SDK 5 | admin.initializeApp({ 6 | projectId: "vertex-testing-1efc3", 7 | }); 8 | 9 | const firestore = admin.firestore(); 10 | 11 | async function countDocuments(collectionPath) { 12 | try { 13 | const collectionRef = firestore.collection(collectionPath); 14 | 15 | // Perform an aggregate query to count the documents 16 | const snapshot = await collectionRef.count().get(); 17 | 18 | // Access the count from the snapshot 19 | const docCount = snapshot.data().count; 20 | 21 | console.log( 22 | `Number of documents in collection '${collectionPath}':`, 23 | docCount 24 | ); 25 | return docCount; 26 | } catch (error) { 27 | console.error("Error counting documents:", error); 28 | throw error; 29 | } 30 | } 31 | 32 | // Call the function and pass the collection path 33 | countDocuments("posts_2"); 34 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": ["./__tests__/test.types.d.ts"], 3 | "compilerOptions": { 4 | "lib": ["esnext.asynciterable"], 5 | "outDir": "lib", 6 | "types": ["node", "jest", "chai"], 7 | "target": "ES2020", 8 | "module": "commonjs", 9 | "skipLibCheck": true, 10 | "noImplicitReturns": true, 11 | "sourceMap": false 12 | }, 13 | "compileOnSave": true, 14 | "include": ["src"], 15 | "exclude": ["src/**/*.test.ts"] 16 | } 17 | -------------------------------------------------------------------------------- /firestore-bigquery-export/functions/tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "types": ["node", "jest"] 5 | }, 6 | "include": ["src/**/*.ts", "src/**/*.test.ts"] 7 | } 8 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/.gitignore: -------------------------------------------------------------------------------- 1 | lib 2 | test-schema.json -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/README.md: -------------------------------------------------------------------------------- 1 | The `fs-bq-schema-views` script is for use with the official Firebase Extension 2 | [_Stream Firestore to BigQuery_](https://github.com/firebase/extensions/tree/master/firestore-bigquery-export). 3 | 4 | This script creates a BigQuery view based on a provided JSON schema configuration file. It queries the data from the `firestore-bigquery-export` extension changelog table to generate the view. 5 | 6 | A guide on how to use the script can be found [here](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md). -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/jest.config.js: -------------------------------------------------------------------------------- 1 | const packageJson = require("./package.json"); 2 | 3 | module.exports = { 4 | name: packageJson.name, 5 | displayName: packageJson.name, 6 | rootDir: "./", 7 | globals: { 8 | "ts-jest": { 9 | tsConfig: "/tsconfig.json", 10 | }, 11 | }, 12 | preset: "ts-jest", 13 | testEnvironment: "node", 14 | testMatch: [ 15 | "/src/__tests__/**/*.test.ts", 16 | "/src/__tests__/schema-loader-utils/*.test.ts", 17 | ], 18 | testPathIgnorePatterns: process.env.CI_TEST === "true" ? ["e2e"] : [], 19 | moduleNameMapper: { 20 | "firebase-admin/eventarc": 21 | "/node_modules/firebase-admin/lib/eventarc", 22 | "firebase-functions/v2": "/node_modules/firebase-functions/lib/v2", 23 | "firebase-admin/auth": "/node_modules/firebase-admin/lib/auth", 24 | "firebase-admin/app": "/node_modules/firebase-admin/lib/app", 25 | "firebase-admin/database": 26 | "/node_modules/firebase-admin/lib/database", 27 | "firebase-admin/firestore": 28 | "/node_modules/firebase-admin/lib/firestore", 29 | }, 30 | }; 31 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/schemas/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "testing", 5 | "type": "string", 6 | "description": "A testing field!" 7 | }, 8 | { 9 | "name": "testing_two", 10 | "type": "number", 11 | "description": "A second testing field!" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/schemas/user-schema-version-login.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "last_login", 5 | "type": "timestamp", 6 | "description": "A timestamp field." 7 | }, 8 | { 9 | "name": "version", 10 | "type": "number", 11 | "description": "The version of the last_login field." 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/schemas/user_array.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "test_array", 5 | "type": "array", 6 | "description": "0 1 1 2 3 5 8 and so on." 7 | } 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/schemas/user_complex.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "name", 5 | "type": "string" 6 | }, 7 | { 8 | "name": "date", 9 | "type": "string" 10 | }, 11 | { 12 | "name": "total", 13 | "type": "string" 14 | }, 15 | { 16 | "name": "cartItems", 17 | "type": "array_map", 18 | "fields": [ 19 | { 20 | "name": "productName", 21 | "type": "string" 22 | }, 23 | { 24 | "name": "quantity", 25 | "type": "string" 26 | }, 27 | { 28 | "name": "isGift", 29 | "type": "string" 30 | } 31 | ] 32 | } 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/schemas/user_full.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "name", 5 | "type": "string" 6 | }, 7 | { 8 | "name": "favorite_numbers", 9 | "type": "array" 10 | }, 11 | { 12 | "name": "last_login", 13 | "type": "timestamp" 14 | }, 15 | { 16 | "name": "last_location", 17 | "type": "geopoint" 18 | }, 19 | { 20 | "fields": [ 21 | { 22 | "name": "name", 23 | "type": "string", 24 | "description": "This is a nested name!" 25 | } 26 | ], 27 | "name": "friends", 28 | "type": "map", 29 | "description": "Maps don't need a description because a column is not created for a map. Instead, their fields are added as columns to the resulting dataset." 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/e2e/helpers/deleteDatasets.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // deleteDatasets.js 18 | const { BigQuery } = require("@google-cloud/bigquery"); 19 | 20 | // Create a BigQuery client for the specified project. 21 | const bq = new BigQuery({ projectId: "dev-extensions-testing" }); 22 | 23 | async function deleteAllDatasets() { 24 | try { 25 | // List all datasets in the project. 26 | const [datasets] = await bq.getDatasets(); 27 | console.log(`Found ${datasets.length} datasets.`); 28 | 29 | // Iterate over each dataset and delete it. 30 | for (const dataset of datasets) { 31 | if (dataset.id !== "2025_stress_test") { 32 | console.log(`Deleting dataset: ${dataset.id}...`); 33 | // The force option will delete the dataset along with all its tables. 34 | await dataset.delete({ force: true }); 35 | console.log(`Dataset ${dataset.id} deleted successfully.`); 36 | } 37 | } 38 | 39 | console.log("All datasets have been deleted."); 40 | } catch (error) { 41 | console.error("Error deleting datasets:", error); 42 | } 43 | } 44 | 45 | // Execute the deletion function. 46 | deleteAllDatasets(); 47 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/e2e/schemas/arraysNestedInMapsSchema/arraysNestedInMapsSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "map", 5 | "type": "map", 6 | "fields": [ 7 | { 8 | "name": "array", 9 | "type": "array" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "map2", 15 | "type": "map", 16 | "fields": [ 17 | { 18 | "name": "array", 19 | "type": "array" 20 | } 21 | ] 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/e2e/schemas/basic/basic.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "name", 5 | "column_name": "name", 6 | "type": "string" 7 | } 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/e2e/schemas/mappedArray/mappedArray.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "name", 5 | "type": "string" 6 | }, 7 | { 8 | "name": "date", 9 | "type": "string" 10 | }, 11 | { 12 | "name": "total", 13 | "type": "string" 14 | }, 15 | { 16 | "name": "cartItems", 17 | "type": "array_map", 18 | "fields": [ 19 | { 20 | "name": "productName", 21 | "type": "string" 22 | }, 23 | { 24 | "name": "quantity", 25 | "type": "string" 26 | }, 27 | { 28 | "name": "isGift", 29 | "type": "string" 30 | } 31 | ] 32 | } 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/e2e/schemas/nestedMapSchema/nestedMapSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "super", 5 | "type": "map", 6 | "fields": [ 7 | { 8 | "name": "nested", 9 | "type": "map", 10 | "fields": [ 11 | { 12 | "name": "schema", 13 | "type": "map", 14 | "fields": [ 15 | { 16 | "name": "value", 17 | "type": "number" 18 | } 19 | ] 20 | } 21 | ] 22 | } 23 | ] 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/deep-directory/1/2/leaf-schemas-1.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/deep-directory/1/2/leaf-schemas-2.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/deep-directory/1/2/leaf-schemas-3.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/deep-directory/1/2/leaf-schemas-4.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/deep-directory/1/2/leaf-schemas-5.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/deep-directory/1/other-schemas-1.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/deep-directory/1/other-schemas-2.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/deep-directory/1/other-schemas-3.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/deep-directory/1/other-schemas-4.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/deep-directory/1/other-schemas-5.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/full-directory/schema-1.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/full-directory/schema-1.txt: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/full-directory/schema-2.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/full-directory/schema-2.txt: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/full-directory/schema-3.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/full-directory/schema-3.txt: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/full-directory/schema-4.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/full-directory/schema-4.txt: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/full-directory/schema-5.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schema-files/full-directory/schema-5.txt: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schemas/arraysNestedInMapsSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "map", 5 | "type": "map", 6 | "fields": [ 7 | { 8 | "name": "array", 9 | "type": "array" 10 | } 11 | ] 12 | }, 13 | { 14 | "name": "map2", 15 | "type": "map", 16 | "fields": [ 17 | { 18 | "name": "array", 19 | "type": "array" 20 | } 21 | ] 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schemas/complexSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "name", 5 | "type": "string" 6 | }, 7 | { 8 | "name": "order", 9 | "type": "map", 10 | "fields": [ 11 | { 12 | "name": "orderNumber", 13 | "type": "int" 14 | }, 15 | { 16 | "name": "orderDate", 17 | "type": "string" 18 | }, 19 | { 20 | "name": "orderTotal", 21 | "type": "double" 22 | }, 23 | { 24 | "name": "cartItems", 25 | "type": "array", 26 | "fields": [ 27 | { 28 | "name": "productName", 29 | "type": "string" 30 | }, 31 | { 32 | "name": "quantity", 33 | "type": "int" 34 | }, 35 | { 36 | "name": "isGift", 37 | "type": "boolean" 38 | } 39 | ] 40 | } 41 | ] 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schemas/emptyMapSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "map", 5 | "type": "map", 6 | "fields": [] 7 | } 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schemas/emptySchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [] 3 | } 4 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schemas/fullSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "name", 5 | "type": "string" 6 | }, 7 | { 8 | "name": "favorite_numbers", 9 | "type": "array" 10 | }, 11 | { 12 | "name": "last_login", 13 | "type": "timestamp" 14 | }, 15 | { 16 | "name": "last_location", 17 | "type": "geopoint" 18 | }, 19 | { 20 | "name": "friends", 21 | "type": "map", 22 | "fields": [ 23 | { 24 | "name": "name", 25 | "type": "string" 26 | } 27 | ] 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schemas/fullSchemaSquared.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "schema1", 5 | "type": "map", 6 | "fields": [ 7 | { 8 | "name": "people", 9 | "type": "stringified_map" 10 | }, 11 | { 12 | "name": "name", 13 | "type": "string" 14 | }, 15 | { 16 | "name": "favorite_numbers", 17 | "type": "array" 18 | }, 19 | { 20 | "name": "last_login", 21 | "type": "timestamp" 22 | }, 23 | { 24 | "name": "last_location", 25 | "type": "geopoint" 26 | }, 27 | { 28 | "fields": [ 29 | { 30 | "name": "name", 31 | "type": "string" 32 | } 33 | ], 34 | "name": "friends", 35 | "type": "map" 36 | } 37 | ] 38 | }, 39 | { 40 | "name": "schema2", 41 | "type": "map", 42 | "fields": [ 43 | { 44 | "name": "people", 45 | "type": "stringified_map" 46 | }, 47 | { 48 | "name": "name", 49 | "type": "string" 50 | }, 51 | { 52 | "name": "favorite_numbers", 53 | "type": "array" 54 | }, 55 | { 56 | "name": "last_login", 57 | "type": "timestamp" 58 | }, 59 | { 60 | "name": "last_location", 61 | "type": "geopoint" 62 | }, 63 | { 64 | "fields": [ 65 | { 66 | "name": "name", 67 | "type": "string" 68 | } 69 | ], 70 | "name": "friends", 71 | "type": "map" 72 | } 73 | ] 74 | } 75 | ] 76 | } 77 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schemas/jsonSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "friends", 5 | "type": "stringified_map" 6 | } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schemas/nestedMapSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "super", 5 | "type": "map", 6 | "fields": [ 7 | { 8 | "name": "nested", 9 | "type": "map", 10 | "fields": [ 11 | { 12 | "name": "schema", 13 | "type": "map", 14 | "fields": [ 15 | { 16 | "name": "value", 17 | "type": "number" 18 | } 19 | ] 20 | } 21 | ] 22 | } 23 | ] 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/schemas/referenceSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "fields": [ 3 | { 4 | "name": "reference", 5 | "type": "reference" 6 | }, 7 | { 8 | "name": "map", 9 | "type": "map", 10 | "fields": [ 11 | { 12 | "name": "reference", 13 | "type": "reference" 14 | } 15 | ] 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/arraysNestedInMapsSchema.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | * 3 | FROM 4 | ( 5 | SELECT 6 | document_name, 7 | document_id, 8 | timestamp, 9 | operation, 10 | `test.test_dataset.firestoreArray`(JSON_EXTRACT(data, '$.map.array')) AS map_array, 11 | `test.test_dataset.firestoreArray`(JSON_EXTRACT(data, '$.map2.array')) AS map2_array 12 | FROM 13 | `test.test_dataset.test_table` 14 | ) test_table 15 | LEFT JOIN UNNEST(test_table.map_array) AS map_array_member WITH OFFSET map_array_index 16 | LEFT JOIN UNNEST(test_table.map2_array) AS map2_array_member WITH OFFSET map2_array_index -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/emptyMapSchema.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | document_name, 3 | document_id, 4 | timestamp, 5 | operation 6 | FROM 7 | `test.test_dataset.test_table` -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/emptySchemaChangeLog.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | document_name, 3 | document_id, 4 | timestamp, 5 | operation 6 | FROM 7 | `test.test_dataset.test_table` -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/emptySchemaLatest.sql: -------------------------------------------------------------------------------- 1 | -- Given a user-defined schema over a raw JSON changelog, returns the 2 | -- schema elements of the latest set of live documents in the collection. 3 | -- timestamp: The Firestore timestamp at which the event took place. 4 | -- operation: One of INSERT, UPDATE, DELETE, IMPORT. 5 | -- event_id: The event that wrote this row. 6 | -- : This can be one, many, or no typed-columns 7 | -- corresponding to fields defined in the schema. 8 | SELECT 9 | document_name, 10 | document_id, 11 | timestamp, 12 | operation 13 | FROM 14 | ( 15 | SELECT 16 | document_name, 17 | document_id, 18 | FIRST_VALUE(timestamp) OVER( 19 | PARTITION BY document_name 20 | ORDER BY 21 | timestamp DESC 22 | ) AS timestamp, 23 | FIRST_VALUE(operation) OVER( 24 | PARTITION BY document_name 25 | ORDER BY 26 | timestamp DESC 27 | ) AS operation, 28 | FIRST_VALUE(operation) OVER( 29 | PARTITION BY document_name 30 | ORDER BY 31 | timestamp DESC 32 | ) = "DELETE" AS is_deleted 33 | FROM 34 | `test.test_dataset.test_table` 35 | ) 36 | WHERE 37 | NOT is_deleted 38 | GROUP BY 39 | document_name, 40 | document_id, 41 | timestamp, 42 | operation -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/emptySchemaLatestFromView.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | * 3 | FROM 4 | ( 5 | SELECT 6 | document_name, 7 | document_id, 8 | timestamp, 9 | operation, 10 | JSON_EXTRACT_SCALAR(data, '$.name') AS name, 11 | `test.test_dataset.firestoreArray`(JSON_EXTRACT(data, '$.favorite_numbers')) AS favorite_numbers, 12 | `test.test_dataset.firestoreTimestamp`(JSON_EXTRACT(data, '$.last_login')) AS last_login, 13 | `test.test_dataset.firestoreGeopoint`(JSON_EXTRACT(data, '$.last_location')) AS last_location, 14 | SAFE_CAST( 15 | JSON_EXTRACT_SCALAR(data, '$.last_location._latitude') AS NUMERIC 16 | ) AS last_location_latitude, 17 | SAFE_CAST( 18 | JSON_EXTRACT_SCALAR(data, '$.last_location._longitude') AS NUMERIC 19 | ) AS last_location_longitude, 20 | JSON_EXTRACT_SCALAR(data, '$.friends.name') AS friends_name 21 | FROM 22 | `test.test_dataset.test_table_latest` 23 | ) test_table_latest 24 | LEFT JOIN UNNEST(test_table_latest.favorite_numbers) AS favorite_numbers_member WITH OFFSET favorite_numbers_index -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/fullSchemaChangeLog.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | * 3 | FROM 4 | ( 5 | SELECT 6 | document_name, 7 | document_id, 8 | timestamp, 9 | operation, 10 | JSON_EXTRACT_SCALAR(data, '$.name') AS name, 11 | `test.test_dataset.firestoreArray`(JSON_EXTRACT(data, '$.favorite_numbers')) AS favorite_numbers, 12 | `test.test_dataset.firestoreTimestamp`(JSON_EXTRACT(data, '$.last_login')) AS last_login, 13 | `test.test_dataset.firestoreGeopoint`(JSON_EXTRACT(data, '$.last_location')) AS last_location, 14 | SAFE_CAST( 15 | JSON_EXTRACT_SCALAR(data, '$.last_location._latitude') AS NUMERIC 16 | ) AS last_location_latitude, 17 | SAFE_CAST( 18 | JSON_EXTRACT_SCALAR(data, '$.last_location._longitude') AS NUMERIC 19 | ) AS last_location_longitude, 20 | JSON_EXTRACT_SCALAR(data, '$.friends.name') AS friends_name 21 | FROM 22 | `test.test_dataset.test_table` 23 | ) test_table 24 | LEFT JOIN UNNEST(test_table.favorite_numbers) AS favorite_numbers_member WITH OFFSET favorite_numbers_index -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/fullSchemaLatestFromView.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | * 3 | FROM 4 | ( 5 | SELECT 6 | document_name, 7 | document_id, 8 | timestamp, 9 | operation, 10 | JSON_EXTRACT_SCALAR(data, '$.name') AS name, 11 | `test.test_dataset.firestoreArray`(JSON_EXTRACT(data, '$.favorite_numbers')) AS favorite_numbers, 12 | `test.test_dataset.firestoreTimestamp`(JSON_EXTRACT(data, '$.last_login')) AS last_login, 13 | `test.test_dataset.firestoreGeopoint`(JSON_EXTRACT(data, '$.last_location')) AS last_location, 14 | SAFE_CAST( 15 | JSON_EXTRACT_SCALAR(data, '$.last_location._latitude') AS NUMERIC 16 | ) AS last_location_latitude, 17 | SAFE_CAST( 18 | JSON_EXTRACT_SCALAR(data, '$.last_location._longitude') AS NUMERIC 19 | ) AS last_location_longitude, 20 | JSON_EXTRACT_SCALAR(data, '$.friends.name') AS friends_name 21 | FROM 22 | `test.test_dataset.test_table_latest` 23 | ) test_table_latest 24 | LEFT JOIN UNNEST(test_table_latest.favorite_numbers) AS favorite_numbers_member WITH OFFSET favorite_numbers_index -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/fullSchemaLatestLatestFromView.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | * 3 | FROM 4 | ( 5 | SELECT 6 | document_name, 7 | document_id, 8 | timestamp, 9 | operation, 10 | JSON_EXTRACT_SCALAR(data, '$.name') AS name, 11 | `test.test_dataset.firestoreArray`(JSON_EXTRACT(data, '$.favorite_numbers')) AS favorite_numbers, 12 | `test.test_dataset.firestoreTimestamp`(JSON_EXTRACT(data, '$.last_login')) AS last_login, 13 | `test.test_dataset.firestoreGeopoint`(JSON_EXTRACT(data, '$.last_location')) AS last_location, 14 | SAFE_CAST( 15 | JSON_EXTRACT_SCALAR(data, '$.last_location._latitude') AS NUMERIC 16 | ) AS last_location_latitude, 17 | SAFE_CAST( 18 | JSON_EXTRACT_SCALAR(data, '$.last_location._longitude') AS NUMERIC 19 | ) AS last_location_longitude, 20 | JSON_EXTRACT_SCALAR(data, '$.friends.name') AS friends_name 21 | FROM 22 | `test.test_dataset.test_table_latest` 23 | ) test_table_latest 24 | LEFT JOIN UNNEST(test_table_latest.favorite_numbers) AS favorite_numbers_member WITH OFFSET favorite_numbers_index -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/jsonColumn.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | document_name, 3 | document_id, 4 | timestamp, 5 | operation, 6 | JSON_EXTRACT(data, '$.friends') AS friends 7 | FROM 8 | `test.test_dataset.test_table` -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/latestConsistentSnapshot.sql: -------------------------------------------------------------------------------- 1 | -- Retrieves the latest document change events for all live documents. 2 | -- timestamp: The Firestore timestamp at which the event took place. 3 | -- operation: One of INSERT, UPDATE, DELETE, IMPORT. 4 | -- event_id: The id of the event that triggered the cloud function mirrored the event. 5 | -- data: A raw JSON payload of the current state of the document. 6 | SELECT 7 | document_name, 8 | document_id, 9 | timestamp, 10 | event_id, 11 | operation, 12 | data 13 | FROM 14 | ( 15 | SELECT 16 | document_name, 17 | FIRST_VALUE(timestamp) OVER( 18 | PARTITION BY document_name 19 | ORDER BY 20 | timestamp DESC 21 | ) AS timestamp, 22 | FIRST_VALUE(event_id) OVER( 23 | PARTITION BY document_name 24 | ORDER BY 25 | timestamp DESC 26 | ) AS event_id, 27 | FIRST_VALUE(operation) OVER( 28 | PARTITION BY document_name 29 | ORDER BY 30 | timestamp DESC 31 | ) AS operation, 32 | FIRST_VALUE(data) OVER( 33 | PARTITION BY document_name 34 | ORDER BY 35 | timestamp DESC 36 | ) AS data, 37 | FIRST_VALUE(operation) OVER( 38 | PARTITION BY document_name 39 | ORDER BY 40 | timestamp DESC 41 | ) = "DELETE" AS is_deleted 42 | FROM 43 | `test.test_dataset.test_table` 44 | ORDER BY 45 | document_name, 46 | timestamp DESC 47 | ) 48 | WHERE 49 | NOT is_deleted 50 | GROUP BY 51 | document_name, 52 | document_id, 53 | timestamp, 54 | event_id, 55 | operation, 56 | data -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/latestConsistentSnapshotNoGroupBy.sql: -------------------------------------------------------------------------------- 1 | -- Retrieves the latest document change events for all live documents. 2 | -- timestamp: The Firestore timestamp at which the event took place. 3 | -- operation: One of INSERT, UPDATE, DELETE, IMPORT. 4 | -- event_id: The id of the event that triggered the cloud function mirrored the event. 5 | -- data: A raw JSON payload of the current state of the document. 6 | SELECT 7 | document_name, 8 | document_id 9 | FROM 10 | ( 11 | SELECT 12 | document_name, 13 | document_id, 14 | FIRST_VALUE(operation) OVER( 15 | PARTITION BY document_name 16 | ORDER BY 17 | timestamp DESC 18 | ) = "DELETE" AS is_deleted 19 | FROM 20 | `test.test_dataset.test_table` 21 | ORDER BY 22 | document_name, 23 | timestamp DESC 24 | ) 25 | WHERE 26 | NOT is_deleted 27 | GROUP BY 28 | document_name, 29 | document_id -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/nestedMapSchemaChangeLog.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | document_name, 3 | document_id, 4 | timestamp, 5 | operation, 6 | `test.test_dataset.firestoreNumber`( 7 | JSON_EXTRACT_SCALAR(data, '$.super.nested.schema.value') 8 | ) AS super_nested_schema_value 9 | FROM 10 | `test.test_dataset.test_table` -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/fixtures/sql/referenceSchema.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | document_name, 3 | document_id, 4 | timestamp, 5 | operation, 6 | JSON_EXTRACT_SCALAR(data, '$.reference') AS reference, 7 | JSON_EXTRACT_SCALAR(data, '$.map.reference') AS map_reference 8 | FROM 9 | `test.test_dataset.test_table` -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/src/logs.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { FirestoreSchema } from "./schema"; 18 | 19 | export const bigQuerySchemaViewCreating = ( 20 | name: string, 21 | schema: FirestoreSchema, 22 | query 23 | ) => { 24 | console.log( 25 | `BigQuery creating schema view ${name}:\nSchema:\n` + 26 | `${JSON.stringify(schema)}\nQuery:\n${query}` 27 | ); 28 | }; 29 | 30 | export const bigQuerySchemaViewCreated = (name: string) => { 31 | console.log(`BigQuery created schema view ${name}\n`); 32 | }; 33 | 34 | export const bigQueryViewCreating = (view: string) => { 35 | console.log(`BigQuery created view ${view}`); 36 | }; 37 | 38 | export const bigQueryViewCreated = (view: string) => { 39 | console.log(`BigQuery created view ${view}`); 40 | }; 41 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/gen-schema-view/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["ESNext"], 4 | "module": "CommonJS", 5 | "moduleResolution": "node", 6 | "noImplicitReturns": true, 7 | "outDir": "lib", 8 | "sourceMap": false, 9 | "target": "ES2020", 10 | "esModuleInterop": true, 11 | "skipLibCheck": true 12 | }, 13 | "compileOnSave": true, 14 | "include": ["src"] 15 | } 16 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/import/.gitignore: -------------------------------------------------------------------------------- 1 | lib 2 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/import/README.md: -------------------------------------------------------------------------------- 1 | The `fs-bq-import-collection` script is for use with the official Firebase Extension [_Stream Firestore to BigQuery_](https://github.com/firebase/extensions/tree/master/firestore-bigquery-export). 2 | 3 | This script reads all existing documents in a specified Firestore Collection, and updates the changelog table used by the `firestore-bigquery-export` extension. 4 | 5 | A guide on how to use the script can be found [here](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/IMPORT_EXISTING_DOCUMENTS.md). -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/import/__tests__/firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "emulators": { 3 | "auth": { 4 | "port": 9099 5 | }, 6 | "firestore": { 7 | "port": 8080 8 | }, 9 | "functions": { 10 | "source": "../../../functions", 11 | "port": 5001 12 | }, 13 | "ui": { 14 | "enabled": true 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/import/__tests__/helpers/waitFor.ts: -------------------------------------------------------------------------------- 1 | export async function repeat( 2 | fn: { (): Promise; (): any }, 3 | until: { ($: any): any; (arg0: any): any }, 4 | retriesLeft = 5, 5 | interval = 1000 6 | ) { 7 | const result = await fn(); 8 | 9 | if (!until(result)) { 10 | if (retriesLeft) { 11 | await new Promise((r) => setTimeout(r, interval)); 12 | return repeat(fn, until, retriesLeft - 1, interval); 13 | } 14 | throw new Error("Max repeats count reached"); 15 | } 16 | 17 | return result; 18 | } 19 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/import/__tests__/test.types.d.ts: -------------------------------------------------------------------------------- 1 | declare namespace NodeJS { 2 | interface Global { 3 | config: () => jest.ModuleMocker; 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/import/__tests__/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "files": ["test.types.d.ts"], 4 | "include": ["**/*"] 5 | } 6 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/import/jest.config.js: -------------------------------------------------------------------------------- 1 | const packageJson = require("./package.json"); 2 | 3 | module.exports = { 4 | displayName: packageJson.name, 5 | rootDir: "./", 6 | globals: { 7 | "ts-jest": { 8 | tsConfig: "/__tests__/tsconfig.json", 9 | }, 10 | }, 11 | preset: "ts-jest", 12 | testEnvironment: "node", 13 | testMatch: ["**/__tests__/**/*.test.ts"], 14 | testEnvironment: "node", 15 | testTimeout: 180000, 16 | collectCoverage: true, 17 | testPathIgnorePatterns: process.env.CI_TEST === "true" ? ["example"] : [], 18 | }; 19 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/import/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@firebaseextensions/fs-bq-import-collection", 3 | "version": "0.1.24", 4 | "description": "Import a Firestore Collection into a BigQuery Changelog Table", 5 | "main": "./lib/index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "github.com/firebase/extensions.git", 9 | "directory": "firestore-bigquery-export/scripts/import" 10 | }, 11 | "scripts": { 12 | "build": "npm run clean && npm run compile", 13 | "build-watch": "npm run clean && tsc --watch", 14 | "clean": "rimraf ./lib", 15 | "compile": "tsc", 16 | "import": "node ./lib/index.js", 17 | "prepare": "npm run build", 18 | "test:local": "jest" 19 | }, 20 | "files": [ 21 | "lib" 22 | ], 23 | "bin": { 24 | "fs-bq-import-collection": "./lib/index.js" 25 | }, 26 | "author": "Jan Wyszynski ", 27 | "license": "Apache-2.0", 28 | "dependencies": { 29 | "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.40", 30 | "@google-cloud/bigquery": "^5.6.0", 31 | "commander": "5.0.0", 32 | "filenamify": "^4.2.0", 33 | "firebase-admin": "^12.1.0", 34 | "firebase-functions": "^4.2.0", 35 | "generate-schema": "^2.6.0", 36 | "inquirer": "^6.4.0", 37 | "sql-formatter": "^2.3.3", 38 | "workerpool": "^6.1.4" 39 | }, 40 | "devDependencies": { 41 | "@types/chai": "^4.2.0", 42 | "@types/jest": "29.5.0", 43 | "@types/workerpool": "^6.0.0", 44 | "chai": "^4.2.0", 45 | "dotenv": "^16.3.1", 46 | "jest": "29.5.0", 47 | "nanoid": "^5.0.9", 48 | "rimraf": "^2.6.3", 49 | "ts-jest": "29.1.2", 50 | "ts-node": "^10.9.1", 51 | "typescript": "^4.2.4" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/import/src/types.ts: -------------------------------------------------------------------------------- 1 | import * as admin from "firebase-admin"; 2 | 3 | export interface CliConfig { 4 | kind: "CONFIG"; 5 | projectId: string; 6 | bigQueryProjectId: string; 7 | sourceCollectionPath: string; 8 | datasetId: string; 9 | tableId: string; 10 | batchSize: number; 11 | queryCollectionGroup: boolean; 12 | datasetLocation: string; 13 | multiThreaded: boolean; 14 | useNewSnapshotQuerySyntax: boolean; 15 | useEmulator: boolean; 16 | rawChangeLogName: string; 17 | cursorPositionFile: string; 18 | failedBatchOutput?: string; 19 | transformFunctionUrl?: string; 20 | } 21 | 22 | export interface CliConfigError { 23 | kind: "ERROR"; 24 | errors: string[]; 25 | } 26 | 27 | export interface SerializableQuery { 28 | startAt?: { 29 | before: boolean; 30 | values: Array<{ 31 | referenceValue: string; 32 | valueType: string; 33 | }>; 34 | }; 35 | endAt?: { 36 | before: boolean; 37 | values: Array<{ 38 | referenceValue: string; 39 | valueType: string; 40 | }>; 41 | }; 42 | limit?: number; 43 | offset?: number; 44 | } 45 | 46 | export interface QueryOptions 47 | extends admin.firestore.Query> { 48 | _queryOptions: SerializableQuery; 49 | } 50 | -------------------------------------------------------------------------------- /firestore-bigquery-export/scripts/import/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2018"], 4 | "module": "commonjs", 5 | "noImplicitReturns": true, 6 | "outDir": "lib", 7 | "sourceMap": false, 8 | "target": "es2018", 9 | "types": ["node", "chai", "jest"] 10 | }, 11 | "compileOnSave": true, 12 | "include": ["src"], 13 | "exclude": ["**/node_modules/@types/jest/**"] 14 | } 15 | -------------------------------------------------------------------------------- /firestore-counter/.gitignore: -------------------------------------------------------------------------------- 1 | test-project-key.json 2 | 3 | .vscode 4 | node_modules 5 | *.js.map 6 | functions/lib/test/*.js 7 | stress_test/**/*.js 8 | 9 | functions/tasks.json 10 | clients/dart/.** -------------------------------------------------------------------------------- /firestore-counter/clients/android/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'com.android.library' 2 | apply plugin: 'com.github.sherter.google-java-format' 3 | 4 | buildscript { 5 | repositories { 6 | google() 7 | jcenter() 8 | maven { 9 | url "https://plugins.gradle.org/m2/" 10 | } 11 | } 12 | dependencies { 13 | classpath 'com.android.tools.build:gradle:3.4.2' 14 | classpath 'gradle.plugin.com.github.sherter.google-java-format:google-java-format-gradle-plugin:0.8' 15 | } 16 | } 17 | 18 | allprojects { 19 | repositories { 20 | google() 21 | jcenter() 22 | } 23 | } 24 | 25 | android { 26 | compileSdkVersion 26 27 | 28 | defaultConfig { 29 | minSdkVersion 16 30 | targetSdkVersion 26 31 | versionCode 1 32 | versionName "1.0" 33 | } 34 | 35 | buildTypes { 36 | release { 37 | minifyEnabled false 38 | proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' 39 | } 40 | } 41 | } 42 | 43 | dependencies { 44 | implementation fileTree(dir: 'libs', include: ['*.jar']) 45 | 46 | implementation 'com.android.support:appcompat-v7:26.1.0' 47 | implementation 'com.google.firebase:firebase-core:17.1.0' 48 | implementation 'com.google.firebase:firebase-firestore:21.0.0' 49 | } 50 | 51 | task sourcesJar(type: Jar) { 52 | classifier = 'sources' 53 | from android.sourceSets.main.java.srcDirs 54 | } 55 | 56 | artifacts { 57 | archives sourcesJar 58 | } 59 | 60 | googleJavaFormat { 61 | toolVersion = '1.6' 62 | } 63 | tasks.googleJavaFormat { 64 | source '.' 65 | include '**/*.java' 66 | exclude '**/generated/**' 67 | } 68 | tasks.verifyGoogleJavaFormat { 69 | source '.' 70 | include '**/*.java' 71 | exclude '**/generated/**' 72 | } 73 | -------------------------------------------------------------------------------- /firestore-counter/clients/android/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/firestore-counter/clients/android/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /firestore-counter/clients/android/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Fri Aug 23 07:25:31 BST 2019 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip 7 | -------------------------------------------------------------------------------- /firestore-counter/clients/android/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | -------------------------------------------------------------------------------- /firestore-counter/clients/dart/README.md: -------------------------------------------------------------------------------- 1 | # Flutter/Dart Client 2 | 3 | The Flutter/Dart client for Distributed Counter. 4 | -------------------------------------------------------------------------------- /firestore-counter/clients/dart/analysis_options.yaml: -------------------------------------------------------------------------------- 1 | # This file configures the analyzer, which statically analyzes Dart code to 2 | # check for errors, warnings, and lints. 3 | # 4 | # The issues identified by the analyzer are surfaced in the UI of Dart-enabled 5 | # IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be 6 | # invoked from the command line by running `flutter analyze`. 7 | 8 | # The following line activates a set of recommended lints for Flutter apps, 9 | # packages, and plugins designed to encourage good coding practices. 10 | include: package:flutter_lints/flutter.yaml 11 | 12 | linter: 13 | # The lint rules applied to this project can be customized in the 14 | # section below to disable rules from the `package:flutter_lints/flutter.yaml` 15 | # included above or to enable additional rules. A list of all available lints 16 | # and their documentation is published at 17 | # https://dart-lang.github.io/linter/lints/index.html. 18 | # 19 | # Instead of disabling a lint rule for the entire project in the 20 | # section below, it can also be suppressed for a single line of code 21 | # or a specific dart file by using the `// ignore: name_of_lint` and 22 | # `// ignore_for_file: name_of_lint` syntax on the line or in the file 23 | # producing the lint. 24 | rules: 25 | # avoid_print: false # Uncomment to disable the `avoid_print` rule 26 | # prefer_single_quotes: true # Uncomment to enable the `prefer_single_quotes` rule 27 | # Additional information about this file can be found at 28 | # https://dart.dev/guides/language/analysis-options 29 | -------------------------------------------------------------------------------- /firestore-counter/clients/dart/integration_test/e2e_distributed_counter_test.dart: -------------------------------------------------------------------------------- 1 | import 'package:cloud_firestore/cloud_firestore.dart'; 2 | import 'package:firebase_core/firebase_core.dart'; 3 | import 'package:firestore_counter/distributed_counter.dart'; 4 | import 'package:flutter_test/flutter_test.dart'; 5 | import 'package:integration_test/integration_test.dart'; 6 | 7 | void main() { 8 | IntegrationTestWidgetsFlutterBinding.ensureInitialized(); 9 | 10 | late final FirebaseFirestore firestore; 11 | late final DocumentReference document; 12 | 13 | setUpAll(() async { 14 | await Firebase.initializeApp( 15 | options: FirebaseOptions( 16 | apiKey: '123', 17 | appId: '123', 18 | messagingSenderId: '123', 19 | projectId: 'demo', 20 | ), 21 | ); 22 | firestore = FirebaseFirestore.instance; 23 | firestore.useFirestoreEmulator('localhost', 8080); 24 | document = firestore.doc('pages/hello-world'); 25 | document.set({'visits': 0}); 26 | }); 27 | 28 | test('Count is initially 0', () async { 29 | final distCounter = DistributedCounter(document, 'visits'); 30 | 31 | expectLater(await distCounter.get(), equals(0)); 32 | }); 33 | 34 | test('Increment by 1', () async { 35 | final distCounter = DistributedCounter(document, 'visits'); 36 | distCounter.incrementBy(1); 37 | expectLater(await distCounter.get(), equals(2)); 38 | }); 39 | } 40 | -------------------------------------------------------------------------------- /firestore-counter/clients/dart/pubspec.yaml: -------------------------------------------------------------------------------- 1 | name: firestore_counter 2 | description: A Dart client for using Firestore Distributed Counter. 3 | version: 0.0.1 4 | 5 | publish_to: none 6 | 7 | environment: 8 | sdk: ">=2.17.0 <3.0.0" 9 | flutter: ">=3.0.0 <3.3.4" 10 | 11 | dependencies: 12 | flutter: 13 | sdk: flutter 14 | async: ^2.9.0 15 | cloud_firestore: ^4.3.1 16 | firebase_core: ^2.4.1 17 | uuid: ^3.0.7 18 | 19 | dev_dependencies: 20 | flutter_test: 21 | sdk: flutter 22 | integration_test: 23 | sdk: flutter 24 | -------------------------------------------------------------------------------- /firestore-counter/clients/dart/test_driver/integration_test.dart: -------------------------------------------------------------------------------- 1 | import 'package:integration_test/integration_test_driver.dart'; 2 | 3 | Future main() => integrationDriver(); 4 | -------------------------------------------------------------------------------- /firestore-counter/clients/dart/web/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/firestore-counter/clients/dart/web/favicon.png -------------------------------------------------------------------------------- /firestore-counter/clients/dart/web/icons/Icon-192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/firestore-counter/clients/dart/web/icons/Icon-192.png -------------------------------------------------------------------------------- /firestore-counter/clients/dart/web/icons/Icon-512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/firestore-counter/clients/dart/web/icons/Icon-512.png -------------------------------------------------------------------------------- /firestore-counter/clients/dart/web/icons/Icon-maskable-192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/firestore-counter/clients/dart/web/icons/Icon-maskable-192.png -------------------------------------------------------------------------------- /firestore-counter/clients/dart/web/icons/Icon-maskable-512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/firestore-counter/clients/dart/web/icons/Icon-maskable-512.png -------------------------------------------------------------------------------- /firestore-counter/clients/dart/web/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dart", 3 | "short_name": "dart", 4 | "start_url": ".", 5 | "display": "standalone", 6 | "background_color": "#0175C2", 7 | "theme_color": "#0175C2", 8 | "description": "A new Flutter project.", 9 | "orientation": "portrait-primary", 10 | "prefer_related_applications": false, 11 | "icons": [ 12 | { 13 | "src": "icons/Icon-192.png", 14 | "sizes": "192x192", 15 | "type": "image/png" 16 | }, 17 | { 18 | "src": "icons/Icon-512.png", 19 | "sizes": "512x512", 20 | "type": "image/png" 21 | }, 22 | { 23 | "src": "icons/Icon-maskable-192.png", 24 | "sizes": "192x192", 25 | "type": "image/png", 26 | "purpose": "maskable" 27 | }, 28 | { 29 | "src": "icons/Icon-maskable-512.png", 30 | "sizes": "512x512", 31 | "type": "image/png", 32 | "purpose": "maskable" 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /firestore-counter/clients/ios/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.3 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "FirestoreCounter", 8 | platforms: [ 9 | .iOS(.v10) 10 | ], 11 | products: [ 12 | // Products define the executables and libraries a package produces, and make them visible to other packages. 13 | .library( 14 | name: "FirestoreCounter", 15 | targets: ["FirestoreCounter"]), 16 | ], 17 | dependencies: [ 18 | // Dependencies declare other packages that this package depends on. 19 | // .package(url: /* package url */, from: "1.0.0"), 20 | .package(name: "Firebase", 21 | url: "https://github.com/firebase/firebase-ios-sdk.git", 22 | .branch("7.0.0")) 23 | ], 24 | targets: [ 25 | // Targets are the basic building blocks of a package. A target can define a module or a test suite. 26 | // Targets can depend on other targets in this package, and on products in packages this package depends on. 27 | .target( 28 | name: "FirestoreCounter", 29 | dependencies: [ 30 | .product(name: "FirebaseFirestore", package: "Firebase") 31 | ]), 32 | ] 33 | ) 34 | -------------------------------------------------------------------------------- /firestore-counter/clients/node/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firestore-counter-node", 3 | "version": "1.0.0", 4 | "description": "Node.js admin SDK to access sharded counters", 5 | "main": "index.js", 6 | "author": "Mansehej Singh ", 7 | "license": "Apache-2.0", 8 | "devDependencies": { 9 | "firebase-admin": "^11.4.1" 10 | }, 11 | "dependencies": { 12 | "uuid": "^8.3.2" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /firestore-counter/clients/web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firestore-counter-web", 3 | "version": "1.0.0", 4 | "main": "src/index.js", 5 | "author": "patryk@google.com", 6 | "license": "Apache-2.0", 7 | "description": "Web SDK to access sharded counters.", 8 | "dependencies": { 9 | "@types/uuid": "^3.4.4", 10 | "uuid": "^3.3.2" 11 | }, 12 | "devDependencies": { 13 | "firebase": "^9.6.1", 14 | "prettier": "1.15.3", 15 | "ts-loader": "^6.0.4", 16 | "typescript": "^4.5.4", 17 | "webpack": "^4.35.0", 18 | "webpack-cli": "^4.9.1" 19 | }, 20 | "scripts": { 21 | "build": "npx webpack", 22 | "format": "prettier --write {,**/}*.{yaml,ts,md}" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /firestore-counter/clients/web/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2020", 4 | "lib": ["es2020", "dom"], 5 | "module": "commonjs", 6 | "outDir": "./dist/", 7 | "sourceMap": true, 8 | "noImplicitAny": true, 9 | "types": ["uuid"] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /firestore-counter/clients/web/webpack.config.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const path = require("path"); 18 | 19 | module.exports = { 20 | entry: "./src/index.ts", 21 | devtool: "inline-source-map", 22 | mode: "production", 23 | module: { 24 | rules: [ 25 | { 26 | test: /\.tsx?$/, 27 | use: "ts-loader", 28 | exclude: /node_modules/, 29 | }, 30 | ], 31 | }, 32 | resolve: { 33 | extensions: [".tsx", ".ts", ".js"], 34 | }, 35 | output: { 36 | filename: "sharded-counter.js", 37 | library: "sharded", 38 | libraryTarget: "var", 39 | path: path.resolve(__dirname, "dist"), 40 | }, 41 | }; 42 | -------------------------------------------------------------------------------- /firestore-counter/functions/.gitignore: -------------------------------------------------------------------------------- 1 | lib -------------------------------------------------------------------------------- /firestore-counter/functions/__tests__/test.types.d.ts: -------------------------------------------------------------------------------- 1 | declare namespace NodeJS { 2 | interface Global { 3 | config: () => jest.ModuleMocker; 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /firestore-counter/functions/__tests__/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "files": ["test.types.d.ts"], 4 | "include": ["**/*"], 5 | "compilerOptions": { 6 | "types": ["jest"], 7 | "esModuleInterop": true 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /firestore-counter/functions/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | rootDir: "./", 3 | preset: "ts-jest", 4 | testMatch: ["**/__tests__/*.test.ts"], 5 | globals: { 6 | "ts-jest": { 7 | tsConfig: "/__tests__/tsconfig.json", 8 | }, 9 | }, 10 | snapshotFormat: { 11 | escapeString: true, 12 | printBasicPrototype: true, 13 | }, 14 | testEnvironment: "node", 15 | testPathIgnorePatterns: ["e2e"], 16 | moduleNameMapper: { 17 | "firebase-admin/app": "/node_modules/firebase-admin/lib/app", 18 | "firebase-admin/eventarc": 19 | "/node_modules/firebase-admin/lib/eventarc", 20 | "firebase-admin/database": 21 | "/node_modules/firebase-admin/lib/database", 22 | "firebase-admin/auth": "/node_modules/firebase-admin/lib/auth", 23 | "firebase-functions/encoder": 24 | "/node_modules/firebase-functions/lib/encoder", 25 | "firebase-admin/database": 26 | "/node_modules/firebase-admin/lib/database", 27 | "firebase-functions/lib/encoder": 28 | "/node_modules/firebase-functions-test/lib/providers/firestore.js", 29 | "firebase-admin/firestore": 30 | "/node_modules/firebase-functions/lib/v1/providers/firestore.js", 31 | }, 32 | }; 33 | -------------------------------------------------------------------------------- /firestore-counter/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firestore-counter-functions", 3 | "main": "lib/index.js", 4 | "author": "patryk@google.com", 5 | "license": "Apache-2.0", 6 | "description": "Auto-scalable counters for your app.", 7 | "dependencies": { 8 | "deep-equal": "^1.0.1", 9 | "firebase-admin": "^12.1.0", 10 | "firebase-functions": "^4.9.0", 11 | "uuid": "^3.3.2", 12 | "rimraf": "^2.6.3", 13 | "typescript": "^4.9.4", 14 | "@types/express-serve-static-core": "4.17.30" 15 | }, 16 | "devDependencies": { 17 | "@types/deep-equal": "^1.0.1", 18 | "prettier": "1.15.3", 19 | "ts-node": "^7.0.1", 20 | "wait-for-expect": "^3.0.2", 21 | "@types/jest": "29.5.0", 22 | "jest": "29.5.0", 23 | "ts-jest": "29.1.2" 24 | }, 25 | "scripts": { 26 | "prepare": "npm run build", 27 | "build": "npm run clean && npm run compile", 28 | "clean": "rimraf lib", 29 | "compile": "tsc", 30 | "format": "prettier --write {,**/}*.{yaml,ts,md}", 31 | "test:local": "jest", 32 | "generate-readme": "firebase ext:info .. --markdown > ../README.md" 33 | }, 34 | "private": true 35 | } 36 | -------------------------------------------------------------------------------- /firestore-counter/functions/src/events.ts: -------------------------------------------------------------------------------- 1 | import * as eventArc from "firebase-admin/eventarc"; 2 | 3 | const { getEventarc } = eventArc; 4 | 5 | const EXTENSION_NAME = "firestore-counter"; 6 | 7 | const getEventType = (eventName: string) => 8 | `firebase.extensions.${EXTENSION_NAME}.v1.${eventName}`; 9 | 10 | let eventChannel: eventArc.Channel; 11 | 12 | /** setup events */ 13 | export const setupEventChannel = () => { 14 | eventChannel = 15 | process.env.EVENTARC_CHANNEL && 16 | getEventarc().channel(process.env.EVENTARC_CHANNEL, { 17 | allowedEventTypes: process.env.EXT_SELECTED_EVENTS, 18 | }); 19 | }; 20 | 21 | export const recordStartEvent = async (data: string | object) => { 22 | if (!eventChannel) return; 23 | 24 | return eventChannel.publish({ 25 | type: getEventType("onStart"), 26 | data, 27 | }); 28 | }; 29 | 30 | export const recordErrorEvent = async (err: Error, subject?: string) => { 31 | if (!eventChannel) return; 32 | 33 | return eventChannel.publish({ 34 | type: getEventType("onError"), 35 | data: { message: err.message }, 36 | subject, 37 | }); 38 | }; 39 | 40 | export const recordSuccessEvent = async ({ 41 | subject, 42 | data, 43 | }: { 44 | subject: string; 45 | data: string | object; 46 | }) => { 47 | if (!eventChannel) return; 48 | 49 | return eventChannel.publish({ 50 | type: getEventType("onSuccess"), 51 | subject, 52 | data, 53 | }); 54 | }; 55 | 56 | export const recordCompletionEvent = async (data: string | object) => { 57 | if (!eventChannel) return; 58 | 59 | return eventChannel.publish({ 60 | type: getEventType("onCompletion"), 61 | data, 62 | }); 63 | }; 64 | -------------------------------------------------------------------------------- /firestore-counter/functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "lib": ["ES2020"], 5 | "outDir": "lib", 6 | "types": ["node"], 7 | "module": "commonjs", 8 | "noImplicitReturns": true, 9 | "sourceMap": false, 10 | "esModuleInterop": true 11 | }, 12 | "include": ["src"] 13 | } 14 | -------------------------------------------------------------------------------- /firestore-counter/stress_test/bin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stress-test-driver", 3 | "author": "patryk@google.com", 4 | "description": "Stress test for sharded counter.", 5 | "dependencies": { 6 | "@google-cloud/firestore": "^1.3.0", 7 | "@types/uuid": "^3.4.4", 8 | "firebase-admin": "^11.4.1", 9 | "uuid": "^3.3.2" 10 | }, 11 | "devDependencies": { 12 | "prettier": "1.15.3", 13 | "typescript": "^3.5.2" 14 | }, 15 | "scripts": { 16 | "build": "tsc", 17 | "format": "prettier --write {,**/}*.{yaml,ts,md}" 18 | }, 19 | "private": true 20 | } 21 | -------------------------------------------------------------------------------- /firestore-counter/stress_test/bin/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "noImplicitReturns": true, 5 | "noUnusedLocals": true, 6 | "strict": true, 7 | "target": "es2015" 8 | }, 9 | "compileOnSave": true 10 | } 11 | -------------------------------------------------------------------------------- /firestore-counter/stress_test/firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "functions": { 3 | "predeploy": ["npm --prefix \"$RESOURCE_DIR\" run build"] 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /firestore-counter/stress_test/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firestore-counter-stress-test-functions", 3 | "scripts": { 4 | "lint": "tslint --project tsconfig.json", 5 | "build": "tsc", 6 | "serve": "npm run build && firebase serve --only functions", 7 | "shell": "npm run build && firebase functions:shell", 8 | "start": "npm run shell", 9 | "deploy": "firebase deploy --only functions", 10 | "logs": "firebase functions:log" 11 | }, 12 | "main": "lib/index.js", 13 | "dependencies": { 14 | "firebase-admin": "^11.4.1", 15 | "firebase-functions": "^2.2.0" 16 | }, 17 | "devDependencies": { 18 | "tslint": "^5.12.0", 19 | "typescript": "^3.5.2" 20 | }, 21 | "private": true 22 | } 23 | -------------------------------------------------------------------------------- /firestore-counter/stress_test/functions/src/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import * as functions from "firebase-functions"; 18 | 19 | interface TaskDoc { 20 | path: string; 21 | object: { [key: string]: any }; 22 | } 23 | 24 | interface TaskInfo { 25 | docs: TaskDoc[]; 26 | } 27 | 28 | export const stressTestRunner = functions.firestore 29 | .document("stress_test_queue/{taskId}") 30 | .onCreate(async (snap, context) => { 31 | const db = snap.ref.firestore; 32 | let done = false; 33 | while (!done) { 34 | done = await db.runTransaction(async (t) => { 35 | const task = await t.get(snap.ref); 36 | if (!task.exists) return true; 37 | const taskInfo = task.data(); 38 | const doc = taskInfo.docs.pop(); 39 | if (doc) t.set(db.doc(doc.path), doc.object); 40 | if (taskInfo.docs.length > 0) { 41 | t.set(snap.ref, taskInfo); 42 | return false; 43 | } else { 44 | t.delete(snap.ref); 45 | return true; 46 | } 47 | }); 48 | } 49 | }); 50 | -------------------------------------------------------------------------------- /firestore-counter/stress_test/functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "noImplicitReturns": true, 5 | "noUnusedLocals": true, 6 | "outDir": "lib", 7 | "sourceMap": true, 8 | "strict": true, 9 | "target": "es2015" 10 | }, 11 | "compileOnSave": true, 12 | "include": ["src"] 13 | } 14 | -------------------------------------------------------------------------------- /firestore-send-email/.gitignore: -------------------------------------------------------------------------------- 1 | .gcloudignore 2 | !emulator-params.env -------------------------------------------------------------------------------- /firestore-send-email/functions/.gitignore: -------------------------------------------------------------------------------- 1 | lib -------------------------------------------------------------------------------- /firestore-send-email/functions/__tests__/createSMTPServer.ts: -------------------------------------------------------------------------------- 1 | const SMTPServer = require("smtp-server").SMTPServer; 2 | 3 | export const smtpServer = () => { 4 | const server = new SMTPServer({ 5 | secure: false, 6 | authOptional: true, 7 | closeTimeout: 1000, 8 | ignoreTLS: true, 9 | onData(stream, session, callback) { 10 | stream.pipe(process.stdout); // print message to console 11 | stream.on("end", () => { 12 | callback(null, "Accepted"); 13 | }); 14 | }, 15 | }); 16 | 17 | const port = 8132; 18 | const host = "127.0.0.1"; 19 | 20 | server.listen(port, () => console.log(`Server listening on ${host}:${port}`)); 21 | 22 | return server; 23 | }; 24 | -------------------------------------------------------------------------------- /firestore-send-email/functions/__tests__/functions.test.ts: -------------------------------------------------------------------------------- 1 | const { logger } = require("firebase-functions"); 2 | 3 | const consoleLogSpy = jest.spyOn(logger, "log").mockImplementation(); 4 | 5 | import { obfuscatedConfig } from "../src/logs"; 6 | import * as exportedFunctions from "../src"; 7 | 8 | describe("extension", () => { 9 | test("functions configuration is logged on initialize", async () => { 10 | expect(consoleLogSpy).toBeCalledWith( 11 | "Initializing extension with configuration", 12 | obfuscatedConfig 13 | ); 14 | }); 15 | 16 | test("functions are exported", async () => { 17 | expect(exportedFunctions.processQueue).toBeInstanceOf(Function); 18 | }); 19 | }); 20 | -------------------------------------------------------------------------------- /firestore-send-email/functions/__tests__/jest.setup.ts: -------------------------------------------------------------------------------- 1 | global.config = () => require("../src/config").default; 2 | -------------------------------------------------------------------------------- /firestore-send-email/functions/__tests__/test.types.d.ts: -------------------------------------------------------------------------------- 1 | declare namespace NodeJS { 2 | interface Global { 3 | config: () => jest.ModuleMocker; 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /firestore-send-email/functions/__tests__/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "files": ["test.types.d.ts"], 4 | "compilerOptions": { 5 | "outDir": "lib", 6 | "types": ["node", "jest"], 7 | "target": "ES2020", 8 | "skipLibCheck": true 9 | }, 10 | "include": ["**/*"] 11 | } 12 | -------------------------------------------------------------------------------- /firestore-send-email/functions/jest.config.js: -------------------------------------------------------------------------------- 1 | const packageJson = require("./package.json"); 2 | 3 | module.exports = { 4 | name: packageJson.name, 5 | displayName: packageJson.name, 6 | rootDir: "./", 7 | preset: "ts-jest", 8 | globals: { 9 | "ts-jest": { 10 | tsConfig: "/__tests__/tsconfig.json", 11 | }, 12 | }, 13 | snapshotFormat: { 14 | escapeString: true, 15 | printBasicPrototype: true, 16 | }, 17 | setupFiles: ["/__tests__/jest.setup.ts"], 18 | testMatch: ["**/__tests__/**/*.test.ts"], 19 | testEnvironment: "node", 20 | collectCoverageFrom: [ 21 | "src/**/*.{js,ts}", 22 | "!src/**/*.d.ts", 23 | "!**/__tests__/**", 24 | ], 25 | coverageDirectory: "coverage", 26 | coverageReporters: ["text", "lcov", "html"], 27 | moduleNameMapper: { 28 | "firebase-admin/app": "/node_modules/firebase-admin/lib/app", 29 | "firebase-admin/eventarc": 30 | "/node_modules/firebase-admin/lib/eventarc", 31 | "firebase-admin/database": 32 | "/node_modules/firebase-admin/lib/database", 33 | "firebase-admin/auth": "/node_modules/firebase-admin/lib/auth", 34 | "firebase-functions/encoder": 35 | "/node_modules/firebase-functions/lib/encoder", 36 | "firebase-admin/database": 37 | "/node_modules/firebase-admin/lib/database", 38 | "firebase-functions/lib/encoder": 39 | "/node_modules/firebase-functions-test/lib/providers/firestore.js", 40 | "firebase-admin/firestore": 41 | "/node_modules/firebase-functions/lib/v1/providers/firestore.js", 42 | }, 43 | }; 44 | -------------------------------------------------------------------------------- /firestore-send-email/functions/jest.teardown.js: -------------------------------------------------------------------------------- 1 | module.exports = async function () { 2 | delete process.env.LOCATION; 3 | delete process.env.MAIL_COLLECTION; 4 | delete process.env.SMTP_CONNECTION_URI; 5 | delete process.env.SMTP_PASSWORD; 6 | delete process.env.DEFAULT_FROM; 7 | delete process.env.DEFAULT_REPLY_TO; 8 | delete process.env.USERS_COLLECTION; 9 | delete process.env.TEMPLATES_COLLECTION; 10 | }; 11 | -------------------------------------------------------------------------------- /firestore-send-email/functions/src/config.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { AuthenticatonType, Config } from "./types"; 18 | 19 | const config: Config = { 20 | location: process.env.LOCATION, 21 | database: process.env.DATABASE || "(default)", 22 | databaseRegion: process.env.DATABASE_REGION || "us-central1", 23 | mailCollection: process.env.MAIL_COLLECTION, 24 | smtpConnectionUri: process.env.SMTP_CONNECTION_URI, 25 | smtpPassword: process.env.SMTP_PASSWORD, 26 | defaultFrom: process.env.DEFAULT_FROM, 27 | defaultReplyTo: process.env.DEFAULT_REPLY_TO, 28 | usersCollection: process.env.USERS_COLLECTION, 29 | templatesCollection: process.env.TEMPLATES_COLLECTION, 30 | testing: process.env.TESTING === "true", 31 | TTLExpireType: process.env.TTL_EXPIRE_TYPE, 32 | TTLExpireValue: parseInt(process.env.TTL_EXPIRE_VALUE), 33 | tls: process.env.TLS_OPTIONS || "{}", 34 | host: process.env.HOST, 35 | port: parseInt(process.env.OAUTH_PORT, null), 36 | secure: process.env.OAUTH_SECURE === "true", 37 | user: process.env.USER, 38 | clientId: process.env.CLIENT_ID, 39 | clientSecret: process.env.CLIENT_SECRET, 40 | refreshToken: process.env.REFRESH_TOKEN, 41 | authenticationType: process.env.AUTH_TYPE as AuthenticatonType, 42 | }; 43 | 44 | export default config; 45 | -------------------------------------------------------------------------------- /firestore-send-email/functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "lib", 4 | "target": "ES2020", 5 | "module": "commonjs", 6 | "noImplicitReturns": true, 7 | "sourceMap": false 8 | }, 9 | "compileOnSave": true, 10 | "include": ["src"] 11 | } 12 | -------------------------------------------------------------------------------- /firestore-shorten-urls-bitly/.gitignore: -------------------------------------------------------------------------------- 1 | lib -------------------------------------------------------------------------------- /firestore-shorten-urls-bitly/POSTINSTALL.md: -------------------------------------------------------------------------------- 1 | ### See it in action 2 | 3 | You can test out this extension right away! 4 | 5 | 1. Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/firestore/data) in the Firebase console. Note that, if you have configured a non-default firestore database, you may have to view it via the [Google Cloud Console](https://console.cloud.google.com/firestore/databases/${param:DATABASE}). 6 | 7 | 1. If it doesn't exist already, create a collection called `${param:COLLECTION_PATH}`. 8 | 9 | 1. Create a document with a field named `${param:URL_FIELD_NAME}` and make its value a URL such as `https://github.com/firebase/firebase-tools`. Make sure that the URL always includes the `https` or `http` protocol. 10 | 11 | 1. In a few seconds, you'll see a new field called `${param:SHORT_URL_FIELD_NAME}` pop up in the same document you just created; it will contain the shortened URL. 12 | 13 | ### Using the extension 14 | 15 | This extension listens to the Cloud Firestore collection `${param:COLLECTION_PATH}`. If you add a URL to the field `${param:URL_FIELD_NAME}` in any document within that collection, this extension: 16 | 17 | - Shortens the URL. 18 | - Saves the shortened URL in the `${param:SHORT_URL_FIELD_NAME}` field of the same document like so: 19 | 20 | ``` 21 | { 22 | ${param:URL_FIELD_NAME}: 'https://my.super.long-link.example.com/api/user/profile/-jEHitne10395-k3593085', 23 | ${param:SHORT_URL_FIELD_NAME}: 'https://bit.ly/EKDdza', 24 | } 25 | ``` 26 | 27 | If the original URL in a document is updated, then the shortened URL will be automatically updated, too. 28 | 29 | ### Monitoring 30 | 31 | As a best practice, you can [monitor the activity](https://firebase.google.com/docs/extensions/manage-installed-extensions#monitor) of your installed extension, including checks on its health, usage, and logs. 32 | -------------------------------------------------------------------------------- /firestore-shorten-urls-bitly/PREINSTALL.md: -------------------------------------------------------------------------------- 1 | Use this extension to create shortened URLs from URLs written to Cloud Firestore. These shortened URLs are useful as display URLs. 2 | 3 | This extension listens to your specified Cloud Firestore collection. If you add a URL to a specified field in any document within that collection, this extension: 4 | 5 | - Shortens the URL. 6 | - Saves the shortened URL in a new specified field in the same document. 7 | 8 | If the original URL in a document is updated, then the shortened URL will be automatically updated, too. 9 | 10 | This extension uses Bitly to shorten URLs, so you'll need to supply your Bitly access token as part of this extension's installation. You can generate this access token using [Bitly](https://bitly.com/a/oauth_apps). 11 | 12 | #### Additional setup 13 | 14 | Before installing this extension, make sure that you've [set up a Cloud Firestore database](https://firebase.google.com/docs/firestore/quickstart) in your Firebase project. 15 | 16 | You must also have a Bitly account and access token before installing this extension. 17 | 18 | #### Billing 19 | To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) 20 | 21 | - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s no-cost tier: 22 | - Cloud Firestore 23 | - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#extensions-pricing)) 24 | - This extension also uses these services: 25 | - [Bitly](https://bitly.com/). You must have a Bitly account and you're responsible for any associated charges. 26 | -------------------------------------------------------------------------------- /firestore-shorten-urls-bitly/functions/jest.config.js: -------------------------------------------------------------------------------- 1 | const packageJson = require("./package.json"); 2 | 3 | module.exports = { 4 | name: packageJson.name, 5 | displayName: packageJson.name, 6 | rootDir: "./", 7 | preset: "ts-jest", 8 | }; 9 | -------------------------------------------------------------------------------- /firestore-shorten-urls-bitly/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firestore-shorten-urls-bitly-functions", 3 | "description": "Automatically shorten url links", 4 | "author": "Chris Bianca ", 5 | "license": "Apache-2.0", 6 | "main": "lib/index.js", 7 | "scripts": { 8 | "prepare": "npm run build", 9 | "build": "npm run clean && npm run compile", 10 | "clean": "rimraf lib", 11 | "compile": "tsc", 12 | "test": "echo \"Error: no test specified\" && exit 1", 13 | "generate-readme": "firebase ext:info .. --markdown > ../README.md" 14 | }, 15 | "dependencies": { 16 | "@types/express-serve-static-core": "4.17.30", 17 | "@types/node": "^20.10.3", 18 | "firebase-admin": "^12.1.0", 19 | "firebase-functions": "^4.9.0", 20 | "rimraf": "^2.6.3", 21 | "typescript": "^4.8.4" 22 | }, 23 | "private": true 24 | } 25 | -------------------------------------------------------------------------------- /firestore-shorten-urls-bitly/functions/src/config.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export default { 18 | bitlyAccessToken: process.env.BITLY_ACCESS_TOKEN, 19 | collectionPath: process.env.COLLECTION_PATH, 20 | location: process.env.LOCATION, 21 | shortUrlFieldName: process.env.SHORT_URL_FIELD_NAME, 22 | urlFieldName: process.env.URL_FIELD_NAME, 23 | database: process.env.DATABASE, 24 | databaseRegion: process.env.DATABASE_REGION, 25 | }; 26 | -------------------------------------------------------------------------------- /firestore-shorten-urls-bitly/functions/src/events.ts: -------------------------------------------------------------------------------- 1 | import { Channel, getEventarc } from "firebase-admin/eventarc"; 2 | 3 | const EXTENSION_NAME = "firestore-shorten-urls-bitly"; 4 | 5 | const getEventType = (eventName: string) => 6 | `firebase.extensions.${EXTENSION_NAME}.v1.${eventName}`; 7 | 8 | let eventChannel: Channel | undefined; 9 | 10 | /** setup events */ 11 | export const setupEventChannel = () => { 12 | eventChannel = 13 | process.env.EVENTARC_CHANNEL && 14 | getEventarc().channel(process.env.EVENTARC_CHANNEL, { 15 | allowedEventTypes: process.env.EXT_SELECTED_EVENTS, 16 | }); 17 | }; 18 | 19 | export const recordStartEvent = async (data: string | object) => { 20 | if (!eventChannel) return; 21 | 22 | return eventChannel.publish({ 23 | type: getEventType("onStart"), 24 | data, 25 | }); 26 | }; 27 | 28 | export const recordErrorEvent = async (err: Error) => { 29 | if (!eventChannel) return; 30 | 31 | return eventChannel.publish({ 32 | type: getEventType("onError"), 33 | data: { message: err.message }, 34 | }); 35 | }; 36 | 37 | export const recordSuccessEvent = async ({ 38 | subject, 39 | data, 40 | }: { 41 | subject: string; 42 | data: string | object; 43 | }) => { 44 | if (!eventChannel) return; 45 | 46 | return eventChannel.publish({ 47 | type: getEventType("onSuccess"), 48 | subject, 49 | data, 50 | }); 51 | }; 52 | 53 | export const recordCompletionEvent = async (data: string | object) => { 54 | if (!eventChannel) return; 55 | 56 | return eventChannel.publish({ 57 | type: getEventType("onCompletion"), 58 | data, 59 | }); 60 | }; 61 | -------------------------------------------------------------------------------- /firestore-shorten-urls-bitly/functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es6"], 4 | "module": "commonjs", 5 | "noImplicitReturns": true, 6 | "sourceMap": false, 7 | "outDir": "lib", 8 | "target": "ES2020" 9 | }, 10 | "compileOnSave": true, 11 | 12 | "include": ["src"] 13 | } 14 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/.gitignore: -------------------------------------------------------------------------------- 1 | lib -------------------------------------------------------------------------------- /firestore-translate-text/functions/__tests__/__snapshots__/config.test.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`extension config config loaded from environment variables 1`] = ` 4 | Object { 5 | "doBackfill": false, 6 | "inputFieldName": "input", 7 | "languages": Array [ 8 | "en", 9 | "es", 10 | "de", 11 | "fr", 12 | ], 13 | "languagesFieldName": undefined, 14 | "location": "us-central1", 15 | "outputFieldName": "translated", 16 | } 17 | `; 18 | 19 | exports[`extension config config.LANGUAGES param exists 1`] = ` 20 | Object { 21 | "default": "en,es,de,fr", 22 | "description": "Into which target languages do you want to translate new strings? The languages are identified using ISO-639-1 codes in a comma-separated list, for example: en,es,de,fr. For these codes, visit the [supported languages list](https://cloud.google.com/translate/docs/languages). 23 | ", 24 | "example": "en,es,de,fr", 25 | "label": "Target languages for translations, as a comma-separated list", 26 | "param": "LANGUAGES", 27 | "required": true, 28 | "validationErrorMessage": "Languages must be a comma-separated list of ISO-639-1 language codes.", 29 | "validationRegex": "^[a-zA-Z,-]*[a-zA-Z-]{2,}$", 30 | } 31 | `; 32 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/__tests__/jest.setup.ts: -------------------------------------------------------------------------------- 1 | import { 2 | snapshot, 3 | mockDocumentSnapshotFactory, 4 | mockFirestoreUpdate, 5 | mockFirestoreTransaction, 6 | } from "./mocks/firestore"; 7 | import { 8 | testTranslations, 9 | mockTranslate, 10 | mockTranslateClassMethod, 11 | mockTranslateClass, 12 | mockTranslateModuleFactory, 13 | } from "./mocks/translate"; 14 | 15 | global.config = () => require("../src/config").default; 16 | 17 | global.snapshot = snapshot; 18 | 19 | global.testTranslations = testTranslations; 20 | 21 | global.mockDocumentSnapshotFactory = mockDocumentSnapshotFactory; 22 | 23 | global.mockTranslate = mockTranslate; 24 | 25 | global.mockTranslateClassMethod = mockTranslateClassMethod; 26 | 27 | global.mockTranslateClass = mockTranslateClass; 28 | 29 | global.mockTranslateModule = () => 30 | jest.mock("@google-cloud/translate", mockTranslateModuleFactory); 31 | 32 | global.mockFirestoreUpdate = mockFirestoreUpdate; 33 | 34 | global.mockFirestoreTransaction = mockFirestoreTransaction; 35 | 36 | global.clearMocks = () => { 37 | mockFirestoreUpdate.mockClear(); 38 | mockTranslateClassMethod.mockClear(); 39 | }; 40 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/__tests__/mocks/firestore.ts: -------------------------------------------------------------------------------- 1 | import * as functionsTestInit from "firebase-functions-test"; 2 | 3 | export const snapshot = ( 4 | input = { input: "hello" }, 5 | path = "translations/id1" 6 | ) => { 7 | let functionsTest = functionsTestInit(); 8 | return functionsTest.firestore.makeDocumentSnapshot(input, path); 9 | }; 10 | 11 | export const mockDocumentSnapshotFactory = (documentSnapshot) => { 12 | return jest.fn().mockImplementation(() => { 13 | return { 14 | exists: true, 15 | get: documentSnapshot.get.bind(documentSnapshot), 16 | ref: { path: documentSnapshot.ref.path }, 17 | }; 18 | })(); 19 | }; 20 | 21 | export const makeChange = (before, after) => { 22 | let functionsTest = functionsTestInit(); 23 | return functionsTest.makeChange(before, after); 24 | }; 25 | 26 | export const mockFirestoreTransaction = jest.fn().mockImplementation(() => { 27 | return (transactionHandler) => { 28 | transactionHandler({ 29 | update(ref, field, data) { 30 | mockFirestoreUpdate(field, data); 31 | }, 32 | }); 33 | }; 34 | }); 35 | 36 | export const mockFirestoreUpdate = jest.fn(); 37 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/__tests__/mocks/translate.ts: -------------------------------------------------------------------------------- 1 | import * as functionsTestInit from "firebase-functions-test"; 2 | 3 | export const testTranslations = { 4 | de: "hallo", 5 | en: "hello", 6 | es: "hola", 7 | fr: "salut", 8 | }; 9 | 10 | export const mockTranslate = () => { 11 | let functionsTest = functionsTestInit(); 12 | return functionsTest.wrap(require("../../src").fstranslate); 13 | }; 14 | 15 | // await translate.translate('hello', 'de'); 16 | export const mockTranslateClassMethod = jest 17 | .fn() 18 | .mockImplementation((string: string, targetLanguage: string) => { 19 | return Promise.resolve([testTranslations[targetLanguage]]); 20 | }); 21 | 22 | // new Translate(opts); 23 | export const mockTranslateClass = jest.fn().mockImplementation(() => { 24 | return { translate: mockTranslateClassMethod }; 25 | }); 26 | 27 | // import { Translate } from "@google-cloud/translate"; 28 | export function mockTranslateModuleFactory() { 29 | return { 30 | Translate: mockTranslateClass, 31 | }; 32 | } 33 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/__tests__/test.types.d.ts: -------------------------------------------------------------------------------- 1 | declare namespace NodeJS { 2 | interface Global { 3 | config: () => jest.ModuleMocker; 4 | snapshot: ( 5 | input?: { input?: any; changed?: number; notTheInput?: string }, 6 | path?: string 7 | ) => any; 8 | testTranslations: { 9 | de: string; 10 | en: string; 11 | es: string; 12 | fr: string; 13 | }; 14 | mockDocumentSnapshotFactory: ( 15 | documentSnapshot: any 16 | ) => jest.MockedFunction; 17 | mockTranslate: () => jest.MockedFunction; 18 | mockTranslateClassMethod: jest.MockedFunction; 19 | mockTranslateClass: jest.MockedClass; 20 | mockTranslateModule: () => jest.ModuleMocker; 21 | mockConsoleLog: jest.MockedFunction; 22 | mockConsoleError: jest.MockedFunction; 23 | mockFirestoreUpdate: jest.MockedFunction; 24 | mockFirestoreTransaction: jest.MockedFunction; 25 | clearMocks: () => void; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/__tests__/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "lib", 5 | "target": "es2018" 6 | }, 7 | "files": ["test.types.d.ts"], 8 | "include": ["**/*"] 9 | } 10 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/jest.config.js: -------------------------------------------------------------------------------- 1 | const packageJson = require("./package.json"); 2 | 3 | module.exports = { 4 | name: packageJson.name, 5 | displayName: packageJson.name, 6 | rootDir: "./", 7 | globals: { 8 | "ts-jest": { 9 | tsConfig: "/__tests__/tsconfig.json", 10 | }, 11 | }, 12 | snapshotFormat: { 13 | escapeString: true, 14 | printBasicPrototype: true, 15 | }, 16 | preset: "ts-jest", 17 | setupFiles: ["/__tests__/jest.setup.ts"], 18 | testMatch: ["**/__tests__/unit/translateMultipleBackfill.test.ts"], 19 | moduleNameMapper: { 20 | "firebase-admin/eventarc": 21 | "/node_modules/firebase-admin/lib/eventarc", 22 | "firebase-admin/auth": "/node_modules/firebase-admin/lib/auth", 23 | "firebase-admin/app": "/node_modules/firebase-admin/lib/app", 24 | "firebase-admin/database": 25 | "/node_modules/firebase-admin/lib/database", 26 | "firebase-admin/firestore": 27 | "/node_modules/firebase-admin/lib/firestore", 28 | "firebase-admin/functions": 29 | "/node_modules/firebase-admin/lib/functions", 30 | "firebase-functions/v2": "/node_modules/firebase-functions/lib/v2", 31 | "firebase-admin/extensions": 32 | "/node_modules/firebase-admin/lib/extensions", 33 | }, 34 | }; 35 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firestore-translate-text-functions", 3 | "description": "Firebase Cloud Functions for the Firestore Translate Text in Firestore Extension", 4 | "main": "lib/index.js", 5 | "license": "Apache-2.0", 6 | "scripts": { 7 | "prepare": "npm run build", 8 | "build": "npm run clean && npm run compile", 9 | "clean": "rimraf lib", 10 | "compile": "tsc", 11 | "test": "jest", 12 | "generate-readme": "firebase ext:info .. --markdown > ../README.md" 13 | }, 14 | "dependencies": { 15 | "@genkit-ai/googleai": "^0.9.7", 16 | "@genkit-ai/vertexai": "^0.9.7", 17 | "@google-cloud/translate": "^8.2.0", 18 | "@google-cloud/vertexai": "^1.9.2", 19 | "@types/express-serve-static-core": "4.19.0", 20 | "@types/node": "^20.10.3", 21 | "firebase-admin": "^12.1.0", 22 | "firebase-functions": "^4.9.0", 23 | "genkit": "^0.9.7", 24 | "rimraf": "^2.6.3", 25 | "typescript": "^4.8.4" 26 | }, 27 | "devDependencies": { 28 | "@types/jest": "29.5.0", 29 | "firebase-functions-test": "3.2.0", 30 | "jest": "29.5.0", 31 | "js-yaml": "^3.13.1", 32 | "mocked-env": "^1.3.1", 33 | "ts-jest": "29.1.2" 34 | }, 35 | "private": true 36 | } 37 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/src/config.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export default { 18 | doBackfill: false, 19 | languages: Array.from(new Set(process.env.LANGUAGES.split(","))), 20 | location: process.env.LOCATION, 21 | inputFieldName: process.env.INPUT_FIELD_NAME, 22 | outputFieldName: process.env.OUTPUT_FIELD_NAME, 23 | languagesFieldName: process.env.LANGUAGES_FIELD_NAME, 24 | useGenkit: process.env.TRANSLATION_MODEL === "gemini", 25 | geminiProvider: "googleai", 26 | googleAIAPIKey: process.env.GOOGLE_AI_API_KEY, 27 | }; 28 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/src/events.ts: -------------------------------------------------------------------------------- 1 | import * as eventArc from "firebase-admin/eventarc"; 2 | const { getEventarc } = eventArc; 3 | 4 | const EXTENSION_NAME = "firestore-translate-text"; 5 | 6 | const getEventType = (eventName: string) => 7 | `firebase.extensions.${EXTENSION_NAME}.v1.${eventName}`; 8 | 9 | let eventChannel: eventArc.Channel | undefined; 10 | 11 | /** setup events */ 12 | export const setupEventChannel = () => { 13 | eventChannel = 14 | process.env.EVENTARC_CHANNEL && 15 | getEventarc().channel(process.env.EVENTARC_CHANNEL, { 16 | allowedEventTypes: process.env.EXT_SELECTED_EVENTS, 17 | }); 18 | }; 19 | 20 | export const recordStartEvent = async (data: string | object) => { 21 | if (!eventChannel) return; 22 | 23 | return eventChannel.publish({ 24 | type: getEventType("onStart"), 25 | data, 26 | }); 27 | }; 28 | 29 | export const recordErrorEvent = async (err: Error) => { 30 | if (!eventChannel) return; 31 | 32 | return eventChannel.publish({ 33 | type: getEventType("onError"), 34 | data: { message: err.message }, 35 | }); 36 | }; 37 | 38 | export const recordSuccessEvent = async ({ 39 | subject, 40 | data, 41 | }: { 42 | subject: string; 43 | data: string | object; 44 | }) => { 45 | if (!eventChannel) return; 46 | 47 | return eventChannel.publish({ 48 | type: getEventType("onSuccess"), 49 | subject, 50 | data, 51 | }); 52 | }; 53 | 54 | export const recordCompletionEvent = async (data: string | object) => { 55 | if (!eventChannel) return; 56 | 57 | return eventChannel.publish({ 58 | type: getEventType("onCompletion"), 59 | data, 60 | }); 61 | }; 62 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/src/translate/index.ts: -------------------------------------------------------------------------------- 1 | export { 2 | translateDocument, 3 | translateDocumentBackfill, 4 | } from "./translateDocument"; 5 | 6 | export { updateTranslations, extractLanguages, extractInput } from "./common"; 7 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/src/translate/translateDocument.ts: -------------------------------------------------------------------------------- 1 | import * as logs from "../logs"; 2 | import * as admin from "firebase-admin"; 3 | import * as validators from "../validators"; 4 | import config from "../config"; 5 | import { 6 | extractInput, 7 | extractLanguages, 8 | extractOutput, 9 | filterLanguagesFn, 10 | translateString, 11 | Translation, 12 | updateTranslations, 13 | } from "./common"; 14 | import { 15 | translateMultiple, 16 | translateMultipleBackfill, 17 | } from "./translateMultiple"; 18 | import { translateSingle, translateSingleBackfill } from "./translateSingle"; 19 | 20 | export const translateDocumentBackfill = async ( 21 | snapshot: admin.firestore.DocumentSnapshot, 22 | bulkWriter: admin.firestore.BulkWriter 23 | ): Promise => { 24 | const input: any = extractInput(snapshot); 25 | 26 | if (typeof input === "object") { 27 | return translateMultipleBackfill(input, snapshot, bulkWriter); 28 | } 29 | 30 | await translateSingleBackfill(input, snapshot, bulkWriter); 31 | }; 32 | 33 | export const translateDocument = async ( 34 | snapshot: admin.firestore.DocumentSnapshot 35 | ): Promise => { 36 | const input: any = extractInput(snapshot); 37 | const languages = extractLanguages(snapshot); 38 | 39 | if ( 40 | validators.fieldNameIsTranslationPath( 41 | config.inputFieldName, 42 | config.outputFieldName, 43 | languages 44 | ) 45 | ) { 46 | logs.inputFieldNameIsOutputPath(); 47 | return; 48 | } 49 | 50 | if (typeof input === "object") { 51 | return translateMultiple(input, languages, snapshot); 52 | } 53 | 54 | await translateSingle(input, languages, snapshot); 55 | }; 56 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/src/validators.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const fieldNamesMatch = (field1: string, field2: string): boolean => 18 | field1 === field2; 19 | 20 | export const fieldNameIsTranslationPath = ( 21 | inputFieldName: string, 22 | outputFieldName: string, 23 | languages: string[] 24 | ): boolean => { 25 | for (const language of languages) { 26 | if (inputFieldName === `${outputFieldName}.${language}`) { 27 | return true; 28 | } 29 | } 30 | return false; 31 | }; 32 | -------------------------------------------------------------------------------- /firestore-translate-text/functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "lib": ["ES2020"], 5 | "module": "commonjs", 6 | "noImplicitReturns": true, 7 | "sourceMap": false, 8 | "outDir": "lib", 9 | "skipLibCheck": true 10 | }, 11 | "compileOnSave": true, 12 | "include": ["src"] 13 | } 14 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | projects: [ 3 | "/*/functions/jest.config.js", 4 | "/firestore-bigquery-export/scripts/gen-schema-view/jest.config.js", 5 | ], 6 | testPathIgnorePatterns: [ 7 | ".*/bin/", 8 | ".*/lib/", 9 | ".*/firestore-counter/", 10 | "/node_modules/", 11 | // Ignoring otherwise tests duplicate due to Jest `projects` 12 | ".*/__tests__/.*.ts", 13 | "/firestore-send-email/functions/__tests__/e2e.test.ts", 14 | ], 15 | preset: "ts-jest", 16 | testEnvironment: "node", 17 | collectCoverageFrom: [ 18 | "**/*.{ts,tsx}", 19 | "!**/node_modules/**", 20 | "!**/test-data/**", 21 | ], 22 | maxConcurrency: 10, 23 | }; 24 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": [ 3 | "*", 4 | "firestore-translate-text/functions", 5 | "rtdb-limit-child-nodes/functions", 6 | "firestore-shorten-urls-bitly/functions", 7 | "auth-mailchimp-sync/functions", 8 | "firestore-send-email/functions", 9 | "firestore-counter/functions", 10 | "delete-user-data/functions", 11 | "delete-user-data/test-data", 12 | "storage-resize-images/functions", 13 | "firestore-bigquery-export/functions", 14 | "firestore-bigquery-export/scripts/gen-schema-view", 15 | "firestore-bigquery-export/scripts/import", 16 | "firestore-bigquery-export/firestore-bigquery-change-tracker" 17 | ], 18 | "version": "0.0.0" 19 | } 20 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firebase-extensions", 3 | "version": "1.0.0", 4 | "description": "Repository of sample Firebase Extensions.", 5 | "private": true, 6 | "scripts": { 7 | "format": "prettier --write \"**/*.{js,md,yml,ts,json,yaml}\"", 8 | "lint": "prettier --list-different \"**/*.{js,md,yml,ts,json,yaml}\"", 9 | "clean": "lerna run --parallel clean && lerna clean", 10 | "build": "lerna run build", 11 | "local:emulator": "cd _emulator && firebase emulators:start -P demo-test", 12 | "test": "cd _emulator && firebase emulators:exec jest -P demo-test", 13 | "test:ci": "cd _emulator && firebase emulators:exec \"CI_TEST=true jest --detectOpenHandles --verbose --forceExit\" -P demo-test", 14 | "test:local": "concurrently \"npm run local:emulator\" \"jest\"", 15 | "test:watch": "concurrently \"npm run local:emulator\" \"jest --watch\"", 16 | "test-coverage": "jest --coverage --detectOpenHandles --forceExit", 17 | "postinstall": "if test \"$SKIP_POSTINSTALL\" != \"yes\" ; then lerna bootstrap --no-ci && lerna run --parallel clean && npm run build && npm run generate-package-locks ; fi", 18 | "generate-package-locks": "lerna exec -- npm i --package-lock-only", 19 | "generate-readmes": "lerna run --parallel generate-readme", 20 | "prepare": "husky install" 21 | }, 22 | "repository": "", 23 | "author": "Firebase (https://firebase.google.com/)", 24 | "license": "Apache-2.0", 25 | "bugs": { 26 | "url": "" 27 | }, 28 | "devDependencies": { 29 | "@types/jest": "29.5.0", 30 | "codecov": "^3.8.1", 31 | "concurrently": "^7.2.1", 32 | "husky": "^7.0.4", 33 | "jest": "^29.7.0", 34 | "lerna": "^3.4.3", 35 | "lint-staged": "^12.4.0", 36 | "prettier": "2.7.1", 37 | "ts-jest": "29.1.2", 38 | "typescript": "^4.8.4" 39 | }, 40 | "lint-staged": { 41 | "*.{js,md,yml,ts,json,yaml}": "prettier --write" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /rtdb-limit-child-nodes/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Version 0.1.15 2 | 3 | feat - move to Node.js 20 runtimes 4 | 5 | ## Version 0.1.14 6 | 7 | fixed - bump dependencies to fix vulnerabilities 8 | 9 | ## Version 0.1.13 10 | 11 | fixed - bump dependencies, fix vulnerabilities (#2061) 12 | 13 | ## Version 0.1.12 14 | 15 | fixed - updated vulnerable dependencies 16 | 17 | ## Version 0.1.11 18 | 19 | build - updated depenencies 20 | 21 | ## Version 0.1.10 22 | 23 | feature - bump to node 18 24 | 25 | ## Version 0.1.9 26 | 27 | feature - bump to nodejs16 28 | 29 | ## Version 0.1.8 30 | 31 | No changes. 32 | 33 | ## Version 0.1.7 34 | 35 | feature - upgrade extensions to the latest firebase-admin sdk 36 | 37 | ## Version 0.1.6 38 | 39 | fixed - generate correct `package-lock.json` files after `lerna bootstrap` (#779) 40 | 41 | fixed - update validate workflow to use node14 42 | 43 | ## Version 0.1.5 44 | 45 | feature - add Taiwan and Singapore Cloud Function locations (#729) 46 | 47 | ## Version 0.1.4 48 | 49 | feature - added Warsaw (europe-central2) location (#677) 50 | 51 | feature - updated Cloud Functions runtime to Node.js 14 (#660) 52 | 53 | fixed - Removed code coverage check on ci 54 | 55 | ## Version 0.1.3 56 | 57 | feature - Adds an option to select an alternative database instance (#504) 58 | 59 | ## Version 0.1.2 60 | 61 | feature - Add new Cloud Functions locations. For more information about locations and their pricing tiers, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). 62 | 63 | ## Version 0.1.1 64 | 65 | feature - Update Cloud Functions runtime to Node.js 10. 66 | 67 | ## Version 0.1.0 68 | 69 | Initial release of the _Limit Child Nodes_ extension. 70 | -------------------------------------------------------------------------------- /rtdb-limit-child-nodes/POSTINSTALL.md: -------------------------------------------------------------------------------- 1 | ### See it in action 2 | 3 | You can test out this extension right away! 4 | 5 | 1. Go to your [Realtime Database dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/${param:PROJECT_ID}/data) in the Firebase console. 6 | 7 | 1. In the path `${param:NODE_PATH}`, add more than ${param:MAX_COUNT} child nodes. 8 | 9 | 1. In a few seconds, you'll see the number of child nodes reduced to ${param:MAX_COUNT}. 10 | 11 | ### Using the extension 12 | 13 | If the number of nodes in `${param:NODE_PATH}` exceeds the max count of ${param:MAX_COUNT}, this extension deletes the oldest nodes first until there are ${param:MAX_COUNT} nodes remaining. 14 | 15 | We recommend adding data via pushing, for example `firebase.database().ref().child('${param:NODE_PATH}').push()`, because pushing assigns an automatically generated ID to the node in the database. During retrieval, these nodes are guaranteed to be ordered by the time they were added. Learn more about reading and writing data for your platform (iOS, Android, or Web) in the [Realtime Database documentation](https://firebase.google.com/docs/database/). 16 | 17 | ### Monitoring 18 | 19 | As a best practice, you can [monitor the activity](https://firebase.google.com/docs/extensions/manage-installed-extensions#monitor) of your installed extension, including checks on its health, usage, and logs. 20 | -------------------------------------------------------------------------------- /rtdb-limit-child-nodes/PREINSTALL.md: -------------------------------------------------------------------------------- 1 | Use this extension to control the maximum number of nodes stored in a Firebase Realtime Database path. 2 | 3 | If the number of nodes in your specified Realtime Database path exceeds the specified max count, this extension deletes the oldest nodes first until there are the max count number of nodes remaining. 4 | 5 | #### Additional setup 6 | 7 | Before installing this extension, make sure that you've [set up a Realtime Database instance](https://firebase.google.com/docs/database) in your Firebase project. 8 | 9 | ### Billing 10 | To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) 11 | 12 | - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s no-cost tier: 13 | - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#extensions-pricing)) 14 | - Firebase Realtime Database 15 | -------------------------------------------------------------------------------- /rtdb-limit-child-nodes/functions/.gitignore: -------------------------------------------------------------------------------- 1 | lib -------------------------------------------------------------------------------- /rtdb-limit-child-nodes/functions/jest.config.js: -------------------------------------------------------------------------------- 1 | const packageJson = require("./package.json"); 2 | 3 | module.exports = { 4 | name: packageJson.name, 5 | displayName: packageJson.name, 6 | rootDir: "./", 7 | preset: "ts-jest", 8 | }; 9 | -------------------------------------------------------------------------------- /rtdb-limit-child-nodes/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rtdb-limit-child-nodes-functions", 3 | "description": "Limit number of child nodes Firebase Functions sample", 4 | "main": "lib/index.js", 5 | "scripts": { 6 | "prepare": "npm run build", 7 | "build": "npm run clean && npm run compile", 8 | "clean": "rimraf lib", 9 | "compile": "tsc", 10 | "test": "echo \"Error: no test specified\" && exit 1", 11 | "generate-readme": "firebase ext:info .. --markdown > ../README.md" 12 | }, 13 | "license": "Apache-2.0", 14 | "dependencies": { 15 | "rimraf": "^2.6.3", 16 | "typescript": "^4.9.4", 17 | "@types/express-serve-static-core": "4.17.30", 18 | "@types/node": "^20.10.3", 19 | "firebase-admin": "^12.1.0", 20 | "firebase-functions": "^4.9.0" 21 | }, 22 | "private": true 23 | } 24 | -------------------------------------------------------------------------------- /rtdb-limit-child-nodes/functions/src/config.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export default { 18 | location: process.env.LOCATION, 19 | maxCount: Number(process.env.MAX_COUNT), 20 | databaseInstance: process.env.SELECTED_DATABASE_INSTANCE, 21 | nodePath: process.env.NODE_PATH, 22 | }; 23 | -------------------------------------------------------------------------------- /rtdb-limit-child-nodes/functions/src/logs.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { logger } from "firebase-functions"; 18 | import config from "./config"; 19 | 20 | export const childCount = (path: string, childCount: number) => { 21 | logger.log(`Node: '${path}' has: ${childCount} children`); 22 | }; 23 | 24 | export const complete = () => { 25 | logger.log("Completed execution of extension"); 26 | }; 27 | 28 | export const error = (err: Error) => { 29 | logger.error("Error when truncating the database node", err); 30 | }; 31 | 32 | export const init = () => { 33 | logger.log("Initializing extension with configuration", config); 34 | }; 35 | 36 | export const pathSkipped = (path: string) => { 37 | logger.log(`Path: '${path}' does not need to be truncated`); 38 | }; 39 | 40 | export const pathTruncated = (path: string, count: number) => { 41 | logger.log(`Truncated path: '${path}' to ${count} items`); 42 | }; 43 | 44 | export const pathTruncating = (path: string, count: number) => { 45 | logger.log(`Truncating path: '${path}' to ${count} items`); 46 | }; 47 | 48 | export const start = () => { 49 | logger.log("Started execution of extension with configuration", config); 50 | }; 51 | -------------------------------------------------------------------------------- /rtdb-limit-child-nodes/functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "lib", 4 | "target": "ES2020", 5 | "module": "commonjs", 6 | "noImplicitReturns": true, 7 | "sourceMap": false 8 | }, 9 | "compileOnSave": true, 10 | "include": ["src"] 11 | } 12 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Version 0.0.1 2 | 3 | - Initial Version 4 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/POSTINSTALL.md: -------------------------------------------------------------------------------- 1 | ### See it in action 2 | 3 | You can test out this extension right away! 4 | 5 | 1. Go to your [Realtime Database dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/${param:PROJECT_ID}/data) in the Firebase console. 6 | 7 | 1. Add a message string to a path that matches the pattern `${param:MESSAGE_PATH}`. 8 | 9 | 1. In a few seconds, you'll see a sibling node named `upper` that contains the message in upper case. 10 | 11 | ### Using the extension 12 | 13 | We recommend adding data by pushing -- for example, `firebase.database().ref().push()` -- because pushing assigns an automatically generated ID to the node in the database. During retrieval, these nodes are guaranteed to be ordered by the time they were added. Learn more about reading and writing data for your platform (iOS, Android, or Web) in the [Realtime Database documentation](https://firebase.google.com/docs/database/). 14 | 15 | ### Monitoring 16 | 17 | As a best practice, you can [monitor the activity](https://firebase.google.com/docs/extensions/manage-installed-extensions#monitor) of your installed extension, including checks on its health, usage, and logs. 18 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/PREINSTALL.md: -------------------------------------------------------------------------------- 1 | Use this extension to automatically convert strings to upper case when added to a specified Realtime Database path. 2 | 3 | This extension expects a database layout like the following example: 4 | 5 | "messages": { 6 | MESSAGE_ID: { 7 | "original": MESSAGE_TEXT 8 | }, 9 | MESSAGE_ID: { 10 | "original": MESSAGE_TEXT 11 | }, 12 | } 13 | 14 | When you create new string records, this extension creates a new sibling record with upper-cased 15 | text: 16 | 17 | MESSAGE_ID: { 18 | "original": MESSAGE_TEXT, 19 | "upper": UPPERCASE_MESSAGE_TEXT, 20 | } 21 | 22 | #### Additional setup 23 | 24 | Before installing this extension, make sure that you've 25 | [set up Realtime Database](https://firebase.google.com/docs/databaae/quickstart) 26 | in your Firebase project. 27 | 28 | #### Billing 29 | 30 | To install an extension, your project must be on the 31 | [Blaze (pay as you go) plan](https://firebase.google.com/pricing) 32 | 33 | - This extension uses other Firebase and Google Cloud Platform services, which have associated 34 | charges if you exceed the service’s no-cost tier: 35 | - Realtime Database 36 | - Cloud Functions (Node.js 10+ runtime) 37 | [See FAQs](https://firebase.google.com/support/faq#extensions-pricing) 38 | - If you enable events, [Eventarc fees apply](https://cloud.google.com/eventarc/pricing). 39 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/functions/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/functions/integration-tests/.firebaserc: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/functions/integration-tests/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | firebase-debug.log* 8 | firebase-debug.*.log* 9 | 10 | # Firebase cache 11 | .firebase/ 12 | 13 | # Firebase config 14 | 15 | # Uncomment this if you'd like others to create their own Firebase project. 16 | # For a team working on the same Firebase project(s), it is recommended to leave 17 | # it commented so all members can deploy to the same project(s) in .firebaserc. 18 | # .firebaserc 19 | 20 | # Runtime data 21 | pids 22 | *.pid 23 | *.seed 24 | *.pid.lock 25 | 26 | # Directory for instrumented libs generated by jscoverage/JSCover 27 | lib-cov 28 | 29 | # Coverage directory used by tools like istanbul 30 | coverage 31 | 32 | # nyc test coverage 33 | .nyc_output 34 | 35 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 36 | .grunt 37 | 38 | # Bower dependency directory (https://bower.io/) 39 | bower_components 40 | 41 | # node-waf configuration 42 | .lock-wscript 43 | 44 | # Compiled binary addons (http://nodejs.org/api/addons.html) 45 | build/Release 46 | 47 | # Dependency directories 48 | node_modules/ 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Optional REPL history 57 | .node_repl_history 58 | 59 | # Output of 'npm pack' 60 | *.tgz 61 | 62 | # Yarn Integrity file 63 | .yarn-integrity 64 | 65 | # dotenv environment variables file 66 | .env 67 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/functions/integration-tests/extensions/.gitignore: -------------------------------------------------------------------------------- 1 | !rtdb-uppercase-messages.env 2 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/functions/integration-tests/extensions/rtdb-uppercase-messages.env: -------------------------------------------------------------------------------- 1 | MESSAGE_PATH=/msgs/{pushId}/original 2 | EVENTARC_CHANNEL=locations/us-central1/channels/firebase 3 | EXT_SELECTED_EVENTS=test-publisher.rtdb-uppercase-messages.v1.complete 4 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/functions/integration-tests/firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "emulators": { 3 | "functions": { 4 | "port": 5001 5 | }, 6 | "database": { 7 | "port": 9000 8 | }, 9 | "eventarc": { 10 | "port": 9299 11 | }, 12 | "ui": { 13 | "enabled": true 14 | }, 15 | "singleProjectMode": true 16 | }, 17 | "extensions": { 18 | "rtdb-uppercase-messages": "../.." 19 | }, 20 | "functions": [ 21 | { 22 | "source": "functions", 23 | "codebase": "default", 24 | "ignore": [ 25 | "node_modules", 26 | ".git", 27 | "firebase-debug.log", 28 | "firebase-debug.*.log" 29 | ] 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/functions/integration-tests/functions/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/functions/integration-tests/functions/index.js: -------------------------------------------------------------------------------- 1 | import { logger } from "firebase-functions/v1"; 2 | import { onCustomEventPublished } from "firebase-functions/v2/eventarc"; 3 | 4 | import { initializeApp } from "firebase-admin/app"; 5 | import { getDatabase } from "firebase-admin/database"; 6 | 7 | const app = initializeApp(); 8 | 9 | export const extraemphasis = onCustomEventPublished( 10 | "test-publisher.rtdb-uppercase-messages.v1.complete", 11 | async (event) => { 12 | logger.info("Received makeuppercase completed event", event); 13 | 14 | const refUrl = event.subject; 15 | const ref = getDatabase().refFromURL(refUrl); 16 | const upper = (await ref.get()).val(); 17 | return ref.set(`${upper}!!!`); 18 | } 19 | ); 20 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/functions/integration-tests/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "functions", 3 | "description": "Cloud Functions for Firebase", 4 | "scripts": { 5 | "serve": "firebase emulators:start --only functions", 6 | "shell": "firebase functions:shell", 7 | "start": "npm run shell", 8 | "deploy": "firebase deploy --only functions", 9 | "logs": "firebase functions:log" 10 | }, 11 | "engines": { 12 | "node": "18" 13 | }, 14 | "main": "index.js", 15 | "type": "module", 16 | "dependencies": { 17 | "firebase-admin": "^12.1.0", 18 | "firebase-functions": "^4.9.0" 19 | }, 20 | "devDependencies": { 21 | "firebase-functions-test": "^3.2.0" 22 | }, 23 | "private": true 24 | } 25 | -------------------------------------------------------------------------------- /samples/rtdb-uppercase-messages/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rtdb-uppercase-messages", 3 | "description": "Uppercases RTDB messages", 4 | "main": "index.js", 5 | "type": "module", 6 | "dependencies": { 7 | "firebase-admin": "^12.1.0", 8 | "firebase-functions": "^4.9.0" 9 | }, 10 | "private": true 11 | } 12 | -------------------------------------------------------------------------------- /scripts/publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Fail on any error. 4 | set -e 5 | 6 | PUBLISHER_ID=$1 7 | shift 8 | ALL_EXTENSIONS=( 9 | "auth-mailchimp-sync" 10 | "delete-user-data" 11 | "firestore-bigquery-export" 12 | "firestore-counter" 13 | "firestore-send-email" 14 | "firestore-shorten-urls-bitly" 15 | "firestore-translate-text" 16 | "rtdb-limit-child-nodes" 17 | "storage-resize-images" 18 | ) 19 | EXTENSIONS=${@:-${ALL_EXTENSIONS[@]}} 20 | 21 | if [ -z "$PUBLISHER_ID" ] 22 | then 23 | echo "\$PUBLISHER_ID is not defined" 24 | exit 1 25 | else 26 | echo "Publishing $EXTENSIONS into $PUBLISHER_ID" 27 | fi 28 | 29 | REPO_ROOT="`( cd \`dirname \"$0\"\` && cd .. && pwd )`" 30 | 31 | cd "$REPO_ROOT" 32 | npm ci 33 | if [ ! -z "$IGNORE_TEST_FAILURES" ] 34 | then 35 | echo "Ignoring failing tests." 36 | set +e 37 | fi 38 | 39 | npm test 40 | 41 | set -e 42 | 43 | for i in ${EXTENSIONS[@]}; do 44 | echo "" 45 | echo "- Publishing $PUBLISHER_ID/$i" 46 | echo "" 47 | 48 | cd "$REPO_ROOT/$i" 49 | set +e 50 | firebase ext:dev:publish $PUBLISHER_ID/$i $FIREX_PUBLISH_EXTRA_FLAGS --non-interactive --force 51 | EXIT_CODE=$? 52 | set -e 53 | echo "Exit code: $EXIT_CODE" 54 | # Exit code 103 means that version already published, move on. 55 | [ $EXIT_CODE -eq 0 ] || [ $EXIT_CODE -eq 103 ] || exit 1 56 | done 57 | -------------------------------------------------------------------------------- /storage-resize-images/functions/.gitignore: -------------------------------------------------------------------------------- 1 | lib -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/__mocks__/src/index.ts: -------------------------------------------------------------------------------- 1 | import * as functionsTestInit from "firebase-functions-test"; 2 | import { generateResizedImage as originalGenerateResizedImage } from "../../../src"; 3 | 4 | export const generateResizedImage = () => { 5 | let functionsTest = functionsTestInit(); 6 | return functionsTest.wrap(originalGenerateResizedImage); 7 | }; 8 | -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`extension configuration detected from environment variables 1`] = ` 4 | Object { 5 | "animated": false, 6 | "bucket": "extensions-testing.appspot.com", 7 | "cacheControlHeader": undefined, 8 | "contentFilterLevel": null, 9 | "customFilterPrompt": null, 10 | "deleteOriginalFile": 0, 11 | "doBackfill": false, 12 | "excludePathList": undefined, 13 | "failedImagesPath": undefined, 14 | "imageSizes": Array [ 15 | "200x200", 16 | ], 17 | "imageTypes": undefined, 18 | "includePathList": undefined, 19 | "location": "us-central1", 20 | "makePublic": false, 21 | "outputOptions": undefined, 22 | "placeholderImagePath": null, 23 | "projectId": undefined, 24 | "regenerateToken": false, 25 | "resizedImagesPath": undefined, 26 | "sharpOptions": "{}", 27 | } 28 | `; 29 | -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/config.test.ts: -------------------------------------------------------------------------------- 1 | import mockedEnv from "mocked-env"; 2 | 3 | const environment = { 4 | LOCATION: "us-central1", 5 | IMG_BUCKET: "extensions-testing.appspot.com", 6 | CACHE_CONTROL_HEADER: undefined, 7 | IMG_SIZES: `200x200`, 8 | RESIZED_IMAGES_PATH: undefined, 9 | DELETE_ORIGINAL_FILE: "true", 10 | CONTENT_FILTER_LEVEL: "OFF", 11 | }; 12 | 13 | let restoreEnv; 14 | 15 | let deleteTypeCounter = 0; 16 | 17 | let config; 18 | let deleteImage; 19 | 20 | describe("extension", () => { 21 | beforeEach(() => { 22 | jest.resetModules(); 23 | if (deleteTypeCounter === 0) { 24 | restoreEnv = mockedEnv(environment); 25 | } else if (deleteTypeCounter === 1) { 26 | restoreEnv = mockedEnv({ ...environment, DELETE_ORIGINAL_FILE: "false" }); 27 | } else if (deleteTypeCounter === 2) { 28 | restoreEnv = mockedEnv({ 29 | ...environment, 30 | DELETE_ORIGINAL_FILE: "on_success", 31 | }); 32 | } 33 | const actualConfigModule = jest.requireActual("../src/config"); 34 | config = actualConfigModule.config; 35 | deleteImage = actualConfigModule.deleteImage; 36 | }); 37 | 38 | afterEach(() => restoreEnv()); 39 | 40 | test("configuration detected from environment variables", async () => { 41 | expect(config).toMatchSnapshot({}); 42 | }); 43 | 44 | test("always delete original file", async () => { 45 | deleteTypeCounter++; 46 | expect(config.deleteOriginalFile).toEqual(deleteImage.always); 47 | }); 48 | 49 | test("never delete original file", async () => { 50 | deleteTypeCounter++; 51 | expect(config.deleteOriginalFile).toEqual(deleteImage.never); 52 | }); 53 | test("delete original file on success", async () => { 54 | expect(config.deleteOriginalFile).toEqual(deleteImage.onSuccess); 55 | }); 56 | }); 57 | -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/fixtures/config.json: -------------------------------------------------------------------------------- 1 | { "some": "secret" } 2 | -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/fixtures/config_hack.txt: -------------------------------------------------------------------------------- 1 | data:text/plain;,'Hello from hacker' -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/fixtures/settings.json: -------------------------------------------------------------------------------- 1 | { "status": "active" } 2 | -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/function.test.ts: -------------------------------------------------------------------------------- 1 | import { config } from "../src/config"; 2 | jest.mock("../src/config"); 3 | jest.mock("../src"); 4 | 5 | // Define your mock functions first 6 | const logMock = jest.fn().mockReturnValue(0); 7 | const errorLogMock = jest.fn().mockReturnValue(0); 8 | const warnLogMock = jest.fn().mockReturnValue(0); 9 | 10 | jest.mock("firebase-functions", () => { 11 | return { 12 | ...jest.requireActual("firebase-functions"), 13 | logger: { 14 | log: jest.fn((...args) => logMock(...args)), // Spread operator to pass all arguments 15 | error: jest.fn((...args) => errorLogMock(...args)), 16 | warn: jest.fn((...args) => warnLogMock(...args)), 17 | }, 18 | }; 19 | }); 20 | 21 | jest.mock("../src/config", () => { 22 | return { 23 | config: { 24 | location: "us-central1", 25 | imgBucket: "extensions-testing.appspot.com", 26 | cacheControlHeader: undefined, 27 | imgSizes: ["200x200"], 28 | resizedImagesPath: undefined, 29 | deleteOriginalFile: "true", 30 | }, 31 | }; 32 | }); 33 | 34 | import { generateResizedImage } from "../src"; 35 | 36 | describe("extension", () => { 37 | beforeEach(() => { 38 | jest.clearAllMocks(); 39 | }); 40 | 41 | test("'generateResizedImage' function is exported", () => { 42 | const exportedFunctions = jest.requireActual("../src"); 43 | expect(exportedFunctions.generateResizedImage).toBeInstanceOf(Function); 44 | }); 45 | }); 46 | -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/gun-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/storage-resize-images/functions/__tests__/gun-image.png -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/not-an-image.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/storage-resize-images/functions/__tests__/not-an-image.jpeg -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/storage.rules: -------------------------------------------------------------------------------- 1 | rules_version = '2'; 2 | service firebase.storage { 3 | match /b/{bucket}/o { 4 | 5 | match /users/{userId}/images/{documents=**} { 6 | allow read, write; 7 | } 8 | 9 | match /users/{userId}/images/thumbs/{image} { 10 | allow read: if image.matches('.*\\.png'); 11 | } 12 | 13 | match /config.json { 14 | allow read, write; 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/test-image.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/storage-resize-images/functions/__tests__/test-image.gif -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/test-image.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/storage-resize-images/functions/__tests__/test-image.jpeg -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/test-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/storage-resize-images/functions/__tests__/test-image.png -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/test-img.jfif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/storage-resize-images/functions/__tests__/test-img.jfif -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/test-jpg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/storage-resize-images/functions/__tests__/test-jpg.jpg -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/test.types.d.ts: -------------------------------------------------------------------------------- 1 | declare namespace NodeJS { 2 | interface Global { 3 | config: () => jest.ModuleMocker; 4 | deleteImage: () => jest.ModuleMocker; 5 | mockGenerateResizedImage: () => jest.MockedFunction; 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "files": ["test.types.d.ts"], 4 | "compilerOptions": { 5 | "outDir": "lib", 6 | "types": ["node", "jest"], 7 | "target": "ES2020", 8 | "skipLibCheck": true 9 | }, 10 | "include": ["**/*"] 11 | } 12 | -------------------------------------------------------------------------------- /storage-resize-images/functions/__tests__/util.ts: -------------------------------------------------------------------------------- 1 | import * as admin from "firebase-admin"; 2 | 3 | export const waitForFile = async ( 4 | storage: admin.storage.Storage, 5 | filePath: string, 6 | timeout: number = 1000, 7 | maxAttempts: number = 20 8 | ) => { 9 | let exists: [boolean]; 10 | 11 | const promise = new Promise((resolve, reject) => { 12 | let timesRun = 0; 13 | const interval = setInterval(async () => { 14 | timesRun += 1; 15 | try { 16 | exists = await storage.bucket().file(filePath).exists(); 17 | } catch (e) {} 18 | if (exists && exists[0]) { 19 | clearInterval(interval); 20 | resolve(exists[0]); 21 | } 22 | if (timesRun > maxAttempts) { 23 | clearInterval(interval); 24 | reject("timed out without finding file " + filePath); 25 | } 26 | }, timeout); 27 | }); 28 | 29 | return await promise; 30 | }; 31 | -------------------------------------------------------------------------------- /storage-resize-images/functions/jest.config.js: -------------------------------------------------------------------------------- 1 | const packageJson = require("./package.json"); 2 | 3 | module.exports = { 4 | name: packageJson.name, 5 | displayName: packageJson.name, 6 | rootDir: "./", 7 | globals: { 8 | "ts-jest": { 9 | tsConfig: "/__tests__/tsconfig.json", 10 | }, 11 | }, 12 | snapshotFormat: { 13 | escapeString: true, 14 | printBasicPrototype: true, 15 | }, 16 | preset: "ts-jest", 17 | testMatch: ["**/__tests__/**/*.test.ts"], 18 | testPathIgnorePatterns: 19 | process.env.CI_TEST === "true" ? ["vulnerability"] : [], 20 | moduleNameMapper: { 21 | "firebase-admin/eventarc": 22 | "/node_modules/firebase-admin/lib/eventarc", 23 | "firebase-admin/functions": 24 | "/node_modules/firebase-admin/lib/functions", 25 | "firebase-admin/extensions": 26 | "/node_modules/firebase-admin/lib/extensions", 27 | }, 28 | }; 29 | -------------------------------------------------------------------------------- /storage-resize-images/functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "storage-resize-images-functions", 3 | "description": "Resized Image Generator for Firebase", 4 | "author": "Firebase", 5 | "license": "Apache-2.0", 6 | "main": "lib/index.js", 7 | "scripts": { 8 | "prepare": "npm run build", 9 | "build": "npm run clean && npm run compile && shx cp src/placeholder.png lib/", 10 | "build:watch": "npm run clean && tsc --watch", 11 | "clean": "rimraf lib", 12 | "compile": "tsc", 13 | "test": "jest", 14 | "test:vulnerability": "RUN_VULNERABILITY_TEST=true jest", 15 | "generate-readme": "firebase ext:info .. --markdown > ../README.md" 16 | }, 17 | "dependencies": { 18 | "@genkit-ai/vertexai": "^1.2.0", 19 | "@types/node": "^20.10.3", 20 | "firebase-admin": "^13.1.0", 21 | "firebase-functions": "^6.3.1", 22 | "genkit": "^1.2.0", 23 | "mkdirp": "^3.0.1", 24 | "p-queue": "^6.6.2", 25 | "rimraf": "^6.0.1", 26 | "sharp": "^0.33.5", 27 | "shx": "^0.4.0", 28 | "typescript": "^5.7.3", 29 | "uuid": "^11.0.5", 30 | "uuidv4": "^6.1.0" 31 | }, 32 | "devDependencies": { 33 | "@types/jest": "^29.5.14", 34 | "@types/mkdirp": "^1.0.1", 35 | "child_process": "^1.0.2", 36 | "dotenv": "^16.4.7", 37 | "firebase": "^11.3.1", 38 | "firebase-functions-test": "^3.4.0", 39 | "image-size": "^1.2.0", 40 | "image-type": "^4.1.0", 41 | "jest": "^29.7.0", 42 | "mocked-env": "^1.3.5", 43 | "ts-jest": "^29.2.5" 44 | }, 45 | "private": true 46 | } 47 | -------------------------------------------------------------------------------- /storage-resize-images/functions/src/events.ts: -------------------------------------------------------------------------------- 1 | import * as eventArc from "firebase-admin/eventarc"; 2 | 3 | const { getEventarc } = eventArc; 4 | 5 | const EXTENSION_NAME = "storage-resize-images"; 6 | 7 | const getEventType = (eventName: string) => 8 | `firebase.extensions.${EXTENSION_NAME}.v1.${eventName}`; 9 | 10 | let eventChannel: eventArc.Channel; 11 | 12 | /** setup events */ 13 | export const setupEventChannel = () => { 14 | eventChannel = 15 | process.env.EVENTARC_CHANNEL && 16 | getEventarc().channel(process.env.EVENTARC_CHANNEL, { 17 | allowedEventTypes: process.env.EXT_SELECTED_EVENTS, 18 | }); 19 | }; 20 | 21 | export const recordStartEvent = async (data: string | object) => { 22 | if (!eventChannel) return; 23 | 24 | return eventChannel.publish({ 25 | type: getEventType("onStart"), 26 | data, 27 | }); 28 | }; 29 | 30 | export const recordErrorEvent = async (err: Error) => { 31 | if (!eventChannel) return; 32 | 33 | return eventChannel.publish({ 34 | type: getEventType("onError"), 35 | data: { message: err.message }, 36 | }); 37 | }; 38 | 39 | export const recordSuccessEvent = async ({ 40 | subject, 41 | data, 42 | }: { 43 | subject: string; 44 | data: string | object; 45 | }) => { 46 | if (!eventChannel) return; 47 | 48 | return eventChannel.publish({ 49 | type: getEventType("onSuccess"), 50 | subject, 51 | data, 52 | }); 53 | }; 54 | 55 | export const recordCompletionEvent = async (data: string | object) => { 56 | if (!eventChannel) return; 57 | 58 | return eventChannel.publish({ 59 | type: getEventType("onCompletion"), 60 | data, 61 | }); 62 | }; 63 | -------------------------------------------------------------------------------- /storage-resize-images/functions/src/filters.ts: -------------------------------------------------------------------------------- 1 | import * as path from "path"; 2 | 3 | import * as logs from "./logs"; 4 | import { config } from "./config"; 5 | import { supportedContentTypes } from "./global"; 6 | import { convertPathToPosix, startsWithArray } from "./util"; 7 | import { ObjectMetadata } from "firebase-functions/v1/storage"; 8 | 9 | export function shouldResize(object: ObjectMetadata): boolean { 10 | const { contentType } = object; // This is the image MIME type 11 | 12 | let tmpFilePath = convertPathToPosix( 13 | path.resolve("/", path.dirname(object.name)), 14 | true 15 | ); // Absolute path to dirname 16 | 17 | if (!contentType) { 18 | logs.noContentType(); 19 | return false; 20 | } 21 | 22 | if (!contentType.startsWith("image/")) { 23 | logs.contentTypeInvalid(contentType); 24 | return false; 25 | } 26 | 27 | if (object.contentEncoding === "gzip") { 28 | logs.gzipContentEncoding(); 29 | return false; 30 | } 31 | 32 | if (!supportedContentTypes.includes(contentType)) { 33 | logs.unsupportedType(supportedContentTypes, contentType); 34 | return false; 35 | } 36 | 37 | if ( 38 | config.includePathList && 39 | !startsWithArray(config.includePathList, tmpFilePath) 40 | ) { 41 | logs.imageOutsideOfPaths(config.includePathList, tmpFilePath); 42 | return false; 43 | } 44 | 45 | if ( 46 | config.excludePathList && 47 | startsWithArray(config.excludePathList, tmpFilePath) 48 | ) { 49 | logs.imageInsideOfExcludedPaths(config.excludePathList, tmpFilePath); 50 | return false; 51 | } 52 | 53 | if (object.metadata && object.metadata.resizedImage === "true") { 54 | logs.imageAlreadyResized(); 55 | return false; 56 | } 57 | if (object.metadata && object.metadata.resizeFailed) { 58 | logs.imageFailedAttempt(); 59 | return false; 60 | } 61 | 62 | return true; 63 | } 64 | -------------------------------------------------------------------------------- /storage-resize-images/functions/src/global.ts: -------------------------------------------------------------------------------- 1 | import PQueue from "p-queue"; 2 | 3 | /** 4 | * Supported file types 5 | */ 6 | export const supportedContentTypes = [ 7 | "image/jpg", 8 | "image/jpeg", 9 | "image/png", 10 | "image/tiff", 11 | "image/webp", 12 | "image/gif", 13 | "image/avif", 14 | ]; 15 | 16 | export const supportedImageContentTypeMap = { 17 | jpg: "image/jpeg", 18 | jpeg: "image/jpeg", 19 | png: "image/png", 20 | tif: "image/tif", 21 | tiff: "image/tiff", 22 | webp: "image/webp", 23 | gif: "image/gif", 24 | avif: "image/avif", 25 | jfif: "image/jpeg", 26 | }; 27 | 28 | export const supportedExtensions = Object.keys( 29 | supportedImageContentTypeMap 30 | ).map((type) => `.${type}`); 31 | 32 | export type RetryQueueItem = { 33 | priority: number; 34 | task: () => Promise; 35 | }; 36 | 37 | export const globalRetryQueue = new PQueue({ concurrency: 3, autoStart: true }); 38 | -------------------------------------------------------------------------------- /storage-resize-images/functions/src/placeholder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firebase/extensions/2f23c5a7efa657aca8ee7b24f9809644687d5c08/storage-resize-images/functions/src/placeholder.png -------------------------------------------------------------------------------- /storage-resize-images/functions/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "lib", 4 | "target": "ES2020", 5 | "module": "commonjs", 6 | "noImplicitReturns": true, 7 | "skipLibCheck": true, 8 | "sourceMap": false, 9 | "esModuleInterop": true 10 | }, 11 | "compileOnSave": true, 12 | "include": ["src"] 13 | } 14 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es6"], 4 | "module": "commonjs", 5 | "noImplicitReturns": true, 6 | "sourceMap": false, 7 | "target": "es6" 8 | }, 9 | "compileOnSave": true, 10 | "exclude": ["node_modules"] 11 | } 12 | --------------------------------------------------------------------------------