├── .npmrc ├── test ├── .npmrc ├── setup.js ├── .gitignore ├── helpers │ ├── create-context.js │ ├── request.js │ ├── waitRestart.js │ ├── builder │ │ ├── context.js │ │ ├── index.js │ │ └── action-registry.js │ ├── auth.js │ ├── strapi.js │ ├── utils.js │ ├── agent.js │ ├── generators.js │ └── models.js ├── config │ ├── server.js │ └── database.js ├── README.md ├── jest.config.js ├── teardown.js ├── utils │ ├── setup-test.js │ ├── firestore.js │ └── app.js ├── package.json └── report.js ├── .github ├── FUNDING.yml └── workflows │ ├── publish.yml │ ├── codeql-analysis.yml │ └── test.yml ├── examples └── cloud-run-and-hosting │ ├── .npmrc │ ├── public │ ├── robots.txt │ └── uploads │ │ └── README.md │ ├── api │ └── README.md │ ├── .firebaserc │ ├── extensions │ └── README.md │ ├── favicon.ico │ ├── .gitignore │ ├── storage.rules │ ├── firestore.rules │ ├── config │ ├── middleware.js │ ├── database.js │ ├── plugins.js │ └── server.js │ ├── firebase.json │ ├── patches │ └── strapi-utils+3.6.8.patch │ ├── middlewares │ └── api │ │ └── index.js │ ├── .gcloudignore │ ├── scripts │ └── deploy.js │ ├── .dockerignore │ ├── Dockerfile │ ├── package.json │ └── README.md ├── src ├── utils │ ├── status-error.ts │ ├── map-not-null.ts │ ├── prefix-query.ts │ ├── components.ts │ ├── manual-filter.ts │ ├── lifecycle.ts │ ├── transaction-runner.ts │ ├── read-repository.ts │ ├── components-indexing.ts │ └── relation-handler.ts ├── db │ ├── query-error.ts │ ├── component-collection.ts │ ├── collection.ts │ ├── morph-reference.ts │ ├── normal-reference.ts │ ├── reference.ts │ ├── field-operation.ts │ ├── virtual-reference.ts │ ├── virtual-collection.ts │ ├── deep-reference.ts │ ├── readonly-transaction.ts │ ├── transaction.ts │ ├── flat-collection.ts │ ├── normal-collection.ts │ └── readwrite-transaction.ts ├── coerce │ └── coerce-to-firestore.ts ├── populate.ts ├── index.ts ├── build-query.ts ├── relations.ts └── queries.ts ├── tsconfig.json ├── .vscode └── launch.json ├── LICENSE.md ├── package.json └── .gitignore /.npmrc: -------------------------------------------------------------------------------- 1 | lockfile-version=3 2 | -------------------------------------------------------------------------------- /test/.npmrc: -------------------------------------------------------------------------------- 1 | lockfile-version=3 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [brettwillis] 2 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/.npmrc: -------------------------------------------------------------------------------- 1 | lockfile-version=3 2 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/public/robots.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: / 3 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/api/README.md: -------------------------------------------------------------------------------- 1 | > This directory exists to prevent Strapi crash while compiling 2 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/.firebaserc: -------------------------------------------------------------------------------- 1 | { 2 | "projects": { 3 | "default": "project-id" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /test/setup.js: -------------------------------------------------------------------------------- 1 | 'use-strict'; 2 | 3 | const { setupTestApp } = require('./utils/app'); 4 | 5 | setupTestApp(); 6 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/extensions/README.md: -------------------------------------------------------------------------------- 1 | > This directory exists to prevent Strapi crash while compiling 2 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arrowheadapps/strapi-connector-firestore/HEAD/examples/cloud-run-and-hosting/favicon.ico -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | .cache/ 2 | .temp/ 3 | api/ 4 | components/ 5 | extensions/ 6 | public/ 7 | build/ 8 | node_modules/ 9 | tests/ 10 | .strapi-updater.json 11 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/public/uploads/README.md: -------------------------------------------------------------------------------- 1 | > This directory exists to prevent Strapi crash when running in development, but is not used or required in production. 2 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | build/ 3 | .cache/ 4 | .temp/ 5 | .firebase/ 6 | extensions/**/jwt.js 7 | .strapi-updater.json 8 | .env 9 | *-debug.log 10 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/storage.rules: -------------------------------------------------------------------------------- 1 | service firebase.storage { 2 | match /b/{bucket}/o { 3 | match /{allPaths=**} { 4 | allow read, write: if false; 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/utils/status-error.ts: -------------------------------------------------------------------------------- 1 | 2 | export class StatusError extends Error { 3 | status: number 4 | 5 | constructor(message: string, status: number) { 6 | super(message); 7 | this.status = status; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/firestore.rules: -------------------------------------------------------------------------------- 1 | rules_version = '2'; 2 | service cloud.firestore { 3 | match /databases/{database}/documents { 4 | match /{document=**} { 5 | allow read, write: if false; 6 | } 7 | } 8 | } -------------------------------------------------------------------------------- /test/helpers/create-context.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = ({ params = {}, query = {}, body = {} }, overrides = {}) => ({ 4 | params, 5 | query, 6 | request: { 7 | query, 8 | body, 9 | }, 10 | ...overrides, 11 | }); 12 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/config/middleware.js: -------------------------------------------------------------------------------- 1 | module.exports = ({ env }) => ({ 2 | load: { 3 | before: ['api'], 4 | }, 5 | settings: { 6 | api: { 7 | enabled: true, 8 | }, 9 | logger: { 10 | level: env('NODE_ENV') === 'production' ? 'info' : 'debug', 11 | requests: env('NODE_ENV') !== 'production', 12 | }, 13 | }, 14 | }); 15 | -------------------------------------------------------------------------------- /test/config/server.js: -------------------------------------------------------------------------------- 1 | module.exports = ({ env }) => ({ 2 | host: 'localhost', 3 | port: 1337, 4 | admin: { 5 | serveAdminPanel: false, 6 | watchIgnoreFiles: [ 7 | // Prevent Firestore log file from triggering Strapi restart 8 | '*-debug.log' 9 | ], 10 | auth: { 11 | secret: env('ADMIN_JWT_SECRET') || 'd399f313-cdde-4abd-86e7-1b040b441d09', 12 | }, 13 | }, 14 | }); 15 | -------------------------------------------------------------------------------- /src/utils/map-not-null.ts: -------------------------------------------------------------------------------- 1 | 2 | export function mapNotNull(arr: T[], fn: (item: T, i: number) => R | null | undefined): R[] { 3 | return arr.map(fn).filter(isNotNull); 4 | } 5 | 6 | export function filterNotNull(arr: (T | null | undefined)[]): T[] { 7 | return arr.filter(isNotNull); 8 | } 9 | 10 | export function isNotNull(value: R | null | undefined): value is R { 11 | return value != null; 12 | } 13 | -------------------------------------------------------------------------------- /test/helpers/request.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { createAgent } = require('./agent'); 4 | const { superAdmin } = require('./strapi'); 5 | 6 | const createRequest = ({ strapi } = {}) => createAgent(strapi); 7 | const createAuthRequest = ({ strapi, userInfo = superAdmin.credentials }) => 8 | createAgent(strapi).login(userInfo); 9 | 10 | module.exports = { 11 | createRequest, 12 | createAuthRequest, 13 | }; 14 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to NPM 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | npm-publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - uses: actions/setup-node@v2 13 | with: 14 | registry-url: https://registry.npmjs.org 15 | - run: npm ci 16 | - run: npm publish 17 | env: 18 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 19 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/node12/tsconfig.json", 3 | "compilerOptions": { 4 | "esModuleInterop": false, 5 | "strict": true, 6 | "useUnknownInCatchVariables": false, 7 | "noImplicitReturns": true, 8 | "noUnusedLocals": true, 9 | "outDir": "lib", 10 | "types": ["node"], 11 | "noImplicitAny": false, 12 | "sourceMap": false, 13 | "declaration": true 14 | }, 15 | "compileOnSave": true, 16 | "include": [ 17 | "src" 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # Tests 2 | 3 | This directory contains a skeleton Strapi project configuration using this Firestore connector, and the test setup scripts copied from the Strapi repo. 4 | 5 | ## Test procedure 6 | 7 | 1. Clean the Strapi project configuration 8 | 2. Copy the end-to-end tests out of the `strapi` package (Jest seemingly refuses to run them while inside `node_modules`) 9 | 3. Start the Firestore emulator 10 | 4. Start Strapi 11 | 5. Run the end-to-end tests from the `strapi` package 12 | -------------------------------------------------------------------------------- /src/utils/prefix-query.ts: -------------------------------------------------------------------------------- 1 | 2 | export interface PrefixQueryTerms { 3 | gte: string, 4 | lt: string 5 | } 6 | 7 | /** 8 | * Firestore-native method to query for prefix. 9 | * See: https://stackoverflow.com/a/46574143/1513557 10 | * @param value 11 | */ 12 | export function buildPrefixQuery(value: string): PrefixQueryTerms { 13 | return { 14 | gte: value, 15 | lt: value.slice(0, -1) + String.fromCharCode(value.charCodeAt(value.length - 1) + 1), // Lexicographically increment the last character 16 | }; 17 | } 18 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "hosting": { 3 | "public": "build", 4 | "rewrites": [ 5 | { 6 | "source": "/api/**", 7 | "run": { 8 | "serviceId": "api-admin", 9 | "region": "us-central1" 10 | } 11 | }, 12 | { 13 | "source": "**", 14 | "destination": "/index.html" 15 | } 16 | ] 17 | }, 18 | "storage": { 19 | "rules": "storage.rules" 20 | }, 21 | "firestore": { 22 | "rules": "firestore.rules", 23 | "indexes": "firestore.indexes.json" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/config/database.js: -------------------------------------------------------------------------------- 1 | module.exports = ({ env }) => ({ 2 | defaultConnection: 'default', 3 | connections: { 4 | default: { 5 | connector: 'firestore', 6 | settings: { 7 | // This not required for production 8 | // But it is required for the Firestore emulator UI otherwise it won't show any data 9 | projectId: env('GCP_PROJECT'), 10 | }, 11 | options: { 12 | useEmulator: env('NODE_ENV') !== 'production', 13 | logQueries: env('NODE_ENV') !== 'production', 14 | } 15 | } 16 | }, 17 | }); 18 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/config/plugins.js: -------------------------------------------------------------------------------- 1 | module.exports = ({ env }) => { 2 | 3 | // Use Cloud Storage for production environment only 4 | if (env('NODE_ENV') === 'production') { 5 | return { 6 | upload: { 7 | provider: 'google-cloud-storage', 8 | providerOptions: { 9 | // The GCP_PROJECT variable is set by the deployment script in production 10 | bucketName: `${env('GCP_PROJECT')}.appspot.com`, 11 | basePath: '/', 12 | publicFiles: true, 13 | uniform: false, 14 | }, 15 | }, 16 | }; 17 | } 18 | }; 19 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/config/server.js: -------------------------------------------------------------------------------- 1 | module.exports = ({ env }) => { 2 | const config = { 3 | host: env('HOST', '0.0.0.0'), 4 | port: env.int('PORT', 8081), 5 | admin: { 6 | //url: '/admin', 7 | auth: { 8 | secret: env('ADMIN_JWT_SECRET'), 9 | }, 10 | } 11 | }; 12 | 13 | // Don't serve the admin panel in production 14 | // Instead it is deployed on Firebase hosting 15 | if (env('NODE_ENV') === 'production') { 16 | config.url = '/api'; 17 | config.admin.url = '/'; 18 | config.admin.serveAdminPanel = false; 19 | } 20 | 21 | return config; 22 | }; 23 | -------------------------------------------------------------------------------- /test/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | name: 'API integration tests', 3 | rootDir: '../', 4 | testEnvironment: 'node', 5 | verbose: true, 6 | transform: { }, 7 | testMatch: ['/test/tests/*.test*.js'], 8 | setupFilesAfterEnv: ['/test/utils/setup-test.js'], 9 | globalTeardown: '/test/teardown.js', 10 | 11 | collectCoverage: true, 12 | coverageReporters: ['json', 'text-summary'], 13 | collectCoverageFrom: [ 14 | '/lib/**/*.js', 15 | ], 16 | 17 | moduleNameMapper: { 18 | // Tests are copied from the Strapi module so the relative imports are broken 19 | // So map them to the correct place 20 | '\\.\\./test/helpers/(.*)$': '/test/helpers/$1', 21 | }, 22 | }; 23 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/patches/strapi-utils+3.6.8.patch: -------------------------------------------------------------------------------- 1 | diff --git a/node_modules/strapi-utils/lib/config.js b/node_modules/strapi-utils/lib/config.js 2 | index 0e1f2ae..3c6a4fa 100644 3 | --- a/node_modules/strapi-utils/lib/config.js 4 | +++ b/node_modules/strapi-utils/lib/config.js 5 | @@ -35,7 +35,9 @@ const getConfigUrls = (serverConfig, forAdminBuild = false) => { 6 | throw new Error('Invalid admin url config. Make sure the url defined in server.js is valid.'); 7 | } 8 | } else { 9 | - adminUrl = `${serverUrl}/${adminUrl}`; 10 | + // For Firebase hosting, we want the API available at `/_api` 11 | + // yet the panel served at `/` 12 | + adminUrl = '/' + adminUrl; 13 | } 14 | 15 | // Defines adminPath value 16 | -------------------------------------------------------------------------------- /src/db/query-error.ts: -------------------------------------------------------------------------------- 1 | import { Query } from '@google-cloud/firestore'; 2 | 3 | export class QueryError extends Error { 4 | constructor(readonly cause: any, readonly query: Query) { 5 | super(`Firestore query failed: ${cause.message}`); 6 | } 7 | 8 | getQueryInfo() { 9 | // HACK: Using private API, can break if Firestore internal changes. 10 | // @ts-expect-error 11 | const { parentPath, collectionId, limit, offset, fieldFilters, fieldOrders, startAt, endAt } = this.query._queryOptions; 12 | return { 13 | parentPath, collectionId, limit, offset, fieldFilters, fieldOrders, startAt, endAt 14 | }; 15 | } 16 | 17 | describeQuery() { 18 | return JSON.stringify(this.getQueryInfo(), undefined, 2); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /test/helpers/waitRestart.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const request = require('request-promise-native'); 4 | 5 | module.exports = function(initTime = 200) { 6 | const ping = async () => { 7 | return new Promise((resolve, reject) => { 8 | // ping _health 9 | request({ 10 | url: 'http://localhost:1337/_health', 11 | method: 'HEAD', 12 | mode: 'no-cors', 13 | json: true, 14 | headers: { 15 | 'Content-Type': 'application/json', 16 | 'Keep-Alive': false, 17 | }, 18 | }).then(resolve, reject); 19 | }).catch(() => { 20 | return new Promise(resolve => setTimeout(resolve, 200)).then(ping); 21 | }); 22 | }; 23 | 24 | return new Promise(resolve => setTimeout(resolve, initTime)).then(ping); 25 | }; 26 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/middlewares/api/index.js: -------------------------------------------------------------------------------- 1 | module.exports = strapi => ({ 2 | initialize: () => { 3 | const prefix = '/api'; 4 | 5 | // Install a middleware that removes '/api' from the start of the URL 6 | // This happens when called via the Firebase Hosting proxy 7 | strapi.app.use(async (ctx, next) => { 8 | if (ctx.path.startsWith(prefix)) { 9 | ctx.path = ctx.path.slice(prefix.length); 10 | } 11 | 12 | // I don't know why but the Firebase CDN seems to cache everything by default (?) 13 | // So explicitly set no caching on all requests 14 | // Caching can be re-enabled explicity by setting the header in a route handler 15 | ctx.set('Cache-Control', 'private, max-age=0'); 16 | 17 | await next(); 18 | }); 19 | }, 20 | }); 21 | -------------------------------------------------------------------------------- /test/teardown.js: -------------------------------------------------------------------------------- 1 | 'use-strict'; 2 | 3 | const { loadCoverage, remap, writeReport } = require('remap-istanbul'); 4 | 5 | /** 6 | * Remaps the source coverage to the original TypeScript source. 7 | */ 8 | module.exports = async () => { 9 | 10 | // Remap coverage 11 | // The implementation only sorts out the relative paths correctly if 12 | // the working directory is the root directory 13 | const cwd = process.cwd(); 14 | try { 15 | process.chdir('../'); 16 | const coverage = await loadCoverage('coverage/coverage-final.json'); 17 | const remapped = await remap(coverage); 18 | await writeReport(remapped, 'json', {}, 'coverage/coverage.json'); 19 | } catch (err) { 20 | console.log('Error while remapping code coverage'); 21 | console.log(err); 22 | } finally { 23 | process.chdir(cwd); 24 | } 25 | }; 26 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/.gcloudignore: -------------------------------------------------------------------------------- 1 | # Exclude scripts 2 | /scripts 3 | 4 | # Exclude Strapi build outputs 5 | # These are for the admin font-end which is deployed to Firebase Hosting 6 | # so we don't want them in the Cloud Run image 7 | /admin 8 | /build 9 | /.cache 10 | /.temp 11 | .strapi-updater.json 12 | 13 | # Exclude environment variables because we set variables using Cloud Run 14 | .env 15 | 16 | # Don't serve uploads from Cloud Run because we use Cloud Storage 17 | /public/uploads 18 | 19 | # Firebase exclusions 20 | firebase.json 21 | .firebaserc 22 | .firebase 23 | *.rules 24 | 25 | # Other exclusions 26 | node_modules 27 | *-debug.log 28 | .dockerignore 29 | .git 30 | .gitignore 31 | .gcloudignore 32 | 33 | # Edit this to exclude your dev service account keys 34 | # because Cloud Run supports default credentials 35 | #key.json 36 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/scripts/deploy.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs-extra'); 2 | const execa = require('execa'); 3 | 4 | 5 | deploy().catch(err => console.error(err)); 6 | 7 | async function deploy() { 8 | // Load GCP project ID and name 9 | const { projects: { default: projectId } } = await fs.readJSON('.firebaserc'); 10 | const { name } = await fs.readJSON('package.json') 11 | const tag = `us.gcr.io/${projectId}/${name}`; 12 | 13 | // Submit build to gcloud 14 | await execa.command(`gcloud builds submit --tag ${tag} --project ${projectId}`, { stdio: 'inherit' }); 15 | 16 | // Deploy to Cloud Run 17 | await execa.command(`gcloud run deploy ${name} --quiet --image ${tag} --project ${projectId} --update-env-vars GCP_PROJECT=${projectId} --platform managed --region us-central1 --allow-unauthenticated`, { stdio: 'inherit' }); 18 | } 19 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/.dockerignore: -------------------------------------------------------------------------------- 1 | # Exclude scripts 2 | /scripts 3 | 4 | # Exclude Strapi build outputs 5 | # These are for the admin font-end which is deployed to Firebase Hosting 6 | # so we don't want them in the Cloud Run image 7 | /admin 8 | /build 9 | /.cache 10 | /.temp 11 | .strapi-updater.json 12 | 13 | # Exclude environment variables because we set variables using Cloud Run 14 | .env 15 | 16 | # Don't serve uploads from Cloud Run because we use Cloud Storage 17 | /public/uploads 18 | 19 | # Firebase exclusions 20 | firebase.json 21 | .firebaserc 22 | .firebase 23 | *.rules 24 | 25 | # Other exclusions 26 | node_modules 27 | *-debug.log 28 | Dockerfile 29 | .dockerignore 30 | .git 31 | .gitignore 32 | .gcloudignore 33 | 34 | # Edit this to exclude your dev service account keys 35 | # because Cloud Run supports default credentials 36 | #key.json 37 | -------------------------------------------------------------------------------- /test/helpers/builder/context.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { merge } = require('lodash/fp'); 4 | 5 | const getDefaultState = () => ({ actions: [], models: [], fixtures: {} }); 6 | 7 | const createContext = initialState => { 8 | let state; 9 | 10 | const contextApi = { 11 | get state() { 12 | return state; 13 | }, 14 | 15 | addAction(action) { 16 | state.actions.push(action); 17 | return this; 18 | }, 19 | 20 | addModel(model) { 21 | state.models.push(model); 22 | return this; 23 | }, 24 | 25 | addFixtures(modelName, entries) { 26 | state.fixtures = merge(state.fixtures, { [modelName]: entries }); 27 | return this; 28 | }, 29 | 30 | resetState() { 31 | return this.setState({ ...getDefaultState(), ...initialState }); 32 | }, 33 | 34 | setState(newState) { 35 | state = newState; 36 | return this; 37 | }, 38 | }; 39 | 40 | return contextApi.resetState(); 41 | }; 42 | 43 | module.exports = { createContext }; 44 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "node", 6 | "request": "launch", 7 | "name": "Debug tests", 8 | "cwd": "${workspaceRoot}/test", 9 | "runtimeArgs": [ 10 | "--inspect-brk", 11 | "${workspaceRoot}/test/node_modules/jest/bin/jest", 12 | "--runInBand", 13 | "--forceExit", 14 | "--detectOpenHandles" 15 | ], 16 | "console": "integratedTerminal", 17 | "internalConsoleOptions": "neverOpen", 18 | "autoAttachChildProcesses": true 19 | }, 20 | { 21 | "type": "node", 22 | "request": "launch", 23 | "name": "Debug example", 24 | "cwd": "${workspaceFolder}/examples/cloud-run-and-hosting", 25 | "program": "${workspaceFolder}/examples/cloud-run-and-hosting/node_modules/strapi/bin/strapi", 26 | "args": [ 27 | "develop" 28 | ], 29 | "envFile": "${workspaceFolder}/examples/cloud-run-and-hosting/.env", 30 | "autoAttachChildProcesses": true 31 | } 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /src/utils/components.ts: -------------------------------------------------------------------------------- 1 | import type { FirestoreConnectorModel } from "../model"; 2 | import type { AttributeKey } from "../types"; 3 | 4 | export interface Component { 5 | key: string 6 | value: any 7 | model: FirestoreConnectorModel 8 | } 9 | 10 | export function getComponentModel(componentName: string): FirestoreConnectorModel 11 | export function getComponentModel(hostModel: FirestoreConnectorModel, key: AttributeKey, value: T[AttributeKey]): FirestoreConnectorModel 12 | export function getComponentModel(hostModelOrName: FirestoreConnectorModel | string, key?: AttributeKey, value?: any): FirestoreConnectorModel { 13 | const modelName = typeof hostModelOrName === 'string' 14 | ? hostModelOrName 15 | : value!.__component || hostModelOrName.attributes[key!].component; 16 | 17 | const model = strapi.components[modelName]; 18 | if (!model) { 19 | throw new Error(`Cannot find model for component "${modelName}"`); 20 | } 21 | return model; 22 | } 23 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Arrowhead Apps Ltd. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/config/database.js: -------------------------------------------------------------------------------- 1 | const flattening = { 2 | flatten_all: true, 3 | flatten_none: false, 4 | 5 | // Flatten models that are referred to 6 | // by non-flattened models 7 | flatten_mixed_src: [ 8 | /category/, 9 | /tag/, 10 | /user/, 11 | /collector/, 12 | ], 13 | 14 | // Flatten models that refer to 15 | // non-flattened models 16 | flatten_mixed_target: [ 17 | /reference/, 18 | /article/, 19 | /paniniCard/, 20 | ], 21 | }; 22 | 23 | module.exports = ({ env }) => ({ 24 | defaultConnection: 'default', 25 | connections: { 26 | default: { 27 | connector: 'firestore', 28 | settings: { 29 | projectId: 'test-project-id', 30 | }, 31 | options: { 32 | useEmulator: true, 33 | maxQuerySize: 0, 34 | // logQueries: true, 35 | // logTransactionStats: true, 36 | 37 | // TODO: Disable if test requirements allow 38 | allowNonNativeQueries: true, 39 | 40 | // Use flattening config from env variable 41 | // Default to no flattening 42 | flattenModels: flattening[process.env.FLATTENING] || [] 43 | }, 44 | } 45 | }, 46 | }); 47 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/Dockerfile: -------------------------------------------------------------------------------- 1 | # Images can be manually built and deployed to GCP with the following commands 2 | # > docker build . --tag gcr.io/${PROJECT_ID}/${IMAGE_NAME} 3 | # > docker push gcr.io/${PROJECT_ID}/${IMAGE_NAME} 4 | 5 | # To clean up afterwards, keeping the new image, run: 6 | # > docker system prune -f 7 | 8 | # To remove all images, run: 9 | # > docker system prune -f -a 10 | 11 | 12 | # Use the official lightweight LTS Node.js image. 13 | # https://hub.docker.com/_/node 14 | FROM node:lts-alpine 15 | 16 | # Create and change to the app directory. 17 | WORKDIR /usr/src/app 18 | 19 | # Copy application dependency manifests to the container image. 20 | # A wildcard is used to ensure both package.json AND package-lock.json are copied. 21 | # Copying this separately prevents re-running npm install on every code change. 22 | COPY package*.json ./ 23 | 24 | # Install production dependencies. 25 | RUN npm ci --only=production 26 | 27 | # Copy local code to the container image. 28 | COPY . ./ 29 | 30 | # Change to run as non-privileged user 31 | USER node 32 | 33 | # Run the web service on container startup as an unpriviledged user. 34 | CMD [ "npm", "start" ] 35 | -------------------------------------------------------------------------------- /test/utils/setup-test.js: -------------------------------------------------------------------------------- 1 | 'user-strict'; 2 | 3 | const { startFirestore, stopFirestore } = require('./firestore'); 4 | const { cleanTestApp } = require('./app'); 5 | 6 | let firestore = null; 7 | 8 | beforeAll(async () => { 9 | await cleanTestApp(); 10 | firestore = await startFirestore(); 11 | }); 12 | 13 | afterAll(async () => { 14 | await stopFirestore(firestore); 15 | firestore = null; 16 | }); 17 | 18 | 19 | // From https://github.com/strapi/strapi/blob/23bd0226a594058f5b0b25c82aa03f90b691df9b/test/jest2e2.setup.js 20 | 21 | const isoDateRegex = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/; 22 | 23 | jest.setTimeout(60000); 24 | 25 | expect.extend({ 26 | stringOrNull(received) { 27 | const pass = typeof received === 'string' || received === null; 28 | return { 29 | message: () => `expected ${received} ${pass ? 'not ' : ''}to be null or a string`, 30 | pass, 31 | }; 32 | }, 33 | toBeISODate(received) { 34 | const pass = isoDateRegex.test(received) && new Date(received).toISOString() === received; 35 | return { 36 | pass, 37 | message: () => `Expected ${received} ${pass ? 'not ' : ''}to be a valid ISO date string`, 38 | }; 39 | }, 40 | }); 41 | -------------------------------------------------------------------------------- /src/coerce/coerce-to-firestore.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { FieldOperation } from '../db/field-operation'; 3 | import type { FirestoreConnectorModel } from '../model'; 4 | 5 | /** 6 | * Lightweight converter for a root model object. Ensures that the 7 | * `primaryKey` is not set on the Firestore data. 8 | */ 9 | export function coerceModelToFirestore(model: FirestoreConnectorModel, data: T): T { 10 | const obj = coerceToFirestore(data); 11 | _.unset(obj, model.primaryKey); 12 | return obj; 13 | } 14 | 15 | /** 16 | * Lightweight converter that converts known custom classes 17 | * to Firestore-compatible values. 18 | */ 19 | export function coerceToFirestore(data: T): T { 20 | return _.cloneDeepWith(data, value => { 21 | 22 | // Coerce values within FieldOperation 23 | // and convert to its native counterpart 24 | if (value instanceof FieldOperation) { 25 | return value 26 | .coerceWith(coerceToFirestore) 27 | .toFirestoreValue(); 28 | } 29 | 30 | if (value && (typeof value === 'object') && ('toFirestoreValue' in value)) { 31 | return value.toFirestoreValue() 32 | } 33 | 34 | return undefined; 35 | }); 36 | } 37 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "strapi-connector-firestore", 3 | "version": "3.0.0-alpha.47", 4 | "description": "Strapi database connector for Firestore database on Google Cloud Platform.", 5 | "keywords": [ 6 | "firestore", 7 | "hook", 8 | "orm", 9 | "nosql", 10 | "strapi" 11 | ], 12 | "author": "Arrowhead Apps Ltd", 13 | "license": "MIT", 14 | "repository": "github:arrowheadapps/strapi-connector-firestore", 15 | "main": "lib/index.js", 16 | "engines": { 17 | "node": ">=12.0.0", 18 | "npm": ">=6.0.0" 19 | }, 20 | "files": [ 21 | "lib/**" 22 | ], 23 | "scripts": { 24 | "build": "rimraf lib && tsc", 25 | "prepare": "tsc --skipLibCheck", 26 | "test": "rimraf lib && tsc --sourceMap && npm test --prefix test" 27 | }, 28 | "dependencies": { 29 | "@google-cloud/firestore": "^6.0.0", 30 | "@types/pino": "^7.0.5", 31 | "fs-extra": "^10.0.1", 32 | "lodash": "^4.17.21", 33 | "p-queue": "^6.6.2", 34 | "strapi-utils": "^3.6.10" 35 | }, 36 | "devDependencies": { 37 | "@tsconfig/node12": "^1.0.11", 38 | "@types/fs-extra": "^9.0.13", 39 | "@types/lodash": "^4.14.184", 40 | "@types/node": "^12.20.55", 41 | "rimraf": "^3.0.2", 42 | "typescript": "^4.7.4" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /test/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "scripts": { 4 | "pretest": "node setup.js", 5 | "test": "jest --runInBand --forceExit --detectOpenHandles --json --outputFile=../coverage/results.json", 6 | "coverage-combine": "istanbul-combine -d ../coverage/combined -p detail -r json ../coverage/*/coverage.json" 7 | }, 8 | "dependencies": { 9 | "degit": "^2.8.4", 10 | "firebase-tools": "^10.6.0", 11 | "jest": "^27.5.1", 12 | "lodash": "^4.17.21", 13 | "qs": "^6.10.3", 14 | "remap-istanbul": "^0.13.0", 15 | "request-promise-native": "^1.0.9", 16 | "strapi": "^3.6.9", 17 | "strapi-admin": "^3.6.9", 18 | "strapi-connector-firestore": "file:..", 19 | "strapi-plugin-content-manager": "^3.6.9", 20 | "strapi-plugin-content-type-builder": "^3.6.9", 21 | "strapi-plugin-upload": "^3.6.9", 22 | "strapi-plugin-users-permissions": "^3.6.9", 23 | "strapi-utils": "^3.6.9", 24 | "supertest": "^6.2.2", 25 | "wait-on": "^6.0.1" 26 | }, 27 | "devDependencies": { 28 | "@actions/core": "^1.6.0", 29 | "@actions/github": "^5.0.1", 30 | "@actions/io": "^1.1.2", 31 | "fs-extra": "^10.0.1", 32 | "glob": "^7.2.0", 33 | "glob-promise": "^4.2.2", 34 | "istanbul-combine": "^0.3.0" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /test/helpers/auth.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { createRequest } = require('./request'); 4 | 5 | const auth = { 6 | email: 'admin@strapi.io', 7 | firstname: 'admin', 8 | lastname: 'admin', 9 | password: 'Password123', 10 | }; 11 | 12 | const rq = createRequest(); 13 | 14 | const register = async () => { 15 | await rq({ 16 | url: '/admin/register-admin', 17 | method: 'POST', 18 | body: auth, 19 | }).catch(err => { 20 | console.error(err); 21 | if (err.message === 'You cannot register a new super admin') return; 22 | throw err; 23 | }); 24 | }; 25 | 26 | const login = async () => { 27 | const { body } = await rq({ 28 | url: '/admin/login', 29 | method: 'POST', 30 | body: { 31 | email: auth.email, 32 | password: auth.password, 33 | }, 34 | }); 35 | 36 | return body.data; 37 | }; 38 | 39 | module.exports = { 40 | async registerAndLogin() { 41 | // register 42 | await register(); 43 | 44 | // login 45 | const res = await login(); 46 | 47 | return res && res.token; 48 | }, 49 | async login() { 50 | const res = await login(); 51 | 52 | return res && res.token; 53 | }, 54 | async getUser() { 55 | const res = await login(); 56 | 57 | return res.user; 58 | }, 59 | }; 60 | -------------------------------------------------------------------------------- /test/helpers/strapi.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const path = require('path'); 4 | const _ = require('lodash'); 5 | const strapi = require('strapi/lib'); 6 | const { createUtils } = require('./utils'); 7 | 8 | const superAdminCredentials = { 9 | email: 'admin@strapi.io', 10 | firstname: 'admin', 11 | lastname: 'admin', 12 | password: 'Password123', 13 | }; 14 | 15 | const superAdminLoginInfo = _.pick(superAdminCredentials, ['email', 'password']); 16 | 17 | const TEST_APP_URL = path.resolve(__dirname, '../'); 18 | 19 | const createStrapiInstance = async ({ ensureSuperAdmin = true, logLevel = 'fatal' } = {}) => { 20 | const options = { dir: TEST_APP_URL }; 21 | const instance = strapi(options); 22 | 23 | await instance.load(); 24 | 25 | instance.log.level = logLevel; 26 | 27 | await instance.app 28 | // Populate Koa routes 29 | .use(instance.router.routes()) 30 | // Populate Koa methods 31 | .use(instance.router.allowedMethods()); 32 | 33 | const utils = createUtils(instance); 34 | 35 | if (ensureSuperAdmin) { 36 | await utils.createUserIfNotExists(superAdminCredentials); 37 | } 38 | 39 | return instance; 40 | }; 41 | 42 | module.exports = { 43 | createStrapiInstance, 44 | superAdmin: { 45 | loginInfo: superAdminLoginInfo, 46 | credentials: superAdminCredentials, 47 | }, 48 | }; 49 | -------------------------------------------------------------------------------- /src/db/component-collection.ts: -------------------------------------------------------------------------------- 1 | import type { CollectionReference } from '@google-cloud/firestore'; 2 | import type { FirestoreConnectorModel } from '../model'; 3 | import type { Collection } from './collection'; 4 | 5 | 6 | export class ComponentCollection implements Collection { 7 | 8 | private collection: CollectionReference 9 | constructor(readonly model: FirestoreConnectorModel) { 10 | this.collection = model.firestore.collection(model.collectionName); 11 | } 12 | 13 | private throw(): never { 14 | throw new Error( 15 | 'Operations are not supported on component collections. ' + 16 | 'This connector embeds components directly into the parent document.' 17 | ); 18 | } 19 | 20 | get converter() { 21 | return this.throw(); 22 | } 23 | 24 | get path() { 25 | return this.throw(); 26 | } 27 | 28 | autoId(): string { 29 | // This is used to generate IDs for components 30 | return this.collection.doc().id; 31 | } 32 | 33 | doc() { 34 | return this.throw(); 35 | } 36 | 37 | get() { 38 | return this.throw(); 39 | } 40 | 41 | where() { 42 | return this.throw(); 43 | } 44 | 45 | orderBy() { 46 | return this.throw(); 47 | } 48 | 49 | limit() { 50 | return this.throw(); 51 | } 52 | 53 | offset() { 54 | return this.throw(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "api-admin", 3 | "private": true, 4 | "scripts": { 5 | "develop": "npm run clean && strapi develop", 6 | "emulator": "firebase emulators:start --only=firestore --import=./.tmp --export-on-exit", 7 | "start": "NODE_ENV=production strapi start", 8 | "build": "strapi build", 9 | "build:prod": "npm run clean && patch-package && NODE_ENV=production strapi build", 10 | "deploy": "npm run deploy:frontend && npm run deploy:backend", 11 | "deploy:frontend": "npm run build:prod && firebase deploy --only hosting", 12 | "deploy:backend": "node scripts/deploy.js", 13 | "strapi": "strapi", 14 | "clean": "rimraf .temp .cache build" 15 | }, 16 | "dependencies": { 17 | "strapi": "^3.6.9", 18 | "strapi-admin": "^3.6.9", 19 | "strapi-connector-firestore": "^3.0.0-alpha.42", 20 | "strapi-plugin-content-manager": "^3.6.9", 21 | "strapi-plugin-content-type-builder": "^3.6.9", 22 | "strapi-plugin-email": "^3.6.9", 23 | "strapi-plugin-upload": "^3.6.9", 24 | "strapi-plugin-users-permissions": "^3.6.9", 25 | "strapi-provider-upload-google-cloud-storage": "^4.0.0", 26 | "strapi-utils": "^3.6.9" 27 | }, 28 | "devDependencies": { 29 | "execa": "^5.1.1", 30 | "fs-extra": "^10.0.1", 31 | "patch-package": "^6.4.7", 32 | "rimraf": "^3.0.2" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /test/utils/firestore.js: -------------------------------------------------------------------------------- 1 | 'use-strict'; 2 | 3 | const { FirestoreEmulator } = require('firebase-tools/lib/emulator/firestoreEmulator'); 4 | 5 | 6 | const startFirestore = async () => { 7 | const firestore = new FirestoreEmulator({ 8 | host: '127.0.0.1', 9 | port: '8080', 10 | projectId: 'test-project-id', 11 | }); 12 | const stop = async () => { 13 | process.stdout.write('Stopping Firestore because process is exiting....'); 14 | await firestore.stop(); 15 | process.exit(); 16 | }; 17 | 18 | process.once('SIGINT', stop); 19 | process.once('SIGTERM', stop); 20 | process.once('SIGABRT', stop); 21 | process.once('beforeExit', stop); 22 | 23 | try { 24 | await firestore.start(); 25 | process.stdout.write('Firestore online.\r\n'); 26 | return firestore; 27 | } catch (err) { 28 | process.stdout.write('Firestore failed to start.\r\n'); 29 | console.error(err); 30 | throw err; 31 | } 32 | }; 33 | 34 | /** 35 | * 36 | * @param {FirestoreEmulator} firestore 37 | */ 38 | const stopFirestore = async (firestore) => { 39 | try { 40 | await firestore.stop(); 41 | process.stdout.write('Firestore stopped.\r\n'); 42 | } catch (err) { 43 | process.stdout.write('Firestore failed to stop.\r\n'); 44 | console.error(err); 45 | throw err; 46 | } 47 | }; 48 | 49 | module.exports = { 50 | startFirestore, 51 | stopFirestore, 52 | }; 53 | -------------------------------------------------------------------------------- /src/db/collection.ts: -------------------------------------------------------------------------------- 1 | import type { OrderByDirection, FieldPath, FirestoreDataConverter } from '@google-cloud/firestore'; 2 | import type { FirestoreFilter, StrapiOrFilter, StrapiWhereFilter } from '../types'; 3 | import type { DeepReference } from './deep-reference'; 4 | import type { FirestoreConnectorModel } from '../model'; 5 | import type { Snapshot } from './reference'; 6 | import type { NormalReference } from './normal-reference'; 7 | import type { ReadRepository } from '../utils/read-repository'; 8 | import { VirtualReference } from './virtual-reference'; 9 | 10 | 11 | export interface QuerySnapshot { 12 | docs: Snapshot[] 13 | empty: boolean 14 | } 15 | 16 | 17 | export interface Queryable { 18 | get(trans?: ReadRepository): Promise>; 19 | 20 | where(filter: StrapiWhereFilter | StrapiOrFilter | FirestoreFilter): Queryable; 21 | orderBy(field: string | FieldPath, directionStr?: OrderByDirection): Queryable; 22 | limit(limit: number): Queryable; 23 | offset(offset: number): Queryable; 24 | } 25 | 26 | export interface Collection extends Queryable { 27 | readonly model: FirestoreConnectorModel 28 | readonly path: string 29 | readonly converter: FirestoreDataConverter 30 | 31 | autoId(): string; 32 | doc(): NormalReference | DeepReference | VirtualReference; 33 | doc(id: string): NormalReference | DeepReference | VirtualReference; 34 | 35 | get(repo?: ReadRepository): Promise>; 36 | } 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | lib/ 2 | 3 | # Ignore any APIs created in the example projects 4 | # for testing 5 | examples/*/api/*/** 6 | examples/*/components/** 7 | examples/*/extensions/*/models/** 8 | examples/*/public/uploads/** 9 | 10 | ############################ 11 | # OS X 12 | ############################ 13 | 14 | .DS_Store 15 | .AppleDouble 16 | .LSOverride 17 | Icon 18 | .Spotlight-V100 19 | .Trashes 20 | ._* 21 | 22 | 23 | ############################ 24 | # Linux 25 | ############################ 26 | 27 | *~ 28 | 29 | 30 | ############################ 31 | # Windows 32 | ############################ 33 | 34 | Thumbs.db 35 | ehthumbs.db 36 | Desktop.ini 37 | $RECYCLE.BIN/ 38 | *.cab 39 | *.msi 40 | *.msm 41 | *.msp 42 | 43 | 44 | ############################ 45 | # Packages 46 | ############################ 47 | 48 | *.7z 49 | *.csv 50 | *.dat 51 | *.dmg 52 | *.gz 53 | *.iso 54 | *.jar 55 | *.rar 56 | *.tar 57 | *.zip 58 | *.com 59 | *.class 60 | *.dll 61 | *.exe 62 | *.o 63 | *.seed 64 | *.so 65 | *.swo 66 | *.swp 67 | *.swn 68 | *.swm 69 | *.out 70 | *.pid 71 | 72 | 73 | ############################ 74 | # Logs and databases 75 | ############################ 76 | 77 | .tmp 78 | *.log 79 | *.sql 80 | *.sqlite 81 | 82 | 83 | ############################ 84 | # Misc. 85 | ############################ 86 | 87 | *# 88 | .idea 89 | nbproject 90 | 91 | 92 | ############################ 93 | # Node.js 94 | ############################ 95 | 96 | lib-cov 97 | lcov.info 98 | pids 99 | logs 100 | results 101 | build 102 | node_modules 103 | .node_history 104 | coverage 105 | .nyc_output 106 | -------------------------------------------------------------------------------- /src/utils/manual-filter.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import type { FieldPath, OrderByDirection } from '@google-cloud/firestore'; 3 | import type { Snapshot } from '../db/reference'; 4 | import type { FirestoreConnectorModel } from '../model'; 5 | import type { QuerySnapshot } from '../db/collection'; 6 | 7 | 8 | export type PartialSnapshot = Pick, 'id' | 'data'>; 9 | 10 | export interface ManualFilter { 11 | (data: PartialSnapshot): boolean 12 | } 13 | 14 | export interface OrderSpec { 15 | field: string | FieldPath 16 | directionStr: OrderByDirection 17 | } 18 | 19 | export interface ManualFilterArgs { 20 | model: FirestoreConnectorModel 21 | data: { [id: string]: T } 22 | filters: ManualFilter[] 23 | orderBy: OrderSpec[] 24 | offset: number | undefined 25 | limit: number | undefined 26 | } 27 | 28 | export function applyManualFilters(args: ManualFilterArgs): QuerySnapshot { 29 | let docs: Snapshot[] = []; 30 | for (const [id, data] of Object.entries(args.data)) { 31 | // Must match every 'AND' filter (if any exist) 32 | // and at least one 'OR' filter (if any exists) 33 | const snap: Snapshot = { 34 | id, 35 | ref: args.model.db.doc(id), 36 | exists: data != null, 37 | data: () => data, 38 | }; 39 | if (args.filters.every(f => f(snap))) { 40 | docs.push(snap); 41 | } 42 | } 43 | 44 | for (const { field, directionStr } of args.orderBy) { 45 | docs = _.orderBy(docs, d => args.model.getAttributeValue(field, d), directionStr); 46 | } 47 | 48 | // Offset and limit after sorting 49 | const offset = Math.max(args.offset || 0, 0); 50 | const limit = Math.max(args.limit || 0, 0) || docs.length; 51 | docs = docs.slice(offset, offset + limit); 52 | 53 | return { 54 | docs, 55 | empty: docs.length === 0 56 | }; 57 | } 58 | -------------------------------------------------------------------------------- /test/utils/app.js: -------------------------------------------------------------------------------- 1 | 'use-strict'; 2 | 3 | const path = require('path'); 4 | const fs = require('fs-extra'); 5 | const degit = require('degit'); 6 | 7 | 8 | const testsDir = 'tests'; 9 | const flattenExcludes = { 10 | flatten_all: [], 11 | flatten_none: [], 12 | 13 | // For mixed-flattening tests we only need to test relations 14 | // so skip all the other tests 15 | flatten_mixed_src: [ 16 | /api/, 17 | /filtering/, 18 | /search/, 19 | /single-type/ 20 | ], 21 | flatten_mixed_target: [ 22 | /api/, 23 | /filtering/, 24 | /search/, 25 | /single-type/ 26 | ] 27 | }; 28 | 29 | const cleanTestApp = async () => { 30 | await Promise.all([ 31 | fs.remove('.cache'), 32 | fs.remove('.temp'), 33 | fs.remove('public'), 34 | fs.remove('build'), 35 | fs.remove('components'), 36 | fs.emptyDir('api'), 37 | fs.emptyDir('extensions'), 38 | ]); 39 | }; 40 | 41 | /** 42 | * Removes the Strapi tests. 43 | */ 44 | const cleanTests = async () => { 45 | await fs.remove(path.resolve(testsDir)); 46 | }; 47 | 48 | /** 49 | * Jest seemingly refuses to run tests located under `node_modules`, 50 | * so we copy Strapi's tests out into our test dir. 51 | */ 52 | const copyTests = async () => { 53 | // Determine installed Strapi version 54 | const { version } = require('strapi/package.json'); 55 | 56 | // Download the tests from GitHub 57 | await fs.emptyDir(testsDir); 58 | await degit(`strapi/strapi#v${version}`).clone(`${testsDir}/.strapi`); 59 | await fs.copy(`${testsDir}/.strapi/packages/strapi/${testsDir}`, testsDir); 60 | await fs.remove(`${testsDir}/.strapi`); 61 | 62 | // Remove excluded tests 63 | console.log(`Collection flattening: "${process.env.FLATTENING || 'flatten_none'}"`) 64 | const excludes = flattenExcludes[process.env.FLATTENING] || []; 65 | for (const p of await fs.readdir(testsDir)) { 66 | if (excludes.some(e => e.test(p))) { 67 | await fs.remove(path.join(testsDir, p)); 68 | } 69 | } 70 | }; 71 | 72 | const setupTestApp = async () => { 73 | await cleanTestApp(); 74 | await copyTests(); 75 | 76 | // Clean coverage outputs 77 | // Jest seems to fail to write the JSON results otherwise 78 | await Promise.all([ 79 | fs.emptyDir(path.resolve('../coverage')), 80 | ]); 81 | }; 82 | 83 | 84 | module.exports = { 85 | cleanTestApp, 86 | copyTests, 87 | cleanTests, 88 | setupTestApp, 89 | }; 90 | -------------------------------------------------------------------------------- /test/helpers/utils.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | 5 | const createUtils = strapi => { 6 | const login = async userInfo => { 7 | const sanitizedUserInfo = _.pick(userInfo, ['email', 'id']); 8 | const user = await strapi.admin.services.user.findOne(sanitizedUserInfo); 9 | if (!user) { 10 | throw new Error('User not found'); 11 | } 12 | const token = strapi.admin.services.token.createJwtToken(user); 13 | 14 | return { token, user }; 15 | }; 16 | const registerOrLogin = async userCredentials => { 17 | await createUserIfNotExists(userCredentials); 18 | return login(userCredentials); 19 | }; 20 | 21 | const findUser = strapi.admin.services.user.findOne; 22 | const userExists = strapi.admin.services.user.exists; 23 | const createUser = async userInfo => { 24 | const superAdminRole = await strapi.admin.services.role.getSuperAdminWithUsersCount(); 25 | 26 | if (superAdminRole.usersCount === 0) { 27 | const userRoles = _.uniq((userInfo.roles || []).concat(superAdminRole.id)); 28 | Object.assign(userInfo, { roles: userRoles }); 29 | } 30 | 31 | return strapi.admin.services.user.create({ 32 | registrationToken: null, 33 | isActive: true, 34 | ...userInfo, 35 | }); 36 | }; 37 | const deleteUserById = strapi.admin.services.user.deleteById; 38 | const deleteUsersById = strapi.admin.services.user.deleteByIds; 39 | const createUserIfNotExists = async userInfo => { 40 | const sanitizedUserInfo = _.pick(userInfo, ['email', 'id']); 41 | const exists = await userExists(sanitizedUserInfo); 42 | 43 | return !exists ? createUser(userInfo) : null; 44 | }; 45 | 46 | const createRole = strapi.admin.services.role.create; 47 | const getRole = strapi.admin.services.role.find; 48 | const deleteRolesById = strapi.admin.services.role.deleteByIds; 49 | const getSuperAdminRole = strapi.admin.services.role.getSuperAdmin; 50 | const assignPermissionsToRole = strapi.admin.services.role.assignPermissions; 51 | 52 | return { 53 | // Auth 54 | login, 55 | registerOrLogin, 56 | // Users 57 | findUser, 58 | createUser, 59 | createUserIfNotExists, 60 | userExists, 61 | deleteUserById, 62 | deleteUsersById, 63 | // Roles 64 | getRole, 65 | getSuperAdminRole, 66 | createRole, 67 | deleteRolesById, 68 | assignPermissionsToRole, 69 | }; 70 | }; 71 | 72 | module.exports = { createUtils }; 73 | -------------------------------------------------------------------------------- /test/helpers/agent.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { clone, has, concat, isNil } = require('lodash/fp'); 4 | const qs = require('qs'); 5 | const request = require('supertest'); 6 | const { createUtils } = require('./utils'); 7 | 8 | const createAgent = (strapi, initialState = {}) => { 9 | const state = clone(initialState); 10 | const utils = createUtils(strapi); 11 | 12 | const agent = options => { 13 | const { method, url, body, formData, qs: queryString } = options; 14 | const supertestAgent = request.agent(strapi.server); 15 | 16 | if (has('token', state)) { 17 | supertestAgent.auth(state.token, { type: 'bearer' }); 18 | } 19 | 20 | const fullUrl = concat(state.urlPrefix, url).join(''); 21 | 22 | const rq = supertestAgent[method.toLowerCase()](fullUrl); 23 | 24 | if (queryString) { 25 | rq.query(qs.stringify(queryString)); 26 | } 27 | 28 | if (body) { 29 | rq.send(body); 30 | } 31 | 32 | if (formData) { 33 | const attachFieldToRequest = field => rq.field(field, formData[field]); 34 | Object.keys(formData).forEach(attachFieldToRequest); 35 | } 36 | 37 | if (isNil(formData)) { 38 | rq.type('application/json'); 39 | } 40 | 41 | return rq; 42 | }; 43 | 44 | const createShorthandMethod = method => (url, options = {}) => { 45 | return agent({ ...options, url, method }); 46 | }; 47 | 48 | Object.assign(agent, { 49 | assignState(newState) { 50 | Object.assign(state, newState); 51 | return agent; 52 | }, 53 | 54 | setURLPrefix(path) { 55 | return this.assignState({ urlPrefix: path }); 56 | }, 57 | 58 | setToken(token) { 59 | return this.assignState({ token }); 60 | }, 61 | 62 | setLoggedUser(loggedUser) { 63 | return this.assignState({ loggedUser }); 64 | }, 65 | 66 | getLoggedUser() { 67 | return state.loggedUser; 68 | }, 69 | 70 | async login(userInfo) { 71 | const { token, user } = await utils.login(userInfo); 72 | 73 | this.setToken(token).setLoggedUser(user); 74 | 75 | return agent; 76 | }, 77 | 78 | async registerOrLogin(userCredentials) { 79 | const { token, user } = await utils.registerOrLogin(userCredentials); 80 | 81 | this.setToken(token).setLoggedUser(user); 82 | 83 | return agent; 84 | }, 85 | 86 | get: createShorthandMethod('GET'), 87 | post: createShorthandMethod('POST'), 88 | put: createShorthandMethod('PUT'), 89 | delete: createShorthandMethod('DELETE'), 90 | }); 91 | 92 | return agent; 93 | }; 94 | 95 | module.exports = { 96 | createAgent, 97 | }; 98 | -------------------------------------------------------------------------------- /test/helpers/builder/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { get } = require('lodash/fp'); 4 | 5 | const _ = require('lodash'); 6 | const modelsUtils = require('../models'); 7 | const { sanitizeEntity } = require('strapi-utils'); 8 | const actionRegistry = require('./action-registry'); 9 | const { createContext } = require('./context'); 10 | 11 | const createTestBuilder = (options = {}) => { 12 | const { initialState } = options; 13 | const ctx = createContext(initialState); 14 | 15 | return { 16 | get models() { 17 | return ctx.state.models; 18 | }, 19 | 20 | get fixtures() { 21 | return ctx.state.fixtures; 22 | }, 23 | 24 | sanitizedFixtures(strapi) { 25 | return _.mapValues(this.fixtures, (value, key) => this.sanitizedFixturesFor(key, strapi)); 26 | }, 27 | 28 | sanitizedFixturesFor(modelName, strapi) { 29 | const model = strapi.getModel(modelName); 30 | const fixtures = this.fixturesFor(modelName); 31 | 32 | return sanitizeEntity(fixtures, { model }); 33 | }, 34 | 35 | fixturesFor(modelName) { 36 | return this.fixtures[modelName]; 37 | }, 38 | 39 | addAction(code, ...params) { 40 | const actionCreator = get(code, actionRegistry); 41 | 42 | ctx.addAction(actionCreator(...params)); 43 | 44 | return this; 45 | }, 46 | 47 | addContentType(contentType) { 48 | return this.addAction('contentType.create', contentType); 49 | }, 50 | 51 | addContentTypes(contentTypes, { batch = true } = {}) { 52 | return this.addAction( 53 | batch ? 'contentType.createBatch' : 'contentType.createMany', 54 | contentTypes 55 | ); 56 | }, 57 | 58 | addComponent(component) { 59 | return this.addAction('component.create', component); 60 | }, 61 | 62 | addFixtures(model, entries) { 63 | return this.addAction('fixtures.create', model, entries, () => this.fixtures); 64 | }, 65 | 66 | async build() { 67 | for (const action of ctx.state.actions) { 68 | await action.build(ctx); 69 | } 70 | 71 | return this; 72 | }, 73 | 74 | async cleanup(options = {}) { 75 | const { enableTestDataAutoCleanup = true } = options; 76 | const { models, actions } = ctx.state; 77 | 78 | if (enableTestDataAutoCleanup) { 79 | for (const model of models.reverse()) { 80 | await modelsUtils.cleanupModel(model.uid || model.modelName); 81 | } 82 | } 83 | 84 | for (const action of actions.reverse()) { 85 | await action.cleanup(ctx); 86 | } 87 | 88 | ctx.resetState(); 89 | 90 | return this; 91 | }, 92 | }; 93 | }; 94 | 95 | module.exports = { createTestBuilder }; 96 | -------------------------------------------------------------------------------- /test/helpers/generators.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | article: { 5 | attributes: { 6 | title: { 7 | type: 'string', 8 | }, 9 | date: { 10 | type: 'date', 11 | }, 12 | jsonField: { 13 | type: 'json', 14 | }, 15 | content: { 16 | type: 'richtext', 17 | }, 18 | author: { 19 | nature: 'manyToOne', 20 | target: 'plugins::users-permissions.user', 21 | targetAttribute: 'articles', 22 | }, 23 | }, 24 | connection: 'default', 25 | name: 'article', 26 | description: '', 27 | collectionName: '', 28 | }, 29 | tag: { 30 | attributes: { 31 | name: { 32 | type: 'string', 33 | }, 34 | articles: { 35 | dominant: true, 36 | nature: 'manyToMany', 37 | target: 'application::article.article', 38 | targetAttribute: 'tags', 39 | }, 40 | }, 41 | connection: 'default', 42 | name: 'tag', 43 | description: '', 44 | collectionName: '', 45 | }, 46 | category: { 47 | attributes: { 48 | name: { 49 | type: 'string', 50 | }, 51 | articles: { 52 | nature: 'oneToMany', 53 | target: 'application::article.article', 54 | targetAttribute: 'category', 55 | }, 56 | }, 57 | connection: 'default', 58 | name: 'category', 59 | description: '', 60 | collectionName: '', 61 | }, 62 | reference: { 63 | attributes: { 64 | name: { 65 | type: 'string', 66 | }, 67 | article: { 68 | target: 'application::article.article', 69 | targetAttribute: 'reference', 70 | nature: 'oneToOne', 71 | }, 72 | tag: { 73 | nature: 'oneWay', 74 | target: 'application::tag.tag', 75 | }, 76 | }, 77 | connection: 'default', 78 | name: 'reference', 79 | description: '', 80 | collectionName: '', 81 | }, 82 | product: { 83 | attributes: { 84 | name: { 85 | type: 'string', 86 | }, 87 | description: { 88 | type: 'richtext', 89 | }, 90 | published: { 91 | type: 'boolean', 92 | }, 93 | }, 94 | connection: 'default', 95 | name: 'product', 96 | description: '', 97 | collectionName: '', 98 | }, 99 | articlewithtag: { 100 | attributes: { 101 | title: { 102 | type: 'string', 103 | }, 104 | tags: { 105 | nature: 'manyWay', 106 | target: 'application::tag.tag', 107 | }, 108 | }, 109 | connection: 'default', 110 | name: 'articlewithtag', 111 | description: '', 112 | collectionName: '', 113 | }, 114 | }; 115 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | name: "CodeQL" 7 | 8 | on: 9 | push: 10 | branches: [master] 11 | pull_request: 12 | # The branches below must be a subset of the branches above 13 | branches: [master] 14 | schedule: 15 | - cron: '0 11 * * 1' 16 | 17 | jobs: 18 | analyze: 19 | name: Analyze 20 | runs-on: ubuntu-latest 21 | 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | # Override automatic language detection by changing the below list 26 | # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] 27 | language: ['javascript'] 28 | # Learn more... 29 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 30 | 31 | steps: 32 | - name: Checkout repository 33 | uses: actions/checkout@v2 34 | with: 35 | # We must fetch at least the immediate parents so that if this is 36 | # a pull request then we can checkout the head. 37 | fetch-depth: 2 38 | 39 | # If this run was triggered by a pull request event, then checkout 40 | # the head of the pull request instead of the merge commit. 41 | - run: git checkout HEAD^2 42 | if: ${{ github.event_name == 'pull_request' }} 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v1 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v1 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v1 72 | -------------------------------------------------------------------------------- /examples/cloud-run-and-hosting/README.md: -------------------------------------------------------------------------------- 1 | # Example Strapi project using Firestore, Cloud Run, Firebase Hosting, and Cloud Storage 2 | 3 | - [x] The Strapi backend/API is deployed to Google Cloud Run 4 | - [x] The Strapi front-end is deployed to Firebase Hosting 5 | - [x] The backend is aliased at `/api` in Firebase Hosting (see `middlewares/api/index.js`, currently a patch to Strapi is required for this functionality) 6 | - [x] The database is configured to use Firestore 7 | - [x] The upload plugin is configured to use Google Cloud Storage 8 | 9 | ## Pre-requisites 10 | 11 | - Install the Firebase CLI tools 12 | - Install the `gcloud` CLI tools 13 | - Optionally, install the Firestore emulator 14 | - Optionally, install the Docker CLI tools 15 | 16 | ## How to use 17 | 18 | 1. Create a Firebase project. 19 | 2. Insert your Firebase/GCP project ID in `.firebaserc`, and create a file at `./.env` setting `GCP_PROJECT` variable to your project ID (required to run locally; the deployment script automatically sets the environment variable on the Cloud Run container). 20 | 3. Configure the admin JWT secret, as outlined [here](https://strapi.io/documentation/v3.x/migration-guide/migration-guide-3.0.x-to-3.1.x.html#_2-define-the-admin-jwt-token), but also apply it to the Cloud Run container using the GCP console. If this is a new container, you may need to deploy it first before assigning the environment variable, and the first deploy will fail to start without the environment variable. 21 | 22 | ## Run locally 23 | 24 | Start the Firestore emulator (in a separate shell) 25 | 26 | `$ npm run emulator` 27 | 28 | Start Strapi 29 | 30 | `$ npm run develop` 31 | 32 | 33 | ## Deploy backend 34 | 35 | Deploys only the files required to run the Strapi backend to a Cloud Run container: 36 | 37 | 1. Build the image (two options) 38 | 2. Deploy to Cloud Run (excluding the front-end files) 39 | 40 | > NOTE: The example package.json includes scripts to automate deployment. You can try running `$ npm run deploy:backend` 41 | 42 | **Build using Docker** 43 | 44 | `$ docker build . --tag gcr.io/{PROJECT_ID}/api-admin` 45 | 46 | `$ docker push us.gcr.io/{PROJECT_ID}/api-admin` 47 | 48 | 49 | **Build using `gcloud`** 50 | 51 | `$ gcloud builds submit --tag us.gcr.io/{PROJECT-ID}/api-admin` 52 | 53 | 54 | **Deploy to Cloud Run** 55 | 56 | `$ gcloud run deploy api-admin --image us.gcr.io/{PROJECT-ID}/api-admin --project {PROJECT_ID} --platform managed --region us-central1 --allow-unauthenticated` 57 | 58 | 59 | 60 | ## Deploy front-end 61 | 62 | Deploys a production build of the Strapi front-end to to Firebase Hosting. 63 | 64 | 1. Build the front-end 65 | 2. Deploy to Firebase Hosting (only the contents of `./build/`) 66 | 67 | > NOTE: The example package.json includes scripts to automate deployment. You can try running `$ npm run deploy:frontend` 68 | 69 | `$ npm run build:prod` 70 | 71 | `$ firebase deploy --only hosting` 72 | -------------------------------------------------------------------------------- /test/helpers/builder/action-registry.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { isFunction, map } = require('lodash/fp'); 4 | const modelsUtils = require('../models'); 5 | 6 | const stringifyDates = object => 7 | JSON.parse( 8 | JSON.stringify(object, (key, value) => { 9 | if (this[key] instanceof Date) { 10 | return this[key].toUTCString(); 11 | } 12 | return value; 13 | }) 14 | ); 15 | 16 | const formatFixtures = map(stringifyDates); 17 | 18 | module.exports = { 19 | contentType: { 20 | create: contentType => { 21 | let createdModel; 22 | 23 | return { 24 | async build(ctx) { 25 | createdModel = await modelsUtils.createContentType(contentType); 26 | ctx.addModel(createdModel); 27 | }, 28 | cleanup: () => modelsUtils.deleteContentType(createdModel.modelName), 29 | }; 30 | }, 31 | 32 | createBatch: contentTypes => { 33 | let createdModels = []; 34 | 35 | return { 36 | async build(ctx) { 37 | createdModels = await modelsUtils.createContentTypes(contentTypes); 38 | createdModels.forEach(ctx.addModel); 39 | }, 40 | async cleanup() { 41 | for (const model of createdModels) { 42 | await modelsUtils.deleteContentType(model.modelName); 43 | } 44 | }, 45 | }; 46 | }, 47 | 48 | createMany: contentTypes => { 49 | const createdModels = []; 50 | 51 | return { 52 | async build(ctx) { 53 | for (const contentType of contentTypes) { 54 | const model = await modelsUtils.createContentType(contentType); 55 | 56 | createdModels.push(model); 57 | ctx.addModel(model); 58 | } 59 | }, 60 | async cleanup() { 61 | for (const model of createdModels) { 62 | await modelsUtils.deleteContentType(model.modelName); 63 | } 64 | }, 65 | }; 66 | }, 67 | }, 68 | component: { 69 | create: component => { 70 | let createdModel; 71 | 72 | return { 73 | async build(ctx) { 74 | createdModel = await modelsUtils.createComponent(component); 75 | ctx.addModel(createdModel); 76 | }, 77 | cleanup: () => modelsUtils.deleteComponent(createdModel.uid), 78 | }; 79 | }, 80 | }, 81 | fixtures: { 82 | create(modelName, entries, getFixtures) { 83 | let createdEntries = []; 84 | 85 | return { 86 | async build(ctx) { 87 | createdEntries = formatFixtures( 88 | await modelsUtils.createFixturesFor( 89 | modelName, 90 | isFunction(entries) ? entries(getFixtures()) : entries 91 | ) 92 | ); 93 | 94 | ctx.addFixtures(modelName, createdEntries); 95 | }, 96 | cleanup: () => modelsUtils.deleteFixturesFor(modelName, createdEntries), 97 | }; 98 | }, 99 | }, 100 | }; 101 | -------------------------------------------------------------------------------- /src/db/morph-reference.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { DeepReference } from './deep-reference'; 3 | import type { Collection } from './collection'; 4 | import { MorphReferenceShape, Reference, SetOpts } from './reference'; 5 | import { NormalReference } from './normal-reference'; 6 | import { VirtualReference } from './virtual-reference'; 7 | 8 | 9 | /** 10 | * Acts as a wrapper around a `NormalReference` or a `DeepReference` 11 | * with additional field/filter information for polymorphic references. 12 | */ 13 | export class MorphReference extends Reference { 14 | 15 | constructor(readonly ref: NormalReference | DeepReference | VirtualReference, readonly filter: string | null) { 16 | super(); 17 | } 18 | 19 | get parent(): Collection { 20 | return this.ref.parent; 21 | } 22 | 23 | get id(): string { 24 | return this.ref.id; 25 | } 26 | 27 | get path() { 28 | return this.ref.path; 29 | } 30 | 31 | get firestore() { 32 | return this.ref.firestore; 33 | } 34 | 35 | delete(opts?: SetOpts) { 36 | return this.ref.delete(); 37 | }; 38 | 39 | create(data: T, opts?: SetOpts): Promise 40 | create(data: Partial, opts?: SetOpts): Promise> 41 | create(data: T | Partial, opts?: SetOpts): Promise> { 42 | return this.ref.create(data, opts); 43 | }; 44 | 45 | update(data: T, opts?: SetOpts): Promise 46 | update(data: Partial, opts?: SetOpts): Promise> 47 | update(data: Partial, opts?: SetOpts) { 48 | return this.ref.update(data, opts); 49 | } 50 | 51 | /** 52 | * Performs a `create()`, `update()`, or `delete()` operation without any coercion or lifecycles. 53 | * @private 54 | * @deprecated For internal connector use only 55 | */ 56 | writeInternal(data: Partial | undefined, editMode: 'create' | 'update') { 57 | return this.ref.writeInternal(data, editMode); 58 | } 59 | 60 | get() { 61 | return this.ref.get(); 62 | } 63 | 64 | isEqual(other: any) { 65 | return (this === other) || 66 | (other instanceof MorphReference 67 | && this.ref.isEqual(other.ref as any) 68 | && (this.filter === other.filter)); 69 | } 70 | 71 | /** 72 | * Allow serialising to JSON. 73 | */ 74 | toJSON() { 75 | // This Strapi behaviour isn't really documented 76 | const { model } = this.ref.parent; 77 | return { 78 | ref: model.modelName, 79 | kind: model.globalId, 80 | source: model.plugin, 81 | refId: this.id, 82 | field: this.filter || undefined, 83 | }; 84 | } 85 | 86 | /** 87 | * Returns a value that can be serialised 88 | * to Firestore. 89 | */ 90 | toFirestoreValue(): MorphReferenceShape { 91 | const value: MorphReferenceShape = this.ref instanceof DeepReference 92 | ? { ...this.ref.toFirestoreValue(), filter: this.filter } 93 | : { ref: this.ref.toFirestoreValue(), filter: this.filter }; 94 | 95 | return value; 96 | } 97 | 98 | toString() { 99 | return this.id; 100 | } 101 | 102 | 103 | } -------------------------------------------------------------------------------- /src/db/normal-reference.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import type { DocumentReference, DocumentSnapshot } from '@google-cloud/firestore'; 3 | import { Reference, SetOpts, Snapshot } from './reference'; 4 | import type { NormalCollection } from './normal-collection'; 5 | import { runUpdateLifecycle } from '../utils/lifecycle'; 6 | 7 | /** 8 | * Acts as a wrapper around a native `DocumentReference`, 9 | */ 10 | export class NormalReference extends Reference { 11 | 12 | constructor(readonly ref: DocumentReference, readonly parent: NormalCollection) { 13 | super(); 14 | } 15 | 16 | get id(): string { 17 | return this.ref.id; 18 | } 19 | 20 | get path() { 21 | return this.ref.path; 22 | } 23 | 24 | get firestore() { 25 | return this.ref.firestore; 26 | } 27 | 28 | async delete(opts?: SetOpts) { 29 | await runUpdateLifecycle({ 30 | editMode: 'update', 31 | ref: this, 32 | data: undefined, 33 | opts, 34 | timestamp: new Date(), 35 | }); 36 | } 37 | 38 | async create(data: T, opts?: SetOpts): Promise 39 | async create(data: Partial, opts?: SetOpts): Promise> 40 | async create(data: T | Partial, opts?: SetOpts) { 41 | return await runUpdateLifecycle({ 42 | editMode: 'create', 43 | ref: this, 44 | data, 45 | opts, 46 | timestamp: new Date(), 47 | }); 48 | } 49 | 50 | update(data: T, opts?: SetOpts): Promise 51 | update(data: Partial, opts?: SetOpts): Promise> 52 | async update(data: T | Partial, opts?: SetOpts) { 53 | return await runUpdateLifecycle({ 54 | editMode: 'update', 55 | ref: this, 56 | data, 57 | opts, 58 | timestamp: new Date(), 59 | }); 60 | } 61 | 62 | 63 | /** 64 | * Performs a `create()`, `update()`, or `delete()` operation without any coercion or lifecycles. 65 | * @private 66 | * @deprecated For internal connector use only 67 | */ 68 | async writeInternal(data: Partial | undefined, editMode: 'create' | 'update') { 69 | if (data) { 70 | if (editMode === 'create') { 71 | await this.ref.create(data as T); 72 | } else { 73 | // Firestore does not run the converter on update operations 74 | const out = this.parent.converter.toFirestore(data as T); 75 | await this.ref.update(out as any); 76 | } 77 | } else { 78 | await this.ref.delete(); 79 | } 80 | } 81 | 82 | async get() { 83 | return makeNormalSnap(this, await this.ref.get()); 84 | } 85 | 86 | isEqual(other: any) { 87 | return (this === other) || ((other instanceof NormalReference) 88 | && this.ref.isEqual(other.ref)); 89 | } 90 | 91 | /** 92 | * Allow serialising to JSON. 93 | */ 94 | toJSON() { 95 | return this.id; 96 | } 97 | 98 | /** 99 | * Returns a value that can be serialised 100 | * to Firestore. 101 | */ 102 | toFirestoreValue(): DocumentReference { 103 | return this.ref; 104 | } 105 | 106 | toString() { 107 | return this.path; 108 | } 109 | } 110 | 111 | export function makeNormalSnap(ref: NormalReference, snap: DocumentSnapshot): Snapshot { 112 | const data = snap.data(); 113 | return { 114 | ref, 115 | data: () => data, 116 | id: snap.id, 117 | exists: snap.exists, 118 | }; 119 | } 120 | -------------------------------------------------------------------------------- /src/db/reference.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import type { DocumentReference, Firestore } from '@google-cloud/firestore'; 3 | import type { Collection } from './collection'; 4 | 5 | 6 | /** 7 | * Deep equality algorithm based on `_.isEqual()` with special handling 8 | * of objects that have their own `isEqual()` method, such as `Reference`. 9 | */ 10 | export function isEqualHandlingRef(a: any, b: any): boolean { 11 | return _.isEqualWith(a, b, (aValue, bValue) => { 12 | if (aValue && (typeof aValue === 'object')) { 13 | const { isEqual } = aValue; 14 | if (typeof isEqual === 'function') { 15 | return (isEqual as Function).bind(aValue)(bValue); 16 | } 17 | } 18 | return undefined; 19 | }); 20 | } 21 | 22 | /** 23 | * The shape of references as stored in Firestore. 24 | */ 25 | export type ReferenceShape = 26 | DocumentReference | 27 | FlatReferenceShape | 28 | MorphReferenceShape; 29 | 30 | export interface FlatReferenceShape { 31 | ref: DocumentReference<{ [id: string]: T }> 32 | id: string 33 | } 34 | 35 | export type MorphReferenceShape = NormalMorphReferenceShape | FlatMorphReferenceShape; 36 | 37 | export interface NormalMorphReferenceShape { 38 | ref: DocumentReference 39 | filter: string | null 40 | } 41 | 42 | export interface FlatMorphReferenceShape extends FlatReferenceShape { 43 | filter: string | null 44 | } 45 | 46 | 47 | 48 | 49 | export interface Snapshot { 50 | data(): T | undefined 51 | ref: Reference 52 | id: string 53 | exists: boolean 54 | } 55 | 56 | 57 | export interface SetOpts { 58 | /** 59 | * Indicates whether relation links should be updated on 60 | * other related documents. This can extra reads, queries 61 | * and writes to multiple documents. 62 | * 63 | * If not updated, reference links will get into invalid states, 64 | * so it should be done unless you know what you're doing. 65 | * 66 | * Defaults to `true`. 67 | */ 68 | updateRelations?: boolean 69 | 70 | /** 71 | * Whether or not the `onChange(...)` hook will be run. If not provided, 72 | * it defaults to the value of `updateRelations` (which defaults to `true`). 73 | * 74 | * This default behaviour means that it will run for all explicit changes, but not 75 | * for implicit changes internally caused to related documents when relations are updated. 76 | */ 77 | runOnChangeHook?: boolean; 78 | } 79 | 80 | /** 81 | * Common interface for normal, flattened, and polymorphic references. 82 | * References perform coercion on input data according to the model 83 | * schema that they belong to. 84 | */ 85 | export abstract class Reference { 86 | 87 | abstract readonly parent: Collection; 88 | abstract readonly id: string; 89 | abstract readonly path: string; 90 | 91 | abstract readonly firestore: Firestore; 92 | 93 | abstract delete(opts?: SetOpts): Promise; 94 | 95 | /** 96 | * @returns The coerced data 97 | */ 98 | abstract create(data: T, opts?: SetOpts): Promise; 99 | abstract create(data: Partial, opts?: SetOpts): Promise>; 100 | 101 | /** 102 | * @returns The coerced data 103 | */ 104 | abstract update(data: T, opts?: SetOpts): Promise; 105 | abstract update(data: Partial, opts?: SetOpts): Promise>; 106 | 107 | abstract get(): Promise>; 108 | 109 | abstract isEqual(other: any): boolean; 110 | 111 | /** 112 | * Allow serialising to JSON. 113 | */ 114 | abstract toJSON(): any; 115 | 116 | /** 117 | * Returns a value that can be serialised to Firestore. 118 | */ 119 | abstract toFirestoreValue(): ReferenceShape; 120 | 121 | toString(): string { 122 | return this.path; 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/populate.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { getComponentModel } from './utils/components'; 3 | import type { Transaction } from './db/transaction'; 4 | import type { Reference, Snapshot } from './db/reference'; 5 | import { FirestoreConnectorModel } from './model'; 6 | import { StatusError } from './utils/status-error'; 7 | 8 | /** 9 | * Defines a type where all Reference members of T are populated as their referred types (including arrays of references). 10 | * Note: This typing is not accurate for components, where all references would be populated, but this type does not populate 11 | * any keys inside components. 12 | */ 13 | export type Populated = { 14 | [Key in keyof T]: T[Key] extends Reference ? R : T[Key] extends Reference[] ? R[] : T[Key]; 15 | } 16 | 17 | /** 18 | * Picks the keys of T whose values are References or arrays of references. 19 | */ 20 | export type PickReferenceKeys = Extract<{ [Key in keyof T]-?: T[Key] extends Reference ? Key : T[Key] extends Reference[] ? Key : never; }[keyof T], string> 21 | 22 | /** 23 | * Defines a type where all Reference members amongst those with the given keys are populated as their referred types. 24 | * Note: This typing is not accurate for components, where all references would be populated, but this type does not populate 25 | * any keys inside components. 26 | */ 27 | export type PopulatedByKeys> = Omit & Populated> 28 | 29 | /** 30 | * Populates all the requested relational field on the given documents. 31 | */ 32 | export async function populateSnapshots>(snaps: Snapshot[], populate: K[], transaction: Transaction): Promise[]> { 33 | return await Promise.all( 34 | snaps.map(async snap => { 35 | const data = snap.data(); 36 | if (!data) { 37 | throw new StatusError('entry.notFound', 404); 38 | } 39 | return await populateDoc(snap.ref.parent.model, snap.ref, data, populate, transaction); 40 | }) 41 | ); 42 | } 43 | 44 | /** 45 | * Populates all the requested relational field on the given document. 46 | * All references in components are populated by default. 47 | */ 48 | export async function populateDoc>(model: FirestoreConnectorModel, ref: Reference, data: T, populateKeys: K[], transaction: Transaction): Promise> { 49 | const promises: Promise[] = []; 50 | 51 | // Shallow copy the object 52 | const newData = Object.assign({}, data); 53 | 54 | // Populate own relations 55 | for (const key of populateKeys) { 56 | const relation = model.relations.find(r => r.alias === key); 57 | if (relation) { 58 | promises.push(relation.populateRelated(ref, newData, transaction)); 59 | } 60 | } 61 | 62 | // Recursively populate components 63 | promises.push( 64 | ...model.componentKeys.map(async componentKey => { 65 | const component: any = _.get(newData, componentKey); 66 | if (component) { 67 | if (Array.isArray(component)) { 68 | const values = await Promise.all( 69 | component.map(c => { 70 | const componentModel = getComponentModel(model, componentKey, c); 71 | return populateDoc(componentModel, ref, c, componentModel.defaultPopulate, transaction); 72 | }) 73 | ); 74 | _.set(newData, componentKey, values); 75 | } else { 76 | const componentModel = getComponentModel(model, componentKey, component); 77 | const value = await populateDoc(componentModel, ref, component, componentModel.defaultPopulate, transaction); 78 | _.set(newData, componentKey, value); 79 | } 80 | } 81 | }) 82 | ); 83 | 84 | await Promise.all(promises); 85 | 86 | // TODO: Better type safety 87 | return newData as any; 88 | } 89 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: 8 | - master 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | flattening: [flatten_all, flatten_none, flatten_mixed_src, flatten_mixed_target] 17 | steps: 18 | - name: Checkout project 19 | uses: actions/checkout@v2 20 | - name: Setup Node.js 21 | uses: actions/setup-node@v2 22 | with: 23 | node-version: '12' 24 | - name: Cache NPM packages 25 | uses: actions/cache@v2 26 | with: 27 | path: ~/.npm 28 | key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} 29 | restore-keys: | 30 | ${{ runner.os }}-node- 31 | 32 | # FIXME: 33 | # Lockfile version 2 with npm@7 broke compatibility with installation 34 | - name: Upgrade npm 35 | run: npm i -g npm 36 | 37 | # Install the package 38 | - name: Install root package 39 | run: npm ci 40 | 41 | # Setup and install the test environment 42 | - name: Setup Java JDK 43 | uses: actions/setup-java@v1 44 | with: 45 | java-version: 1.8 46 | - name: Install test package 47 | run: npm ci --prefix test 48 | - name: Cache emulators 49 | uses: actions/cache@v2.0.0 50 | with: 51 | path: ~/.cache/firebase/emulators 52 | key: ${{ runner.os }} 53 | - name: Install emulators 54 | run: test/node_modules/.bin/firebase setup:emulators:firestore 55 | 56 | # Run tests 57 | # Set output coverage JSON regardless if test failed or not 58 | - name: Run tests 59 | run: npm test 60 | env: 61 | FLATTENING: ${{ matrix.flattening }} 62 | - name: Upload coverage 63 | uses: actions/upload-artifact@v2 64 | if: ${{ always() }} 65 | with: 66 | name: coverage_${{ matrix.flattening }} 67 | path: | 68 | coverage/coverage.json 69 | coverage/results.json 70 | 71 | results: 72 | runs-on: ubuntu-latest 73 | if: ${{ always() }} 74 | needs: test 75 | steps: 76 | - name: Checkout project 77 | uses: actions/checkout@v2 78 | - name: Setup Node.js 79 | uses: actions/setup-node@v2 80 | - name: Cache NPM packages 81 | uses: actions/cache@v2 82 | with: 83 | path: ~/.npm 84 | key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} 85 | restore-keys: | 86 | ${{ runner.os }}-node- 87 | 88 | # FIXME: 89 | # Lockfile version 2 with npm@7 broke compatibility with installation 90 | - name: Upgrade npm 91 | run: sudo npm i -g npm 92 | 93 | # Install the package 94 | - name: Install root package 95 | run: npm ci 96 | 97 | # Download individual coverage reports 98 | - uses: actions/download-artifact@v2 99 | with: 100 | path: coverage 101 | 102 | # Install the test package 103 | - name: Install test package 104 | run: npm ci --prefix test 105 | 106 | # Report pass/fail results 107 | - name: Report results 108 | uses: actions/github-script@v3 109 | with: 110 | result-encoding: string 111 | script: | 112 | const report = require(`${process.env.GITHUB_WORKSPACE}/test/report.js`); 113 | return await report({ github, context, core, io }); 114 | 115 | # Combine coverage 116 | - name: Combine coverage 117 | run: npm run coverage-combine -s --prefix test 118 | 119 | # Upload report 120 | - name: Codecov 121 | uses: codecov/codecov-action@v1 122 | with: 123 | directory: coverage/combined 124 | fail_ci_if_error: true 125 | -------------------------------------------------------------------------------- /src/utils/lifecycle.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { CoerceOpts, coerceToModel } from '../coerce/coerce-to-model'; 3 | import { DeepReference } from '../db/deep-reference'; 4 | import { MorphReference } from '../db/morph-reference'; 5 | import { NormalReference } from '../db/normal-reference'; 6 | import type { Reference, SetOpts } from '../db/reference'; 7 | import type { Transaction } from '../db/transaction'; 8 | import type { ReadWriteTransaction } from '../db/readwrite-transaction'; 9 | import type { ReadOnlyTransaction } from '../db/readonly-transaction'; 10 | import { VirtualReference } from '../db/virtual-reference'; 11 | import { relationsUpdate, shouldUpdateRelations } from '../relations'; 12 | 13 | export interface LifecycleArgs extends Required { 14 | ref: Reference 15 | data: T | Partial | undefined 16 | transaction?: ReadWriteTransaction | ReadOnlyTransaction 17 | opts: SetOpts | undefined 18 | timestamp: Date 19 | } 20 | 21 | /** 22 | * Runs the full lifecycle on the given reference including coercion and updating relations. 23 | * @returns The coerced data 24 | */ 25 | export async function runUpdateLifecycle({ ref, data, editMode, opts, timestamp, transaction }: LifecycleArgs): Promise | undefined> { 26 | const db = ref.parent; 27 | const newData = data ? coerceToModel(db.model, ref.id, data, null, { editMode, timestamp }) : undefined; 28 | 29 | const updateRelations = shouldUpdateRelations(opts); 30 | const runOnChangeHook = (typeof opts?.runOnChangeHook === 'boolean') ? opts.runOnChangeHook: updateRelations; 31 | 32 | if (updateRelations) { 33 | 34 | const runUpdateWithRelations = async (trans: Transaction) => { 35 | const prevData = editMode === 'update' 36 | ? await trans.getAtomic(ref).then(snap => snap.data()) 37 | : undefined; 38 | 39 | if (runOnChangeHook) { 40 | // Run the change hook before relations are updated 41 | // so any changes made by the hook will be included 42 | const onSuccess = await ref.parent.model.options.onChange(prevData, newData, trans, ref); 43 | if (onSuccess) { 44 | trans.addSuccessHook(() => onSuccess(newData, ref)); 45 | } 46 | } 47 | 48 | await relationsUpdate(db.model, ref, prevData, newData, editMode, trans); 49 | (trans as (ReadWriteTransaction | ReadOnlyTransaction)).mergeWriteInternal(ref, newData, editMode); 50 | }; 51 | 52 | if (transaction) { 53 | await runUpdateWithRelations(transaction); 54 | } else { 55 | await db.model.runTransaction(runUpdateWithRelations); 56 | } 57 | } else { 58 | if (runOnChangeHook) { 59 | // We always need a transaction when we need to run the change hook 60 | const runUpdateWithoutRelations = async (trans: Transaction) => { 61 | const prevData = editMode === 'update' 62 | ? await trans.getAtomic(ref).then(snap => snap.data()) 63 | : undefined; 64 | 65 | // Run the change hook before relations are updated 66 | // so any changes made by the hook will be included 67 | const onSuccess = await ref.parent.model.options.onChange(prevData, newData, trans, ref); 68 | if (onSuccess) { 69 | trans.addSuccessHook(() => onSuccess(newData, ref)); 70 | } 71 | 72 | (trans as (ReadWriteTransaction | ReadOnlyTransaction)).mergeWriteInternal(ref, newData, editMode); 73 | }; 74 | 75 | if (transaction) { 76 | await runUpdateWithoutRelations(transaction); 77 | } else { 78 | await db.model.runTransaction(runUpdateWithoutRelations); 79 | } 80 | } else { 81 | if (transaction) { 82 | transaction.mergeWriteInternal(ref, newData, editMode); 83 | } else { 84 | if ((ref instanceof NormalReference) 85 | || (ref instanceof DeepReference) 86 | || (ref instanceof MorphReference) 87 | || (ref instanceof VirtualReference)) { 88 | await ref.writeInternal(newData, editMode); 89 | } else { 90 | throw new Error(`Unknown type of reference: ${ref}`); 91 | } 92 | } 93 | } 94 | } 95 | 96 | return newData; 97 | } 98 | -------------------------------------------------------------------------------- /src/utils/transaction-runner.ts: -------------------------------------------------------------------------------- 1 | import type { Firestore } from '@google-cloud/firestore'; 2 | import type { ConnectorOptions, ModelOptions } from '../types'; 3 | import type { FirestoreConnectorModel } from '../model'; 4 | import { ReadWriteTransaction } from '../db/readwrite-transaction'; 5 | import PQueue from 'p-queue'; 6 | import { ReadOnlyTransaction } from '../db/readonly-transaction'; 7 | 8 | export type TransactionRunner = FirestoreConnectorModel['runTransaction']; 9 | 10 | /** 11 | * Makes a function that runs a transaction. 12 | * If the connector is using an emulator then the return runner queues transactions 13 | * with a maximum concurrency limit. 14 | */ 15 | export function makeTransactionRunner(firestore: Firestore, options: Required>, connectorOptions: Required): TransactionRunner { 16 | const { useEmulator, logTransactionStats } = connectorOptions; 17 | const isVirtual = options.virtualDataSource != null; 18 | 19 | const normalRunner: TransactionRunner = async (fn, opts) => { 20 | const isReadOnly = isVirtual || (opts && opts.readOnly); 21 | let hooks: (() => (void | PromiseLike))[]; 22 | let result: Awaited>; 23 | if (isReadOnly) { 24 | // Always use read-only transactions for virtual collections 25 | // The only scenario where a virtual collection may want to perform a Firestore write is if it has 26 | // a dominant relation to a non-virtual collection. However, because of the (potentially) transient nature of 27 | // references to virtual collections, dominant relations to a virtual collection are not supported. 28 | // Don't log stats for virtual collections 29 | const trans = new ReadOnlyTransaction(firestore, logTransactionStats && !isVirtual); 30 | result = await fn(trans); 31 | hooks = trans.successHooks; 32 | await trans.commit(); 33 | } else { 34 | let attempt = 0; 35 | const r = await firestore.runTransaction(async (trans) => { 36 | if ((attempt > 0) && useEmulator) { 37 | // Random back-off for contested transactions only when running on the emulator 38 | // The production server has deadlock avoidance but the emulator currently doesn't 39 | // See https://github.com/firebase/firebase-tools/issues/1629#issuecomment-525464351 40 | // See https://github.com/firebase/firebase-tools/issues/2452 41 | const ms = Math.random() * 5000; 42 | strapi.log.warn(`There is contention on a document and the Firestore emulator is getting deadlocked. Waiting ${ms.toFixed(0)}ms.`); 43 | await new Promise(resolve => setTimeout(resolve, ms)); 44 | } 45 | 46 | const wrapper = new ReadWriteTransaction(firestore, trans, logTransactionStats, ++attempt); 47 | const result = await fn(wrapper); 48 | await wrapper.commit(); 49 | return { result, hooks: wrapper.successHooks }; 50 | }, { maxAttempts: opts?.maxAttempts }); 51 | 52 | result = r.result; 53 | hooks = r.hooks; 54 | } 55 | 56 | // Run the hooks 57 | if (hooks.length) { 58 | await Promise.all(hooks.map(async hook => { 59 | try { 60 | await hook(); 61 | } catch (err) { 62 | strapi.log.error(`Error running transaction success hook: ${err.message}`, { err }); 63 | } 64 | })); 65 | } 66 | 67 | return result; 68 | }; 69 | 70 | // Virtual option overrides flatten option 71 | if (options.flatten && !isVirtual) { 72 | const queue = new PQueue({ concurrency: 1 }); 73 | return async (fn, opts) => { 74 | if (opts && opts.readOnly) { 75 | // Read-only transactions can succeed concurrently because they don't lock any documents 76 | // and will not contend with any other transactions 77 | return normalRunner(fn, opts); 78 | } else { 79 | // When using flattened collections, only one read-write transaction can be executed at a time 80 | // So we queue them up rather than allowing them to contend 81 | // Contention which would introduce 30-second timeout delays on the emulator, and cause unnecessary 82 | // read and write operations on the production server 83 | return await queue.add(() => normalRunner(fn)); 84 | } 85 | }; 86 | } else { 87 | return normalRunner; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/utils/read-repository.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { DocumentReference, DocumentSnapshot, FieldPath, Query, QuerySnapshot } from '@google-cloud/firestore'; 3 | 4 | 5 | export interface ReadRepositoryHandler { 6 | getAll(refs: DocumentReference[], fieldMasks?: (string | FieldPath)[]): Promise[]> 7 | getQuery(query: Query): Promise> 8 | } 9 | 10 | export interface RefAndMask { 11 | ref: DocumentReference 12 | fieldMasks?: (string | FieldPath)[] 13 | } 14 | 15 | interface GroupedReadOps { 16 | fieldMasks: (string | FieldPath)[] | undefined 17 | ops: ReadOp[] 18 | } 19 | 20 | interface ReadOp { 21 | ref: DocumentReference 22 | resolve: ((r: DocumentSnapshot) => void) 23 | reject: ((reason: any) => void) 24 | } 25 | 26 | /** 27 | * Utility class for transactions that acts as a caching proxy for read operations. 28 | */ 29 | export class ReadRepository { 30 | 31 | private readCounter = 0; 32 | private readonly cache = new Map>>(); 33 | 34 | constructor( 35 | private readonly handler: ReadRepositoryHandler, 36 | private readonly delegate?: ReadRepository, 37 | ) { } 38 | 39 | get size(): number { 40 | return this.cache.size; 41 | } 42 | 43 | get readCount(): number { 44 | return this.readCounter; 45 | } 46 | 47 | /** 48 | * Gets the given documents, first from this repository's cache, then 49 | * from the delegate repository's cache, or finally from the database. 50 | * Documents fetched from the database are stored in the cache. 51 | * 52 | * If field masks are provided, then results can be fulfilled from non-masked 53 | * cache entries, but masked requests from the database will not be stored in the cache. 54 | */ 55 | async getAll(items: RefAndMask[]): Promise[]> { 56 | 57 | const toRead: GroupedReadOps[] = []; 58 | const results: Promise[] = new Array(items.length); 59 | for (let i = 0; i < items.length; i++) { 60 | const { ref, fieldMasks } = items[i]; 61 | let result = this.cache.get(ref.path) 62 | || (this.delegate && this.delegate.cache.get(ref.path)); 63 | 64 | if (!result) { 65 | // Create a new read operation grouped by field masks 66 | result = new Promise((resolve, reject) => { 67 | const op: ReadOp = { ref, resolve, reject }; 68 | for (const entry of toRead) { 69 | if (isFieldPathsEqual(entry.fieldMasks, fieldMasks)) { 70 | entry.ops.push(op); 71 | return; 72 | } 73 | } 74 | toRead.push({ 75 | fieldMasks, 76 | ops: [op], 77 | }); 78 | }); 79 | 80 | // Only cache the new read operation if there is no field mask 81 | if (!fieldMasks) { 82 | this.cache.set(ref.path, result); 83 | } 84 | } 85 | 86 | results[i] = result; 87 | } 88 | 89 | // Fetch and resolve all of the newly required read operations 90 | await Promise.all(toRead.map(ops => fetchGroupedReadOp(ops, this.handler))); 91 | this.readCounter += toRead.length; 92 | 93 | return Promise.all(results); 94 | } 95 | 96 | async getQuery(query: Query): Promise> { 97 | const result = await this.handler.getQuery(query); 98 | for (const d of result.docs) { 99 | const { path } = d.ref; 100 | if (!this.cache.has(path)) { 101 | this.cache.set(path, Promise.resolve(d)); 102 | } 103 | } 104 | 105 | this.readCounter += (result.docs.length || 1); 106 | 107 | return result; 108 | } 109 | } 110 | 111 | async function fetchGroupedReadOp({ fieldMasks, ops }: GroupedReadOps, handler: ReadRepositoryHandler) { 112 | try { 113 | const snaps = await handler.getAll(ops.map(({ ref }) => ref), fieldMasks); 114 | let i = ops.length; 115 | while (i--) { 116 | ops[i].resolve(snaps[i]); 117 | } 118 | } catch (err) { 119 | for (const { reject } of ops) { 120 | reject(err); 121 | } 122 | } 123 | } 124 | 125 | function isFieldPathsEqual(a: (string | FieldPath)[] | undefined, b: (string | FieldPath)[] | undefined) { 126 | return _.isEqualWith(a, b, (aVal, bVal) => { 127 | if (aVal instanceof FieldPath) { 128 | return aVal.isEqual(bVal); 129 | } 130 | return undefined; 131 | }); 132 | } 133 | -------------------------------------------------------------------------------- /src/db/field-operation.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { FieldValue } from '@google-cloud/firestore'; 3 | import { isEqualHandlingRef } from './reference'; 4 | 5 | /** 6 | * Acts as a wrapper for Firestore's `FieldValue` but allows 7 | * manual implementation (where Firestore's) API is not public. 8 | */ 9 | export abstract class FieldOperation { 10 | 11 | static delete(): FieldOperation { 12 | return new DeleteFieldOperation(); 13 | } 14 | 15 | static increment(n: number): FieldOperation { 16 | return new IncrementFieldOperation(n); 17 | } 18 | 19 | static arrayRemove(...items: any[]): FieldOperation { 20 | return new ArrayRemoveFieldOperation(items); 21 | } 22 | 23 | static arrayUnion(...items: any[]): FieldOperation { 24 | return new ArrayUnionFieldOperation(items); 25 | } 26 | 27 | /** 28 | * Sets the given value and the given path or applies the 29 | * transform if the value is a transform. 30 | */ 31 | static apply(data: any, fieldPath: string, valueOrOperation: any): void { 32 | const value = _.get(data, fieldPath); 33 | const result = valueOrOperation instanceof FieldOperation 34 | ? valueOrOperation.transform(value) 35 | : valueOrOperation; 36 | if (result === undefined) { 37 | _.unset(data, fieldPath); 38 | } else { 39 | _.set(data, fieldPath, result); 40 | } 41 | } 42 | 43 | /** 44 | * Converts the operation to its Firestore-native 45 | * `FieldValue` equivalent. 46 | */ 47 | abstract toFirestoreValue(): FieldValue; 48 | 49 | /** 50 | * Performs the operation on the given data. 51 | */ 52 | abstract transform(value: any): any; 53 | 54 | 55 | /** 56 | * Returns another instance of this operation which 57 | * has any values coerced using the given function. 58 | * @param coerceFn The function that coerces each value 59 | */ 60 | abstract coerceWith(coerceFn: (value: any) => any): FieldOperation; 61 | 62 | 63 | /** 64 | * @deprecated Unsupported operation 65 | */ 66 | toJSON(): never { 67 | throw new Error('Instance of FieldOperation class cannot be serialised to JSON') 68 | } 69 | } 70 | 71 | 72 | class DeleteFieldOperation extends FieldOperation { 73 | 74 | constructor() { 75 | super() 76 | } 77 | 78 | toFirestoreValue(): FieldValue { 79 | return FieldValue.delete(); 80 | } 81 | 82 | transform(): undefined { 83 | return undefined; 84 | } 85 | 86 | coerceWith() { 87 | return this; 88 | } 89 | } 90 | 91 | class IncrementFieldOperation extends FieldOperation { 92 | 93 | constructor(readonly n: number) { 94 | super() 95 | } 96 | 97 | toFirestoreValue(): FieldValue { 98 | return FieldValue.increment(this.n); 99 | } 100 | 101 | transform(value: any): number { 102 | const number = (typeof value === 'number') ? value : 0; 103 | return number + this.n; 104 | } 105 | 106 | coerceWith() { 107 | return this; 108 | } 109 | } 110 | 111 | class ArrayUnionFieldOperation extends FieldOperation { 112 | 113 | constructor(readonly elements: any[]) { 114 | super() 115 | } 116 | 117 | toFirestoreValue(): FieldValue { 118 | return FieldValue.arrayUnion(...this.elements); 119 | } 120 | 121 | transform(value: any): any[] { 122 | // Add any instances that aren't already existing 123 | // If the value was not an array then it is overwritten with 124 | // an empty array 125 | const arr = (Array.isArray(value) ? value : []); 126 | const toAdd = this.elements 127 | .filter(e => !arr.some(value => isEqualHandlingRef(value, e))); 128 | return arr.concat(toAdd); 129 | } 130 | 131 | coerceWith(coerceFn: (value: any) => any) { 132 | return new ArrayUnionFieldOperation(this.elements.map(coerceFn)); 133 | } 134 | } 135 | 136 | 137 | class ArrayRemoveFieldOperation extends FieldOperation { 138 | 139 | constructor(readonly elements: any[]) { 140 | super() 141 | } 142 | 143 | toFirestoreValue(): FieldValue { 144 | return FieldValue.arrayRemove(...this.elements); 145 | } 146 | 147 | transform(value: any): any[] { 148 | // Remove all instances from the array 149 | // If the value was not an array then it is overwritten with 150 | // an empty array 151 | return (Array.isArray(value) ? value : []) 152 | .filter(value => !this.elements.some(e => isEqualHandlingRef(value, e))); 153 | } 154 | 155 | coerceWith(coerceFn: (value: any) => any) { 156 | return new ArrayRemoveFieldOperation(this.elements.map(coerceFn)); 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/db/virtual-reference.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { Reference, SetOpts, Snapshot } from './reference'; 3 | import { runUpdateLifecycle } from '../utils/lifecycle'; 4 | import { VirtualCollection } from './virtual-collection'; 5 | import { DocumentReference } from '@google-cloud/firestore'; 6 | import { StatusError } from '../utils/status-error'; 7 | import { FieldOperation } from './field-operation'; 8 | 9 | /** 10 | * References an item in a virtual collection. 11 | */ 12 | export class VirtualReference extends Reference { 13 | 14 | 15 | constructor(readonly id: string, readonly parent: VirtualCollection) { 16 | super(); 17 | if (!id) { 18 | throw new Error('Document ID must not be empty'); 19 | } 20 | } 21 | 22 | get path() { 23 | return `${this.parent.path}/${this.id}`; 24 | } 25 | 26 | 27 | get firestore() { 28 | return this.parent.model.firestore; 29 | } 30 | 31 | async delete(opts?: SetOpts) { 32 | await runUpdateLifecycle({ 33 | editMode: 'update', 34 | ref: this, 35 | data: undefined, 36 | opts, 37 | timestamp: new Date(), 38 | }); 39 | }; 40 | 41 | create(data: T, opts?: SetOpts): Promise 42 | create(data: Partial, opts?: SetOpts): Promise> 43 | async create(data: T | Partial, opts?: SetOpts) { 44 | return await runUpdateLifecycle({ 45 | editMode: 'create', 46 | ref: this, 47 | data, 48 | opts, 49 | timestamp: new Date(), 50 | }); 51 | }; 52 | 53 | update(data: T, opts?: SetOpts): Promise 54 | update(data: Partial, opts?: SetOpts): Promise> 55 | async update(data: T | Partial, opts?: SetOpts) { 56 | return await runUpdateLifecycle({ 57 | editMode: 'update', 58 | ref: this, 59 | data, 60 | opts, 61 | timestamp: new Date(), 62 | }); 63 | } 64 | 65 | /** 66 | * Performs a `create()`, `update()`, or `delete()` operation without any coercion or lifecycles. 67 | * @private 68 | * @deprecated For internal connector use only 69 | */ 70 | async writeInternal(data: Partial | undefined, editMode: 'create' | 'update') { 71 | const virtualData = await this.parent.getData(); 72 | 73 | if (data === undefined) { 74 | delete virtualData[this.id]; 75 | } else { 76 | const existingData = virtualData[this.id]; 77 | if (editMode === 'create') { 78 | if (existingData !== undefined) { 79 | throw new StatusError(`Cannot create a new document that already exists (document: ${this.path})`, 400); 80 | } 81 | } else { 82 | if (existingData === undefined) { 83 | throw new StatusError(`Cannot update a document that does not exist (document: ${this.path})`, 400); 84 | } 85 | } 86 | 87 | // Don't coerce back to native Firestore values because we don't need to 88 | // The data has already been coerced to the model schema 89 | 90 | const newData = virtualData[this.id] = (existingData || {}) 91 | 92 | for (const key of Object.keys(data)) { 93 | // TODO: Manually handle FieldOperation instances deeper in the data 94 | FieldOperation.apply(newData, key, data[key]); 95 | } 96 | 97 | await this.parent.updateData(); 98 | } 99 | } 100 | 101 | 102 | async get(): Promise> { 103 | const virtualData = await this.parent.getData(); 104 | const data = virtualData[this.id]; 105 | const converted = data ? this.parent.converter.fromFirestore(data) : undefined; 106 | return makeVirtualSnap(this, converted); 107 | } 108 | 109 | isEqual(other: any) { 110 | return (this === other) || 111 | (other instanceof VirtualReference 112 | && this.id === other.id); 113 | } 114 | 115 | /** 116 | * Allow serialising to JSON. 117 | */ 118 | toJSON() { 119 | return this.id; 120 | } 121 | 122 | /** 123 | * Returns a value that can be serialised 124 | * to Firestore. 125 | */ 126 | toFirestoreValue(): DocumentReference { 127 | // If other collections reference this virtual collection, then it looks like a normal reference. 128 | return this.parent.model.firestore.collection(this.parent.path).doc(this.id) as DocumentReference; 129 | } 130 | 131 | toString() { 132 | return this.path; 133 | } 134 | } 135 | 136 | 137 | export function makeVirtualSnap(ref: VirtualReference, data: T | undefined): Snapshot { 138 | return { 139 | ref, 140 | data: () => data, 141 | id: ref.id, 142 | exists: data !== undefined, 143 | }; 144 | } 145 | -------------------------------------------------------------------------------- /src/db/virtual-collection.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { convertWhere } from '../utils/convert-where'; 3 | import { OrderByDirection, FieldPath, DocumentData } from '@google-cloud/firestore'; 4 | import type { Collection, QuerySnapshot } from './collection'; 5 | import type { Converter, DataSource, FirestoreFilter, StrapiOrFilter, StrapiWhereFilter } from '../types'; 6 | import type { FirestoreConnectorModel } from '../model'; 7 | import { coerceModelToFirestore } from '../coerce/coerce-to-firestore'; 8 | import { coerceToModel } from '../coerce/coerce-to-model'; 9 | import { VirtualReference } from './virtual-reference'; 10 | import { applyManualFilters, ManualFilter, OrderSpec } from '../utils/manual-filter'; 11 | 12 | 13 | export class VirtualCollection implements Collection { 14 | 15 | readonly model: FirestoreConnectorModel 16 | readonly converter: Required> 17 | 18 | private dataSource: DataSource; 19 | private data: Promise<{ [id: string]: T }> | undefined; 20 | 21 | private readonly manualFilters: ManualFilter[] = []; 22 | private readonly _orderBy: OrderSpec[] = []; 23 | private _limit?: number; 24 | private _offset?: number; 25 | 26 | 27 | 28 | constructor(model: FirestoreConnectorModel) 29 | constructor(other: VirtualCollection) 30 | constructor(modelOrOther: FirestoreConnectorModel | VirtualCollection) { 31 | if (modelOrOther instanceof VirtualCollection) { 32 | // Copy the values 33 | this.model = modelOrOther.model; 34 | this.converter = modelOrOther.converter; 35 | this.dataSource = modelOrOther.dataSource; 36 | this.data = modelOrOther.data; 37 | this.manualFilters = modelOrOther.manualFilters.slice(); 38 | this._orderBy = modelOrOther._orderBy.slice(); 39 | this._limit = modelOrOther._limit; 40 | this._offset = modelOrOther._offset; 41 | } else { 42 | this.model = modelOrOther; 43 | this.dataSource = modelOrOther.options.virtualDataSource!; 44 | const { 45 | toFirestore = (value) => value, 46 | fromFirestore = (value) => value, 47 | } = modelOrOther.options.converter; 48 | this.converter = { 49 | toFirestore: data => { 50 | const d = coerceModelToFirestore(modelOrOther, data as T); 51 | return toFirestore(d); 52 | }, 53 | fromFirestore: snap => { 54 | const d = fromFirestore(snap); 55 | return coerceToModel(modelOrOther, snap.id, d, null, {}); 56 | }, 57 | }; 58 | } 59 | } 60 | 61 | get path(): string { 62 | return this.model.collectionName; 63 | } 64 | 65 | autoId() { 66 | return this.model.firestore.collection(this.path).doc().id 67 | } 68 | 69 | doc(): VirtualReference; 70 | doc(id: string): VirtualReference; 71 | doc(id?: string) { 72 | return new VirtualReference(id?.toString() || this.autoId(), this); 73 | }; 74 | 75 | async getData(): Promise<{ [id: string]: T }> { 76 | if (!this.data) { 77 | this.data = Promise.resolve().then(() => this.dataSource.getData()); 78 | } 79 | return this.data; 80 | } 81 | 82 | /** 83 | * Notifies the data source when the data has been updated. 84 | */ 85 | async updateData() { 86 | // Data is modified in place on the original object instance 87 | if (this.dataSource.setData) { 88 | await this.dataSource.setData(await this.getData()); 89 | } 90 | } 91 | 92 | async get(): Promise> { 93 | return applyManualFilters({ 94 | model: this.model, 95 | data: await this.getData(), 96 | filters: this.manualFilters, 97 | orderBy: this._orderBy, 98 | limit: this._limit, 99 | offset: this._offset, 100 | }); 101 | } 102 | 103 | where(clause: StrapiWhereFilter | StrapiOrFilter | FirestoreFilter): VirtualCollection { 104 | const filter = convertWhere(this.model, clause, 'manualOnly'); 105 | if (!filter) { 106 | return this; 107 | } 108 | const other = new VirtualCollection(this); 109 | other.manualFilters.push(filter); 110 | return other; 111 | } 112 | 113 | orderBy(field: string | FieldPath, directionStr: OrderByDirection = 'asc'): VirtualCollection { 114 | const other = new VirtualCollection(this); 115 | other._orderBy.push({ field, directionStr }); 116 | return other; 117 | } 118 | 119 | limit(limit: number): VirtualCollection { 120 | const other = new VirtualCollection(this); 121 | other._limit = limit; 122 | return other; 123 | } 124 | 125 | offset(offset: number): VirtualCollection { 126 | const other = new VirtualCollection(this); 127 | other._offset = offset; 128 | return other; 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/db/deep-reference.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import type { DocumentReference, DocumentSnapshot } from "@google-cloud/firestore"; 3 | import type { FlatCollection } from './flat-collection'; 4 | import { FlatReferenceShape, Reference, SetOpts, Snapshot } from './reference'; 5 | import { FieldOperation } from './field-operation'; 6 | import { runUpdateLifecycle } from '../utils/lifecycle'; 7 | 8 | /** 9 | * References an item in a flattened collection 10 | * (i.e.) a field within a document. 11 | */ 12 | export class DeepReference extends Reference { 13 | 14 | readonly doc: DocumentReference<{ [id: string]: T }> 15 | 16 | constructor(readonly id: string, readonly parent: FlatCollection) { 17 | super(); 18 | if (!id) { 19 | throw new Error('Document ID must not be empty'); 20 | } 21 | 22 | this.doc = parent.document; 23 | } 24 | 25 | get path() { 26 | return `${this.doc.path}/${this.id}`; 27 | } 28 | 29 | 30 | get firestore() { 31 | return this.doc.firestore; 32 | } 33 | 34 | async delete(opts?: SetOpts) { 35 | await runUpdateLifecycle({ 36 | editMode: 'update', 37 | ref: this, 38 | data: undefined, 39 | opts, 40 | timestamp: new Date(), 41 | }); 42 | }; 43 | 44 | create(data: T, opts?: SetOpts): Promise 45 | create(data: Partial, opts?: SetOpts): Promise> 46 | async create(data: T | Partial, opts?: SetOpts) { 47 | return await runUpdateLifecycle({ 48 | editMode: 'create', 49 | ref: this, 50 | data, 51 | opts, 52 | timestamp: new Date(), 53 | }); 54 | }; 55 | 56 | update(data: T, opts?: SetOpts): Promise 57 | update(data: Partial, opts?: SetOpts): Promise> 58 | async update(data: T | Partial, opts?: SetOpts) { 59 | return await runUpdateLifecycle({ 60 | editMode: 'update', 61 | ref: this, 62 | data, 63 | opts, 64 | timestamp: new Date(), 65 | }); 66 | } 67 | 68 | 69 | /** 70 | * Performs a `create()`, `update()`, or `delete()` operation without any coercion or lifecycles. 71 | * @private 72 | * @deprecated For internal connector use only 73 | */ 74 | async writeInternal(data: Partial | undefined, editMode: 'create' | 'update') { 75 | const d = mapToFlattenedDoc(this, data, editMode === 'update'); 76 | await this.parent.ensureDocument(); 77 | 78 | // TODO: Fail on create if document already exists 79 | // TODO: Fail on update if document doesn't exist 80 | 81 | // Firestore does not run the converter on update operations 82 | const out = this.parent.converter.toFirestore(d); 83 | await this.doc.update(out as any); 84 | } 85 | 86 | 87 | async get(): Promise> { 88 | // Apply a field mask so only the specific entry in flattened document is returned 89 | // This saves bandwidth from the database 90 | const [snap] = await this.doc.firestore.getAll(this.doc, { fieldMask: [this.id] }); 91 | return makeDeepSnap(this, snap); 92 | } 93 | 94 | isEqual(other: any) { 95 | return (this === other) || 96 | (other instanceof DeepReference 97 | && this.id === other.id 98 | && this.doc.isEqual(other.doc)); 99 | } 100 | 101 | /** 102 | * Allow serialising to JSON. 103 | */ 104 | toJSON() { 105 | return this.id; 106 | } 107 | 108 | /** 109 | * Returns a value that can be serialised 110 | * to Firestore. 111 | */ 112 | toFirestoreValue(): FlatReferenceShape { 113 | return { 114 | ref: this.doc, 115 | id: this.id, 116 | }; 117 | } 118 | 119 | toString() { 120 | return this.path; 121 | } 122 | } 123 | 124 | 125 | export function makeDeepSnap(ref: DeepReference, snap: DocumentSnapshot<{[id: string]: T}>): Snapshot { 126 | const data = snap.data()?.[ref.id]; 127 | return { 128 | ref, 129 | data: () => data, 130 | id: ref.id, 131 | exists: data !== undefined, 132 | }; 133 | } 134 | 135 | export function mapToFlattenedDoc({ id }: DeepReference, data: Partial | undefined, merge: boolean): { [id: string]: any } { 136 | if ((data !== undefined) && (typeof data !== 'object')) { 137 | throw new Error(`Invalid data provided to Firestore. It must be an object but it was: ${JSON.stringify(data)}`); 138 | } 139 | 140 | if (!data) { 141 | return { 142 | [id]: FieldOperation.delete(), 143 | }; 144 | } else { 145 | if (merge) { 146 | // Flatten into key-value pairs to merge the fields 147 | return _.toPairs(data).reduce((d, [path, value]) => { 148 | d[`${id}.${path}`] = value; 149 | return d; 150 | }, {}); 151 | } else { 152 | return { [id]: data }; 153 | } 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/db/readonly-transaction.ts: -------------------------------------------------------------------------------- 1 | import type { Transaction as FirestoreTransaction, Firestore } from '@google-cloud/firestore'; 2 | import { ReadRepository } from '../utils/read-repository'; 3 | import type { Queryable, QuerySnapshot } from './collection'; 4 | import type { Reference, SetOpts, Snapshot } from './reference'; 5 | import { VirtualReference } from './virtual-reference'; 6 | import type { GetOpts, Transaction } from './transaction'; 7 | import { get, getAll, getRefInfo } from './readwrite-transaction'; 8 | 9 | 10 | export class ReadOnlyTransaction implements Transaction { 11 | 12 | private readonly nonAtomicReads: ReadRepository; 13 | 14 | 15 | /** 16 | * @private 17 | * @deprecated For internal connector use only 18 | */ 19 | readonly successHooks: (() => (void | PromiseLike))[] = []; 20 | 21 | 22 | /** 23 | * @deprecated Not supported on ReadonlyTransaction 24 | */ 25 | get nativeTransaction(): FirestoreTransaction { 26 | throw new Error('nativeTransaction is not supported on ReadonlyTransaction'); 27 | } 28 | 29 | 30 | constructor( 31 | readonly firestore: Firestore, 32 | private readonly logStats: boolean, 33 | ) { 34 | this.nonAtomicReads = new ReadRepository({ 35 | getAll: (refs, fieldMask) => firestore.getAll(...refs, { fieldMask }), 36 | getQuery: query => query.get(), 37 | }); 38 | } 39 | 40 | /** 41 | * @deprecated Not supported on ReadOnlyTransaction 42 | */ 43 | getAtomic(ref: Reference, opts?: GetOpts): Promise> 44 | getAtomic(refs: Reference[], opts?: GetOpts): Promise[]> 45 | getAtomic(query: Queryable): Promise> 46 | getAtomic(): Promise | Snapshot[] | QuerySnapshot> { 47 | throw new Error('getAtomic() is not supported on ReadOnlyTransaction'); 48 | } 49 | 50 | getNonAtomic(ref: Reference, opts?: GetOpts): Promise> 51 | getNonAtomic(refs: Reference[], opts?: GetOpts): Promise[]> 52 | getNonAtomic(query: Queryable): Promise> 53 | getNonAtomic(refOrQuery: Reference | Reference[] | Queryable, opts?: GetOpts): Promise | Snapshot[] | QuerySnapshot> { 54 | if (Array.isArray(refOrQuery)) { 55 | return getAll(refOrQuery, this.nonAtomicReads, opts); 56 | } else { 57 | return get(refOrQuery, this.nonAtomicReads, opts); 58 | } 59 | } 60 | 61 | /** 62 | * @private 63 | * @deprecated For internal connector use only 64 | */ 65 | commit() { 66 | if (this.logStats && this.nonAtomicReads.readCount) { 67 | strapi.log.debug(`TRANSACTION (read-only): ${this.nonAtomicReads.readCount} reads.`); 68 | } 69 | return Promise.resolve(); 70 | } 71 | 72 | 73 | create(ref: Reference, data: T, opts?: SetOpts): Promise 74 | create(ref: Reference, data: Partial, opts?: SetOpts): Promise> 75 | async create(ref: Reference, data: T | Partial, opts?: SetOpts): Promise> { 76 | if (ref instanceof VirtualReference) { 77 | return await ref.create(data, opts); 78 | } else { 79 | throw new Error('create() is not supported on ReadOnlyTransaction'); 80 | } 81 | } 82 | 83 | update(ref: Reference, data: T, opts?: SetOpts): Promise 84 | update(ref: Reference, data: Partial, opts?: SetOpts): Promise> 85 | async update(ref: Reference, data: T | Partial, opts?: SetOpts): Promise> { 86 | if (ref instanceof VirtualReference) { 87 | return await ref.update(data, opts); 88 | } else { 89 | throw new Error('update() is not supported on ReadOnlyTransaction'); 90 | } 91 | } 92 | 93 | async delete(ref: Reference): Promise { 94 | if (ref instanceof VirtualReference) { 95 | return await ref.delete(); 96 | } else { 97 | throw new Error('delete() is not supported on ReadOnlyTransaction'); 98 | } 99 | } 100 | 101 | /** 102 | * @deprecated Not supported on ReadOnlyTransaction 103 | */ 104 | addNativeWrite(): never { 105 | throw new Error('Writes are not supported on ReadOnlyTransaction'); 106 | } 107 | 108 | addSuccessHook(cb: () => (void | PromiseLike)): void { 109 | this.successHooks.push(cb); 110 | } 111 | 112 | /** 113 | * Performs write operations only for virtual references. All other write operations 114 | * are not supported. 115 | * @private 116 | * @deprecated For internal connector use only 117 | */ 118 | mergeWriteInternal(ref: Reference, data: Partial | undefined, editMode: 'create' | 'update') { 119 | const { docRef } = getRefInfo(ref); 120 | if (!docRef) { 121 | (ref as VirtualReference).writeInternal(data, editMode); 122 | return; 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/db/transaction.ts: -------------------------------------------------------------------------------- 1 | import type { Transaction as FirestoreTransaction } from '@google-cloud/firestore'; 2 | import type { Queryable, QuerySnapshot } from './collection'; 3 | import type { Reference, SetOpts, Snapshot } from './reference'; 4 | 5 | export interface TransactionOpts { 6 | /** 7 | * Makes this a read-only transaction. Read-only transactions do not run any native Firestore transaction, 8 | * and cannot perform any writes, they are only useful for caching read data. 9 | * Furthermore, atomic-read operations are not supported. 10 | * @default false 11 | */ 12 | readOnly?: boolean; 13 | 14 | /** 15 | * The maximum number of attempts for this transaction. 16 | * Does not apply to read-only transactions, which only have a single attempt. 17 | */ 18 | maxAttempts?: number; 19 | } 20 | 21 | export interface GetOpts { 22 | /** 23 | * Relevant in flattened collections only (when getting a `DeepReference`), 24 | * ignored for normal collections. 25 | * 26 | * If `true`, field masks will be applied so that only the requested entries in the flattened collection 27 | * are returned, saving bandwidth and processing, but the entries will not be cached in the transaction. 28 | * 29 | * If `false`, the entire flattened collection (stored in a single document) will be 30 | * fetched and cached in the transaction, meaning to only a single read operation 31 | * is used even when multiple entries are fetched (in the same transaction). 32 | * 33 | * **Caution:** Use this only when you know that this is the request that will 34 | * be fetched from this collection within the scope of the transaction. Otherwise you 35 | * may be defeating the purpose of flattened collections. 36 | */ 37 | isSingleRequest?: boolean 38 | } 39 | 40 | /** 41 | * Acts as a conduit for all read and write operations, with the 42 | * following behaviours: 43 | * - Caches all read operations so that any document is read 44 | * only once within the batch (excludes queries). 45 | * - Merges all writes within a single document so that each 46 | * document is written only once. 47 | * - Performs all writes within an atomic transaction. 48 | * - The `getNonAtomic(...)` methods will reuse a cached read 49 | * from a `getAtomic(...)` call, but not the other way around. 50 | * - The `getAtomic(...)` functions perform the read operations within 51 | * a `Transaction`, which holds a lock on those documents for the 52 | * duration of the transaction. 53 | */ 54 | export interface Transaction { 55 | 56 | /** 57 | * The underlying Firestore `Transaction`. 58 | */ 59 | readonly nativeTransaction: FirestoreTransaction; 60 | 61 | /** 62 | * Enqueues a callback which will be executed after the transaction block is run 63 | * and before the transaction is committed. 64 | * 65 | * Firestore does not allow any reads from the transaction subsequent to any write. 66 | * This Transaction wrapper allows any order of reads or writes by batching and merging the 67 | * writes at the end of the transaction. If you add a write directly to the transaction, 68 | * you could cause reads and queries from relation updates to fail. 69 | * 70 | * Therefore, if you wish to add a write to the transaction, add the write inside this callback. 71 | * 72 | * For example: 73 | * 74 | * ``` 75 | * transaction.addNativeWrite(trans => trans.update(…)); 76 | * ``` 77 | */ 78 | addNativeWrite(cb: (transaction: FirestoreTransaction) => void): void; 79 | 80 | /** 81 | * Adds a callback that will be run after the transaction is successfully committed. 82 | * The callback must not throw. 83 | */ 84 | addSuccessHook(cb: () => (void | PromiseLike)): void; 85 | 86 | /** 87 | * Reads the given document and holds lock on it 88 | * for the duration of this transaction. 89 | * 90 | * Returns a cached response if the document has already been 91 | * fetched and locked using `getAtomic(...)` within this transaction. 92 | * Does not return a cached response from `getNonAtomic(...)` because 93 | * that would not establish a lock. 94 | */ 95 | getAtomic(ref: Reference, opts?: GetOpts): Promise> 96 | getAtomic(refs: Reference[], opts?: GetOpts): Promise[]> 97 | getAtomic(query: Queryable): Promise> 98 | 99 | /** 100 | * Reads the given document. Returns a cached response if the document 101 | * has been read *(with `getNonAtomic(...)` or `getAtomic(...)`)* within 102 | * this transaction. 103 | */ 104 | getNonAtomic(ref: Reference, opts?: GetOpts): Promise> 105 | getNonAtomic(refs: Reference[], opts?: GetOpts): Promise[]> 106 | getNonAtomic(query: Queryable): Promise> 107 | 108 | /** 109 | * Creates the given document, merging the data with any other `create()` or `update()` 110 | * operations on the document within this transaction. 111 | * 112 | * @returns The coerced data 113 | */ 114 | create(ref: Reference, data: T, opts?: SetOpts): Promise 115 | create(ref: Reference, data: Partial, opts?: SetOpts): Promise> 116 | 117 | /** 118 | * Updates the given document, merging the data with any other `create()` or `update()` 119 | * operations on the document within this transaction. 120 | * 121 | * @returns The coerced data 122 | */ 123 | update(ref: Reference, data: Partial, opts?: SetOpts): Promise 124 | update(ref: Reference, data: Partial>, opts?: SetOpts): Promise> 125 | 126 | /** 127 | * Deletes the given document, overriding all other write operations 128 | * on the document in this transaction. 129 | */ 130 | delete(ref: Reference, opts?: SetOpts): Promise 131 | } 132 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path'; 2 | import * as fs from 'fs-extra'; 3 | import * as _ from 'lodash'; 4 | import { Firestore, Settings, DocumentReference, Timestamp, FieldPath } from '@google-cloud/firestore'; 5 | import { allModels, DEFAULT_CREATE_TIME_KEY, DEFAULT_UPDATE_TIME_KEY, mountModels } from './model'; 6 | import { queries } from './queries'; 7 | import type { Strapi, ConnectorOptions } from './types'; 8 | import { FlatCollection } from './db/flat-collection'; 9 | 10 | export type { 11 | Strapi, 12 | Connector, 13 | ConnectorOptions, 14 | ModelOptions, 15 | Converter, 16 | StrapiQuery, 17 | IndexerFn, 18 | FlattenFn, 19 | ModelTestFn, 20 | } from './types'; 21 | export type { 22 | Reference, 23 | Snapshot 24 | } from './db/reference'; 25 | export type { 26 | Queryable, 27 | Collection, 28 | QuerySnapshot, 29 | } from './db/collection'; 30 | export type { FirestoreConnectorModel } from './model'; 31 | export type { Transaction } from './db/transaction'; 32 | export type { 33 | PopulatedByKeys, 34 | PickReferenceKeys, 35 | } from './populate'; 36 | 37 | 38 | 39 | const defaults = { 40 | defaultConnection: 'default', 41 | }; 42 | 43 | const defaultOptions: Required = { 44 | useEmulator: false, 45 | singleId: 'default', 46 | flattenModels: [], 47 | allowNonNativeQueries: false, 48 | ensureComponentIds: true, 49 | logTransactionStats: process.env.NODE_ENV === 'development', 50 | logQueries: false, 51 | metadataField: '$meta', 52 | creatorUserModel: { model: 'user', plugin: 'admin' }, 53 | ignoreMismatchedReferences: false, 54 | beforeMountModel: () => {}, 55 | afterMountModel: () => {}, 56 | 57 | // Default to 200 because of query size used in admin permissions query 58 | // https://github.com/strapi/strapi/blob/be4d5556936cf923aa3e23d5da82a6c60a5a42bc/packages/strapi-admin/services/permission.js 59 | maxQuerySize: 200, 60 | } 61 | 62 | module.exports = (strapi: Strapi) => { 63 | 64 | // Patch BigInt to allow JSON serialization 65 | if (!(BigInt.prototype as any).toJSON) { 66 | (BigInt.prototype as any).toJSON = function() { return this.toString() }; 67 | } 68 | 69 | // Patch Firestore types to allow JSON serialization 70 | (DocumentReference.prototype as any).toJSON = function() { return (this as DocumentReference).id; }; 71 | (Timestamp.prototype as any).toJSON = function() { return (this as Timestamp).toDate().toJSON(); }; 72 | (FieldPath.prototype as any).toJSON = function() { return (this as FieldPath).toString(); }; 73 | 74 | const { connections } = strapi.config; 75 | const firestoreConnections = Object.keys(connections) 76 | .filter(connectionName => { 77 | const connection = connections[connectionName]; 78 | if (connection.connector !== 'firestore') { 79 | strapi.log.warn( 80 | 'You are using the Firestore connector alongside ' + 81 | 'other connector types. The Firestore connector is not ' + 82 | 'designed for this, so you will likely run into problems.' 83 | ); 84 | return false; 85 | } else { 86 | return true; 87 | } 88 | }); 89 | 90 | 91 | const initialize = async () => { 92 | await Promise.all( 93 | firestoreConnections.map(async connectionName => { 94 | const connection = connections[connectionName]; 95 | 96 | _.defaults(connection.settings, strapi.config.hook.settings.firestore); 97 | const options = _.defaults(connection.options, defaultOptions); 98 | 99 | const settings: Settings = { 100 | ignoreUndefinedProperties: true, 101 | useBigInt: true, 102 | ...connection.settings, 103 | }; 104 | 105 | if (options.useEmulator) { 106 | // Direct the Firestore instance to connect to a local emulator 107 | Object.assign(settings, { 108 | port: 8080, 109 | host: 'localhost', 110 | sslCreds: require('@grpc/grpc-js').credentials.createInsecure(), 111 | customHeaders: { 112 | "Authorization": "Bearer owner" 113 | }, 114 | }); 115 | } 116 | 117 | const firestore = new Firestore(settings); 118 | _.set(strapi, `connections.${connectionName}`, firestore); 119 | 120 | const initFunctionPath = path.resolve( 121 | strapi.config.appPath, 122 | 'config', 123 | 'functions', 124 | 'firebase.js' 125 | ); 126 | 127 | if (await fs.pathExists(initFunctionPath)) { 128 | require(initFunctionPath)(firestore, connection); 129 | } 130 | 131 | // Mount all models 132 | await mountModels({ 133 | strapi, 134 | firestore, 135 | connectorOptions: options, 136 | }); 137 | 138 | // TODO: Find a way to initialise the connection lazily, and avoid performing a write operation on 139 | // every startup, because read operations are cheaper. 140 | // Initialise all flat collections 141 | // We do it here rather than lazily, otherwise the write which 142 | // ensures the existence will contend with the transaction that 143 | // operates on the document 144 | // In the Firestore production server this resolves and retries 145 | // but in the emulator it results in deadlock 146 | const tasks: Promise[] = []; 147 | for (const { model: { db } } of allModels()) { 148 | if (db instanceof FlatCollection) { 149 | tasks.push(db.ensureDocument()); 150 | } 151 | } 152 | await Promise.all(tasks); 153 | }) 154 | ); 155 | }; 156 | 157 | const destroy = async () => { 158 | await Promise.all( 159 | firestoreConnections.map(async connectionName => { 160 | const firestore = strapi.connections[connectionName]; 161 | if (firestore instanceof Firestore) { 162 | await firestore.terminate(); 163 | } 164 | }) 165 | ); 166 | }; 167 | 168 | return { 169 | defaults, 170 | initialize, 171 | destroy, 172 | queries, 173 | defaultTimestamps: [DEFAULT_CREATE_TIME_KEY, DEFAULT_UPDATE_TIME_KEY], 174 | }; 175 | }; 176 | -------------------------------------------------------------------------------- /src/db/flat-collection.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import * as path from 'path'; 3 | import { convertWhere } from '../utils/convert-where'; 4 | import { DocumentReference, OrderByDirection, FieldPath, DocumentData, FirestoreDataConverter } from '@google-cloud/firestore'; 5 | import type { Collection, QuerySnapshot } from './collection'; 6 | import type { FirestoreFilter, StrapiOrFilter, StrapiWhereFilter } from '../types'; 7 | import { DeepReference } from './deep-reference'; 8 | import type { FirestoreConnectorModel } from '../model'; 9 | import { coerceModelToFirestore, coerceToFirestore } from '../coerce/coerce-to-firestore'; 10 | import { coerceToModel } from '../coerce/coerce-to-model'; 11 | import type { ReadRepository } from '../utils/read-repository'; 12 | import { applyManualFilters, ManualFilter, OrderSpec } from '../utils/manual-filter'; 13 | 14 | 15 | export class FlatCollection implements Collection { 16 | 17 | readonly model: FirestoreConnectorModel 18 | readonly document: DocumentReference<{ [id: string]: T }>; 19 | readonly converter: FirestoreDataConverter<{ [id: string]: T }>; 20 | 21 | private readonly manualFilters: ManualFilter[] = []; 22 | private readonly _orderBy: OrderSpec[] = []; 23 | private _limit?: number; 24 | private _offset?: number; 25 | 26 | private _ensureDocument: Promise | null; 27 | 28 | 29 | constructor(model: FirestoreConnectorModel) 30 | constructor(other: FlatCollection) 31 | constructor(modelOrOther: FirestoreConnectorModel | FlatCollection) { 32 | if (modelOrOther instanceof FlatCollection) { 33 | // Copy the values 34 | this.model = modelOrOther.model; 35 | this.document = modelOrOther.document; 36 | this.converter = modelOrOther.converter; 37 | this._ensureDocument = modelOrOther._ensureDocument; 38 | this.manualFilters = modelOrOther.manualFilters.slice(); 39 | this._orderBy = modelOrOther._orderBy.slice(); 40 | this._limit = modelOrOther._limit; 41 | this._offset = modelOrOther._offset; 42 | } else { 43 | this.model = modelOrOther; 44 | this._ensureDocument = null; 45 | const { 46 | toFirestore = (value) => value, 47 | fromFirestore = (value) => value, 48 | } = modelOrOther.options.converter; 49 | this.converter = { 50 | toFirestore: data => { 51 | return _.mapValues(data, (d, path) => { 52 | // Remove the document ID component from the field path 53 | const { fieldPath } = splitId(path); 54 | if (fieldPath === modelOrOther.primaryKey) { 55 | return undefined; 56 | } 57 | 58 | // If the field path exists then the value isn't a root model object 59 | const obj: T = fieldPath ? coerceToFirestore(d) : coerceModelToFirestore(modelOrOther, d); 60 | return toFirestore(obj); 61 | }); 62 | }, 63 | fromFirestore: data => { 64 | return _.mapValues(data.data(), (d, path) => { 65 | const { id, fieldPath } = splitId(path); 66 | return coerceToModel(modelOrOther, id, fromFirestore(d), fieldPath, {}); 67 | }); 68 | }, 69 | }; 70 | 71 | const docPath = path.posix.join(modelOrOther.collectionName, modelOrOther.options.singleId); 72 | this.document = modelOrOther.firestore 73 | .doc(docPath) 74 | .withConverter(this.converter); 75 | } 76 | } 77 | 78 | get path(): string { 79 | return this.document.parent.path; 80 | } 81 | 82 | autoId() { 83 | return this.document.parent.doc().id; 84 | } 85 | 86 | doc(): DeepReference; 87 | doc(id: string): DeepReference; 88 | doc(id?: string) { 89 | return new DeepReference(id?.toString() || this.autoId(), this); 90 | }; 91 | 92 | 93 | /** 94 | * Ensures that the document containing this flat collection exists. 95 | * This operation is cached, so that it will happen at most once 96 | * for the life of the model instance. 97 | */ 98 | async ensureDocument(): Promise { 99 | // Set and merge with empty object 100 | // This will ensure that the document exists using 101 | // as single write operation 102 | if (!this._ensureDocument) { 103 | this._ensureDocument = this.document.set({}, { merge: true }) 104 | .catch((err) => { 105 | this._ensureDocument = null; 106 | throw err; 107 | }); 108 | } 109 | return this._ensureDocument; 110 | } 111 | 112 | async get(repo?: ReadRepository): Promise> { 113 | const snap = repo 114 | ? (await repo.getAll([{ ref: this.document }]))[0] 115 | : await this.document.get(); 116 | 117 | return applyManualFilters({ 118 | model: this.model, 119 | data: snap.data() || {}, 120 | filters: this.manualFilters, 121 | orderBy: this._orderBy, 122 | limit: this._limit, 123 | offset: this._offset, 124 | }); 125 | } 126 | 127 | where(clause: StrapiWhereFilter | StrapiOrFilter | FirestoreFilter): FlatCollection { 128 | const filter = convertWhere(this.model, clause, 'manualOnly'); 129 | if (!filter) { 130 | return this; 131 | } 132 | const other = new FlatCollection(this); 133 | other.manualFilters.push(filter); 134 | return other; 135 | } 136 | 137 | orderBy(field: string | FieldPath, directionStr: OrderByDirection = 'asc'): FlatCollection { 138 | const other = new FlatCollection(this); 139 | other._orderBy.push({ field, directionStr }); 140 | return other; 141 | } 142 | 143 | limit(limit: number): FlatCollection { 144 | const other = new FlatCollection(this); 145 | other._limit = limit; 146 | return other; 147 | } 148 | 149 | offset(offset: number): FlatCollection { 150 | const other = new FlatCollection(this); 151 | other._offset = offset; 152 | return other; 153 | } 154 | } 155 | 156 | function splitId(path: string): { id: string, fieldPath: string | null } { 157 | const i = path.indexOf('.'); 158 | if (i === -1) { 159 | return { 160 | id: path, 161 | fieldPath: null, 162 | }; 163 | } else { 164 | return { 165 | id: path.slice(0, i), 166 | fieldPath: path.slice(i + 1), 167 | }; 168 | } 169 | } -------------------------------------------------------------------------------- /src/utils/components-indexing.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { isEqualHandlingRef } from '../db/reference'; 3 | import type { FirestoreConnectorModel } from '../model'; 4 | import type { AttributeKey, IndexerFn, StrapiAttribute, StrapiModel } from '../types'; 5 | import { getComponentModel } from './components'; 6 | 7 | 8 | 9 | export function doesComponentRequireMetadata(attr: StrapiAttribute): boolean { 10 | return (attr.type === 'dynamiczone') 11 | || ((attr.type === 'component') && (attr.repeatable === true)); 12 | } 13 | 14 | export function updateComponentsMetadata(model: FirestoreConnectorModel, data: T, output: T = data) { 15 | if (model.isComponent) { 16 | return; 17 | } 18 | 19 | for (const parentAlias of model.componentKeys) { 20 | // Don't overwrite metadata with empty map if the value 21 | // doesn't exist because this could be a partial update 22 | if (!_.has(data, parentAlias)) { 23 | continue; 24 | } 25 | 26 | Object.assign(output, generateMetadataForComponent(model, parentAlias, data)); 27 | } 28 | } 29 | 30 | export function generateMetadataForComponent(model: FirestoreConnectorModel, parentAlias: AttributeKey, data: T): object | undefined { 31 | 32 | const parentAttr = model.attributes[parentAlias]; 33 | if (doesComponentRequireMetadata(parentAttr)) { 34 | const metaField = model.getMetadataMapKey(parentAlias); 35 | 36 | // Initialise the map will null for all known keys 37 | const meta: { [key: string]: any[] | null } = {}; 38 | const componentModels = parentAttr.component ? [parentAttr.component] : (parentAttr.components || []); 39 | for (const modelName of componentModels) { 40 | const { indexers = [] } = getComponentModel(modelName); 41 | for (const info of indexers) { 42 | for (const key of Object.keys(info.indexers)) { 43 | meta[key] = null; 44 | } 45 | } 46 | } 47 | 48 | // Make an array containing the value of this attribute from all the components in the array 49 | // If the value itself is an array then is is concatenated/flattened 50 | const components: any[] = _.castArray(_.get(data, parentAlias) || []); 51 | for (const component of components) { 52 | const componentModel = getComponentModel(model, parentAlias, component); 53 | if (!componentModel.indexers) { 54 | continue; 55 | } 56 | 57 | for (const { alias, indexers } of componentModel.indexers) { 58 | const values = _.castArray(_.get(component, alias, [])); 59 | 60 | for (const key of Object.keys(indexers)) { 61 | const arr = meta[key] = meta[key] || []; 62 | const indexer = indexers[key]; 63 | 64 | for (let value of values) { 65 | const result = indexer(value, component); 66 | 67 | // Only add if the element doesn't already exist 68 | // and is not undefined 69 | if ((result !== undefined) 70 | && (!arr.some(v => isEqualHandlingRef(v, result)))) { 71 | arr.push(result); 72 | } 73 | } 74 | } 75 | } 76 | } 77 | 78 | // Ensure all empty indexes are null 79 | for (const key of Object.keys(meta)) { 80 | const arr = meta[key]; 81 | if (!arr || !arr.length) { 82 | meta[key] = null; 83 | } 84 | 85 | // Add the empty indicator 86 | meta[`${key}$empty`] = (arr?.length === 0) as any; 87 | } 88 | 89 | // Only assign meta if there are any keys 90 | if (Object.keys(meta).length) { 91 | return { [metaField]: meta }; 92 | } else { 93 | return {}; 94 | } 95 | 96 | } 97 | 98 | return undefined; 99 | } 100 | 101 | 102 | 103 | export interface AttributeIndexInfo { 104 | alias: string 105 | attr: StrapiAttribute 106 | defaultIndexer?: string 107 | indexers: { 108 | [key: string]: IndexerFn 109 | } 110 | } 111 | 112 | /** 113 | * Build indexers for all the indexed attributes 114 | * in a component model. 115 | */ 116 | export function buildIndexers(model: StrapiModel): AttributeIndexInfo[] | undefined { 117 | if (model.modelType !== 'component') { 118 | return undefined; 119 | } 120 | 121 | const infos: AttributeIndexInfo[] = []; 122 | 123 | for (const alias of Object.keys(model.attributes)) { 124 | const attr = model.attributes[alias]; 125 | const isRelation = attr.model || attr.collection; 126 | 127 | if (isRelation || attr.index) { 128 | let defaultIndexer: string | undefined; 129 | let indexers: { [key: string]: IndexerFn }; 130 | 131 | if (typeof attr.index === 'object') { 132 | indexers = {}; 133 | for (const key of Object.keys(attr.index)) { 134 | const indexer = attr.index[key]; 135 | if (indexer) { 136 | if (typeof indexer === 'function') { 137 | indexers[key] = indexer; 138 | } else { 139 | indexers[key] = defaultIndexerFn; 140 | if (!defaultIndexer) { 141 | defaultIndexer = key; 142 | } 143 | } 144 | } 145 | } 146 | 147 | // Ensure there is a default indexer for relation types 148 | if (isRelation && !defaultIndexer) { 149 | defaultIndexer = alias; 150 | indexers[alias] = defaultIndexerFn; 151 | } 152 | 153 | } else { 154 | const key = (typeof attr.index === 'string') ? attr.index : alias; 155 | defaultIndexer = key; 156 | indexers = { 157 | [key]: defaultIndexerFn, 158 | }; 159 | } 160 | 161 | // Delete index info from the original attribute definition 162 | // no longer needed and it clutters API metadata etc 163 | delete attr.index; 164 | 165 | // If this indexer is for the primary key and only the indexer is defined 166 | // then this is a special case and we remove the attribute once 167 | // the indexers have been absorbed 168 | if ((alias === model.primaryKey) && !Object.keys(attr).length) { 169 | delete model.attributes[alias]; 170 | } 171 | 172 | infos.push({ 173 | alias, 174 | attr, 175 | defaultIndexer, 176 | indexers, 177 | }); 178 | } 179 | } 180 | 181 | return infos; 182 | } 183 | 184 | function defaultIndexerFn(value: any) { 185 | return value; 186 | } -------------------------------------------------------------------------------- /src/build-query.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { convertRestQueryParams } from 'strapi-utils'; 3 | import { FieldPath } from '@google-cloud/firestore'; 4 | import { EmptyQueryError } from './utils/convert-where'; 5 | import { StatusError } from './utils/status-error'; 6 | import { buildPrefixQuery } from './utils/prefix-query'; 7 | import type { Queryable } from './db/collection'; 8 | import type { StrapiFilter, StrapiOrFilter } from './types'; 9 | import type { FirestoreConnectorModel } from './model'; 10 | import type { Reference } from './db/reference'; 11 | 12 | export interface QueryArgs { 13 | model: FirestoreConnectorModel 14 | params: any 15 | allowSearch?: boolean 16 | } 17 | 18 | export function buildQuery(query: Queryable, { model, params, allowSearch }: QueryArgs): Queryable | Reference[] { 19 | 20 | // Capture the search term and remove it so it doesn't appear as filter 21 | const searchTerm = allowSearch ? params._q : undefined; 22 | delete params._q; 23 | 24 | const { where, limit, sort, start }: StrapiFilter = convertRestQueryParams(params); 25 | 26 | try { 27 | if (searchTerm !== undefined) { 28 | query = buildSearchQuery(model, searchTerm, query); 29 | } 30 | 31 | // Check for special case where querying for document IDs 32 | // In this case it is more effective to fetch the documents by id 33 | // because the "in" operator only supports ten arguments 34 | if (where && (where.length === 1)) { 35 | const [{ field, operator, value }] = where; 36 | if ((field === model.primaryKey) && ((operator === 'eq') || (operator === 'in'))) { 37 | return _.castArray(value || []) 38 | .slice(start || 0, (limit || -1) < 1 ? undefined : limit) 39 | .map(v => { 40 | if (!v || (typeof v !== 'string')) { 41 | throw new StatusError(`Argument for "${model.primaryKey}" must be an array of strings`, 400); 42 | } 43 | return model.db.doc(v) 44 | }); 45 | } 46 | } 47 | 48 | // Apply filters 49 | for (const clause of (where || [])) { 50 | query = query.where(clause); 51 | } 52 | 53 | for (const { field, order } of (sort || [])) { 54 | if (field === model.primaryKey) { 55 | if ((searchTerm !== undefined) || 56 | (where || []).some(w => w.field !== model.primaryKey)) { 57 | // Ignore sort by document ID when there are other filers 58 | // on fields other than the document ID 59 | // Document ID is the default sort for all queries 60 | // And more often than not, it interferes with Firestore inequality filter 61 | // or indexing rules 62 | } else { 63 | query = query.orderBy(FieldPath.documentId() as any, order); 64 | } 65 | } else { 66 | query = query.orderBy(field, order); 67 | } 68 | }; 69 | 70 | if (start! > 0) { 71 | query = query.offset(start!); 72 | } 73 | 74 | if (limit! > 0) { 75 | query = query.limit(limit!); 76 | } 77 | 78 | return query; 79 | } catch (err) { 80 | if (err instanceof EmptyQueryError) 81 | return []; 82 | else 83 | throw err; 84 | } 85 | } 86 | 87 | function buildSearchQuery(model: FirestoreConnectorModel, value: any, query: Queryable) { 88 | 89 | // Special case: empty query will match all entries 90 | if (value === '') { 91 | return query; 92 | } 93 | 94 | if (model.options.searchAttribute) { 95 | const field = model.options.searchAttribute; 96 | const type = model.getAttribute(field)?.type; 97 | 98 | // Build a native implementation of primitive search 99 | switch (type) { 100 | case 'integer': 101 | case 'float': 102 | case 'decimal': 103 | case 'biginteger': 104 | case 'date': 105 | case 'time': 106 | case 'datetime': 107 | case 'timestamp': 108 | case 'json': 109 | case 'boolean': 110 | // Use equality operator 111 | return query.where({ field, operator: 'eq', value }); 112 | 113 | case 'string': 114 | case 'text': 115 | case 'richtext': 116 | case 'email': 117 | case 'enumeration': 118 | case 'uid': 119 | case 'password': 120 | // Use prefix operator 121 | const { gte, lt } = buildPrefixQuery(value); 122 | return query 123 | .where({ field, operator: 'gte', value: gte }) 124 | .where({ field, operator: 'lt', value: lt }); 125 | 126 | default: 127 | throw new StatusError(`Search attribute "${field}" is an of an unsupported type`, 400); 128 | } 129 | 130 | } else { 131 | 132 | // Build a manual implementation of fully-featured search 133 | const filters: StrapiOrFilter['value'] = []; 134 | 135 | if (value != null) { 136 | filters.push([{ field: model.primaryKey, operator: 'eq', value }]); 137 | } 138 | 139 | for (const field of Object.keys(model.attributes)) { 140 | const attr = model.attributes[field]; 141 | switch (attr.type) { 142 | case 'integer': 143 | case 'float': 144 | case 'decimal': 145 | case 'biginteger': 146 | try { 147 | // Use equality operator for numbers 148 | filters.push([{ field, operator: 'eq', value }]); 149 | } catch { 150 | // Ignore if the query can't be coerced to this type 151 | } 152 | break; 153 | 154 | case 'string': 155 | case 'text': 156 | case 'richtext': 157 | case 'email': 158 | case 'enumeration': 159 | case 'uid': 160 | try { 161 | // User contains operator for strings 162 | filters.push([{ field, operator: 'contains', value }]); 163 | } catch { 164 | // Ignore if the query can't be coerced to this type 165 | } 166 | break; 167 | 168 | case 'date': 169 | case 'time': 170 | case 'datetime': 171 | case 'timestamp': 172 | case 'json': 173 | case 'boolean': 174 | case 'password': 175 | // Explicitly don't search in these fields 176 | break; 177 | 178 | default: 179 | // Unsupported field type for search 180 | // Don't search in these fields 181 | break; 182 | } 183 | } 184 | 185 | // Apply OR filter 186 | return query.where({ operator: 'or', value: filters }); 187 | } 188 | }; 189 | -------------------------------------------------------------------------------- /test/report.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs-extra'); 2 | const path = require('path'); 3 | const glob = require('glob-promise'); 4 | 5 | const fingerprint = 'strapi-connector-firestore-bot-results'; 6 | 7 | 8 | /** 9 | * Pull in type definitions for the available context variables. 10 | * 11 | * @typedef {InstanceType} GitHub 12 | * @typedef {import('@actions/github').context} Context 13 | * @typedef {import('@actions/core')} Core 14 | * @typedef {import('@actions/io')} Io 15 | * 16 | * @typedef {{ 17 | * github: GitHub, 18 | * context: Context, 19 | * core: Core, 20 | * io: Io 21 | * }} GitHubContext 22 | */ 23 | 24 | /** 25 | * Define result structure 26 | * @typedef {{ [suite: string]: { [flattening: string]: { pass: number, fail: number, skipped: number } } }} TestResult 27 | */ 28 | 29 | 30 | /** 31 | * This is a GitHub Script which reports the pass/fail results 32 | * of each test run onto the PR. 33 | * 34 | * See: https://github.com/marketplace/actions/github-script 35 | * 36 | * @param {GitHubContext} 37 | */ 38 | module.exports = async ({ github, context, core, io }) => { 39 | let totalPassed = 0, grandTotal = 0; 40 | const [results, baseResults] = await Promise.all([ 41 | loadResults(), 42 | findBaseResults({ github, context }) 43 | ]); 44 | const rows = [ 45 | `## Test results`, 46 | ``, 47 | ]; 48 | 49 | if (!baseResults) { 50 | rows.unshift( 51 | `> :exclamation: No results found for base commit. Results delta is unknown.`, 52 | ``, 53 | ); 54 | } 55 | 56 | // Build a markdown table 57 | 58 | Object.keys(results).forEach((suiteName, row) => { 59 | const colKeys = Object.keys(results[suiteName]).sort(); 60 | 61 | if (!row) { 62 | // Write header 63 | rows.push( 64 | `| Test suite | ${colKeys.join(' | ')} |`, 65 | `|------------| ${colKeys.map(() => '---').join('|')}|`, 66 | ); 67 | } 68 | 69 | // Write the row 70 | 71 | const cols = colKeys 72 | .map(flattening => { 73 | const { pass = 0, fail = 0, skipped = 0 } = results[suiteName][flattening]; 74 | const total = pass + fail + skipped; 75 | // const percent = (pass / total) * 100; 76 | 77 | totalPassed += pass; 78 | grandTotal += total; 79 | let row = `\`${pass} / ${total}\``; 80 | 81 | const base = ((baseResults || {})[suiteName] || {})[flattening]; 82 | if (base) { 83 | // const basePercent = (base.pass / (base.pass + base.skipped + base.fail)) * 100; 84 | const diff = pass - base.pass; 85 | const sign = (diff > 0) ? '+' : ''; 86 | row += ` (Δ \`${sign}${diff}\`)`; 87 | } else { 88 | row += ` (Δ ?)`; 89 | } 90 | 91 | return row; 92 | }) 93 | .join(' | '); 94 | 95 | rows.push( 96 | `| ${suiteName} | ${cols} |`, 97 | ); 98 | }); 99 | 100 | // Post the comment 101 | await updateComment(rows.join('\n'), results, { github, context }); 102 | 103 | return `Total passed ${totalPassed} out of ${grandTotal}`; 104 | }; 105 | 106 | 107 | 108 | /** 109 | * @returns {TestResult} 110 | */ 111 | const loadResults = async () => { 112 | const results = {}; 113 | const resultFiles = await glob.promise('coverage/*/results.json'); 114 | await Promise.all(resultFiles.map(async file => { 115 | const obj = await fs.readJSON(file); 116 | const flattening = /(flatten_\w+)\/results.json$/.exec(file)[1]; 117 | 118 | results['Total'] = results['Total'] || {}; 119 | results['Total'][flattening] = results['Total'][flattening] || {}; 120 | results['Total'][flattening].pass = obj.numPassedTests; 121 | results['Total'][flattening].fail = obj.numFailedTests; 122 | results['Total'][flattening].skipped = obj.numPendingTests; 123 | })); 124 | 125 | return results; 126 | }; 127 | 128 | /** 129 | * @param {GitHubContext} 130 | * @returns {TestResult} 131 | */ 132 | const findBaseResults = async ({ github, context }) => { 133 | 134 | const opts = github.repos.listCommentsForCommit.endpoint.merge({ 135 | ...context.repo, 136 | commit_sha: context.payload.pull_request 137 | ? context.payload.pull_request.base.sha 138 | : context.payload.before, 139 | }); 140 | 141 | const comment = await paginateFilteringFingerprint(opts, { github }); 142 | if (comment) { 143 | const regexp = new RegExp(`\n`); 144 | const match = regexp.exec(comment.body); 145 | if (match) { 146 | return JSON.parse(match[1]) || null; 147 | } 148 | } 149 | 150 | return null; 151 | }; 152 | 153 | /** 154 | * 155 | * @param {string} body 156 | * @param {TestResult} meta 157 | * @param {GitHubContext} 158 | */ 159 | const updateComment = async (body, meta, { github, context }) => { 160 | 161 | const opts = context.payload.pull_request 162 | ? github.issues.listComments.endpoint.merge({ 163 | ...context.repo, 164 | issue_number: context.issue.number, 165 | }) 166 | : github.repos.listCommentsForCommit.endpoint.merge({ 167 | ...context.repo, 168 | commit_sha: context.sha, 169 | }); 170 | 171 | const comment = await paginateFilteringFingerprint(opts, { github }); 172 | 173 | const fingerprintedBody = `\n${body}`; 174 | if (comment) { 175 | const endpoint = context.payload.pull_request 176 | ? github.issues.updateComment 177 | : github.repos.updateCommitComment; 178 | 179 | await endpoint({ 180 | ...context.repo, 181 | comment_id: comment.id, 182 | body: fingerprintedBody, 183 | }); 184 | } else { 185 | if (context.payload.pull_request) { 186 | await github.issues.createComment({ 187 | ...context.repo, 188 | issue_number: context.issue.number, 189 | body: fingerprintedBody, 190 | }); 191 | } else { 192 | await github.repos.createCommitComment({ 193 | ...context.repo, 194 | commit_sha: context.sha, 195 | body: fingerprintedBody, 196 | }); 197 | } 198 | } 199 | }; 200 | 201 | 202 | 203 | /** 204 | * 205 | * @param {any} opts 206 | * @param {GitHubContext} 207 | */ 208 | const paginateFilteringFingerprint = async (opts, { github }) => { 209 | 210 | // Paginate and filter for comments that contain 211 | // the fingerprint of this bot 212 | // There should be one or zero 213 | const [comment] = await github.paginate(opts, (resp, done) => { 214 | const filtered = resp.data.filter(cmt => (cmt.body || '').includes(fingerprint)); 215 | if (filtered.length) { 216 | done(); 217 | } 218 | return filtered; 219 | }); 220 | 221 | return comment || null; 222 | }; 223 | -------------------------------------------------------------------------------- /test/helpers/models.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { isFunction, isNil, prop } = require('lodash/fp'); 4 | const { createStrapiInstance } = require('./strapi'); 5 | 6 | const createHelpers = async ({ strapi: strapiInstance = null, ...options } = {}) => { 7 | const strapi = strapiInstance || (await createStrapiInstance(options)); 8 | const contentTypeService = strapi.plugins['content-type-builder'].services.contenttypes; 9 | const componentsService = strapi.plugins['content-type-builder'].services.components; 10 | 11 | const cleanup = async () => { 12 | if (isNil(strapiInstance)) { 13 | await strapi.destroy(); 14 | } 15 | }; 16 | 17 | return { 18 | strapi, 19 | contentTypeService, 20 | componentsService, 21 | cleanup, 22 | }; 23 | }; 24 | 25 | const createContentType = async (model, { strapi } = {}) => { 26 | const { contentTypeService, cleanup } = await createHelpers({ strapi }); 27 | 28 | const contentType = await contentTypeService.createContentType({ 29 | contentType: { 30 | connection: 'default', 31 | ...model, 32 | }, 33 | }); 34 | 35 | await cleanup(); 36 | 37 | return contentType; 38 | }; 39 | 40 | const createContentTypes = async (models, { strapi } = {}) => { 41 | const { contentTypeService, cleanup } = await createHelpers({ strapi }); 42 | 43 | const contentTypes = await contentTypeService.createContentTypes( 44 | models.map(model => ({ 45 | contentType: { 46 | connection: 'default', 47 | ...model, 48 | }, 49 | })) 50 | ); 51 | 52 | await cleanup(); 53 | 54 | return contentTypes; 55 | }; 56 | 57 | const createComponent = async (component, { strapi } = {}) => { 58 | const { componentsService, cleanup } = await createHelpers({ strapi }); 59 | 60 | const createdComponent = await componentsService.createComponent({ 61 | component: { 62 | category: 'default', 63 | icon: 'default', 64 | connection: 'default', 65 | ...component, 66 | }, 67 | }); 68 | 69 | await cleanup(); 70 | 71 | return createdComponent; 72 | }; 73 | 74 | const createComponents = async (components, { strapi } = {}) => { 75 | const createdComponents = []; 76 | 77 | for (const component of components) { 78 | createdComponents.push(await createComponent(component, { strapi })); 79 | } 80 | 81 | return createdComponents; 82 | }; 83 | 84 | const deleteComponent = async (componentUID, { strapi } = {}) => { 85 | const { componentsService, cleanup } = await createHelpers({ strapi }); 86 | 87 | const component = await componentsService.deleteComponent(componentUID); 88 | 89 | await cleanup(); 90 | 91 | return component; 92 | }; 93 | 94 | const deleteComponents = async (componentsUID, { strapi } = {}) => { 95 | const deletedComponents = []; 96 | 97 | for (const componentUID of componentsUID) { 98 | deletedComponents.push(await deleteComponent(componentUID, { strapi })); 99 | } 100 | 101 | return deletedComponents; 102 | }; 103 | 104 | const deleteContentType = async (modelName, { strapi } = {}) => { 105 | const { contentTypeService, cleanup } = await createHelpers({ strapi }); 106 | const uid = `application::${modelName}.${modelName}`; 107 | 108 | const contentType = await contentTypeService.deleteContentType(uid); 109 | 110 | await cleanup(); 111 | 112 | return contentType; 113 | }; 114 | 115 | const deleteContentTypes = async (modelsName, { strapi } = {}) => { 116 | const { contentTypeService, cleanup } = await createHelpers({ strapi }); 117 | const toUID = name => `application::${name}.${name}`; 118 | 119 | const contentTypes = await contentTypeService.deleteContentTypes(modelsName.map(toUID)); 120 | 121 | await cleanup(); 122 | 123 | return contentTypes; 124 | }; 125 | 126 | async function cleanupModels(models, { strapi } = {}) { 127 | for (const model of models) { 128 | await cleanupModel(model, { strapi }); 129 | } 130 | } 131 | 132 | async function cleanupModel(model, { strapi: strapiIst } = {}) { 133 | const { strapi, cleanup } = await createHelpers({ strapi: strapiIst }); 134 | 135 | await strapi.query(model).delete(); 136 | 137 | await cleanup(); 138 | } 139 | 140 | async function createFixtures(dataMap, { strapi: strapiIst } = {}) { 141 | const { strapi, cleanup } = await createHelpers({ strapi: strapiIst }); 142 | const models = Object.keys(dataMap); 143 | const resultMap = {}; 144 | 145 | for (const model of models) { 146 | const entries = []; 147 | 148 | for (const data of dataMap[model]) { 149 | entries.push(await strapi.query(model).create(data)); 150 | } 151 | 152 | resultMap[model] = entries; 153 | } 154 | 155 | await cleanup(); 156 | 157 | return resultMap; 158 | } 159 | 160 | async function createFixturesFor(model, entries, { strapi: strapiIst } = {}) { 161 | const { strapi, cleanup } = await createHelpers({ strapi: strapiIst }); 162 | const results = []; 163 | 164 | for (const entry of entries) { 165 | const dataToCreate = isFunction(entry) ? entry(results) : entry; 166 | results.push(await strapi.query(model).create(dataToCreate)); 167 | } 168 | 169 | await cleanup(); 170 | 171 | return results; 172 | } 173 | 174 | async function deleteFixturesFor(model, entries, { strapi: strapiIst } = {}) { 175 | const { strapi, cleanup } = await createHelpers({ strapi: strapiIst }); 176 | 177 | await strapi.query(model).delete({ id_in: entries.map(prop('id')) }); 178 | 179 | await cleanup(); 180 | } 181 | 182 | async function modifyContentType(data, { strapi } = {}) { 183 | const { contentTypeService, cleanup } = await createHelpers({ strapi }); 184 | 185 | const sanitizedData = { ...data }; 186 | delete sanitizedData.editable; 187 | delete sanitizedData.restrictRelationsTo; 188 | 189 | const uid = `application::${sanitizedData.name}.${sanitizedData.name}`; 190 | 191 | const ct = await contentTypeService.editContentType(uid, { 192 | contentType: { 193 | connection: 'default', 194 | ...sanitizedData, 195 | }, 196 | }); 197 | 198 | await cleanup(); 199 | 200 | return ct; 201 | } 202 | 203 | async function getContentTypeSchema(modelName, { strapi: strapiIst } = {}) { 204 | const { strapi, contentTypeService, cleanup } = await createHelpers({ strapi: strapiIst }); 205 | 206 | const uid = `application::${modelName}.${modelName}`; 207 | const ct = contentTypeService.formatContentType(strapi.contentTypes[uid]); 208 | 209 | await cleanup(); 210 | 211 | return (ct || {}).schema; 212 | } 213 | 214 | module.exports = { 215 | // Create Content-Types 216 | createContentType, 217 | createContentTypes, 218 | // Delete Content-Types 219 | deleteContentType, 220 | deleteContentTypes, 221 | // Cleanup Models 222 | cleanupModel, 223 | cleanupModels, 224 | // Create Components 225 | createComponent, 226 | createComponents, 227 | // Delete Components 228 | deleteComponent, 229 | deleteComponents, 230 | // Fixtures 231 | createFixtures, 232 | createFixturesFor, 233 | deleteFixturesFor, 234 | // Update Content-Types 235 | modifyContentType, 236 | // Misc 237 | getContentTypeSchema, 238 | }; 239 | -------------------------------------------------------------------------------- /src/relations.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { allModels, FirestoreConnectorModel } from './model'; 3 | import type { StrapiModel, StrapiAttribute } from './types'; 4 | import type { Transaction } from './db/transaction'; 5 | import { RelationAttrInfo, RelationHandler, RelationInfo } from './utils/relation-handler'; 6 | import { doesComponentRequireMetadata } from './utils/components-indexing'; 7 | import type { Reference, SetOpts } from './db/reference'; 8 | 9 | export function shouldUpdateRelations(opts: SetOpts | undefined): boolean { 10 | return !opts || (opts.updateRelations !== false); 11 | } 12 | 13 | 14 | /** 15 | * Parse relation attributes on this updated model and update the referred-to 16 | * models accordingly. 17 | */ 18 | export async function relationsUpdate(model: FirestoreConnectorModel, ref: Reference, prevData: T | undefined, newData: T | undefined, editMode: 'create' | 'update', transaction: Transaction) { 19 | await Promise.all( 20 | model.relations.map(r => r.update(ref, prevData, newData, editMode, transaction)) 21 | ); 22 | } 23 | 24 | export function buildRelations(model: FirestoreConnectorModel, strapiInstance = strapi) { 25 | 26 | // Build the dominant relations (these exist as attributes on this model) 27 | // The non-dominant relations will be populated as a matter of course 28 | // when the other models are built 29 | for (const alias of Object.keys(model.attributes)) { 30 | const attr = model.attributes[alias]; 31 | if (attr.isMeta) { 32 | continue; 33 | } 34 | 35 | const targetModelName = attr.model || attr.collection; 36 | if (!targetModelName) { 37 | // Not a relation attribute 38 | continue; 39 | } 40 | 41 | const isMorph = targetModelName === '*'; 42 | const attrInfo = makeAttrInfo(alias, attr); 43 | const thisEnd: RelationInfo = { 44 | model, 45 | parentModels: findParentModels(model, attrInfo, strapiInstance), 46 | attr: attrInfo, 47 | }; 48 | 49 | let otherEnds: RelationInfo[]; 50 | if (isMorph) { 51 | otherEnds = findModelsRelatingTo( 52 | { model, attr, alias }, 53 | strapiInstance 54 | ); 55 | } else { 56 | const targetModel = strapiInstance.db.getModel(targetModelName, attr.plugin); 57 | if (!targetModel) { 58 | throw new Error( 59 | `Problem building relations. The model targetted by attribute "${alias}" ` + 60 | `on model "${model.uid}" does not exist.` 61 | ); 62 | } 63 | const attrInfo = findOtherAttr(model, alias, attr, targetModel); 64 | otherEnds = [{ 65 | model: targetModel, 66 | parentModels: findParentModels(model, attrInfo, strapiInstance), 67 | attr: attrInfo, 68 | }]; 69 | } 70 | 71 | model.relations.push(new RelationHandler(thisEnd, otherEnds)); 72 | 73 | // If there are any non-dominant other ends 74 | // Then we add them to the other model also 75 | // so that the other model knows about the relation 76 | // (I.e. This is necessary when that model is deleting itself) 77 | for (const other of otherEnds) { 78 | if (!other.attr) { 79 | other.model.relations = other.model.relations || []; 80 | other.model.relations.push(new RelationHandler(other, [thisEnd])); 81 | } 82 | } 83 | } 84 | } 85 | 86 | 87 | 88 | function findModelsRelatingTo(info: { model: FirestoreConnectorModel, attr: StrapiAttribute, alias: string }, strapiInstance = strapi): RelationInfo[] { 89 | const related: RelationInfo[] = []; 90 | for (const { model } of allModels(strapiInstance)) { 91 | if (model.isComponent) { 92 | // Dominant relations to components not supported 93 | // Quietly ignore this on polymorphic relations because it 94 | // isn't specifically directed to this component model 95 | continue; 96 | } 97 | 98 | for (const alias of Object.keys(model.attributes)) { 99 | const attr = model.attributes[alias]; 100 | const otherModelName = attr.model || attr.collection; 101 | if (otherModelName 102 | && (otherModelName === info.model.modelName) 103 | && ((attr.via === info.alias) || (info.attr.via === alias))) { 104 | const attrInfo = makeAttrInfo(alias, attr); 105 | related.push({ 106 | model: model, 107 | parentModels: findParentModels(model, attrInfo, strapiInstance), 108 | attr: attrInfo, 109 | }); 110 | } 111 | } 112 | } 113 | return related; 114 | } 115 | 116 | function findOtherAttr(thisModel: StrapiModel, key: string, attr: StrapiAttribute, otherModel: StrapiModel): RelationAttrInfo | undefined { 117 | const alias = Object.keys(otherModel.attributes).find(alias => { 118 | const otherAttr = otherModel.attributes[alias]; 119 | if ((otherAttr.model || otherAttr.collection) === thisModel.modelName) { 120 | if (attr.via && (attr.via === alias)) { 121 | return true; 122 | } 123 | if (otherAttr.via && (otherAttr.via === key)) { 124 | return true; 125 | } 126 | } 127 | return false; 128 | }); 129 | 130 | if (alias) { 131 | const otherAttr = otherModel.attributes[alias]; 132 | return makeAttrInfo(alias, otherAttr); 133 | } 134 | return undefined; 135 | } 136 | 137 | function findParentModels(componentModel: FirestoreConnectorModel, componentAttr: RelationAttrInfo | undefined, strapiInstance = strapi): RelationInfo[] | undefined { 138 | const relations: RelationInfo[] = []; 139 | if (componentModel.isComponent && componentAttr) { 140 | const indexer = (componentModel.indexers || []).find(info => info.alias === componentAttr.alias); 141 | if (!indexer || !indexer.defaultIndexer) { 142 | // This should not be able to happen because it is guaranteed by buildIndexers() 143 | throw new Error('Relation in component does not have a default indexer'); 144 | } 145 | 146 | for (const { model: otherModel } of allModels(strapiInstance)) { 147 | if (componentModel.uid !== otherModel.uid) { 148 | for (const alias of Object.keys(otherModel.attributes)) { 149 | const attr = otherModel.attributes[alias]; 150 | if ((attr.component === componentModel.uid) 151 | || (attr.components && attr.components.includes(componentModel.uid))) { 152 | const isRepeatable = doesComponentRequireMetadata(attr); 153 | relations.push({ 154 | model: otherModel, 155 | attr: componentAttr ? { 156 | ...componentAttr, 157 | isMeta: isRepeatable, 158 | actualAlias: { 159 | componentAlias: componentAttr.alias, 160 | parentAlias: alias, 161 | }, 162 | alias: isRepeatable 163 | ? `${otherModel.getMetadataMapKey(alias)}.${indexer.defaultIndexer}` 164 | : `${alias}.${componentAttr.alias}`, 165 | } : undefined, 166 | parentModels: undefined, 167 | }); 168 | } 169 | } 170 | } 171 | } 172 | } 173 | return relations.length ? relations : undefined; 174 | } 175 | 176 | function makeAttrInfo(alias: string, attr: StrapiAttribute): RelationAttrInfo { 177 | return { 178 | alias, 179 | isArray: !attr.model || Boolean(attr.collection) || attr.repeatable || (attr.type === 'dynamiczone'), 180 | isMorph: (attr.model || attr.collection) === '*', 181 | filter: attr.filter, 182 | actualAlias: undefined, 183 | isMeta: false, 184 | }; 185 | } 186 | -------------------------------------------------------------------------------- /src/queries.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { PickReferenceKeys, populateDoc, populateSnapshots } from './populate'; 3 | import { StatusError } from './utils/status-error'; 4 | import type { StrapiQuery, StrapiContext } from './types'; 5 | import type { Queryable } from './db/collection'; 6 | import type { Transaction } from './db/transaction'; 7 | import type { Reference, Snapshot } from './db/reference'; 8 | import { buildQuery, QueryArgs } from './build-query'; 9 | 10 | 11 | /** 12 | * Firestore connector implementation of the Strapi query interface. 13 | */ 14 | export interface FirestoreConnectorQueries extends StrapiQuery { 15 | 16 | } 17 | 18 | 19 | export function queries({ model, strapi }: StrapiContext): FirestoreConnectorQueries { 20 | 21 | const log = model.options.logQueries 22 | ? (name: string, details: object) => { strapi.log.debug(`QUERY ${name}[${model.uid}] ${JSON.stringify(details)}`) } 23 | : () => {}; 24 | 25 | const find: FirestoreConnectorQueries['find'] = async (params, populate = (model.defaultPopulate as any)) => { 26 | log('find', { params, populate }); 27 | 28 | return await model.runTransaction(async trans => { 29 | const snaps = await buildAndFetchQuery({ model, params }, trans); 30 | 31 | // Populate relations 32 | return await populateSnapshots(snaps, populate, trans); 33 | }, { readOnly: true }); 34 | }; 35 | 36 | const findOne: FirestoreConnectorQueries['findOne'] = async (params, populate) => { 37 | const [entry] = await find({ ...params, _limit: 1 }, populate); 38 | return entry || null; 39 | }; 40 | 41 | const count: FirestoreConnectorQueries['count'] = async (params) => { 42 | log('count', { params }); 43 | return await model.runTransaction(async trans => { 44 | return await buildAndCountQuery({ model, params }, trans); 45 | }, { readOnly: true }); 46 | }; 47 | 48 | const create: FirestoreConnectorQueries['create'] = async (values, populate = (model.defaultPopulate as any)) => { 49 | log('create', { populate }); 50 | 51 | const ref = model.hasPK(values) 52 | ? model.db.doc(model.getPK(values)) 53 | : model.db.doc(); 54 | 55 | return await model.runTransaction(async trans => { 56 | // Create while coercing data and updating relations 57 | const data = await trans.create(ref, values); 58 | 59 | // Populate relations 60 | return await populateDoc(model, ref, data, populate, trans); 61 | }); 62 | }; 63 | 64 | const update: FirestoreConnectorQueries['update'] = async (params, values, populate = (model.defaultPopulate as any)) => { 65 | log('update', { params, populate }); 66 | 67 | return await model.runTransaction(async trans => { 68 | const [snap] = await buildAndFetchQuery({ 69 | model, 70 | params: { ...params, _limit: 1 }, 71 | }, trans); 72 | 73 | const prevData = snap && snap.data(); 74 | if (!prevData) { 75 | throw new StatusError('entry.notFound', 404); 76 | } 77 | 78 | // Update and merge coerced data (shallow merge) 79 | const data = { 80 | ...snap.data(), 81 | ...await trans.update(snap.ref, values), 82 | }; 83 | 84 | // Populate relations 85 | return await populateDoc(model, snap.ref, data, populate, trans); 86 | }); 87 | }; 88 | 89 | const deleteMany: FirestoreConnectorQueries['delete'] = async (params, populate = (model.defaultPopulate as any)) => { 90 | log('delete', { params, populate }); 91 | 92 | return await model.runTransaction(async trans => { 93 | const query = buildQuery(model.db, { model, params }); 94 | const snaps = await fetchQuery(query, trans); 95 | 96 | // The defined behaviour is unusual 97 | // Official connectors return a single item if queried by primary key or an array otherwise 98 | if (Array.isArray(query) && (query.length === 1)) { 99 | return await deleteOne(snaps[0], populate, trans); 100 | } else { 101 | return await Promise.all( 102 | snaps.map(snap => deleteOne(snap, populate, trans)) 103 | ); 104 | } 105 | }); 106 | }; 107 | 108 | async function deleteOne>(snap: Snapshot, populate: K[], trans: Transaction) { 109 | const prevData = snap.data(); 110 | if (!prevData) { 111 | // Delete API returns `null` rather than throwing an error for non-existent documents 112 | return null; 113 | } 114 | 115 | // Delete while updating relations 116 | await trans.delete(snap.ref); 117 | 118 | // Populate relations 119 | return await populateDoc(model, snap.ref, prevData, populate, trans); 120 | }; 121 | 122 | const search: FirestoreConnectorQueries['search'] = async (params, populate = (model.defaultPopulate as any)) => { 123 | log('search', { params, populate }); 124 | 125 | return await model.runTransaction(async trans => { 126 | const snaps = await buildAndFetchQuery({ model, params, allowSearch: true }, trans); 127 | return await populateSnapshots(snaps, populate, trans); 128 | }, { readOnly: true }); 129 | }; 130 | 131 | const countSearch: FirestoreConnectorQueries['countSearch'] = async (params) => { 132 | log('countSearch', { params }); 133 | return await model.runTransaction(async trans => { 134 | return await buildAndCountQuery({ model, params, allowSearch: true }); 135 | }, { readOnly: true }); 136 | }; 137 | 138 | const fetchRelationCounters: FirestoreConnectorQueries['fetchRelationCounters'] = async (attribute, entitiesIds = []) => { 139 | log('fetchRelationCounters', { attribute, entitiesIds }); 140 | 141 | const relation = model.relations.find(a => a.alias === attribute); 142 | if (!relation) { 143 | throw new StatusError(`Could not find relation "${attribute}" in model "${model.globalId}".`, 400); 144 | } 145 | 146 | if (!entitiesIds.length) { 147 | return []; 148 | } 149 | 150 | return await model.runTransaction(async trans => { 151 | const snaps = await trans.getNonAtomic(entitiesIds.map(id => model.db.doc(id))); 152 | 153 | return Promise.all(snaps.map(async snap => { 154 | const data = snap.data(); 155 | const count = data ? (await relation.findRelated(snap.ref, data, trans)).length : 0; 156 | return { 157 | id: snap.id, 158 | count, 159 | }; 160 | })); 161 | }, { readOnly: true }); 162 | }; 163 | 164 | const queries: FirestoreConnectorQueries = { 165 | model, 166 | find, 167 | findOne, 168 | count, 169 | create, 170 | update, 171 | delete: deleteMany, 172 | search, 173 | countSearch, 174 | fetchRelationCounters, 175 | }; 176 | return queries; 177 | } 178 | 179 | async function buildAndCountQuery(args: QueryArgs, transaction?: Transaction): Promise { 180 | const queryOrIds = buildQuery(args.model.db, args); 181 | if (!queryOrIds) { 182 | return 0; 183 | } 184 | 185 | if (Array.isArray(queryOrIds)) { 186 | // Don't do any read operations if we already know the count 187 | return queryOrIds.length; 188 | } else { 189 | const result = transaction 190 | ? await transaction.getNonAtomic(queryOrIds) 191 | : await queryOrIds.get(); 192 | return result.docs.length; 193 | } 194 | } 195 | 196 | async function buildAndFetchQuery(args: QueryArgs, transaction: Transaction): Promise[]> { 197 | const queryOrRefs = buildQuery(args.model.db, args); 198 | return await fetchQuery(queryOrRefs, transaction); 199 | } 200 | 201 | async function fetchQuery(queryOrRefs: Queryable | Reference[], transaction: Transaction): Promise[]> { 202 | if (Array.isArray(queryOrRefs)) { 203 | if (queryOrRefs.length) { 204 | return await transaction.getNonAtomic(queryOrRefs, { isSingleRequest: true }); 205 | } else { 206 | return []; 207 | } 208 | } else { 209 | const result = await transaction.getNonAtomic(queryOrRefs); 210 | return result.docs; 211 | } 212 | } 213 | 214 | -------------------------------------------------------------------------------- /src/db/normal-collection.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import { convertWhere } from '../utils/convert-where'; 3 | import { Query, QueryDocumentSnapshot, FieldPath, DocumentData, CollectionReference, FirestoreDataConverter } from '@google-cloud/firestore'; 4 | import type { Collection, QuerySnapshot } from './collection'; 5 | import type { FirestoreFilter, StrapiOrFilter, StrapiWhereFilter } from '../types'; 6 | import type { FirestoreConnectorModel } from '../model'; 7 | import { coerceModelToFirestore, coerceToFirestore } from '../coerce/coerce-to-firestore'; 8 | import { coerceToModel } from '../coerce/coerce-to-model'; 9 | import { makeNormalSnap, NormalReference } from './normal-reference'; 10 | import type { ReadRepository } from '../utils/read-repository'; 11 | import { QueryError } from './query-error'; 12 | import type { ManualFilter } from '../utils/manual-filter'; 13 | 14 | 15 | export class NormalCollection implements Collection { 16 | 17 | readonly model: FirestoreConnectorModel 18 | readonly converter: FirestoreDataConverter; 19 | 20 | readonly collection: CollectionReference 21 | 22 | private readonly allowNonNativeQueries: boolean 23 | private readonly maxQuerySize: number 24 | private readonly manualFilters: ManualFilter[]; 25 | private query: Query 26 | private _limit?: number; 27 | private _offset?: number; 28 | 29 | constructor(other: NormalCollection) 30 | constructor(model: FirestoreConnectorModel) 31 | constructor(modelOrOther: FirestoreConnectorModel | NormalCollection) { 32 | if (modelOrOther instanceof NormalCollection) { 33 | this.model = modelOrOther.model; 34 | this.collection = modelOrOther.collection; 35 | this.converter = modelOrOther.converter; 36 | this.allowNonNativeQueries = modelOrOther.allowNonNativeQueries; 37 | this.maxQuerySize = modelOrOther.maxQuerySize; 38 | this.query = modelOrOther.query; 39 | this.manualFilters = modelOrOther.manualFilters.slice(); 40 | this._limit = modelOrOther._limit; 41 | this._offset = modelOrOther._offset; 42 | } else { 43 | 44 | this.model = modelOrOther; 45 | const { 46 | toFirestore = (value) => value, 47 | fromFirestore = (value) => value, 48 | } = modelOrOther.options.converter; 49 | this.converter = { 50 | toFirestore: data => { 51 | const d = coerceModelToFirestore(modelOrOther, data); 52 | return toFirestore(d); 53 | }, 54 | fromFirestore: snap => { 55 | const d = fromFirestore(snap.data()); 56 | return coerceToModel(modelOrOther, snap.id, d, null, {}); 57 | }, 58 | }; 59 | 60 | this.collection = modelOrOther.firestore 61 | .collection(modelOrOther.collectionName) 62 | .withConverter(this.converter); 63 | 64 | this.query = this.collection; 65 | this.allowNonNativeQueries = modelOrOther.options.allowNonNativeQueries; 66 | this.maxQuerySize = modelOrOther.options.maxQuerySize; 67 | this.manualFilters = []; 68 | 69 | if (this.maxQuerySize < 0) { 70 | throw new Error("maxQuerySize cannot be less than zero"); 71 | } 72 | } 73 | } 74 | 75 | 76 | 77 | get path() { 78 | return this.collection.path; 79 | } 80 | 81 | autoId() { 82 | return this.collection.doc().id; 83 | } 84 | 85 | doc(): NormalReference; 86 | doc(id: string): NormalReference; 87 | doc(id?: string) { 88 | const doc = id ? this.collection.doc(id.toString()) : this.collection.doc(); 89 | return new NormalReference(doc, this); 90 | } 91 | 92 | 93 | private warnQueryLimit(limit: number | 'unlimited') { 94 | const msg = 95 | `The query limit of "${limit}" has been capped to "${this.maxQuerySize}".` + 96 | 'Adjust the strapi-connector-firestore \`maxQuerySize\` configuration option ' + 97 | 'if this is not the desired behaviour.'; 98 | 99 | if (limit === 'unlimited') { 100 | // Log at debug level if no limit was set 101 | strapi.log.debug(msg); 102 | } else { 103 | // Log at warning level if a limit was explicitly 104 | // set beyond the maximum limit 105 | strapi.log.warn(msg); 106 | } 107 | } 108 | 109 | async get(trans?: ReadRepository): Promise> { 110 | try { 111 | // Ensure the maximum limit is set if no limit has been set yet 112 | let q: NormalCollection = this; 113 | if (this.maxQuerySize && (this._limit === undefined)) { 114 | // Log a warning when the limit is applied where no limit was requested 115 | this.warnQueryLimit('unlimited'); 116 | q = q.limit(this.maxQuerySize); 117 | } 118 | 119 | const docs = q.manualFilters.length 120 | ? await queryWithManualFilters(q.query, q.manualFilters, q._limit || 0, q._offset || 0, this.maxQuerySize, trans) 121 | : await (trans ? trans.getQuery(q.query) : q.query.get()).then(snap => snap.docs); 122 | 123 | 124 | return { 125 | empty: docs.length === 0, 126 | docs: docs.map(s => { 127 | const ref = this.doc(s.id); 128 | return makeNormalSnap(ref, s); 129 | }), 130 | }; 131 | } catch (err) { 132 | throw new QueryError(err, this.query); 133 | } 134 | } 135 | 136 | where(clause: StrapiWhereFilter | StrapiOrFilter | FirestoreFilter): NormalCollection { 137 | const filter = convertWhere(this.model, clause, this.allowNonNativeQueries ? 'preferNative' : 'nativeOnly'); 138 | if (!filter) { 139 | return this; 140 | } 141 | const other = new NormalCollection(this); 142 | if (typeof filter === 'function') { 143 | other.manualFilters.push(filter); 144 | } else { 145 | // Convert the value for Firestore-native query 146 | const value = coerceToFirestore(filter.value); 147 | other.query = this.query.where(filter.field, filter.operator, value); 148 | } 149 | return other; 150 | } 151 | 152 | orderBy(field: string | FieldPath, directionStr: "desc" | "asc" = 'asc'): NormalCollection { 153 | const other = new NormalCollection(this); 154 | other.query = this.query.orderBy(field, directionStr); 155 | return other; 156 | } 157 | 158 | limit(limit: number): NormalCollection { 159 | if (this.maxQuerySize && (this.maxQuerySize < limit)) { 160 | // Log a warning when a limit is explicitly requested larger 161 | // than than the configured limit 162 | this.warnQueryLimit(limit); 163 | limit = this.maxQuerySize; 164 | } 165 | 166 | const other = new NormalCollection(this); 167 | other.query = this.query.limit(limit); 168 | other._limit = limit; 169 | return other; 170 | } 171 | 172 | offset(offset: number): NormalCollection { 173 | const other = new NormalCollection(this); 174 | other.query = this.query.offset(offset); 175 | other._offset = offset; 176 | return other; 177 | } 178 | } 179 | 180 | 181 | async function* queryChunked(query: Query, chunkSize: number, maxQuerySize: number, transaction: ReadRepository | undefined) { 182 | let cursor: QueryDocumentSnapshot | undefined 183 | let totalReads = 0; 184 | 185 | while (true) { 186 | if (maxQuerySize) { 187 | chunkSize = Math.max(chunkSize, maxQuerySize - totalReads); 188 | } 189 | if (chunkSize === 0) { 190 | return; 191 | } 192 | let q = query.limit(chunkSize); 193 | if (cursor) { 194 | // WARNING: 195 | // Usage of a cursor implicitly applies field ordering by document ID 196 | // and this can cause queries to fail 197 | // E.g. inequality filters require the first sort field to be the same 198 | // field as the inequality filter (see issue #29) 199 | // This scenario only manifests when manual queries are used 200 | q = q.startAfter(cursor); 201 | } 202 | 203 | const { docs } = await (transaction ? transaction.getQuery(q) : q.get()); 204 | cursor = docs[docs.length - 1]; 205 | totalReads += docs.length; 206 | 207 | for (const d of docs) { 208 | yield d; 209 | } 210 | 211 | if (docs.length < chunkSize) { 212 | return; 213 | } 214 | } 215 | } 216 | 217 | async function queryWithManualFilters(query: Query, filters: ManualFilter[], limit: number, offset: number, maxQuerySize: number, transaction: ReadRepository | undefined): Promise[]> { 218 | 219 | // Use a chunk size of 10 for the native query 220 | // E.g. if we only want 1 result, we will still query 221 | // ten at a time to improve performance for larger queries 222 | // But it will increase read usage (at most 9 reads will be unused) 223 | const chunkSize = Math.max(10, limit); 224 | 225 | // Improve performance by performing some native offset 226 | const q = query.offset(offset); 227 | 228 | const docs: QueryDocumentSnapshot[] = []; 229 | 230 | for await (const doc of queryChunked(q, chunkSize, maxQuerySize, transaction)) { 231 | if (filters.every(op => op(doc))) { 232 | if (offset) { 233 | offset--; 234 | } else { 235 | docs.push(doc); 236 | if (docs.length >= limit) { 237 | break; 238 | } 239 | } 240 | } 241 | } 242 | 243 | return docs; 244 | } -------------------------------------------------------------------------------- /src/db/readwrite-transaction.ts: -------------------------------------------------------------------------------- 1 | import { DocumentReference, Transaction as FirestoreTransaction, DocumentData, Firestore, DocumentSnapshot, FirestoreDataConverter } from '@google-cloud/firestore'; 2 | import { DeepReference, makeDeepSnap, mapToFlattenedDoc } from './deep-reference'; 3 | import { ReadRepository, RefAndMask } from '../utils/read-repository'; 4 | import type { Queryable, QuerySnapshot } from './collection'; 5 | import { Reference, SetOpts, Snapshot } from './reference'; 6 | import { MorphReference } from './morph-reference'; 7 | import { makeNormalSnap, NormalReference } from './normal-reference'; 8 | import { runUpdateLifecycle } from '../utils/lifecycle'; 9 | import { VirtualReference } from './virtual-reference'; 10 | import type { GetOpts, Transaction } from './transaction'; 11 | 12 | 13 | export class ReadWriteTransaction implements Transaction { 14 | 15 | private readonly writes = new Map(); 16 | private readonly nativeWrites: ((trans: FirestoreTransaction) => void)[] = []; 17 | 18 | /** 19 | * @private 20 | * @deprecated For internal connector use only 21 | */ 22 | readonly successHooks: (() => (void | PromiseLike))[] = []; 23 | 24 | private readonly timestamp = new Date(); 25 | private readonly atomicReads: ReadRepository; 26 | private readonly nonAtomicReads: ReadRepository; 27 | private readonly ensureFlatCollections: Promise[] = []; 28 | 29 | 30 | constructor( 31 | readonly firestore: Firestore, 32 | readonly nativeTransaction: FirestoreTransaction, 33 | private readonly logStats: boolean, 34 | private readonly attempt: number, 35 | ) { 36 | 37 | this.atomicReads = new ReadRepository({ 38 | getAll: (refs, fieldMask) => this.nativeTransaction.getAll(...refs, { fieldMask }), 39 | getQuery: query => this.nativeTransaction.get(query), 40 | }); 41 | 42 | this.nonAtomicReads = new ReadRepository({ 43 | getAll: (refs, fieldMask) => firestore.getAll(...refs, { fieldMask }), 44 | getQuery: query => query.get(), 45 | }, this.atomicReads); 46 | } 47 | 48 | getAtomic(ref: Reference, opts?: GetOpts): Promise> 49 | getAtomic(refs: Reference[], opts?: GetOpts): Promise[]> 50 | getAtomic(query: Queryable): Promise> 51 | getAtomic(refOrQuery: Reference | Reference[] | Queryable, opts?: GetOpts): Promise | Snapshot[] | QuerySnapshot> { 52 | if (Array.isArray(refOrQuery)) { 53 | return getAll(refOrQuery, this.atomicReads, opts); 54 | } else { 55 | return get(refOrQuery, this.atomicReads, opts); 56 | } 57 | } 58 | 59 | getNonAtomic(ref: Reference, opts?: GetOpts): Promise> 60 | getNonAtomic(refs: Reference[], opts?: GetOpts): Promise[]> 61 | getNonAtomic(query: Queryable): Promise> 62 | getNonAtomic(refOrQuery: Reference | Reference[] | Queryable, opts?: GetOpts): Promise | Snapshot[] | QuerySnapshot> { 63 | if (Array.isArray(refOrQuery)) { 64 | return getAll(refOrQuery, this.nonAtomicReads, opts); 65 | } else { 66 | return get(refOrQuery, this.nonAtomicReads, opts); 67 | } 68 | } 69 | 70 | /** 71 | * @private 72 | * @deprecated For internal connector use only 73 | */ 74 | async commit() { 75 | if (this.logStats) { 76 | strapi.log.debug(`TRANSACTION (attempt #${this.attempt}): ${this.writes.size} writes, ${this.atomicReads.readCount + this.nonAtomicReads.readCount} reads (${this.atomicReads.readCount} atomic).`); 77 | } 78 | 79 | // If we have fetched flat documents then we need to wait to 80 | // ensure that the document exists so that the update 81 | // operate will succeed 82 | await Promise.all(this.ensureFlatCollections); 83 | 84 | for (const op of this.writes.values()) { 85 | if (op.data === null) { 86 | this.nativeTransaction.delete(op.ref) 87 | } else { 88 | if (op.create) { 89 | this.nativeTransaction.create(op.ref, op.data); 90 | } else { 91 | // Firestore does not run the converter on update operations 92 | op.data = op.converter.toFirestore(op.data); 93 | this.nativeTransaction.update(op.ref, op.data); 94 | } 95 | } 96 | } 97 | 98 | // Commit any native writes 99 | for (const cb of this.nativeWrites) { 100 | cb(this.nativeTransaction); 101 | } 102 | } 103 | 104 | 105 | create(ref: Reference, data: T, opts?: SetOpts): Promise 106 | create(ref: Reference, data: Partial, opts?: SetOpts): Promise> 107 | async create(ref: Reference, data: T | Partial, opts?: SetOpts): Promise> { 108 | return (await runUpdateLifecycle({ 109 | editMode: 'create', 110 | ref, 111 | data, 112 | opts, 113 | transaction: this, 114 | timestamp: this.timestamp, 115 | }))!; 116 | } 117 | 118 | update(ref: Reference, data: T, opts?: SetOpts): Promise 119 | update(ref: Reference, data: Partial, opts?: SetOpts): Promise> 120 | async update(ref: Reference, data: T | Partial, opts?: SetOpts): Promise> { 121 | return (await runUpdateLifecycle({ 122 | editMode: 'update', 123 | ref, 124 | data, 125 | opts, 126 | transaction: this, 127 | timestamp: this.timestamp, 128 | }))!; 129 | } 130 | 131 | async delete(ref: Reference, opts?: SetOpts): Promise { 132 | await runUpdateLifecycle({ 133 | editMode: 'update', 134 | ref, 135 | data: undefined, 136 | opts, 137 | transaction: this, 138 | timestamp: this.timestamp, 139 | }); 140 | } 141 | 142 | addNativeWrite(cb: (transaction: FirestoreTransaction) => void): void { 143 | this.nativeWrites.push(cb); 144 | } 145 | 146 | 147 | addSuccessHook(cb: () => (void | PromiseLike)): void { 148 | this.successHooks.push(cb); 149 | } 150 | 151 | /** 152 | * Merges a create, update, or delete operation into pending writes for a given 153 | * reference in this transaction, without any coercion or lifecycles. 154 | * @private 155 | * @deprecated For internal connector use only 156 | */ 157 | mergeWriteInternal(ref: Reference, data: Partial | undefined, editMode: 'create' | 'update') { 158 | 159 | const { docRef, deepRef } = getRefInfo(ref); 160 | if (!docRef) { 161 | (ref as VirtualReference).writeInternal(data, editMode); 162 | return; 163 | } 164 | 165 | const { path } = docRef; 166 | 167 | let op: WriteOp; 168 | if (this.writes.has(path)) { 169 | op = this.writes.get(path)!; 170 | } else { 171 | op = { 172 | ref: docRef, 173 | data: {}, 174 | create: false, 175 | converter: ref.parent.converter, 176 | }; 177 | this.writes.set(path, op); 178 | 179 | // If the write is for a flattened collection 180 | // then pre-emptively start ensuring that the document exists 181 | if (deepRef) { 182 | this.ensureFlatCollections.push(deepRef.parent.ensureDocument()); 183 | } 184 | } 185 | 186 | if (op.data === null) { 187 | // Deletion overrides all other operations 188 | return; 189 | } 190 | 191 | // Don't create documents for flattened collections 192 | // because we use ensureDocument() and then update() 193 | op.create = op.create || ((editMode === 'create') && !deepRef) || false; 194 | 195 | if (deepRef) { 196 | Object.assign(op.data, mapToFlattenedDoc(deepRef, data, true)); 197 | } else { 198 | if (!data) { 199 | op.data = null; 200 | } else { 201 | Object.assign(op.data, data); 202 | } 203 | } 204 | } 205 | } 206 | 207 | 208 | interface WriteOp { 209 | ref: DocumentReference 210 | data: DocumentData | null 211 | create: boolean 212 | converter: FirestoreDataConverter 213 | } 214 | 215 | interface RefInfo { 216 | docRef?: DocumentReference, 217 | deepRef?: DeepReference, 218 | morphRef?: MorphReference, 219 | } 220 | 221 | 222 | export async function get(refOrQuery: Reference | Queryable, repo: ReadRepository, opts: GetOpts | undefined): Promise | QuerySnapshot> { 223 | if (refOrQuery instanceof Reference) { 224 | return (await getAll([refOrQuery], repo, opts))[0]; 225 | } else { 226 | // Queryable 227 | return await refOrQuery.get(repo); 228 | } 229 | } 230 | 231 | 232 | export async function getAll(refs: Reference[], repo: ReadRepository, opts: GetOpts | undefined): Promise[]> { 233 | const isSingleRequest = opts && opts.isSingleRequest; 234 | 235 | // Collect the masks for each native document 236 | const getters: ((args: { ref: Reference, results: DocumentSnapshot[], virtualSnaps: Snapshot[] }) => Snapshot)[] = new Array(refs.length); 237 | const docRefs = new Map(); 238 | const virtualGets: Promise>[] = []; 239 | 240 | for (let i = 0; i < refs.length; i++) { 241 | const { docRef, deepRef } = getRefInfo(refs[i]); 242 | if (!docRef) { 243 | const index = virtualGets.length; 244 | const ref = refs[i] as VirtualReference; 245 | virtualGets.push(ref.get()); 246 | getters[i] = ({ virtualSnaps }) => virtualSnaps[index]; 247 | } else { 248 | let entry = docRefs.get(docRef.path); 249 | if (!entry) { 250 | entry = { ref: docRef, i: docRefs.size }; 251 | docRefs.set(docRef.path, entry); 252 | } 253 | if (isSingleRequest && deepRef) { 254 | if (!entry.fieldMasks) { 255 | entry.fieldMasks = [deepRef.id]; 256 | } else if (!entry.fieldMasks.includes(deepRef.id)) { 257 | entry.fieldMasks.push(deepRef.id); 258 | } 259 | } 260 | 261 | getters[i] = ({ ref, results }) => makeSnap(ref, results[entry!.i]); 262 | } 263 | } 264 | 265 | const refsWithMasks = Array.from(docRefs.values()); 266 | const virtualSnaps = await Promise.all(virtualGets); 267 | const results = await repo.getAll(refsWithMasks); 268 | 269 | return refs.map((ref, i) => getters[i]({ ref, results, virtualSnaps })); 270 | } 271 | 272 | export function getRefInfo(ref: Reference): RefInfo { 273 | if (ref instanceof NormalReference) { 274 | return { docRef: ref.ref }; 275 | } 276 | if (ref instanceof DeepReference) { 277 | return { docRef: ref.doc, deepRef: ref }; 278 | } 279 | if (ref instanceof VirtualReference) { 280 | return {}; 281 | } 282 | if (ref instanceof MorphReference) { 283 | return { 284 | ...getRefInfo(ref.ref), 285 | morphRef: ref, 286 | }; 287 | } 288 | throw new Error('Unknown type of reference'); 289 | } 290 | 291 | function makeSnap(ref: Reference, snap: DocumentSnapshot): Snapshot { 292 | if (ref instanceof NormalReference) { 293 | return makeNormalSnap(ref, snap); 294 | } 295 | if (ref instanceof DeepReference) { 296 | return makeDeepSnap(ref, snap); 297 | } 298 | if (ref instanceof MorphReference) { 299 | return { 300 | ...makeSnap(ref, snap), 301 | ref, 302 | } 303 | } 304 | throw new Error('Unknown type of reference'); 305 | } 306 | -------------------------------------------------------------------------------- /src/utils/relation-handler.ts: -------------------------------------------------------------------------------- 1 | import * as _ from 'lodash'; 2 | import type { FirestoreConnectorModel } from '../model'; 3 | import type { Transaction } from '../db/transaction'; 4 | import { StatusError } from './status-error'; 5 | import { MorphReference } from '../db/morph-reference'; 6 | import { FieldOperation } from '../db/field-operation'; 7 | import { isEqualHandlingRef, Reference } from '../db/reference'; 8 | import { NormalReference } from '../db/normal-reference'; 9 | import { DeepReference } from '../db/deep-reference'; 10 | import { mapNotNull } from './map-not-null'; 11 | 12 | export interface RelationInfo { 13 | model: FirestoreConnectorModel 14 | attr: RelationAttrInfo | undefined 15 | parentModels: RelationInfo[] | undefined 16 | } 17 | 18 | export interface RelationAttrInfo { 19 | alias: string 20 | isArray: boolean 21 | filter: string | undefined 22 | isMorph: boolean 23 | 24 | /** 25 | * Indicates that this is a "virtual" attribute 26 | * which is metadata/index map for a repeatable component, 27 | * or a deep path to a non-repeatable component, 28 | * and the actual alias inside the component is this value. 29 | */ 30 | actualAlias: { 31 | componentAlias: string 32 | parentAlias: string 33 | } | undefined 34 | /** 35 | * Indicates that this is a metadata/index map, not a path to 36 | * an actual attribute. 37 | */ 38 | isMeta: boolean 39 | } 40 | 41 | export class RelationHandler { 42 | 43 | constructor( 44 | private readonly thisEnd: RelationInfo, 45 | private readonly otherEnds: RelationInfo[], 46 | ) { 47 | if (!thisEnd.attr && !otherEnds.some(e => e.attr)) { 48 | throw new Error('Relation does not have any dominant ends defined'); 49 | } 50 | 51 | if (otherEnds.some(e => e.model.isComponent && (!e.attr || thisEnd.attr))) { 52 | throw new Error('Relation cannot have a dominant reference to a component'); 53 | } 54 | 55 | // Virtual collections are probably transient/unstable unless `hasStableIds` is explicitly set 56 | // so references to a virtual collection should not be stored in the database 57 | // But we allow virtual collections to refer to other virtual collections because they are not stored in the database 58 | if (thisEnd.attr && !thisEnd.model.options.virtualDataSource && otherEnds.some(e => e.model.options.virtualDataSource && !e.model.options.virtualDataSource.hasStableIds)) { 59 | throw new Error('Non-virtual collection cannot have a dominant relation to a virtual collection without stable IDs'); 60 | } 61 | } 62 | 63 | /** 64 | * Gets the alias of this relation, or `undefined` 65 | * if this end of the relation is not dominant. 66 | */ 67 | get alias(): string | undefined { 68 | return this.thisEnd.attr?.alias; 69 | } 70 | 71 | /** 72 | * Finds references to the related models on the given object. 73 | * The related models are not necessarily fetched. 74 | */ 75 | async findRelated(ref: Reference, data: T, transaction: Transaction): Promise[]> { 76 | const { attr } = this.thisEnd; 77 | const related = attr 78 | ? this._getRefInfo(data, attr) 79 | : await this._queryRelated(ref, transaction, false); 80 | return related.map(r => r.ref); 81 | } 82 | 83 | 84 | /** 85 | * Updates the the related models on the given object. 86 | */ 87 | async update(ref: Reference, prevData: T | undefined, newData: T | undefined, editMode: 'create' | 'update', transaction: Transaction): Promise { 88 | const { attr: thisAttr } = this.thisEnd; 89 | if (thisAttr) { 90 | // This end is dominant 91 | // So we know all the other ends directly without querying 92 | 93 | // Update operations will not touch keys that don't exist 94 | // If the data doesn't have the key, then don't update the relation because we aren't touching it 95 | // If newData is undefined then we are deleting and we do need to update the relation 96 | if ((editMode === 'update') && newData && (_.get(newData, thisAttr.alias) === undefined)) { 97 | return; 98 | } 99 | 100 | const prevValues = this._getRefInfo(prevData, thisAttr); 101 | const newValues = this._getRefInfo(newData, thisAttr); 102 | 103 | // Set the value stored in this document appropriately 104 | this._setThis(newData, thisAttr, newValues.map(v => this._makeRefToOther(v.ref, thisAttr))); 105 | 106 | // Set the value stored in the references documents appropriately 107 | const removed = _.differenceWith(prevValues, newValues, (a, b) => isEqualHandlingRef(a.ref, b.ref)); 108 | const added = _.differenceWith(newValues, prevValues, (a, b) => isEqualHandlingRef(a.ref, b.ref)); 109 | 110 | const related = [ 111 | ...removed.map(info => ({ info, set: false })), 112 | ...added.map(info => ({ info, set: true })), 113 | ]; 114 | 115 | await this._setAllRelated(related, ref, transaction); 116 | 117 | } else { 118 | // I.e. thisAttr == null (meaning this end isn't dominant) 119 | 120 | if (!newData) { 121 | // This end is being deleted and it is not dominant 122 | // so we need to search for the dangling references existing on other models 123 | const related = await this._queryRelated(ref, transaction, true); 124 | await this._setAllRelated(related.map(info => ({ info, set: false })), ref, transaction); 125 | 126 | } else { 127 | // This end isn't dominant 128 | // But it isn't being deleted so there is 129 | // no action required on the other side 130 | } 131 | } 132 | } 133 | 134 | 135 | /** 136 | * Populates the related models onto the given object for this relation. 137 | */ 138 | async populateRelated(ref: Reference, data: T, transaction: Transaction): Promise { 139 | const { attr } = this.thisEnd; 140 | // This could be a partial data update 141 | // If the attribute value does not exist in the data, then we ignore it 142 | if (attr && (_.get(data, attr.alias) !== undefined)) { 143 | const related = await this.findRelated(ref, data, transaction); 144 | const results = related.length ? await transaction.getNonAtomic(related) : []; 145 | 146 | const values = mapNotNull(results, snap => { 147 | const data = snap.data(); 148 | if (!data) { 149 | // TODO: 150 | // Should we throw an error if the reference can't be found or just silently omit it? 151 | strapi.log.warn( 152 | `Could not populate the reference "${snap.ref.path}" because it no longer exists. ` + 153 | 'This may be because the database has been modified outside of Strapi, or there is a bug in the Firestore connector.' 154 | ); 155 | } 156 | return data; 157 | }); 158 | 159 | // The values will be correctly coerced 160 | // into and array or single value by the method below 161 | this._setThis(data, attr, values); 162 | } 163 | } 164 | 165 | 166 | 167 | private get _singleOtherEnd(): RelationInfo | undefined { 168 | return (this.otherEnds.length === 1) ? this.otherEnds[0] : undefined; 169 | } 170 | 171 | /** 172 | * Creates an appropriate `ReferenceShape` to store in the documents 173 | * at the other end, properly handling polymorphic references. 174 | * 175 | * @param ref The reference to this 176 | * @param otherAttr Attribute info of the other end (which refers to this) 177 | */ 178 | private _makeRefToThis(ref: Reference, otherAttr: RelationAttrInfo): Reference { 179 | if (otherAttr.isMorph && !(ref instanceof MorphReference)) { 180 | const { attr } = this.thisEnd; 181 | if (!attr && otherAttr.filter) { 182 | throw new Error('Polymorphic reference does not have the required information'); 183 | } 184 | if ((ref instanceof NormalReference) || (ref instanceof DeepReference)) { 185 | ref = new MorphReference(ref, attr ? attr.alias : null); 186 | } else { 187 | throw new Error(`Unknown type of reference: ${ref}`); 188 | } 189 | } 190 | 191 | return ref; 192 | } 193 | 194 | /** 195 | * Checks the `Reference` to store in this document, 196 | * properly handling polymorphic references. 197 | * 198 | * @param otherRef The reference to the other end 199 | */ 200 | private _makeRefToOther(otherRef: Reference | null | undefined, thisAttr: RelationAttrInfo): Reference | null { 201 | if (otherRef) { 202 | if (thisAttr.isMorph && !(otherRef instanceof MorphReference)) { 203 | // The reference would have been coerced to an instance of MorphReference 204 | // only if it was an object with the required info 205 | throw new Error('Polymorphic reference does not have the required information'); 206 | } else { 207 | return otherRef; 208 | } 209 | } 210 | return null; 211 | } 212 | 213 | private _setThis(data: T | undefined, { alias, isArray }: RelationAttrInfo, value: any) { 214 | if (data) { 215 | if (isArray) { 216 | const val = value ? _.castArray(value) : []; 217 | _.set(data, alias, val); 218 | } else { 219 | const val = value ? (Array.isArray(value) ? value[0] || null : value) : null; 220 | _.set(data, alias, val); 221 | } 222 | } 223 | } 224 | 225 | private async _setAllRelated(refs: { info: RefInfo, set: boolean }[], thisRef: Reference, transaction: Transaction) { 226 | refs = refs.filter(r => r.info.attr); 227 | 228 | // Batch-get all the references that we need to fetch 229 | // I.e. the ones inside component arrays that required manual manipulation 230 | const toGet: Reference[] = []; 231 | const infos = new Array<{ attr: RelationAttrInfo, ref: Reference, set: boolean, thisRefValue: Reference | undefined, snapIndex?: number } | undefined>(refs.length); 232 | for (let i = 0; i < refs.length; i++) { 233 | const { info, set } = refs[i]; 234 | // Filter to those that have a dominant other end 235 | if (info.attr) { 236 | infos[i] = { 237 | attr: info.attr, 238 | ref: info.ref, 239 | thisRefValue: info.thisRefValue, 240 | set, 241 | }; 242 | // Set aside to fetch this relation 243 | if (info.attr.isMeta) { 244 | infos[i]!.snapIndex = toGet.length; 245 | toGet.push(info.ref); 246 | } 247 | } 248 | } 249 | 250 | const snaps = toGet.length ? await transaction.getAtomic(toGet) : []; 251 | 252 | // Perform all the write operations on the relations 253 | await Promise.all( 254 | infos.map(async info => { 255 | if (info) { 256 | const data = info.snapIndex !== undefined ? snaps[info.snapIndex].data() : undefined; 257 | const thisRefValue = info.thisRefValue || this._makeRefToThis(thisRef, info.attr); 258 | await this._setRelated(info.ref, info.attr, data, thisRefValue, info.set, transaction) 259 | } 260 | }) 261 | ); 262 | } 263 | 264 | private async _setRelated(ref: Reference, attr: RelationAttrInfo, prevData: R | undefined, thisRefValue: Reference, set: boolean, transaction: Transaction) { 265 | const value = set 266 | ? (attr.isArray ? FieldOperation.arrayUnion(thisRefValue) : thisRefValue) 267 | : (attr.isArray ? FieldOperation.arrayRemove(thisRefValue) : null); 268 | 269 | if (attr.isMeta) { 270 | if (!prevData) { 271 | // Relation no longer exists, do not update 272 | return; 273 | } 274 | 275 | const { componentAlias, parentAlias } = attr.actualAlias!; 276 | // The attribute is a metadata map for an array of components 277 | // This requires special handling 278 | // We need to atomically fetch and process the data then update 279 | // Extract a new object with only the fields that are being updated 280 | const newData: any = {}; 281 | const components = _.get(prevData, parentAlias); 282 | _.set(newData, parentAlias, components); 283 | for (const component of _.castArray(components)) { 284 | if (component) { 285 | FieldOperation.apply(component, componentAlias, value); 286 | } 287 | } 288 | 289 | await transaction.update(ref, newData, { updateRelations: false }); 290 | } else { 291 | // TODO: Safely handle relations that no longer exist 292 | await transaction.update(ref, { [attr.alias]: value } as object, { updateRelations: false }); 293 | } 294 | } 295 | 296 | private async _queryRelated(ref: Reference, transaction: Transaction, atomic: boolean, otherEnds = this.otherEnds): Promise[]> { 297 | const snaps = otherEnds.map(async otherEnd => { 298 | const { model, attr, parentModels } = otherEnd; 299 | if (parentModels && parentModels.length) { 300 | // Find instances of the parent document containing 301 | // a component instance that references this 302 | return await this._queryRelated(ref, transaction, atomic, parentModels); 303 | } 304 | 305 | if (attr) { 306 | // The refValue will be coerced appropriately 307 | // by the model that is performing the query 308 | const refValue = this._makeRefToThis(ref, attr); 309 | const operator = attr.isArray ? 'array-contains' : '=='; 310 | let q = model.db.where({ field: attr.alias, operator, value: refValue }); 311 | if (model.options.maxQuerySize) { 312 | q = q.limit(model.options.maxQuerySize); 313 | } 314 | const snap = atomic 315 | ? await transaction.getAtomic(q) 316 | : await transaction.getNonAtomic(q); 317 | return snap.docs.map(d => makeRefInfo(otherEnd, d.ref, refValue)); 318 | } else { 319 | return []; 320 | } 321 | }); 322 | return (await Promise.all(snaps)).flat(); 323 | } 324 | 325 | private _getRefInfo(data: T | undefined, thisAttr: RelationAttrInfo) { 326 | return mapNotNull( 327 | _.castArray(_.get(data, thisAttr.alias) || []), 328 | v => this._getSingleRefInfo(v) 329 | ); 330 | } 331 | 332 | private _getSingleRefInfo(ref: any): RefInfo | null { 333 | let other = this._singleOtherEnd; 334 | if (ref) { 335 | if (!(ref instanceof Reference)) { 336 | throw new Error('Value is not an instance of Reference. Data must be coerced before updating relations.') 337 | } 338 | 339 | if (!other) { 340 | // Find the end which this reference relates to 341 | other = this.otherEnds.find(({ model }) => model.db.path === ref.parent.path); 342 | if (!other) { 343 | throw new StatusError( 344 | `Reference "${ref.path}" does not refer to any of the available models: ` 345 | + this.otherEnds.map(e => `"${e.model.uid}"`).join(', '), 346 | 400, 347 | ); 348 | } 349 | } 350 | 351 | return makeRefInfo(other, ref, undefined); 352 | } 353 | return null; 354 | } 355 | } 356 | 357 | 358 | 359 | interface RefInfo { 360 | ref: Reference 361 | model: FirestoreConnectorModel 362 | 363 | /** 364 | * If the snapshot was found by querying, then this is the 365 | * reference value that was used in the query. 366 | */ 367 | thisRefValue: Reference | undefined 368 | 369 | /** 370 | * The attribute info of the other end (referred to by `ref`). 371 | */ 372 | attr: RelationAttrInfo | undefined 373 | } 374 | 375 | function makeRefInfo(info: RelationInfo, ref: Reference, thisRefValue: Reference | undefined): RefInfo { 376 | return { 377 | ...info, 378 | ref, 379 | thisRefValue, 380 | }; 381 | } 382 | --------------------------------------------------------------------------------