├── test ├── init.ts ├── demo.test.ts ├── service │ └── logger.test.ts ├── memory-mongo-client.ts ├── mongodb-collection │ ├── record-query.test.ts │ ├── record-storage.test.ts │ └── util.test.ts └── bll │ └── server-ratelimit.test.ts ├── docs ├── interface.md ├── roadmap.md ├── er.md └── api.yml ├── develop ├── .gitignore ├── stop.sh └── start.sh ├── src ├── global.d.ts ├── interface │ ├── query.ts │ ├── record.ts │ ├── config.d.ts │ ├── record-storage.ts │ ├── field.ts │ ├── entity.ts │ ├── storage-engine.ts │ └── record-query.ts ├── service │ ├── async-storage.ts │ ├── mongodb.ts │ ├── jaeger.ts │ └── logger.ts ├── app.ts ├── bll │ ├── auth.ts │ ├── mongodb-collection-engine │ │ ├── record-transfer.ts │ │ ├── record-storage.ts │ │ ├── util.ts │ │ └── record-query.ts │ ├── record-auth.ts │ ├── record.ts │ ├── entity.ts │ └── server-ratelimit.ts ├── http-server │ ├── logger.ts │ ├── error-handler.ts │ └── index.ts └── api │ ├── doc.ts │ ├── version.ts │ ├── record-transfer.ts │ ├── record-count.ts │ ├── record-group.ts │ ├── table-record.ts │ └── record.ts ├── script └── mongodb │ ├── mongodb.init.sh │ ├── 2021-08-21-upsert-record-id.js │ └── mongo-collection-engine.index.js ├── .vscode └── settings.json ├── .nycrc.yml ├── .gitignore ├── config ├── test.yml └── default.yml ├── tsconfig.json ├── README.md ├── .eslintrc.yml ├── .github └── workflows │ └── node.js.yml ├── package.json └── .mocharc.yml /test/init.ts: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/interface.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/roadmap.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /develop/.gitignore: -------------------------------------------------------------------------------- 1 | mongodb -------------------------------------------------------------------------------- /src/global.d.ts: -------------------------------------------------------------------------------- 1 | import './interface/config' 2 | -------------------------------------------------------------------------------- /script/mongodb/mongodb.init.sh: -------------------------------------------------------------------------------- 1 | mongo mongo-collection-engine.index.js 2 | -------------------------------------------------------------------------------- /develop/stop.sh: -------------------------------------------------------------------------------- 1 | podman stop gem-next-mongodb 2 | podman rm -f gem-next-mongodb 3 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "coverage-gutters.coverageBaseDir": "coverage/**" 3 | } -------------------------------------------------------------------------------- /.nycrc.yml: -------------------------------------------------------------------------------- 1 | --- 2 | reporter: 3 | - lcov 4 | - html 5 | - text-summary 6 | exclude: 7 | - test 8 | -------------------------------------------------------------------------------- /src/interface/query.ts: -------------------------------------------------------------------------------- 1 | export interface QueryResult { 2 | result: T[] 3 | nextPageToken?: string 4 | } 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | .git-credentials 3 | /coverage 4 | /.nyc_output 5 | /config/local* 6 | /config/dev* 7 | /debug 8 | /lib -------------------------------------------------------------------------------- /test/demo.test.ts: -------------------------------------------------------------------------------- 1 | import 'mocha' 2 | import { strict as assert } from 'assert' 3 | 4 | describe('demo', () => { 5 | it('demo', async () => { 6 | assert.ok('demo') 7 | }) 8 | }) 9 | -------------------------------------------------------------------------------- /develop/start.sh: -------------------------------------------------------------------------------- 1 | mkdir -p $PWD/develop/mongodb 2 | podman run -d \ 3 | --name gem-next-mongodb \ 4 | --volume $PWD/develop/mongodb:/data/db:rw \ 5 | -p 127.0.0.1:27017:27017 \ 6 | w5ccbqcv.mirror.aliyuncs.com/library/mongo:4.4.6 7 | -------------------------------------------------------------------------------- /src/service/async-storage.ts: -------------------------------------------------------------------------------- 1 | import { AsyncLocalStorage } from 'async_hooks' 2 | 3 | export const asyncLocalStorage = new AsyncLocalStorage>() 4 | export const als = asyncLocalStorage 5 | export default als 6 | -------------------------------------------------------------------------------- /config/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | PORT: 3000 3 | MONGODB: 4 | URL: 'mongodb://127.0.0.1:27017/test' 5 | SERVER_RATELIMIT: 6 | '^spaceId:000000000000000000000000-entityId:000000000000000000000000$': 5 7 | '^spaceId:[0-9a-fA-F]{24}-entityId:[0-9a-fA-F]{24}$': 10 8 | -------------------------------------------------------------------------------- /script/mongodb/2021-08-21-upsert-record-id.js: -------------------------------------------------------------------------------- 1 | /* global db print */ 2 | 3 | let count = 0 4 | 5 | db.record.find({ 6 | id: null 7 | }).forEach(record => { 8 | db.record.updateOne({ 9 | _id: record._id, 10 | }, { 11 | $set: { 12 | id: record._id.str 13 | } 14 | }) 15 | count++ 16 | }) 17 | 18 | print(`total count: ${count}`) 19 | -------------------------------------------------------------------------------- /src/app.ts: -------------------------------------------------------------------------------- 1 | import * as config from 'config' 2 | import app from './http-server' 3 | import { createLogger } from './service/logger' 4 | const logger = createLogger({ label: 'app' }) 5 | 6 | app.listen(config.PORT, () => { 7 | logger.info('http serving ...') 8 | }) 9 | 10 | process.on('unhandledRejection', (err) => { 11 | logger.error(err, 'unhandledRejection') 12 | }) 13 | -------------------------------------------------------------------------------- /src/interface/record.ts: -------------------------------------------------------------------------------- 1 | export interface RecordCustomfieldMap { 2 | [x: string]: any 3 | } 4 | 5 | export interface RecordData { 6 | id: string 7 | spaceId: string 8 | labels: string[] 9 | entityId: string 10 | updateTime: Date 11 | createTime: Date 12 | cf: RecordCustomfieldMap 13 | } 14 | 15 | 16 | export interface GroupDate { 17 | [x: string]: any 18 | } 19 | -------------------------------------------------------------------------------- /src/bll/auth.ts: -------------------------------------------------------------------------------- 1 | import { MiddlewareFn } from '@tng/koa-controller' 2 | import recordAuthBll from './record-auth' 3 | 4 | export function authMW(): MiddlewareFn { 5 | return async (ctx) => { 6 | let token = ctx.get('authorization') as string || '' 7 | token = token.replace(/^Bearer /, '') 8 | const { spaceId, entityId } = ctx.request.body as any 9 | recordAuthBll.verify({ spaceId, entityId, token }) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "ts-node": { 3 | "transpileOnly": true, 4 | "files": true 5 | }, 6 | "compilerOptions": { 7 | "target": "ESNext", 8 | "lib": ["es7"], 9 | "rootDir": "src", 10 | "outDir": "lib", 11 | "module": "commonjs", 12 | "moduleResolution": "node", 13 | "sourceMap": true, 14 | "allowJs": true, 15 | "inlineSources": true, 16 | "experimentalDecorators": true, 17 | "skipLibCheck": true, 18 | }, 19 | "include": [ 20 | "src/**/*", 21 | "@types/**/*", 22 | ], 23 | } 24 | -------------------------------------------------------------------------------- /docs/er.md: -------------------------------------------------------------------------------- 1 | 2 | ### Entity 3 | - id 4 | - name 5 | - labels 6 | - storageEngineId 7 | - spaceId 8 | - createTime 9 | - updateTime 10 | 11 | ### RecordData 12 | - id 13 | - entityId 14 | - labels 15 | - spaceId 16 | - createTime 17 | - updateTime 18 | - externalCustomfields 19 | - cf:[ID]:[TYPE] 20 | 21 | ### Field 22 | - id 23 | - entityId 24 | - labels 25 | - spaceId 26 | - name 27 | - fieldType 28 | - fieldOptions 29 | - createTime 30 | - updateTime 31 | 32 | ### StorageEngine 33 | - id 34 | - labels 35 | - data 36 | - createTime 37 | - updateTime -------------------------------------------------------------------------------- /script/mongodb/mongo-collection-engine.index.js: -------------------------------------------------------------------------------- 1 | /* global db */ 2 | 3 | // labels index 4 | db.record.createIndex({ 5 | labels: 1, 6 | }, { 7 | background: true 8 | }) 9 | 10 | // spaceId, entityId, labels compond index 11 | db.record.createIndex({ 12 | spaceId: 1, 13 | entityId: 1, 14 | labels: 1, 15 | }, { 16 | background: true 17 | }) 18 | 19 | // spaceId, entityId, id compond index 20 | db.record.createIndex({ 21 | spaceId: 1, 22 | entityId: 1, 23 | id: 1, 24 | }, { 25 | // unique: true, 26 | background: true, 27 | }) 28 | 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | General Entity Management (Next) 2 | ==== 3 | 4 | General Entity Management (2nd) or gem-next is a multi-tenant data partition service based on mongodb. 5 | 6 | ## Purpose 7 | ### why Another database service ? 8 | TODO 9 | 10 | ## Roadmap 11 | 12 | ## API 13 | - POST /api/record/create 14 | - POST /api/record/update 15 | - POST /api/record/remove 16 | - POST /api/record/query 17 | - POST /api/record/query-array 18 | 19 | ## Reference 20 | - https://www.mongodb.com/presentations/managing-multitenant-saas-applications-at-scale 21 | - https://supabase.io/ 22 | - https://fauna.com/ 23 | - https://firebase.google.com/ 24 | -------------------------------------------------------------------------------- /src/service/mongodb.ts: -------------------------------------------------------------------------------- 1 | import * as mongodb from 'mongodb' 2 | import * as config from 'config' 3 | import { createLogger } from './logger' 4 | 5 | const logger = createLogger({ label: 'mongodb' }) 6 | const options: mongodb.MongoClientOptions = { } 7 | Object.assign(options, config.MONGODB?.OPTIONS) 8 | export const client = new mongodb.MongoClient(config.MONGODB?.URL, options) 9 | 10 | client.on('error', (err) => { 11 | logger.error(err) 12 | }) 13 | 14 | client.connect().then(() => { 15 | logger.info('mongodb connection success') 16 | }, err => { 17 | logger.error(err, 'mongo connection error') 18 | process.exit(1) 19 | }) 20 | 21 | export default client 22 | -------------------------------------------------------------------------------- /src/http-server/logger.ts: -------------------------------------------------------------------------------- 1 | import { Middleware } from 'koa' 2 | import { createLogger } from '../service/logger' 3 | const logger = createLogger({ label: 'http-request' }) 4 | 5 | export function loggerMW(): Middleware { 6 | return async (ctx, next) => { 7 | const start = Date.now() 8 | try { 9 | await next() 10 | } finally { 11 | if (!ctx.skipLogger) { 12 | logger.info({ 13 | status: ctx.status, 14 | method: ctx.method, 15 | duration: Date.now() - start, 16 | url: ctx.originalUrl, 17 | userAgent: ctx.get('user-agent'), 18 | xRequestId: ctx.response.get('x-request-id'), 19 | }) 20 | } 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/http-server/error-handler.ts: -------------------------------------------------------------------------------- 1 | import { Middleware } from 'koa' 2 | import { createLogger } from '../service/logger' 3 | const logger = createLogger({ label: 'http-error' }) 4 | 5 | export function errorHandlerMW(): Middleware { 6 | return async (ctx, next) => { 7 | try { 8 | await next() 9 | } catch (e) { 10 | const status = ctx.status = e.status || 500 11 | ctx.body = { 12 | error: e.message 13 | } 14 | if (status >= 500) { 15 | logger.error(Object.assign(e, { 16 | method: ctx.method, 17 | url: ctx.url, 18 | headers: ctx.headers, 19 | reqBody: JSON.stringify(ctx.request.body), 20 | })) 21 | } 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /src/api/doc.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs' 2 | import * as path from 'path' 3 | import * as yaml from 'js-yaml' 4 | import * as config from 'config' 5 | import { before, controller, get } from '@tng/koa-controller' 6 | import * as createHttpError from 'http-errors' 7 | 8 | let openapi: any 9 | 10 | @controller('/doc') 11 | @before(async (ctx) => { 12 | if (config.DISABLE_DOC) throw createHttpError(403) 13 | }) 14 | @before(async (ctx) => { 15 | if (ctx.get('origin')) { 16 | ctx.set('Access-Control-Allow-Origin', '*') 17 | } 18 | }) 19 | export class VersionAPI { 20 | @get('/') 21 | async doc() { 22 | if (!openapi) { 23 | openapi = yaml.load(fs.readFileSync(path.resolve(__dirname, '../../docs/api.yml'), {encoding: 'utf-8'})) 24 | } 25 | return openapi 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/interface/config.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'config' { 2 | interface Config { 3 | HOST: string 4 | PORT: number 5 | DISABLE_DOC: boolean 6 | JWTKEYS: string[] 7 | MONGODB: { 8 | URL: string 9 | OPTIONS: { [x: string]: any } 10 | VERSION?: string 11 | } 12 | LOGGERS: { 13 | base: Record 14 | MONGODB: Record 15 | HTTP: Record 16 | [x: string]: Record 17 | } 18 | MONGODB_QUERY_OPTIONS: { [x: string]: any } 19 | SERVER_RATELIMIT: { [x: string]: number } 20 | SERVER_RATELIMIT_RESET_INTERVAL_MS?: number 21 | SERVER_RATELIMIT_RETRY_INTERVAL_MS?: number 22 | JAEGER?: { 23 | RATE?: number 24 | ENDPOINT?: string 25 | } 26 | } 27 | 28 | const config: Config 29 | export = config 30 | } 31 | -------------------------------------------------------------------------------- /.eslintrc.yml: -------------------------------------------------------------------------------- 1 | --- 2 | env: 3 | node: true 4 | es6: true 5 | es2017: true 6 | plugins: 7 | - '@typescript-eslint' 8 | parser: '@typescript-eslint/parser' 9 | extends: 10 | - eslint:recommended 11 | - plugin:@typescript-eslint/eslint-recommended 12 | - plugin:@typescript-eslint/recommended 13 | rules: 14 | semi: [error, never] 15 | require-yield: [off] 16 | no-unused-vars: [off] 17 | dot-notation: warn 18 | comma-dangle: [off, always] 19 | array-bracket-spacing: [off] 20 | object-curly-spacing: [off] 21 | no-multiple-empty-lines: [warn] 22 | func-call-spacing: [error, never] 23 | space-before-function-paren: [off, never] 24 | '@typescript-eslint/no-var-requires': [off] 25 | '@typescript-eslint/no-this-alias': [off] 26 | '@typescript-eslint/explicit-module-boundary-types': [off] 27 | '@typescript-eslint/no-empty-interface': [off] 28 | # padded-blocks: [error] -------------------------------------------------------------------------------- /config/default.yml: -------------------------------------------------------------------------------- 1 | --- 2 | PORT: 3000 3 | MONGODB: 4 | URL: 'mongodb://127.0.0.1:27017/test' 5 | OPTIONS: 6 | readPreference: secondaryPreferred 7 | 8 | # for production 9 | # MONGODB_QUERY_OPTIONS: 10 | # maxTimeMs: 10000 # mongodb 查询超时时间 11 | # maxResultWindow: 10000 # 最大查询数的限制 12 | # SERVER_RATELIMIT: 13 | # '^spaceId:[0-9a-fA-F]{24}-entityId:[0-9a-fA-F]{24}-RecordGroupAPI/group$': 1000 14 | # '^spaceId:[0-9a-fA-F]{24}-entityId:[0-9a-fA-F]{24}-RecordCountAPI/count': 1000 15 | # '^spaceId:[0-9a-fA-F]{24}-entityId:[0-9a-fA-F]{24}-RecordAPI/queryArray': 1000 16 | # '^spaceId:[0-9a-fA-F]{24}-entityId:[0-9a-fA-F]{24}$': 1000 # spaceId + entityId 支持正则表达式 17 | # 单机限流时间 18 | # SERVER_RATELIMIT_RESET_INTERVAL_MS: 20 19 | # 单机限流重试时间 20 | # SERVER_RATELIMIT_RETRY_INTERVAL_MS: 10 21 | 22 | LOGGERS: 23 | base: 24 | level: info 25 | transport: 26 | target: pino-pretty 27 | -------------------------------------------------------------------------------- /src/service/jaeger.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs' 2 | import * as path from 'path' 3 | import * as config from 'config' 4 | import { initTracer, ZipkinB3TextMapCodec } from 'jaeger-client' 5 | 6 | const pkg = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../../', 'package.json'), { encoding: 'utf-8' })) 7 | 8 | export const tracer = initTracer({ 9 | serviceName: pkg.name, 10 | sampler: { 11 | type: 'probabilistic', // 百分比 12 | param: config.JAEGER?.RATE || 0.01, 13 | }, 14 | reporter: { 15 | collectorEndpoint: config.JAEGER?.ENDPOINT, 16 | }, 17 | }, { 18 | tags: { 19 | version: pkg.version 20 | }, 21 | logger: console, 22 | }) 23 | 24 | const zipkinCodec = new ZipkinB3TextMapCodec({ urlEncoding: true }) 25 | tracer.registerInjector('ZIPKIN_HTTP_HEADERS', zipkinCodec) 26 | tracer.registerExtractor('ZIPKIN_HTTP_HEADERS', zipkinCodec) 27 | 28 | export default tracer 29 | -------------------------------------------------------------------------------- /src/interface/record-storage.ts: -------------------------------------------------------------------------------- 1 | import { QueryResult } from './query' 2 | import { RecordCustomfieldMap, RecordData } from './record' 3 | 4 | export interface CreateRecord { 5 | id?: string 6 | spaceId: string 7 | entityId: string 8 | labels?: string[] 9 | cf: RecordCustomfieldMap 10 | } 11 | 12 | export interface UpdateRecord { 13 | id: string 14 | spaceId: string 15 | entityId: string 16 | update: any 17 | removeLabels?: string[] 18 | setLabels?: string[] 19 | addLabels?: string[] 20 | options?: any 21 | } 22 | 23 | export interface RemoveRecord { 24 | id: string 25 | spaceId: string 26 | entityId: string 27 | } 28 | 29 | // export interface ListRecord { 30 | // spaceId: string 31 | // entityId?: string 32 | // } 33 | 34 | export interface RecordStorageBll { 35 | create(createRecord: CreateRecord): Promise 36 | // list(listRecord: ListRecord): Promise> 37 | update(updateRecord: UpdateRecord): Promise 38 | remove(removeRecord: RemoveRecord): Promise 39 | } 40 | -------------------------------------------------------------------------------- /src/api/version.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs' 2 | import * as path from 'path' 3 | import { controller, get, before } from '@tng/koa-controller' 4 | import { createLogger } from '../service/logger' 5 | const logger = createLogger({ label: 'version' }) 6 | 7 | const version = { 8 | name: 'unknown', 9 | version: 'unknown', 10 | startTime: new Date(), 11 | buildTime: new Date(), 12 | commit: 'unknown', 13 | } 14 | 15 | // generate version 16 | const pkg = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../../package.json'), {encoding: 'utf-8'})) 17 | version.name = pkg.name 18 | version.version = pkg.version 19 | 20 | try { 21 | const buildInfo = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../../version.json'), {encoding: 'utf-8'})) 22 | version.buildTime = new Date(buildInfo.TIME) 23 | version.commit = buildInfo.COMMIT || 'unknown' 24 | } catch (e) { 25 | logger.warn('invaild version.json for /version: ' + e.message) 26 | } 27 | 28 | @controller('/version') 29 | @before(async ctx => { 30 | ctx.skipLogger = true 31 | }) 32 | export class VersionAPI { 33 | @get('/') 34 | async version() { 35 | return version 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/interface/field.ts: -------------------------------------------------------------------------------- 1 | import { QueryResult } from './query' 2 | 3 | export interface Field { 4 | id: string 5 | spaceId: string 6 | labels: string[] 7 | entityId: string 8 | name: string 9 | fieldType: string 10 | fieldOptions: any 11 | updateTime: Date 12 | createTime: Date 13 | } 14 | 15 | export interface CreateField { 16 | name: string 17 | spaceId: string 18 | entityId: string 19 | labels?: string[] 20 | } 21 | 22 | export interface UpdateField { 23 | id: string 24 | spaceId: string 25 | entityId: string 26 | name?: string 27 | removeLabels?: string[] 28 | setLabels?: string[] 29 | addLabels?: string[] 30 | fieldOptions?: any 31 | } 32 | 33 | export interface RemoveField { 34 | id: string 35 | spaceId: string 36 | entityId: string 37 | } 38 | 39 | export interface ListField { 40 | spaceId: string 41 | entityId?: string 42 | limit?: number 43 | skip?: number 44 | } 45 | 46 | export interface FieldBll { 47 | create(createField: CreateField): Promise 48 | list(listField: ListField): Promise> 49 | update(updateField: UpdateField): Promise 50 | remove(removeField: RemoveField): Promise 51 | } 52 | -------------------------------------------------------------------------------- /src/interface/entity.ts: -------------------------------------------------------------------------------- 1 | import { QueryResult } from './query' 2 | 3 | export interface Entity { 4 | id: string 5 | name: string 6 | spaceId: string 7 | storageEngineId: string 8 | labels: string[] 9 | updateTime: Date 10 | createTime: Date 11 | } 12 | 13 | export type CreateEntity = Partial> & Pick 14 | // export interface CreateEntity { 15 | // name: string 16 | // spaceId: string 17 | // storageEngineId?: string 18 | // labels?: string[] 19 | // } 20 | export interface UpdateEntity { 21 | id: string 22 | spaceId: string 23 | name?: string 24 | removeLabels?: string[] 25 | setLabels?: string[] 26 | addLabels?: string[] 27 | } 28 | 29 | export interface RemoveEntity { 30 | id: string 31 | spaceId: string 32 | } 33 | 34 | export interface ListEntity { 35 | spaceId: string 36 | labels?: string[] 37 | limit?: number 38 | skip?: number 39 | } 40 | 41 | export interface EntityBll { 42 | create(createEntity: CreateEntity): Promise 43 | list(listEntity: ListEntity): Promise> 44 | update(updateEntity: UpdateEntity): Promise 45 | remove(removeEntity: RemoveEntity): Promise 46 | } 47 | -------------------------------------------------------------------------------- /src/http-server/index.ts: -------------------------------------------------------------------------------- 1 | import * as Koa from 'koa' 2 | import * as path from 'path' 3 | import * as http from 'http' 4 | import * as koaBody from 'koa-bodyparser' 5 | import * as createError from 'http-errors' 6 | import { randomUUID } from 'crypto' 7 | import { getRouterSync, traceMW, alsMW } from '@tng/koa-controller' 8 | import { loggerMW } from './logger' 9 | import { errorHandlerMW } from './error-handler' 10 | import { createLogger } from '../service/logger' 11 | import { als } from '../service/async-storage' 12 | import { tracer } from '../service/jaeger' 13 | 14 | export const app = new Koa() 15 | app.use(alsMW(als)) 16 | app.use((ctx, next) => { 17 | ctx.response.set('x-request-id', ctx.get('x-request-id') || randomUUID()) 18 | return next() 19 | }) 20 | app.use(traceMW(tracer, { als })) 21 | app.use(koaBody()) 22 | app.use(loggerMW()) 23 | app.use(errorHandlerMW()) 24 | app.use(getRouterSync({ 25 | files: path.resolve(__dirname, '../api/**/*.[jt]s'), 26 | logger: createLogger({ label: 'http-router' }), 27 | }).routes()) 28 | 29 | app.use((ctx) => { 30 | if (!ctx.matched.length) throw createError(404, 'api not found') 31 | }) 32 | 33 | export const httpServer = http.createServer(app.callback()) 34 | export default httpServer 35 | -------------------------------------------------------------------------------- /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Node.js CI 5 | 6 | on: 7 | push: 8 | branches: [ master, dev ] 9 | pull_request: 10 | branches: [ master, dev ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | matrix: 19 | node-version: [20.x] 20 | # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ 21 | 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | - uses: pnpm/action-setup@v4 26 | with: 27 | version: 8 28 | - name: Use Node.js ${{ matrix.node-version }} 29 | uses: actions/setup-node@v2 30 | with: 31 | node-version: ${{ matrix.node-version }} 32 | cache: 'pnpm' 33 | - name: Install dependencies 34 | run: pnpm install 35 | - run: pnpm run build 36 | - run: pnpm test 37 | - run: pnpm run test:coverage 38 | - name: Codecov 39 | uses: codecov/codecov-action@v2 40 | with: 41 | token: ${{ secrets.CODECOV_TOKEN }} 42 | flags: unittests 43 | -------------------------------------------------------------------------------- /src/interface/storage-engine.ts: -------------------------------------------------------------------------------- 1 | // import { QueryResult } from './query' 2 | 3 | export interface StorageEngineData { 4 | engine: string 5 | version: string 6 | partitionType: string // data partition, collection partition, database partition 7 | engineClass: string 8 | } 9 | 10 | export interface StorageEngine { 11 | id: string 12 | labels: string[] 13 | data: StorageEngineData 14 | updateTime: Date 15 | createTime: Date 16 | } 17 | 18 | export interface CreateStorageEngine { 19 | data: any 20 | labels?: string[] 21 | } 22 | 23 | // export interface UpdateStorageEngine { 24 | // id: string 25 | // spaceId: string 26 | // name?: string 27 | // removeLabels?: string[] 28 | // setLabels?: string[] 29 | // addLabels?: string[] 30 | // } 31 | 32 | export interface RemoveStorageEngine { 33 | id: string 34 | spaceId: string 35 | } 36 | 37 | export interface FindStorageEngine { 38 | labels?: string[] 39 | } 40 | 41 | export interface StorageEngineBll { 42 | findOne(findStorageEngine: FindStorageEngine): Promise 43 | // create(createStorageEngine: CreateStorageEngine): Promise 44 | // list(listStorageEngine: ListStorageEngine): Promise> 45 | // update(updateStorageEngine: UpdateStorageEngine): Promise 46 | // remove(removeStorageEngine: RemoveStorageEngine): Promise 47 | } 48 | -------------------------------------------------------------------------------- /src/bll/mongodb-collection-engine/record-transfer.ts: -------------------------------------------------------------------------------- 1 | import { Collection as MongodbCollection, MongoClient, Db as MongodbDatabase } from 'mongodb' 2 | import dbClient from '../../service/mongodb' 3 | 4 | export class MongodbCollectionRecordTransferBllImpl { 5 | private dbClient: MongoClient 6 | constructor(options: { dbClient?: MongoClient } = {}) { 7 | this.dbClient = options.dbClient || dbClient 8 | } 9 | 10 | private get db(): MongodbDatabase { 11 | return this.dbClient.db() 12 | } 13 | 14 | private get collection(): MongodbCollection { 15 | return this.db.collection('record') 16 | } 17 | 18 | /** 19 | * @deprecated Not Recommend to trasfer entity to another 20 | * @param opt 21 | * @returns 22 | */ 23 | async transfer(opt: { 24 | id?: string 25 | sourceEntityId: string 26 | sourceSpaceId: string 27 | targetEntityId: string 28 | targetSpaceId: string 29 | }): Promise { 30 | const conds: any = { 31 | entityId: opt.sourceEntityId, 32 | spaceId: opt.sourceSpaceId, 33 | } 34 | if (opt.id) { conds.id = opt.id } 35 | const resp = await this.collection.updateMany(conds, { 36 | $set: { 37 | entityId: opt.targetEntityId, 38 | spaceId: opt.targetSpaceId, 39 | } 40 | }) 41 | return resp.modifiedCount 42 | } 43 | } 44 | 45 | export default new MongodbCollectionRecordTransferBllImpl() 46 | -------------------------------------------------------------------------------- /test/service/logger.test.ts: -------------------------------------------------------------------------------- 1 | import 'mocha' 2 | import { strict as assert } from 'assert' 3 | import * as path from 'path' 4 | import * as fs from 'fs' 5 | import { createLogger } from '../../src/service/logger' 6 | 7 | const logfilepath = path.resolve(__dirname, 'tmp.log') 8 | 9 | describe('createLogger', () => { 10 | afterEach(async () => { 11 | try { 12 | fs.rmSync(logfilepath) 13 | } catch (err) { 14 | // ignore error 15 | } 16 | }) 17 | 18 | it('createLogger()', async () => { 19 | const logger = createLogger({ 20 | label: 'this is label', 21 | options: { 22 | transport: { 23 | target: 'pino/file', 24 | options: { destination: path.resolve(__dirname, 'tmp.log') }, 25 | } 26 | } 27 | }) 28 | const loggerTime = Date.now() 29 | logger.info({key: 'value'}, 'this is message') 30 | const waitTime = 300 31 | await new Promise(resolve => setTimeout(() => resolve(1), waitTime)) 32 | const resultString = fs.readFileSync(logfilepath, { encoding: 'utf-8' }) 33 | const result = JSON.parse(resultString) 34 | 35 | assert.ok(new Date(result.time).getTime() - loggerTime < 10, 'log time difference should less than 10ms') 36 | assert.equal(result.message, 'this is message') 37 | assert.equal(result.level, 30) 38 | assert.equal(result.label, 'this is label') 39 | assert.equal(result.key, 'value') 40 | }) 41 | }) 42 | -------------------------------------------------------------------------------- /test/memory-mongo-client.ts: -------------------------------------------------------------------------------- 1 | import { Query } from 'mingo' 2 | 3 | const database = {} 4 | 5 | export function replaceCollection(name, collection) { 6 | database[name] = collection 7 | } 8 | 9 | export class MemoryMongoClient { 10 | static connect(any) { return new MemoryMongoClient() } 11 | db () { 12 | return new MemoryMongoDb() 13 | } 14 | } 15 | 16 | export class MemoryMongoDb { 17 | collection (collection: string) { 18 | return new MemoryMongoCollection({ collection }) 19 | } 20 | } 21 | 22 | export class MemoryMongoCollection { 23 | collection = '' 24 | constructor({collection}) { 25 | this.collection = collection 26 | } 27 | 28 | find (condition) { 29 | const query = new Query(condition) 30 | const result = query.find(database[this.collection] || []) 31 | Object.assign(result, { stream: () => result }) 32 | return result 33 | // return new MemoryMongoCursor({ collection: this.collection, condition }) 34 | } 35 | } 36 | 37 | export class MongoClient extends MemoryMongoClient {} 38 | 39 | export class MemoryMongoCursor { 40 | start = 0 41 | end = 0 42 | collection = '' 43 | condition = '' 44 | 45 | constructor({ collection, condition }) { 46 | 47 | } 48 | skip(n: number) { 49 | this.start = n 50 | } 51 | 52 | limit(n: number) { 53 | this.end = n 54 | } 55 | 56 | stream() { 57 | return this 58 | } 59 | 60 | // [Symbol.asyncIterator] () { 61 | // // database[] 62 | // } 63 | } -------------------------------------------------------------------------------- /test/mongodb-collection/record-query.test.ts: -------------------------------------------------------------------------------- 1 | import 'mocha' 2 | // import _ from 'lodash' 3 | // import * as MemoryMongoDB from 'mongo-mock' 4 | import * as MemoryMongoDB from '../memory-mongo-client' 5 | import { strict as assert } from 'assert' 6 | import { MongodbCollectionRecordQueryBllImpl } from '../../src/bll/mongodb-collection-engine/record-query' 7 | 8 | describe('mongodb-collection-engine/record-query', () => { 9 | beforeEach(() => { 10 | MemoryMongoDB.replaceCollection('record', []) 11 | }) 12 | it('record-query/query()', async () => { 13 | const { MongoClient } = MemoryMongoDB 14 | const client: any = await MongoClient.connect('mongodb://localhost:27017/unitest') 15 | const bll = new MongodbCollectionRecordQueryBllImpl({ 16 | dbClient: client, 17 | }) 18 | 19 | MemoryMongoDB.replaceCollection('record', [ 20 | { entityId: '2', spaceId: '1', id: 'id-1', createTime: new Date(), updateTime: new Date() }, 21 | { entityId: '2', spaceId: '1', id: 'id-2', createTime: new Date(), updateTime: new Date() }, 22 | { entityId: '2', spaceId: '1', id: 'id-3', createTime: new Date(), updateTime: new Date() }, 23 | ]) 24 | 25 | const cursor = await bll.query({ 26 | entityId: '2', 27 | spaceId: '1', 28 | filter: {}, 29 | sort: { id: -1, created: 1 }, 30 | limit: 10, 31 | skip: 1, 32 | }) 33 | 34 | const result = [] 35 | for await (const doc of cursor) { 36 | result.push(doc) 37 | } 38 | assert.equal(result.length, 2) 39 | }) 40 | }) 41 | -------------------------------------------------------------------------------- /src/bll/record-auth.ts: -------------------------------------------------------------------------------- 1 | import * as Debug from 'debug' 2 | import * as config from 'config' 3 | import * as jwt from 'jsonwebtoken' 4 | import { strict as assert } from 'assert' 5 | 6 | const debug = Debug('recordAuthBll') 7 | 8 | export class RecordAuthBll { 9 | sign({ spaceId, entityId, expireSeconds = 300 }) { 10 | return jwt.sign({ 11 | spaceId, 12 | entityId, 13 | }, config.JWTKEYS[0], { 14 | expiresIn: expireSeconds 15 | }) 16 | } 17 | 18 | verify({ spaceId, entityId, token }) { 19 | debug('verify', spaceId, entityId, token) 20 | try { 21 | let claim: {[x: string]: any} 22 | for (const key of config.JWTKEYS || []) { 23 | claim = jwt.verify(token, key) as {[x: string]: any} 24 | } 25 | debug('verify claim', claim) 26 | assert.ok(claim.spaceId && claim.entityId) 27 | const spaceIdRegExps = this.generateRegExp(claim.spaceId) 28 | const entityIdRegExps = this.generateRegExp(claim.entityId) 29 | assert.ok(spaceIdRegExps.some(regex => regex.test(spaceId))) 30 | assert.ok(entityIdRegExps.some(regex => regex.test(entityId))) 31 | } catch (e) { 32 | const error = new Error('invalid jwt token') 33 | Object.assign(error, { status: 403, origin: e.message }) 34 | throw error 35 | } 36 | } 37 | 38 | generateRegExp(m: string): RegExp[] { 39 | return m.split(',') 40 | .filter(v => v) 41 | .map(m => new RegExp('^' + m.replace(/\*/, '[\\w:_-]*').trim() + '$')) 42 | } 43 | } 44 | 45 | export default new RecordAuthBll() 46 | -------------------------------------------------------------------------------- /src/api/record-transfer.ts: -------------------------------------------------------------------------------- 1 | import { after, before, controller, MiddlewareFn, post, state, validateState } from '@tng/koa-controller' 2 | import { MongodbCollectionRecordTransferBllImpl } from '../bll/mongodb-collection-engine/record-transfer' 3 | import { authMW } from '../bll/auth' 4 | 5 | interface RecordTransferRequest { 6 | id?: string 7 | spaceId: string 8 | entityId: string 9 | targetSpaceId: string 10 | targetEntityId: string 11 | } 12 | // const pipelinePromise = promisify(pipeline) 13 | 14 | export function resultMW(): MiddlewareFn { 15 | return async (ctx) => { 16 | ctx.body = { result: ctx.body } 17 | } 18 | } 19 | 20 | @controller('/api/record') 21 | @state() 22 | @before(authMW()) 23 | export class RecordTransferAPI { 24 | private recordTransferBll = new MongodbCollectionRecordTransferBllImpl() 25 | 26 | @post('/transfer') 27 | @validateState({ 28 | required: ['spaceId', 'entityId', 'targetSpaceId', 'targetEntityId'], 29 | properties: { 30 | id: { type: 'string' }, 31 | spaceId: { type: 'string' }, 32 | entityId: { type: 'string' }, 33 | targetSpaceId: { type: 'string' }, 34 | targetEntityId: { type: 'string' }, 35 | } 36 | }) 37 | @after(resultMW()) 38 | async transfer({ id, spaceId: sourceSpaceId, entityId: sourceEntityId, targetSpaceId, targetEntityId, }: RecordTransferRequest) { 39 | const resp = await this.recordTransferBll.transfer({ 40 | id, 41 | sourceSpaceId, 42 | sourceEntityId, 43 | targetSpaceId, 44 | targetEntityId, 45 | }) 46 | 47 | return resp 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/api/record-count.ts: -------------------------------------------------------------------------------- 1 | import { RecordQueryBll } from '../interface/record-query' 2 | import { RecordStorageBll } from '../interface/record-storage' 3 | import { after, before, controller, MiddlewareFn, post, state, validateState } from '@tng/koa-controller' 4 | import recordBll from '../bll/record' 5 | import { authMW } from '../bll/auth' 6 | import { checkEntityRateLimitMW } from '../bll/server-ratelimit' 7 | 8 | interface RecordCountQueryRequest { 9 | spaceId: string 10 | entityId: string 11 | filter?: any 12 | options?: any 13 | } 14 | 15 | export function resultMW(): MiddlewareFn { 16 | return async (ctx) => { 17 | ctx.body = { result: ctx.body } 18 | } 19 | } 20 | 21 | @controller('/api/record') 22 | @state() 23 | @before(authMW()) 24 | export class RecordCountAPI { 25 | private recordBll: RecordStorageBll & RecordQueryBll 26 | 27 | constructor(options: { recordBll?: RecordStorageBll & RecordQueryBll } = {}) { 28 | this.recordBll = options.recordBll || recordBll 29 | } 30 | 31 | @post('/count') 32 | @validateState({ 33 | type: 'object', 34 | required: ['spaceId', 'entityId'], 35 | properties: { 36 | spaceId: { type: 'string' }, 37 | entityId: { type: 'string' }, 38 | filter: { type: 'object' }, 39 | options: { type: 'object' }, 40 | }, 41 | }) 42 | @before(checkEntityRateLimitMW()) 43 | @after(resultMW()) 44 | async count({spaceId, entityId, filter, options}: RecordCountQueryRequest) { 45 | const result = await this.recordBll.count({ spaceId, entityId, filter, options }) 46 | return result 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gem-server-next", 3 | "version": "0.0.1", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "build": "tsc", 8 | "dev": "nodemon -w src -w config -e ts,js,yml,json -x ts-node src/app", 9 | "start": "node lib/app", 10 | "test": "NODE_ENV=test mocha", 11 | "lint": "eslint src", 12 | "test:coverage": "NODE_ENV=test nyc mocha" 13 | }, 14 | "author": "", 15 | "license": "ISC", 16 | "devDependencies": { 17 | "@types/config": "^0.0.41", 18 | "@types/debug": "^4.1.7", 19 | "@types/glob": "^7.2.0", 20 | "@types/js-yaml": "^4.0.5", 21 | "@types/jsonwebtoken": "^8.5.8", 22 | "@types/koa-bodyparser": "^4.3.7", 23 | "@types/koa-router": "^7.4.4", 24 | "@types/lodash": "^4.14.182", 25 | "@types/mocha": "^9.1.1", 26 | "@types/sinon": "^10.0.12", 27 | "@typescript-eslint/eslint-plugin": "^5.30.5", 28 | "eslint": "^8.19.0", 29 | "mingo": "^6.0.6", 30 | "mocha": "^10.0.0", 31 | "nodemon": "^2.0.19", 32 | "nyc": "^15.1.0", 33 | "pino-pretty": "^8.1.0", 34 | "sinon": "^14.0.0", 35 | "ts-node": "^10.8.2", 36 | "typescript": "^4.7.4" 37 | }, 38 | "dependencies": { 39 | "@tng/koa-controller": "^1.0.0-alpha.1", 40 | "ajv": "^8.11.0", 41 | "config": "^3.3.7", 42 | "debug": "^4.3.4", 43 | "glob": "^8.0.3", 44 | "http-errors": "^2.0.0", 45 | "jaeger-client": "^3.19.0", 46 | "js-yaml": "^4.1.0", 47 | "jsonwebtoken": "^8.5.1", 48 | "koa": "^2.13.4", 49 | "koa-bodyparser": "^4.3.0", 50 | "koa-router": "^11.0.1", 51 | "lodash": "^4.17.21", 52 | "mongodb": "^4.7.0", 53 | "pino": "^8.1.0" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/interface/record-query.ts: -------------------------------------------------------------------------------- 1 | import { RecordData, GroupDate } from './record' 2 | 3 | export interface Options { 4 | maxTimeMs?: number 5 | hint?: string 6 | readPreference?: string 7 | } 8 | export interface ScanQuery { 9 | filter: {[x:string]: any} 10 | skip?: number 11 | limit?: number 12 | options?: Options 13 | } 14 | 15 | export interface RecordQuery { 16 | spaceId: string 17 | entityId: string 18 | filter: FilterType 19 | // queryEngine?: string 20 | sort?: SortType 21 | skip?: number 22 | limit?: number 23 | options?: Options 24 | } 25 | 26 | export interface RecordAggregateByPatchSort { 27 | conds: FilterType 28 | addFields?: any 29 | sort?: SortType 30 | skip?: number 31 | limit?: number 32 | options?: Options 33 | } 34 | 35 | export interface RecordGroup { 36 | spaceId: string 37 | entityId: string 38 | filter: FilterType 39 | group?: Group 40 | sort?: SortType 41 | limit?: number 42 | options?: Options 43 | } 44 | 45 | export interface Group { 46 | groupField?: string 47 | aggField?: string 48 | aggFunc?: 'sum' | 'count' | 'avg' | 'max' | 'min' 49 | } 50 | 51 | export interface Sort { 52 | [key: string]: { 53 | falseField?: string 54 | isArrField?: boolean 55 | order: 1 | -1 56 | } 57 | } 58 | 59 | export interface RecordQueryBll { 60 | scan(query: ScanQuery): Promise> 61 | query(query: RecordQuery): Promise> 62 | count?(query: RecordQuery): Promise 63 | group?(query: RecordGroup): Promise> 64 | } 65 | -------------------------------------------------------------------------------- /.mocharc.yml: -------------------------------------------------------------------------------- 1 | --- 2 | spec: 3 | # - test/**/*.test.js 4 | - test/**/*.test.ts 5 | require: 6 | - ts-node/register 7 | - test/init.ts 8 | recursive: true 9 | exit: true 10 | 11 | # This is an example Mocha config containing every Mocha option plus others. 12 | # FROM: https://github.com/mochajs/mocha/blob/master/example/config/.mocharc.yml 13 | # allow-uncaught: false 14 | # async-only: false 15 | # bail: false 16 | # check-leaks: false 17 | # color: true 18 | # delay: false 19 | # diff: true 20 | # exit: false # could be expressed as "no-exit: true" 21 | # extension: 22 | # - 'js' 23 | # # fgrep and grep are mutually exclusive 24 | # # fgrep: something 25 | # file: 26 | # - '/path/to/some/file' 27 | # - '/path/to/some/other/file' 28 | # forbid-only: false 29 | # forbid-pending: false 30 | # full-trace: false 31 | # global: 32 | # - 'jQuery' 33 | # - '$' 34 | # # fgrep and grep are mutually exclusive 35 | # # grep: something 36 | # growl: false 37 | # ignore: 38 | # - '/path/to/some/ignored/file' 39 | # inline-diffs: false 40 | # # needs to be used with grep or fgrep 41 | # # invert: false 42 | # jobs: 1 43 | # package: './package.json' 44 | # parallel: false 45 | # recursive: false 46 | # reporter: 'spec' 47 | # reporter-option: 48 | # - 'foo=bar' 49 | # - 'baz=quux' 50 | # require: '@babel/register' 51 | # retries: 1 52 | # slow: '75' 53 | # sort: false 54 | # spec: 55 | # - 'test/**/*.spec.js' # the positional arguments! 56 | # timeout: '2000' # same as "timeout: '2s'" 57 | # # timeout: false # same as "no-timeout: true" or "timeout: 0" 58 | # trace-warnings: true # node flags ok 59 | # ui: 'bdd' 60 | # v8-stack-trace-limit: 100 # V8 flags are prepended with "v8-" 61 | # watch: false 62 | # watch-files: 63 | # - 'lib/**/*.js' 64 | # - 'test/**/*.js' 65 | # watch-ignore: 66 | # - 'lib/vendor' -------------------------------------------------------------------------------- /src/service/logger.ts: -------------------------------------------------------------------------------- 1 | import pino from 'pino' 2 | import { merge } from 'lodash' 3 | import * as config from 'config' 4 | import { als } from './async-storage' 5 | 6 | // https://github.com/pinojs/pino/issues/78 7 | // pino官方不喜欢使用环境变量控制日志等级 8 | // 参考debug代码，使用 `LOGGER_DEBUG=xxx` 的方式支持环境变量增加日志的方式 9 | // 这里支持各个等级的日志等级控制，主要用于DEBUG和trace控制。 10 | // 在不改变配置文件的情况下，通过环境变量控制就可以做到 11 | const levels = ['ERROR', 'WARNING', 'INFO', 'DEBUG', 'TRACE'] 12 | const envLabelLevelMap: Record = {} 13 | levels.forEach(level => { 14 | (process.env['LOGGER_' + level] || '').split(',').filter(Boolean).forEach(label => { 15 | envLabelLevelMap[label] = level 16 | }) 17 | }) 18 | 19 | export function createLogger({ 20 | label = 'app', 21 | options, 22 | // destination = 1, 23 | }: { 24 | label?: string 25 | options?: pino.LoggerOptions 26 | // destination?: string | number | pino.DestinationObjectOptions | pino.DestinationStream | NodeJS.WritableStream 27 | } = {}) { 28 | return pino(merge({ 29 | base: { label }, 30 | messageKey: 'message', 31 | mixin () { 32 | return { traceId: als.getStore()?.traceId } 33 | }, 34 | formatters: { 35 | // 由于 pino 会把 Error Object 全部序列化到 err 这个字段 36 | // 代码中给 Error 对象添加的额外属性 也全部到 err 这个字段里了 37 | // 这样日志格式字段中如果需要统计一个类别的日志 38 | // 只能 label 字段 和 mixin() 方法加入字段，没有其他自由添加字段的方法 39 | // 所以 弃用 err 字段，直接将 Error 对象序列化之后作为日志对象 40 | // 已知可能造成的问题: 41 | // logger.error(new Error('a'), 'b') 42 | // 这样会产生2个一摸一样的 message 43 | // {"level":50, 44 | // "time":"2022-04-07T17:03:52.534Z", 45 | // "label":"app", 46 | // "type":"Error", 47 | // "message":"a", 48 | // "stack":"...", 49 | // "message":"b"} // 这里多了一个message 50 | log (obj) { 51 | if (obj?.err) { 52 | const { traceId } = obj 53 | return Object.assign(pino.stdSerializers.err(obj.err), { traceId }) 54 | } 55 | return obj 56 | } 57 | }, 58 | timestamp: pino.stdTimeFunctions.isoTime, 59 | transport: null, 60 | }, config.LOGGERS?.base, options, config.LOGGERS?.[label], { level: envLabelLevelMap[label] })) 61 | } 62 | 63 | export default createLogger 64 | -------------------------------------------------------------------------------- /src/api/record-group.ts: -------------------------------------------------------------------------------- 1 | import * as createHttpError from 'http-errors' 2 | 3 | import { RecordQueryBll } from '../interface/record-query' 4 | import { RecordStorageBll } from '../interface/record-storage' 5 | import { before, after, controller, MiddlewareFn, post, state, validateState } from '@tng/koa-controller' 6 | import recordBll from '../bll/record' 7 | import { authMW } from '../bll/auth' 8 | import { checkEntityRateLimitMW } from '../bll/server-ratelimit' 9 | import { GroupDate } from '../interface/record' 10 | import { Group } from '../interface/record-query' 11 | 12 | interface RecordGroupQueryRequest { 13 | spaceId: string 14 | entityId: string 15 | filter?: any 16 | options?: any 17 | group?: Group 18 | sort?: {[x: string]: 1 | -1 } 19 | limit?: number 20 | } 21 | 22 | export function resultMW(): MiddlewareFn { 23 | return async (ctx) => { 24 | ctx.body = { result: ctx.body } 25 | } 26 | } 27 | 28 | @controller('/api/record') 29 | @state() 30 | @before(authMW()) 31 | export class RecordGroupAPI { 32 | private recordBll: RecordStorageBll & RecordQueryBll 33 | 34 | constructor(options: { recordBll?: RecordStorageBll & RecordQueryBll } = {}) { 35 | this.recordBll = options.recordBll || recordBll 36 | } 37 | 38 | @post('/group') 39 | @validateState({ 40 | type: 'object', 41 | required: ['spaceId', 'entityId'], 42 | properties: { 43 | spaceId: { type: 'string' }, 44 | entityId: { type: 'string' }, 45 | filter: { type: 'object' }, 46 | group: { 47 | type: 'object', 48 | properties: { 49 | groupField: { type: 'string' }, 50 | aggField: { type: 'string' }, 51 | aggFunc: { type: 'string', enum: ['sum', 'count', 'avg', 'max', 'min'] } 52 | } 53 | }, 54 | sort: { type: 'object' }, 55 | limit: { type: 'number' }, 56 | options: { type: 'object' }, 57 | }, 58 | }) 59 | @before(checkEntityRateLimitMW()) 60 | @before(async (ctx) => { 61 | const { group } = ctx.request.body as any 62 | for (const key in group) { 63 | if (group[key]?.includes('$')) throw createHttpError(400, `invalid ${key}`) 64 | } 65 | }) 66 | @after(resultMW()) 67 | async group({spaceId, entityId, group, sort, limit, filter, options}: RecordGroupQueryRequest) { 68 | const resp = await this.recordBll.group({ spaceId, entityId, group, sort, limit, filter, options }) 69 | const records: GroupDate[] = [] 70 | for await (const doc of resp) { 71 | records.push(doc) 72 | } 73 | return records 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/bll/mongodb-collection-engine/record-storage.ts: -------------------------------------------------------------------------------- 1 | import { Collection as MongodbCollection, MongoClient, Db as MongodbDatabase, ObjectId, UpdateOptions } from 'mongodb' 2 | import { RecordData } from '../../interface/record' 3 | import { CreateRecord, RecordStorageBll, RemoveRecord, UpdateRecord } from '../../interface/record-storage' 4 | import dbClient from '../../service/mongodb' 5 | import { decodeBsonUpdate, decodeBsonValue, transform } from './util' 6 | 7 | export class MongodbCollectionRecordStorageBllImpl implements RecordStorageBll { 8 | private dbClient: MongoClient 9 | constructor(options: { dbClient?: MongoClient } = {}) { 10 | this.dbClient = options.dbClient || dbClient 11 | } 12 | 13 | private get db(): MongodbDatabase { 14 | return this.dbClient.db() 15 | } 16 | 17 | private get collection(): MongodbCollection { 18 | return this.db.collection('record') 19 | } 20 | 21 | async create(createRecord: CreateRecord): Promise { 22 | const labels = createRecord.labels || [] 23 | const labelSet = new Set() 24 | labels.forEach(label => labelSet.add(label)) 25 | labelSet.add(`space:${createRecord.spaceId}`) 26 | labelSet.add(`entity:${createRecord.entityId}`) 27 | 28 | const doc: Record = { 29 | id: createRecord.id || new ObjectId().toHexString(), 30 | spaceId: createRecord.spaceId, 31 | entityId: createRecord.entityId, 32 | labels: Array.from(labels), 33 | createTime: new Date(), 34 | updateTime: new Date(), 35 | } 36 | 37 | for (const cfKey in createRecord.cf) { 38 | const value = createRecord.cf[cfKey] 39 | doc['cf:' + cfKey] = decodeBsonValue(value) 40 | } 41 | const resp = await this.collection.insertOne(doc) 42 | doc._id = String(resp.insertedId) 43 | return transform(doc) 44 | } 45 | 46 | async update(updateRecord: UpdateRecord): Promise { 47 | const { id, spaceId, entityId, update, options } = updateRecord 48 | const updateOptions: UpdateOptions = { 49 | upsert: options?.upsert, 50 | } 51 | const recordUpdate = decodeBsonUpdate(update) 52 | recordUpdate.$set = recordUpdate.$set || {} 53 | recordUpdate.$set.updateTime = new Date() 54 | const resp = await this.collection.updateOne({ 55 | id: id, 56 | spaceId, 57 | entityId, 58 | }, recordUpdate, updateOptions) 59 | return resp.modifiedCount > 0 60 | } 61 | 62 | async remove(removeRecord: RemoveRecord): Promise { 63 | const { id, spaceId, entityId } = removeRecord 64 | const resp = await this.collection.deleteOne({ 65 | id, 66 | spaceId, 67 | entityId, 68 | }) 69 | return resp.deletedCount > 0 70 | } 71 | } 72 | 73 | export default new MongodbCollectionRecordStorageBllImpl() 74 | -------------------------------------------------------------------------------- /test/mongodb-collection/record-storage.test.ts: -------------------------------------------------------------------------------- 1 | import 'mocha' 2 | import { strict as assert } from 'assert' 3 | import * as sinon from 'sinon' 4 | import { MongodbCollectionRecordStorageBllImpl } from '../../src/bll/mongodb-collection-engine/record-storage' 5 | 6 | describe('mongodb-collection-engine/record-storage', () => { 7 | afterEach(() => { 8 | sinon.restore() 9 | }) 10 | 11 | it('record-storage.create()', async () => { 12 | 13 | const insertOne = sinon.fake.resolves({ insertedId: 'id-1' }) 14 | 15 | const dbClient: any = { 16 | db () { 17 | return { 18 | collection: () => ({ 19 | insertOne, 20 | }) 21 | } 22 | } 23 | } 24 | 25 | const storage = new MongodbCollectionRecordStorageBllImpl({ 26 | dbClient, 27 | }) 28 | 29 | const result = await storage.create({ 30 | entityId: '1', 31 | spaceId: '2', 32 | cf: { 33 | abc: 123 34 | } 35 | }) 36 | 37 | assert.ok(result) 38 | assert.ok(result.id) 39 | assert.equal(result.cf.abc, 123) 40 | assert.equal(insertOne.callCount, 1) 41 | assert.equal(insertOne.args[0][0]['cf:abc'], 123) 42 | }) 43 | 44 | it('record-storage.update()', async () => { 45 | 46 | const updateOne = sinon.fake.resolves({ modifiedCount: 1 }) 47 | 48 | const dbClient: any = { 49 | db () { 50 | return { 51 | collection: () => ({ 52 | updateOne, 53 | }) 54 | } 55 | } 56 | } 57 | 58 | const storage = new MongodbCollectionRecordStorageBllImpl({ 59 | dbClient, 60 | }) 61 | 62 | const result = await storage.update({ 63 | id: '123456789012345678901234', 64 | entityId: '1', 65 | spaceId: '2', 66 | update: { 67 | abc: 123 68 | } 69 | }) 70 | 71 | assert.equal(result, true) 72 | assert.equal(updateOne.args[0][1].$set['cf:abc'], 123) 73 | assert.ok(!updateOne.args[0][2].upsert) 74 | }) 75 | 76 | 77 | it('record-storage.remove()', async () => { 78 | 79 | const deleteOne = sinon.fake.resolves({ deletedCount: 1 }) 80 | 81 | const dbClient: any = { 82 | db () { 83 | return { 84 | collection: () => ({ 85 | deleteOne, 86 | }) 87 | } 88 | } 89 | } 90 | 91 | const storage = new MongodbCollectionRecordStorageBllImpl({ 92 | dbClient, 93 | }) 94 | 95 | const result = await storage.remove({ 96 | id: '123456789012345678901234', 97 | entityId: '1', 98 | spaceId: '2', 99 | }) 100 | 101 | assert.equal(result, true) 102 | assert.equal(deleteOne.args[0][0].id.toString(), '123456789012345678901234') 103 | }) 104 | }) 105 | -------------------------------------------------------------------------------- /src/bll/record.ts: -------------------------------------------------------------------------------- 1 | import { RecordData, GroupDate } from '../interface/record' 2 | import { RecordQuery, RecordQueryBll, ScanQuery } from '../interface/record-query' 3 | import { CreateRecord, RecordStorageBll, RemoveRecord, UpdateRecord } from '../interface/record-storage' 4 | import mongodbCollectionRecordStorageBll from './mongodb-collection-engine/record-storage' 5 | import mongodbCollectionRecordQueryBll from './mongodb-collection-engine/record-query' 6 | 7 | export class RecordBllImpl implements RecordStorageBll, RecordQueryBll { 8 | private recordStorageBll: RecordStorageBll 9 | private recordQueryBll: RecordQueryBll 10 | 11 | constructor(options: { recordStorageBll?: RecordStorageBll, recordQueryBll?: RecordQueryBll } = {}) { 12 | this.recordStorageBll = options.recordStorageBll || mongodbCollectionRecordStorageBll 13 | this.recordQueryBll = options.recordQueryBll || mongodbCollectionRecordQueryBll 14 | } 15 | 16 | async create(createRecord: CreateRecord): Promise { 17 | // TODO: get query.entityId and find entity 18 | // TODO: get entity.engineId and find engine 19 | // TODO: get engineClass Info get queryEngine or storageEngine 20 | return this.recordStorageBll.create(createRecord) 21 | } 22 | 23 | async update(updateRecord: UpdateRecord): Promise { 24 | // TODO: get query.entityId and find entity 25 | // TODO: get entity.engineId and find engine 26 | // TODO: get engineClass Info get queryEngine or storageEngine 27 | return this.recordStorageBll.update(updateRecord) 28 | } 29 | 30 | async remove(removeRecord: RemoveRecord): Promise { 31 | // TODO: get query.entityId and find entity 32 | // TODO: get entity.engineId and find engine 33 | // TODO: get engineClass Info get queryEngine or storageEngine 34 | return this.recordStorageBll.remove(removeRecord) 35 | } 36 | 37 | async query(query: RecordQuery): Promise> { 38 | // TODO: get query.entityId and find entity 39 | // TODO: get entity.engineId and find engine 40 | // TODO: get engineClass Info get queryEngine or storageEngine 41 | return this.recordQueryBll.query(query) 42 | } 43 | 44 | async count(query: RecordQuery): Promise { 45 | // TODO: get query.entityId and find entity 46 | // TODO: get entity.engineId and find engine 47 | // TODO: get engineClass Info get queryEngine or storageEngine 48 | return this.recordQueryBll.count(query) 49 | } 50 | 51 | async group(query: RecordQuery): Promise> { 52 | return this.recordQueryBll.group(query) 53 | } 54 | 55 | async scan(query: ScanQuery): Promise> { 56 | return this.recordQueryBll.scan(query) 57 | } 58 | } 59 | 60 | export default new RecordBllImpl() 61 | -------------------------------------------------------------------------------- /src/bll/entity.ts: -------------------------------------------------------------------------------- 1 | import { Collection as MongodbCollection, MongoClient, Db as MongodbDatabase, ObjectId } from 'mongodb' 2 | import { CreateEntity, Entity, EntityBll, ListEntity, RemoveEntity, UpdateEntity } from '../interface/entity' 3 | import { QueryResult } from '../interface/query' 4 | import dbClient from '../service/mongodb' 5 | 6 | export class EntityBllImpl implements EntityBll { 7 | private dbClient: MongoClient 8 | constructor(options: { dbClient?: MongoClient } = {}) { 9 | this.dbClient = options.dbClient || dbClient 10 | } 11 | private get db(): MongodbDatabase { 12 | return this.dbClient.db() 13 | } 14 | 15 | private get collection(): MongodbCollection & { _id?: ObjectId, isDeleted: boolean }> { 16 | return this.db.collection('entity') 17 | } 18 | 19 | private defaultStorageEngineId = '' 20 | 21 | async create(createEntity: CreateEntity): Promise { 22 | const insertDoc: Omit & { isDeleted: boolean } = { 23 | spaceId: createEntity.spaceId, 24 | name: createEntity.name, 25 | labels: createEntity.labels || [], 26 | storageEngineId: createEntity.storageEngineId || this.defaultStorageEngineId, 27 | createTime: new Date(), 28 | updateTime: new Date(), 29 | isDeleted: false, 30 | } 31 | 32 | // add default labels 33 | insertDoc.labels.push('space:' + insertDoc.spaceId) 34 | 35 | const resp = await this.collection.insertOne(insertDoc) 36 | 37 | return Object.assign({}, insertDoc, { 38 | id: String(resp.insertedId), 39 | }) 40 | } 41 | async list(listEntity: ListEntity): Promise> { 42 | const { spaceId, labels, limit } = listEntity 43 | const resp = await this.collection.find({ 44 | spaceId, 45 | labels, 46 | isDeleted: false, 47 | }).toArray() 48 | const result = resp.map(({ name, spaceId, labels, storageEngineId, createTime, updateTime, _id }) => ({ 49 | id: String(_id), name, spaceId, labels, storageEngineId, createTime, updateTime, 50 | })) 51 | return { 52 | result, 53 | nextPageToken: null 54 | } 55 | } 56 | async update(updateEntity: UpdateEntity): Promise { 57 | // TODO 58 | // const doc = await this.collection.findOneAndUpdate({}, { 59 | // $set: { 60 | 61 | // } 62 | // }, { returnDocument: 'after' }) 63 | throw new Error('Method not implemented.') 64 | } 65 | async remove(removeEntity: RemoveEntity): Promise { 66 | const resp = await this.collection.updateOne({ 67 | _id: new ObjectId(removeEntity.id), 68 | spaceId: removeEntity.spaceId, 69 | isDeleted: false, 70 | }, { 71 | $set: { 72 | updateTime: new Date(), 73 | isDeleted: true, 74 | } 75 | }) 76 | return !!resp.modifiedCount 77 | } 78 | } 79 | 80 | export default new EntityBllImpl() 81 | -------------------------------------------------------------------------------- /src/bll/server-ratelimit.ts: -------------------------------------------------------------------------------- 1 | import * as config from 'config' 2 | import { MiddlewareFn } from '@tng/koa-controller' 3 | import * as createHttpError from 'http-errors' 4 | import { createLogger } from '../service/logger' 5 | 6 | const logger = createLogger({ label: 'ratelimit' }) 7 | export class MemoryStore { 8 | hits: Record = {} 9 | resetIntervalMS: number 10 | resetTime: number 11 | 12 | resetAll () { 13 | this.hits = {} 14 | this.resetTime = Date.now() + this.resetIntervalMS 15 | } 16 | 17 | constructor ({ 18 | resetIntervalMS, 19 | }: { 20 | resetIntervalMS?: number 21 | } = {}) { 22 | this.init({ resetIntervalMS }) 23 | } 24 | 25 | init ({ 26 | resetIntervalMS = config.SERVER_RATELIMIT_RESET_INTERVAL_MS || 60 * 1000, 27 | }: { 28 | resetIntervalMS?: number 29 | } = {}) { 30 | this.resetIntervalMS = resetIntervalMS 31 | this.resetAll() 32 | } 33 | 34 | count ({ 35 | key, 36 | }) { 37 | if (Date.now() > this.resetTime) this.resetAll() 38 | return this.hits[key] || 0 39 | } 40 | 41 | incr ({ 42 | key, 43 | incr = 1, 44 | }: { 45 | key: string, 46 | incr?: number, 47 | }) { 48 | this.hits[key] = this.hits[key] + incr || incr 49 | 50 | return { 51 | count: this.hits[key], 52 | } 53 | } 54 | } 55 | 56 | export const globalMemoryStore = new MemoryStore() 57 | 58 | export function checkEntityRateLimitMW ({ memoryStore = globalMemoryStore, leftTryCount = 3, waitInterval = config.SERVER_RATELIMIT_RETRY_INTERVAL_MS || 200 }: { memoryStore?: MemoryStore, leftTryCount?: number, waitInterval?: number } = {}): MiddlewareFn { 59 | return async (ctx) => { 60 | if (!config.SERVER_RATELIMIT) return 61 | 62 | const { spaceId, entityId } = ctx.request.body as any 63 | if (!spaceId || !entityId) return 64 | 65 | let prefixKey = spaceId + '-' + entityId 66 | // implement controller 67 | if (ctx.routerName) { 68 | prefixKey = prefixKey + '-' + ctx.routerName 69 | } 70 | 71 | // 先匹配到的规则优先级高 72 | const key = Object.keys(config.SERVER_RATELIMIT).find(key => new RegExp(key).test(prefixKey)) 73 | const limitCount = config.SERVER_RATELIMIT?.[key] 74 | if (!limitCount) return 75 | 76 | const count = memoryStore.count({ key: prefixKey }) 77 | logger.debug({ prefixKey, limitCount, count }) 78 | 79 | // 正常访问 + 计数 80 | if (count < limitCount) { 81 | memoryStore.incr({ key: prefixKey }) 82 | return 83 | } 84 | 85 | // 等资源释放 86 | while (leftTryCount-- > 0) { 87 | await new Promise(resolve => setTimeout(() => resolve(1), waitInterval)) // 等 100ms 再次判断 88 | return checkEntityRateLimitMW({ leftTryCount })(ctx) 89 | } 90 | 91 | logger.warn({ message: 'ServerRequestLimit', spaceId, entityId, prefixKey, limitCount, count }) 92 | throw createHttpError(400, 'Service unavailable') 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /test/bll/server-ratelimit.test.ts: -------------------------------------------------------------------------------- 1 | import 'mocha' 2 | import { strict as assert } from 'assert' 3 | import { Context } from 'koa' 4 | import * as serverRatelimitBll from '../../src/bll/server-ratelimit' 5 | 6 | describe('serverRatelimitBll', () => { 7 | it('serverRatelimitBll.checkEntityRateLimitMW()', async () => { 8 | const mw = serverRatelimitBll.checkEntityRateLimitMW({ 9 | memoryStore: new serverRatelimitBll.MemoryStore({ resetIntervalMS: 20 }), 10 | waitInterval: 1 11 | }) 12 | 13 | let i 14 | let throwErr = false 15 | for (i = 0; i < 20; i++) { 16 | try { 17 | await mw({ 18 | request: { 19 | body: { 20 | spaceId: 'spaceId:000000000000000000000001', 21 | entityId: 'entityId:000000000000000000000001' 22 | }, 23 | } 24 | } as Context) 25 | } catch (err) { 26 | throwErr = true 27 | } 28 | } 29 | 30 | 31 | assert.deepEqual(i >= 10, true) 32 | assert.deepEqual(throwErr, true) 33 | }) 34 | 35 | it('serverRatelimitBll.checkEntityRateLimitMW() and retry success', async () => { 36 | const mw = serverRatelimitBll.checkEntityRateLimitMW({ 37 | memoryStore: new serverRatelimitBll.MemoryStore({ resetIntervalMS: 20 }), 38 | waitInterval: 10 39 | }) 40 | 41 | for (let i = 0; i < 10; i++) { 42 | await mw({ 43 | request: { 44 | body: { 45 | spaceId: 'spaceId:000000000000000000000001', 46 | entityId: 'entityId:000000000000000000000001' 47 | }, 48 | } 49 | } as Context) 50 | } 51 | await mw({ 52 | request: { 53 | body: { 54 | spaceId: 'spaceId:000000000000000000000001', 55 | entityId: 'entityId:000000000000000000000001' 56 | }, 57 | } 58 | } as Context) 59 | }) 60 | 61 | it('serverRatelimitBll.checkEntityRateLimitMW() restore after resetInterval', async () => { 62 | const mw = serverRatelimitBll.checkEntityRateLimitMW({ 63 | memoryStore: new serverRatelimitBll.MemoryStore({ resetIntervalMS: 10 }), 64 | waitInterval: 1 65 | }) 66 | let i 67 | let throwErr = false 68 | for (i = 0; i < 10; i++) { 69 | try { 70 | await mw({ 71 | request: { 72 | body: { 73 | spaceId: 'spaceId:000000000000000000000000', 74 | entityId: 'entityId:000000000000000000000000' 75 | }, 76 | } 77 | } as Context) 78 | } catch (err) { 79 | throwErr = true 80 | } 81 | } 82 | 83 | assert.deepEqual(i >= 5, true) 84 | assert.deepEqual(throwErr, true) 85 | 86 | // wait 11ms 87 | await new Promise(resolve => setTimeout(resolve, 11)) 88 | await mw({ 89 | request: { 90 | body: { 91 | spaceId: 'spaceId:000000000000000000000000', 92 | entityId: 'entityId:000000000000000000000000' 93 | }, 94 | } 95 | } as Context) 96 | }) 97 | }) 98 | -------------------------------------------------------------------------------- /src/api/table-record.ts: -------------------------------------------------------------------------------- 1 | import { } from 'koa' 2 | import { RecordQueryBll } from '../interface/record-query' 3 | import { RecordStorageBll } from '../interface/record-storage' 4 | import { post } from '@tng/koa-controller' 5 | import recordBll from '../bll/record' 6 | 7 | interface RecordQueryRequest { 8 | spaceId: string 9 | entityId: string 10 | limit?: number 11 | skip?: number 12 | sort?: any 13 | filter?: any 14 | options?: any 15 | } 16 | 17 | interface RecordCreateRequest { 18 | spaceId: string 19 | entityId: string 20 | cf: { 21 | [x: string]: any 22 | } 23 | options?: any 24 | } 25 | 26 | interface RecordUpdateRequest { 27 | spaceId: string 28 | entityId: string 29 | id: string 30 | update: { 31 | [x: string]: any 32 | } 33 | options?: any 34 | } 35 | 36 | interface RecordRemoveRequest { 37 | spaceId: string 38 | entityId: string 39 | id: string 40 | } 41 | 42 | // @controller('/table-record') 43 | // @after(async ctx => { 44 | // // console.log('req', ctx.method, ctx.url, ctx.status, ctx.state) 45 | // }) 46 | export class TableRecordAPI { 47 | private recordBll: RecordStorageBll & RecordQueryBll 48 | 49 | constructor(options: { recordBll?: RecordStorageBll & RecordQueryBll } = {}) { 50 | this.recordBll = options.recordBll || recordBll 51 | } 52 | 53 | @post('/query') 54 | async query({ spaceId, entityId, limit = 10, skip = 0, sort, filter }: RecordQueryRequest) { 55 | const resp = await this.recordBll.query({ 56 | spaceId, 57 | entityId, 58 | limit, 59 | skip, 60 | sort, 61 | filter, 62 | }) 63 | 64 | const result = [] 65 | for await (const doc of resp) { 66 | result.push(doc) 67 | } 68 | 69 | return result 70 | } 71 | 72 | @post('/query-array') 73 | async queryArray({ spaceId, entityId, limit, skip, sort, filter }: RecordQueryRequest) { 74 | const resp = await this.recordBll.query({ 75 | spaceId, 76 | entityId, 77 | limit, 78 | skip, 79 | sort, 80 | filter, 81 | }) 82 | 83 | const result = [] 84 | for await (const doc of resp) { 85 | result.push(doc) 86 | } 87 | 88 | return result 89 | } 90 | 91 | @post('/create') 92 | async create({ spaceId, entityId, cf }: RecordCreateRequest) { 93 | const resp = await this.recordBll.create({ 94 | spaceId, 95 | entityId, 96 | cf, 97 | }) 98 | 99 | return resp 100 | } 101 | 102 | @post('/update') 103 | async update({ spaceId, entityId, id, update, options }: RecordUpdateRequest) { 104 | const resp = await this.recordBll.update({ 105 | spaceId, 106 | entityId, 107 | id, 108 | update, 109 | options, 110 | }) 111 | 112 | return resp 113 | } 114 | 115 | @post('/remove') 116 | async remove({ spaceId, entityId, id }: RecordRemoveRequest) { 117 | const resp = await this.recordBll.remove({ 118 | spaceId: spaceId, 119 | entityId: entityId, 120 | id: id, 121 | }) 122 | 123 | return resp 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /test/mongodb-collection/util.test.ts: -------------------------------------------------------------------------------- 1 | import 'mocha' 2 | // import _ from 'lodash' 3 | import { strict as assert } from 'assert' 4 | import * as mceUtil from '../../src/bll/mongodb-collection-engine/util' 5 | 6 | describe('mongodb-collection-engine/util', () => { 7 | describe('decodeBsonValue', () => { 8 | function decodeBsonValueTest (type: string, value: any, expect: any) { 9 | it(`util.decodeBsonValue(${type})`, () => { 10 | const result = mceUtil.decodeBsonValue(value) 11 | if (expect instanceof Date) { 12 | assert.deepEqual(result.getTime(), expect.getTime()) 13 | } else { 14 | assert.deepEqual(result, expect) 15 | } 16 | }) 17 | } 18 | 19 | decodeBsonValueTest('null', null, null) 20 | decodeBsonValueTest('number', 1, 1) 21 | decodeBsonValueTest('float-number', 1.1, 1.1) 22 | decodeBsonValueTest('string', 'abc', 'abc') 23 | decodeBsonValueTest('boolean:true', true, true) 24 | decodeBsonValueTest('boolean:false', false, false) 25 | decodeBsonValueTest('date', {$date: '2015-01-23T04:56:17.893Z'}, new Date('2015-01-23T04:56:17.893Z')) 26 | decodeBsonValueTest('array:empty', [], []) 27 | decodeBsonValueTest('array:multiData', ['abc', true, 123], ['abc', true, 123]) 28 | }) 29 | 30 | describe('decodeBsonQuery', () => { 31 | function decodeBsonQueryTest (type: string, query: any, expect: any) { 32 | it(`util.decodeBsonQuery(${type})`, () => { 33 | const result = mceUtil.decodeBsonQuery(query) 34 | assert.deepEqual(result, expect) 35 | }) 36 | } 37 | decodeBsonQueryTest('emptyObj', {}, {}) 38 | decodeBsonQueryTest('{key:value}', {a: 'b'}, {$and: [{'cf:a': {$eq: 'b'}}]}) 39 | decodeBsonQueryTest('{key:value, ...}', {a: 1, c: 'd', e: null}, {"$and":[ 40 | {"cf:a":{"$eq":1}}, 41 | {"cf:c":{"$eq":"d"}}, 42 | {"cf:e":{"$eq":null}}, 43 | ]}) 44 | decodeBsonQueryTest('{key:{$ne: value}}', {a: {$ne: 'b'}}, {$and: [{'cf:a': {$ne: 'b'}}]}) 45 | decodeBsonQueryTest('{key:{$gt: value}}', {a: {$gt: 'b'}}, {$and: [{'cf:a': {$gt: 'b'}}]}) 46 | decodeBsonQueryTest('{key:{$lt: value}}', {a: {$lt: 'b'}}, {$and: [{'cf:a': {$lt: 'b'}}]}) 47 | decodeBsonQueryTest('{key:{$gte: value}}', {a: {$gte: 'b'}}, {$and: [{'cf:a': {$gte: 'b'}}]}) 48 | decodeBsonQueryTest('{key:{$lte: value}}', {a: {$lte: 'b'}}, {$and: [{'cf:a': {$lte: 'b'}}]}) 49 | decodeBsonQueryTest('{key:{$in: [value]}}', {a: {$in: ['b']}}, {$and: [{'cf:a': {$in: ['b']}}]}) 50 | decodeBsonQueryTest('{key:{$nin: [value]}}', {a: {$nin: ['b']}}, {$and: [{'cf:a': {$nin: ['b']}}]}) 51 | decodeBsonQueryTest('{key:{$like: value}}', {a: {$like: 'b'}}, {$and: [{'cf:a': {$regex: 'b', '$options': 'i'}}]}) 52 | decodeBsonQueryTest('{key:{$nlike: value}}', {a: {$nlike: 'b'}}, {$and: [{'cf:a': {$not: {$regex: 'b', '$options': 'i'}}}]}) 53 | decodeBsonQueryTest('{$and: [{key:value}]}', {$and: [{a: 'b'}]}, {$and: [{$and: [{'cf:a': {$eq: 'b'}}]}]}) 54 | decodeBsonQueryTest('{$or: [{key:value}]}', {$or: [{a: 'b'}]}, {$or: [{$and: [{'cf:a': {$eq: 'b'}}]}]}) 55 | 56 | // id or date test should not use deep equal ... 57 | // decodeBsonQueryTest('{id:value}', {id: '123456789012345678901234'}, {$and: [{_id: {$eq: '123456789012345678901234'}}]}) 58 | }) 59 | 60 | describe('decodeBsonUpdate', () => { 61 | function decodeBsonUpdateTest (type: string, query: any, expect: any) { 62 | it(`util.decodeBsonUpdate(${type})`, () => { 63 | const result = mceUtil.decodeBsonUpdate(query) 64 | assert.deepEqual(result, expect) 65 | }) 66 | } 67 | decodeBsonUpdateTest('{key:value}', {a: 'b'}, {$set: {'cf:a': 'b'}}) 68 | decodeBsonUpdateTest('{key:[value]}', {a: ['b1','b2']}, {$set: {'cf:a': ['b1', 'b2']}}) 69 | decodeBsonUpdateTest('{key:{$set: value}}', {a: {$set: 'b'}}, {$set: {'cf:a': 'b'}}) 70 | decodeBsonUpdateTest('{key:{$addToSet: value}}', {a: {$addToSet: 'b'}}, {$addToSet: {'cf:a': {$each: ['b']}}}) 71 | decodeBsonUpdateTest('{key:{$pull: value}}', {a: {$pull: 'b'}}, {$pullAll: {'cf:a': ['b']}}) 72 | 73 | }) 74 | describe('transform', () => { 75 | it('transform()', () => { 76 | const result = mceUtil.transform({ 77 | id: 'abc', 78 | spaceId: 's1', 79 | entityId: 'e1', 80 | 'cf:abc': 123, 81 | 'cf:def': 'def', 82 | }) 83 | assert.ok(result) 84 | assert.equal(result.id, 'abc') 85 | assert.equal(result.spaceId, 's1') 86 | assert.equal(result.entityId, 'e1') 87 | assert.deepEqual(result.cf, { 88 | abc: 123, 89 | def: 'def', 90 | }) 91 | }) 92 | }) 93 | }) 94 | -------------------------------------------------------------------------------- /src/bll/mongodb-collection-engine/util.ts: -------------------------------------------------------------------------------- 1 | import { strict as assert } from 'assert' 2 | import { ObjectId } from 'mongodb' 3 | import * as config from 'config' 4 | import { RecordData } from '../../interface/record' 5 | 6 | const INTERNAL_KEYS = ['id', 'createTime', 'updateTime', 'spaceId', 'entityId'] 7 | const QUERY_OPS = [ 8 | '$eq', 9 | '$ne', 10 | '$gt', 11 | '$gte', 12 | '$lt', 13 | '$lte', 14 | '$in', 15 | '$nin', 16 | '$like', 17 | '$nlike', 18 | ] 19 | const MANIPULDATE_OPS = [ 20 | '$addToSet', // { $addToSet: { field: { $each: [] } } } 21 | '$pull', // { $pullAll: {field: [] } } 22 | '$set', 23 | '$unset', 24 | '$inc', 25 | ] 26 | 27 | interface TransformOptions { 28 | encodeBson?: boolean 29 | } 30 | 31 | export function transform(doc: any, options: TransformOptions = {}): RecordData { 32 | return { 33 | id: String(doc.id), 34 | spaceId: String(doc.spaceId), 35 | entityId: String(doc.entityId), 36 | labels: doc.labels || [], 37 | createTime: new Date(doc.createTime), 38 | updateTime: new Date(doc.updateTime), 39 | cf: Object.keys(doc) 40 | .filter(key => /^cf:/.test(key)) 41 | .reduce<{[x: string]: any}>((result, key) => { 42 | const cfKey = key.slice(3) // omit 'cf:' 43 | let value = doc[key] 44 | if (options.encodeBson) value = encodeBsonValue(value) 45 | Object.assign(result, {[cfKey]: value}) 46 | return result 47 | }, {}) 48 | } 49 | } 50 | 51 | export function decodeBsonValue(value: any): any { 52 | // string: 'abc' 53 | // number: 12 4.5 54 | // boolean: true, false 55 | // date: { $date: '2015-01-23T04:56:17.893Z' } 56 | // array: ['abc', 12, 4.5, false, { $date: '2015-01-23T04:56:17.893Z' }] 57 | 58 | if (value === undefined || value === null) return value 59 | if (Array.isArray(value)) return value.map(v => decodeBsonValue(v)) 60 | 61 | const tov = typeof value // type of value 62 | if (['string', 'number', 'boolean'].includes(tov)) return value 63 | 64 | const keys = Object.keys(value) 65 | const objKey = keys[0] 66 | assert.equal(keys.length, 1) 67 | // assert.equal(keys[0][0], '$') 68 | // const dataType = keys[0].slice(1) 69 | if (objKey === '$date') { 70 | value = value[objKey] 71 | assert.ok(isFinite(new Date(value).getTime())) 72 | return new Date(value) 73 | } 74 | throw new Error(`invalid action ${JSON.stringify(objKey)}`) 75 | } 76 | 77 | export function encodeBsonValue(value: any): any { 78 | if (value === undefined || value === null) return value 79 | if (Array.isArray(value)) return value.map(v => encodeBsonValue(v)) 80 | 81 | const tov = typeof value 82 | if (['string', 'number', 'boolean'].includes(tov)) return value 83 | if (value instanceof Date) { 84 | return { $date: value.toISOString() } 85 | } 86 | throw new Error(`invalid value to encode: ${JSON.stringify(value)}`) 87 | } 88 | 89 | export function decodeField(key: string) { 90 | if (INTERNAL_KEYS.includes(key)) return key 91 | return 'cf:' + key 92 | } 93 | 94 | // export function decodeObjectIdValue(value: string) { 95 | // return new ObjectId(value) 96 | // } 97 | 98 | export function decodeBsonQuery(query: Record = {}): any { 99 | // field: {$gt: {$date: '2020-01-1'}} 100 | // field: 1 101 | // field: {$eq: 1} 102 | // field: [1] 103 | // field: {$date: '2021-12-03'} 104 | // field: [{$date: '2021-12-03'}] 105 | const result: any = { } 106 | for (const key in query) { 107 | let value = query[key] 108 | if (key === '$and' || key === '$or') { 109 | assert.ok(Array.isArray(value)) 110 | const arrays = query[key].map(child => decodeBsonQuery(child)) 111 | result[key] = result[key] || [] 112 | result[key].push(...arrays) 113 | continue 114 | } 115 | 116 | let op = '$eq' 117 | if (typeof value === 'object' && !Array.isArray(value)) { 118 | const objKey = value && Object.keys(value)[0] 119 | if (QUERY_OPS.includes(objKey)) { 120 | op = objKey 121 | value = value[objKey] 122 | } 123 | } 124 | value = decodeBsonValue(value) 125 | // TODO: assert op match value type ($in/$nin should follow array) 126 | 127 | result.$and = result.$and || [] 128 | const field = decodeField(key) 129 | 130 | // special field value format for _id 131 | if (field === '_id') value = new ObjectId(value) 132 | 133 | // mongodb version strict limits 134 | if (config.MONGODB.VERSION && config.MONGODB.VERSION <= '4.0.6') { 135 | // In 4.0.6 and earlier, could use $not operator with regular expression objects (i.e. /pattern/) but not with $regex operator expressions. 136 | if (op === '$nlike') { 137 | result.$and.push({ 138 | [field]: { 139 | $not: new RegExp(value) 140 | } 141 | }) 142 | continue 143 | } 144 | } 145 | if (op === '$like') { 146 | result.$and.push({ 147 | [field]: { 148 | $regex: value, 149 | $options: 'i' 150 | } 151 | }) 152 | continue 153 | } 154 | if (op === '$nlike') { 155 | result.$and.push({ 156 | [field]: { 157 | $not: { 158 | $regex: value, 159 | $options: 'i' 160 | } 161 | } 162 | }) 163 | continue 164 | } 165 | result.$and.push({ [field]: { [op]: value } }) 166 | } 167 | 168 | return result 169 | } 170 | 171 | export function decodeBsonUpdate(cond: any): any { 172 | // key: {$unset: 1} 173 | // key: 1 174 | // key: {$date: '2021-12-03'} 175 | // key: {$addSet: 'abc'} 176 | // key: ['abc'] 177 | // key: [{$date: '2021-12-03'}] 178 | // key: {$set: ['abc']} 179 | // const result = 180 | const result: any = {} 181 | 182 | for (const key in cond) { 183 | let value = cond[key] 184 | let op = '$set' 185 | 186 | if (typeof value === 'object' && !Array.isArray(value)) { 187 | const objKey = value && Object.keys(value)[0] 188 | if (MANIPULDATE_OPS.includes(objKey)) { 189 | op = objKey 190 | value = value[objKey] 191 | } 192 | } 193 | 194 | value = decodeBsonValue(value) 195 | 196 | // addToSet and pull need a value of array 197 | if ((op === '$addToSet' || op === '$pull') && !Array.isArray(value)) { 198 | value = [value] 199 | } 200 | 201 | // mongodb need pullAll insteadof pull 202 | if (op === '$pull') { 203 | op = '$pullAll' 204 | } 205 | 206 | const one: any = result[op] = result[op] || {} 207 | const field = 'cf:' + key 208 | 209 | // addToSet need '$each' as modifier 210 | if (op === '$addToSet') { 211 | one[field] = { $each: value } 212 | } else { 213 | one[field] = value 214 | } 215 | } 216 | return result 217 | } 218 | -------------------------------------------------------------------------------- /docs/api.yml: -------------------------------------------------------------------------------- 1 | openapi: "3.0.0" 2 | info: 3 | title: General Entity Manager(Next) 4 | description: | 5 | General Entity Manager (Next) 6 | version: '0.0.1' 7 | # termsOfService: "http://swagger.io/terms/" 8 | contact: 9 | email: "orangemiwj@gmail.com" 10 | license: 11 | name: "Apache 2.0" 12 | url: "http://www.apache.org/licenses/LICENSE-2.0.html" 13 | servers: [] 14 | 15 | tags: 16 | - name: API 17 | - name: Record API 18 | 19 | paths: 20 | /api/record/create: 21 | post: 22 | summary: Create Record 23 | tags: 24 | - Record API 25 | requestBody: 26 | content: 27 | application/json: 28 | schema: 29 | type: object 30 | required: [entityId, spaceId, cf] 31 | properties: 32 | entityId: 33 | type: string 34 | spaceId: 35 | type: string 36 | cf: 37 | type: object 38 | responses: 39 | '200': 40 | description: 'Create Success' 41 | content: 42 | application/json: 43 | schema: 44 | type: object 45 | properties: 46 | result: 47 | type: object 48 | properties: 49 | id: 50 | type: string 51 | createTime: 52 | type: string 53 | format: date 54 | updateTime: 55 | type: string 56 | format: date 57 | cf: 58 | type: object 59 | /api/record/update: 60 | post: 61 | summary: Update Record 62 | tags: 63 | - Record API 64 | requestBody: 65 | content: 66 | application/json: 67 | schema: 68 | type: object 69 | required: [entityId, spaceId, cf] 70 | properties: 71 | entityId: 72 | type: string 73 | spaceId: 74 | type: string 75 | id: 76 | type: string 77 | update: 78 | type: object 79 | responses: 80 | '200': 81 | description: 'Update Record Success Or Fail' 82 | content: 83 | application/json: 84 | schema: 85 | type: object 86 | properties: 87 | result: 88 | type: boolean 89 | /api/record/remove: 90 | post: 91 | summary: Remove Record 92 | tags: 93 | - Record API 94 | requestBody: 95 | content: 96 | application/json: 97 | schema: 98 | type: object 99 | required: [entityId, spaceId, cf] 100 | properties: 101 | entityId: 102 | type: string 103 | spaceId: 104 | type: string 105 | id: 106 | type: string 107 | responses: 108 | '200': 109 | description: 'Remove Record Success Or Fail' 110 | content: 111 | application/json: 112 | schema: 113 | type: object 114 | properties: 115 | result: 116 | type: boolean 117 | /api/record/query: 118 | post: 119 | summary: Query Record in Stream Mode 120 | tags: 121 | - Record API 122 | requestBody: 123 | content: 124 | application/json: 125 | schema: 126 | type: object 127 | required: [entityId, spaceId, cf] 128 | properties: 129 | entityId: 130 | type: string 131 | spaceId: 132 | type: string 133 | filter: 134 | type: object 135 | sort: 136 | type: object 137 | skip: 138 | type: number 139 | default: 0 140 | limit: 141 | type: number 142 | default: 10 143 | disableBsonEncode: 144 | type: boolean 145 | default: false 146 | responses: 147 | '200': 148 | description: Query Record Success in JSON-lined Stream 149 | content: 150 | application/json: 151 | schema: 152 | type: object 153 | properties: 154 | id: 155 | type: string 156 | createTime: 157 | type: string 158 | format: date 159 | updateTime: 160 | type: string 161 | format: date 162 | cf: 163 | type: object 164 | 165 | /api/record/query-array: 166 | post: 167 | summary: Query Record in JSON Array Mode 168 | tags: 169 | - Record API 170 | requestBody: 171 | content: 172 | application/json: 173 | schema: 174 | type: object 175 | required: [entityId, spaceId, cf] 176 | properties: 177 | entityId: 178 | type: string 179 | spaceId: 180 | type: string 181 | filter: 182 | type: object 183 | sort: 184 | type: object 185 | skip: 186 | type: number 187 | default: 0 188 | limit: 189 | type: number 190 | default: 10 191 | disableBsonEncode: 192 | type: boolean 193 | default: false 194 | responses: 195 | '200': 196 | description: Query Record Success in JSON Array 197 | content: 198 | application/json: 199 | schema: 200 | type: object 201 | properties: 202 | result: 203 | type: array 204 | items: 205 | type: object 206 | properties: 207 | id: 208 | type: string 209 | createTime: 210 | type: string 211 | format: date 212 | updateTime: 213 | type: string 214 | format: date 215 | cf: 216 | type: object 217 | -------------------------------------------------------------------------------- /src/bll/mongodb-collection-engine/record-query.ts: -------------------------------------------------------------------------------- 1 | import * as config from 'config' 2 | import { promisify } from 'util' 3 | import { Transform as StreamTransform, pipeline } from 'stream' 4 | import { Collection as MongodbCollection, FindCursor, Db as MongodbDatabase, MongoClient, AggregationCursor, ReadPreferenceLike, CountDocumentsOptions } from 'mongodb' 5 | import { RecordData, GroupDate } from '../../interface/record' 6 | import { transform, decodeBsonQuery, decodeField } from './util' 7 | import { RecordQueryBll, RecordQuery, RecordAggregateByPatchSort, RecordGroup, Sort, ScanQuery } from '../../interface/record-query' 8 | import dbClient from '../../service/mongodb' 9 | import { createLogger } from '../../service/logger' 10 | const logger = createLogger({ label: 'mongodb-collection-engine' }) 11 | 12 | const pipelinePromise = promisify(pipeline) 13 | 14 | enum PipelineKeyEnum { 15 | $match, 16 | $addFields, 17 | $group, 18 | $project, 19 | $sort, 20 | $limit, 21 | $skip 22 | } 23 | 24 | export class MongodbCollectionRecordQueryBllImpl implements RecordQueryBll { 25 | private dbClient: MongoClient 26 | constructor(options: { dbClient?: MongoClient } = {}) { 27 | this.dbClient = options.dbClient || dbClient 28 | } 29 | 30 | private get db(): MongodbDatabase { 31 | return this.dbClient.db() 32 | } 33 | 34 | private get collection(): MongodbCollection { 35 | return this.db.collection('record') 36 | } 37 | 38 | async scan({ filter, limit, skip = 0, options = {} }: ScanQuery): Promise> { 39 | // filter transform to mongo query condition 40 | const conds = decodeBsonQuery(filter || {}) 41 | const cursor = this.collection.find(conds) 42 | 43 | if (skip) cursor.skip(skip) 44 | if (limit) cursor.limit(limit) 45 | 46 | // add time limit for query 47 | const maxTimeMs = options?.maxTimeMs || config.MONGODB_QUERY_OPTIONS?.maxTimeMs 48 | if (maxTimeMs) cursor.maxTimeMS(maxTimeMs) 49 | 50 | // add hint for query 51 | if (options?.hint) cursor.hint(options.hint) 52 | 53 | // add readPreference for query 54 | if (options?.readPreference) cursor.withReadPreference(options.readPreference as ReadPreferenceLike) 55 | 56 | // cursor transform to RecordData 57 | return this.stream(cursor) 58 | } 59 | 60 | async query({ filter, sort, spaceId, entityId, limit, skip = 0, options = {} }: RecordQuery): Promise> { 61 | // filter transform to mongo query condition 62 | let conds = decodeBsonQuery(filter || {}) 63 | conds = Object.assign({ 64 | spaceId, 65 | entityId, 66 | }, conds) 67 | 68 | const addFields = {} 69 | const singleSort = {} 70 | 71 | Object.entries(sort || {}).map(([key, value]) => { 72 | let sortField = decodeField(key) 73 | if (value.falseField) { 74 | addFields[sortField] = { 75 | $ifNull: ['$' + decodeField(key), '$' + decodeField(value.falseField)] 76 | } 77 | } else if (value.isArrField) { 78 | sortField = decodeField('sort:' + key) 79 | addFields[sortField] = { 80 | $cond: { 81 | if: { $isArray: '$' + decodeField(key) }, 82 | then: { $reduce: { 83 | input: '$' + decodeField(key), 84 | initialValue: '', 85 | in: { $concat: ["$$value", { $toString: "$$this" }] } 86 | }}, 87 | else: '$' + decodeField(key), 88 | } 89 | } 90 | } 91 | singleSort[sortField] = value.order || 1 92 | }) 93 | 94 | if (Object.keys(addFields).length) { 95 | return this.aggregateByFalseField({ conds, sort: singleSort, addFields, limit, skip, options }) 96 | } 97 | 98 | const cursor = this.collection.find(conds) 99 | 100 | // TODO: this is unsafe to use user's sort as mongo sort directly 101 | if (singleSort) cursor.sort(singleSort) 102 | if (skip) cursor.skip(skip) 103 | if (limit) cursor.limit(limit) 104 | 105 | // add time limit for query 106 | const maxTimeMs = options?.maxTimeMs || config.MONGODB_QUERY_OPTIONS?.maxTimeMs 107 | if (maxTimeMs) cursor.maxTimeMS(maxTimeMs) 108 | 109 | // add hint for query 110 | if (options?.hint) cursor.hint(options.hint) 111 | 112 | // add readPreference for query 113 | if (options?.readPreference) cursor.withReadPreference(options.readPreference as ReadPreferenceLike) 114 | 115 | // cursor transform to RecordData 116 | return this.stream(cursor) 117 | } 118 | 119 | async aggregateByFalseField ({ conds, addFields, sort, limit, skip = 0, options = {} }: RecordAggregateByPatchSort): Promise> { 120 | const pipeline: {[key in keyof typeof PipelineKeyEnum]?: any}[] = [{ 121 | $match: conds 122 | }] 123 | const aggOption = {} 124 | 125 | if (addFields) { 126 | pipeline.push({ 127 | $addFields: addFields 128 | }) 129 | } 130 | if (sort) { 131 | pipeline.push({ 132 | $sort: sort 133 | }) 134 | } 135 | if (skip) { 136 | pipeline.push({ 137 | $skip: skip 138 | }) 139 | } 140 | if (limit) { 141 | pipeline.push({ 142 | $limit: limit 143 | }) 144 | } 145 | 146 | const maxTimeMs = options?.maxTimeMs || config.MONGODB_QUERY_OPTIONS?.maxTimeMs 147 | if (maxTimeMs) { 148 | Object.assign(aggOption, { maxTimeMS: maxTimeMs }) 149 | } 150 | 151 | // add hint for query 152 | if (options?.hint) { 153 | Object.assign(aggOption, { hint: options.hint }) 154 | } 155 | 156 | const cursor = this.collection.aggregate(pipeline, aggOption) 157 | 158 | // add readPreference for aggregate 159 | if (options?.readPreference) cursor.withReadPreference(options.readPreference as ReadPreferenceLike) 160 | 161 | return this.stream(cursor) 162 | } 163 | 164 | private stream(cursor: FindCursor | AggregationCursor, transformDoc = true) { 165 | const result = new StreamTransform({ 166 | readableObjectMode: true, 167 | objectMode: true, 168 | transform(doc: any, _, cb) { 169 | cb(null, transformDoc ? transform(doc) : doc) 170 | } 171 | }) 172 | // cursor.pipe(transform) 173 | pipelinePromise(cursor.stream(), result).catch(err => { 174 | logger.error(err) 175 | result.emit('error', err) 176 | }) 177 | return result 178 | } 179 | 180 | async count({ filter, spaceId, entityId, options }: RecordQuery): Promise { 181 | let conds = decodeBsonQuery(filter || {}) 182 | conds = Object.assign({ 183 | spaceId, 184 | entityId, 185 | }, conds) 186 | 187 | const result = await this.collection.countDocuments(conds, Object.assign({}, config.MONGODB_QUERY_OPTIONS, options) as CountDocumentsOptions) 188 | return result 189 | } 190 | 191 | async group ({ spaceId, entityId, filter, group = {}, sort, limit, options }: RecordGroup): Promise> { 192 | let conds = decodeBsonQuery(filter || {}) 193 | conds = Object.assign({ 194 | spaceId, 195 | entityId, 196 | }, conds) 197 | 198 | const { groupField, aggField, aggFunc = 'sum' } = group 199 | const pipeline: {[key in keyof typeof PipelineKeyEnum]?: any}[] = [{ 200 | $match: conds 201 | }, { 202 | $group: { 203 | _id: groupField ? `$${decodeField(groupField)}` : null, 204 | [aggFunc]: { [`$${aggFunc}`]: aggField || 1 } 205 | } 206 | }, { 207 | $project: { 208 | _id: 0, 209 | [aggFunc]: 1, 210 | [groupField || 'id']: '$_id', 211 | }, 212 | }] 213 | 214 | if (sort) { 215 | pipeline.push({ 216 | $sort: sort 217 | }) 218 | } 219 | 220 | if (limit) { 221 | pipeline.push({ 222 | $limit: limit 223 | }) 224 | } 225 | 226 | const aggOption = {} 227 | 228 | // add hint for aggregate 229 | if (options?.hint) { 230 | Object.assign(aggOption, { hint: options.hint }) 231 | } 232 | 233 | const cursor = this.collection.aggregate(pipeline, aggOption) 234 | 235 | const maxTimeMs = options?.maxTimeMs || config.MONGODB_QUERY_OPTIONS?.maxTimeMs 236 | if (maxTimeMs) { 237 | cursor.maxTimeMS(maxTimeMs) 238 | } 239 | // add readPreference for aggregate 240 | if (options?.readPreference) cursor.withReadPreference(options.readPreference as ReadPreferenceLike) 241 | 242 | return this.stream(cursor, false) 243 | } 244 | } 245 | 246 | export default new MongodbCollectionRecordQueryBllImpl() 247 | -------------------------------------------------------------------------------- /src/api/record.ts: -------------------------------------------------------------------------------- 1 | import { promisify } from 'util' 2 | import { Transform, pipeline } from 'stream' 3 | import * as createHttpError from 'http-errors' 4 | import * as config from 'config' 5 | import { RecordQueryBll, ScanQuery } from '../interface/record-query' 6 | import { RecordStorageBll } from '../interface/record-storage' 7 | import { after, before, controller, MiddlewareFn, get, post, state, validateState } from '@tng/koa-controller' 8 | import recordBll from '../bll/record' 9 | import { RecordData } from '../interface/record' 10 | import { encodeBsonValue } from '../bll/mongodb-collection-engine/util' 11 | import { createLogger } from '../service/logger' 12 | import { authMW } from '../bll/auth' 13 | import { checkEntityRateLimitMW } from '../bll/server-ratelimit' 14 | 15 | 16 | const logger = createLogger({ label: 'record-api' }) 17 | const pipelinePromise = promisify(pipeline) 18 | const maxResultWindow = config.MONGODB_QUERY_OPTIONS?.maxResultWindow || 10000 19 | 20 | interface PatchSort { 21 | order: 1| -1, 22 | falseField?: string 23 | isArrField?: boolean 24 | } 25 | interface RecordQueryRequest { 26 | spaceId: string 27 | entityId: string 28 | limit?: number 29 | skip?: number 30 | sort?: {[x: string]: 1 | -1 | PatchSort } 31 | filter?: any 32 | options?: any 33 | } 34 | 35 | interface RecordCreateRequest { 36 | spaceId: string 37 | entityId: string 38 | id?: string 39 | cf: { 40 | [x: string]: any 41 | } 42 | options?: any 43 | } 44 | 45 | interface RecordUpdateRequest { 46 | spaceId: string 47 | entityId: string 48 | id: string 49 | update: { 50 | [x: string]: any 51 | } 52 | options?: any 53 | } 54 | 55 | interface RecordRemoveRequest { 56 | spaceId: string 57 | entityId: string 58 | id: string 59 | options?: any 60 | } 61 | 62 | type BatchAction = { method: string } & RecordQueryRequest & RecordCreateRequest & RecordUpdateRequest & RecordRemoveRequest 63 | 64 | interface BatchRequest { 65 | spaceId: string 66 | entityId: string 67 | actions: BatchAction[] 68 | } 69 | // const pipelinePromise = promisify(pipeline) 70 | 71 | export function resultMW(): MiddlewareFn { 72 | return async (ctx) => { 73 | ctx.body = { result: ctx.body } 74 | } 75 | } 76 | 77 | @controller('/api/record') 78 | @state() 79 | @before(authMW()) 80 | export class RecordAPI { 81 | private recordBll: RecordStorageBll & RecordQueryBll 82 | 83 | constructor(options: { recordBll?: RecordStorageBll & RecordQueryBll } = {}) { 84 | this.recordBll = options.recordBll || recordBll 85 | } 86 | 87 | @post('/query') 88 | @validateState({ 89 | type: 'object', 90 | required: ['spaceId', 'entityId'], 91 | properties: { 92 | spaceId: { type: 'string' }, 93 | entityId: { type: 'string' }, 94 | filter: { type: 'object' }, 95 | sort: { type: 'object' }, 96 | skip: { type: 'integer', minimum: 0, default: 0 }, 97 | limit: { type: 'integer', minimum: 0, default: 10 }, 98 | disableBsonEncode: { type: 'boolean', default: false }, 99 | } 100 | }) 101 | @before(async (ctx) => { 102 | const { skip, limit } = ctx.state as any 103 | if (skip + limit > maxResultWindow) { 104 | throw createHttpError(400, `Result window is too large, skip + limit must be less than or equal to:[${maxResultWindow}] but was [${skip + limit}]`) 105 | } 106 | }) 107 | @before(async (ctx) => { 108 | const { sort = {} } = ctx.state as any 109 | ctx.state.sort = Object.keys(sort).reduce((map, key) => { 110 | map[key] = { 111 | falseField: sort[key]?.falseField || null, 112 | isArrField: sort[key]?.isArrField, 113 | order: sort[key]?.order || sort[key] || 1, 114 | } 115 | return map 116 | }, {}) 117 | }) 118 | @after(async (ctx) => { 119 | if (ctx.state.disableBsonEncode) return 120 | const origin: AsyncIterable = ctx.body as any 121 | const target = new Transform({ 122 | // readableObjectMode: true, 123 | objectMode: true, 124 | transform(record: RecordData, _, cb) { 125 | record = {...record, cf: Object.keys(record.cf).reduce((cf, key) => { 126 | cf[key] = encodeBsonValue(record.cf[key]) 127 | return cf 128 | }, {})} 129 | const data = JSON.stringify(record) + '\n' 130 | cb(null, data) 131 | } 132 | }) 133 | ctx.body = target 134 | pipelinePromise(origin, target).catch(err => { 135 | logger.error(err, 'pipeline-error') 136 | }) 137 | }) 138 | async query({ spaceId, entityId, limit = 10, skip = 0, sort, filter, options }: RecordQueryRequest) { 139 | const resp = await this.recordBll.query({ 140 | spaceId, 141 | entityId, 142 | limit, 143 | skip, 144 | sort, 145 | filter, 146 | options, 147 | }) 148 | 149 | return resp 150 | } 151 | 152 | @post('/scan') 153 | @validateState({ 154 | type: 'object', 155 | required: ['filter'], 156 | properties: { 157 | filter: { type: 'object' }, 158 | skip: { type: 'integer', minimum: 0, default: 0 }, 159 | limit: { type: 'integer', minimum: 0, default: 10 }, 160 | options: { type: 'object' }, 161 | disableBsonEncode: { type: 'boolean', default: false }, 162 | } 163 | }) 164 | @before(async (ctx) => { 165 | const { skip, limit } = ctx.state as any 166 | if (skip + limit > maxResultWindow) { 167 | throw createHttpError(400, `Result window is too large, skip + limit must be less than or equal to:[${maxResultWindow}] but was [${skip + limit}]`) 168 | } 169 | }) 170 | @after(async (ctx) => { 171 | if (ctx.state.disableBsonEncode) return 172 | const records = ctx.body as RecordData[] 173 | ctx.body = records.map(record => { 174 | return {...record, cf: Object.keys(record.cf).reduce((cf, key) => { 175 | cf[key] = encodeBsonValue(record.cf[key]) 176 | return cf 177 | }, {})} 178 | }) 179 | }) 180 | @after(resultMW()) 181 | async scan({ filter, limit = 10, skip = 0, options }: ScanQuery) { 182 | const resp = await this.recordBll.scan({ 183 | filter, 184 | limit, 185 | skip, 186 | options, 187 | }) 188 | 189 | const records: RecordData[] = [] 190 | for await (const doc of resp) { 191 | records.push(doc) 192 | } 193 | return records 194 | } 195 | 196 | @post('/query-array') 197 | @validateState({ 198 | type: 'object', 199 | required: ['spaceId', 'entityId'], 200 | properties: { 201 | spaceId: { type: 'string' }, 202 | entityId: { type: 'string' }, 203 | filter: { type: 'object' }, 204 | sort: { type: 'object' }, 205 | skip: { type: 'integer', minimum: 0, default: 0 }, 206 | limit: { type: 'integer', minimum: 0, default: 10 }, 207 | options: { type: 'object' }, 208 | disableBsonEncode: { type: 'boolean', default: false }, 209 | } 210 | }) 211 | @before(checkEntityRateLimitMW()) 212 | @before(async (ctx) => { 213 | const { skip, limit } = ctx.state as any 214 | if (skip + limit > maxResultWindow) { 215 | throw createHttpError(400, `Result window is too large, skip + limit must be less than or equal to:[${maxResultWindow}] but was [${skip + limit}]`) 216 | } 217 | }) 218 | @before(async (ctx) => { 219 | const { sort = {} } = ctx.state as any 220 | ctx.state.sort = Object.keys(sort).reduce((map, key) => { 221 | map[key] = { 222 | falseField: sort[key]?.falseField || null, 223 | isArrField: sort[key]?.isArrField, 224 | order: sort[key]?.order || sort[key] || 1, 225 | } 226 | return map 227 | }, {}) 228 | }) 229 | @after(async (ctx) => { 230 | if (ctx.state.disableBsonEncode) return 231 | const records = ctx.body as RecordData[] 232 | ctx.body = records.map(record => { 233 | return {...record, cf: Object.keys(record.cf).reduce((cf, key) => { 234 | cf[key] = encodeBsonValue(record.cf[key]) 235 | return cf 236 | }, {})} 237 | }) 238 | }) 239 | @after(resultMW()) 240 | async queryArray(request: RecordQueryRequest) { 241 | const resp = await this.query(request) 242 | const records: RecordData[] = [] 243 | for await (const doc of resp) { 244 | records.push(doc) 245 | } 246 | return records 247 | } 248 | 249 | @post('/create') 250 | @validateState({ 251 | type: 'object', 252 | required: ['spaceId', 'entityId'], 253 | properties: { 254 | id: { type: 'string' }, 255 | spaceId: { type: 'string' }, 256 | entityId: { type: 'string' }, 257 | cf: { type: 'object' }, 258 | options: { type: 'object' }, 259 | } 260 | }) 261 | @after(resultMW()) 262 | async create({ spaceId, entityId, cf, id }: RecordCreateRequest) { 263 | const record = await this.recordBll.create({ 264 | id, 265 | spaceId: spaceId, 266 | entityId: entityId, 267 | cf: cf, 268 | }) 269 | 270 | return record 271 | } 272 | 273 | @post('/update') 274 | @validateState({ 275 | type: 'object', 276 | required: ['id', 'spaceId', 'entityId', 'update'], 277 | properties: { 278 | id: { type: 'string' }, 279 | spaceId: { type: 'string' }, 280 | entityId: { type: 'string' }, 281 | update: { type: 'object' }, 282 | options: { type: 'object' }, 283 | } 284 | }) 285 | @after(resultMW()) 286 | async update({ spaceId, entityId, id, update, options }: RecordUpdateRequest) { 287 | const result = await this.recordBll.update({ 288 | spaceId, 289 | entityId, 290 | id, 291 | update, 292 | options, 293 | }) 294 | 295 | return result 296 | } 297 | 298 | @post('/remove') 299 | @validateState({ 300 | type: 'object', 301 | required: ['id', 'spaceId', 'entityId'], 302 | properties: { 303 | id: { type: 'string' }, 304 | spaceId: { type: 'string' }, 305 | entityId: { type: 'string' }, 306 | options: { type: 'object' }, 307 | } 308 | }) 309 | @after(resultMW()) 310 | async remove({ spaceId, entityId, id }: RecordRemoveRequest) { 311 | const result = await this.recordBll.remove({ 312 | spaceId: spaceId, 313 | entityId: entityId, 314 | id: id, 315 | }) 316 | 317 | return result 318 | } 319 | 320 | @post('/batch') 321 | @validateState({ 322 | type: 'object', 323 | required: ['spaceId', 'entityId'], 324 | properties: { 325 | spaceId: { type: 'string' }, 326 | entityId: { type: 'string' }, 327 | actions: { 328 | type: 'array', 329 | items: { 330 | type: 'object', 331 | properties: { options: { type: 'object' } }, 332 | } 333 | } 334 | } 335 | }) 336 | @after(resultMW()) 337 | async batch({ spaceId, entityId, actions }: BatchRequest) { 338 | const result = await Promise.all(actions.map(async action => { 339 | try { 340 | switch (action.method) { 341 | case 'create': return await this.create({ ...action, entityId, spaceId }) 342 | case 'update': return await this.update({ ...action, entityId, spaceId }) 343 | case 'remove': return await this.remove({ ...action, entityId, spaceId }) 344 | case 'queryArray': return await this.queryArray({ ...action, entityId, spaceId }) 345 | default: throw new Error('invalid method: ' + action.method) 346 | } 347 | } catch (e) { 348 | return { status: e.status || 500, error: e.message } 349 | } 350 | })) 351 | return result 352 | } 353 | } 354 | --------------------------------------------------------------------------------