= { [P in keyof T]?: FieldFilter }
2 |
3 | export type FieldFilter
=
4 | | {
5 | $in?: Array
6 | $nin?: Array
7 | $eq?: T[P]
8 | // TODO: Review if the null type on $neq is necessary
9 | $neq?: T[P] | null
10 | $gt?: T[P]
11 | $gte?: T[P]
12 | $lt?: T[P]
13 | $lte?: T[P]
14 | $contains?: T[P]
15 | $arr_contains?: T[P] // Could be made conditional so it only appears where T[P] is an Array
16 | }
17 | | T[P]
18 |
19 | export type Filter = {
20 | $and?: Array>
21 | $or?: Array>
22 | } & FilterObject
23 |
--------------------------------------------------------------------------------
/src/query/abstract/FilterCompiler.ts:
--------------------------------------------------------------------------------
1 | import { Filter } from "./Filter"
2 |
3 | export type FilterCompiler = (filter: Filter) => R
4 |
--------------------------------------------------------------------------------
/src/query/abstract/Query.ts:
--------------------------------------------------------------------------------
1 | export abstract class Query {
2 | abstract execute(...args: Array): Promise
3 | }
4 |
--------------------------------------------------------------------------------
/src/query/abstract/Repository.ts:
--------------------------------------------------------------------------------
1 | import { Filter } from "./Filter"
2 |
3 | /**
4 | * A Repository represents an access mechanism for the underlying data in a store.
5 | *
6 | * It is responsible for fetching the data from a store and using it to instantiate Model objects.
7 | * Repositories in the abstract sense are not tied to any given type of datastore however classes
8 | * that extend Repository may begin to make more assertions about the nature of the underlying store.
9 | */
10 | export abstract class Repository {
11 | /**
12 | * Create a new record in the underlying data store.
13 | *
14 | * @param payload The objects to be created in the datastore
15 | */
16 | abstract async create(payload: Partial): Promise
17 |
18 | /**
19 | * Read a collection of records from the underlying data store which match the provided filter.
20 | *
21 | * @param filter A filter to select which objects should be returned in the read response
22 | */
23 | abstract async read(filter: Filter): Promise>
24 |
25 | /**
26 | * Update a collection of records in the underlying data store to the provided values where
27 | * the original record matches the filter provided.
28 | *
29 | * @param payload An payload object representing the delta that should be applied to updated records.
30 | * @param filter A filter to select which records should be updated.
31 | */
32 | abstract async update(payload: Partial, filter: Filter): Promise
33 |
34 | /**
35 | * Delete a collection of records from the underlying data store where the record matches
36 | * the filter provided.
37 | *
38 | * @param filter A filter to select which records should be deleted
39 | */
40 | abstract async delete(filter: Filter): Promise
41 | }
42 |
--------------------------------------------------------------------------------
/src/query/drivers/pg/PGQueryPostProcessor.ts:
--------------------------------------------------------------------------------
1 | export const pgQueryPostProcessor = (
2 | queryString: string,
3 | queryParameters: Array
4 | ): [string, Array] => {
5 | let parameterCount = 0
6 | let tokenizedQuery = queryString.split("")
7 | let outputQuery = ""
8 | let outputParameters = []
9 |
10 | for (let i = 0; i < tokenizedQuery.length; i++) {
11 | if (tokenizedQuery[i] === "?") {
12 | if (tokenizedQuery[i + 1] === "?") {
13 | // If the MySQL style "??" is used then pass the parameter in directly as
14 | // PostgreSQL doesn't support column parameter injection
15 | outputQuery += queryParameters[parameterCount]
16 | parameterCount++
17 | i = i + 1
18 | } else {
19 | // Add one because Postgres parameters are 1 indexed not 0
20 | outputQuery += `$${outputParameters.length + 1}`
21 | outputParameters.push(queryParameters[parameterCount])
22 | parameterCount++
23 | }
24 | } else if (tokenizedQuery[i] === "$") {
25 | // Search for the first character that isn't a number
26 | for (let j = i + 1; j < tokenizedQuery.length; j++) {
27 | if (Number.isNaN(Number(tokenizedQuery[j]))) {
28 | i = j + 1
29 | break
30 | } else if (j === tokenizedQuery.length - 1) {
31 | i = j
32 | break
33 | }
34 | }
35 |
36 | // Add the new parameter to the query
37 | // Add one because Postgres parameters are 1 indexed not 0
38 | outputQuery += `$${outputParameters.length + 1}`
39 | outputParameters.push(queryParameters[parameterCount])
40 | parameterCount++
41 | } else {
42 | outputQuery += tokenizedQuery[i]
43 | }
44 | }
45 |
46 | return [outputQuery, outputParameters]
47 | }
48 |
--------------------------------------------------------------------------------
/src/query/drivers/sql/SQLFilterCompiler.ts:
--------------------------------------------------------------------------------
1 | import { Filter, FilterCompiler } from "../.."
2 |
3 | /**
4 | * The SQL query compiler takes a standard Strontium Query and returns a SQL
5 | * query with arguments. It uses a MySQL dialect which requires Post Processing to operate
6 | * in other databases.
7 | */
8 | export const compileSQLFilter: FilterCompiler<[string, Array]> = (
9 | filter: Filter
10 | ): [string, Array] => {
11 | let queries: Array<[string, Array]> = []
12 |
13 | if (filter.$or) {
14 | let subqueries = filter.$or.map(compileSQLFilter)
15 |
16 | let orQuery = concatQueryStringsWithConjunction(subqueries, "OR")
17 |
18 | queries.push(orQuery)
19 | }
20 |
21 | if (filter.$and) {
22 | let subqueries = filter.$and.map(compileSQLFilter)
23 |
24 | let andQuery = concatQueryStringsWithConjunction(subqueries, "AND")
25 |
26 | queries.push(andQuery)
27 | }
28 |
29 | for (let field in filter) {
30 | if (field === "$or" || field === "$and") {
31 | continue
32 | }
33 |
34 | // Don't process prototype values - separated from the special
35 | // keywords for TypeScript's benefit
36 | if (!filter.hasOwnProperty(field)) {
37 | continue
38 | }
39 |
40 | let subquery = filter[field]
41 |
42 | if (subquery === undefined) {
43 | continue
44 | } else if (subquery === null) {
45 | queries.push(["?? IS NULL", [field]])
46 | } else if (subquery.$in !== undefined) {
47 | if (subquery.$in.length === 0) {
48 | // IN with an empty array typically causes an error - just make a tautological filter instead
49 | queries.push(["TRUE = FALSE", []])
50 | } else {
51 | queries.push([
52 | `?? IN (${subquery.$in.map((p: any) => "?").join(", ")})`,
53 | [field, ...subquery.$in],
54 | ])
55 | }
56 | } else if (subquery.$nin !== undefined) {
57 | if (subquery.$nin.length === 0) {
58 | queries.push(["TRUE = TRUE", []])
59 | } else {
60 | queries.push([
61 | `?? NOT IN (${subquery.$nin
62 | .map((p: any) => "?")
63 | .join(", ")})`,
64 | [field, ...subquery.$nin],
65 | ])
66 | }
67 | } else if (subquery.$neq !== undefined) {
68 | if (subquery.$neq === null) {
69 | queries.push(["?? IS NOT NULL", [field]])
70 | } else {
71 | queries.push(["?? != ?", [field, subquery.$neq]])
72 | }
73 | } else if (subquery.$gt !== undefined) {
74 | queries.push(["?? > ?", [field, subquery.$gt]])
75 | } else if (subquery.$gte !== undefined) {
76 | queries.push(["?? >= ?", [field, subquery.$gte]])
77 | } else if (subquery.$lt !== undefined) {
78 | queries.push(["?? < ?", [field, subquery.$lt]])
79 | } else if (subquery.$lte !== undefined) {
80 | queries.push(["?? <= ?", [field, subquery.$lte]])
81 | } else if (subquery.$contains !== undefined) {
82 | queries.push(["?? LIKE ?", [field, `%${subquery.$contains}%`]])
83 | } else if (subquery.$arr_contains !== undefined) {
84 | // This implementation is currently unique to PostgreSQL - We should consider how to add similar functionality to MySQL
85 | queries.push(["?? @> ?", [field, subquery.$arr_contains]])
86 | } else if (subquery.$eq !== undefined) {
87 | queries.push(["?? = ?", [field, subquery.$eq]])
88 | } else {
89 | queries.push(["?? = ?", [field, subquery]])
90 | }
91 | }
92 |
93 | // Submit the final queries AND'd together
94 | return concatQueryStringsWithConjunction(queries, "AND")
95 | }
96 |
97 | export const concatQueryStringsWithConjunction = (
98 | queries: Array<[string, Array]>,
99 | conjunction: "AND" | "OR"
100 | ): [string, Array] => {
101 | if (queries.length === 1) {
102 | return [queries[0][0], queries[0][1]]
103 | }
104 |
105 | return queries.reduce(
106 | (memo, [subqueryString, subqueryArguments], idx) => {
107 | if (idx !== 0) {
108 | memo[0] += ` ${conjunction} `
109 | }
110 |
111 | memo[0] += "("
112 | memo[0] += subqueryString
113 | memo[0] += ")"
114 |
115 | memo[1].push(...subqueryArguments)
116 |
117 | return memo
118 | },
119 | ["", []]
120 | )
121 | }
122 |
--------------------------------------------------------------------------------
/src/query/drivers/sql/TableRepository.ts:
--------------------------------------------------------------------------------
1 | import { pgQueryPostProcessor } from "../pg/PGQueryPostProcessor"
2 | import { Repository } from "../../abstract/Repository"
3 | import {
4 | MySQLStore,
5 | MySQLTransaction,
6 | PGStore,
7 | PGTransaction,
8 | SQLStore,
9 | } from "../../../datastore"
10 | import { injectable } from "inversify"
11 | import { isUndefined, omitBy } from "lodash"
12 | import { Logger } from "../../../logging"
13 |
14 | import { Filter, compileSQLFilter } from "../.."
15 |
16 | /**
17 | * A TableRepository represents a one to one mapping with an underlying SQL table.
18 | *
19 | * It is designed to provide an 80% solution for common SQL workloads with the other 20% being taken up by custom
20 | * Repository classes or direct queries.
21 | */
22 | @injectable()
23 | export abstract class TableRepository<
24 | T extends any,
25 | K extends keyof T
26 | > extends Repository {
27 | protected postProcessor: (
28 | query: string,
29 | parameters: Array
30 | ) => [string, Array] = (q, p) => [q, p]
31 |
32 | constructor(
33 | protected store: SQLStore,
34 | protected tableName: string,
35 | protected queryFields: Array,
36 | protected primaryKeyField: K,
37 | protected logger?: Logger
38 | ) {
39 | super()
40 |
41 | if (store instanceof PGStore || store instanceof PGTransaction) {
42 | this.postProcessor = pgQueryPostProcessor
43 | this.tableName = `"${this.tableName}"`
44 | }
45 | }
46 |
47 | async create(
48 | payload: Partial,
49 | connection: SQLStore = this.store
50 | ): Promise {
51 | // Generate an ID for the new record
52 | let id = await this.generateID()
53 | payload[this.primaryKeyField] = payload[this.primaryKeyField] || id
54 |
55 | // Filter the payload for any undefined keys
56 | let filteredPayload = (omitBy(payload, isUndefined) as unknown) as T
57 |
58 | if (
59 | connection instanceof MySQLStore ||
60 | connection instanceof MySQLTransaction
61 | ) {
62 | let insertQuery = `
63 | INSERT INTO
64 | ??
65 | SET
66 | ?
67 | `
68 |
69 | // This can throw a SQL error which will be returned directly to the caller rather than handled here.
70 | let result: any = await connection.query(insertQuery, [
71 | this.tableName,
72 | filteredPayload,
73 | ])
74 |
75 | return result.insertId || id
76 | } else {
77 | let query = `
78 | INSERT INTO
79 | ?? (${Object.keys(filteredPayload).map(() => `"??"`)})
80 | VALUES
81 | (${Object.keys(filteredPayload).map(() => "?")})
82 | RETURNING ??
83 | `
84 |
85 | let parameters: Array = [this.tableName]
86 |
87 | Object.keys(filteredPayload).forEach((k: string) => {
88 | parameters.push(k)
89 | })
90 |
91 | Object.keys(filteredPayload).forEach((k: string) => {
92 | parameters.push(filteredPayload[k as keyof T])
93 | })
94 |
95 | parameters.push(this.primaryKeyField)
96 |
97 | let [processedQuery, processedParameters] = this.postProcessor(
98 | query,
99 | parameters
100 | )
101 |
102 | let results = await connection.query<{ [key: string]: any }>(
103 | processedQuery,
104 | processedParameters
105 | )
106 |
107 | return results[0][this.primaryKeyField as string] || id
108 | }
109 | }
110 |
111 | async read(
112 | filter: Filter,
113 | pagination: {
114 | order?: [keyof T, "DESC" | "ASC"]
115 | limit?: number
116 | offset?: number
117 | } = {},
118 | connection: SQLStore = this.store
119 | ): Promise> {
120 | let startTime = process.hrtime()
121 | let [filterQuery, filterParameters] = compileSQLFilter(filter)
122 | let parameters = [this.tableName, ...filterParameters]
123 |
124 | let lookupQuery = `
125 | SELECT
126 | ${this.queryFields.map((f) => `"${f}"`).join(", ")}
127 | FROM
128 | ??
129 | ${filterQuery !== "" ? "WHERE" : ""}
130 | ${filterQuery}
131 | `
132 |
133 | if (pagination.order) {
134 | lookupQuery = `${lookupQuery}
135 | ORDER BY ?? ${pagination.order[1]}`
136 | parameters.push(pagination.order[0])
137 | }
138 |
139 | if (pagination.limit) {
140 | lookupQuery = `${lookupQuery}
141 | LIMIT ${pagination.limit}`
142 | }
143 |
144 | if (pagination.offset) {
145 | lookupQuery = `${lookupQuery}
146 | OFFSET ${pagination.offset}`
147 | }
148 |
149 | let [processedQuery, processedParameters] = this.postProcessor(
150 | lookupQuery,
151 | parameters
152 | )
153 | let results = await connection.query(
154 | processedQuery,
155 | processedParameters
156 | )
157 |
158 | this.recordQueryTime(processedQuery, startTime)
159 | return results
160 | }
161 |
162 | async update(
163 | payload: Partial,
164 | filter: Filter,
165 | connection: SQLStore = this.store
166 | ): Promise {
167 | // Strip the object of the undefined parameters
168 | let filteredPayload = (omitBy(payload, isUndefined) as unknown) as T
169 | let [filterQuery, filterParameters] = compileSQLFilter(filter)
170 |
171 | let lookupQuery = `
172 | UPDATE
173 | ??
174 | SET
175 | ${Object.keys(filteredPayload).map(() => "?? = ?")}
176 | ${filterQuery !== "" ? "WHERE" : ""}
177 | ${filterQuery}
178 | `
179 |
180 | let payloadParameters: Array = []
181 | Object.keys(filteredPayload).forEach((k) => {
182 | payloadParameters.push(k)
183 | payloadParameters.push(filteredPayload[k])
184 | })
185 |
186 | let [processedQuery, processedParameters] = this.postProcessor(
187 | lookupQuery,
188 | [this.tableName, ...payloadParameters, ...filterParameters]
189 | )
190 | await connection.query(processedQuery, processedParameters)
191 | }
192 |
193 | async delete(
194 | filter: Filter,
195 | connection: SQLStore = this.store
196 | ): Promise {
197 | let [filterQuery, filterParameters] = compileSQLFilter(filter)
198 | let parameters = [this.tableName, ...filterParameters]
199 |
200 | let lookupQuery = `
201 | DELETE FROM
202 | ??
203 | ${filterQuery !== "" ? "WHERE" : ""}
204 | ${filterQuery}
205 | `
206 |
207 | let [processedQuery, processedParameters] = this.postProcessor(
208 | lookupQuery,
209 | parameters
210 | )
211 | await connection.query(processedQuery, processedParameters)
212 | }
213 |
214 | async generateID(): Promise {
215 | return undefined
216 | }
217 |
218 | protected recordQueryTime(
219 | queryString: string,
220 | startTime: [number, number]
221 | ): void {
222 | if (this.logger !== undefined) {
223 | let runtime = process.hrtime(startTime)
224 |
225 | this.logger.debug(
226 | `[REPOSITORY - QUERY - DIAGNOSTICS] ${
227 | this.tableName
228 | } query complete - ${runtime[0]} s and ${runtime[1] /
229 | 1000000} ms`,
230 | {
231 | query: queryString,
232 | seconds: runtime[0],
233 | milliseconds: runtime[1] / 1000000,
234 | }
235 | )
236 | }
237 | }
238 | }
239 |
--------------------------------------------------------------------------------
/src/query/index.ts:
--------------------------------------------------------------------------------
1 | export * from "./abstract/Filter"
2 | export * from "./abstract/FilterCompiler"
3 | export * from "./abstract/Query"
4 | export * from "./abstract/Repository"
5 |
6 | export * from "./drivers/sql/SQLFilterCompiler"
7 | export * from "./drivers/sql/TableRepository"
8 |
--------------------------------------------------------------------------------
/src/queue/abstract/QueueHandler.ts:
--------------------------------------------------------------------------------
1 | import { ValidatorFunction } from "../../validation"
2 |
3 | export abstract class QueueHandler {
4 | public abstract inputValidator: ValidatorFunction
5 |
6 | public abstract async handle(message: P): Promise
7 | }
8 |
9 | export type QueueHanderPayload<
10 | Q extends QueueHandler
11 | > = Q extends QueueHandler ? P : never
12 |
--------------------------------------------------------------------------------
/src/queue/abstract/QueuePublisher.ts:
--------------------------------------------------------------------------------
1 | import { QueueHanderPayload, QueueHandler } from "./QueueHandler"
2 |
3 | export abstract class QueuePublisher {
4 | public abstract publish>(
5 | queueName: string,
6 | eventName: string,
7 | message: QueueHanderPayload
8 | ): Promise
9 | }
10 |
--------------------------------------------------------------------------------
/src/queue/abstract/SerializedTask.ts:
--------------------------------------------------------------------------------
1 | export interface SerializedTask {
2 | message: any
3 | }
4 |
--------------------------------------------------------------------------------
/src/queue/drivers/gcps/GCPSClient.ts:
--------------------------------------------------------------------------------
1 | import Axios from "axios"
2 | import { AsymmetricJWTSigner, RSASHA256Signer } from "../../../cryptography"
3 |
4 | export interface GCPSMessage {
5 | data: any
6 | attributes: {
7 | [key: string]: string
8 | }
9 | }
10 |
11 | export interface GCPSSubscription {
12 | name: string
13 | topic: string
14 | pushConfig: {
15 | pushEndpoint?: string
16 | }
17 | ackDeadlineSeconds: number
18 | retainAckedMessages: boolean
19 | messageRetentionDuration: string
20 | }
21 |
22 | export class GCPSClient {
23 | private signer: AsymmetricJWTSigner
24 |
25 | constructor(
26 | private serviceAccountEmail: string,
27 | private keyId: string,
28 | private privateKey: string
29 | ) {
30 | // Sanitize the private key
31 | let sanitizedPrivateKey = privateKey.replace(
32 | /\\n/g,
33 | `
34 | `
35 | )
36 |
37 | this.signer = new AsymmetricJWTSigner(
38 | new RSASHA256Signer(
39 | // Public key is empty as this will never be used to validate a token
40 | new Buffer(""),
41 | new Buffer(sanitizedPrivateKey)
42 | ),
43 | "RS256",
44 | keyId
45 | )
46 | }
47 |
48 | public async signRequest(
49 | audience:
50 | | "https://pubsub.googleapis.com/google.pubsub.v1.Publisher"
51 | | "https://pubsub.googleapis.com/google.pubsub.v1.Subscriber"
52 | ): Promise {
53 | let currentUnixTimestamp = Math.round(new Date().getTime() / 1000)
54 |
55 | return this.signer.sign({
56 | iss: this.serviceAccountEmail,
57 | sub: this.serviceAccountEmail,
58 | aud: audience,
59 | iat: currentUnixTimestamp,
60 | exp: currentUnixTimestamp + 3600,
61 | })
62 | }
63 |
64 | public async publish(
65 | topic: string,
66 | messages: Array
67 | ): Promise {
68 | await Axios.post(
69 | `https://pubsub.googleapis.com/v1/${topic}:publish`,
70 | {
71 | messages: messages.map((m) => ({
72 | attributes: m.attributes,
73 | data: Buffer.from(JSON.stringify(m.data)).toString(
74 | "base64"
75 | ),
76 | })),
77 | },
78 | {
79 | headers: {
80 | Authorization: `Bearer ${await this.signRequest(
81 | "https://pubsub.googleapis.com/google.pubsub.v1.Publisher"
82 | )}`,
83 | },
84 | maxContentLength: 10 * 1024 * 1024, // 10 MB in Bytes
85 | }
86 | )
87 | }
88 |
89 | public async getSubscriptionData(
90 | subscriptionName: string
91 | ): Promise {
92 | let subscriptionResp = await Axios.get(
93 | `https://pubsub.googleapis.com/v1/${subscriptionName}`,
94 | {
95 | headers: {
96 | Authorization: `Bearer ${await this.signRequest(
97 | "https://pubsub.googleapis.com/google.pubsub.v1.Subscriber"
98 | )}`,
99 | },
100 | }
101 | )
102 |
103 | return subscriptionResp.data
104 | }
105 |
106 | public async pullTasks(
107 | subscriptionName: string,
108 | maxMessages: number = 10,
109 | returnImmediately: boolean = false
110 | ): Promise<
111 | Array<{
112 | ackId: string
113 | message: GCPSMessage
114 | }>
115 | > {
116 | let taskResp = await Axios.post(
117 | `https://pubsub.googleapis.com/v1/${subscriptionName}:pull`,
118 | {
119 | returnImmediately: returnImmediately,
120 | maxMessages: maxMessages,
121 | },
122 | {
123 | headers: {
124 | Authorization: `Bearer ${await this.signRequest(
125 | "https://pubsub.googleapis.com/google.pubsub.v1.Subscriber"
126 | )}`,
127 | },
128 | // Set a 90 second timeout to take advantage of long polling
129 | timeout: 120 * 1000,
130 | }
131 | )
132 |
133 | return taskResp.data.receivedMessages
134 | ? taskResp.data.receivedMessages.map((m: any) => {
135 | return {
136 | ackId: m.ackId,
137 | message: {
138 | attributes: m.attributes,
139 | data: JSON.parse(
140 | Buffer.from(m.message.data, "base64").toString()
141 | ),
142 | },
143 | }
144 | })
145 | : []
146 | }
147 |
148 | public async modifyAckDeadline(
149 | subscriptionName: string,
150 | ackIds: Array,
151 | ackExtensionSeconds: number
152 | ): Promise {
153 | await Axios.post(
154 | `https://pubsub.googleapis.com/v1/${subscriptionName}:modifyAckDeadline`,
155 | {
156 | ackIds: ackIds,
157 | ackDeadlineSeconds: ackExtensionSeconds,
158 | },
159 | {
160 | headers: {
161 | Authorization: `Bearer ${await this.signRequest(
162 | "https://pubsub.googleapis.com/google.pubsub.v1.Subscriber"
163 | )}`,
164 | },
165 | }
166 | )
167 | }
168 |
169 | public async acknowledge(
170 | subscriptionName: string,
171 | ackIds: Array
172 | ): Promise {
173 | await Axios.post(
174 | `https://pubsub.googleapis.com/v1/${subscriptionName}:acknowledge`,
175 | {
176 | ackIds: ackIds,
177 | },
178 | {
179 | headers: {
180 | Authorization: `Bearer ${await this.signRequest(
181 | "https://pubsub.googleapis.com/google.pubsub.v1.Subscriber"
182 | )}`,
183 | },
184 | }
185 | )
186 | }
187 | }
188 |
--------------------------------------------------------------------------------
/src/queue/drivers/gcps/GCPSConsumer.ts:
--------------------------------------------------------------------------------
1 | import { GCPSClient } from "./GCPSClient"
2 | import { QueueHandler } from "../../abstract/QueueHandler"
3 | import { SerializedTask } from "../../abstract/SerializedTask"
4 | import { TransientError } from "../../../errors/TransientError"
5 | import { Container } from "inversify"
6 | import { isEmpty } from "lodash"
7 | import { Logger } from "../../../logging"
8 | import { Process } from "../../../runtime"
9 | import { ConstructorOf } from "../../../utils/types"
10 | import Timer = NodeJS.Timer
11 |
12 | export class GCPSConsumer implements Process {
13 | public isEnabled: boolean = false
14 | private ackDeadlineSeconds: number = 0
15 | private client: GCPSClient
16 | private logger?: Logger
17 |
18 | constructor(
19 | serviceAccountEmail: string,
20 | keyId: string,
21 | privateKey: string,
22 | public subscriptionName: string,
23 | public taskHandler: ConstructorOf>,
24 | public prefetchCount: number = 15
25 | ) {
26 | this.client = new GCPSClient(serviceAccountEmail, keyId, privateKey)
27 | }
28 |
29 | public isHealthy(): boolean {
30 | return this.isEnabled
31 | }
32 |
33 | public async shutdown(container: Container): Promise {
34 | this.isEnabled = false
35 | this.logger = undefined
36 | }
37 |
38 | public async startup(container: Container): Promise {
39 | // Start the process
40 | this.isEnabled = true
41 |
42 | if (container.isBound(Logger)) {
43 | this.logger = container.get(Logger)
44 | }
45 |
46 | // Fetch the subscription configuration
47 | let subscription = await this.client.getSubscriptionData(
48 | this.subscriptionName
49 | )
50 |
51 | if (!isEmpty(subscription.pushConfig)) {
52 | throw new Error(
53 | "The Strontium GCPS Consumer does not support Push based GCPS subscriptions. " +
54 | "Please change the subscription inside Google Cloud Platform to operate on a Pull Based model if you wish " +
55 | "to use this queue processor."
56 | )
57 | }
58 |
59 | this.ackDeadlineSeconds = subscription.ackDeadlineSeconds
60 |
61 | this.pollAndExecute(container)
62 | return
63 | }
64 |
65 | public async ack(ackId: string): Promise {
66 | return this.client.acknowledge(this.subscriptionName, [ackId])
67 | }
68 |
69 | public async nack(ackId: string, requeue: boolean = false): Promise {
70 | if (requeue) {
71 | return this.client.modifyAckDeadline(
72 | this.subscriptionName,
73 | [ackId],
74 | 0
75 | )
76 | } else {
77 | return this.ack(ackId)
78 | }
79 | }
80 |
81 | public async extendAck(ackId: string): Promise {
82 | return this.client.modifyAckDeadline(
83 | this.subscriptionName,
84 | [ackId],
85 | this.ackDeadlineSeconds
86 | )
87 | }
88 |
89 | public async pollAndExecute(container: Container): Promise {
90 | while (this.isEnabled) {
91 | let messages = await this.client.pullTasks(
92 | this.subscriptionName,
93 | this.prefetchCount,
94 | false
95 | )
96 |
97 | await Promise.all(
98 | messages.map(async (m) => {
99 | return this.executeTask(
100 | m.ackId,
101 | {
102 | message: m.message.data,
103 | },
104 | container
105 | )
106 | })
107 | )
108 | }
109 | }
110 |
111 | public async executeTask(
112 | ackId: string,
113 | task: SerializedTask,
114 | applicationContainer: Container
115 | ): Promise {
116 | // Create a new DI Container for the life of this request
117 | let requestContainer = new Container({
118 | autoBindInjectable: true,
119 | skipBaseClassChecks: true,
120 | })
121 |
122 | requestContainer.parent = applicationContainer
123 |
124 | // Spawn a handler for the Task type
125 | let handlerType = this.taskHandler
126 | if (this.logger) {
127 | this.logger.info(
128 | `[GCPS - TASK - START] Event received by Consumer for topic.`,
129 | {
130 | subscription: this.subscriptionName,
131 | }
132 | )
133 | }
134 |
135 | if (handlerType === undefined) {
136 | if (this.logger) {
137 | this.logger.error(
138 | `[GCPS - TASK - NO_IMPLEMENTATION_FAIL] No implementation found for topic.`,
139 | {
140 | subscription: this.subscriptionName,
141 | }
142 | )
143 | }
144 | await this.nack(ackId)
145 | return
146 | }
147 |
148 | let requestHandler = requestContainer.get(handlerType)
149 |
150 | // Set a regular task to extend the lifespan of the job until we are done processing it.
151 | let ackInterval: Timer = setInterval(() => {
152 | this.extendAck(ackId)
153 | }, this.ackDeadlineSeconds * 1000)
154 |
155 | try {
156 | // Validation errors aren't caught explicitly as they should never happen in Production.
157 | // They are instead designed to prevent edge case errors and are thrown as such.
158 | let validatedMessage = await requestHandler.inputValidator(
159 | task.message
160 | )
161 |
162 | await requestHandler.handle(validatedMessage)
163 |
164 | await this.ack(ackId)
165 | if (this.logger) {
166 | this.logger.info(
167 | `[GCPS - TASK - SUCCESS] Event successfully completed by Consumer.`,
168 | {
169 | subscription: this.subscriptionName,
170 | }
171 | )
172 | }
173 | } catch (e) {
174 | if (e instanceof TransientError) {
175 | if (this.logger) {
176 | this.logger.error(
177 | "[GCPS - TASK - TRANSIENT_FAIL] Task failed with transient error. Attempting to reschedule execution.",
178 | e
179 | )
180 | }
181 |
182 | await this.nack(ackId, true)
183 | } else {
184 | if (this.logger) {
185 | this.logger.error(
186 | "[GCPS - TASK - PERMANENT_FAIL] Task failed with permanent error.",
187 | e
188 | )
189 | }
190 |
191 | // For permanent errors we stop attempting to process the object
192 | await this.nack(ackId)
193 | }
194 | } finally {
195 | clearInterval(ackInterval)
196 | }
197 | }
198 | }
199 |
--------------------------------------------------------------------------------
/src/queue/drivers/gcps/GCPSPublisher.ts:
--------------------------------------------------------------------------------
1 | import { GCPSClient } from "./GCPSClient"
2 | import { QueuePublisher } from "../../abstract/QueuePublisher"
3 | import { Container } from "inversify"
4 | import { Process } from "../../../runtime"
5 |
6 | export class GCPSPublisher extends QueuePublisher implements Process {
7 | private client: GCPSClient
8 |
9 | constructor(
10 | serviceAccountEmail: string,
11 | keyId: string,
12 | privateKey: string
13 | ) {
14 | super()
15 |
16 | this.client = new GCPSClient(serviceAccountEmail, keyId, privateKey)
17 | }
18 |
19 | public isHealthy(): boolean {
20 | // GCPS is a REST service - it is incapable of having a fundamentally unhealthy state.
21 | return true
22 | }
23 |
24 | public async publish(
25 | queueName: string,
26 | eventName: string,
27 | messages: Q | Array
28 | ): Promise {
29 | if (!Array.isArray(messages)) {
30 | messages = [messages]
31 | }
32 |
33 | return this.client.publish(
34 | queueName,
35 | messages.map((m) => ({
36 | attributes: {
37 | STRONTIUM_EVENT_NAME: eventName,
38 | },
39 | data: m,
40 | }))
41 | )
42 | }
43 |
44 | public async shutdown(container: Container): Promise {
45 | container.unbind(QueuePublisher)
46 | container.unbind(GCPSPublisher)
47 | }
48 |
49 | public async startup(container: Container): Promise {
50 | container.bind(QueuePublisher).toConstantValue(this)
51 | container.bind(GCPSPublisher).toConstantValue(this)
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/src/queue/index.ts:
--------------------------------------------------------------------------------
1 | export * from "./abstract/QueueHandler"
2 | export * from "./abstract/QueuePublisher"
3 | export * from "./abstract/SerializedTask"
4 |
5 | export * from "./drivers/gcps/GCPSClient"
6 | export * from "./drivers/gcps/GCPSConsumer"
7 | export * from "./drivers/gcps/GCPSPublisher"
8 |
--------------------------------------------------------------------------------
/src/runtime/abstract/Process.ts:
--------------------------------------------------------------------------------
1 | import { Container } from "inversify"
2 |
3 | /**
4 | * A Process represents a long running logical process within the wider scope
5 | * of an application runtime.
6 | *
7 | * An example might be a Web Server or a Database Connection pool.
8 | *
9 | * Conceptually a Process has a startup procedure, a shutdown procedure and an
10 | * ongoing state.
11 | *
12 | * The lifetime expectations of a Process are as follows:
13 | * 1. It should not have any effect until started
14 | * 2. All effects should cease and be cleaned once shutdown is closed.
15 | * 3. The isHealthy check should only return true if the process is functioning nominally. Any abnormal behaviour or errors
16 | * should trigger an error.
17 | */
18 | export interface Process {
19 | /**
20 | * Start the process.
21 | *
22 | * Implementing processes should take care to ensure that startup is roughly idempotent ( i.e subsequent calls will
23 | * not cause issues in an already started process ).
24 | *
25 | * Implementations should also ensure not to cause any side effects prior to Startup being called.
26 | *
27 | * @param container {Container} - The Inversify container used by the Runtime for type resolution. This should
28 | * be used by implementations to register the started process with the Runtime for use.
29 | */
30 | startup(container: Container): Promise
31 |
32 | /**
33 | * Stop the process.
34 | *
35 | * Implementing processes should use this hook to close all open connections, sockets and event loop items
36 | * ( intervals, timeouts, etc. ).
37 | *
38 | * Runtimes will expect that upon the completion of the Promise shutdown is complete to the level that node
39 | * will gracefully terminate ( the event loop is empty ).
40 | *
41 | * @param container {Container} - The Inversify container used by the Runtime for type resolution. This should
42 | * be used by implementations to deregister the stopped process from the Runtime.
43 | */
44 | shutdown(container: Container): Promise
45 |
46 | /**
47 | * Return the health status of the Process.
48 | *
49 | * If the process is unable to function as originally anticipated then it should return false.
50 | *
51 | * Runtime implementations will decide what to do in the case of health check failure but actions may include
52 | * attempting to restart the process using Shutdown and Startup sequentially or simply killing the entire runtime.
53 | */
54 | isHealthy(): boolean
55 | }
56 |
57 | export const isProcess = (p: any): p is Process => {
58 | return p !== undefined && typeof p.isHealthy === "function"
59 | }
60 |
--------------------------------------------------------------------------------
/src/runtime/drivers/Runtime.ts:
--------------------------------------------------------------------------------
1 | import { Process } from "../abstract/Process"
2 | import { Container } from "inversify"
3 |
4 | /**
5 | * A Runtime represents a collection of Processes run together to form an Application.
6 | *
7 | * Runtimes are designed to provide an easy to work with wrapper to build reliable
8 | * applications and abstract the nasty underlayers of DI and subprocess monitoring
9 | * that are often discarded due to their complexity.
10 | */
11 | export class Runtime implements Process {
12 | private container: Container = new Container()
13 |
14 | constructor(private processes: Array) {}
15 |
16 | public async startup(): Promise {
17 | // Start each process in order, waiting for it to be fully booted before moving to the next.
18 | for (let p of this.processes) {
19 | await p.startup(this.container)
20 | }
21 | }
22 |
23 | public async shutdown(): Promise {
24 | // Stop each process in reverse order, waiting for it to be full closed before moving to the next.
25 | for (let p of this.processes.reverse()) {
26 | await p.shutdown(this.container)
27 | }
28 | }
29 |
30 | public isHealthy(): boolean {
31 | // Aggregate the health status of each of the sub processes
32 | return this.processes.reduce((memo: boolean, p) => {
33 | return memo && p.isHealthy()
34 | }, true)
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/src/runtime/index.ts:
--------------------------------------------------------------------------------
1 | export { Process, isProcess } from "./abstract/Process"
2 | export { Runtime } from "./drivers/Runtime"
3 |
--------------------------------------------------------------------------------
/src/utils/list.ts:
--------------------------------------------------------------------------------
1 | import { notMissing } from "./typeGuard"
2 |
3 | export const compact = (input: T[]): Exclude[] => {
4 | return input.filter(notMissing)
5 | }
6 |
--------------------------------------------------------------------------------
/src/utils/typeGuard.ts:
--------------------------------------------------------------------------------
1 | export type TypeGuard = (val: T) => val is V
2 |
3 | export function notMissing(input: T): input is Exclude {
4 | return input !== null && input !== undefined
5 | }
6 |
--------------------------------------------------------------------------------
/src/utils/types.ts:
--------------------------------------------------------------------------------
1 | export type ConstructorOf = {
2 | new (...arg: any[]): T
3 | }
4 |
5 | export type UUID = string
6 |
7 | export type Nullable = T | null
8 |
--------------------------------------------------------------------------------
/src/validation/abstract/ObjectValidator.ts:
--------------------------------------------------------------------------------
1 | import { ValidatorFunction, ValidatorOutput } from "./ValidatorFunction"
2 |
3 | export type ObjectValidator = {
4 | [key: string]: ValidatorFunction
5 | } & Object
6 |
7 | export type ValidatedObject = {
8 | [P in keyof O]: ValidatorOutput
9 | }
10 |
--------------------------------------------------------------------------------
/src/validation/abstract/ValidatorFunction.ts:
--------------------------------------------------------------------------------
1 | export type ValidatorFunction = (input: I) => O | Promise
2 |
3 | export type ValidatorOutput<
4 | I,
5 | P extends ValidatorFunction
6 | > = P extends ValidatorFunction ? O : ReturnType
7 |
8 | export type ValidatorInput<
9 | P extends ValidatorFunction
10 | > = P extends ValidatorFunction ? I : ReturnType
11 |
--------------------------------------------------------------------------------
/src/validation/drivers/helpers/combineValidators.ts:
--------------------------------------------------------------------------------
1 | import { ValidatorFunction } from "../../abstract/ValidatorFunction"
2 |
3 | export function combineValidators(
4 | V1: ValidatorFunction,
5 | V2: ValidatorFunction
6 | ): ValidatorFunction
7 | export function combineValidators(
8 | V1: ValidatorFunction,
9 | V2: ValidatorFunction,
10 | V3: ValidatorFunction
11 | ): ValidatorFunction
12 | export function combineValidators(
13 | V1: ValidatorFunction,
14 | V2: ValidatorFunction,
15 | V3: ValidatorFunction,
16 | V4: ValidatorFunction
17 | ): ValidatorFunction
18 | export function combineValidators(
19 | V1: ValidatorFunction,
20 | V2: ValidatorFunction,
21 | V3: ValidatorFunction,
22 | V4: ValidatorFunction,
23 | V5: ValidatorFunction
24 | ): ValidatorFunction
25 |
26 | export function combineValidators(
27 | V1: ValidatorFunction,
28 | V2: ValidatorFunction,
29 | V3?: ValidatorFunction,
30 | V4?: ValidatorFunction,
31 | V5?: ValidatorFunction
32 | ):
33 | | ValidatorFunction
34 | | ValidatorFunction
35 | | ValidatorFunction
36 | | ValidatorFunction {
37 | // This is split into if statements so Type completion is rigid. It's possible this could
38 | // be better written in the future but for now TypeScript is happy.
39 | if (V5 !== undefined && V4 !== undefined && V3 !== undefined) {
40 | return async (i: I) => {
41 | let r1 = await V1(i)
42 | let r2 = await V2(r1)
43 | let r3 = await V3(r2)
44 | let r4 = await V4(r3)
45 | return await V5(r4)
46 | }
47 | } else if (V4 !== undefined && V3 !== undefined) {
48 | return async (i: I) => {
49 | let r1 = await V1(i)
50 | let r2 = await V2(r1)
51 | let r3 = await V3(r2)
52 | return await V4(r3)
53 | }
54 | } else if (V3 !== undefined) {
55 | return async (i: I) => {
56 | let r1 = await V1(i)
57 | let r2 = await V2(r1)
58 | return await V3(r2)
59 | }
60 | } else {
61 | return async (i: I) => {
62 | let r1 = await V1(i)
63 | return await V2(r1)
64 | }
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/src/validation/drivers/helpers/either.ts:
--------------------------------------------------------------------------------
1 | import { ValidationError } from "../../../errors/http/ValidationError"
2 | import { ValidatorFunction } from "../../abstract/ValidatorFunction"
3 |
4 | export function either(
5 | V1: ValidatorFunction,
6 | V2: ValidatorFunction
7 | ): ValidatorFunction
8 | export function either(
9 | V1: ValidatorFunction,
10 | V2: ValidatorFunction,
11 | V3: ValidatorFunction
12 | ): ValidatorFunction
13 | export function either(
14 | V1: ValidatorFunction,
15 | V2: ValidatorFunction,
16 | V3: ValidatorFunction,
17 | V4: ValidatorFunction
18 | ): ValidatorFunction
19 | export function either(
20 | V1: ValidatorFunction,
21 | V2: ValidatorFunction,
22 | V3: ValidatorFunction,
23 | V4: ValidatorFunction,
24 | V5: ValidatorFunction
25 | ): ValidatorFunction
26 |
27 | export function either(
28 | V1: ValidatorFunction,
29 | V2: ValidatorFunction,
30 | V3?: ValidatorFunction,
31 | V4?: ValidatorFunction,
32 | V5?: ValidatorFunction
33 | ):
34 | | ValidatorFunction
35 | | ValidatorFunction
36 | | ValidatorFunction
37 | | ValidatorFunction {
38 | return async (i: I) => {
39 | let errors: Array = []
40 |
41 | // Iterate over each validator in descending order until one succeeds.
42 | for (let validator of [V1, V2, V3, V4, V5]) {
43 | if (validator !== undefined) {
44 | try {
45 | return await validator(i)
46 | } catch (e) {
47 | errors.push(e)
48 | }
49 | }
50 | }
51 |
52 | // If we get to this stage then we have failed the validator - throw a Validation Error unless one of
53 | // the validators threw a different error type
54 | let failedConstraints: Array = []
55 | let failedInternalMessages: Array = []
56 | let failedExternalMessages: Array = []
57 |
58 | for (let error of errors) {
59 | if (error instanceof ValidationError) {
60 | failedConstraints.push(error.constraintName)
61 |
62 | if (error.internalMessage) {
63 | failedInternalMessages.push(error.internalMessage)
64 | }
65 |
66 | if (error.externalMessage) {
67 | failedExternalMessages.push(error.externalMessage)
68 | }
69 | } else {
70 | throw error
71 | }
72 | }
73 |
74 | throw new ValidationError(
75 | `EITHER(${failedConstraints.join(",")})`,
76 | `No compatible validators found: (${failedInternalMessages.join(
77 | ", "
78 | )})`,
79 | `This value did not match any of the following validators: (${failedExternalMessages.join(
80 | " | "
81 | )})`
82 | )
83 | }
84 | }
85 |
--------------------------------------------------------------------------------
/src/validation/drivers/helpers/isOptional.ts:
--------------------------------------------------------------------------------
1 | import { either } from "./either"
2 | import { isUndefined } from "../validators/isUndefined"
3 |
4 | import { ValidatorFunction } from "../.."
5 |
6 | export const isOptional = , I, O>(
7 | validator: V
8 | ) => either(isUndefined, validator)
9 |
--------------------------------------------------------------------------------
/src/validation/drivers/sanitizers/defaultValue.ts:
--------------------------------------------------------------------------------
1 | export const defaultValue = (defaultValue: D) => (input?: I): I | D => {
2 | if (input === undefined) {
3 | return defaultValue
4 | }
5 |
6 | return input
7 | }
8 |
--------------------------------------------------------------------------------
/src/validation/drivers/sanitizers/normalizeEmail.ts:
--------------------------------------------------------------------------------
1 | import { ValidationError } from "../../../errors/http/ValidationError"
2 | import * as Validator from "validator"
3 | import NormalizeEmailOptions = ValidatorJS.NormalizeEmailOptions
4 |
5 | export const normalizeEmail = (options?: NormalizeEmailOptions) =>