├── .eslintrc.js ├── .gitignore ├── .husky ├── .gitignore └── pre-commit ├── .prettierignore ├── .prettierrc ├── LICENSE ├── README.md ├── esm ├── AbstractFetcher.d.ts ├── AbstractFetcher.js ├── BatchFetcher.d.ts ├── BatchFetcher.js ├── BatchWriter.d.ts ├── BatchWriter.js ├── Pipeline.d.ts ├── Pipeline.js ├── QueryFetcher.d.ts ├── QueryFetcher.js ├── ScanQueryPipeline.d.ts ├── ScanQueryPipeline.js ├── TableIterator.d.ts ├── TableIterator.js ├── helpers │ ├── index.d.ts │ └── index.js ├── index.d.ts ├── index.js ├── mocks │ ├── index.d.ts │ └── index.js ├── types.d.ts └── types.js ├── example ├── .eslintrc.js ├── package.json ├── src │ └── example-lambda.ts ├── test │ └── example-lambda.test.ts ├── tsconfig.json └── yarn.lock ├── jest.config.ts ├── lib ├── AbstractFetcher.d.ts ├── AbstractFetcher.js ├── BatchFetcher.d.ts ├── BatchFetcher.js ├── BatchWriter.d.ts ├── BatchWriter.js ├── Pipeline.d.ts ├── Pipeline.js ├── QueryFetcher.d.ts ├── QueryFetcher.js ├── ScanQueryPipeline.d.ts ├── ScanQueryPipeline.js ├── TableIterator.d.ts ├── TableIterator.js ├── helpers │ ├── index.d.ts │ └── index.js ├── index.d.ts ├── index.js ├── mocks │ ├── index.d.ts │ └── index.js ├── types.d.ts └── types.js ├── package.json ├── pnpm-lock.yaml ├── src ├── AbstractFetcher.ts ├── BatchFetcher.ts ├── BatchWriter.ts ├── Pipeline.ts ├── QueryFetcher.ts ├── ScanQueryPipeline.ts ├── TableIterator.ts ├── helpers │ └── index.ts ├── index.ts ├── mocks │ └── index.ts └── types.ts ├── test ├── dynamo-pipeline.test.ts ├── dynamodb.setup.ts └── jest.setup.ts ├── tsconfig-cjs.json ├── tsconfig.eslint.json └── tsconfig.json /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: [ 3 | "plugin:@typescript-eslint/eslint-recommended", 4 | "plugin:@typescript-eslint/recommended", 5 | "plugin:@typescript-eslint/recommended-requiring-type-checking", 6 | "standard", 7 | "prettier", 8 | ], 9 | parser: "@typescript-eslint/parser", 10 | plugins: ["@typescript-eslint"], 11 | parserOptions: { 12 | project: "./tsconfig.eslint.json", 13 | ecmaVersion: 10, 14 | sourceType: "module", 15 | }, 16 | env: { 17 | node: true, 18 | jest: true, 19 | }, 20 | globals: { 21 | __DEV__: false, 22 | beforeAll: false, 23 | afterAll: false, 24 | beforeEach: false, 25 | afterEach: false, 26 | test: false, 27 | expect: false, 28 | describe: false, 29 | jest: false, 30 | it: false, 31 | }, 32 | ignorePatterns: [ 33 | "**/node_modules/**/*", 34 | "node_modules/**/*", 35 | "lib/**/*", 36 | "esm/**/*", 37 | "coverage/**/*", 38 | "**/coverage/**/*", 39 | ".*", 40 | ], 41 | rules: { 42 | complexity: ["warn", 30], 43 | "default-case": 2, 44 | "dot-notation": 2, 45 | eqeqeq: 2, 46 | "guard-for-in": 2, 47 | "no-constant-condition": 2, 48 | "no-dupe-keys": 2, 49 | "no-eval": 2, 50 | "no-unreachable": 2, 51 | "no-unused-vars": 0, 52 | "no-void": ["error", { allowAsStatement: true }], 53 | "prefer-destructuring": [ 54 | "warn", 55 | { 56 | object: true, 57 | array: true, 58 | }, 59 | ], 60 | camelcase: 0, 61 | "@typescript-eslint/camelcase": 0, 62 | "@typescript-eslint/explicit-member-accessibility": 0, 63 | "@typescript-eslint/explicit-function-return-type": 0, 64 | "@typescript-eslint/indent": 0, 65 | "@typescript-eslint/no-explicit-any": 0, 66 | "@typescript-eslint/no-empty-interface": 0, 67 | "@typescript-eslint/no-object-literal-type-assertion": 0, 68 | "@typescript-eslint/no-use-before-define": 0, 69 | "@typescript-eslint/no-var-requires": 0, 70 | "@typescript-eslint/no-floating-promises": 2, 71 | "standard/no-callback-literal": 0, 72 | "node/no-callback-literal": 0, 73 | "@typescript-eslint/no-unused-vars": 0, 74 | }, 75 | }; 76 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # Dependency directories 26 | node_modules/ 27 | 28 | # TypeScript cache 29 | *.tsbuildinfo 30 | 31 | # Optional npm cache directory 32 | .npm 33 | 34 | # Optional eslint cache 35 | .eslintcache 36 | 37 | # Optional REPL history 38 | .node_repl_history 39 | 40 | # Output of 'npm pack' 41 | *.tgz 42 | 43 | # Yarn Integrity file 44 | .yarn-integrity 45 | 46 | # parcel-bundler cache (https://parceljs.org/) 47 | .cache 48 | 49 | # DynamoDB Local files 50 | .dynamodb/ 51 | 52 | # Typescript build cache 53 | .buildcache/ 54 | 55 | # VSCode Settings 56 | .vscode/ 57 | 58 | example/dist/ 59 | example/node_modules/ -------------------------------------------------------------------------------- /.husky/.gitignore: -------------------------------------------------------------------------------- 1 | _ 2 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | pnpm run format 5 | pnpm run lint 6 | pnpm run build 7 | pnpm run test 8 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | lib 2 | esm -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/prettierrc", 3 | "printWidth": 120, 4 | "trailingComma": "es5", 5 | "semi": true 6 | } 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dynamo-pipeline 2 | 3 | Alternative API for DynamoDB DocumentClient (v2) to improve types, allow easy iteration and paging of data, and reduce developer mistakes. From "So complex there are no obvious mistakes" to "So simple there are obviously no mistakes". 4 | 5 | 5KB gzipped (excluding aws-sdk DocumentClient). 6 | 7 | ### Limitations 8 | 9 | 1. Partition Keys and Sort Keys only support string type 10 | 1. Limited transaction support 11 | 1. Some dynamodb request options are not available 12 | 13 | ## Example 14 | 15 | Suppose you wish to find the first 5000 form items with sk > "0000" which are not deleted, and for each item, add 'gsi1pk' and 'gsi1sk' attributes to each item. 16 | 17 | ### Dynamo Pipeline 18 | 19 | ```typescript 20 | import { Pipeline } from "dynamo-pipeline"; 21 | 22 | interface Item { 23 | id: string; 24 | sk: string; 25 | _isDeleted: boolean; 26 | data: { 27 | attr1: string; 28 | attr2: string; 29 | }; 30 | } 31 | 32 | const privatePipeline = new Pipeline("PrivateTableName-xxx", { 33 | pk: "id", 34 | sk: "sk", 35 | }); 36 | 37 | await privatePipeline 38 | .query( 39 | { pk: "FormId", sk: sortKey(">", "0000") }, 40 | { 41 | limit: 5000, 42 | filters: { 43 | lhs: { property: "_isDeleted" }, 44 | logical: "<>", 45 | rhs: false, 46 | }, 47 | } 48 | ) 49 | .forEach((item, _index, pipeline) => pipeline.update(item, { gsi1pk: data.attr1, gsi1sk: data.attr2 })); 50 | 51 | privatePipeline.handleUnprocessed((item) => console.error(`Update Failed: id: ${item.id} , sk: ${item.sk}`)); 52 | ``` 53 | 54 | ### NoSQL Workbench Generated Code + Looping logic 55 | 56 | ```javascript 57 | const AWS = require("aws-sdk"); 58 | 59 | const dynamoDbClient = createDynamoDbClient(); 60 | const queryInput = createQueryInput(); 61 | let isFirstQuery = true; 62 | let itemsProcessed = 0; 63 | const updateErrors = []; 64 | 65 | while ((isFirstQuery || queryInput.LastEvaluatedKey) && itemsProcessed < 5000) { 66 | isFirstQuery = false; 67 | const result = await executeQuery(dynamoDbClient, queryInput); 68 | await Promise.all( 69 | result.Items.map((item) => { 70 | itemsProcessed += 1; 71 | if (itemsProcessed <= 5000) { 72 | return executeUpdateItem( 73 | dynamoDbClient, 74 | createUpdateItemInput(item.id, item.sk, item.data.attr1, item.data.attr2) 75 | ); 76 | } 77 | }) 78 | ); 79 | 80 | if (result.LastEvaluatedKey) { 81 | queryInput.LastEvaluatedKey = result.LastEvaluatedKey; 82 | } 83 | } 84 | 85 | updateErrors.forEach((err) => console.error(`Update Failed: id: ${item.id} , sk: ${item.sk}`)); 86 | 87 | function createDynamoDbClient() { 88 | return new AWS.DynamoDB(); 89 | } 90 | 91 | function createQueryInput() { 92 | return { 93 | TableName: "Private-xxx-xxx", 94 | ScanIndexForward: false, 95 | ConsistentRead: false, 96 | KeyConditionExpression: "#254c0 = :254c0 And #254c1 > :254c1", 97 | FilterExpression: "#254c2 <> :254c2", 98 | ExpressionAttributeValues: { 99 | ":254c0": { 100 | S: "FormId", 101 | }, 102 | ":254c1": { 103 | S: "0000", 104 | }, 105 | ":254c2": { 106 | BOOL: true, 107 | }, 108 | }, 109 | ExpressionAttributeNames: { 110 | "#254c0": "id", 111 | "#254c1": "sk", 112 | "#254c2": "_isDeleted", 113 | }, 114 | }; 115 | } 116 | 117 | function createUpdateItemInput(id, sk, gsi1pk, gsi1sk) { 118 | return { 119 | TableName: "Form-xxx-xxx", 120 | Key: { 121 | id: { 122 | S: id, 123 | }, 124 | sk: { 125 | S: sk, 126 | }, 127 | }, 128 | UpdateExpression: "SET #4bd90 = :4bd90, #4bd91 = :4bd91", 129 | ExpressionAttributeValues: { 130 | ":4bd90": { 131 | S: gsi1pk, 132 | }, 133 | ":4bd91": { 134 | S: gsi1sk, 135 | }, 136 | }, 137 | ExpressionAttributeNames: { 138 | "#4bd90": "gsi1pk", 139 | "#4bd91": "gsi1sk", 140 | }, 141 | }; 142 | } 143 | 144 | async function executeUpdateItem(dynamoDbClient, updateItemInput) { 145 | // Call DynamoDB's updateItem API 146 | try { 147 | const updateItemOutput = await dynamoDbClient.updateItem(updateItemInput).promise(); 148 | return updateItemOutput; 149 | } catch (err) { 150 | handleUpdateItemError(err); 151 | } 152 | } 153 | 154 | async function executeQuery(dynamoDbClient, queryInput) { 155 | try { 156 | const queryOutput = await dynamoDbClient.query(queryInput).promise(); 157 | return queryOutput; 158 | } catch (err) { 159 | // handleQueryError(err); 160 | } 161 | } 162 | 163 | function handleUpdateItemError(err) { 164 | updateErrors.push(err); 165 | } 166 | ``` 167 | 168 | ## Default Options 169 | 170 | ### Assumptions 171 | 172 | Safe assumptions for lambda -> dynamodb round trip times to avoid excessive throttling 173 | 174 | - Items are small, <= 2KB 175 | - On-Demand billing 176 | - default On-Demand max capacity of 12,000 RRUs and 4,000 WRUs. 177 | - Batch Writes consume 50 WRUs and complete in 7ms 178 | - Transact writes consume 100 WRUs and complete in 20ms 179 | - Single item write operations consume 2 WRUs and complete in 5ms 180 | - Queries and scans consume 125 RRUs and complete in 10ms 181 | - Batch Get operations consume 50 RRUs and complete in 5ms 182 | 183 | ### Writes 184 | 185 | - For all items, divide below by 1 + # of GSIs where the item will appear 186 | - Assume other application traffic is consuming no more than 100 WRUs 187 | - Batch writes should use a buffer size of 1, or 2 for small items. A buffer of 3 will result in throttles and retries. Reduce batch write size if desired buffer size is < 1. 188 | - Query and Scan reads which result in 1:1 writes should be batched into sizes of 25, or 50 for small items. A (read) batch size of 75 will experience write throttling. 189 | 190 | ### Read 191 | 192 | - Assume other application traffic is consuming no more than 500 RRUs 193 | - Batch Gets should use a read buffer of 1 or reduce the batch size 194 | - Queries and scans are usually not a bottleneck and read buffers can be set to high values except in memory constrained environments 195 | -------------------------------------------------------------------------------- /esm/AbstractFetcher.d.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | export declare abstract class AbstractFetcher { 3 | protected activeRequests: Promise[]; 4 | protected bufferSize: number; 5 | protected bufferCapacity: number; 6 | protected batchSize: number; 7 | protected limit?: number; 8 | protected totalReturned: number; 9 | protected nextToken: number | Record | null; 10 | protected documentClient: DocumentClient; 11 | protected results: T[]; 12 | protected errors: Error | null; 13 | constructor(client: DocumentClient, options: { 14 | batchSize: number; 15 | bufferCapacity: number; 16 | limit?: number; 17 | }); 18 | abstract fetchStrategy(): Promise | null; 19 | abstract processResult(data: Record): void; 20 | protected fetchNext(): Promise | null; 21 | private setupFetchProcessor; 22 | execute(): AsyncGenerator; 24 | } | void, void>; 25 | getResultBatch(batchSize: number): T[]; 26 | processError(e: Error): void; 27 | hasDataReady(): boolean; 28 | isDone(): boolean; 29 | isActive(): boolean; 30 | } 31 | -------------------------------------------------------------------------------- /esm/AbstractFetcher.js: -------------------------------------------------------------------------------- 1 | export class AbstractFetcher { 2 | constructor(client, options) { 3 | this.activeRequests = []; 4 | this.bufferSize = 0; 5 | this.bufferCapacity = 1; 6 | this.totalReturned = 0; 7 | this.results = []; 8 | this.errors = null; 9 | this.documentClient = client; 10 | this.bufferCapacity = options.bufferCapacity; 11 | this.batchSize = options.batchSize; 12 | this.limit = options.limit; 13 | this.nextToken = null; 14 | } 15 | // take in a promise to allow recursive calls, 16 | // batch fetcher can immediately create many requests 17 | fetchNext() { 18 | const fetchResponse = this.fetchStrategy(); 19 | if (fetchResponse instanceof Promise && !this.activeRequests.includes(fetchResponse)) { 20 | return this.setupFetchProcessor(fetchResponse); 21 | } 22 | return fetchResponse; 23 | } 24 | setupFetchProcessor(promise) { 25 | this.activeRequests.push(promise); 26 | this.bufferSize += 1; 27 | return promise 28 | .then((data) => { 29 | this.activeRequests = this.activeRequests.filter((r) => r !== promise); 30 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 31 | this.processResult(data); 32 | }) 33 | .catch((e) => { 34 | this.activeRequests = this.activeRequests.filter((r) => r !== promise); 35 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 36 | this.processError(e); 37 | }); 38 | } 39 | // Entry point. 40 | async *execute() { 41 | let count = 0; 42 | do { 43 | if (this.errors) { 44 | return Promise.reject(this.errors); 45 | } 46 | if (!this.hasDataReady()) { 47 | await this.fetchNext(); 48 | } 49 | // check for errors again after running another fetch 50 | if (this.errors) { 51 | return Promise.reject(this.errors); 52 | } 53 | const batch = this.getResultBatch(Math.min(this.batchSize, this.limit ? this.limit - count : 1000000000000)); 54 | count += batch.length; 55 | if (!this.isDone() && (!this.limit || count < this.limit)) { 56 | // do not await here, background process the next set of data 57 | void this.fetchNext(); 58 | } 59 | yield batch; 60 | if (this.limit && count >= this.limit) { 61 | if (typeof this.nextToken === "object" && this.nextToken !== null) { 62 | return { lastEvaluatedKey: this.nextToken }; 63 | } 64 | return; 65 | } 66 | } while (!this.isDone()); 67 | } 68 | getResultBatch(batchSize) { 69 | const items = (this.results.length && this.results.splice(0, batchSize)) || []; 70 | if (!items.length) { 71 | this.bufferSize = this.activeRequests.length; 72 | } 73 | else { 74 | this.bufferSize -= 1; 75 | } 76 | return items; 77 | } 78 | processError(e) { 79 | this.errors = e; 80 | } 81 | hasDataReady() { 82 | return this.results.length > 0; 83 | } 84 | isDone() { 85 | return !this.isActive() && this.nextToken === null && this.results.length === 0; 86 | } 87 | isActive() { 88 | return this.activeRequests.length > 0; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /esm/BatchFetcher.d.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { Key, KeyDefinition } from "./types"; 3 | import { AbstractFetcher } from "./AbstractFetcher"; 4 | type BatchGetItems = { 5 | tableName: string; 6 | keys: Key[]; 7 | }; 8 | type TransactGetItems = { 9 | tableName: string; 10 | keys: Key; 11 | }[]; 12 | export declare class BatchGetFetcher extends AbstractFetcher { 13 | protected operation: "batchGet" | "transactGet"; 14 | protected chunks: BatchGetItems[] | TransactGetItems[]; 15 | protected retryKeys: BatchGetItems[] | null; 16 | protected onUnprocessedKeys: ((keys: Key[]) => void) | undefined; 17 | protected consistentRead: boolean; 18 | constructor(client: DocumentClient, operation: "batchGet" | "transactGet", items: BatchGetItems | TransactGetItems, options: { 19 | onUnprocessedKeys?: (keys: Key[]) => void; 20 | batchSize: number; 21 | bufferCapacity: number; 22 | consistentRead?: boolean; 23 | }); 24 | private chunkBatchRequests; 25 | retry(): Promise | null; 26 | fetchStrategy(): Promise | null; 27 | processResult(data: DocumentClient.BatchGetItemOutput | DocumentClient.TransactGetItemsOutput | void): void; 28 | processError(err: Error | { 29 | tableName: string; 30 | errorKeys: Key[]; 31 | }): void; 32 | isDone(): boolean; 33 | private createTransactionRequest; 34 | private createBatchGetRequest; 35 | private hasNextChunk; 36 | } 37 | export {}; 38 | -------------------------------------------------------------------------------- /esm/BatchFetcher.js: -------------------------------------------------------------------------------- 1 | import { AbstractFetcher } from "./AbstractFetcher"; 2 | export class BatchGetFetcher extends AbstractFetcher { 3 | constructor(client, operation, items, options) { 4 | super(client, options); 5 | this.retryKeys = []; 6 | this.consistentRead = false; 7 | this.operation = operation; 8 | this.onUnprocessedKeys = options.onUnprocessedKeys; 9 | this.consistentRead = Boolean(options.consistentRead); 10 | if (operation === "batchGet" && !Array.isArray(items)) { 11 | this.chunks = this.chunkBatchRequests(items); 12 | } 13 | else { 14 | // Transactions don't support chunking, its a transaction 15 | this.chunks = [items]; 16 | } 17 | this.nextToken = 0; 18 | } 19 | chunkBatchRequests(items) { 20 | const chunks = []; 21 | const n = items.keys.length; 22 | let i = 0; 23 | while (i < n) { 24 | chunks.push({ 25 | tableName: items.tableName, 26 | keys: items.keys.slice(i, (i += this.batchSize)), 27 | }); 28 | } 29 | return chunks; 30 | } 31 | retry() { 32 | this.chunks = this.retryKeys || []; 33 | this.nextToken = 0; 34 | this.retryKeys = null; 35 | return this.fetchNext(); 36 | // TODO: Batch Get needs to be tested with chunk size of 1 and three items 37 | } 38 | fetchStrategy() { 39 | if (this.retryKeys && this.retryKeys.length && this.nextToken === null && !this.isActive()) { 40 | // if finished fetching initial requests, begin to process the retry keys 41 | return this.retry(); 42 | } 43 | else if (this.bufferSize >= this.bufferCapacity || 44 | (typeof this.nextToken === "number" && this.chunks.length <= this.nextToken) || 45 | this.nextToken === null) { 46 | // return the current promise if buffer at capacity, or if there are no more items to fetch 47 | return this.activeRequests[0] || null; 48 | } 49 | else if (!this.hasNextChunk()) { 50 | /* istanbul ignore next */ 51 | return null; 52 | } 53 | let promise = null; 54 | if (this.operation === "transactGet") { 55 | const transactionRequest = this.createTransactionRequest(); 56 | if (transactionRequest === null) { 57 | /* istanbul ignore next */ 58 | return null; 59 | } 60 | promise = this.documentClient.transactGet(transactionRequest).promise(); 61 | } 62 | else if (this.operation === "batchGet") { 63 | const batchGetRequest = this.createBatchGetRequest(); 64 | if (batchGetRequest === null) { 65 | /* istanbul ignore next */ 66 | return null; 67 | } 68 | promise = this.documentClient.batchGet(batchGetRequest).promise(); 69 | } 70 | if (typeof this.nextToken === "number" && typeof this.chunks[this.nextToken + 1] !== "undefined") { 71 | this.nextToken = this.nextToken + 1; 72 | } 73 | else { 74 | this.nextToken = null; 75 | } 76 | return promise; 77 | } 78 | processResult(data) { 79 | let responseItems = []; 80 | if (data && data.Responses && Array.isArray(data.Responses)) { 81 | // transaction 82 | responseItems = data.Responses.map((r) => r.Item).filter(notEmpty); 83 | } 84 | else if (data && data.Responses && !Array.isArray(data.Responses)) { 85 | // batch, flatten each table response 86 | responseItems = [] 87 | .concat(...Object.values(data.Responses)) 88 | .filter(notEmpty); 89 | } 90 | if (data) { 91 | const unprocessedKeys = "UnprocessedKeys" in data && data.UnprocessedKeys; 92 | if (unprocessedKeys) { 93 | Object.entries(unprocessedKeys).forEach(([tableName, keys]) => { 94 | this.processError({ tableName, errorKeys: keys.Keys }); 95 | }); 96 | } 97 | } 98 | this.totalReturned += responseItems.length; 99 | this.results.push(...responseItems); 100 | } 101 | processError(err) { 102 | if (err && "tableName" in err && Array.isArray(this.retryKeys)) { 103 | const retryItems = splitInHalf(err.errorKeys) 104 | .filter(notEmpty) 105 | .map((k) => ({ 106 | tableName: err.tableName, 107 | keys: k, 108 | })); 109 | this.retryKeys.push(...[].concat(...retryItems)); 110 | } 111 | else if (err && "errorKeys" in err && typeof this.onUnprocessedKeys !== "undefined") { 112 | this.onUnprocessedKeys(err.errorKeys); 113 | } 114 | } 115 | isDone() { 116 | return super.isDone() && (!this.retryKeys || this.retryKeys.length === 0); 117 | } 118 | createTransactionRequest() { 119 | const currentChunk = typeof this.nextToken === "number" 120 | ? this.chunks[this.nextToken] 121 | : undefined; 122 | if (!currentChunk) { 123 | /* istanbul ignore next */ 124 | return null; 125 | } 126 | const transaction = { 127 | TransactItems: currentChunk.map((item) => ({ 128 | Get: { 129 | Key: item.keys, 130 | TableName: item.tableName, 131 | }, 132 | })), 133 | }; 134 | return transaction; 135 | } 136 | // each batch handles a single table for now... 137 | createBatchGetRequest() { 138 | const currentChunk = typeof this.nextToken === "number" ? this.chunks[this.nextToken] : undefined; 139 | if (!currentChunk) { 140 | /* istanbul ignore next */ 141 | return null; 142 | } 143 | // when multiple tables are supported in a single batch 144 | // switch to items.reduce(acc, curr) => ({...acc, [curr.tableName]: curr.keyItems,}),{}) 145 | const request = { 146 | RequestItems: { 147 | [currentChunk.tableName]: { 148 | ConsistentRead: this.consistentRead, 149 | Keys: currentChunk.keys, 150 | }, 151 | }, 152 | }; 153 | return request; 154 | } 155 | hasNextChunk() { 156 | if (typeof this.nextToken !== "number" || this.nextToken >= this.chunks.length) { 157 | return false; 158 | } 159 | return true; 160 | } 161 | } 162 | function notEmpty(val) { 163 | if (Array.isArray(val) && !val.length) { 164 | return false; 165 | } 166 | return !!val; 167 | } 168 | function splitInHalf(arr) { 169 | return [arr.slice(0, Math.ceil(arr.length / 2)), arr.slice(Math.ceil(arr.length / 2), arr.length)]; 170 | } 171 | -------------------------------------------------------------------------------- /esm/BatchWriter.d.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { Key, KeyDefinition } from "./types"; 3 | type BatchWriteItems = { 4 | tableName: string; 5 | records: Key[]; 6 | }; 7 | export declare class BatchWriter { 8 | private client; 9 | private tableName; 10 | private activeRequests; 11 | private chunks; 12 | private nextToken; 13 | private retryKeys; 14 | private errors; 15 | private batchSize; 16 | private bufferCapacity; 17 | private backoffActive; 18 | private onUnprocessedItems; 19 | constructor(client: DocumentClient, items: BatchWriteItems, options: { 20 | onUnprocessedItems?: (keys: Key[]) => void; 21 | batchSize: number; 22 | bufferCapacity: number; 23 | }); 24 | execute(): Promise; 25 | private chunkBatchWrites; 26 | private writeChunk; 27 | private getNextChunk; 28 | private isActive; 29 | private processResult; 30 | private retry; 31 | private isDone; 32 | private hasNextChunk; 33 | } 34 | export {}; 35 | -------------------------------------------------------------------------------- /esm/BatchWriter.js: -------------------------------------------------------------------------------- 1 | export class BatchWriter { 2 | constructor(client, items, options) { 3 | this.activeRequests = []; 4 | this.retryKeys = []; 5 | this.errors = null; 6 | this.batchSize = 25; 7 | this.bufferCapacity = 3; 8 | this.backoffActive = false; 9 | this.client = client; 10 | this.tableName = items.tableName; 11 | this.batchSize = options.batchSize; 12 | this.bufferCapacity = options.bufferCapacity; 13 | this.onUnprocessedItems = options.onUnprocessedItems; 14 | this.chunks = this.chunkBatchWrites(items); 15 | this.nextToken = 0; 16 | } 17 | async execute() { 18 | do { 19 | if (this.errors) { 20 | return Promise.reject(this.errors); 21 | } 22 | if (!this.isDone()) { 23 | await this.writeChunk(); 24 | } 25 | } while (!this.isDone()); 26 | await Promise.all(this.activeRequests); 27 | } 28 | chunkBatchWrites(items) { 29 | const chunks = []; 30 | let i = 0; 31 | const n = items.records.length; 32 | while (i < n) { 33 | chunks.push(items.records.slice(i, (i += this.batchSize || 25))); 34 | } 35 | return chunks; 36 | } 37 | async writeChunk() { 38 | if (this.retryKeys && this.retryKeys.length && this.nextToken === null && !this.isActive()) { 39 | // if finished fetching initial requests, begin to process the retry keys 40 | return this.retry(); 41 | } 42 | else if (this.activeRequests.length >= this.bufferCapacity || this.nextToken === null || this.backoffActive) { 43 | // return the current promise if buffer at capacity, or if there are no more items to fetch 44 | return this.activeRequests[0] || null; 45 | } 46 | else if (!this.hasNextChunk()) { 47 | this.nextToken = null; 48 | // let the caller wait until all active requests are finished 49 | return Promise.all(this.activeRequests).then(); 50 | } 51 | const chunk = this.getNextChunk(); 52 | if (chunk) { 53 | const request = this.client.batchWrite({ 54 | RequestItems: { 55 | [this.tableName]: chunk.map((item) => ({ 56 | PutRequest: { 57 | Item: item, 58 | }, 59 | })), 60 | }, 61 | }); 62 | if (request && typeof request.on === "function") { 63 | request.on("retry", (e) => { 64 | var _a; 65 | if ((_a = e === null || e === void 0 ? void 0 : e.error) === null || _a === void 0 ? void 0 : _a.retryable) { 66 | // reduce buffer capacity on retryable error 67 | this.bufferCapacity = Math.max(Math.floor((this.bufferCapacity * 3) / 4), 5); 68 | this.backoffActive = true; 69 | } 70 | }); 71 | } 72 | const promise = request 73 | .promise() 74 | .catch((e) => { 75 | console.error("Error: AWS Error, Put Items", e); 76 | if (this.onUnprocessedItems) { 77 | this.onUnprocessedItems(chunk); 78 | } 79 | this.errors = e; 80 | }) 81 | .then((results) => { 82 | this.processResult(results, promise); 83 | }); 84 | this.activeRequests.push(promise); 85 | } 86 | } 87 | getNextChunk() { 88 | if (this.nextToken === null) { 89 | /* istanbul ignore next */ 90 | return null; 91 | } 92 | const chunk = this.chunks[this.nextToken] || null; 93 | this.nextToken += 1; 94 | return chunk; 95 | } 96 | isActive() { 97 | return this.activeRequests.length > 0; 98 | } 99 | processResult(data, request) { 100 | var _a; 101 | this.activeRequests = this.activeRequests.filter((r) => r !== request); 102 | if (!this.activeRequests.length || !data || !data.UnprocessedItems) { 103 | this.backoffActive = false; 104 | } 105 | if (data && data.UnprocessedItems && (((_a = data.UnprocessedItems[this.tableName]) === null || _a === void 0 ? void 0 : _a.length) || 0) > 0) { 106 | // eslint-disable-next-line 107 | const unprocessedItems = data.UnprocessedItems[this.tableName].map((ui) => { var _a; return (_a = ui.PutRequest) === null || _a === void 0 ? void 0 : _a.Item; }); 108 | if (Array.isArray(this.retryKeys)) { 109 | const retryItems = splitInHalf(unprocessedItems).filter(notEmpty); 110 | this.retryKeys.push(...retryItems); 111 | } 112 | else if (this.onUnprocessedItems) { 113 | this.onUnprocessedItems(unprocessedItems); 114 | } 115 | } 116 | } 117 | retry() { 118 | this.chunks = this.retryKeys || []; 119 | this.nextToken = 0; 120 | this.retryKeys = null; 121 | return this.writeChunk(); 122 | } 123 | isDone() { 124 | return !this.isActive() && (!this.retryKeys || this.retryKeys.length === 0) && this.nextToken === null; 125 | } 126 | hasNextChunk() { 127 | if (this.nextToken === null || this.nextToken >= this.chunks.length) { 128 | return false; 129 | } 130 | return true; 131 | } 132 | } 133 | function notEmpty(val) { 134 | if (Array.isArray(val) && !val.length) { 135 | return false; 136 | } 137 | return !!val; 138 | } 139 | function splitInHalf(arr) { 140 | return [arr.slice(0, Math.ceil(arr.length / 2)), arr.slice(Math.ceil(arr.length / 2), arr.length)]; 141 | } 142 | -------------------------------------------------------------------------------- /esm/Pipeline.d.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { ConditionExpression, UpdateReturnValues, PrimitiveType, Key } from "./types"; 3 | import { TableIterator } from "./TableIterator"; 4 | import { ScanQueryPipeline } from "./ScanQueryPipeline"; 5 | export declare class Pipeline extends ScanQueryPipeline { 12 | constructor(tableName: string, keys: { 13 | pk: PK; 14 | sk?: SK; 15 | }, config?: { 16 | client?: DocumentClient; 17 | readBuffer?: number; 18 | writeBuffer?: number; 19 | readBatchSize?: number; 20 | writeBatchSize?: number; 21 | }); 22 | withWriteBuffer(writeBuffer?: number): this; 23 | withWriteBatchSize(writeBatchSize?: number): this; 24 | createIndex(name: string, definition: { 25 | pk: PK2; 26 | sk?: SK2; 27 | }): ScanQueryPipeline; 28 | transactGet(keys: Key[] | { 29 | tableName: string; 30 | keys: Key; 31 | keyDefinition: KD2; 32 | }[], options?: { 33 | bufferCapacity?: number; 34 | }): TableIterator; 35 | getItems(keys: Key[], options?: { 36 | batchSize?: number; 37 | bufferCapacity?: number; 38 | consistentRead?: boolean; 39 | }): TableIterator; 40 | putItems>(items: I[], options?: { 41 | bufferCapacity?: number; 42 | disableSlowStart?: boolean; 43 | batchSize?: number; 44 | }): Promise>; 45 | put>(item: Item, condition?: ConditionExpression): Promise>; 46 | putIfNotExists>(item: Item): Promise>; 47 | buildUpdateRequest(key: Key, attributes: Record, options?: { 48 | condition?: ConditionExpression; 49 | returnType?: UpdateReturnValues; 50 | }): DocumentClient.UpdateItemInput & { 51 | UpdateExpression: string; 52 | }; 53 | update(key: Key, attributes: Record, options?: { 54 | condition?: ConditionExpression; 55 | returnType?: UpdateReturnValues; 56 | }): Promise; 57 | delete(key: Key, options?: { 58 | condition?: ConditionExpression | undefined; 59 | returnType?: "ALL_OLD"; 60 | reportError?: boolean; 61 | }): Promise; 62 | handleUnprocessed(callback: (item: Record) => void): Pipeline; 63 | private keyAttributesOnlyFromArray; 64 | private keyAttributesOnly; 65 | } 66 | -------------------------------------------------------------------------------- /esm/Pipeline.js: -------------------------------------------------------------------------------- 1 | import { BatchGetFetcher } from "./BatchFetcher"; 2 | import { TableIterator } from "./TableIterator"; 3 | import { BatchWriter } from "./BatchWriter"; 4 | import { conditionToDynamo, pkName } from "./helpers"; 5 | import { ScanQueryPipeline } from "./ScanQueryPipeline"; 6 | export class Pipeline extends ScanQueryPipeline { 7 | constructor(tableName, keys, config) { 8 | super(tableName, keys, undefined, config); 9 | return this; 10 | } 11 | withWriteBuffer(writeBuffer = 30) { 12 | if (writeBuffer < 0) { 13 | throw new Error("Write buffer out of range"); 14 | } 15 | this.config.writeBuffer = writeBuffer; 16 | return this; 17 | } 18 | withWriteBatchSize(writeBatchSize = 25) { 19 | if (writeBatchSize < 1 || writeBatchSize > 25) { 20 | throw new Error("Write batch size out of range"); 21 | } 22 | this.config.writeBatchSize = writeBatchSize; 23 | return this; 24 | } 25 | createIndex(name, definition) { 26 | const { keys, ...config } = this.config; 27 | return new ScanQueryPipeline(this.config.table, definition, name, config); 28 | } 29 | transactGet(keys, options) { 30 | // get keys into a standard format, filter out any non-key attributes 31 | const transactGetItems = typeof keys[0] !== "undefined" && !("tableName" in keys[0]) && !("keys" in keys[0]) 32 | ? keys.map((k) => ({ 33 | tableName: this.config.table, 34 | keys: this.keyAttributesOnly(k, this.config.keys), 35 | })) 36 | : keys.map((key) => ({ 37 | tableName: key.tableName, 38 | keys: this.keyAttributesOnly(key.keys, key.keyDefinition), 39 | })); 40 | return new TableIterator(new BatchGetFetcher(this.config.client, "transactGet", transactGetItems, { 41 | bufferCapacity: this.config.readBuffer, 42 | batchSize: this.config.readBatchSize, 43 | ...options, 44 | }), this); 45 | } 46 | getItems(keys, options) { 47 | const handleUnprocessed = (keys) => { 48 | this.unprocessedItems.push(...keys); 49 | }; 50 | if (typeof (options === null || options === void 0 ? void 0 : options.batchSize) === "number" && (options.batchSize > 100 || options.batchSize < 1)) { 51 | throw new Error("Batch size out of range"); 52 | } 53 | if (typeof (options === null || options === void 0 ? void 0 : options.bufferCapacity) === "number" && options.bufferCapacity < 0) { 54 | throw new Error("Buffer capacity is out of range"); 55 | } 56 | // filter out any non-key attributes 57 | const tableKeys = this.keyAttributesOnlyFromArray(keys, this.config.keys); 58 | const batchGetItems = { tableName: this.config.table, keys: tableKeys }; 59 | return new TableIterator(new BatchGetFetcher(this.config.client, "batchGet", batchGetItems, { 60 | batchSize: this.config.readBatchSize, 61 | bufferCapacity: this.config.readBuffer, 62 | onUnprocessedKeys: handleUnprocessed, 63 | ...options, 64 | }), this); 65 | } 66 | async putItems(items, options) { 67 | const handleUnprocessed = (keys) => { 68 | this.unprocessedItems.push(...keys); 69 | }; 70 | if (typeof (options === null || options === void 0 ? void 0 : options.bufferCapacity) === "number" && options.bufferCapacity < 0) { 71 | throw new Error("Buffer capacity is out of range"); 72 | } 73 | if (typeof (options === null || options === void 0 ? void 0 : options.batchSize) === "number" && (options.batchSize < 1 || options.batchSize > 25)) { 74 | throw new Error("Batch size is out of range"); 75 | } 76 | const writer = new BatchWriter(this.config.client, { tableName: this.config.table, records: items }, { 77 | batchSize: this.config.writeBatchSize, 78 | bufferCapacity: this.config.writeBuffer, 79 | onUnprocessedItems: handleUnprocessed, 80 | ...options, 81 | }); 82 | await writer.execute(); 83 | return this; 84 | } 85 | put(item, condition) { 86 | const request = { 87 | TableName: this.config.table, 88 | Item: item, 89 | }; 90 | if (condition) { 91 | const compiledCondition = conditionToDynamo(condition); 92 | request.ConditionExpression = compiledCondition.Condition; 93 | request.ExpressionAttributeNames = compiledCondition.ExpressionAttributeNames; 94 | request.ExpressionAttributeValues = compiledCondition.ExpressionAttributeValues; 95 | } 96 | return this.config.client 97 | .put(request) 98 | .promise() 99 | .catch((e) => { 100 | console.error("Error: AWS Error, Put,", e); 101 | this.unprocessedItems.push(item); 102 | }) 103 | .then(() => this); 104 | } 105 | putIfNotExists(item) { 106 | const pkCondition = { 107 | operator: "attribute_not_exists", 108 | property: pkName(this.config.keys), 109 | }; 110 | return this.put(item, pkCondition); 111 | } 112 | buildUpdateRequest(key, attributes, options) { 113 | const attributesToUpdate = Object.entries(attributes).filter(([_key, val]) => val !== undefined); 114 | const attributesToRemove = Object.entries(attributes).filter(([_key, val]) => val === undefined); 115 | const setExpression = attributesToUpdate 116 | .map(([k, _v]) => `#${k.replace(/[#.:]/g, "")} = :${k.replace(/[#.:]/g, "")}`) 117 | .join(", "); 118 | const removeExpression = attributesToRemove.map(([k, _v]) => `#${k.replace(/[#.:]/g, "")}`).join(", "); 119 | const expressionNames = Object.keys(attributes).reduce((acc, curr) => ({ ...acc, ["#" + curr.replace(/[#.:]/g, "")]: curr }), {}); 120 | const expressionValues = attributesToUpdate.reduce((acc, curr) => ({ 121 | ...acc, 122 | [":" + curr[0].replace(/[#.:]/g, "")]: curr[1], 123 | }), {}); 124 | const updateExpression = [ 125 | setExpression.length ? `SET ${setExpression}` : "", 126 | removeExpression.length ? `REMOVE ${removeExpression}` : "", 127 | ] 128 | .filter((i) => i === null || i === void 0 ? void 0 : i.length) 129 | .join(" "); 130 | const request = { 131 | TableName: this.config.table, 132 | Key: this.keyAttributesOnly(key, this.config.keys), 133 | UpdateExpression: updateExpression, 134 | ...(Object.keys(expressionNames).length > 0 && { 135 | ExpressionAttributeNames: expressionNames, 136 | }), 137 | ...(Object.keys(expressionValues).length > 0 && { 138 | ExpressionAttributeValues: expressionValues, 139 | }), 140 | ...((options === null || options === void 0 ? void 0 : options.returnType) && { ReturnValues: options.returnType }), 141 | }; 142 | if (options === null || options === void 0 ? void 0 : options.condition) { 143 | const compiledCondition = conditionToDynamo(options.condition, { 144 | Condition: "", 145 | ...(Object.keys(expressionNames).length > 0 && { 146 | ExpressionAttributeNames: expressionNames, 147 | }), 148 | ...(Object.keys(expressionValues).length > 0 && { 149 | ExpressionAttributeValues: expressionValues, 150 | }), 151 | }); 152 | request.ConditionExpression = compiledCondition.Condition; 153 | request.ExpressionAttributeNames = compiledCondition.ExpressionAttributeNames; 154 | request.ExpressionAttributeValues = compiledCondition.ExpressionAttributeValues; 155 | } 156 | return request; 157 | } 158 | update(key, attributes, options) { 159 | // TODO: Cleanup and extact 160 | const request = this.buildUpdateRequest(key, attributes, options); 161 | return this.config.client 162 | .update(request) 163 | .promise() 164 | .catch((e) => { 165 | console.error("Error: AWS Error, Update", e); 166 | this.unprocessedItems.push(key); 167 | }) 168 | .then((d) => { 169 | return d && "Attributes" in d && d.Attributes ? d.Attributes : null; 170 | }); 171 | } 172 | delete(key, options) { 173 | const request = { 174 | TableName: this.config.table, 175 | Key: this.keyAttributesOnly(key, this.config.keys), 176 | ...((options === null || options === void 0 ? void 0 : options.returnType) && { ReturnValues: options.returnType }), 177 | }; 178 | if (options === null || options === void 0 ? void 0 : options.condition) { 179 | const compiledCondition = conditionToDynamo(options.condition); 180 | request.ConditionExpression = compiledCondition.Condition; 181 | request.ExpressionAttributeNames = compiledCondition.ExpressionAttributeNames; 182 | request.ExpressionAttributeValues = compiledCondition.ExpressionAttributeValues; 183 | } 184 | else { 185 | const compiledCondition = conditionToDynamo({ 186 | operator: "attribute_exists", 187 | property: pkName(this.config.keys), 188 | }); 189 | request.ConditionExpression = compiledCondition.Condition; 190 | request.ExpressionAttributeNames = compiledCondition.ExpressionAttributeNames; 191 | request.ExpressionAttributeValues = compiledCondition.ExpressionAttributeValues; 192 | } 193 | return this.config.client 194 | .delete(request) 195 | .promise() 196 | .catch((e) => { 197 | if (options === null || options === void 0 ? void 0 : options.reportError) { 198 | console.error("Error: AWS Error, Delete", e, request); 199 | this.unprocessedItems.push(key); 200 | } 201 | }) 202 | .then((old) => (old && "Attributes" in old && old.Attributes ? old.Attributes : null)); 203 | } 204 | handleUnprocessed(callback) { 205 | this.unprocessedItems.map(callback); 206 | return this; 207 | } 208 | keyAttributesOnlyFromArray(items, keyDefinition) { 209 | return items.map((item) => this.keyAttributesOnly(item, keyDefinition)); 210 | } 211 | keyAttributesOnly(item, keyDefinition) { 212 | return { 213 | [keyDefinition.pk]: item[keyDefinition.pk], 214 | ...(typeof this.config.keys.sk === "string" && { 215 | [this.config.keys.sk]: item[this.config.keys.sk], 216 | }), 217 | }; 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /esm/QueryFetcher.d.ts: -------------------------------------------------------------------------------- 1 | import { AbstractFetcher } from "./AbstractFetcher"; 2 | import { ScanInput, QueryInput, DocumentClient } from "aws-sdk/clients/dynamodb"; 3 | export declare class QueryFetcher extends AbstractFetcher { 4 | private request; 5 | private operation; 6 | constructor(request: ScanInput | QueryInput, client: DocumentClient, operation: "query" | "scan", options: { 7 | batchSize: number; 8 | bufferCapacity: number; 9 | limit?: number; 10 | nextToken?: DocumentClient.Key; 11 | }); 12 | fetchStrategy(): null | Promise; 13 | processResult(data: DocumentClient.ScanOutput | DocumentClient.QueryOutput | void): void; 14 | getResultBatch(batchSize: number): T[]; 15 | } 16 | -------------------------------------------------------------------------------- /esm/QueryFetcher.js: -------------------------------------------------------------------------------- 1 | import { AbstractFetcher } from "./AbstractFetcher"; 2 | export class QueryFetcher extends AbstractFetcher { 3 | constructor(request, client, operation, options) { 4 | super(client, options); 5 | this.request = request; 6 | this.operation = operation; 7 | if (options.nextToken) { 8 | this.nextToken = options.nextToken; 9 | } 10 | else { 11 | this.nextToken = 1; 12 | } 13 | } 14 | // TODO: remove null response type 15 | fetchStrategy() { 16 | // no support for parallel query 17 | // 1. 1 active request allowed at a time 18 | // 2. Do not create a new request when the buffer is full 19 | // 3. If there are no more items to fetch, exit 20 | if (this.activeRequests.length > 0 || this.bufferSize > this.bufferCapacity || !this.nextToken) { 21 | return this.activeRequests[0] || null; 22 | } 23 | const request = { 24 | ...(this.request.Limit && { 25 | Limit: this.request.Limit - this.totalReturned, 26 | }), 27 | ...this.request, 28 | ...(Boolean(this.nextToken) && 29 | typeof this.nextToken === "object" && { 30 | ExclusiveStartKey: this.nextToken, 31 | }), 32 | }; 33 | const promise = this.documentClient[this.operation](request).promise(); 34 | return promise; 35 | } 36 | processResult(data) { 37 | this.nextToken = (data && data.LastEvaluatedKey) || null; 38 | if (data && data.Items) { 39 | this.totalReturned += data.Items.length; 40 | this.results.push(...data.Items); 41 | } 42 | } 43 | // override since filtering results in inconsistent result set size, base buffer on the items returned last 44 | // this may give surprising results if the returned list varies considerably, but errs on the side of caution. 45 | getResultBatch(batchSize) { 46 | const items = super.getResultBatch(batchSize); 47 | if (items.length > 0) { 48 | this.bufferSize = this.results.length / items.length; 49 | } 50 | else if (!this.activeRequests.length) { 51 | // if we don't have any items to process, and no active requests, buffer size should be zero. 52 | this.bufferSize = 0; 53 | } 54 | return items; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /esm/ScanQueryPipeline.d.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { TableIterator } from "./TableIterator"; 3 | import { ComparisonOperator, ConditionExpression, Key, KeyConditions, QueryTemplate, Scalar } from "./types"; 4 | export type SortArgs = [Exclude">, Scalar] | ["between", Scalar, Scalar]; 5 | export declare const sortKey: (...args: SortArgs) => QueryTemplate; 6 | export declare class ScanQueryPipeline { 13 | config: { 14 | client: DocumentClient; 15 | table: string; 16 | keys: KD; 17 | index?: string; 18 | readBuffer: number; 19 | writeBuffer: number; 20 | readBatchSize: number; 21 | writeBatchSize: number; 22 | }; 23 | unprocessedItems: Key[]; 24 | constructor(tableName: string, keys: { 25 | pk: PK; 26 | sk?: SK; 27 | }, index?: string, config?: { 28 | client?: DocumentClient; 29 | readBuffer?: number; 30 | writeBuffer?: number; 31 | readBatchSize?: number; 32 | writeBatchSize?: number; 33 | }); 34 | static sortKey: (...args: SortArgs) => QueryTemplate; 35 | sortKey: (...args: SortArgs) => QueryTemplate; 36 | withReadBuffer(readBuffer: number): this; 37 | withReadBatchSize(readBatchSize: number): this; 38 | query(keyConditions: KeyConditions<{ 39 | pk: PK; 40 | sk: SK; 41 | }>, options?: { 42 | sortDescending?: true; 43 | batchSize?: number; 44 | bufferCapacity?: number; 45 | limit?: number; 46 | filters?: ConditionExpression; 47 | consistentRead?: boolean; 48 | nextToken?: Key; 49 | }): TableIterator; 50 | scan(options?: { 51 | batchSize?: number; 52 | bufferCapacity?: number; 53 | limit?: number; 54 | filters?: ConditionExpression; 55 | consistentRead?: boolean; 56 | nextToken?: Key; 57 | }): TableIterator; 58 | private buildQueryScanRequest; 59 | } 60 | -------------------------------------------------------------------------------- /esm/ScanQueryPipeline.js: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { conditionToDynamo, skQueryToDynamoString } from "./helpers"; 3 | import { QueryFetcher } from "./QueryFetcher"; 4 | import { TableIterator } from "./TableIterator"; 5 | export const sortKey = (...args) => { 6 | if (args.length === 3) { 7 | return ["between", "and", args[1], args[2]]; 8 | } 9 | return args; 10 | }; 11 | export class ScanQueryPipeline { 12 | constructor(tableName, keys, index, config) { 13 | this.sortKey = sortKey; 14 | this.config = { 15 | table: tableName, 16 | readBuffer: 1, 17 | writeBuffer: 3, 18 | readBatchSize: 100, 19 | writeBatchSize: 25, 20 | ...config, 21 | // shortcut to use KD, otherwise type definitions throughout the 22 | // class are too long 23 | keys: keys, 24 | index, 25 | client: (config && config.client) || new DocumentClient(), 26 | }; 27 | this.unprocessedItems = []; 28 | return this; 29 | } 30 | withReadBuffer(readBuffer) { 31 | if (readBuffer < 0) { 32 | throw new Error("Read buffer out of range"); 33 | } 34 | this.config.readBuffer = readBuffer; 35 | return this; 36 | } 37 | withReadBatchSize(readBatchSize) { 38 | if (readBatchSize < 1) { 39 | throw new Error("Read batch size out of range"); 40 | } 41 | this.config.readBatchSize = readBatchSize; 42 | return this; 43 | } 44 | query(keyConditions, options) { 45 | const request = this.buildQueryScanRequest({ ...options, keyConditions }); 46 | const fetchOptions = { 47 | bufferCapacity: this.config.readBuffer, 48 | batchSize: this.config.readBatchSize, 49 | ...options, 50 | }; 51 | return new TableIterator(new QueryFetcher(request, this.config.client, "query", fetchOptions), this); 52 | } 53 | scan(options) { 54 | const request = this.buildQueryScanRequest(options !== null && options !== void 0 ? options : {}); 55 | const fetchOptions = { 56 | bufferCapacity: this.config.readBuffer, 57 | batchSize: this.config.readBatchSize, 58 | ...options, 59 | }; 60 | return new TableIterator(new QueryFetcher(request, this.config.client, "scan", fetchOptions), this); 61 | } 62 | buildQueryScanRequest(options) { 63 | const pkName = this.config.keys.pk; 64 | const skName = this.config.keys.sk; 65 | const skValue = options.keyConditions && typeof skName !== "undefined" && options.keyConditions && skName in options.keyConditions 66 | ? options.keyConditions[skName] 67 | : null; 68 | const request = { 69 | TableName: this.config.table, 70 | ...(options.limit && { 71 | Limit: options.limit, 72 | }), 73 | ...(this.config.index && { IndexName: this.config.index }), 74 | ...(options.keyConditions && { 75 | KeyConditionExpression: "#p0 = :v0" + (skValue ? ` AND ${skQueryToDynamoString(skValue)}` : ""), 76 | }), 77 | ConsistentRead: Boolean(options.consistentRead), 78 | ScanIndexForward: Boolean(!options.sortDescending), 79 | }; 80 | const [skVal1, skVal2] = (skValue === null || skValue === void 0 ? void 0 : skValue.length) === 4 ? [skValue[2], skValue[3]] : (skValue === null || skValue === void 0 ? void 0 : skValue.length) === 2 ? [skValue[1], null] : [null, null]; 81 | const keySubstitues = { 82 | Condition: "", 83 | ExpressionAttributeNames: options.keyConditions 84 | ? { 85 | "#p0": pkName, 86 | ...(skValue && { 87 | "#p1": skName, 88 | }), 89 | } 90 | : undefined, 91 | ExpressionAttributeValues: options.keyConditions 92 | ? { 93 | ":v0": options.keyConditions[pkName], 94 | ...(skVal1 !== null && { 95 | ":v1": skVal1, 96 | }), 97 | ...(skVal2 !== null && { 98 | ":v2": skVal2, 99 | }), 100 | } 101 | : undefined, 102 | }; 103 | if (options.filters) { 104 | const compiledCondition = conditionToDynamo(options.filters, keySubstitues); 105 | request.FilterExpression = compiledCondition.Condition; 106 | request.ExpressionAttributeNames = compiledCondition.ExpressionAttributeNames; 107 | request.ExpressionAttributeValues = compiledCondition.ExpressionAttributeValues; 108 | } 109 | else { 110 | request.ExpressionAttributeNames = keySubstitues.ExpressionAttributeNames; 111 | request.ExpressionAttributeValues = keySubstitues.ExpressionAttributeValues; 112 | } 113 | return request; 114 | } 115 | } 116 | ScanQueryPipeline.sortKey = sortKey; 117 | -------------------------------------------------------------------------------- /esm/TableIterator.d.ts: -------------------------------------------------------------------------------- 1 | import DynamoDB from "aws-sdk/clients/dynamodb"; 2 | interface IteratorExecutor { 3 | execute(): AsyncGenerator; 5 | } | void, void>; 6 | } 7 | export declare class TableIterator { 8 | private lastEvaluatedKeyHandlers; 9 | config: { 10 | parent: P; 11 | fetcher: IteratorExecutor; 12 | }; 13 | constructor(fetcher: IteratorExecutor, parent?: P); 14 | forEachStride(iterator: (items: T[], index: number, parent: P, cancel: () => void) => Promise | void): Promise

; 15 | onLastEvaluatedKey(handler: (lastEvaluatedKey: Record) => void): this; 16 | private iterate; 17 | private handleDone; 18 | forEach(iterator: (item: T, index: number, pipeline: P, cancel: () => void) => Promise | void): Promise

; 19 | map(iterator: (item: T, index: number) => U): Promise; 20 | filterLazy(predicate: (item: T, index: number) => boolean): TableIterator; 21 | mapLazy(iterator: (item: T, index: number) => U): TableIterator; 22 | all(): Promise; 23 | iterator(): AsyncGenerator; 24 | strideIterator(): AsyncGenerator | void, void>; 25 | } 26 | export {}; 27 | -------------------------------------------------------------------------------- /esm/TableIterator.js: -------------------------------------------------------------------------------- 1 | export class TableIterator { 2 | constructor(fetcher, parent) { 3 | this.lastEvaluatedKeyHandlers = []; 4 | this.config = { parent: parent, fetcher }; 5 | } 6 | async forEachStride(iterator) { 7 | let index = 0; 8 | await this.iterate(this.config.fetcher, async (stride, cancel) => { 9 | await iterator(stride, index, this.config.parent, cancel); 10 | index += 1; 11 | }); 12 | return this.config.parent; 13 | } 14 | onLastEvaluatedKey(handler) { 15 | this.lastEvaluatedKeyHandlers.push(handler); 16 | return this; 17 | } 18 | async iterate(fetcher, iterator) { 19 | let cancelled = false; 20 | const cancel = () => { 21 | cancelled = true; 22 | }; 23 | const executor = fetcher.execute(); 24 | while (true) { 25 | if (cancelled) { 26 | break; 27 | } 28 | const stride = await executor.next(); 29 | const { value } = stride; 30 | if (stride.done) { 31 | this.handleDone(stride); 32 | break; 33 | } 34 | await iterator(value, cancel); 35 | } 36 | } 37 | handleDone(iteratorResponse) { 38 | const { value } = iteratorResponse; 39 | if (value && "lastEvaluatedKey" in value) { 40 | this.lastEvaluatedKeyHandlers.forEach((h) => h(value.lastEvaluatedKey)); 41 | this.lastEvaluatedKeyHandlers = []; 42 | } 43 | } 44 | // when a promise is returned, all promises are resolved in the batch before processing the next batch 45 | async forEach(iterator) { 46 | let index = 0; 47 | let iteratorPromises = []; 48 | let cancelled = false; 49 | const cancelForEach = () => { 50 | cancelled = true; 51 | }; 52 | await this.iterate(this.config.fetcher, async (stride, cancel) => { 53 | iteratorPromises = []; 54 | for (const item of stride) { 55 | const iteratorResponse = iterator(item, index, this.config.parent, cancelForEach); 56 | index += 1; 57 | if (cancelled) { 58 | await Promise.all(iteratorPromises); 59 | cancel(); 60 | break; 61 | } 62 | else if (typeof iteratorResponse === "object" && iteratorResponse instanceof Promise) { 63 | iteratorPromises.push(iteratorResponse); 64 | } 65 | } 66 | await Promise.all(iteratorPromises); 67 | }); 68 | await Promise.all(iteratorPromises); 69 | return this.config.parent; 70 | } 71 | async map(iterator) { 72 | const results = []; 73 | let index = 0; 74 | await this.iterate(this.config.fetcher, (stride, _cancel) => { 75 | for (const item of stride) { 76 | results.push(iterator(item, index)); 77 | index += 1; 78 | } 79 | return Promise.resolve(); 80 | }); 81 | return results; 82 | } 83 | filterLazy(predicate) { 84 | const existingFetcher = this.config.fetcher; 85 | let index = 0; 86 | // eslint-disable-next-line @typescript-eslint/no-this-alias 87 | const that = this; 88 | const fetcher = async function* () { 89 | const executor = existingFetcher.execute(); 90 | while (true) { 91 | const stride = await executor.next(); 92 | if (stride.done) { 93 | that.handleDone(stride); 94 | break; 95 | } 96 | yield stride.value.filter((val, i) => { 97 | const filtered = predicate(val, index); 98 | index += 1; 99 | return filtered; 100 | }); 101 | } 102 | }; 103 | return new TableIterator({ execute: fetcher }, this.config.parent); 104 | } 105 | mapLazy(iterator) { 106 | const existingFetcher = this.config.fetcher; 107 | let results = []; 108 | let index = 0; 109 | // eslint-disable-next-line @typescript-eslint/no-this-alias 110 | const that = this; 111 | const fetcher = async function* () { 112 | const executor = existingFetcher.execute(); 113 | while (true) { 114 | const stride = await executor.next(); 115 | if (stride.done) { 116 | that.handleDone(stride); 117 | break; 118 | } 119 | results = stride.value.map((item) => { 120 | const result = iterator(item, index); 121 | index += 1; 122 | return result; 123 | }); 124 | yield results; 125 | } 126 | }; 127 | return new TableIterator({ execute: fetcher }, this.config.parent); 128 | } 129 | all() { 130 | const result = this.map((i) => i); 131 | return result; 132 | } 133 | async *iterator() { 134 | const executor = this.config.fetcher.execute(); 135 | while (true) { 136 | const stride = await executor.next(); 137 | if (stride.done) { 138 | this.handleDone(stride); 139 | return; 140 | } 141 | for (const item of stride.value) { 142 | yield item; 143 | } 144 | } 145 | } 146 | strideIterator() { 147 | return this.config.fetcher.execute(); 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /esm/helpers/index.d.ts: -------------------------------------------------------------------------------- 1 | import { ConditionExpression, DynamoCondition, KeyDefinition, QueryTemplate } from "../types"; 2 | export declare function conditionToDynamo(condition: ConditionExpression | undefined, mergeCondition?: DynamoCondition): DynamoCondition; 3 | export declare const pkName: (keys: KeyDefinition) => string; 4 | export declare function skQueryToDynamoString(template: QueryTemplate): string; 5 | -------------------------------------------------------------------------------- /esm/helpers/index.js: -------------------------------------------------------------------------------- 1 | export function conditionToDynamo(condition, mergeCondition) { 2 | const result = mergeCondition || 3 | { 4 | Condition: "", 5 | }; 6 | if (!condition) { 7 | return result; 8 | } 9 | if ("logical" in condition) { 10 | const preCondition = result.Condition; 11 | const logicalLhs = conditionToDynamo(condition.lhs, result); 12 | const logicalRhs = conditionToDynamo(condition.rhs, { 13 | Condition: preCondition, 14 | ExpressionAttributeNames: { 15 | ...result.ExpressionAttributeNames, 16 | ...logicalLhs.ExpressionAttributeNames, 17 | }, 18 | ExpressionAttributeValues: { 19 | ...result.ExpressionAttributeValues, 20 | ...logicalLhs.ExpressionAttributeValues, 21 | }, 22 | }); 23 | if (condition.lhs && "logical" in condition.lhs) { 24 | logicalLhs.Condition = `(${logicalLhs.Condition})`; 25 | } 26 | if (condition.rhs && "logical" in condition.rhs) { 27 | logicalRhs.Condition = `(${logicalRhs.Condition})`; 28 | } 29 | result.Condition = `${logicalLhs.Condition + (logicalLhs.Condition.length ? " " : "")}${condition.logical} ${logicalRhs.Condition}`; 30 | Object.entries({ 31 | ...logicalRhs.ExpressionAttributeNames, 32 | ...logicalLhs.ExpressionAttributeNames, 33 | }).forEach(([name, value]) => { 34 | if (!result.ExpressionAttributeNames) { 35 | result.ExpressionAttributeNames = {}; 36 | } 37 | // @ts-expect-error: Object.entries hard codes string as the key type, 38 | // and indexing by template strings is invalid in ts 4.2.0 39 | result.ExpressionAttributeNames[name] = value; 40 | }); 41 | Object.entries({ 42 | ...logicalRhs.ExpressionAttributeValues, 43 | ...logicalLhs.ExpressionAttributeValues, 44 | }).forEach(([name, value]) => { 45 | if (!result.ExpressionAttributeValues) { 46 | result.ExpressionAttributeValues = {}; 47 | } 48 | result.ExpressionAttributeValues[name] = value; 49 | }); 50 | return result; 51 | } 52 | const names = conditionToAttributeNames(condition, result.ExpressionAttributeNames ? Object.keys(result.ExpressionAttributeNames).length : 0); 53 | const values = conditionToAttributeValues(condition, result.ExpressionAttributeValues ? Object.keys(result.ExpressionAttributeValues).length : 0); 54 | const conditionString = conditionToConditionString(condition, result.ExpressionAttributeNames ? Object.keys(result.ExpressionAttributeNames).length : 0, result.ExpressionAttributeValues ? Object.keys(result.ExpressionAttributeValues).length : 0); 55 | return { 56 | ...((Object.keys(names).length > 0 || Object.keys(result.ExpressionAttributeNames || {}).length > 0) && { 57 | ExpressionAttributeNames: { 58 | ...names, 59 | ...result.ExpressionAttributeNames, 60 | }, 61 | }), 62 | ...((Object.keys(values).length > 0 || Object.keys(result.ExpressionAttributeValues || {}).length > 0) && { 63 | ExpressionAttributeValues: { 64 | ...values, 65 | ...result.ExpressionAttributeValues, 66 | }, 67 | }), 68 | Condition: conditionString, 69 | }; 70 | } 71 | export const pkName = (keys) => keys.pk; 72 | export function skQueryToDynamoString(template) { 73 | const expression = template[0] === "begins_with" 74 | ? { operator: template[0], property: "sk", value: template[1] } 75 | : template[0] === "between" 76 | ? { 77 | operator: template[0], 78 | property: "sk", 79 | start: template[2], 80 | end: template[3], 81 | } 82 | : { operator: template[0], lhs: "sk", rhs: { value: template[1] } }; 83 | const result = conditionToConditionString(expression, 1, 1); 84 | return result; 85 | } 86 | function comparisonOperator(condition, nameStart, valueStart) { 87 | const lhs = typeof condition.lhs === "string" ? "#p" + nameStart.toString() : "#p" + nameStart.toString(); 88 | (typeof condition.lhs === "string" || "property" in condition.lhs) && (nameStart += 1); 89 | const rhs = "property" in condition.rhs ? "#p" + nameStart.toString() : ":v" + valueStart.toString(); 90 | return `${typeof condition.lhs !== "string" && "function" in condition.lhs ? condition.lhs.function + "(" : ""}${lhs}${typeof condition.lhs !== "string" && "function" in condition.lhs ? ")" : ""} ${condition.operator} ${"function" in condition.rhs ? condition.rhs.function + "(" : ""}${rhs}${"function" in condition.rhs ? ")" : ""}`; 91 | } 92 | function conditionToConditionString(condition, nameCountStart, valueCountStart) { 93 | // TODO: HACK: the name and value conversions follow the same operator flow 94 | // as the condition to values and condition to names to keep the numbers in sync 95 | // lhs, rhs, start,end,list 96 | // lhs, rhs, property, arg2 97 | if ("logical" in condition) { 98 | /* istanbul ignore next */ 99 | throw new Error("Unimplemented"); 100 | } 101 | const nameStart = nameCountStart; 102 | let valueStart = valueCountStart; 103 | switch (condition.operator) { 104 | case ">": 105 | case "<": 106 | case ">=": 107 | case "<=": 108 | case "=": 109 | case "<>": 110 | // TODO: fix any type 111 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 112 | return comparisonOperator(condition, nameStart, valueStart); 113 | case "begins_with": 114 | case "contains": 115 | case "attribute_type": 116 | return `${condition.operator}(#p${nameStart}, :v${valueStart})`; 117 | case "attribute_exists": 118 | case "attribute_not_exists": 119 | return `${condition.operator}(#p${nameStart})`; 120 | case "between": 121 | return `#p${nameStart} BETWEEN :v${valueStart} AND :v${valueStart + 1}`; 122 | case "in": 123 | return `${"#p" + nameStart.toString()} IN (${condition.list 124 | .map(() => { 125 | valueStart += 1; 126 | return `:v${valueStart - 1}`; 127 | }) 128 | .join(",")})`; 129 | default: 130 | /* istanbul ignore next */ 131 | throw new Error("Operator does not exist"); 132 | } 133 | } 134 | function conditionToAttributeValues(condition, countStart = 0) { 135 | const values = {}; 136 | if ("rhs" in condition && condition.rhs && "value" in condition.rhs) { 137 | setPropertyValue(condition.rhs.value, values, countStart); 138 | } 139 | if ("value" in condition) { 140 | setPropertyValue(condition.value, values, countStart); 141 | } 142 | if ("start" in condition) { 143 | setPropertyValue(condition.start, values, countStart); 144 | } 145 | if ("end" in condition) { 146 | setPropertyValue(condition.end, values, countStart); 147 | } 148 | if ("list" in condition) { 149 | condition.list.forEach((l) => setPropertyValue(l, values, countStart)); 150 | } 151 | return values; 152 | } 153 | function setPropertyValue(value, values, countStart) { 154 | // note this is the main place to change if we switch from document client to the regular dynamodb client 155 | const dynamoValue = Array.isArray(value) 156 | ? value.join("") 157 | : typeof value === "boolean" || typeof value === "string" || typeof value === "number" 158 | ? value 159 | : value === null 160 | ? true 161 | : (value === null || value === void 0 ? void 0 : value.toString()) || true; 162 | return setRawPropertyValue(dynamoValue, values, countStart); 163 | } 164 | function setRawPropertyValue(value, values, countStart) { 165 | const name = ":v" + (Object.keys(values).length + countStart).toString(); 166 | values[name] = value; 167 | return values; 168 | } 169 | function conditionToAttributeNames(condition, countStart = 0) { 170 | const names = {}; 171 | if ("lhs" in condition && condition.lhs && (typeof condition.lhs === "string" || "property" in condition.lhs)) { 172 | splitAndSetPropertyName(typeof condition.lhs === "string" ? condition.lhs : condition.lhs.property, names, countStart); 173 | } 174 | // TODO: Test if this is possible in a scan wih dynamo? 175 | if ("rhs" in condition && condition.rhs && "property" in condition.rhs) { 176 | splitAndSetPropertyName(condition.rhs.property, names, countStart); 177 | } 178 | if ("property" in condition) { 179 | splitAndSetPropertyName(condition.property, names, countStart); 180 | } 181 | return names; 182 | } 183 | function splitAndSetPropertyName(propertyName, names, countStart) { 184 | return propertyName 185 | .split(".") 186 | .forEach((prop) => (names["#p" + (Object.keys(names).length + countStart).toString()] = prop)); 187 | } 188 | -------------------------------------------------------------------------------- /esm/index.d.ts: -------------------------------------------------------------------------------- 1 | export { Pipeline } from "./Pipeline"; 2 | export { sortKey } from "./ScanQueryPipeline"; 3 | export { TableIterator } from "./TableIterator"; 4 | export * as helpers from "./helpers"; 5 | -------------------------------------------------------------------------------- /esm/index.js: -------------------------------------------------------------------------------- 1 | export { Pipeline } from "./Pipeline"; 2 | export { sortKey } from "./ScanQueryPipeline"; 3 | export { TableIterator } from "./TableIterator"; 4 | export * as helpers from "./helpers"; 5 | -------------------------------------------------------------------------------- /esm/mocks/index.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 3 | import { Request } from "aws-sdk/lib/request"; 4 | type Spy = jest.MockContext, [TInput, any?]>; 5 | type WrappedFn = (client: DocumentClient, spy: Spy) => Promise; 6 | type MockReturn = { 7 | err?: Error; 8 | data?: TOutput; 9 | } | { 10 | err?: Error; 11 | data?: TOutput; 12 | }[]; 13 | export declare function setMockOn(on: boolean): void; 14 | type DynamoClientCommandName = "scan" | "query" | "delete" | "update" | "put" | "batchGet" | "batchWrite" | "transactGet"; 15 | interface MockSet> { 16 | name: DynamoClientCommandName; 17 | returns?: MockReturn; 18 | delay?: number; 19 | } 20 | export declare function multiMock(fn: (client: DocumentClient, spies: jest.MockContext[]) => Promise, mockSet: MockSet[]): () => Promise; 21 | export declare function mockScan(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 22 | export declare function alwaysMockScan(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 23 | export declare function mockQuery(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 24 | export declare function alwaysMockQuery(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 25 | export declare function alwaysMockBatchGet(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 26 | export declare function mockPut(fn: WrappedFn, returns?: MockReturn): () => Promise; 27 | export declare function mockUpdate(fn: WrappedFn, returns?: MockReturn): () => Promise; 28 | export declare function mockDelete(fn: WrappedFn, returns?: MockReturn): () => Promise; 29 | export declare function alwaysMockBatchWrite(fn: WrappedFn, returns?: MockReturn): () => Promise; 30 | export declare function mockBatchWrite(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 31 | export declare function mockBatchGet(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 32 | export declare function mockTransactGet(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 33 | export {}; 34 | -------------------------------------------------------------------------------- /esm/mocks/index.js: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import AWS from "aws-sdk"; 3 | import AWSMock from "aws-sdk-mock"; 4 | let mockOn = true; 5 | export function setMockOn(on) { 6 | mockOn = on; 7 | } 8 | export function multiMock(fn, mockSet) { 9 | return async () => { 10 | const spies = mockSet.map((ms) => setupMock(ms.name, ms.returns, true, ms.delay).mock); 11 | const client = new DocumentClient(); 12 | await fn(client, spies); 13 | mockSet.forEach((ms) => teardownMock(ms.name, true)); 14 | }; 15 | } 16 | export function mockScan(fn, returns, delay) { 17 | return mockCall("scan", fn, returns, false, delay); 18 | } 19 | export function alwaysMockScan(fn, returns, delay) { 20 | return mockCall("scan", fn, returns, true, delay); 21 | } 22 | export function mockQuery(fn, returns, delay) { 23 | return mockCall("query", fn, returns, false, delay); 24 | } 25 | export function alwaysMockQuery(fn, returns, delay) { 26 | return mockCall("query", fn, returns, true, delay); 27 | } 28 | export function alwaysMockBatchGet(fn, returns, delay) { 29 | return mockCall("batchGet", fn, returns, true, delay); 30 | } 31 | export function mockPut(fn, returns) { 32 | return mockCall("put", fn, returns); 33 | } 34 | export function mockUpdate(fn, returns) { 35 | return mockCall("update", fn, returns); 36 | } 37 | export function mockDelete(fn, returns) { 38 | return mockCall("delete", fn, returns); 39 | } 40 | export function alwaysMockBatchWrite(fn, returns) { 41 | return mockCall("batchWrite", fn, returns, true); 42 | } 43 | export function mockBatchWrite(fn, returns, delay) { 44 | return mockCall("batchWrite", fn, returns, false, delay); 45 | } 46 | export function mockBatchGet(fn, returns, delay) { 47 | return mockCall("batchGet", fn, returns, false, delay); 48 | } 49 | export function mockTransactGet(fn, returns, delay) { 50 | return mockCall("transactGet", fn, returns, false, delay); 51 | } 52 | function mockCall(name, fn, returns = {}, alwaysMock = false, delay) { 53 | return async () => { 54 | const spy = setupMock(name, returns, alwaysMock, delay); 55 | // TODO: Type cleanup 56 | // eslint-disable-next-line 57 | const client = new DocumentClient(); 58 | if (!mockOn && !alwaysMock) { 59 | // TODO: Type cleanup 60 | await fn(client, jest.spyOn(client, name).mock); 61 | } 62 | else { 63 | await fn(client, spy.mock); 64 | } 65 | teardownMock(name, alwaysMock); 66 | }; 67 | } 68 | function setupMock(name, returns = {}, alwaysMock, delay) { 69 | const spy = jest.fn(); 70 | let callCount = 0; 71 | if (mockOn || alwaysMock) { 72 | AWSMock.setSDKInstance(AWS); 73 | AWSMock.mock("DynamoDB.DocumentClient", name, function (input, callback) { 74 | var _a, _b, _c, _d; 75 | spy(input); 76 | if (Array.isArray(returns)) { 77 | if (typeof delay === "number") { 78 | setTimeout(() => { 79 | var _a, _b, _c; 80 | callback((_b = (_a = returns[callCount]) === null || _a === void 0 ? void 0 : _a.err) !== null && _b !== void 0 ? _b : undefined, (_c = returns[callCount]) === null || _c === void 0 ? void 0 : _c.data); 81 | callCount += 1; 82 | }, delay); 83 | } 84 | else { 85 | callback((_b = (_a = returns[callCount]) === null || _a === void 0 ? void 0 : _a.err) !== null && _b !== void 0 ? _b : undefined, (_c = returns[callCount]) === null || _c === void 0 ? void 0 : _c.data); 86 | callCount += 1; 87 | } 88 | } 89 | else if (typeof delay === "number") { 90 | setTimeout(() => { var _a; return callback((_a = returns === null || returns === void 0 ? void 0 : returns.err) !== null && _a !== void 0 ? _a : undefined, returns === null || returns === void 0 ? void 0 : returns.data); }, delay); 91 | } 92 | else { 93 | callback((_d = returns === null || returns === void 0 ? void 0 : returns.err) !== null && _d !== void 0 ? _d : undefined, returns === null || returns === void 0 ? void 0 : returns.data); 94 | } 95 | }); 96 | } 97 | return spy; 98 | } 99 | function teardownMock(name, alwaysMock) { 100 | if (mockOn || alwaysMock) { 101 | AWSMock.restore("DynamoDB.DocumentClient", name); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /esm/types.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | export type Scalar = string | number; 3 | export type ComparisonOperator = "=" | "<" | ">" | "<=" | ">=" | "<>"; 4 | export type Logical = "AND" | "OR" | "NOT"; 5 | export type QueryOperator = "begins_with" | "between"; 6 | export type ConditionOperator = ComparisonOperator | "contains" | "attribute_type"; 7 | export type ConditionFunction = "attribute_exists" | "attribute_not_exists"; 8 | export type SKQuery = `${Exclude">} ${Scalar}` | `begins_with ${Scalar}` | `between ${Scalar} and ${Scalar}`; 9 | export type SKQueryParts = [Exclude"> | "begins_with", Scalar] | ["between", Scalar, "and", Scalar]; 10 | export type QueryTemplate = [Exclude"> | "begins_with", Scalar] | ["between", "and", Scalar, Scalar]; 11 | export type DynamoConditionAttributeName = `#p${number}`; 12 | export type DynamoConditionAttributeValue = `:v${number}`; 13 | export type DynamoCondition = { 14 | Condition: string; 15 | ExpressionAttributeNames?: Record; 16 | ExpressionAttributeValues?: Record; 17 | }; 18 | export type SimpleKey = { 19 | pk: string; 20 | }; 21 | export type CompoundKey = { 22 | pk: string; 23 | sk: string; 24 | }; 25 | export type KeyDefinition = SimpleKey | CompoundKey; 26 | export type IndexDefinition = (SimpleKey & { 27 | name: string; 28 | }) | (CompoundKey & { 29 | name: string; 30 | }); 31 | export type KeyType = string | number | Buffer | Uint8Array; 32 | export type KeyTypeName = "N" | "S" | "B"; 33 | export type Key = Keyset extends CompoundKey ? Record : Record; 34 | export type KeyConditions = Keyset extends CompoundKey ? Record & Partial> : Record; 35 | export type PrimitiveType = string | number | null | boolean | Buffer | Uint8Array; 36 | export type PrimitiveTypeName = KeyTypeName | "NULL" | "BOOL"; 37 | export type PropertyTypeName = PrimitiveTypeName | "M" | "L"; 38 | export type DynamoValue = { 39 | [_key in PropertyTypeName]?: string | boolean | Array | Record; 40 | }; 41 | export type DynamoPrimitiveValue = { 42 | [_key in PrimitiveTypeName]?: string | boolean | number; 43 | }; 44 | export type Operand = { 45 | property: string; 46 | } | { 47 | value: PrimitiveType; 48 | } | { 49 | property: string; 50 | function: "size"; 51 | }; 52 | export type LHSOperand = string | { 53 | property: string; 54 | function: "size"; 55 | }; 56 | type BeginsWith = { 57 | property: string; 58 | operator: "begins_with"; 59 | value: PrimitiveType; 60 | }; 61 | type Between = { 62 | property: string; 63 | start: PrimitiveType; 64 | end: PrimitiveType; 65 | operator: "between"; 66 | }; 67 | type In = { 68 | property: string; 69 | list: PrimitiveType[]; 70 | operator: "in"; 71 | }; 72 | type BaseExpression = { 73 | lhs: LHSOperand; 74 | rhs: Operand; 75 | operator: T; 76 | } | Between | BeginsWith; 77 | export type ConditionExpression = BaseExpression | In | { 78 | operator: ConditionFunction; 79 | property: string; 80 | } | { 81 | lhs?: ConditionExpression; 82 | logical: Logical; 83 | rhs: ConditionExpression; 84 | }; 85 | export type UpdateReturnValues = "ALL_OLD" | "UPDATED_OLD" | "ALL_NEW" | "UPDATED_NEW"; 86 | export {}; 87 | -------------------------------------------------------------------------------- /esm/types.js: -------------------------------------------------------------------------------- 1 | export {}; 2 | -------------------------------------------------------------------------------- /example/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: [ 3 | "plugin:@typescript-eslint/eslint-recommended", 4 | "plugin:@typescript-eslint/recommended", 5 | "plugin:@typescript-eslint/recommended-requiring-type-checking", 6 | "standard", 7 | "prettier", 8 | ], 9 | parser: "@typescript-eslint/parser", 10 | plugins: ["@typescript-eslint"], 11 | parserOptions: { 12 | tsconfigRootDir: __dirname, 13 | project: "./tsconfig.json", 14 | ecmaVersion: 10, 15 | sourceType: "module", 16 | }, 17 | env: { 18 | node: true, 19 | jest: true, 20 | }, 21 | globals: { 22 | __DEV__: false, 23 | beforeAll: false, 24 | afterAll: false, 25 | beforeEach: false, 26 | afterEach: false, 27 | test: false, 28 | expect: false, 29 | describe: false, 30 | jest: false, 31 | it: false, 32 | }, 33 | ignorePatterns: ["**/node_modules/**/*", "node_modules/**/*", "dist/**/*"], 34 | rules: { 35 | complexity: ["warn", 30], 36 | "default-case": 2, 37 | "dot-notation": 2, 38 | eqeqeq: 2, 39 | "guard-for-in": 2, 40 | "no-constant-condition": 2, 41 | "no-dupe-keys": 2, 42 | "no-eval": 2, 43 | "no-unreachable": 2, 44 | "no-unused-vars": 0, 45 | "no-void": ["error", { allowAsStatement: true }], 46 | "prefer-destructuring": [ 47 | "warn", 48 | { 49 | object: true, 50 | array: true, 51 | }, 52 | ], 53 | camelcase: 0, 54 | "@typescript-eslint/camelcase": 0, 55 | "@typescript-eslint/explicit-member-accessibility": 0, 56 | "@typescript-eslint/explicit-function-return-type": 0, 57 | "@typescript-eslint/indent": 0, 58 | "@typescript-eslint/no-explicit-any": 0, 59 | "@typescript-eslint/no-empty-interface": 0, 60 | "@typescript-eslint/no-object-literal-type-assertion": 0, 61 | "@typescript-eslint/no-use-before-define": 0, 62 | "@typescript-eslint/no-var-requires": 0, 63 | "@typescript-eslint/no-floating-promises": 2, 64 | "standard/no-callback-literal": 0, 65 | "node/no-callback-literal": 0, 66 | "@typescript-eslint/no-unused-vars": 0, 67 | }, 68 | }; 69 | -------------------------------------------------------------------------------- /example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamo-pipeline-example", 3 | "version": "1.0.0", 4 | "description": "Example use of dynamo-pipeline", 5 | "main": "example-lambda.ts", 6 | "author": "RossWilliams", 7 | "license": "Apache-2.0", 8 | "private": true, 9 | "scripts": { 10 | "test": "NODE_ENV=test jest", 11 | "tsc": "tsc --noEmit", 12 | "build": "esbuild src/example-lambda.ts --bundle --platform=node --external:aws-sdk --target=node12 --outfile=dist/index.js --minify" 13 | }, 14 | "dependencies": { 15 | "@types/jest": "^26.0.20", 16 | "dynamo-pipeline": "git://github.com/RossWilliams/dynamo-pipeline.git", 17 | "esbuild": "^0.8.54" 18 | }, 19 | "devDependencies": { 20 | "typescript": "^4.2.2" 21 | }, 22 | "jest": { 23 | "testRegex": "/(example)/.*\\.test\\.[jt]s?$", 24 | "setupFiles": [ 25 | "../test/jest.setup.ts" 26 | ], 27 | "moduleFileExtensions": [ 28 | "mjs", 29 | "js", 30 | "json", 31 | "ts", 32 | "node" 33 | ], 34 | "transform": { 35 | "^.+\\.mjs$": "babel-jest", 36 | "\\.tsx?$": [ 37 | "ts-jest" 38 | ] 39 | }, 40 | "testPathIgnorePatterns": [ 41 | "node_modules/", 42 | ".buildcache/" 43 | ], 44 | "verbose": true, 45 | "collectCoverage": true, 46 | "collectCoverageFrom": [ 47 | "example-lambda.ts" 48 | ] 49 | }, 50 | "prettier": { 51 | "printWidth": 120, 52 | "trailingComma": "es5" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /example/src/example-lambda.ts: -------------------------------------------------------------------------------- 1 | // import { Pipeline, sortKey } from "dynamo-pipeline"; 2 | import { Pipeline, sortKey } from "../../"; 3 | const TABLE_NAME = process.env.TABLE_NAME || "example-2c19f773"; 4 | interface AddUserCalendarEvent { 5 | userId: string; 6 | startDateTime: string; // ISO 8601 7 | expectedVersion: number; // counting number 8 | } 9 | 10 | interface CalendarItem { 11 | id: string; 12 | sort: string; 13 | gsi1pk: string; 14 | gsi1sk: string; 15 | start: string; // ISO 8601 16 | } 17 | 18 | /** 19 | * 1. Update the user profile with appropriate event metadata 20 | * 2. Remove all existing events for the user for 7 days beyond the start date of the event 21 | * 3. Add the new calendar item 22 | * @param event The Calendar Event to add to the User 23 | */ 24 | export async function handler(event: AddUserCalendarEvent): Promise<{ error: string } | { item: CalendarItem }> { 25 | const table = new Pipeline(TABLE_NAME, { pk: "id", sk: "sort" }).withReadBuffer(20); 26 | const index = table.createIndex("gsi1", { pk: "gsi1pk", sk: "gsi1sk" }); 27 | 28 | const startDate: string = event.startDateTime.split("T")[0] || ""; 29 | const sevenDaysFromStart: string | null = tryGetSevenDaysFromStart(startDate); 30 | 31 | if (!startDate || !sevenDaysFromStart) { 32 | console.error("Error: Data format error. Invalid startDateTime", event.startDateTime); 33 | return { error: "Invalid startDateTime in Event" }; 34 | } 35 | 36 | // update user profile 37 | const updatedUserVersion = await table 38 | .update<{ currentVersion: number }>( 39 | { id: event.userId, sort: event.userId }, // keys for the items to update 40 | { lastEventAdded: event.startDateTime, currentVersion: event.expectedVersion + 1 }, // attributes to update 41 | { 42 | // options object 43 | condition: { 44 | // type-safe and nestable conditions 45 | lhs: "currentVersion", 46 | operator: "=", 47 | rhs: { value: event.expectedVersion }, 48 | }, 49 | returnType: "UPDATED_NEW", 50 | } 51 | ) 52 | .then((updatedValues) => updatedValues?.currentVersion); 53 | 54 | if (updatedUserVersion !== event.expectedVersion + 1) { 55 | console.info(` 56 | Info: Calendar Event not added. Version does not match expected version 57 | User Version: ${event.expectedVersion} 58 | Actual Version: ${updatedUserVersion ?? ""} 59 | `); 60 | return { error: "Version Conflict Error" }; 61 | } 62 | 63 | // delete all other calenar items 64 | await index 65 | // sk string is type checked 66 | .query({ 67 | gsi1pk: event.userId, 68 | gsi1sk: sortKey("between", startDate, sevenDaysFromStart), 69 | }) 70 | // delete method returns a promise which the forEach will await on for us. 71 | .forEach((event, _index) => table.delete(event)); // delete method can take extra object properties and extract keys 72 | 73 | // add in new calendar item 74 | const newItem: CalendarItem = { 75 | id: "1", 76 | sort: "1", 77 | gsi1pk: event.userId, 78 | gsi1sk: event.startDateTime, 79 | start: event.startDateTime, 80 | }; 81 | 82 | // helper extention to put which add a pk not exists condition check to the request 83 | await table.putIfNotExists(newItem); 84 | 85 | return { item: newItem }; 86 | } 87 | 88 | function tryGetSevenDaysFromStart(start: string): string | null { 89 | try { 90 | return new Date(new Date(start).setDate(new Date(start).getDate() + 7)).toISOString().split("T")[0] || null; 91 | } catch { 92 | return null; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /example/test/example-lambda.test.ts: -------------------------------------------------------------------------------- 1 | // import { mocks } from "dynamo-pipeline"; 2 | import DynamoDB from "aws-sdk/clients/dynamodb"; 3 | import * as mocks from "../../lib/mocks"; 4 | import { handler } from "../src/example-lambda"; 5 | 6 | const testEvent = { 7 | userId: "1", 8 | startDateTime: "2020-10-21T01:02:03.001Z", 9 | expectedVersion: 1, 10 | }; 11 | 12 | describe("Example Lambda", () => { 13 | test( 14 | "Throws when invalid startDateTime event property is supplied", 15 | mocks.mockUpdate( 16 | async (_client, spy) => { 17 | const result = await handler({ ...testEvent, startDateTime: "INVALID" }); 18 | 19 | expect(spy.calls.length).toEqual(0); 20 | expect("error" in result && result.error).toEqual("Invalid startDateTime in Event"); 21 | }, 22 | { data: { Attributes: { currentVersion: 2 } } } 23 | ) 24 | ); 25 | 26 | test( 27 | "Attempts to update user profile, returns early if expected version does not match", 28 | mocks.mockUpdate( 29 | async (_client, spy) => { 30 | const result = await handler({ ...testEvent, expectedVersion: 0 }); 31 | const request = spy.calls[0]![0]; // eslint-disable-line 32 | 33 | expect(spy.calls.length).toEqual(1); 34 | expect(request.ConditionExpression).toBeTruthy(); 35 | expect(request.ExpressionAttributeValues?.[":v2"]).toEqual(0); 36 | expect("error" in result && result.error).toEqual("Version Conflict Error"); 37 | }, 38 | { data: { Attributes: { currentVersion: 2 } } } 39 | ) 40 | ); 41 | 42 | test( 43 | "When expected version is valid, queries existing calendar items in next 7 days", 44 | mocks.multiMock( 45 | async (_client, spies) => { 46 | const result = await handler(testEvent); 47 | // eslint-disable-next-line 48 | const querySpy = spies[1]!; 49 | 50 | expect("error" in result).toBeFalsy(); 51 | expect(querySpy.calls.length).toEqual(1); 52 | const request = querySpy.calls[0]?.[0] as DynamoDB.QueryInput; 53 | expect(request.IndexName).toEqual("gsi1"); 54 | expect(request.KeyConditionExpression?.includes("BETWEEN")).toBeTruthy(); 55 | }, 56 | [ 57 | { name: "update", returns: { data: { Attributes: { currentVersion: 2 } } } }, 58 | { name: "query", returns: { data: { Items: [] } } }, 59 | { name: "delete" }, 60 | { name: "put" }, 61 | ] 62 | ) 63 | ); 64 | 65 | test( 66 | "Deletes all existing queried calendar events", 67 | mocks.multiMock( 68 | async (_client, spies) => { 69 | const deleteSpy = spies[2]!; // eslint-disable-line 70 | 71 | await handler(testEvent); 72 | const deleteKeys = (deleteSpy.calls as [[DynamoDB.DeleteItemInput]]).map((call) => call[0].Key); 73 | 74 | expect(deleteSpy.calls.length).toEqual(3); 75 | expect(deleteKeys[0]?.id).toEqual("1"); 76 | expect(deleteKeys[0]?.sort).toEqual("1"); 77 | expect(deleteKeys[1]?.id).toEqual("2"); 78 | expect(deleteKeys[1]?.sort).toEqual("2"); 79 | expect(deleteKeys[2]?.id).toEqual("3"); 80 | expect(deleteKeys[2]?.sort).toEqual("3"); 81 | }, 82 | [ 83 | { 84 | name: "query", 85 | returns: { 86 | data: { 87 | Items: [ 88 | { id: "1", sort: "1" }, 89 | { id: "2", sort: "2" }, 90 | { id: "3", sort: "3" }, 91 | ], 92 | }, 93 | }, 94 | }, 95 | { name: "update", returns: { data: { Attributes: { currentVersion: 2 } } } }, 96 | { name: "delete" }, 97 | { name: "put" }, 98 | ] 99 | ) 100 | ); 101 | 102 | test( 103 | "Adds new calendar item afer deleting other calendar items", 104 | mocks.multiMock( 105 | async (_queryClient, spies) => { 106 | const putSpy = spies[0]!; // eslint-disable-line 107 | 108 | await handler(testEvent); 109 | const request = putSpy.calls[0]![0] as DynamoDB.PutItemInput; // eslint-disable-line 110 | 111 | expect(putSpy.calls.length).toEqual(1); 112 | expect(request.Item.start).toEqual(testEvent.startDateTime); 113 | expect(request.Item.gsi1pk).toEqual(testEvent.userId); 114 | }, 115 | [ 116 | { name: "put" }, 117 | { 118 | name: "query", 119 | returns: { 120 | data: { 121 | Items: [ 122 | { id: "1", sort: "1" }, 123 | { id: "2", sort: "2" }, 124 | { id: "3", sort: "3" }, 125 | ], 126 | }, 127 | }, 128 | }, 129 | { name: "delete" }, 130 | { name: "update", returns: { data: { Attributes: { currentVersion: 2 } } } }, 131 | ] 132 | ) 133 | ); 134 | }); 135 | -------------------------------------------------------------------------------- /example/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "outDir": "dist", 5 | "moduleResolution": "node", 6 | "declaration": true, 7 | "incremental": true, 8 | "types": ["jest", "node"], 9 | "esModuleInterop": true, 10 | "target": "ES2019", 11 | "strict": true, 12 | "noImplicitAny": true, 13 | "noImplicitReturns": true, 14 | "noImplicitThis": true, 15 | "noUnusedLocals": false, 16 | "noUncheckedIndexedAccess": true, 17 | "alwaysStrict": true, 18 | "strictNullChecks": true 19 | }, 20 | "exclude": ["dist", "node_moules"], 21 | "include": ["src", "test"] 22 | } 23 | -------------------------------------------------------------------------------- /jest.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from "jest"; 2 | 3 | const config: Config = { 4 | testRegex: "/(test)/.*\\.test\\.[jt]s?$", 5 | setupFiles: ["./test/jest.setup.ts"], 6 | moduleFileExtensions: ["mjs", "js", "json", "ts", "node"], 7 | transform: { 8 | "^.+\\.(t|j)sx?$": "@swc/jest", 9 | }, 10 | testPathIgnorePatterns: ["node_modules/", ".buildcache/"], 11 | verbose: true, 12 | collectCoverage: true, 13 | collectCoverageFrom: ["src/**/*.ts", "!src/mocks/*.ts"], 14 | testTimeout: 90000, 15 | }; 16 | 17 | export default config; 18 | -------------------------------------------------------------------------------- /lib/AbstractFetcher.d.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | export declare abstract class AbstractFetcher { 3 | protected activeRequests: Promise[]; 4 | protected bufferSize: number; 5 | protected bufferCapacity: number; 6 | protected batchSize: number; 7 | protected limit?: number; 8 | protected totalReturned: number; 9 | protected nextToken: number | Record | null; 10 | protected documentClient: DocumentClient; 11 | protected results: T[]; 12 | protected errors: Error | null; 13 | constructor(client: DocumentClient, options: { 14 | batchSize: number; 15 | bufferCapacity: number; 16 | limit?: number; 17 | }); 18 | abstract fetchStrategy(): Promise | null; 19 | abstract processResult(data: Record): void; 20 | protected fetchNext(): Promise | null; 21 | private setupFetchProcessor; 22 | execute(): AsyncGenerator; 24 | } | void, void>; 25 | getResultBatch(batchSize: number): T[]; 26 | processError(e: Error): void; 27 | hasDataReady(): boolean; 28 | isDone(): boolean; 29 | isActive(): boolean; 30 | } 31 | -------------------------------------------------------------------------------- /lib/AbstractFetcher.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.AbstractFetcher = void 0; 4 | class AbstractFetcher { 5 | constructor(client, options) { 6 | this.activeRequests = []; 7 | this.bufferSize = 0; 8 | this.bufferCapacity = 1; 9 | this.totalReturned = 0; 10 | this.results = []; 11 | this.errors = null; 12 | this.documentClient = client; 13 | this.bufferCapacity = options.bufferCapacity; 14 | this.batchSize = options.batchSize; 15 | this.limit = options.limit; 16 | this.nextToken = null; 17 | } 18 | // take in a promise to allow recursive calls, 19 | // batch fetcher can immediately create many requests 20 | fetchNext() { 21 | const fetchResponse = this.fetchStrategy(); 22 | if (fetchResponse instanceof Promise && !this.activeRequests.includes(fetchResponse)) { 23 | return this.setupFetchProcessor(fetchResponse); 24 | } 25 | return fetchResponse; 26 | } 27 | setupFetchProcessor(promise) { 28 | this.activeRequests.push(promise); 29 | this.bufferSize += 1; 30 | return promise 31 | .then((data) => { 32 | this.activeRequests = this.activeRequests.filter((r) => r !== promise); 33 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 34 | this.processResult(data); 35 | }) 36 | .catch((e) => { 37 | this.activeRequests = this.activeRequests.filter((r) => r !== promise); 38 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 39 | this.processError(e); 40 | }); 41 | } 42 | // Entry point. 43 | async *execute() { 44 | let count = 0; 45 | do { 46 | if (this.errors) { 47 | return Promise.reject(this.errors); 48 | } 49 | if (!this.hasDataReady()) { 50 | await this.fetchNext(); 51 | } 52 | // check for errors again after running another fetch 53 | if (this.errors) { 54 | return Promise.reject(this.errors); 55 | } 56 | const batch = this.getResultBatch(Math.min(this.batchSize, this.limit ? this.limit - count : 1000000000000)); 57 | count += batch.length; 58 | if (!this.isDone() && (!this.limit || count < this.limit)) { 59 | // do not await here, background process the next set of data 60 | void this.fetchNext(); 61 | } 62 | yield batch; 63 | if (this.limit && count >= this.limit) { 64 | if (typeof this.nextToken === "object" && this.nextToken !== null) { 65 | return { lastEvaluatedKey: this.nextToken }; 66 | } 67 | return; 68 | } 69 | } while (!this.isDone()); 70 | } 71 | getResultBatch(batchSize) { 72 | const items = (this.results.length && this.results.splice(0, batchSize)) || []; 73 | if (!items.length) { 74 | this.bufferSize = this.activeRequests.length; 75 | } 76 | else { 77 | this.bufferSize -= 1; 78 | } 79 | return items; 80 | } 81 | processError(e) { 82 | this.errors = e; 83 | } 84 | hasDataReady() { 85 | return this.results.length > 0; 86 | } 87 | isDone() { 88 | return !this.isActive() && this.nextToken === null && this.results.length === 0; 89 | } 90 | isActive() { 91 | return this.activeRequests.length > 0; 92 | } 93 | } 94 | exports.AbstractFetcher = AbstractFetcher; 95 | -------------------------------------------------------------------------------- /lib/BatchFetcher.d.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { Key, KeyDefinition } from "./types"; 3 | import { AbstractFetcher } from "./AbstractFetcher"; 4 | type BatchGetItems = { 5 | tableName: string; 6 | keys: Key[]; 7 | }; 8 | type TransactGetItems = { 9 | tableName: string; 10 | keys: Key; 11 | }[]; 12 | export declare class BatchGetFetcher extends AbstractFetcher { 13 | protected operation: "batchGet" | "transactGet"; 14 | protected chunks: BatchGetItems[] | TransactGetItems[]; 15 | protected retryKeys: BatchGetItems[] | null; 16 | protected onUnprocessedKeys: ((keys: Key[]) => void) | undefined; 17 | protected consistentRead: boolean; 18 | constructor(client: DocumentClient, operation: "batchGet" | "transactGet", items: BatchGetItems | TransactGetItems, options: { 19 | onUnprocessedKeys?: (keys: Key[]) => void; 20 | batchSize: number; 21 | bufferCapacity: number; 22 | consistentRead?: boolean; 23 | }); 24 | private chunkBatchRequests; 25 | retry(): Promise | null; 26 | fetchStrategy(): Promise | null; 27 | processResult(data: DocumentClient.BatchGetItemOutput | DocumentClient.TransactGetItemsOutput | void): void; 28 | processError(err: Error | { 29 | tableName: string; 30 | errorKeys: Key[]; 31 | }): void; 32 | isDone(): boolean; 33 | private createTransactionRequest; 34 | private createBatchGetRequest; 35 | private hasNextChunk; 36 | } 37 | export {}; 38 | -------------------------------------------------------------------------------- /lib/BatchFetcher.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.BatchGetFetcher = void 0; 4 | const AbstractFetcher_1 = require("./AbstractFetcher"); 5 | class BatchGetFetcher extends AbstractFetcher_1.AbstractFetcher { 6 | constructor(client, operation, items, options) { 7 | super(client, options); 8 | this.retryKeys = []; 9 | this.consistentRead = false; 10 | this.operation = operation; 11 | this.onUnprocessedKeys = options.onUnprocessedKeys; 12 | this.consistentRead = Boolean(options.consistentRead); 13 | if (operation === "batchGet" && !Array.isArray(items)) { 14 | this.chunks = this.chunkBatchRequests(items); 15 | } 16 | else { 17 | // Transactions don't support chunking, its a transaction 18 | this.chunks = [items]; 19 | } 20 | this.nextToken = 0; 21 | } 22 | chunkBatchRequests(items) { 23 | const chunks = []; 24 | const n = items.keys.length; 25 | let i = 0; 26 | while (i < n) { 27 | chunks.push({ 28 | tableName: items.tableName, 29 | keys: items.keys.slice(i, (i += this.batchSize)), 30 | }); 31 | } 32 | return chunks; 33 | } 34 | retry() { 35 | this.chunks = this.retryKeys || []; 36 | this.nextToken = 0; 37 | this.retryKeys = null; 38 | return this.fetchNext(); 39 | // TODO: Batch Get needs to be tested with chunk size of 1 and three items 40 | } 41 | fetchStrategy() { 42 | if (this.retryKeys && this.retryKeys.length && this.nextToken === null && !this.isActive()) { 43 | // if finished fetching initial requests, begin to process the retry keys 44 | return this.retry(); 45 | } 46 | else if (this.bufferSize >= this.bufferCapacity || 47 | (typeof this.nextToken === "number" && this.chunks.length <= this.nextToken) || 48 | this.nextToken === null) { 49 | // return the current promise if buffer at capacity, or if there are no more items to fetch 50 | return this.activeRequests[0] || null; 51 | } 52 | else if (!this.hasNextChunk()) { 53 | /* istanbul ignore next */ 54 | return null; 55 | } 56 | let promise = null; 57 | if (this.operation === "transactGet") { 58 | const transactionRequest = this.createTransactionRequest(); 59 | if (transactionRequest === null) { 60 | /* istanbul ignore next */ 61 | return null; 62 | } 63 | promise = this.documentClient.transactGet(transactionRequest).promise(); 64 | } 65 | else if (this.operation === "batchGet") { 66 | const batchGetRequest = this.createBatchGetRequest(); 67 | if (batchGetRequest === null) { 68 | /* istanbul ignore next */ 69 | return null; 70 | } 71 | promise = this.documentClient.batchGet(batchGetRequest).promise(); 72 | } 73 | if (typeof this.nextToken === "number" && typeof this.chunks[this.nextToken + 1] !== "undefined") { 74 | this.nextToken = this.nextToken + 1; 75 | } 76 | else { 77 | this.nextToken = null; 78 | } 79 | return promise; 80 | } 81 | processResult(data) { 82 | let responseItems = []; 83 | if (data && data.Responses && Array.isArray(data.Responses)) { 84 | // transaction 85 | responseItems = data.Responses.map((r) => r.Item).filter(notEmpty); 86 | } 87 | else if (data && data.Responses && !Array.isArray(data.Responses)) { 88 | // batch, flatten each table response 89 | responseItems = [] 90 | .concat(...Object.values(data.Responses)) 91 | .filter(notEmpty); 92 | } 93 | if (data) { 94 | const unprocessedKeys = "UnprocessedKeys" in data && data.UnprocessedKeys; 95 | if (unprocessedKeys) { 96 | Object.entries(unprocessedKeys).forEach(([tableName, keys]) => { 97 | this.processError({ tableName, errorKeys: keys.Keys }); 98 | }); 99 | } 100 | } 101 | this.totalReturned += responseItems.length; 102 | this.results.push(...responseItems); 103 | } 104 | processError(err) { 105 | if (err && "tableName" in err && Array.isArray(this.retryKeys)) { 106 | const retryItems = splitInHalf(err.errorKeys) 107 | .filter(notEmpty) 108 | .map((k) => ({ 109 | tableName: err.tableName, 110 | keys: k, 111 | })); 112 | this.retryKeys.push(...[].concat(...retryItems)); 113 | } 114 | else if (err && "errorKeys" in err && typeof this.onUnprocessedKeys !== "undefined") { 115 | this.onUnprocessedKeys(err.errorKeys); 116 | } 117 | } 118 | isDone() { 119 | return super.isDone() && (!this.retryKeys || this.retryKeys.length === 0); 120 | } 121 | createTransactionRequest() { 122 | const currentChunk = typeof this.nextToken === "number" 123 | ? this.chunks[this.nextToken] 124 | : undefined; 125 | if (!currentChunk) { 126 | /* istanbul ignore next */ 127 | return null; 128 | } 129 | const transaction = { 130 | TransactItems: currentChunk.map((item) => ({ 131 | Get: { 132 | Key: item.keys, 133 | TableName: item.tableName, 134 | }, 135 | })), 136 | }; 137 | return transaction; 138 | } 139 | // each batch handles a single table for now... 140 | createBatchGetRequest() { 141 | const currentChunk = typeof this.nextToken === "number" ? this.chunks[this.nextToken] : undefined; 142 | if (!currentChunk) { 143 | /* istanbul ignore next */ 144 | return null; 145 | } 146 | // when multiple tables are supported in a single batch 147 | // switch to items.reduce(acc, curr) => ({...acc, [curr.tableName]: curr.keyItems,}),{}) 148 | const request = { 149 | RequestItems: { 150 | [currentChunk.tableName]: { 151 | ConsistentRead: this.consistentRead, 152 | Keys: currentChunk.keys, 153 | }, 154 | }, 155 | }; 156 | return request; 157 | } 158 | hasNextChunk() { 159 | if (typeof this.nextToken !== "number" || this.nextToken >= this.chunks.length) { 160 | return false; 161 | } 162 | return true; 163 | } 164 | } 165 | exports.BatchGetFetcher = BatchGetFetcher; 166 | function notEmpty(val) { 167 | if (Array.isArray(val) && !val.length) { 168 | return false; 169 | } 170 | return !!val; 171 | } 172 | function splitInHalf(arr) { 173 | return [arr.slice(0, Math.ceil(arr.length / 2)), arr.slice(Math.ceil(arr.length / 2), arr.length)]; 174 | } 175 | -------------------------------------------------------------------------------- /lib/BatchWriter.d.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { Key, KeyDefinition } from "./types"; 3 | type BatchWriteItems = { 4 | tableName: string; 5 | records: Key[]; 6 | }; 7 | export declare class BatchWriter { 8 | private client; 9 | private tableName; 10 | private activeRequests; 11 | private chunks; 12 | private nextToken; 13 | private retryKeys; 14 | private errors; 15 | private batchSize; 16 | private bufferCapacity; 17 | private backoffActive; 18 | private onUnprocessedItems; 19 | constructor(client: DocumentClient, items: BatchWriteItems, options: { 20 | onUnprocessedItems?: (keys: Key[]) => void; 21 | batchSize: number; 22 | bufferCapacity: number; 23 | }); 24 | execute(): Promise; 25 | private chunkBatchWrites; 26 | private writeChunk; 27 | private getNextChunk; 28 | private isActive; 29 | private processResult; 30 | private retry; 31 | private isDone; 32 | private hasNextChunk; 33 | } 34 | export {}; 35 | -------------------------------------------------------------------------------- /lib/BatchWriter.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.BatchWriter = void 0; 4 | class BatchWriter { 5 | constructor(client, items, options) { 6 | this.activeRequests = []; 7 | this.retryKeys = []; 8 | this.errors = null; 9 | this.batchSize = 25; 10 | this.bufferCapacity = 3; 11 | this.backoffActive = false; 12 | this.client = client; 13 | this.tableName = items.tableName; 14 | this.batchSize = options.batchSize; 15 | this.bufferCapacity = options.bufferCapacity; 16 | this.onUnprocessedItems = options.onUnprocessedItems; 17 | this.chunks = this.chunkBatchWrites(items); 18 | this.nextToken = 0; 19 | } 20 | async execute() { 21 | do { 22 | if (this.errors) { 23 | return Promise.reject(this.errors); 24 | } 25 | if (!this.isDone()) { 26 | await this.writeChunk(); 27 | } 28 | } while (!this.isDone()); 29 | await Promise.all(this.activeRequests); 30 | } 31 | chunkBatchWrites(items) { 32 | const chunks = []; 33 | let i = 0; 34 | const n = items.records.length; 35 | while (i < n) { 36 | chunks.push(items.records.slice(i, (i += this.batchSize || 25))); 37 | } 38 | return chunks; 39 | } 40 | async writeChunk() { 41 | if (this.retryKeys && this.retryKeys.length && this.nextToken === null && !this.isActive()) { 42 | // if finished fetching initial requests, begin to process the retry keys 43 | return this.retry(); 44 | } 45 | else if (this.activeRequests.length >= this.bufferCapacity || this.nextToken === null || this.backoffActive) { 46 | // return the current promise if buffer at capacity, or if there are no more items to fetch 47 | return this.activeRequests[0] || null; 48 | } 49 | else if (!this.hasNextChunk()) { 50 | this.nextToken = null; 51 | // let the caller wait until all active requests are finished 52 | return Promise.all(this.activeRequests).then(); 53 | } 54 | const chunk = this.getNextChunk(); 55 | if (chunk) { 56 | const request = this.client.batchWrite({ 57 | RequestItems: { 58 | [this.tableName]: chunk.map((item) => ({ 59 | PutRequest: { 60 | Item: item, 61 | }, 62 | })), 63 | }, 64 | }); 65 | if (request && typeof request.on === "function") { 66 | request.on("retry", (e) => { 67 | var _a; 68 | if ((_a = e === null || e === void 0 ? void 0 : e.error) === null || _a === void 0 ? void 0 : _a.retryable) { 69 | // reduce buffer capacity on retryable error 70 | this.bufferCapacity = Math.max(Math.floor((this.bufferCapacity * 3) / 4), 5); 71 | this.backoffActive = true; 72 | } 73 | }); 74 | } 75 | const promise = request 76 | .promise() 77 | .catch((e) => { 78 | console.error("Error: AWS Error, Put Items", e); 79 | if (this.onUnprocessedItems) { 80 | this.onUnprocessedItems(chunk); 81 | } 82 | this.errors = e; 83 | }) 84 | .then((results) => { 85 | this.processResult(results, promise); 86 | }); 87 | this.activeRequests.push(promise); 88 | } 89 | } 90 | getNextChunk() { 91 | if (this.nextToken === null) { 92 | /* istanbul ignore next */ 93 | return null; 94 | } 95 | const chunk = this.chunks[this.nextToken] || null; 96 | this.nextToken += 1; 97 | return chunk; 98 | } 99 | isActive() { 100 | return this.activeRequests.length > 0; 101 | } 102 | processResult(data, request) { 103 | var _a; 104 | this.activeRequests = this.activeRequests.filter((r) => r !== request); 105 | if (!this.activeRequests.length || !data || !data.UnprocessedItems) { 106 | this.backoffActive = false; 107 | } 108 | if (data && data.UnprocessedItems && (((_a = data.UnprocessedItems[this.tableName]) === null || _a === void 0 ? void 0 : _a.length) || 0) > 0) { 109 | // eslint-disable-next-line 110 | const unprocessedItems = data.UnprocessedItems[this.tableName].map((ui) => { var _a; return (_a = ui.PutRequest) === null || _a === void 0 ? void 0 : _a.Item; }); 111 | if (Array.isArray(this.retryKeys)) { 112 | const retryItems = splitInHalf(unprocessedItems).filter(notEmpty); 113 | this.retryKeys.push(...retryItems); 114 | } 115 | else if (this.onUnprocessedItems) { 116 | this.onUnprocessedItems(unprocessedItems); 117 | } 118 | } 119 | } 120 | retry() { 121 | this.chunks = this.retryKeys || []; 122 | this.nextToken = 0; 123 | this.retryKeys = null; 124 | return this.writeChunk(); 125 | } 126 | isDone() { 127 | return !this.isActive() && (!this.retryKeys || this.retryKeys.length === 0) && this.nextToken === null; 128 | } 129 | hasNextChunk() { 130 | if (this.nextToken === null || this.nextToken >= this.chunks.length) { 131 | return false; 132 | } 133 | return true; 134 | } 135 | } 136 | exports.BatchWriter = BatchWriter; 137 | function notEmpty(val) { 138 | if (Array.isArray(val) && !val.length) { 139 | return false; 140 | } 141 | return !!val; 142 | } 143 | function splitInHalf(arr) { 144 | return [arr.slice(0, Math.ceil(arr.length / 2)), arr.slice(Math.ceil(arr.length / 2), arr.length)]; 145 | } 146 | -------------------------------------------------------------------------------- /lib/Pipeline.d.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { ConditionExpression, UpdateReturnValues, PrimitiveType, Key } from "./types"; 3 | import { TableIterator } from "./TableIterator"; 4 | import { ScanQueryPipeline } from "./ScanQueryPipeline"; 5 | export declare class Pipeline extends ScanQueryPipeline { 12 | constructor(tableName: string, keys: { 13 | pk: PK; 14 | sk?: SK; 15 | }, config?: { 16 | client?: DocumentClient; 17 | readBuffer?: number; 18 | writeBuffer?: number; 19 | readBatchSize?: number; 20 | writeBatchSize?: number; 21 | }); 22 | withWriteBuffer(writeBuffer?: number): this; 23 | withWriteBatchSize(writeBatchSize?: number): this; 24 | createIndex(name: string, definition: { 25 | pk: PK2; 26 | sk?: SK2; 27 | }): ScanQueryPipeline; 28 | transactGet(keys: Key[] | { 29 | tableName: string; 30 | keys: Key; 31 | keyDefinition: KD2; 32 | }[], options?: { 33 | bufferCapacity?: number; 34 | }): TableIterator; 35 | getItems(keys: Key[], options?: { 36 | batchSize?: number; 37 | bufferCapacity?: number; 38 | consistentRead?: boolean; 39 | }): TableIterator; 40 | putItems>(items: I[], options?: { 41 | bufferCapacity?: number; 42 | disableSlowStart?: boolean; 43 | batchSize?: number; 44 | }): Promise>; 45 | put>(item: Item, condition?: ConditionExpression): Promise>; 46 | putIfNotExists>(item: Item): Promise>; 47 | buildUpdateRequest(key: Key, attributes: Record, options?: { 48 | condition?: ConditionExpression; 49 | returnType?: UpdateReturnValues; 50 | }): DocumentClient.UpdateItemInput & { 51 | UpdateExpression: string; 52 | }; 53 | update(key: Key, attributes: Record, options?: { 54 | condition?: ConditionExpression; 55 | returnType?: UpdateReturnValues; 56 | }): Promise; 57 | delete(key: Key, options?: { 58 | condition?: ConditionExpression | undefined; 59 | returnType?: "ALL_OLD"; 60 | reportError?: boolean; 61 | }): Promise; 62 | handleUnprocessed(callback: (item: Record) => void): Pipeline; 63 | private keyAttributesOnlyFromArray; 64 | private keyAttributesOnly; 65 | } 66 | -------------------------------------------------------------------------------- /lib/QueryFetcher.d.ts: -------------------------------------------------------------------------------- 1 | import { AbstractFetcher } from "./AbstractFetcher"; 2 | import { ScanInput, QueryInput, DocumentClient } from "aws-sdk/clients/dynamodb"; 3 | export declare class QueryFetcher extends AbstractFetcher { 4 | private request; 5 | private operation; 6 | constructor(request: ScanInput | QueryInput, client: DocumentClient, operation: "query" | "scan", options: { 7 | batchSize: number; 8 | bufferCapacity: number; 9 | limit?: number; 10 | nextToken?: DocumentClient.Key; 11 | }); 12 | fetchStrategy(): null | Promise; 13 | processResult(data: DocumentClient.ScanOutput | DocumentClient.QueryOutput | void): void; 14 | getResultBatch(batchSize: number): T[]; 15 | } 16 | -------------------------------------------------------------------------------- /lib/QueryFetcher.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.QueryFetcher = void 0; 4 | const AbstractFetcher_1 = require("./AbstractFetcher"); 5 | class QueryFetcher extends AbstractFetcher_1.AbstractFetcher { 6 | constructor(request, client, operation, options) { 7 | super(client, options); 8 | this.request = request; 9 | this.operation = operation; 10 | if (options.nextToken) { 11 | this.nextToken = options.nextToken; 12 | } 13 | else { 14 | this.nextToken = 1; 15 | } 16 | } 17 | // TODO: remove null response type 18 | fetchStrategy() { 19 | // no support for parallel query 20 | // 1. 1 active request allowed at a time 21 | // 2. Do not create a new request when the buffer is full 22 | // 3. If there are no more items to fetch, exit 23 | if (this.activeRequests.length > 0 || this.bufferSize > this.bufferCapacity || !this.nextToken) { 24 | return this.activeRequests[0] || null; 25 | } 26 | const request = { 27 | ...(this.request.Limit && { 28 | Limit: this.request.Limit - this.totalReturned, 29 | }), 30 | ...this.request, 31 | ...(Boolean(this.nextToken) && 32 | typeof this.nextToken === "object" && { 33 | ExclusiveStartKey: this.nextToken, 34 | }), 35 | }; 36 | const promise = this.documentClient[this.operation](request).promise(); 37 | return promise; 38 | } 39 | processResult(data) { 40 | this.nextToken = (data && data.LastEvaluatedKey) || null; 41 | if (data && data.Items) { 42 | this.totalReturned += data.Items.length; 43 | this.results.push(...data.Items); 44 | } 45 | } 46 | // override since filtering results in inconsistent result set size, base buffer on the items returned last 47 | // this may give surprising results if the returned list varies considerably, but errs on the side of caution. 48 | getResultBatch(batchSize) { 49 | const items = super.getResultBatch(batchSize); 50 | if (items.length > 0) { 51 | this.bufferSize = this.results.length / items.length; 52 | } 53 | else if (!this.activeRequests.length) { 54 | // if we don't have any items to process, and no active requests, buffer size should be zero. 55 | this.bufferSize = 0; 56 | } 57 | return items; 58 | } 59 | } 60 | exports.QueryFetcher = QueryFetcher; 61 | -------------------------------------------------------------------------------- /lib/ScanQueryPipeline.d.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { TableIterator } from "./TableIterator"; 3 | import { ComparisonOperator, ConditionExpression, Key, KeyConditions, QueryTemplate, Scalar } from "./types"; 4 | export type SortArgs = [Exclude">, Scalar] | ["between", Scalar, Scalar]; 5 | export declare const sortKey: (...args: SortArgs) => QueryTemplate; 6 | export declare class ScanQueryPipeline { 13 | config: { 14 | client: DocumentClient; 15 | table: string; 16 | keys: KD; 17 | index?: string; 18 | readBuffer: number; 19 | writeBuffer: number; 20 | readBatchSize: number; 21 | writeBatchSize: number; 22 | }; 23 | unprocessedItems: Key[]; 24 | constructor(tableName: string, keys: { 25 | pk: PK; 26 | sk?: SK; 27 | }, index?: string, config?: { 28 | client?: DocumentClient; 29 | readBuffer?: number; 30 | writeBuffer?: number; 31 | readBatchSize?: number; 32 | writeBatchSize?: number; 33 | }); 34 | static sortKey: (...args: SortArgs) => QueryTemplate; 35 | sortKey: (...args: SortArgs) => QueryTemplate; 36 | withReadBuffer(readBuffer: number): this; 37 | withReadBatchSize(readBatchSize: number): this; 38 | query(keyConditions: KeyConditions<{ 39 | pk: PK; 40 | sk: SK; 41 | }>, options?: { 42 | sortDescending?: true; 43 | batchSize?: number; 44 | bufferCapacity?: number; 45 | limit?: number; 46 | filters?: ConditionExpression; 47 | consistentRead?: boolean; 48 | nextToken?: Key; 49 | }): TableIterator; 50 | scan(options?: { 51 | batchSize?: number; 52 | bufferCapacity?: number; 53 | limit?: number; 54 | filters?: ConditionExpression; 55 | consistentRead?: boolean; 56 | nextToken?: Key; 57 | }): TableIterator; 58 | private buildQueryScanRequest; 59 | } 60 | -------------------------------------------------------------------------------- /lib/ScanQueryPipeline.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.ScanQueryPipeline = exports.sortKey = void 0; 4 | const dynamodb_1 = require("aws-sdk/clients/dynamodb"); 5 | const helpers_1 = require("./helpers"); 6 | const QueryFetcher_1 = require("./QueryFetcher"); 7 | const TableIterator_1 = require("./TableIterator"); 8 | const sortKey = (...args) => { 9 | if (args.length === 3) { 10 | return ["between", "and", args[1], args[2]]; 11 | } 12 | return args; 13 | }; 14 | exports.sortKey = sortKey; 15 | class ScanQueryPipeline { 16 | constructor(tableName, keys, index, config) { 17 | this.sortKey = exports.sortKey; 18 | this.config = { 19 | table: tableName, 20 | readBuffer: 1, 21 | writeBuffer: 3, 22 | readBatchSize: 100, 23 | writeBatchSize: 25, 24 | ...config, 25 | // shortcut to use KD, otherwise type definitions throughout the 26 | // class are too long 27 | keys: keys, 28 | index, 29 | client: (config && config.client) || new dynamodb_1.DocumentClient(), 30 | }; 31 | this.unprocessedItems = []; 32 | return this; 33 | } 34 | withReadBuffer(readBuffer) { 35 | if (readBuffer < 0) { 36 | throw new Error("Read buffer out of range"); 37 | } 38 | this.config.readBuffer = readBuffer; 39 | return this; 40 | } 41 | withReadBatchSize(readBatchSize) { 42 | if (readBatchSize < 1) { 43 | throw new Error("Read batch size out of range"); 44 | } 45 | this.config.readBatchSize = readBatchSize; 46 | return this; 47 | } 48 | query(keyConditions, options) { 49 | const request = this.buildQueryScanRequest({ ...options, keyConditions }); 50 | const fetchOptions = { 51 | bufferCapacity: this.config.readBuffer, 52 | batchSize: this.config.readBatchSize, 53 | ...options, 54 | }; 55 | return new TableIterator_1.TableIterator(new QueryFetcher_1.QueryFetcher(request, this.config.client, "query", fetchOptions), this); 56 | } 57 | scan(options) { 58 | const request = this.buildQueryScanRequest(options !== null && options !== void 0 ? options : {}); 59 | const fetchOptions = { 60 | bufferCapacity: this.config.readBuffer, 61 | batchSize: this.config.readBatchSize, 62 | ...options, 63 | }; 64 | return new TableIterator_1.TableIterator(new QueryFetcher_1.QueryFetcher(request, this.config.client, "scan", fetchOptions), this); 65 | } 66 | buildQueryScanRequest(options) { 67 | const pkName = this.config.keys.pk; 68 | const skName = this.config.keys.sk; 69 | const skValue = options.keyConditions && typeof skName !== "undefined" && options.keyConditions && skName in options.keyConditions 70 | ? options.keyConditions[skName] 71 | : null; 72 | const request = { 73 | TableName: this.config.table, 74 | ...(options.limit && { 75 | Limit: options.limit, 76 | }), 77 | ...(this.config.index && { IndexName: this.config.index }), 78 | ...(options.keyConditions && { 79 | KeyConditionExpression: "#p0 = :v0" + (skValue ? ` AND ${(0, helpers_1.skQueryToDynamoString)(skValue)}` : ""), 80 | }), 81 | ConsistentRead: Boolean(options.consistentRead), 82 | ScanIndexForward: Boolean(!options.sortDescending), 83 | }; 84 | const [skVal1, skVal2] = (skValue === null || skValue === void 0 ? void 0 : skValue.length) === 4 ? [skValue[2], skValue[3]] : (skValue === null || skValue === void 0 ? void 0 : skValue.length) === 2 ? [skValue[1], null] : [null, null]; 85 | const keySubstitues = { 86 | Condition: "", 87 | ExpressionAttributeNames: options.keyConditions 88 | ? { 89 | "#p0": pkName, 90 | ...(skValue && { 91 | "#p1": skName, 92 | }), 93 | } 94 | : undefined, 95 | ExpressionAttributeValues: options.keyConditions 96 | ? { 97 | ":v0": options.keyConditions[pkName], 98 | ...(skVal1 !== null && { 99 | ":v1": skVal1, 100 | }), 101 | ...(skVal2 !== null && { 102 | ":v2": skVal2, 103 | }), 104 | } 105 | : undefined, 106 | }; 107 | if (options.filters) { 108 | const compiledCondition = (0, helpers_1.conditionToDynamo)(options.filters, keySubstitues); 109 | request.FilterExpression = compiledCondition.Condition; 110 | request.ExpressionAttributeNames = compiledCondition.ExpressionAttributeNames; 111 | request.ExpressionAttributeValues = compiledCondition.ExpressionAttributeValues; 112 | } 113 | else { 114 | request.ExpressionAttributeNames = keySubstitues.ExpressionAttributeNames; 115 | request.ExpressionAttributeValues = keySubstitues.ExpressionAttributeValues; 116 | } 117 | return request; 118 | } 119 | } 120 | exports.ScanQueryPipeline = ScanQueryPipeline; 121 | ScanQueryPipeline.sortKey = exports.sortKey; 122 | -------------------------------------------------------------------------------- /lib/TableIterator.d.ts: -------------------------------------------------------------------------------- 1 | import DynamoDB from "aws-sdk/clients/dynamodb"; 2 | interface IteratorExecutor { 3 | execute(): AsyncGenerator; 5 | } | void, void>; 6 | } 7 | export declare class TableIterator { 8 | private lastEvaluatedKeyHandlers; 9 | config: { 10 | parent: P; 11 | fetcher: IteratorExecutor; 12 | }; 13 | constructor(fetcher: IteratorExecutor, parent?: P); 14 | forEachStride(iterator: (items: T[], index: number, parent: P, cancel: () => void) => Promise | void): Promise

; 15 | onLastEvaluatedKey(handler: (lastEvaluatedKey: Record) => void): this; 16 | private iterate; 17 | private handleDone; 18 | forEach(iterator: (item: T, index: number, pipeline: P, cancel: () => void) => Promise | void): Promise

; 19 | map(iterator: (item: T, index: number) => U): Promise; 20 | filterLazy(predicate: (item: T, index: number) => boolean): TableIterator; 21 | mapLazy(iterator: (item: T, index: number) => U): TableIterator; 22 | all(): Promise; 23 | iterator(): AsyncGenerator; 24 | strideIterator(): AsyncGenerator | void, void>; 25 | } 26 | export {}; 27 | -------------------------------------------------------------------------------- /lib/TableIterator.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.TableIterator = void 0; 4 | class TableIterator { 5 | constructor(fetcher, parent) { 6 | this.lastEvaluatedKeyHandlers = []; 7 | this.config = { parent: parent, fetcher }; 8 | } 9 | async forEachStride(iterator) { 10 | let index = 0; 11 | await this.iterate(this.config.fetcher, async (stride, cancel) => { 12 | await iterator(stride, index, this.config.parent, cancel); 13 | index += 1; 14 | }); 15 | return this.config.parent; 16 | } 17 | onLastEvaluatedKey(handler) { 18 | this.lastEvaluatedKeyHandlers.push(handler); 19 | return this; 20 | } 21 | async iterate(fetcher, iterator) { 22 | let cancelled = false; 23 | const cancel = () => { 24 | cancelled = true; 25 | }; 26 | const executor = fetcher.execute(); 27 | while (true) { 28 | if (cancelled) { 29 | break; 30 | } 31 | const stride = await executor.next(); 32 | const { value } = stride; 33 | if (stride.done) { 34 | this.handleDone(stride); 35 | break; 36 | } 37 | await iterator(value, cancel); 38 | } 39 | } 40 | handleDone(iteratorResponse) { 41 | const { value } = iteratorResponse; 42 | if (value && "lastEvaluatedKey" in value) { 43 | this.lastEvaluatedKeyHandlers.forEach((h) => h(value.lastEvaluatedKey)); 44 | this.lastEvaluatedKeyHandlers = []; 45 | } 46 | } 47 | // when a promise is returned, all promises are resolved in the batch before processing the next batch 48 | async forEach(iterator) { 49 | let index = 0; 50 | let iteratorPromises = []; 51 | let cancelled = false; 52 | const cancelForEach = () => { 53 | cancelled = true; 54 | }; 55 | await this.iterate(this.config.fetcher, async (stride, cancel) => { 56 | iteratorPromises = []; 57 | for (const item of stride) { 58 | const iteratorResponse = iterator(item, index, this.config.parent, cancelForEach); 59 | index += 1; 60 | if (cancelled) { 61 | await Promise.all(iteratorPromises); 62 | cancel(); 63 | break; 64 | } 65 | else if (typeof iteratorResponse === "object" && iteratorResponse instanceof Promise) { 66 | iteratorPromises.push(iteratorResponse); 67 | } 68 | } 69 | await Promise.all(iteratorPromises); 70 | }); 71 | await Promise.all(iteratorPromises); 72 | return this.config.parent; 73 | } 74 | async map(iterator) { 75 | const results = []; 76 | let index = 0; 77 | await this.iterate(this.config.fetcher, (stride, _cancel) => { 78 | for (const item of stride) { 79 | results.push(iterator(item, index)); 80 | index += 1; 81 | } 82 | return Promise.resolve(); 83 | }); 84 | return results; 85 | } 86 | filterLazy(predicate) { 87 | const existingFetcher = this.config.fetcher; 88 | let index = 0; 89 | // eslint-disable-next-line @typescript-eslint/no-this-alias 90 | const that = this; 91 | const fetcher = async function* () { 92 | const executor = existingFetcher.execute(); 93 | while (true) { 94 | const stride = await executor.next(); 95 | if (stride.done) { 96 | that.handleDone(stride); 97 | break; 98 | } 99 | yield stride.value.filter((val, i) => { 100 | const filtered = predicate(val, index); 101 | index += 1; 102 | return filtered; 103 | }); 104 | } 105 | }; 106 | return new TableIterator({ execute: fetcher }, this.config.parent); 107 | } 108 | mapLazy(iterator) { 109 | const existingFetcher = this.config.fetcher; 110 | let results = []; 111 | let index = 0; 112 | // eslint-disable-next-line @typescript-eslint/no-this-alias 113 | const that = this; 114 | const fetcher = async function* () { 115 | const executor = existingFetcher.execute(); 116 | while (true) { 117 | const stride = await executor.next(); 118 | if (stride.done) { 119 | that.handleDone(stride); 120 | break; 121 | } 122 | results = stride.value.map((item) => { 123 | const result = iterator(item, index); 124 | index += 1; 125 | return result; 126 | }); 127 | yield results; 128 | } 129 | }; 130 | return new TableIterator({ execute: fetcher }, this.config.parent); 131 | } 132 | all() { 133 | const result = this.map((i) => i); 134 | return result; 135 | } 136 | async *iterator() { 137 | const executor = this.config.fetcher.execute(); 138 | while (true) { 139 | const stride = await executor.next(); 140 | if (stride.done) { 141 | this.handleDone(stride); 142 | return; 143 | } 144 | for (const item of stride.value) { 145 | yield item; 146 | } 147 | } 148 | } 149 | strideIterator() { 150 | return this.config.fetcher.execute(); 151 | } 152 | } 153 | exports.TableIterator = TableIterator; 154 | -------------------------------------------------------------------------------- /lib/helpers/index.d.ts: -------------------------------------------------------------------------------- 1 | import { ConditionExpression, DynamoCondition, KeyDefinition, QueryTemplate } from "../types"; 2 | export declare function conditionToDynamo(condition: ConditionExpression | undefined, mergeCondition?: DynamoCondition): DynamoCondition; 3 | export declare const pkName: (keys: KeyDefinition) => string; 4 | export declare function skQueryToDynamoString(template: QueryTemplate): string; 5 | -------------------------------------------------------------------------------- /lib/helpers/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.skQueryToDynamoString = exports.pkName = exports.conditionToDynamo = void 0; 4 | function conditionToDynamo(condition, mergeCondition) { 5 | const result = mergeCondition || 6 | { 7 | Condition: "", 8 | }; 9 | if (!condition) { 10 | return result; 11 | } 12 | if ("logical" in condition) { 13 | const preCondition = result.Condition; 14 | const logicalLhs = conditionToDynamo(condition.lhs, result); 15 | const logicalRhs = conditionToDynamo(condition.rhs, { 16 | Condition: preCondition, 17 | ExpressionAttributeNames: { 18 | ...result.ExpressionAttributeNames, 19 | ...logicalLhs.ExpressionAttributeNames, 20 | }, 21 | ExpressionAttributeValues: { 22 | ...result.ExpressionAttributeValues, 23 | ...logicalLhs.ExpressionAttributeValues, 24 | }, 25 | }); 26 | if (condition.lhs && "logical" in condition.lhs) { 27 | logicalLhs.Condition = `(${logicalLhs.Condition})`; 28 | } 29 | if (condition.rhs && "logical" in condition.rhs) { 30 | logicalRhs.Condition = `(${logicalRhs.Condition})`; 31 | } 32 | result.Condition = `${logicalLhs.Condition + (logicalLhs.Condition.length ? " " : "")}${condition.logical} ${logicalRhs.Condition}`; 33 | Object.entries({ 34 | ...logicalRhs.ExpressionAttributeNames, 35 | ...logicalLhs.ExpressionAttributeNames, 36 | }).forEach(([name, value]) => { 37 | if (!result.ExpressionAttributeNames) { 38 | result.ExpressionAttributeNames = {}; 39 | } 40 | // @ts-expect-error: Object.entries hard codes string as the key type, 41 | // and indexing by template strings is invalid in ts 4.2.0 42 | result.ExpressionAttributeNames[name] = value; 43 | }); 44 | Object.entries({ 45 | ...logicalRhs.ExpressionAttributeValues, 46 | ...logicalLhs.ExpressionAttributeValues, 47 | }).forEach(([name, value]) => { 48 | if (!result.ExpressionAttributeValues) { 49 | result.ExpressionAttributeValues = {}; 50 | } 51 | result.ExpressionAttributeValues[name] = value; 52 | }); 53 | return result; 54 | } 55 | const names = conditionToAttributeNames(condition, result.ExpressionAttributeNames ? Object.keys(result.ExpressionAttributeNames).length : 0); 56 | const values = conditionToAttributeValues(condition, result.ExpressionAttributeValues ? Object.keys(result.ExpressionAttributeValues).length : 0); 57 | const conditionString = conditionToConditionString(condition, result.ExpressionAttributeNames ? Object.keys(result.ExpressionAttributeNames).length : 0, result.ExpressionAttributeValues ? Object.keys(result.ExpressionAttributeValues).length : 0); 58 | return { 59 | ...((Object.keys(names).length > 0 || Object.keys(result.ExpressionAttributeNames || {}).length > 0) && { 60 | ExpressionAttributeNames: { 61 | ...names, 62 | ...result.ExpressionAttributeNames, 63 | }, 64 | }), 65 | ...((Object.keys(values).length > 0 || Object.keys(result.ExpressionAttributeValues || {}).length > 0) && { 66 | ExpressionAttributeValues: { 67 | ...values, 68 | ...result.ExpressionAttributeValues, 69 | }, 70 | }), 71 | Condition: conditionString, 72 | }; 73 | } 74 | exports.conditionToDynamo = conditionToDynamo; 75 | const pkName = (keys) => keys.pk; 76 | exports.pkName = pkName; 77 | function skQueryToDynamoString(template) { 78 | const expression = template[0] === "begins_with" 79 | ? { operator: template[0], property: "sk", value: template[1] } 80 | : template[0] === "between" 81 | ? { 82 | operator: template[0], 83 | property: "sk", 84 | start: template[2], 85 | end: template[3], 86 | } 87 | : { operator: template[0], lhs: "sk", rhs: { value: template[1] } }; 88 | const result = conditionToConditionString(expression, 1, 1); 89 | return result; 90 | } 91 | exports.skQueryToDynamoString = skQueryToDynamoString; 92 | function comparisonOperator(condition, nameStart, valueStart) { 93 | const lhs = typeof condition.lhs === "string" ? "#p" + nameStart.toString() : "#p" + nameStart.toString(); 94 | (typeof condition.lhs === "string" || "property" in condition.lhs) && (nameStart += 1); 95 | const rhs = "property" in condition.rhs ? "#p" + nameStart.toString() : ":v" + valueStart.toString(); 96 | return `${typeof condition.lhs !== "string" && "function" in condition.lhs ? condition.lhs.function + "(" : ""}${lhs}${typeof condition.lhs !== "string" && "function" in condition.lhs ? ")" : ""} ${condition.operator} ${"function" in condition.rhs ? condition.rhs.function + "(" : ""}${rhs}${"function" in condition.rhs ? ")" : ""}`; 97 | } 98 | function conditionToConditionString(condition, nameCountStart, valueCountStart) { 99 | // TODO: HACK: the name and value conversions follow the same operator flow 100 | // as the condition to values and condition to names to keep the numbers in sync 101 | // lhs, rhs, start,end,list 102 | // lhs, rhs, property, arg2 103 | if ("logical" in condition) { 104 | /* istanbul ignore next */ 105 | throw new Error("Unimplemented"); 106 | } 107 | const nameStart = nameCountStart; 108 | let valueStart = valueCountStart; 109 | switch (condition.operator) { 110 | case ">": 111 | case "<": 112 | case ">=": 113 | case "<=": 114 | case "=": 115 | case "<>": 116 | // TODO: fix any type 117 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 118 | return comparisonOperator(condition, nameStart, valueStart); 119 | case "begins_with": 120 | case "contains": 121 | case "attribute_type": 122 | return `${condition.operator}(#p${nameStart}, :v${valueStart})`; 123 | case "attribute_exists": 124 | case "attribute_not_exists": 125 | return `${condition.operator}(#p${nameStart})`; 126 | case "between": 127 | return `#p${nameStart} BETWEEN :v${valueStart} AND :v${valueStart + 1}`; 128 | case "in": 129 | return `${"#p" + nameStart.toString()} IN (${condition.list 130 | .map(() => { 131 | valueStart += 1; 132 | return `:v${valueStart - 1}`; 133 | }) 134 | .join(",")})`; 135 | default: 136 | /* istanbul ignore next */ 137 | throw new Error("Operator does not exist"); 138 | } 139 | } 140 | function conditionToAttributeValues(condition, countStart = 0) { 141 | const values = {}; 142 | if ("rhs" in condition && condition.rhs && "value" in condition.rhs) { 143 | setPropertyValue(condition.rhs.value, values, countStart); 144 | } 145 | if ("value" in condition) { 146 | setPropertyValue(condition.value, values, countStart); 147 | } 148 | if ("start" in condition) { 149 | setPropertyValue(condition.start, values, countStart); 150 | } 151 | if ("end" in condition) { 152 | setPropertyValue(condition.end, values, countStart); 153 | } 154 | if ("list" in condition) { 155 | condition.list.forEach((l) => setPropertyValue(l, values, countStart)); 156 | } 157 | return values; 158 | } 159 | function setPropertyValue(value, values, countStart) { 160 | // note this is the main place to change if we switch from document client to the regular dynamodb client 161 | const dynamoValue = Array.isArray(value) 162 | ? value.join("") 163 | : typeof value === "boolean" || typeof value === "string" || typeof value === "number" 164 | ? value 165 | : value === null 166 | ? true 167 | : (value === null || value === void 0 ? void 0 : value.toString()) || true; 168 | return setRawPropertyValue(dynamoValue, values, countStart); 169 | } 170 | function setRawPropertyValue(value, values, countStart) { 171 | const name = ":v" + (Object.keys(values).length + countStart).toString(); 172 | values[name] = value; 173 | return values; 174 | } 175 | function conditionToAttributeNames(condition, countStart = 0) { 176 | const names = {}; 177 | if ("lhs" in condition && condition.lhs && (typeof condition.lhs === "string" || "property" in condition.lhs)) { 178 | splitAndSetPropertyName(typeof condition.lhs === "string" ? condition.lhs : condition.lhs.property, names, countStart); 179 | } 180 | // TODO: Test if this is possible in a scan wih dynamo? 181 | if ("rhs" in condition && condition.rhs && "property" in condition.rhs) { 182 | splitAndSetPropertyName(condition.rhs.property, names, countStart); 183 | } 184 | if ("property" in condition) { 185 | splitAndSetPropertyName(condition.property, names, countStart); 186 | } 187 | return names; 188 | } 189 | function splitAndSetPropertyName(propertyName, names, countStart) { 190 | return propertyName 191 | .split(".") 192 | .forEach((prop) => (names["#p" + (Object.keys(names).length + countStart).toString()] = prop)); 193 | } 194 | -------------------------------------------------------------------------------- /lib/index.d.ts: -------------------------------------------------------------------------------- 1 | export { Pipeline } from "./Pipeline"; 2 | export { sortKey } from "./ScanQueryPipeline"; 3 | export { TableIterator } from "./TableIterator"; 4 | export * as helpers from "./helpers"; 5 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { 3 | if (k2 === undefined) k2 = k; 4 | var desc = Object.getOwnPropertyDescriptor(m, k); 5 | if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { 6 | desc = { enumerable: true, get: function() { return m[k]; } }; 7 | } 8 | Object.defineProperty(o, k2, desc); 9 | }) : (function(o, m, k, k2) { 10 | if (k2 === undefined) k2 = k; 11 | o[k2] = m[k]; 12 | })); 13 | var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { 14 | Object.defineProperty(o, "default", { enumerable: true, value: v }); 15 | }) : function(o, v) { 16 | o["default"] = v; 17 | }); 18 | var __importStar = (this && this.__importStar) || function (mod) { 19 | if (mod && mod.__esModule) return mod; 20 | var result = {}; 21 | if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); 22 | __setModuleDefault(result, mod); 23 | return result; 24 | }; 25 | Object.defineProperty(exports, "__esModule", { value: true }); 26 | exports.helpers = exports.TableIterator = exports.sortKey = exports.Pipeline = void 0; 27 | var Pipeline_1 = require("./Pipeline"); 28 | Object.defineProperty(exports, "Pipeline", { enumerable: true, get: function () { return Pipeline_1.Pipeline; } }); 29 | var ScanQueryPipeline_1 = require("./ScanQueryPipeline"); 30 | Object.defineProperty(exports, "sortKey", { enumerable: true, get: function () { return ScanQueryPipeline_1.sortKey; } }); 31 | var TableIterator_1 = require("./TableIterator"); 32 | Object.defineProperty(exports, "TableIterator", { enumerable: true, get: function () { return TableIterator_1.TableIterator; } }); 33 | exports.helpers = __importStar(require("./helpers")); 34 | -------------------------------------------------------------------------------- /lib/mocks/index.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 3 | import { Request } from "aws-sdk/lib/request"; 4 | type Spy = jest.MockContext, [TInput, any?]>; 5 | type WrappedFn = (client: DocumentClient, spy: Spy) => Promise; 6 | type MockReturn = { 7 | err?: Error; 8 | data?: TOutput; 9 | } | { 10 | err?: Error; 11 | data?: TOutput; 12 | }[]; 13 | export declare function setMockOn(on: boolean): void; 14 | type DynamoClientCommandName = "scan" | "query" | "delete" | "update" | "put" | "batchGet" | "batchWrite" | "transactGet"; 15 | interface MockSet> { 16 | name: DynamoClientCommandName; 17 | returns?: MockReturn; 18 | delay?: number; 19 | } 20 | export declare function multiMock(fn: (client: DocumentClient, spies: jest.MockContext[]) => Promise, mockSet: MockSet[]): () => Promise; 21 | export declare function mockScan(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 22 | export declare function alwaysMockScan(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 23 | export declare function mockQuery(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 24 | export declare function alwaysMockQuery(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 25 | export declare function alwaysMockBatchGet(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 26 | export declare function mockPut(fn: WrappedFn, returns?: MockReturn): () => Promise; 27 | export declare function mockUpdate(fn: WrappedFn, returns?: MockReturn): () => Promise; 28 | export declare function mockDelete(fn: WrappedFn, returns?: MockReturn): () => Promise; 29 | export declare function alwaysMockBatchWrite(fn: WrappedFn, returns?: MockReturn): () => Promise; 30 | export declare function mockBatchWrite(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 31 | export declare function mockBatchGet(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 32 | export declare function mockTransactGet(fn: WrappedFn, returns?: MockReturn, delay?: number): () => Promise; 33 | export {}; 34 | -------------------------------------------------------------------------------- /lib/mocks/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __importDefault = (this && this.__importDefault) || function (mod) { 3 | return (mod && mod.__esModule) ? mod : { "default": mod }; 4 | }; 5 | Object.defineProperty(exports, "__esModule", { value: true }); 6 | exports.mockTransactGet = exports.mockBatchGet = exports.mockBatchWrite = exports.alwaysMockBatchWrite = exports.mockDelete = exports.mockUpdate = exports.mockPut = exports.alwaysMockBatchGet = exports.alwaysMockQuery = exports.mockQuery = exports.alwaysMockScan = exports.mockScan = exports.multiMock = exports.setMockOn = void 0; 7 | const dynamodb_1 = require("aws-sdk/clients/dynamodb"); 8 | const aws_sdk_1 = __importDefault(require("aws-sdk")); 9 | const aws_sdk_mock_1 = __importDefault(require("aws-sdk-mock")); 10 | let mockOn = true; 11 | function setMockOn(on) { 12 | mockOn = on; 13 | } 14 | exports.setMockOn = setMockOn; 15 | function multiMock(fn, mockSet) { 16 | return async () => { 17 | const spies = mockSet.map((ms) => setupMock(ms.name, ms.returns, true, ms.delay).mock); 18 | const client = new dynamodb_1.DocumentClient(); 19 | await fn(client, spies); 20 | mockSet.forEach((ms) => teardownMock(ms.name, true)); 21 | }; 22 | } 23 | exports.multiMock = multiMock; 24 | function mockScan(fn, returns, delay) { 25 | return mockCall("scan", fn, returns, false, delay); 26 | } 27 | exports.mockScan = mockScan; 28 | function alwaysMockScan(fn, returns, delay) { 29 | return mockCall("scan", fn, returns, true, delay); 30 | } 31 | exports.alwaysMockScan = alwaysMockScan; 32 | function mockQuery(fn, returns, delay) { 33 | return mockCall("query", fn, returns, false, delay); 34 | } 35 | exports.mockQuery = mockQuery; 36 | function alwaysMockQuery(fn, returns, delay) { 37 | return mockCall("query", fn, returns, true, delay); 38 | } 39 | exports.alwaysMockQuery = alwaysMockQuery; 40 | function alwaysMockBatchGet(fn, returns, delay) { 41 | return mockCall("batchGet", fn, returns, true, delay); 42 | } 43 | exports.alwaysMockBatchGet = alwaysMockBatchGet; 44 | function mockPut(fn, returns) { 45 | return mockCall("put", fn, returns); 46 | } 47 | exports.mockPut = mockPut; 48 | function mockUpdate(fn, returns) { 49 | return mockCall("update", fn, returns); 50 | } 51 | exports.mockUpdate = mockUpdate; 52 | function mockDelete(fn, returns) { 53 | return mockCall("delete", fn, returns); 54 | } 55 | exports.mockDelete = mockDelete; 56 | function alwaysMockBatchWrite(fn, returns) { 57 | return mockCall("batchWrite", fn, returns, true); 58 | } 59 | exports.alwaysMockBatchWrite = alwaysMockBatchWrite; 60 | function mockBatchWrite(fn, returns, delay) { 61 | return mockCall("batchWrite", fn, returns, false, delay); 62 | } 63 | exports.mockBatchWrite = mockBatchWrite; 64 | function mockBatchGet(fn, returns, delay) { 65 | return mockCall("batchGet", fn, returns, false, delay); 66 | } 67 | exports.mockBatchGet = mockBatchGet; 68 | function mockTransactGet(fn, returns, delay) { 69 | return mockCall("transactGet", fn, returns, false, delay); 70 | } 71 | exports.mockTransactGet = mockTransactGet; 72 | function mockCall(name, fn, returns = {}, alwaysMock = false, delay) { 73 | return async () => { 74 | const spy = setupMock(name, returns, alwaysMock, delay); 75 | // TODO: Type cleanup 76 | // eslint-disable-next-line 77 | const client = new dynamodb_1.DocumentClient(); 78 | if (!mockOn && !alwaysMock) { 79 | // TODO: Type cleanup 80 | await fn(client, jest.spyOn(client, name).mock); 81 | } 82 | else { 83 | await fn(client, spy.mock); 84 | } 85 | teardownMock(name, alwaysMock); 86 | }; 87 | } 88 | function setupMock(name, returns = {}, alwaysMock, delay) { 89 | const spy = jest.fn(); 90 | let callCount = 0; 91 | if (mockOn || alwaysMock) { 92 | aws_sdk_mock_1.default.setSDKInstance(aws_sdk_1.default); 93 | aws_sdk_mock_1.default.mock("DynamoDB.DocumentClient", name, function (input, callback) { 94 | var _a, _b, _c, _d; 95 | spy(input); 96 | if (Array.isArray(returns)) { 97 | if (typeof delay === "number") { 98 | setTimeout(() => { 99 | var _a, _b, _c; 100 | callback((_b = (_a = returns[callCount]) === null || _a === void 0 ? void 0 : _a.err) !== null && _b !== void 0 ? _b : undefined, (_c = returns[callCount]) === null || _c === void 0 ? void 0 : _c.data); 101 | callCount += 1; 102 | }, delay); 103 | } 104 | else { 105 | callback((_b = (_a = returns[callCount]) === null || _a === void 0 ? void 0 : _a.err) !== null && _b !== void 0 ? _b : undefined, (_c = returns[callCount]) === null || _c === void 0 ? void 0 : _c.data); 106 | callCount += 1; 107 | } 108 | } 109 | else if (typeof delay === "number") { 110 | setTimeout(() => { var _a; return callback((_a = returns === null || returns === void 0 ? void 0 : returns.err) !== null && _a !== void 0 ? _a : undefined, returns === null || returns === void 0 ? void 0 : returns.data); }, delay); 111 | } 112 | else { 113 | callback((_d = returns === null || returns === void 0 ? void 0 : returns.err) !== null && _d !== void 0 ? _d : undefined, returns === null || returns === void 0 ? void 0 : returns.data); 114 | } 115 | }); 116 | } 117 | return spy; 118 | } 119 | function teardownMock(name, alwaysMock) { 120 | if (mockOn || alwaysMock) { 121 | aws_sdk_mock_1.default.restore("DynamoDB.DocumentClient", name); 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /lib/types.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | export type Scalar = string | number; 3 | export type ComparisonOperator = "=" | "<" | ">" | "<=" | ">=" | "<>"; 4 | export type Logical = "AND" | "OR" | "NOT"; 5 | export type QueryOperator = "begins_with" | "between"; 6 | export type ConditionOperator = ComparisonOperator | "contains" | "attribute_type"; 7 | export type ConditionFunction = "attribute_exists" | "attribute_not_exists"; 8 | export type SKQuery = `${Exclude">} ${Scalar}` | `begins_with ${Scalar}` | `between ${Scalar} and ${Scalar}`; 9 | export type SKQueryParts = [Exclude"> | "begins_with", Scalar] | ["between", Scalar, "and", Scalar]; 10 | export type QueryTemplate = [Exclude"> | "begins_with", Scalar] | ["between", "and", Scalar, Scalar]; 11 | export type DynamoConditionAttributeName = `#p${number}`; 12 | export type DynamoConditionAttributeValue = `:v${number}`; 13 | export type DynamoCondition = { 14 | Condition: string; 15 | ExpressionAttributeNames?: Record; 16 | ExpressionAttributeValues?: Record; 17 | }; 18 | export type SimpleKey = { 19 | pk: string; 20 | }; 21 | export type CompoundKey = { 22 | pk: string; 23 | sk: string; 24 | }; 25 | export type KeyDefinition = SimpleKey | CompoundKey; 26 | export type IndexDefinition = (SimpleKey & { 27 | name: string; 28 | }) | (CompoundKey & { 29 | name: string; 30 | }); 31 | export type KeyType = string | number | Buffer | Uint8Array; 32 | export type KeyTypeName = "N" | "S" | "B"; 33 | export type Key = Keyset extends CompoundKey ? Record : Record; 34 | export type KeyConditions = Keyset extends CompoundKey ? Record & Partial> : Record; 35 | export type PrimitiveType = string | number | null | boolean | Buffer | Uint8Array; 36 | export type PrimitiveTypeName = KeyTypeName | "NULL" | "BOOL"; 37 | export type PropertyTypeName = PrimitiveTypeName | "M" | "L"; 38 | export type DynamoValue = { 39 | [_key in PropertyTypeName]?: string | boolean | Array | Record; 40 | }; 41 | export type DynamoPrimitiveValue = { 42 | [_key in PrimitiveTypeName]?: string | boolean | number; 43 | }; 44 | export type Operand = { 45 | property: string; 46 | } | { 47 | value: PrimitiveType; 48 | } | { 49 | property: string; 50 | function: "size"; 51 | }; 52 | export type LHSOperand = string | { 53 | property: string; 54 | function: "size"; 55 | }; 56 | type BeginsWith = { 57 | property: string; 58 | operator: "begins_with"; 59 | value: PrimitiveType; 60 | }; 61 | type Between = { 62 | property: string; 63 | start: PrimitiveType; 64 | end: PrimitiveType; 65 | operator: "between"; 66 | }; 67 | type In = { 68 | property: string; 69 | list: PrimitiveType[]; 70 | operator: "in"; 71 | }; 72 | type BaseExpression = { 73 | lhs: LHSOperand; 74 | rhs: Operand; 75 | operator: T; 76 | } | Between | BeginsWith; 77 | export type ConditionExpression = BaseExpression | In | { 78 | operator: ConditionFunction; 79 | property: string; 80 | } | { 81 | lhs?: ConditionExpression; 82 | logical: Logical; 83 | rhs: ConditionExpression; 84 | }; 85 | export type UpdateReturnValues = "ALL_OLD" | "UPDATED_OLD" | "ALL_NEW" | "UPDATED_NEW"; 86 | export {}; 87 | -------------------------------------------------------------------------------- /lib/types.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamo-pipeline", 3 | "version": "0.2.9", 4 | "description": "Alternative API for DynamoDB's DocumentClient", 5 | "main": "lib/index.js", 6 | "module": "esm/index.js", 7 | "exports": { 8 | "import": "./esm/index.js", 9 | "require": "./lib/index.js", 10 | "default": "./esm/index.js" 11 | }, 12 | "files": [ 13 | "lib", 14 | "esm" 15 | ], 16 | "repository": "https://github.com/RossWilliams/dynamo-pipeline.git", 17 | "author": "RossWilliams", 18 | "license": "Apache-2.0", 19 | "private": false, 20 | "scripts": { 21 | "clean": "rm -rf lib && rm -rf esm && rm -rf .buildcache", 22 | "build": "tsc -p tsconfig.json && tsc -p tsconfig-cjs.json", 23 | "build:watch": "tsc --build --watch", 24 | "test": "NODE_ENV=test jest", 25 | "test:watch": "NODE_ENV=test jest --watch", 26 | "test:dynamodb": "TEST_WITH_DYNAMO=true NODE_ENV=test jest", 27 | "lint": "eslint . --cache --cache-location '.buildcache/lint/' --fix", 28 | "format": "prettier . --write", 29 | "types": "tsc --noEmit --pretty", 30 | "prepare": "husky install" 31 | }, 32 | "dependencies": { 33 | "aws-sdk": "^2.1515.0", 34 | "aws-sdk-mock": "^5.8.0" 35 | }, 36 | "peerDependencies": { 37 | "@types/jest": ">=26.0.19" 38 | }, 39 | "devDependencies": { 40 | "@swc/core": "^1.3.100", 41 | "@swc/jest": "^0.2.29", 42 | "@types/jest": "*", 43 | "@typescript-eslint/eslint-plugin": "^6.13.2", 44 | "@typescript-eslint/parser": "^6.13.2", 45 | "eslint": "^8.55.0", 46 | "eslint-config-prettier": "^9.1.0", 47 | "eslint-config-standard": "^17.1.0", 48 | "eslint-plugin-import": "^2.29.0", 49 | "eslint-plugin-node": "^11.1.0", 50 | "eslint-plugin-promise": "^6.1.1", 51 | "eslint-plugin-standard": "^5.0.0", 52 | "husky": "^8.0.3", 53 | "jest": "^29.7.0", 54 | "prettier": "^3.1.1", 55 | "ts-jest": "^29.1.1", 56 | "ts-node": "^10.9.2", 57 | "typescript": "5.3.3" 58 | }, 59 | "engines": { 60 | "pnpm": ">=8.10.5", 61 | "node": ">=18", 62 | "yarn": "forbidden, use pnpm", 63 | "npm": "forbidden, use pnpm" 64 | }, 65 | "packageManager": "pnpm@8.10.5", 66 | "volta": { 67 | "node": "18.15.0", 68 | "pnpm": "8.10.5" 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/AbstractFetcher.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | 3 | export abstract class AbstractFetcher { 4 | protected activeRequests: Promise[] = []; 5 | protected bufferSize = 0; 6 | protected bufferCapacity = 1; 7 | protected batchSize: number; 8 | protected limit?: number; 9 | protected totalReturned = 0; 10 | protected nextToken: number | Record | null; 11 | protected documentClient: DocumentClient; 12 | protected results: T[] = []; 13 | protected errors: Error | null = null; 14 | 15 | constructor( 16 | client: DocumentClient, 17 | options: { 18 | batchSize: number; 19 | bufferCapacity: number; 20 | limit?: number; 21 | } 22 | ) { 23 | this.documentClient = client; 24 | this.bufferCapacity = options.bufferCapacity; 25 | this.batchSize = options.batchSize; 26 | this.limit = options.limit; 27 | this.nextToken = null; 28 | } 29 | 30 | /* 31 | 1. Decide if a fetch should take place considering buffer size and capacity. 32 | 2. Perform DocumentClient operation call 33 | 3. Set next token. 34 | */ 35 | abstract fetchStrategy(): Promise | null; 36 | /* 37 | 1. Receive data from DocumentClient operation call in fetch strategy 38 | 2. Set results and totalReturned. 39 | 3. Handle API errors 40 | */ 41 | abstract processResult(data: Record): void; 42 | 43 | // take in a promise to allow recursive calls, 44 | // batch fetcher can immediately create many requests 45 | protected fetchNext(): Promise | null { 46 | const fetchResponse = this.fetchStrategy(); 47 | 48 | if (fetchResponse instanceof Promise && !this.activeRequests.includes(fetchResponse)) { 49 | return this.setupFetchProcessor(fetchResponse); 50 | } 51 | 52 | return fetchResponse; 53 | } 54 | 55 | private setupFetchProcessor(promise: Promise): Promise { 56 | this.activeRequests.push(promise); 57 | this.bufferSize += 1; 58 | return promise 59 | .then((data) => { 60 | this.activeRequests = this.activeRequests.filter((r) => r !== promise); 61 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 62 | this.processResult(data); 63 | }) 64 | .catch((e) => { 65 | this.activeRequests = this.activeRequests.filter((r) => r !== promise); 66 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 67 | this.processError(e); 68 | }); 69 | } 70 | 71 | // Entry point. 72 | async *execute(): AsyncGenerator } | void, void> { 73 | let count = 0; 74 | do { 75 | if (this.errors) { 76 | return Promise.reject(this.errors); 77 | } 78 | 79 | if (!this.hasDataReady()) { 80 | await this.fetchNext(); 81 | } 82 | 83 | // check for errors again after running another fetch 84 | if (this.errors) { 85 | return Promise.reject(this.errors); 86 | } 87 | 88 | const batch = this.getResultBatch(Math.min(this.batchSize, this.limit ? this.limit - count : 1000000000000)); 89 | count += batch.length; 90 | 91 | if (!this.isDone() && (!this.limit || count < this.limit)) { 92 | // do not await here, background process the next set of data 93 | void this.fetchNext(); 94 | } 95 | 96 | yield batch; 97 | 98 | if (this.limit && count >= this.limit) { 99 | if (typeof this.nextToken === "object" && this.nextToken !== null) { 100 | return { lastEvaluatedKey: this.nextToken }; 101 | } 102 | return; 103 | } 104 | } while (!this.isDone()); 105 | } 106 | 107 | getResultBatch(batchSize: number): T[] { 108 | const items = (this.results.length && this.results.splice(0, batchSize)) || []; 109 | 110 | if (!items.length) { 111 | this.bufferSize = this.activeRequests.length; 112 | } else { 113 | this.bufferSize -= 1; 114 | } 115 | 116 | return items; 117 | } 118 | 119 | processError(e: Error): void { 120 | this.errors = e; 121 | } 122 | 123 | hasDataReady(): boolean { 124 | return this.results.length > 0; 125 | } 126 | 127 | isDone(): boolean { 128 | return !this.isActive() && this.nextToken === null && this.results.length === 0; 129 | } 130 | 131 | isActive(): boolean { 132 | return this.activeRequests.length > 0; 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/BatchFetcher.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { Key, KeyDefinition } from "./types"; 3 | import { AbstractFetcher } from "./AbstractFetcher"; 4 | 5 | type BatchGetItems = { 6 | tableName: string; 7 | keys: Key[]; 8 | }; 9 | type TransactGetItems = { 10 | tableName: string; 11 | keys: Key; 12 | }[]; 13 | 14 | export class BatchGetFetcher extends AbstractFetcher { 15 | protected operation: "batchGet" | "transactGet"; 16 | protected chunks: BatchGetItems[] | TransactGetItems[]; 17 | protected retryKeys: BatchGetItems[] | null = []; 18 | protected onUnprocessedKeys: ((keys: Key[]) => void) | undefined; 19 | protected consistentRead = false; 20 | 21 | constructor( 22 | client: DocumentClient, 23 | operation: "batchGet" | "transactGet", 24 | items: BatchGetItems | TransactGetItems, 25 | options: { 26 | onUnprocessedKeys?: (keys: Key[]) => void; 27 | batchSize: number; 28 | bufferCapacity: number; 29 | consistentRead?: boolean; 30 | } 31 | ) { 32 | super(client, options); 33 | 34 | this.operation = operation; 35 | this.onUnprocessedKeys = options.onUnprocessedKeys; 36 | this.consistentRead = Boolean(options.consistentRead); 37 | 38 | if (operation === "batchGet" && !Array.isArray(items)) { 39 | this.chunks = this.chunkBatchRequests(items); 40 | } else { 41 | // Transactions don't support chunking, its a transaction 42 | this.chunks = [items as TransactGetItems]; 43 | } 44 | 45 | this.nextToken = 0; 46 | } 47 | 48 | private chunkBatchRequests(items: BatchGetItems) { 49 | const chunks: BatchGetItems[] = []; 50 | const n = items.keys.length; 51 | let i = 0; 52 | while (i < n) { 53 | chunks.push({ 54 | tableName: items.tableName, 55 | keys: items.keys.slice(i, (i += this.batchSize)), 56 | }); 57 | } 58 | 59 | return chunks; 60 | } 61 | 62 | retry(): Promise | null { 63 | this.chunks = this.retryKeys || []; 64 | this.nextToken = 0; 65 | this.retryKeys = null; 66 | return this.fetchNext(); 67 | // TODO: Batch Get needs to be tested with chunk size of 1 and three items 68 | } 69 | 70 | fetchStrategy(): Promise | null { 71 | if (this.retryKeys && this.retryKeys.length && this.nextToken === null && !this.isActive()) { 72 | // if finished fetching initial requests, begin to process the retry keys 73 | return this.retry(); 74 | } else if ( 75 | this.bufferSize >= this.bufferCapacity || 76 | (typeof this.nextToken === "number" && this.chunks.length <= this.nextToken) || 77 | this.nextToken === null 78 | ) { 79 | // return the current promise if buffer at capacity, or if there are no more items to fetch 80 | return this.activeRequests[0] || null; 81 | } else if (!this.hasNextChunk()) { 82 | /* istanbul ignore next */ 83 | return null; 84 | } 85 | 86 | let promise: Promise | null = null; 87 | 88 | if (this.operation === "transactGet") { 89 | const transactionRequest = this.createTransactionRequest(); 90 | 91 | if (transactionRequest === null) { 92 | /* istanbul ignore next */ 93 | return null; 94 | } 95 | 96 | promise = this.documentClient.transactGet(transactionRequest).promise(); 97 | } else if (this.operation === "batchGet") { 98 | const batchGetRequest = this.createBatchGetRequest(); 99 | 100 | if (batchGetRequest === null) { 101 | /* istanbul ignore next */ 102 | return null; 103 | } 104 | 105 | promise = this.documentClient.batchGet(batchGetRequest).promise(); 106 | } 107 | 108 | if (typeof this.nextToken === "number" && typeof this.chunks[this.nextToken + 1] !== "undefined") { 109 | this.nextToken = this.nextToken + 1; 110 | } else { 111 | this.nextToken = null; 112 | } 113 | 114 | return promise; 115 | } 116 | 117 | processResult(data: DocumentClient.BatchGetItemOutput | DocumentClient.TransactGetItemsOutput | void): void { 118 | let responseItems: ReturnType[] = []; 119 | if (data && data.Responses && Array.isArray(data.Responses)) { 120 | // transaction 121 | responseItems = data.Responses.map((r) => r.Item).filter(notEmpty) as ReturnType[]; 122 | } else if (data && data.Responses && !Array.isArray(data.Responses)) { 123 | // batch, flatten each table response 124 | responseItems = ([] as ReturnType[]) 125 | .concat(...(Object.values(data.Responses) as ReturnType[][])) 126 | .filter(notEmpty); 127 | } 128 | 129 | if (data) { 130 | const unprocessedKeys = 131 | "UnprocessedKeys" in data && (data.UnprocessedKeys as { [tableName: string]: { Keys: Key[] } }); 132 | if (unprocessedKeys) { 133 | Object.entries(unprocessedKeys).forEach(([tableName, keys]) => { 134 | this.processError({ tableName, errorKeys: keys.Keys }); 135 | }); 136 | } 137 | } 138 | 139 | this.totalReturned += responseItems.length; 140 | this.results.push(...responseItems); 141 | } 142 | 143 | processError(err: Error | { tableName: string; errorKeys: Key[] }): void { 144 | if (err && "tableName" in err && Array.isArray(this.retryKeys)) { 145 | const retryItems = splitInHalf(err.errorKeys) 146 | .filter(notEmpty) 147 | .map((k) => ({ 148 | tableName: err.tableName, 149 | keys: k, 150 | })); 151 | 152 | this.retryKeys.push(...([] as BatchGetItems[]).concat(...retryItems)); 153 | } else if (err && "errorKeys" in err && typeof this.onUnprocessedKeys !== "undefined") { 154 | this.onUnprocessedKeys(err.errorKeys); 155 | } 156 | } 157 | 158 | isDone(): boolean { 159 | return super.isDone() && (!this.retryKeys || this.retryKeys.length === 0); 160 | } 161 | 162 | private createTransactionRequest(): DocumentClient.TransactGetItemsInput | null { 163 | const currentChunk: TransactGetItems | undefined = 164 | typeof this.nextToken === "number" 165 | ? (this.chunks[this.nextToken] as TransactGetItems | undefined) 166 | : undefined; 167 | 168 | if (!currentChunk) { 169 | /* istanbul ignore next */ 170 | return null; 171 | } 172 | 173 | const transaction = { 174 | TransactItems: currentChunk.map((item) => ({ 175 | Get: { 176 | Key: item.keys, 177 | TableName: item.tableName, 178 | }, 179 | })), 180 | }; 181 | 182 | return transaction; 183 | } 184 | 185 | // each batch handles a single table for now... 186 | private createBatchGetRequest(): DocumentClient.BatchGetItemInput | null { 187 | const currentChunk: BatchGetItems | undefined = 188 | typeof this.nextToken === "number" ? (this.chunks[this.nextToken] as BatchGetItems | undefined) : undefined; 189 | 190 | if (!currentChunk) { 191 | /* istanbul ignore next */ 192 | return null; 193 | } 194 | // when multiple tables are supported in a single batch 195 | // switch to items.reduce(acc, curr) => ({...acc, [curr.tableName]: curr.keyItems,}),{}) 196 | const request = { 197 | RequestItems: { 198 | [currentChunk.tableName]: { 199 | ConsistentRead: this.consistentRead, 200 | Keys: currentChunk.keys, 201 | }, 202 | }, 203 | }; 204 | return request; 205 | } 206 | 207 | private hasNextChunk(): boolean { 208 | if (typeof this.nextToken !== "number" || this.nextToken >= this.chunks.length) { 209 | return false; 210 | } 211 | 212 | return true; 213 | } 214 | } 215 | 216 | function notEmpty(val: T | null | undefined | []): val is T { 217 | if (Array.isArray(val) && !val.length) { 218 | return false; 219 | } 220 | 221 | return !!val; 222 | } 223 | 224 | function splitInHalf(arr: T[]): T[][] { 225 | return [arr.slice(0, Math.ceil(arr.length / 2)), arr.slice(Math.ceil(arr.length / 2), arr.length)]; 226 | } 227 | -------------------------------------------------------------------------------- /src/BatchWriter.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { Key, KeyDefinition } from "./types"; 3 | 4 | type BatchWriteItems = { 5 | tableName: string; 6 | records: Key[]; 7 | }; 8 | 9 | export class BatchWriter { 10 | private client: DocumentClient; 11 | private tableName: string; 12 | private activeRequests: Promise[] = []; 13 | private chunks: Key[][]; 14 | private nextToken: number | null; 15 | private retryKeys: Key[][] | null = []; 16 | private errors: Error | null = null; 17 | private batchSize = 25; 18 | private bufferCapacity = 3; 19 | private backoffActive = false; 20 | private onUnprocessedItems: ((keys: Key[]) => void) | undefined; 21 | 22 | constructor( 23 | client: DocumentClient, 24 | items: BatchWriteItems, 25 | options: { 26 | onUnprocessedItems?: (keys: Key[]) => void; 27 | batchSize: number; 28 | bufferCapacity: number; 29 | } 30 | ) { 31 | this.client = client; 32 | this.tableName = items.tableName; 33 | this.batchSize = options.batchSize; 34 | this.bufferCapacity = options.bufferCapacity; 35 | this.onUnprocessedItems = options.onUnprocessedItems; 36 | this.chunks = this.chunkBatchWrites(items); 37 | this.nextToken = 0; 38 | } 39 | 40 | async execute(): Promise { 41 | do { 42 | if (this.errors) { 43 | return Promise.reject(this.errors); 44 | } 45 | 46 | if (!this.isDone()) { 47 | await this.writeChunk(); 48 | } 49 | } while (!this.isDone()); 50 | 51 | await Promise.all(this.activeRequests); 52 | } 53 | 54 | private chunkBatchWrites(items: BatchWriteItems): Key[][] { 55 | const chunks = []; 56 | let i = 0; 57 | const n = items.records.length; 58 | 59 | while (i < n) { 60 | chunks.push(items.records.slice(i, (i += this.batchSize || 25))); 61 | } 62 | 63 | return chunks; 64 | } 65 | 66 | private async writeChunk(): Promise { 67 | if (this.retryKeys && this.retryKeys.length && this.nextToken === null && !this.isActive()) { 68 | // if finished fetching initial requests, begin to process the retry keys 69 | return this.retry(); 70 | } else if (this.activeRequests.length >= this.bufferCapacity || this.nextToken === null || this.backoffActive) { 71 | // return the current promise if buffer at capacity, or if there are no more items to fetch 72 | return this.activeRequests[0] || null; 73 | } else if (!this.hasNextChunk()) { 74 | this.nextToken = null; 75 | // let the caller wait until all active requests are finished 76 | return Promise.all(this.activeRequests).then(); 77 | } 78 | 79 | const chunk = this.getNextChunk(); 80 | 81 | if (chunk) { 82 | const request = this.client.batchWrite({ 83 | RequestItems: { 84 | [this.tableName]: chunk.map((item) => ({ 85 | PutRequest: { 86 | Item: item, 87 | }, 88 | })), 89 | }, 90 | }); 91 | 92 | if (request && typeof request.on === "function") { 93 | request.on("retry", (e?: { error?: { retryable?: boolean } }) => { 94 | if (e?.error?.retryable) { 95 | // reduce buffer capacity on retryable error 96 | this.bufferCapacity = Math.max(Math.floor((this.bufferCapacity * 3) / 4), 5); 97 | this.backoffActive = true; 98 | } 99 | }); 100 | } 101 | 102 | const promise = request 103 | .promise() 104 | .catch((e) => { 105 | console.error("Error: AWS Error, Put Items", e); 106 | if (this.onUnprocessedItems) { 107 | this.onUnprocessedItems(chunk); 108 | } 109 | this.errors = e as Error; 110 | }) 111 | .then((results) => { 112 | this.processResult(results, promise); 113 | }); 114 | 115 | this.activeRequests.push(promise); 116 | } 117 | } 118 | 119 | private getNextChunk(): Key[] | null { 120 | if (this.nextToken === null) { 121 | /* istanbul ignore next */ 122 | return null; 123 | } 124 | 125 | const chunk = this.chunks[this.nextToken] || null; 126 | 127 | this.nextToken += 1; 128 | 129 | return chunk; 130 | } 131 | 132 | private isActive(): boolean { 133 | return this.activeRequests.length > 0; 134 | } 135 | 136 | private processResult(data: DocumentClient.BatchWriteItemOutput | void, request: Promise): void { 137 | this.activeRequests = this.activeRequests.filter((r) => r !== request); 138 | 139 | if (!this.activeRequests.length || !data || !data.UnprocessedItems) { 140 | this.backoffActive = false; 141 | } 142 | 143 | if (data && data.UnprocessedItems && (data.UnprocessedItems[this.tableName]?.length || 0) > 0) { 144 | // eslint-disable-next-line 145 | const unprocessedItems = data.UnprocessedItems[this.tableName]!.map((ui) => ui.PutRequest?.Item as Key); 146 | if (Array.isArray(this.retryKeys)) { 147 | const retryItems = splitInHalf(unprocessedItems).filter(notEmpty); 148 | this.retryKeys.push(...retryItems); 149 | } else if (this.onUnprocessedItems) { 150 | this.onUnprocessedItems(unprocessedItems); 151 | } 152 | } 153 | } 154 | 155 | private retry(): Promise { 156 | this.chunks = this.retryKeys || []; 157 | this.nextToken = 0; 158 | this.retryKeys = null; 159 | return this.writeChunk(); 160 | } 161 | 162 | private isDone(): boolean { 163 | return !this.isActive() && (!this.retryKeys || this.retryKeys.length === 0) && this.nextToken === null; 164 | } 165 | 166 | private hasNextChunk(): boolean { 167 | if (this.nextToken === null || this.nextToken >= this.chunks.length) { 168 | return false; 169 | } 170 | 171 | return true; 172 | } 173 | } 174 | 175 | function notEmpty(val: T | null | undefined | []): val is T { 176 | if (Array.isArray(val) && !val.length) { 177 | return false; 178 | } 179 | 180 | return !!val; 181 | } 182 | 183 | function splitInHalf(arr: T[]): T[][] { 184 | return [arr.slice(0, Math.ceil(arr.length / 2)), arr.slice(Math.ceil(arr.length / 2), arr.length)]; 185 | } 186 | -------------------------------------------------------------------------------- /src/QueryFetcher.ts: -------------------------------------------------------------------------------- 1 | import { AbstractFetcher } from "./AbstractFetcher"; 2 | import { ScanInput, QueryInput, DocumentClient } from "aws-sdk/clients/dynamodb"; 3 | 4 | export class QueryFetcher extends AbstractFetcher { 5 | private request: ScanInput | QueryInput; 6 | private operation: "query" | "scan"; 7 | 8 | constructor( 9 | request: ScanInput | QueryInput, 10 | client: DocumentClient, 11 | operation: "query" | "scan", 12 | options: { 13 | batchSize: number; 14 | bufferCapacity: number; 15 | limit?: number; 16 | nextToken?: DocumentClient.Key; 17 | } 18 | ) { 19 | super(client, options); 20 | this.request = request; 21 | this.operation = operation; 22 | if (options.nextToken) { 23 | this.nextToken = options.nextToken; 24 | } else { 25 | this.nextToken = 1; 26 | } 27 | } 28 | 29 | // TODO: remove null response type 30 | fetchStrategy(): null | Promise { 31 | // no support for parallel query 32 | // 1. 1 active request allowed at a time 33 | // 2. Do not create a new request when the buffer is full 34 | // 3. If there are no more items to fetch, exit 35 | if (this.activeRequests.length > 0 || this.bufferSize > this.bufferCapacity || !this.nextToken) { 36 | return this.activeRequests[0] || null; 37 | } 38 | 39 | const request = { 40 | ...(this.request.Limit && { 41 | Limit: this.request.Limit - this.totalReturned, 42 | }), 43 | ...this.request, 44 | ...(Boolean(this.nextToken) && 45 | typeof this.nextToken === "object" && { 46 | ExclusiveStartKey: this.nextToken, 47 | }), 48 | }; 49 | 50 | const promise = this.documentClient[this.operation](request).promise(); 51 | 52 | return promise; 53 | } 54 | 55 | processResult(data: DocumentClient.ScanOutput | DocumentClient.QueryOutput | void): void { 56 | this.nextToken = (data && data.LastEvaluatedKey) || null; 57 | 58 | if (data && data.Items) { 59 | this.totalReturned += data.Items.length; 60 | this.results.push(...(data.Items as T[])); 61 | } 62 | } 63 | 64 | // override since filtering results in inconsistent result set size, base buffer on the items returned last 65 | // this may give surprising results if the returned list varies considerably, but errs on the side of caution. 66 | getResultBatch(batchSize: number): T[] { 67 | const items = super.getResultBatch(batchSize); 68 | 69 | if (items.length > 0) { 70 | this.bufferSize = this.results.length / items.length; 71 | } else if (!this.activeRequests.length) { 72 | // if we don't have any items to process, and no active requests, buffer size should be zero. 73 | this.bufferSize = 0; 74 | } 75 | 76 | return items; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/ScanQueryPipeline.ts: -------------------------------------------------------------------------------- 1 | import { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import { conditionToDynamo, skQueryToDynamoString } from "./helpers"; 3 | import { QueryFetcher } from "./QueryFetcher"; 4 | import { TableIterator } from "./TableIterator"; 5 | import { 6 | ComparisonOperator, 7 | ConditionExpression, 8 | DynamoCondition, 9 | Key, 10 | KeyConditions, 11 | QueryTemplate, 12 | Scalar, 13 | } from "./types"; 14 | 15 | export type SortArgs = [Exclude">, Scalar] | ["between", Scalar, Scalar]; 16 | 17 | export const sortKey = (...args: SortArgs): QueryTemplate => { 18 | if (args.length === 3) { 19 | return ["between", "and", args[1], args[2]]; 20 | } 21 | 22 | return args; 23 | }; 24 | 25 | export class ScanQueryPipeline< 26 | PK extends string, 27 | SK extends string | undefined = undefined, 28 | KD extends { pk: PK; sk: SK } = { pk: PK; sk: SK }, 29 | > { 30 | config: { 31 | client: DocumentClient; 32 | table: string; 33 | keys: KD; 34 | index?: string; 35 | readBuffer: number; 36 | writeBuffer: number; 37 | readBatchSize: number; 38 | writeBatchSize: number; 39 | }; 40 | 41 | unprocessedItems: Key[]; 42 | constructor( 43 | tableName: string, 44 | keys: { pk: PK; sk?: SK }, 45 | index?: string, 46 | config?: { 47 | client?: DocumentClient; 48 | readBuffer?: number; 49 | writeBuffer?: number; 50 | readBatchSize?: number; 51 | writeBatchSize?: number; 52 | } 53 | ) { 54 | this.config = { 55 | table: tableName, 56 | readBuffer: 1, 57 | writeBuffer: 3, 58 | readBatchSize: 100, 59 | writeBatchSize: 25, 60 | ...config, 61 | // shortcut to use KD, otherwise type definitions throughout the 62 | // class are too long 63 | keys: keys as unknown as KD, 64 | index, 65 | client: (config && config.client) || new DocumentClient(), 66 | }; 67 | this.unprocessedItems = []; 68 | 69 | return this; 70 | } 71 | 72 | static sortKey = sortKey; 73 | sortKey = sortKey; 74 | 75 | withReadBuffer(readBuffer: number): this { 76 | if (readBuffer < 0) { 77 | throw new Error("Read buffer out of range"); 78 | } 79 | this.config.readBuffer = readBuffer; 80 | return this; 81 | } 82 | 83 | withReadBatchSize(readBatchSize: number): this { 84 | if (readBatchSize < 1) { 85 | throw new Error("Read batch size out of range"); 86 | } 87 | this.config.readBatchSize = readBatchSize; 88 | return this; 89 | } 90 | 91 | query( 92 | keyConditions: KeyConditions<{ pk: PK; sk: SK }>, 93 | options?: { 94 | sortDescending?: true; 95 | batchSize?: number; 96 | bufferCapacity?: number; 97 | limit?: number; 98 | filters?: ConditionExpression; 99 | consistentRead?: boolean; 100 | nextToken?: Key; 101 | } 102 | ): TableIterator { 103 | const request = this.buildQueryScanRequest({ ...options, keyConditions }); 104 | 105 | const fetchOptions = { 106 | bufferCapacity: this.config.readBuffer, 107 | batchSize: this.config.readBatchSize, 108 | ...options, 109 | }; 110 | 111 | return new TableIterator(new QueryFetcher(request, this.config.client, "query", fetchOptions), this); 112 | } 113 | 114 | scan(options?: { 115 | batchSize?: number; 116 | bufferCapacity?: number; 117 | limit?: number; 118 | filters?: ConditionExpression; 119 | consistentRead?: boolean; 120 | nextToken?: Key; 121 | }): TableIterator { 122 | const request = this.buildQueryScanRequest(options ?? {}); 123 | 124 | const fetchOptions = { 125 | bufferCapacity: this.config.readBuffer, 126 | batchSize: this.config.readBatchSize, 127 | ...options, 128 | }; 129 | 130 | return new TableIterator(new QueryFetcher(request, this.config.client, "scan", fetchOptions), this); 131 | } 132 | 133 | private buildQueryScanRequest(options: { 134 | keyConditions?: KeyConditions; 135 | batchSize?: number; 136 | limit?: number; 137 | filters?: ConditionExpression; 138 | bufferCapacity?: number; 139 | consistentRead?: boolean; 140 | sortDescending?: true; 141 | }): DocumentClient.ScanInput | DocumentClient.QueryInput { 142 | const pkName = this.config.keys.pk; 143 | const skName = this.config.keys.sk; 144 | 145 | const skValue: QueryTemplate | null = 146 | options.keyConditions && typeof skName !== "undefined" && options.keyConditions && skName in options.keyConditions 147 | ? (options.keyConditions as Record, QueryTemplate>)[skName as Exclude] 148 | : null; 149 | 150 | const request: DocumentClient.ScanInput | DocumentClient.QueryInput = { 151 | TableName: this.config.table, 152 | ...(options.limit && { 153 | Limit: options.limit, 154 | }), 155 | ...(this.config.index && { IndexName: this.config.index }), 156 | ...(options.keyConditions && { 157 | KeyConditionExpression: "#p0 = :v0" + (skValue ? ` AND ${skQueryToDynamoString(skValue)}` : ""), 158 | }), 159 | ConsistentRead: Boolean(options.consistentRead), 160 | ScanIndexForward: Boolean(!options.sortDescending), 161 | }; 162 | 163 | const [skVal1, skVal2] = 164 | skValue?.length === 4 ? [skValue[2], skValue[3]] : skValue?.length === 2 ? [skValue[1], null] : [null, null]; 165 | 166 | const keySubstitues: DynamoCondition = { 167 | Condition: "", 168 | ExpressionAttributeNames: options.keyConditions 169 | ? { 170 | "#p0": pkName, 171 | ...(skValue && { 172 | "#p1": skName, 173 | }), 174 | } 175 | : undefined, 176 | ExpressionAttributeValues: options.keyConditions 177 | ? { 178 | ":v0": options.keyConditions[pkName], 179 | ...(skVal1 !== null && { 180 | ":v1": skVal1, 181 | }), 182 | ...(skVal2 !== null && { 183 | ":v2": skVal2, 184 | }), 185 | } 186 | : undefined, 187 | }; 188 | 189 | if (options.filters) { 190 | const compiledCondition = conditionToDynamo(options.filters, keySubstitues); 191 | request.FilterExpression = compiledCondition.Condition; 192 | request.ExpressionAttributeNames = compiledCondition.ExpressionAttributeNames; 193 | request.ExpressionAttributeValues = compiledCondition.ExpressionAttributeValues; 194 | } else { 195 | request.ExpressionAttributeNames = keySubstitues.ExpressionAttributeNames; 196 | request.ExpressionAttributeValues = keySubstitues.ExpressionAttributeValues; 197 | } 198 | 199 | return request; 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /src/TableIterator.ts: -------------------------------------------------------------------------------- 1 | import DynamoDB from "aws-sdk/clients/dynamodb"; 2 | 3 | interface IteratorExecutor { 4 | execute(): AsyncGenerator } | void, void>; 5 | } 6 | export class TableIterator { 7 | private lastEvaluatedKeyHandlers: Array<(k: Record) => void> = []; 8 | 9 | config: { 10 | parent: P; 11 | fetcher: IteratorExecutor; 12 | }; 13 | 14 | constructor(fetcher: IteratorExecutor, parent?: P) { 15 | this.config = { parent: parent as P, fetcher }; 16 | } 17 | 18 | async forEachStride( 19 | iterator: (items: T[], index: number, parent: P, cancel: () => void) => Promise | void 20 | ): Promise

{ 21 | let index = 0; 22 | 23 | await this.iterate(this.config.fetcher, async (stride, cancel) => { 24 | await iterator(stride, index, this.config.parent, cancel); 25 | index += 1; 26 | }); 27 | 28 | return this.config.parent; 29 | } 30 | 31 | onLastEvaluatedKey(handler: (lastEvaluatedKey: Record) => void): this { 32 | this.lastEvaluatedKeyHandlers.push(handler); 33 | return this; 34 | } 35 | 36 | private async iterate( 37 | fetcher: IteratorExecutor, 38 | iterator: (stride: T[], cancel: () => void) => Promise 39 | ): Promise { 40 | let cancelled = false; 41 | const cancel = () => { 42 | cancelled = true; 43 | }; 44 | const executor = fetcher.execute(); 45 | 46 | while (true) { 47 | if (cancelled) { 48 | break; 49 | } 50 | const stride = await executor.next(); 51 | const { value } = stride; 52 | if (stride.done) { 53 | this.handleDone(stride); 54 | break; 55 | } 56 | 57 | await iterator(value as T[], cancel); 58 | } 59 | } 60 | 61 | private handleDone(iteratorResponse: { done: true; value?: void | { lastEvaluatedKey: Record } }) { 62 | const { value } = iteratorResponse; 63 | if (value && "lastEvaluatedKey" in value) { 64 | this.lastEvaluatedKeyHandlers.forEach((h) => h(value.lastEvaluatedKey)); 65 | this.lastEvaluatedKeyHandlers = []; 66 | } 67 | } 68 | 69 | // when a promise is returned, all promises are resolved in the batch before processing the next batch 70 | async forEach( 71 | iterator: (item: T, index: number, pipeline: P, cancel: () => void) => Promise | void 72 | ): Promise

{ 73 | let index = 0; 74 | let iteratorPromises: unknown[] = []; 75 | let cancelled = false; 76 | const cancelForEach = () => { 77 | cancelled = true; 78 | }; 79 | 80 | await this.iterate(this.config.fetcher, async (stride, cancel) => { 81 | iteratorPromises = []; 82 | for (const item of stride) { 83 | const iteratorResponse = iterator(item, index, this.config.parent, cancelForEach); 84 | index += 1; 85 | 86 | if (cancelled) { 87 | await Promise.all(iteratorPromises); 88 | cancel(); 89 | break; 90 | } else if (typeof iteratorResponse === "object" && iteratorResponse instanceof Promise) { 91 | iteratorPromises.push(iteratorResponse); 92 | } 93 | } 94 | 95 | await Promise.all(iteratorPromises); 96 | }); 97 | 98 | await Promise.all(iteratorPromises); 99 | 100 | return this.config.parent; 101 | } 102 | 103 | async map(iterator: (item: T, index: number) => U): Promise { 104 | const results: U[] = []; 105 | 106 | let index = 0; 107 | 108 | await this.iterate(this.config.fetcher, (stride, _cancel) => { 109 | for (const item of stride) { 110 | results.push(iterator(item, index)); 111 | index += 1; 112 | } 113 | 114 | return Promise.resolve(); 115 | }); 116 | return results; 117 | } 118 | 119 | filterLazy(predicate: (item: T, index: number) => boolean): TableIterator { 120 | const existingFetcher = this.config.fetcher; 121 | 122 | let index = 0; 123 | // eslint-disable-next-line @typescript-eslint/no-this-alias 124 | const that = this; 125 | 126 | const fetcher = async function* () { 127 | const executor = existingFetcher.execute(); 128 | while (true) { 129 | const stride = await executor.next(); 130 | 131 | if (stride.done) { 132 | that.handleDone(stride); 133 | break; 134 | } 135 | 136 | yield stride.value.filter((val, i) => { 137 | const filtered = predicate(val, index); 138 | index += 1; 139 | return filtered; 140 | }); 141 | } 142 | }; 143 | 144 | return new TableIterator({ execute: fetcher }, this.config.parent); 145 | } 146 | 147 | mapLazy(iterator: (item: T, index: number) => U): TableIterator { 148 | const existingFetcher = this.config.fetcher; 149 | let results: U[] = []; 150 | let index = 0; 151 | // eslint-disable-next-line @typescript-eslint/no-this-alias 152 | const that = this; 153 | 154 | const fetcher = async function* () { 155 | const executor = existingFetcher.execute(); 156 | 157 | while (true) { 158 | const stride = await executor.next(); 159 | 160 | if (stride.done) { 161 | that.handleDone(stride); 162 | break; 163 | } 164 | 165 | results = stride.value.map((item) => { 166 | const result = iterator(item, index); 167 | index += 1; 168 | return result; 169 | }); 170 | 171 | yield results; 172 | } 173 | }; 174 | 175 | return new TableIterator({ execute: fetcher }, this.config.parent); 176 | } 177 | 178 | all(): Promise { 179 | const result = this.map((i) => i); 180 | return result; 181 | } 182 | 183 | async *iterator(): AsyncGenerator { 184 | const executor = this.config.fetcher.execute(); 185 | 186 | while (true) { 187 | const stride = await executor.next(); 188 | if (stride.done) { 189 | this.handleDone(stride); 190 | return; 191 | } 192 | 193 | for (const item of stride.value) { 194 | yield item; 195 | } 196 | } 197 | } 198 | 199 | strideIterator(): AsyncGenerator | void, void> { 200 | return this.config.fetcher.execute(); 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /src/helpers/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ConditionExpression, 3 | DynamoCondition, 4 | DynamoConditionAttributeValue, 5 | KeyDefinition, 6 | LHSOperand, 7 | Operand, 8 | PrimitiveType, 9 | Scalar, 10 | SKQuery, 11 | SKQueryParts, 12 | QueryTemplate, 13 | } from "../types"; 14 | 15 | export function conditionToDynamo( 16 | condition: ConditionExpression | undefined, 17 | mergeCondition?: DynamoCondition 18 | ): DynamoCondition { 19 | const result: DynamoCondition = 20 | mergeCondition || 21 | ({ 22 | Condition: "", 23 | } as DynamoCondition); 24 | 25 | if (!condition) { 26 | return result; 27 | } 28 | 29 | if ("logical" in condition) { 30 | const preCondition = result.Condition; 31 | const logicalLhs = conditionToDynamo(condition.lhs, result); 32 | 33 | const logicalRhs = conditionToDynamo(condition.rhs, { 34 | Condition: preCondition, 35 | ExpressionAttributeNames: { 36 | ...result.ExpressionAttributeNames, 37 | ...logicalLhs.ExpressionAttributeNames, 38 | }, 39 | ExpressionAttributeValues: { 40 | ...result.ExpressionAttributeValues, 41 | ...logicalLhs.ExpressionAttributeValues, 42 | }, 43 | }); 44 | if (condition.lhs && "logical" in condition.lhs) { 45 | logicalLhs.Condition = `(${logicalLhs.Condition})`; 46 | } 47 | if (condition.rhs && "logical" in condition.rhs) { 48 | logicalRhs.Condition = `(${logicalRhs.Condition})`; 49 | } 50 | result.Condition = `${logicalLhs.Condition + (logicalLhs.Condition.length ? " " : "")}${condition.logical} ${ 51 | logicalRhs.Condition 52 | }`; 53 | 54 | Object.entries({ 55 | ...logicalRhs.ExpressionAttributeNames, 56 | ...logicalLhs.ExpressionAttributeNames, 57 | }).forEach(([name, value]) => { 58 | if (!result.ExpressionAttributeNames) { 59 | result.ExpressionAttributeNames = {}; 60 | } 61 | // @ts-expect-error: Object.entries hard codes string as the key type, 62 | // and indexing by template strings is invalid in ts 4.2.0 63 | result.ExpressionAttributeNames[name] = value; 64 | }); 65 | 66 | ( 67 | Object.entries({ 68 | ...logicalRhs.ExpressionAttributeValues, 69 | ...logicalLhs.ExpressionAttributeValues, 70 | }) as [DynamoConditionAttributeValue, Scalar][] 71 | ).forEach(([name, value]) => { 72 | if (!result.ExpressionAttributeValues) { 73 | result.ExpressionAttributeValues = {}; 74 | } 75 | 76 | result.ExpressionAttributeValues[name] = value; 77 | }); 78 | 79 | return result; 80 | } 81 | 82 | const names = conditionToAttributeNames( 83 | condition, 84 | result.ExpressionAttributeNames ? Object.keys(result.ExpressionAttributeNames).length : 0 85 | ); 86 | const values = conditionToAttributeValues( 87 | condition, 88 | result.ExpressionAttributeValues ? Object.keys(result.ExpressionAttributeValues).length : 0 89 | ); 90 | 91 | const conditionString = conditionToConditionString( 92 | condition, 93 | result.ExpressionAttributeNames ? Object.keys(result.ExpressionAttributeNames).length : 0, 94 | result.ExpressionAttributeValues ? Object.keys(result.ExpressionAttributeValues).length : 0 95 | ); 96 | 97 | return { 98 | ...((Object.keys(names).length > 0 || Object.keys(result.ExpressionAttributeNames || {}).length > 0) && { 99 | ExpressionAttributeNames: { 100 | ...names, 101 | ...result.ExpressionAttributeNames, 102 | }, 103 | }), 104 | ...((Object.keys(values).length > 0 || Object.keys(result.ExpressionAttributeValues || {}).length > 0) && { 105 | ExpressionAttributeValues: { 106 | ...values, 107 | ...result.ExpressionAttributeValues, 108 | }, 109 | }), 110 | Condition: conditionString, 111 | }; 112 | } 113 | 114 | export const pkName = (keys: KeyDefinition): string => keys.pk; 115 | 116 | export function skQueryToDynamoString(template: QueryTemplate): string { 117 | const expression: ConditionExpression = 118 | template[0] === "begins_with" 119 | ? { operator: template[0], property: "sk", value: template[1] } 120 | : template[0] === "between" 121 | ? { 122 | operator: template[0], 123 | property: "sk", 124 | start: template[2], 125 | end: template[3], 126 | } 127 | : { operator: template[0], lhs: "sk", rhs: { value: template[1] } }; 128 | 129 | const result = conditionToConditionString(expression, 1, 1); 130 | return result; 131 | } 132 | 133 | function comparisonOperator( 134 | condition: { 135 | lhs: LHSOperand; 136 | rhs: Operand; 137 | operator: ">" | "<" | ">=" | "<=" | "=" | "<>"; 138 | }, 139 | nameStart: number, 140 | valueStart: number 141 | ) { 142 | const lhs = typeof condition.lhs === "string" ? "#p" + nameStart.toString() : "#p" + nameStart.toString(); 143 | (typeof condition.lhs === "string" || "property" in condition.lhs) && (nameStart += 1); 144 | const rhs = "property" in condition.rhs ? "#p" + nameStart.toString() : ":v" + valueStart.toString(); 145 | return `${ 146 | typeof condition.lhs !== "string" && "function" in condition.lhs ? condition.lhs.function + "(" : "" 147 | }${lhs}${typeof condition.lhs !== "string" && "function" in condition.lhs ? ")" : ""} ${condition.operator} ${ 148 | "function" in condition.rhs ? condition.rhs.function + "(" : "" 149 | }${rhs}${"function" in condition.rhs ? ")" : ""}`; 150 | } 151 | 152 | function conditionToConditionString( 153 | condition: ConditionExpression, 154 | nameCountStart: number, 155 | valueCountStart: number 156 | ): string { 157 | // TODO: HACK: the name and value conversions follow the same operator flow 158 | // as the condition to values and condition to names to keep the numbers in sync 159 | // lhs, rhs, start,end,list 160 | // lhs, rhs, property, arg2 161 | if ("logical" in condition) { 162 | /* istanbul ignore next */ 163 | throw new Error("Unimplemented"); 164 | } 165 | 166 | const nameStart = nameCountStart; 167 | let valueStart = valueCountStart; 168 | 169 | switch (condition.operator) { 170 | case ">": 171 | case "<": 172 | case ">=": 173 | case "<=": 174 | case "=": 175 | case "<>": 176 | // TODO: fix any type 177 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 178 | return comparisonOperator(condition as any, nameStart, valueStart); 179 | case "begins_with": 180 | case "contains": 181 | case "attribute_type": 182 | return `${condition.operator}(#p${nameStart}, :v${valueStart})`; 183 | case "attribute_exists": 184 | case "attribute_not_exists": 185 | return `${condition.operator}(#p${nameStart})`; 186 | case "between": 187 | return `#p${nameStart} BETWEEN :v${valueStart} AND :v${valueStart + 1}`; 188 | case "in": 189 | return `${"#p" + nameStart.toString()} IN (${condition.list 190 | .map(() => { 191 | valueStart += 1; 192 | return `:v${valueStart - 1}`; 193 | }) 194 | .join(",")})`; 195 | default: 196 | /* istanbul ignore next */ 197 | throw new Error("Operator does not exist"); 198 | } 199 | } 200 | 201 | function conditionToAttributeValues(condition: ConditionExpression, countStart = 0): { [key: string]: any } { 202 | const values: { [key: string]: any } = {}; 203 | 204 | if ("rhs" in condition && condition.rhs && "value" in condition.rhs) { 205 | setPropertyValue(condition.rhs.value, values, countStart); 206 | } 207 | 208 | if ("value" in condition) { 209 | setPropertyValue(condition.value, values, countStart); 210 | } 211 | 212 | if ("start" in condition) { 213 | setPropertyValue(condition.start, values, countStart); 214 | } 215 | 216 | if ("end" in condition) { 217 | setPropertyValue(condition.end, values, countStart); 218 | } 219 | 220 | if ("list" in condition) { 221 | condition.list.forEach((l) => setPropertyValue(l, values, countStart)); 222 | } 223 | 224 | return values; 225 | } 226 | 227 | function setPropertyValue(value: PrimitiveType, values: { [key: string]: PrimitiveType }, countStart: number) { 228 | // note this is the main place to change if we switch from document client to the regular dynamodb client 229 | const dynamoValue = Array.isArray(value) 230 | ? value.join("") 231 | : typeof value === "boolean" || typeof value === "string" || typeof value === "number" 232 | ? value 233 | : value === null 234 | ? true 235 | : value?.toString() || true; 236 | 237 | return setRawPropertyValue(dynamoValue, values, countStart); 238 | } 239 | 240 | function setRawPropertyValue(value: PrimitiveType, values: { [key: string]: any }, countStart: number) { 241 | const name: string = ":v" + (Object.keys(values).length + countStart).toString(); 242 | values[name] = value; 243 | return values; 244 | } 245 | 246 | function conditionToAttributeNames(condition: ConditionExpression, countStart = 0): { [key: string]: string } { 247 | const names: { [key: string]: string } = {}; 248 | if ("lhs" in condition && condition.lhs && (typeof condition.lhs === "string" || "property" in condition.lhs)) { 249 | splitAndSetPropertyName( 250 | typeof condition.lhs === "string" ? condition.lhs : condition.lhs.property, 251 | names, 252 | countStart 253 | ); 254 | } 255 | 256 | // TODO: Test if this is possible in a scan wih dynamo? 257 | if ("rhs" in condition && condition.rhs && "property" in condition.rhs) { 258 | splitAndSetPropertyName(condition.rhs.property, names, countStart); 259 | } 260 | 261 | if ("property" in condition) { 262 | splitAndSetPropertyName(condition.property, names, countStart); 263 | } 264 | 265 | return names; 266 | } 267 | 268 | function splitAndSetPropertyName(propertyName: string, names: { [key: string]: string }, countStart: number) { 269 | return propertyName 270 | .split(".") 271 | .forEach((prop) => (names["#p" + (Object.keys(names).length + countStart).toString()] = prop)); 272 | } 273 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { Pipeline } from "./Pipeline"; 2 | export { sortKey } from "./ScanQueryPipeline"; 3 | export { TableIterator } from "./TableIterator"; 4 | export * as helpers from "./helpers"; 5 | -------------------------------------------------------------------------------- /src/mocks/index.ts: -------------------------------------------------------------------------------- 1 | import DynamoDB, { DocumentClient } from "aws-sdk/clients/dynamodb"; 2 | import AWS from "aws-sdk"; 3 | import { Request } from "aws-sdk/lib/request"; 4 | import AWSMock from "aws-sdk-mock"; 5 | 6 | let mockOn = true; 7 | 8 | type Spy = jest.MockContext, [TInput, any?]>; 9 | type WrappedFn = (client: DocumentClient, spy: Spy) => Promise; 10 | type MockReturn = { err?: Error; data?: TOutput } | { err?: Error; data?: TOutput }[]; 11 | 12 | export function setMockOn(on: boolean): void { 13 | mockOn = on; 14 | } 15 | 16 | type DynamoClientCommandName = 17 | | "scan" 18 | | "query" 19 | | "delete" 20 | | "update" 21 | | "put" 22 | | "batchGet" 23 | | "batchWrite" 24 | | "transactGet"; 25 | 26 | interface MockSet> { 27 | name: DynamoClientCommandName; 28 | returns?: MockReturn; 29 | delay?: number; 30 | } 31 | 32 | export function multiMock( 33 | fn: (client: DocumentClient, spies: jest.MockContext[]) => Promise, 34 | mockSet: MockSet[] 35 | ): () => Promise { 36 | return async () => { 37 | const spies = mockSet.map((ms) => setupMock(ms.name, ms.returns, true, ms.delay).mock); 38 | 39 | const client = new DocumentClient(); 40 | 41 | await fn(client, spies); 42 | 43 | mockSet.forEach((ms) => teardownMock(ms.name, true)); 44 | }; 45 | } 46 | 47 | export function mockScan( 48 | fn: WrappedFn, 49 | returns?: MockReturn, 50 | delay?: number 51 | ): () => Promise { 52 | return mockCall("scan", fn, returns, false, delay); 53 | } 54 | 55 | export function alwaysMockScan( 56 | fn: WrappedFn, 57 | returns?: MockReturn, 58 | delay?: number 59 | ): () => Promise { 60 | return mockCall("scan", fn, returns, true, delay); 61 | } 62 | 63 | export function mockQuery( 64 | fn: WrappedFn, 65 | returns?: MockReturn, 66 | delay?: number 67 | ): () => Promise { 68 | return mockCall("query", fn, returns, false, delay); 69 | } 70 | 71 | export function alwaysMockQuery( 72 | fn: WrappedFn, 73 | returns?: MockReturn, 74 | delay?: number 75 | ): () => Promise { 76 | return mockCall("query", fn, returns, true, delay); 77 | } 78 | 79 | export function alwaysMockBatchGet( 80 | fn: WrappedFn, 81 | returns?: MockReturn, 82 | delay?: number 83 | ): () => Promise { 84 | return mockCall("batchGet", fn, returns, true, delay); 85 | } 86 | 87 | export function mockPut( 88 | fn: WrappedFn, 89 | returns?: MockReturn 90 | ): () => Promise { 91 | return mockCall("put", fn, returns); 92 | } 93 | 94 | export function mockUpdate( 95 | fn: WrappedFn, 96 | returns?: MockReturn 97 | ): () => Promise { 98 | return mockCall("update", fn, returns); 99 | } 100 | 101 | export function mockDelete( 102 | fn: WrappedFn, 103 | returns?: MockReturn 104 | ): () => Promise { 105 | return mockCall("delete", fn, returns); 106 | } 107 | 108 | export function alwaysMockBatchWrite( 109 | fn: WrappedFn, 110 | returns?: MockReturn 111 | ): () => Promise { 112 | return mockCall("batchWrite", fn, returns, true); 113 | } 114 | 115 | export function mockBatchWrite( 116 | fn: WrappedFn, 117 | returns?: MockReturn, 118 | delay?: number 119 | ): () => Promise { 120 | return mockCall("batchWrite", fn, returns, false, delay); 121 | } 122 | 123 | export function mockBatchGet( 124 | fn: WrappedFn, 125 | returns?: MockReturn, 126 | delay?: number 127 | ): () => Promise { 128 | return mockCall("batchGet", fn, returns, false, delay); 129 | } 130 | 131 | export function mockTransactGet( 132 | fn: WrappedFn, 133 | returns?: MockReturn, 134 | delay?: number 135 | ): () => Promise { 136 | return mockCall("transactGet", fn, returns, false, delay); 137 | } 138 | 139 | function mockCall( 140 | name: DynamoClientCommandName, 141 | fn: WrappedFn, 142 | returns: MockReturn = {}, 143 | alwaysMock = false, 144 | delay?: number 145 | ) { 146 | return async () => { 147 | const spy = setupMock(name, returns, alwaysMock, delay); 148 | 149 | // TODO: Type cleanup 150 | // eslint-disable-next-line 151 | const client = new DocumentClient(); 152 | 153 | if (!mockOn && !alwaysMock) { 154 | // TODO: Type cleanup 155 | await fn(client, jest.spyOn(client, name).mock as unknown as Spy); 156 | } else { 157 | await fn(client, spy.mock); 158 | } 159 | 160 | teardownMock(name, alwaysMock); 161 | }; 162 | } 163 | 164 | function setupMock( 165 | name: keyof DynamoDB.DocumentClient, 166 | returns: MockReturn = {}, 167 | alwaysMock: boolean, 168 | delay?: number 169 | ) { 170 | const spy = jest.fn, [TInput, any?]>(); 171 | let callCount = 0; 172 | if (mockOn || alwaysMock) { 173 | AWSMock.setSDKInstance(AWS); 174 | AWSMock.mock("DynamoDB.DocumentClient", name, function (input: TInput, callback: (err: any, args: any) => void) { 175 | spy(input); 176 | if (Array.isArray(returns)) { 177 | if (typeof delay === "number") { 178 | setTimeout(() => { 179 | callback(returns[callCount]?.err ?? undefined, returns[callCount]?.data); 180 | callCount += 1; 181 | }, delay); 182 | } else { 183 | callback(returns[callCount]?.err ?? undefined, returns[callCount]?.data); 184 | callCount += 1; 185 | } 186 | } else if (typeof delay === "number") { 187 | setTimeout(() => callback(returns?.err ?? undefined, returns?.data), delay); 188 | } else { 189 | callback(returns?.err ?? undefined, returns?.data); 190 | } 191 | }); 192 | } 193 | 194 | return spy; 195 | } 196 | 197 | function teardownMock(name: keyof DynamoDB.DocumentClient, alwaysMock?: boolean) { 198 | if (mockOn || alwaysMock) { 199 | AWSMock.restore("DynamoDB.DocumentClient", name); 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | export type Scalar = string | number; // binary is base64 encoded before being send 2 | export type ComparisonOperator = "=" | "<" | ">" | "<=" | ">=" | "<>"; 3 | export type Logical = "AND" | "OR" | "NOT"; 4 | export type QueryOperator = "begins_with" | "between"; 5 | export type ConditionOperator = ComparisonOperator | "contains" | "attribute_type"; 6 | export type ConditionFunction = "attribute_exists" | "attribute_not_exists"; 7 | 8 | export type SKQuery = 9 | | `${Exclude">} ${Scalar}` 10 | | `begins_with ${Scalar}` 11 | | `between ${Scalar} and ${Scalar}`; 12 | export type SKQueryParts = 13 | | [Exclude"> | "begins_with", Scalar] 14 | | ["between", Scalar, "and", Scalar]; 15 | 16 | export type QueryTemplate = 17 | | [Exclude"> | "begins_with", Scalar] 18 | | ["between", "and", Scalar, Scalar]; 19 | 20 | export type DynamoConditionAttributeName = `#p${number}`; 21 | export type DynamoConditionAttributeValue = `:v${number}`; 22 | 23 | export type DynamoCondition = { 24 | Condition: string; 25 | ExpressionAttributeNames?: Record; 26 | ExpressionAttributeValues?: Record; 27 | }; 28 | 29 | export type SimpleKey = { 30 | pk: string; 31 | }; 32 | 33 | export type CompoundKey = { 34 | pk: string; 35 | sk: string; 36 | }; 37 | 38 | export type KeyDefinition = SimpleKey | CompoundKey; 39 | 40 | export type IndexDefinition = 41 | | (SimpleKey & { 42 | name: string; 43 | }) 44 | | (CompoundKey & { name: string }); 45 | 46 | export type KeyType = string | number | Buffer | Uint8Array; 47 | export type KeyTypeName = "N" | "S" | "B"; 48 | 49 | export type Key = Keyset extends CompoundKey 50 | ? Record 51 | : Record; 52 | 53 | export type KeyConditions = Keyset extends CompoundKey 54 | ? Record & Partial> 55 | : Record; 56 | 57 | export type PrimitiveType = string | number | null | boolean | Buffer | Uint8Array; 58 | 59 | export type PrimitiveTypeName = KeyTypeName | "NULL" | "BOOL"; 60 | export type PropertyTypeName = PrimitiveTypeName | "M" | "L"; 61 | 62 | export type DynamoValue = { 63 | [_key in PropertyTypeName]?: string | boolean | Array | Record; 64 | }; 65 | 66 | export type DynamoPrimitiveValue = { 67 | [_key in PrimitiveTypeName]?: string | boolean | number; 68 | }; 69 | 70 | export type Operand = { property: string } | { value: PrimitiveType } | { property: string; function: "size" }; 71 | export type LHSOperand = string | { property: string; function: "size" }; 72 | 73 | type BeginsWith = { 74 | property: string; 75 | operator: "begins_with"; 76 | value: PrimitiveType; 77 | }; 78 | 79 | type Between = { 80 | property: string; 81 | start: PrimitiveType; 82 | end: PrimitiveType; 83 | operator: "between"; 84 | }; 85 | 86 | type In = { property: string; list: PrimitiveType[]; operator: "in" }; 87 | 88 | type BaseExpression = { lhs: LHSOperand; rhs: Operand; operator: T } | Between | BeginsWith; 89 | 90 | export type ConditionExpression = 91 | | BaseExpression 92 | | In 93 | | { operator: ConditionFunction; property: string } 94 | | { 95 | lhs?: ConditionExpression; 96 | logical: Logical; 97 | rhs: ConditionExpression; 98 | }; 99 | 100 | // removes the string option from type definition 101 | export type UpdateReturnValues = "ALL_OLD" | "UPDATED_OLD" | "ALL_NEW" | "UPDATED_NEW"; 102 | -------------------------------------------------------------------------------- /test/dynamodb.setup.ts: -------------------------------------------------------------------------------- 1 | import DynamoDB from "aws-sdk/clients/dynamodb"; 2 | 3 | export async function ensureDatabaseExists(tableName: string): Promise { 4 | const dynamodb = new DynamoDB(); 5 | const tables = await dynamodb.listTables().promise(); 6 | if (!tables || !tables.TableNames) { 7 | throw new Error("Could not list account tables\n\n" + (tables.$response.error as unknown as string)); 8 | } 9 | if (!tables.TableNames?.includes(tableName)) { 10 | await dynamodb 11 | .createTable({ 12 | AttributeDefinitions: [ 13 | { AttributeName: "id", AttributeType: "S" }, 14 | { AttributeName: "sk", AttributeType: "S" }, 15 | { AttributeName: "gsi1pk", AttributeType: "S" }, 16 | { AttributeName: "gsi1sk", AttributeType: "S" }, 17 | ], 18 | TableName: tableName, 19 | KeySchema: [ 20 | { AttributeName: "id", KeyType: "HASH" }, 21 | { AttributeName: "sk", KeyType: "RANGE" }, 22 | ], 23 | GlobalSecondaryIndexes: [ 24 | { 25 | IndexName: "gsi1", 26 | KeySchema: [ 27 | { AttributeName: "gsi1pk", KeyType: "HASH" }, 28 | { AttributeName: "gsi1sk", KeyType: "RANGE" }, 29 | ], 30 | Projection: { 31 | ProjectionType: "ALL", 32 | }, 33 | }, 34 | ], 35 | BillingMode: "PAY_PER_REQUEST", 36 | }) 37 | .promise(); 38 | 39 | let status = "CREATING"; 40 | while (status === "CREATING") { 41 | await new Promise((resolve) => setTimeout(resolve, 5000)); 42 | 43 | status = await dynamodb 44 | .describeTable({ TableName: tableName }) 45 | .promise() 46 | .then((result) => { 47 | const indexStatus = result.Table?.GlobalSecondaryIndexes?.pop()?.IndexStatus; 48 | if (indexStatus !== "ACTIVE") { 49 | return "CREATING"; 50 | } 51 | 52 | const tableStatus = result.Table?.TableStatus || "FAILURE"; 53 | return tableStatus; 54 | }); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /test/jest.setup.ts: -------------------------------------------------------------------------------- 1 | global.console = { 2 | ...global.console, 3 | // log: jest.fn(), // console.log are ignored in tests 4 | error: jest.fn(), 5 | warn: jest.fn(), 6 | info: console.info, 7 | debug: console.debug, 8 | }; 9 | -------------------------------------------------------------------------------- /tsconfig-cjs.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "module": "commonjs", 5 | "outDir": "lib", 6 | "tsBuildInfoFile": "./.buildcache/cjs.tsbuildinfo" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tsconfig.eslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["lib", "node_modules"], 4 | "include": ["src", "test", "mocks", "example", "jest.config.ts"], 5 | "compilerOptions": { 6 | "tsBuildInfoFile": "./.buildcache/eslint.tsbuildinfo" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "es2020", 4 | "target": "ES2019", 5 | "moduleResolution": "node", 6 | "declaration": true, 7 | "incremental": true, 8 | "tsBuildInfoFile": "./.buildcache/esm.tsbuildinfo", 9 | "types": ["jest", "node"], 10 | "esModuleInterop": true, 11 | "forceConsistentCasingInFileNames": true, 12 | "strict": true, 13 | "noImplicitAny": true, 14 | "noImplicitReturns": true, 15 | "noImplicitThis": true, 16 | "noUnusedLocals": false, 17 | "noUncheckedIndexedAccess": true, 18 | "alwaysStrict": true, 19 | "strictNullChecks": true, 20 | "outDir": "esm" 21 | }, 22 | "exclude": ["lib", "node_modules", "test"], 23 | "include": ["src", "mocks"] 24 | } 25 | --------------------------------------------------------------------------------