├── .gitignore ├── .npmignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── cdk.json ├── docs ├── ListService-getItemsFromDb.mmd ├── ListService-listIds.mmd ├── ListService-process.mmd ├── acmpca.hierarchy.png ├── architecture.drawio └── images │ ├── ListService-getItemsFromDb.svg │ ├── ListService-listIds.svg │ ├── ListService-process.svg │ └── architecture.png ├── jest.config.js ├── package-lock.json ├── package.json ├── src ├── infra │ ├── .DS_Store │ ├── amazon-dynamodb-item-tagging-stack.ts │ └── amazon-dynamodb-item-tagging.ts ├── lambda │ ├── create.handler.ts │ ├── create.spec.ts │ ├── create.ts │ ├── list.handler.ts │ ├── list.spec.ts │ ├── list.ts │ └── models.ts └── utils │ └── dynamoDb.util.ts └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | 6 | .cdk.staging 7 | cdk.out 8 | dist 9 | .history 10 | cdk-outputs.json 11 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional documentation, we greatly value feedback and contributions from our community. 4 | 5 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary information to effectively respond to your bug report or contribution. 6 | 7 | 8 | ## Reporting Bugs/Feature Requests 9 | 10 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 11 | 12 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 13 | 14 | * A reproducible test case or series of steps 15 | * The version of our code being used 16 | * Any modifications you've made relevant to the bug 17 | * Anything unusual about your environment or deployment 18 | 19 | 20 | ## Contributing via Pull Requests 21 | 22 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 23 | 24 | 1. You are working against the latest source on the *main* branch. 25 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 26 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 27 | 28 | To send us a pull request, please: 29 | 30 | 1. Fork the repository. 31 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 32 | 3. Ensure local tests pass. 33 | 4. Commit to your fork using clear commit messages. 34 | 5. Send us a pull request, answering any default questions in the pull request interface. 35 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 36 | 37 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 38 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 39 | 40 | 41 | ## Finding contributions to work on 42 | 43 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 44 | 45 | 46 | ## Code of Conduct 47 | 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact opensource-codeofconduct@amazon.com with any additional questions or comments. 49 | 50 | 51 | ## Security issue notifications 52 | 53 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 54 | 55 | 56 | ## Licensing 57 | 58 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 4 | software and associated documentation files (the "Software"), to deal in the Software 5 | without restriction, including without limitation the rights to use, copy, modify, 6 | merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 7 | permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 10 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 11 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 12 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 13 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 14 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon DynamoDB Item Tagging 2 | 3 | ## Summary 4 | 5 | Amazon DynamoDB is a fast and flexible NoSQL database service for single-digit millisecond performance at any scale. But in order to provide this scalability and performance, data access patterns must be known up front so that optimum keys and indexes can be designed. This is difficult in scenarios such as allowing the users of your platform to define any attributes for their data, then search that data filtering by any number of those attributes. This pattern outlines an approach to solve this problem by demonstrating how to structure a table and its indexes within DynamoDB to allow searching, then at the application layer how to efficiently aggregate results that match the multiple requests attributes. 6 | 7 | As an example, let's say we have a task management application which allows users to create tasks as follows: 8 | 9 | ```json 10 | { 11 | "id": "TASK_001", 12 | "name": "Read sample", 13 | "description": "Walk through the sample", 14 | "tags": { 15 | "project": "self improvement", 16 | "priority": "high", 17 | "severity": "low" 18 | } 19 | } 20 | ``` 21 | 22 | What is relevant here is the `tags` property. In our application we allow its users to specify their own tags against their tasks (`project`, `priority`, and `severity` in this case), as well as querying their tasks based on any number of tag attribute keys and values they provide. 23 | 24 | ## Prerequisites 25 | 26 | * An active AWS account 27 | * [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) installed, with credential configured using `aws configure` 28 | * Node.js v16. It is recommended to install and use [nvm](https://github.com/nvm-sh/nvm) to manage multiple versions of node.js 29 | * Install AWS CDK using `npm install -g aws-cdk` 30 | * Docker (required by AWS CDK) 31 | 32 | ## Architecture 33 | 34 | ![architecture](docs/images/architecture.png) 35 | 36 | The infrastructure that is deployed as part of this pattern is relatively simple: an *Amazon API Gateway* proxies a `POST /tasks` REST API to a *AWS Lambda* function to save a task to *Amazon DynamoDB*. Likewise, *Amazon API Gateway* proxies a `GET /tasks` REST API to another *AWS Lambda* function that handles the querying of data. The complexity involved with this pattern is in the implementation of the querying logic carried out as part of the *List Items AWS Lambda* function. 37 | 38 | ## Database Table Design 39 | 40 | * We take the approach of using a single Amazon DynamoDB table to store all data for the application. 41 | * To facilitate querying items by any user defined tags, we use the [Adjacency List design pattern](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/bp-adjacency-graphs.html#bp-adjacency-lists) to store both task and tags data as separate items in the same table. 42 | * [Composite sort keys](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/bp-sort-keys.html) are used to allow efficient querying of tag values. 43 | * A single [sparse index](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/bp-indexes-general-sparse-indexes.html) allows querying all task items when no filtering by tags has been requested. 44 | * Within the table we store 2 types of items: `task` and `tag`: 45 | 46 | **Task item** 47 | 48 | Attribute name | Attribute type | Example 49 | ---|---|--- 50 | `pk` (partition key) | String | `task#` e.g. `task#TASK_001` 51 | `sk` (sort key) | String | `task#` e.g. `task#TASK_001` 52 | `siKey1` | String | `task` 53 | `name` | String | `Read sample` 54 | `description` | String | `Walk through the sample` 55 | `done` | Boolean | `false` 56 | `tags` | Map | `{ "project": "self improvement", "priority": "high", "severity": "low" }` 57 | 58 | **Tag item** 59 | 60 | Attribute name | Attribute type | Example 61 | ---|---|--- 62 | `pk` (partition key) | String | `tag#` e.g. `tag#project` 63 | `sk` (sort key) | String | `#task#` e.g. `self improvement#task#001` 64 | 65 | **Global Secondary Index** 66 | 67 | A sparse GSI (named `siKey1-sk-index` exists with partition key `siKey1` and sort key `sk` and a projection type of `ALL` (refer to `src/infra/amazon-dynamodb-item-tagging-stack.ts` for further details). 68 | 69 | **Sample data** 70 | Taking the sample task item as listed in the summary, we store this as 4 separate items within the Amazon DynamoDB table as follows (refer to `src/lambda/create.ts` for further details on the implementation): 71 | 72 | pk (partition key) | sk (sort key) | siKey1 | name | description | done | tags 73 | ---|---|---|---|---|---|--- 74 | `task#TASK--1` | `task#TASK--1` | `task` | `Read sample` | `Walk through the sample` | `false` | `{ "project": "self improvement", "priority": "high", "severity": "low" }` 75 | `tag#project` | `self improvement#task#001` | 76 | `tag#priority` | `high#task#001` | 77 | `tag#severity` | `low#task#001` | 78 | 79 | ## Application implementation walkthrough 80 | 81 | The code files of interest are: 82 | 83 | ``` 84 | src/ // source code 85 | ├── infra/ // infrastructure as code (cdk) 86 | │ └── amazon-dynamodb-item-tagging-stack.ts // stack implementation 87 | │ └── amazon-dynamodb-item-tagging-.spec.ts // stack tests 88 | ├── lambda/ // lambda functions 89 | │ └── create.ts // create task code 90 | │ └── create.spec.ts // create tasks tests 91 | │ └── create.handler.ts // create task lambda handler 92 | │ └── list.ts // list task code 93 | │ └── list.spec.ts // list task tests 94 | │ └── list.handler.ts // list task lambda handler 95 | │ └── models.ts // shared models 96 | ├── utils/ // utils 97 | │ └── dynamodb.util.ts // dynamodb helper utils 98 | ``` 99 | 100 | #### Creating Tasks 101 | 102 | The code for creating tasks as described in the *Database table design* section is contained within the `CreateService process(item:TaskItem)` function located in `src/lambda/create.ts`. 103 | 104 | This class/method is wrapped by the lambda handler defined in `src/create.handler.ts` and is invoked by the *API Gateway* proxy to the *Lambda* function as defined in `src/infra/amazon-dynamodb-item-tagging-stack.ts`. The lambda handler takes the raw `APIGatewayEvent` object and invokes the `process` method with the extracted methods. 105 | 106 | The `CreateService` class is separated from the lambda handler to allow for unit testing (refer to `src/lambda/create.spec.ts`). 107 | 108 | 109 | #### Listing Tasks 110 | 111 | Similar to the create tasks logic, the listing of tasks is implemented in the `ListService process(tags?: Tags, paginationKey?: TaskItemListPaginationKey, count?: number)` function located in `src/lambda/list.ts`, wrapped by the lambda handler defined in `src/lambda/list.handler.ts`, and tested in `src/lambda/list.spec.ts`. 112 | 113 | Finding tasks that match all requested (user defined) tags is not (efficiently or cost effectively) possible in a single query using DynamoDB. Instead we need to query the table for all tasks per each filter, then attempt to find matching tasks across those different result sets at the application layer before returning the final result set. Along the way we may need to obtain the next page of results for any of the provided tags if the requested page size is greater than the number of tasks accumulated so far. The following sequence diagrams illustrate the process that allows this to be done in an efficient and scalable manner: 114 | 115 | ![ListService-process](docs/images/ListService-process.svg) 116 | 117 | ![ListService-listIds](docs/images/ListService-listIds.svg) 118 | 119 | ![ListService-getItemsFromDb](docs/images/ListService-getItemsFromDb.svg) 120 | 121 | 122 | ## Limitations 123 | 124 | The algorithm used to implement the application is optimized for scalability and performance. However, its effectiveness is still heavily dependent on the cardinality of data of those user defined tags. 125 | 126 | As an example, let's say we have the following to indicate a best case scenario: 127 | 128 | * 10,000,000 tasks 129 | * 50 tasks tagged with `project` of `self improvement` 130 | * 80 tasks tagged with `priority` of `high` 131 | * 20 tasks tagged with `severity` of `low` 132 | * 3 tasks that match all tags 133 | 134 | Best case is that the 3 tasks with all matching tags happen to be in the first page of results we return for each tag. This would entail 60 tag item reads (a page of 20 tag items per tag) followed by 3 task item reads. 135 | 136 | Worst case is that the 3 tasks with all matching tags happen to be in the last page of results we return for each tag. This would entail 150 tag item reads (all tag items returned) followed by 3 task item reads. 137 | 138 | As the next example, let's say we have the following to indicate a worst case scenario: 139 | 140 | * 10,000,000 tasks 141 | * 1,000,000 tasks tagged with `project` of `self improvement` 142 | * 9,000,000 tasks tagged with `priority` of `high` 143 | * 2,000,000 tasks tagged with `severity` of `low` 144 | * 3 tasks that match all tags 145 | 146 | Best case is that the 3 tasks with all matching tags happen to be in the first page of results we return for each tag. Like the last example, this would entail 60 tag item reads (a page of 20 tag items per tag) followed by 3 task item reads. 147 | 148 | Worst case is that the 3 tasks with all matching tags happen to be in the last page of results we return for each tag. This would entail 12,000,000 tag item reads (all tag items returned) followed by 3 task item reads. 149 | 150 | That last example would be a very expensive query, as well as likely to exceed the Lambda function execution timeout. To alleviate this, the concept of composite tags could be used to reduce the number of tag item reads. For example, we could have a user defined composite tag `project_priority_severity` in addition to the existing as follows: 151 | 152 | * 10,000,000 tasks 153 | * 1,000,000 tasks tagged with `project` of `self improvement` 154 | * 9,000,000 tasks tagged with `priority` of `high` 155 | * 2,000,000 tasks tagged with `severity` of `low` 156 | * 3 tasks tagged with `project_priority_severity` of `self improvement_high_low` 157 | 158 | Both best and worst case scenarios of instead searching just using the composite tag results in 3 tag item reads and 3 task item reads. 159 | 160 | ## Deployment Steps 161 | 162 | * Ensure all [prerequisites](#prerequisites) are met 163 | * Clone this repository, and `cd` into its directory 164 | * Build the application using `npm install && npm run build` 165 | * Deploy the application using `npx cdk deploy --outputs-file ./cdk-outputs.json` 166 | * Open `./cdk-outputs.json` and make a note of the API Gateway URL where the application's REST API is deployed 167 | * The following is an example of how to create new tasks 168 | 169 | ```http 170 | POST /tasks HTTP/1.1 171 | 172 | Request Headers: 173 | Accept: application/json 174 | Content-Type: application/json 175 | 176 | Request Body: 177 | { 178 | "name": "Read sample", 179 | "description": "Walk through the sample", 180 | "tags": { 181 | "project": "self improvement", 182 | "priority": "high", 183 | "severity": "low" 184 | } 185 | } 186 | 187 | Response Status: 188 | 201 189 | 190 | Response Body: 191 | { 192 | "id": "d72hsy2is", 193 | "name": "Read sample", 194 | "description": "Walk through the sample", 195 | "tags": { 196 | "project": "self improvement", 197 | "priority": "high", 198 | "severity": "low" 199 | } 200 | } 201 | ``` 202 | 203 | * The following is an example of how to query tasks 204 | 205 | ```http 206 | GET /tasks?tag=priority:high&tag=severity:low HTTP/1.1 207 | 208 | Request Headers: 209 | Accept: application/json 210 | Content-Type: application/json 211 | 212 | Response Status: 213 | 200 214 | 215 | Response Body: 216 | { 217 | "items": [ 218 | "id": "d72hsy2is", 219 | "name": "Read sample", 220 | "description": "Walk through the sample", 221 | "tags": { 222 | "project": "self improvement", 223 | "priority": "high", 224 | "severity": "low" 225 | } 226 | ] 227 | } 228 | ``` 229 | 230 | 231 | ## Useful commands 232 | 233 | * `npm install` install the node.js dependencies 234 | * `npm run build` compile typescript to js 235 | * `npm run lint` lint the code 236 | * `npm run test` perform the jest unit tests 237 | * `cdk synth` emits the synthesized CloudFormation template 238 | * `cdk diff` compare deployed stack with current state 239 | * `cdk deploy` deploy this stack to your default AWS account/region 240 | 241 | ## Security 242 | 243 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 244 | 245 | ## License 246 | 247 | This library is licensed under the MIT-0 License. See the LICENSE file. 248 | -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts src/infra/amazon-dynamodb-item-tagging.ts", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "**/*.d.ts", 11 | "**/*.js", 12 | "tsconfig.json", 13 | "package*.json", 14 | "yarn.lock", 15 | "node_modules", 16 | ".history" 17 | ] 18 | }, 19 | "context": { 20 | "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, 21 | "@aws-cdk/core:stackRelativeExports": true, 22 | "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, 23 | "@aws-cdk/aws-lambda:recognizeVersionProps": true, 24 | "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, 25 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 26 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 27 | "@aws-cdk/core:target-partitions": [ 28 | "aws", 29 | "aws-cn" 30 | ] 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /docs/ListService-getItemsFromDb.mmd: -------------------------------------------------------------------------------- 1 | sequenceDiagram 2 | 3 | title: ListService `getItemsFromDb()` Sequence Flow 4 | 5 | participant l1 as ListService 6 | participant l2 as ListService 7 | participant ddbu as DynamoDBUtils 8 | participant ddb as DynamoDB 9 | 10 | l1->>+l2: getItemsFromDb(taskIds) 11 | 12 | l2->>+ddbu: batchGetAll(params) 13 | 14 | note right of ddbu: DynamoDB batchGet call has max limit of 25 items per request, therefore chunk these 15 | ddbu->>+ddbu: splitBatchGetIntoChunks(params) 16 | ddbu-->>-ddbu: chunks 17 | 18 | note right of ddbu: now process each chunk, including retries on failed items... 19 | loop each chunk 20 | 21 | ddbu->>+ddb: results = ddb:batchGet(chunk) 22 | ddb-->>-ddbu: results 23 | 24 | ddbu->>+ddbu: mergeBatchGetOutput(overallResults, chunkResults) 25 | ddbu-->>-ddbu: overallResults 26 | 27 | opt has unprocessed keys? 28 | note right of ddbu: recursive retry until hit max retry limit 29 | ddbu->>+ddbu: retriedResults = batchGetAll(params) 30 | ddbu-->>-ddbu: retriedResults 31 | ddbu->>+ddbu: mergeBatchGetOutput(overallResults, retriedResults) 32 | ddbu-->>-ddbu: overallResults 33 | end 34 | end 35 | 36 | ddbu-->>-l2: overallResults 37 | 38 | l2-->>-l1: items 39 | -------------------------------------------------------------------------------- /docs/ListService-listIds.mmd: -------------------------------------------------------------------------------- 1 | sequenceDiagram 2 | 3 | title: ListService `listIds()` Sequence Flow 4 | 5 | participant l1 as ListService 6 | participant l2 as ListService 7 | participant ddb as DynamoDB 8 | 9 | l1->>+l2: listIds(tags,paginationKey,count) 10 | 11 | par async task per tag 12 | note right of l2: Retrieve the first page of task ids that match each tag 13 | loop each tag 14 | l2->>+l2: listIdsFromDbUsingTags<(tagKey,tagValue,tagPaginationKey,count) 15 | note right of l2: Refer to `listIdsFromDbUsingTags()` section below 16 | l2-->>-l2: task ids 17 | end 18 | end 19 | 20 | note right of l1: if any of the initial results are empty, then we can exit
immediately as there are no common matches across all requested tags 21 | loop each tag 22 | opt no task ids 23 | l2-->>l1: [undefined, undefined] 24 | end 25 | end 26 | 27 | note right of l2: initialize a set of pointers that tracks the current position of each tags page of results 28 | l2->>l2: initialize pointers 29 | note right of l2: loop through each page of results per tag looking for task ids that are found across 30 | loop more item ids still to process and found items < requested count 31 | loop each tag index 32 | note right of l2: retrieve the next task id for the current tag to process 33 | l2->>+l2: currentTagTaskItemId = getNextItemIdFromResults(tagIndex) 34 | note right of l2: Refer to `getNextPageOfResults(tagIndex)` section below 35 | l2-->>-l2: task id 36 | 37 | alt tag index === last tag index 38 | note right of l2: if we reach here it means we found a task id that was matched across all tags 39 | l2->>l2: add currentTagTaskItemId to results 40 | l2->>l2: increment all the pointers to reference the next result for each tag 41 | 42 | else tag index < last tag index 43 | note right of l2: check for matching task ids between this and the next tag to be compared 44 | l2->>+l2: nextTagTaskItemId = getNextItemIdFromResults(tagIndex) 45 | note right of l2: Refer to `getNextPageOfResults(tagIndex)` section below 46 | l2-->>-l2: task id 47 | 48 | alt currentTagTaskItemId === nextTagTaskItemId 49 | note right of l2: we have a match across the tag pair being checked, so lets move onto checking the next tag pair 50 | l2-->>l2: continue loop 51 | 52 | else currentTagTaskItemId < nextTagTaskItemId 53 | note right of l2: this tag has a lower task id, therefore increment the pointer for the current tag and restart the matching flow 54 | l2-->>l2: increment pointer of current tag 55 | l2-->>l2: break loop 56 | 57 | else currentTagTaskItemId > nextTagTaskItemId 58 | note right of l2: this tag has a higher task id, therefore increment the pointer for the next tag and restart the matching flow 59 | l2-->>l2: increment pointer of next tag 60 | l2-->>l2: break loop 61 | 62 | end 63 | 64 | end 65 | end 66 | end 67 | 68 | l2-->>-l1: [taskIds, paginationKey] 69 | 70 | note right of l1: ~~~
`getNextPageOfResults(tagIndex)` sequence flow
~~~ 71 | rect rgb(227, 211, 175) 72 | l1->>+l2: getNextItemIdFromResults(tagIndex) 73 | opt no more task ids in current page of results 74 | l2->>+l2: getNextPageOfResults(tagIndex) 75 | l2->>+l2: listIdsFromDbUsingTags(tagKey,tagValue,paginationKey,count) 76 | note right of l2: Refer to `listIdsFromDbUsingTags()` section below 77 | l2-->>-l2: task ids 78 | l2-->>-l2: has results? 79 | end 80 | l2-->>-l1: task id 81 | opt no next task id for current tag 82 | note right of l1: stop processing 83 | end 84 | end 85 | 86 | note right of l1: ~~~
`listIdsFromDbUsingTags(tagName,tagValue,exclusiveStart?,count?)` sequence flow
~~~ 87 | rect rgb(175, 227, 178) 88 | 89 | l1->>+l2: listIdsFromDbUsingTags
(tagName,tagValue,exclusiveStart?,count?) 90 | 91 | l2->>+ddb:query(params) 92 | ddb-->>-l2: items 93 | 94 | l2->>l2: extract task ids 95 | 96 | l2-->>-l1: [taskIds, paginationKey] 97 | end 98 | -------------------------------------------------------------------------------- /docs/ListService-process.mmd: -------------------------------------------------------------------------------- 1 | sequenceDiagram 2 | 3 | title: ListService `process()` Sequence Flow 4 | 5 | participant lh as Lambda Handler 6 | participant l as ListService 7 | participant ddbu as DynamoDBUtils 8 | participant ddb as DynamoDB 9 | 10 | lh->>+l: process
(tags?,paginationKey?,count?) 11 | 12 | alt tags provided 13 | 14 | l->>+l: listIds() 15 | note right of l: Refer to `listIds()` sequence diagram 16 | l-->>-l: [taskIds, paginationKey] 17 | 18 | opt taskids.length > 0 19 | l->>+l: getItemsFromDb() 20 | note right of l: Refer to `getItemsFromDb()` sequence diagram 21 | l-->>-l: tasks 22 | end 23 | 24 | else no tags provided 25 | l->>+l: listItemsFromDb() 26 | note right of l: Refer to `listItemsFromDb()` section within `listIds()` sequence diagram 27 | l-->>-l: [tasks, paginationKey] 28 | end 29 | 30 | l-->>-lh: [tasks, paginationKey] 31 | -------------------------------------------------------------------------------- /docs/acmpca.hierarchy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-dynamodb-item-tagging/3c381465988e00efc3eaf57c3936cf35f2a6ed9b/docs/acmpca.hierarchy.png -------------------------------------------------------------------------------- /docs/architecture.drawio: -------------------------------------------------------------------------------- 1 | 7Zpbb+I4FMc/DY9b5UrKIwl0phLdraZajfapMsQE75g4a5sC++n3OHGuTmm7hemgQouI/z6+n9/pqcXAjda7LxxlqzsWYzpwrHg3cCcDx7FtawgfStkXyrVjF0LCSayNauGB/Iu1aGl1Q2IsWoaSMSpJ1hYXLE3xQrY0xDnbts2WjLZHzVCCDeFhgaipfiexXOlV+Fatf8UkWclqwbpmjUpjLYgVitm2IbnTgRtxxmTxtN5FmKrNK/elaHfzTG01MY5T+ZoGQvye+eSviNj27J8/vLu7NBv+pk9HyH25YBzD+nWRcbliCUsRndZqyNkmjbHq1YJSbTNjLAPRBvFvLOVeHybaSAbSSq6pri3GVAM9uxQtCbbhC3xg/qVLIJ5gecDOqTYcPBWzNZZ8D+04pkiSp/Y8kHaZpLKrdxUe9Ma+YZN1v0+IbvRIfwrMjZ0XP7BcrMpt3UhKUhxVbq3EJUtlxCjjeQMXfm7UNMKEo5jgui5lqTqoJaG0az4BXUjOfuCOcYzEqjrUJ8wlAQhmaI7pPRNEEpZC3ZxJydYNgzEliaqQ6uhDpEsLmAsssHXoau7aI2ynLOvFqyGRyIqFLslOzSPMGFG9TJ+gM6E7AYYy1WC9S1S4uUJb4V1tRD6W3mSYGN4d9ivTD3QDT6OqY5VTsrytyR9qadWAvtSO7jjBmdPpvJJO95105k3HnKN9wyD3H9Ho+V4J9WG7Tue0u/HzBXs7sDoHXMygPu5qKf/fA64/iQd4v4QHeG/zAGf0EzxgdOYe4L7SA/yP8AD/MNMv2ndixmk8wDXSh4hjJDFotxKvB86QwsaGc8gohol6+orSmL6UYOitgQo/hF9YR1S8fTCNlHLl+D1inxaYom2awYfdN0JX7NMCU7RNM1UqZ90W+7TAN2fcbW33tLY7rf3Je/M1qLsJrqeW16ibEA4dFWlXqhA2E7qJ5Ud2YCR0ULPMX79iVtebwXFchJHbhZpPCMXiqW1F0Xoeo+Nkeu6oA773wale+f/u2cZ575zifOcv98v23k+I854R52dESB3lxSXMX8L8Jcy/M8xX8fujwrxvMD7Zp2jNJuEF4zPC2JsEUPk2jOHlhaNPg3Gc+3U8Pw7I/rAD8kdfzdnmpe74+wMIEWWb2IC5l1+D3S63BrNtXg0WuhwYDLRpNIDtUm2g344OBqBdig3UD/L3zC12w3+3KyLxQ4byRHQLvvYKnwbXlAjG4s1I2rhWLmwoRZkg86oV0LzhAvLRb1gUnVvP4ZBAFp7l0+/DIK99hMfHhXKMR5QncUYIqIJNM2joDTjMOcXLvEfYFZIms7w0cQ/Fs1aUOQKbtt9mMzDR9L0eNl33ZHA6BpzfcKLC54XLC5cNLnnhFT1A2l4wDccnBbIaogLSPhKQQRtI2/JNIq0eIp3APxGRJpDj+1sQviCJt2h/SX7PKPkdef7kxnlb8uuPXSv0P03yizLymGjXPgrSTvdC6nT5LxTrL6UUd1v1V3vc6X8= -------------------------------------------------------------------------------- /docs/images/ListService-getItemsFromDb.svg: -------------------------------------------------------------------------------- 1 | ListService `getItemsFromDb()` Sequence FlowListServiceListServiceDynamoDBUtilsDynamoDBgetItemsFromDb(taskIds)batchGetAll(params)DynamoDB batchGet call has max limit of 25 items per request, therefore chunk thesesplitBatchGetIntoChunks(params)chunksnow process each chunk, including retries on failed items...results = ddb:batchGet(chunk)resultsmergeBatchGetOutput(overallResults, chunkResults)overallResultsrecursive retry until hit max retry limitretriedResults = batchGetAll(params)retriedResultsmergeBatchGetOutput(overallResults, retriedResults)overallResultsopt[has unprocessed keys?]loop[each chunk]overallResultsitemsListServiceListServiceDynamoDBUtilsDynamoDBListService `getItemsFromDb()` Sequence Flow -------------------------------------------------------------------------------- /docs/images/ListService-listIds.svg: -------------------------------------------------------------------------------- 1 | ListService `listIds()` Sequence FlowListServiceListServiceDynamoDBlistIds(tags,paginationKey,count)Retrieve the first page of task ids that match each taglistIdsFromDbUsingTags<(tagKey,tagValue,tagPaginationKey,count)Refer to `listIdsFromDbUsingTags()` section belowtask idsloop[each tag]par[async task per tag]if any of the initial results are empty, then we can exitimmediately as there are no common matches across all requested tags[undefined, undefined]opt[no task ids]loop[each tag]initialize a set of pointers that tracks the current position of each tags page of resultsinitialize pointersloop through each page of results per tag looking for task ids that are found acrossretrieve the next task id for the current tag to processcurrentTagTaskItemId = getNextItemIdFromResults(tagIndex)Refer to `getNextPageOfResults(tagIndex)` section belowtask idif we reach here it means we found a task id that was matched across all tagsadd currentTagTaskItemId to resultsincrement all the pointers to reference the next result for each tagcheck for matching task ids between this and the next tag to be comparednextTagTaskItemId = getNextItemIdFromResults(tagIndex)Refer to `getNextPageOfResults(tagIndex)` section belowtask idwe have a match across the tag pair being checked, so lets move onto checking the next tag paircontinue loopthis tag has a lower task id, therefore increment the pointer for the current tag and restart the matching flowincrement pointer of current tagbreak loopthis tag has a higher task id, therefore increment the pointer for the next tag and restart the matching flowincrement pointer of next tagbreak loopalt[currentTagTaskItemId === nextTagTaskItemId][currentTagTaskItemId < nextTagTaskItemId][currentTagTaskItemId > nextTagTaskItemId]alt[tag index === last tag index][tag index < last tag index]loop[each tag index]loop[more item ids still to process and found items < requested count][taskIds, paginationKey]~~~`getNextPageOfResults(tagIndex)` sequence flow~~~getNextItemIdFromResults(tagIndex)getNextPageOfResults(tagIndex)listIdsFromDbUsingTags(tagKey,tagValue,paginationKey,count)Refer to `listIdsFromDbUsingTags()` section belowtask idshas results?opt[no more task ids in current page of results]task idstop processingopt[no nexttask id forcurrenttag]~~~`listIdsFromDbUsingTags(tagName,tagValue,exclusiveStart?,count?)` sequence flow~~~listIdsFromDbUsingTags(tagName,tagValue,exclusiveStart?,count?)query(params)itemsextract task ids[taskIds, paginationKey]ListServiceListServiceDynamoDBListService `listIds()` Sequence Flow -------------------------------------------------------------------------------- /docs/images/ListService-process.svg: -------------------------------------------------------------------------------- 1 | ListService `process()` Sequence FlowLambda HandlerListServiceDynamoDBUtilsDynamoDBprocess(tags?,paginationKey?,count?)listIds()Refer to `listIds()` sequence diagram[taskIds, paginationKey]getItemsFromDb()Refer to `getItemsFromDb()` sequence diagramtasksopt[taskids.length > 0]listItemsFromDb()Refer to `listItemsFromDb()` section within `listIds()` sequence diagram[tasks, paginationKey]alt[tags provided][no tags provided][tasks, paginationKey]Lambda HandlerListServiceDynamoDBUtilsDynamoDBListService `process()` Sequence Flow -------------------------------------------------------------------------------- /docs/images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-dynamodb-item-tagging/3c381465988e00efc3eaf57c3936cf35f2a6ed9b/docs/images/architecture.png -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /*! 2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | // SPDX-License-Identifier: MIT-0 4 | */ 5 | 6 | module.exports = { 7 | testEnvironment: 'node', 8 | roots: ['/src'], 9 | "testMatch": [ 10 | "/**/?(*.)+(spec|test).ts?(x)" 11 | ], 12 | "transform": { 13 | "^.+\\.tsx?$": "ts-jest" 14 | }, 15 | "moduleFileExtensions": [ 16 | "ts", 17 | "tsx", 18 | "js", 19 | "jsx", 20 | "json", 21 | "node" 22 | ] 23 | }; 24 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "amazon-dynamodb-item-tagging", 3 | "version": "0.1.0", 4 | "license": "MIT-0", 5 | "bin": { 6 | "amazon-dynamodb-item-tagging": "dist/infra/amazon-dynamodb-item-tagging.js" 7 | }, 8 | "scripts": { 9 | "lint": "eslint . --ext '.ts'", 10 | "build": "tsc -b", 11 | "test": "jest", 12 | "cdk": "cdk" 13 | }, 14 | "dependencies": { 15 | "aws-cdk-lib": "2.80.0", 16 | "aws-lambda": "1.0.7", 17 | "constructs": "10.2.12", 18 | "ow": "^0.28.1", 19 | "short-unique-id": "4.4.4", 20 | "source-map-support": "0.5.21" 21 | }, 22 | "devDependencies": { 23 | "@types/aws-lambda": "8.10.114", 24 | "@types/jest": "29.5.1", 25 | "@types/node": "16.18.25", 26 | "@typescript-eslint/eslint-plugin": "5.59.1", 27 | "@typescript-eslint/parser": "5.59.1", 28 | "aws-cdk": "2.77.0", 29 | "cdk-nag": "2.26.11", 30 | "eslint": "8.39.0", 31 | "jest": "29.5.0", 32 | "jest-create-mock-instance": "^2.0.0", 33 | "jest-haste-map": "29.5.0", 34 | "jest-mock": "29.5.0", 35 | "jest-mock-extended": "3.0.4", 36 | "jest-resolve": "29.5.0", 37 | "ts-jest": "29.1.0", 38 | "ts-node": "10.9.1", 39 | "typescript": "4.9.5" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/infra/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-dynamodb-item-tagging/3c381465988e00efc3eaf57c3936cf35f2a6ed9b/src/infra/.DS_Store -------------------------------------------------------------------------------- /src/infra/amazon-dynamodb-item-tagging-stack.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | // SPDX-License-Identifier: MIT-0 4 | */ 5 | 6 | import { Duration, RemovalPolicy, Stack, StackProps } from 'aws-cdk-lib'; 7 | import { AccessLogFormat, Cors, EndpointType, LambdaIntegration, LogGroupLogDestination, MethodLoggingLevel, RestApi } from 'aws-cdk-lib/aws-apigateway'; 8 | import { 9 | AttributeType, BillingMode, ProjectionType, Table, TableEncryption 10 | } from 'aws-cdk-lib/aws-dynamodb'; 11 | import { Runtime } from 'aws-cdk-lib/aws-lambda'; 12 | import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs'; 13 | import { LogGroup } from 'aws-cdk-lib/aws-logs'; 14 | import { Construct } from 'constructs'; 15 | import path from 'path'; 16 | 17 | export class DynamodbItemTaggingStack extends Stack { 18 | constructor(scope: Construct, id: string, props?: StackProps) { 19 | super(scope, id, props); 20 | 21 | // create the DynamoDB table 22 | const tableName = 'tasks'; 23 | const table = new Table(this, tableName, { 24 | tableName, 25 | partitionKey: { 26 | name: 'pk', 27 | type: AttributeType.STRING, 28 | }, 29 | sortKey: { 30 | name: 'sk', 31 | type: AttributeType.STRING, 32 | }, 33 | billingMode: BillingMode.PAY_PER_REQUEST, 34 | encryption: TableEncryption.AWS_MANAGED, 35 | removalPolicy: RemovalPolicy.DESTROY, 36 | pointInTimeRecovery: true, 37 | }); 38 | 39 | // add a global secondary index to allow listing all task items 40 | table.addGlobalSecondaryIndex({ 41 | indexName: 'siKey1-sk-index', 42 | partitionKey: { 43 | name: 'siKey1', 44 | type: AttributeType.STRING, 45 | }, 46 | sortKey: { 47 | name: 'sk', 48 | type: AttributeType.STRING, 49 | }, 50 | projectionType: ProjectionType.ALL, 51 | }); 52 | 53 | // define the lambda function to create new tasks 54 | const createTaskLambda = new NodejsFunction(this, 'createTask', { 55 | memorySize: 256, 56 | timeout: Duration.seconds(5), 57 | runtime: Runtime.NODEJS_16_X, 58 | handler: 'handler', 59 | entry: path.join(__dirname, `/../lambda/create.handler.ts`), 60 | environment: { 61 | TABLE_NAME: tableName 62 | }, 63 | bundling: { 64 | minify: true, 65 | externalModules: ['aws-sdk'], 66 | } 67 | }); 68 | 69 | // define the lambda function to list existing tasks 70 | const listTasksLambda = new NodejsFunction(this, 'listTasks', { 71 | memorySize: 256, 72 | timeout: Duration.seconds(29), 73 | runtime: Runtime.NODEJS_16_X, 74 | handler: 'handler', 75 | entry: path.join(__dirname, `/../lambda/list.handler.ts`), 76 | environment: { 77 | TABLE_NAME: tableName 78 | }, 79 | bundling: { 80 | minify: true, 81 | externalModules: ['aws-sdk'], 82 | }, 83 | }); 84 | 85 | // grant the lambda functions access to the table 86 | table.grantWriteData(createTaskLambda); 87 | table.grantReadData(listTasksLambda); 88 | // define the API Gateway 89 | const logGroup = new LogGroup(this, 'DynamoDBItemTaggingLogs'); 90 | const api = new RestApi(this, 'amazon-dynamodb-item-tagging-api', { 91 | description: 'Amazon DynamoDB Item Tagging API', 92 | deployOptions: { 93 | stageName: 'prod', 94 | accessLogDestination: new LogGroupLogDestination(logGroup), 95 | accessLogFormat: AccessLogFormat.jsonWithStandardFields(), 96 | loggingLevel: MethodLoggingLevel.INFO, 97 | }, 98 | defaultCorsPreflightOptions: { 99 | allowOrigins: Cors.ALL_ORIGINS, 100 | }, 101 | endpointTypes: [EndpointType.REGIONAL] 102 | }); 103 | 104 | // define the rest api endpoints to proxy through to the lambda functions 105 | const tasksResource = api.root.addResource('tasks'); 106 | tasksResource.addMethod('POST', new LambdaIntegration(createTaskLambda, {proxy: true}), {apiKeyRequired: true}); 107 | tasksResource.addMethod('GET', new LambdaIntegration(listTasksLambda, {proxy: true}), {apiKeyRequired: true}); 108 | 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/infra/amazon-dynamodb-item-tagging.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /*! 4 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 5 | // SPDX-License-Identifier: MIT-0 6 | */ 7 | 8 | import 'source-map-support/register'; 9 | import * as cdk from 'aws-cdk-lib'; 10 | import { DynamodbItemTaggingStack } from './amazon-dynamodb-item-tagging-stack'; 11 | import { Aspects } from 'aws-cdk-lib'; 12 | import { AwsSolutionsChecks, NagSuppressions } from 'cdk-nag'; 13 | 14 | const app = new cdk.App(); 15 | const stack = new DynamodbItemTaggingStack(app, 'DynamodbItemTaggingStack', {}); 16 | 17 | 18 | NagSuppressions.addResourceSuppressions(stack, [ 19 | { 20 | id: 'AwsSolutions-IAM4', 21 | reason: 'CDK autogenerated - AWSLambdaBasicExecutionRole is valid in this case', 22 | }, 23 | { 24 | id: 'AwsSolutions-IAM5', 25 | reason: 'CDK autogenerated - Valid to read all indices', 26 | }, 27 | { 28 | id: 'AwsSolutions-APIG1', 29 | reason: 'Access logging not required for this simple walk through', 30 | }, 31 | { 32 | id: 'AwsSolutions-APIG2', 33 | reason: 'Lambdas are running in proxy mode, performing their own request validation', 34 | }, 35 | { 36 | id: 'AwsSolutions-APIG3', 37 | reason: 'WAF not enabled for this simple walk through', 38 | }, 39 | { 40 | id: 'AwsSolutions-APIG4', 41 | reason: 'API Gateway authorization not enabled for this simple walk through', 42 | }, 43 | { 44 | id: 'AwsSolutions-COG4', 45 | reason: 'API Gateway authorization not enabled for this simple walk through', 46 | }, 47 | ], true 48 | ); 49 | 50 | Aspects.of(app).add(new AwsSolutionsChecks()); 51 | -------------------------------------------------------------------------------- /src/lambda/create.handler.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | // SPDX-License-Identifier: MIT-0 4 | */ 5 | 6 | import { APIGatewayEvent, APIGatewayProxyResult, Context } from 'aws-lambda'; 7 | import { DocumentClient } from 'aws-sdk/clients/dynamodb'; 8 | 9 | import { CreateService } from './create'; 10 | import ow from 'ow'; 11 | import { TaskItem } from './models'; 12 | 13 | const dc = new DocumentClient(); 14 | const tableName = process.env.TABLE_NAME as string; 15 | const service = new CreateService(dc, tableName); 16 | 17 | /** 18 | * The lambda handler function to create new task items 19 | */ 20 | exports.handler = async (event: APIGatewayEvent, _context: Context): Promise => { 21 | 22 | try { 23 | // Validate the incoming item. At a minimum, it must have a name 24 | ow(event.body, 'request body', ow.string.nonEmpty); 25 | const item: TaskItem = JSON.parse(event.body); 26 | const saved = await service.process(item); 27 | 28 | // return the saved item 29 | return { 30 | statusCode: 201, 31 | body: JSON.stringify(saved), 32 | }; 33 | } catch (e) { 34 | return handleError(e); 35 | } 36 | } 37 | 38 | /** 39 | * Converts an error to an API Gateway friendly response 40 | * @param e 41 | * @returns 42 | */ 43 | function handleError(e: Error): APIGatewayProxyResult { 44 | if (e.name === 'ArgumentError') { 45 | return { 46 | statusCode: 400, 47 | body: JSON.stringify({ message: e.message }), 48 | }; 49 | } else { 50 | return { 51 | statusCode: 500, 52 | body: JSON.stringify({ message: e.message || e.name }), 53 | }; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/lambda/create.spec.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | // SPDX-License-Identifier: MIT-0 4 | */ 5 | 6 | import { DocumentClient } from 'aws-sdk/clients/dynamodb'; 7 | import { CreateService } from './create'; 8 | import { TaskItem } from './models'; 9 | 10 | 11 | describe('CreateService', () => { 12 | let mockedDocumentClient: DocumentClient; 13 | let underTest: CreateService; 14 | 15 | beforeEach(() => { 16 | mockedDocumentClient = new DocumentClient(); 17 | underTest = new CreateService(mockedDocumentClient, 'myTable'); 18 | }); 19 | 20 | it('creating new task item - happy path', async () => { 21 | 22 | // input data 23 | const toSave: TaskItem = { 24 | name: 'my test', 25 | description: 'my test description', 26 | tags: { 27 | 'tag1': 'value1', 28 | 'tag2': 'value2' 29 | } 30 | }; 31 | 32 | // mocks 33 | const mockedSave = mockedDocumentClient.batchWrite = jest.fn() 34 | .mockImplementationOnce(()=> { 35 | return { 36 | promise: () => { 37 | return { 38 | UnprocessedItems: {} 39 | }; 40 | } 41 | }; 42 | }); 43 | 44 | // test 45 | const saved = await underTest.process(toSave); 46 | 47 | // assertions 48 | expect(saved.id).toBeDefined(); 49 | expect(saved.name).toEqual(toSave.name); 50 | expect(saved.description).toEqual(toSave.description); 51 | expect(saved.tags).toEqual(toSave.tags); 52 | 53 | const expectedBatchWriteItemInput: DocumentClient.BatchWriteItemInput = { 54 | RequestItems: { 55 | myTable: [ 56 | { 57 | PutRequest: { 58 | Item: { 59 | pk: `task#${saved.id}`, 60 | sk: `task#${saved.id}`, 61 | siKey1: 'task', 62 | name: saved.name, 63 | description: saved.description, 64 | tags: saved.tags 65 | } 66 | } 67 | }, { 68 | PutRequest: { 69 | Item: { 70 | pk: 'tag#tag1', 71 | sk: `value1#task#${saved.id}`, 72 | } 73 | } 74 | }, { 75 | PutRequest: { 76 | Item: { 77 | pk: 'tag#tag2', 78 | sk: `value2#task#${saved.id}`, 79 | } 80 | } 81 | } 82 | ] 83 | } 84 | }; 85 | expect(mockedSave).toHaveBeenCalledWith(expectedBatchWriteItemInput); 86 | 87 | }); 88 | }); 89 | 90 | -------------------------------------------------------------------------------- /src/lambda/create.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | // SPDX-License-Identifier: MIT-0 4 | */ 5 | 6 | import { DocumentClient } from 'aws-sdk/clients/dynamodb'; 7 | import ow from 'ow'; 8 | import ShortUniqueId from 'short-unique-id'; 9 | 10 | import { DynamoDbUtils } from '../utils/dynamoDb.util'; 11 | import { TaskItem } from './models'; 12 | 13 | 14 | export class CreateService { 15 | 16 | private readonly dynamoDbUtils: DynamoDbUtils; 17 | private readonly uidGenerator: ShortUniqueId; 18 | 19 | public constructor( 20 | dc: DocumentClient, 21 | private tableName: string 22 | ) { 23 | this.dynamoDbUtils = new DynamoDbUtils(dc); 24 | this.uidGenerator = new ShortUniqueId({ 25 | dictionary: 'alphanum_lower', 26 | length: 9, 27 | }); 28 | } 29 | 30 | public async process(item: TaskItem): Promise { 31 | 32 | // Validate the incoming item. At a minimum, it must have a name 33 | ow(item?.name, 'name', ow.string.nonEmpty); 34 | 35 | // set attributes we need to before saving 36 | item.id = this.uidGenerator(); 37 | 38 | // Save it to the database. 39 | await this.saveToDb(item); 40 | 41 | // return the saved item 42 | return item; 43 | } 44 | 45 | /** 46 | * Saves a new TaskItem to the database 47 | * @param item 48 | */ 49 | private async saveToDb(item: TaskItem): Promise { 50 | 51 | // as we are writing potentially multiple items (the task item, along with its tags), we need to use BatchWriteItem 52 | const params: DocumentClient.BatchWriteItemInput = { 53 | RequestItems: { 54 | } 55 | }; 56 | params.RequestItems[this.tableName] = []; 57 | 58 | // first lets write the TaskItem as its own DynamoDB item. We include the tags to simplify retrieval later 59 | const taskDbItem: DocumentClient.WriteRequest = { 60 | PutRequest: { 61 | Item: { 62 | // we set the pk and sk to the item id. we prefix both with `task#` to allow filtering by task items 63 | pk: `task#${item.id}`, 64 | sk: `task#${item.id}`, 65 | // we are using a gsi to allow listing all items of a certain type, which in this case is task items 66 | // task: GSI key sharding 67 | siKey1: 'task', 68 | name: item.name, 69 | description: item.description, 70 | // tags are duplicated here to simplify retrieval 71 | tags: item.tags 72 | } 73 | } 74 | }; 75 | params.RequestItems[this.tableName].push(taskDbItem); 76 | 77 | // next we write all the tags as separate DynamoDB items. We use the tag name as the partition key, and the tag value and the TaskItem id as a composite sort key. 78 | if (item.tags) { 79 | Object.entries(item.tags).forEach(([tagName, tagValue]) => { 80 | const tagDbItem: DocumentClient.WriteRequest = { 81 | PutRequest: { 82 | Item: { 83 | pk: `tag#${tagName}`, 84 | sk: `${tagValue}#task#${item.id}`, 85 | } 86 | } 87 | }; 88 | params.RequestItems[this.tableName].push(tagDbItem); 89 | }); 90 | } 91 | 92 | const r = await this.dynamoDbUtils.batchWriteAll(params); 93 | if (this.dynamoDbUtils.hasUnprocessedItems(r)) { 94 | throw new Error('SAVE_FAILED'); 95 | } 96 | 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/lambda/list.handler.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | // SPDX-License-Identifier: MIT-0 4 | */ 5 | import { APIGatewayEvent, APIGatewayProxyResult, Context } from 'aws-lambda'; 6 | import { DocumentClient } from 'aws-sdk/clients/dynamodb'; 7 | 8 | import { ListService } from './list'; 9 | import { Tags, TaskItemListPaginationKey, TaskListItem } from './models'; 10 | 11 | const dc = new DocumentClient(); 12 | const tableName = process.env.TABLE_NAME as string; 13 | const service = new ListService(dc, tableName); 14 | 15 | /** 16 | * The lambda handler function to list existing task items, optionally filtering by tags 17 | */ 18 | exports.handler = async (event: APIGatewayEvent, _context: Context): Promise => { 19 | 20 | try { 21 | 22 | // retrieve the provided query string values and convert them to something we can use more easily 23 | const tagsQS = event.multiValueQueryStringParameters?.tag as string | string[]; 24 | const tags = convertTagsQS(tagsQS); 25 | const paginationKeyQS = event.queryStringParameters?.paginationKey; 26 | const paginationKey: TaskItemListPaginationKey = (paginationKeyQS) ? { id: paginationKeyQS } : undefined; 27 | const countQS = event.queryStringParameters?.count as string; 28 | const count = (countQS) ? parseInt(countQS) : undefined; 29 | 30 | // perform the search 31 | const results = await service.process(tags, paginationKey, count); 32 | 33 | // assemble and return the response 34 | const response: TaskListItem = { 35 | items: results[0] 36 | }; 37 | 38 | if (paginationKey?.id || count) { 39 | response['pagination'] = {}; 40 | } 41 | if (paginationKey?.id) { 42 | response['pagination']['nextToken'] = paginationKey?.id; 43 | } 44 | if (count) { 45 | response['pagination']['count'] = count; 46 | } 47 | 48 | return { 49 | statusCode: 200, 50 | body: JSON.stringify(response), 51 | }; 52 | 53 | } catch (e) { 54 | return handleError(e); 55 | } 56 | } 57 | 58 | /** 59 | * Converts the tag multi-valued query string parameter into a map of tag names to tag values 60 | * @param tagsQS 61 | * @returns 62 | */ 63 | function convertTagsQS(tagsQS: string | string[]): Tags { 64 | const tags: Tags = {}; 65 | if (typeof tagsQS === 'string') { 66 | tagsQS = [tagsQS]; 67 | } 68 | if ((tagsQS?.length ?? 0) > 0) { 69 | tagsQS.forEach(t => { 70 | const [key, value] = t.split(':'); 71 | tags[decodeURIComponent(key)] = decodeURIComponent(value); 72 | }); 73 | } 74 | return tags; 75 | } 76 | 77 | function handleError(e: Error): APIGatewayProxyResult { 78 | if (e.name === 'ArgumentError') { 79 | return { 80 | statusCode: 400, 81 | body: JSON.stringify({ message: e.message }), 82 | }; 83 | } else { 84 | return { 85 | statusCode: 500, 86 | body: JSON.stringify({ message: e.message || e.name }), 87 | }; 88 | } 89 | } -------------------------------------------------------------------------------- /src/lambda/list.spec.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | // SPDX-License-Identifier: MIT-0 4 | */ 5 | 6 | import { DocumentClient } from 'aws-sdk/clients/dynamodb'; 7 | import { ListService } from './list'; 8 | import { Tags, TaskItemListPaginationKey } from './models'; 9 | 10 | 11 | describe('ListService', () => { 12 | 13 | let mockedDocumentClient: DocumentClient; 14 | let underTest: ListService; 15 | 16 | beforeEach(() => { 17 | mockedDocumentClient = new DocumentClient(); 18 | underTest = new ListService(mockedDocumentClient, 'myTable'); 19 | }); 20 | 21 | it('listing items with multiple tags - happy path', async () => { 22 | 23 | /* 24 | input data 25 | */ 26 | const tags: Tags = { 27 | 'tag1': 'value1', 28 | 'tag2': 'value2', 29 | 'tag3': 'value3' 30 | }; 31 | 32 | /* 33 | mocks 34 | */ 35 | const mockedQuery = mockedDocumentClient.query = jest.fn() 36 | // mock 1st page of results for tag1 37 | .mockImplementationOnce(() => mockQueryOutput( 38 | ['value1#task#001', 'value1#task#003', 'value1#task#005', 'value1#task#007', 'value1#task#009', 'value1#task#011'].map(v => ({ sk: v })), 39 | { pk: 'tag#tag1', sk: 'value1#task#011' }) 40 | ) 41 | // mock 1st page of results for tag2 42 | .mockImplementationOnce(() => mockQueryOutput( 43 | ['value2#task#001', 'value2#task#005', 'value2#task#009', 'value2#task#013'].map(v => ({ sk: v }))) 44 | ) 45 | // mock 1st page of results for tag3 46 | .mockImplementationOnce(() => mockQueryOutput( 47 | ['value3#task#001', 'value3#task#009', 'value3#task#017', 'value3#task#025'].map(v => ({ sk: v }))) 48 | ) 49 | // mock 2nd page of results for tag1 50 | .mockImplementationOnce(() => mockQueryOutput( 51 | ['value1#task#013', 'value1#task#015', 'value1#task#017', 'value1#task#019', 'value1#task#021', 'value1#task#023'].map(v => ({ sk: v }))) 52 | ); 53 | 54 | 55 | // mock retrieving the items where the tags match 56 | const mockedBatchGet = mockedDocumentClient.batchGet = jest.fn() 57 | .mockImplementationOnce(() => mockedBatchGetItemOutput('myTable', [ 58 | { 59 | pk: 'task#001', 60 | sk: 'task#001', 61 | name: 'item1', 62 | description: 'item1 description', 63 | tags: { 64 | 'tag1': 'value1', 65 | 'tag2': 'value2', 66 | 'tag3': 'value3' 67 | } 68 | }, 69 | { 70 | pk: 'task#009', 71 | sk: 'task#009', 72 | name: 'item9', 73 | description: 'item9 description', 74 | tags: { 75 | 'tag1': 'value1', 76 | 'tag2': 'value2', 77 | 'tag3': 'value3' 78 | } 79 | } 80 | ]) 81 | ); 82 | 83 | /* 84 | test 85 | */ 86 | const results = await underTest.process(tags); 87 | 88 | /* 89 | assertions 90 | */ 91 | // correct number of results returned 92 | expect(results?.[0]?.length).toBe(2); 93 | 94 | // 1st result is as expected 95 | expect(results?.[0]?.[0]).toStrictEqual({ 96 | id: '001', 97 | name: 'item1', 98 | description: 'item1 description', 99 | tags: { 100 | 'tag1': 'value1', 101 | 'tag2': 'value2', 102 | 'tag3': 'value3' 103 | } 104 | }); 105 | 106 | // 2nd result is as expected 107 | expect(results?.[0]?.[1]).toStrictEqual({ 108 | id: '009', 109 | name: 'item9', 110 | description: 'item9 description', 111 | tags: { 112 | 'tag1': 'value1', 113 | 'tag2': 'value2', 114 | 'tag3': 'value3' 115 | } 116 | }); 117 | 118 | // should be no pagination returned 119 | expect(results[1]).toBeUndefined(); 120 | 121 | // 1st database call should be `query` for page 1 of tag 1 items 122 | const expectedQueryInput1: DocumentClient.QueryInput = { 123 | TableName: 'myTable', 124 | KeyConditionExpression: `#hash = :hash AND begins_with(#sort,:sort)`, 125 | ExpressionAttributeNames: { 126 | '#hash': 'pk', 127 | '#sort': 'sk', 128 | }, 129 | ExpressionAttributeValues: { 130 | ':hash': 'tag#tag1', 131 | ':sort': 'value1#task#', 132 | }, 133 | ExclusiveStartKey: undefined, 134 | Limit: 20 135 | }; 136 | expect(mockedQuery.mock.calls[0][0]).toStrictEqual(expectedQueryInput1); 137 | 138 | // 2nd database call should be `query` for page 1 of tag 2 items 139 | const expectedQueryInput2: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 140 | expectedQueryInput2.ExpressionAttributeValues = { 141 | ':hash': 'tag#tag2', 142 | ':sort': 'value2#task#', 143 | }; 144 | expect(mockedQuery.mock.calls[1][0]).toStrictEqual(expectedQueryInput2); 145 | 146 | // 3rd database call should be `query` for page 1 of tag 3 items 147 | const expectedQueryInput3: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 148 | expectedQueryInput3.ExpressionAttributeValues = { 149 | ':hash': 'tag#tag3', 150 | ':sort': 'value3#task#', 151 | }; 152 | expect(mockedQuery.mock.calls[2][0]).toStrictEqual(expectedQueryInput3); 153 | 154 | // 4th database call should be `query` for page 2 of tag 1 items 155 | const expectedQueryInput4: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 156 | expectedQueryInput4.ExclusiveStartKey = { 157 | pk: 'tag#tag1', 158 | sk: 'value1#task#011' 159 | }; 160 | expect(mockedQuery.mock.calls[3][0]).toStrictEqual(expectedQueryInput4); 161 | 162 | // 5th database call should be `batchget` for the matching items 163 | const expectedBatchGet: DocumentClient.BatchGetItemInput = { 164 | RequestItems: { 165 | 'myTable': { 166 | Keys: [ 167 | { 168 | pk: 'task#001', 169 | sk: 'task#001' 170 | }, 171 | { 172 | pk: 'task#009', 173 | sk: 'task#009' 174 | } 175 | ] 176 | } 177 | } 178 | }; 179 | expect(mockedBatchGet.mock.calls[0][0]).toStrictEqual(expectedBatchGet); 180 | }); 181 | 182 | it('listing items with multiple tags involving many pagination\'s of tag items', async () => { 183 | 184 | /* 185 | input data 186 | */ 187 | const tags: Tags = { 188 | 'tag1': 'value1', 189 | 'tag2': 'value2', 190 | 'tag3': 'value3' 191 | }; 192 | 193 | /* 194 | mocks 195 | */ 196 | const mockedQuery = mockedDocumentClient.query = jest.fn() 197 | // mock 1st page of results for tag1 198 | .mockImplementationOnce(() => mockQueryOutput( 199 | ['value1#task#001', 'value1#task#003', 'value1#task#005', 'value1#task#007'].map(v => ({ sk: v })), { pk: 'tag#tag1', sk: 'value1#task#007' }) 200 | ) 201 | // mock 1st page of results for tag2 202 | .mockImplementationOnce(() => mockQueryOutput( 203 | ['value2#task#001', 'value2#task#005', 'value2#task#009', 'value2#task#013'].map(v => ({ sk: v })), { pk: 'tag#tag2', sk: 'value2#task#013' }) 204 | ) 205 | // mock 1st page of results for tag3 206 | .mockImplementationOnce(() => mockQueryOutput( 207 | ['value3#task#001', 'value3#task#009', 'value3#task#017', 'value3#task#025'].map(v => ({ sk: v })), { pk: 'tag#tag3', sk: 'value3#task#025' }) 208 | ) 209 | // mock 2nd page of results for tag1 210 | .mockImplementationOnce(() => mockQueryOutput( 211 | ['value1#task#009', 'value1#task#011', 'value1#task#013', 'value1#task#015'].map(v => ({ sk: v })), { pk: 'tag#tag1', sk: 'value1#task#015' }) 212 | ) 213 | // mock 2nd page of results for tag2 214 | .mockImplementationOnce(() => mockQueryOutput( 215 | ['value2#task#017', 'value2#task#021', 'value2#task#025', 'value2#task#029'].map(v => ({ sk: v })), { pk: 'tag#tag2', sk: 'value2#task#029' }) 216 | ) 217 | // mock 3rd page of results for tag1 218 | .mockImplementationOnce(() => mockQueryOutput( 219 | ['value1#task#017', 'value1#task#019', 'value1#task#021', 'value1#task#023'].map(v => ({ sk: v })), { pk: 'tag#tag1', sk: 'value1#task#023' }) 220 | ) 221 | // mock 4th and final page of results for tag1 222 | .mockImplementationOnce(() => mockQueryOutput( 223 | ['value1#task#025', 'value1#task#027'].map(v => ({ sk: v }))) 224 | ); 225 | 226 | // mock retrieving the items where the tags match 227 | const mockedBatchGet = mockedDocumentClient.batchGet = jest.fn() 228 | .mockImplementationOnce(() => mockedBatchGetItemOutput('myTable', [ 229 | { 230 | pk: 'task#001', 231 | sk: 'task#001', 232 | name: 'item1', 233 | description: 'item1 description', 234 | tags: { 235 | 'tag1': 'value1', 236 | 'tag2': 'value2', 237 | 'tag3': 'value3' 238 | } 239 | }, 240 | { 241 | pk: 'task#009', 242 | sk: 'task#009', 243 | name: 'item9', 244 | description: 'item9 description', 245 | tags: { 246 | 'tag1': 'value1', 247 | 'tag2': 'value2', 248 | 'tag3': 'value3' 249 | } 250 | }, 251 | { 252 | pk: 'task#017', 253 | sk: 'task#017', 254 | name: 'item17', 255 | description: 'item17 description', 256 | tags: { 257 | 'tag1': 'value1', 258 | 'tag2': 'value2', 259 | 'tag3': 'value3' 260 | } 261 | }, 262 | { 263 | pk: 'task#025', 264 | sk: 'task#025', 265 | name: 'item25', 266 | description: 'item25 description', 267 | tags: { 268 | 'tag1': 'value1', 269 | 'tag2': 'value2', 270 | 'tag3': 'value3' 271 | } 272 | } 273 | ]) 274 | ); 275 | 276 | /* 277 | test 278 | */ 279 | const results = await underTest.process(tags); 280 | 281 | /* 282 | assertions 283 | */ 284 | // correct number of results returned 285 | expect(results?.[0]?.length).toBe(4); 286 | 287 | // 1st result is as expected 288 | expect(results?.[0]?.[0]).toStrictEqual({ 289 | id: '001', 290 | name: 'item1', 291 | description: 'item1 description', 292 | tags: { 293 | 'tag1': 'value1', 294 | 'tag2': 'value2', 295 | 'tag3': 'value3' 296 | } 297 | }); 298 | 299 | // 2nd result is as expected 300 | expect(results?.[0]?.[1]).toStrictEqual({ 301 | id: '009', 302 | name: 'item9', 303 | description: 'item9 description', 304 | tags: { 305 | 'tag1': 'value1', 306 | 'tag2': 'value2', 307 | 'tag3': 'value3' 308 | } 309 | }); 310 | 311 | // 3rd result is as expected 312 | expect(results?.[0]?.[2]).toStrictEqual({ 313 | id: '017', 314 | name: 'item17', 315 | description: 'item17 description', 316 | tags: { 317 | 'tag1': 'value1', 318 | 'tag2': 'value2', 319 | 'tag3': 'value3' 320 | } 321 | }); 322 | 323 | // 4th result is as expected 324 | expect(results?.[0]?.[3]).toStrictEqual({ 325 | id: '025', 326 | name: 'item25', 327 | description: 'item25 description', 328 | tags: { 329 | 'tag1': 'value1', 330 | 'tag2': 'value2', 331 | 'tag3': 'value3' 332 | } 333 | }); 334 | 335 | 336 | // should be no pagination returned 337 | expect(results[1]).toBeUndefined(); 338 | 339 | // 1st database call should be `query` for page 1 of tag 1 items 340 | const expectedQueryInput1: DocumentClient.QueryInput = { 341 | TableName: 'myTable', 342 | KeyConditionExpression: `#hash = :hash AND begins_with(#sort,:sort)`, 343 | ExpressionAttributeNames: { 344 | '#hash': 'pk', 345 | '#sort': 'sk', 346 | }, 347 | ExpressionAttributeValues: { 348 | ':hash': 'tag#tag1', 349 | ':sort': 'value1#task#', 350 | }, 351 | ExclusiveStartKey: undefined, 352 | Limit: 20 353 | }; 354 | expect(mockedQuery.mock.calls[0][0]).toStrictEqual(expectedQueryInput1); 355 | 356 | // 2nd database call should be `query` for page 1 of tag 2 items 357 | const expectedQueryInput2: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 358 | expectedQueryInput2.ExpressionAttributeValues = { 359 | ':hash': 'tag#tag2', 360 | ':sort': 'value2#task#', 361 | }; 362 | expect(mockedQuery.mock.calls[1][0]).toStrictEqual(expectedQueryInput2); 363 | 364 | // 3rd database call should be `query` for page 1 of tag 3 items 365 | const expectedQueryInput3: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 366 | expectedQueryInput3.ExpressionAttributeValues = { 367 | ':hash': 'tag#tag3', 368 | ':sort': 'value3#task#', 369 | }; 370 | expect(mockedQuery.mock.calls[2][0]).toStrictEqual(expectedQueryInput3); 371 | 372 | // 4th database call should be `query` for page 2 of tag 1 items 373 | const expectedQueryInput4: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 374 | expectedQueryInput4.ExclusiveStartKey = { 375 | pk: 'tag#tag1', 376 | sk: 'value1#task#007' 377 | }; 378 | expect(mockedQuery.mock.calls[3][0]).toStrictEqual(expectedQueryInput4); 379 | 380 | // 5th database call should be `query` for page 2 of tag 2 items 381 | const expectedQueryInput5: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput2); 382 | expectedQueryInput5.ExclusiveStartKey = { 383 | pk: 'tag#tag2', 384 | sk: 'value2#task#013' 385 | }; 386 | expect(mockedQuery.mock.calls[4][0]).toStrictEqual(expectedQueryInput5); 387 | 388 | // 6th database call should be `query` for page 3 of tag 1 items 389 | const expectedQueryInput6: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 390 | expectedQueryInput6.ExclusiveStartKey = { 391 | pk: 'tag#tag1', 392 | sk: 'value1#task#015' 393 | }; 394 | expect(mockedQuery.mock.calls[5][0]).toStrictEqual(expectedQueryInput6); 395 | 396 | // 7th database call should be `query` for page 4 of tag 1 items 397 | const expectedQueryInput7: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 398 | expectedQueryInput7.ExclusiveStartKey = { 399 | pk: 'tag#tag1', 400 | sk: 'value1#task#023' 401 | }; 402 | expect(mockedQuery.mock.calls[6][0]).toStrictEqual(expectedQueryInput7); 403 | 404 | // 8th database call should be `batchget` for the matching items 405 | const expectedBatchGet: DocumentClient.BatchGetItemInput = { 406 | RequestItems: { 407 | 'myTable': { 408 | Keys: [ 409 | { pk: 'task#001', sk: 'task#001' }, 410 | { pk: 'task#009', sk: 'task#009' }, 411 | { pk: 'task#017', sk: 'task#017' }, 412 | { pk: 'task#025', sk: 'task#025' }, 413 | ] 414 | } 415 | } 416 | }; 417 | expect(mockedBatchGet.mock.calls[0][0]).toStrictEqual(expectedBatchGet); 418 | }); 419 | 420 | it('listing items with multiple tags with pagination', async () => { 421 | 422 | /* 423 | input data 424 | */ 425 | const tags: Tags = { 426 | 'tag1': 'value1', 427 | 'tag2': 'value2', 428 | 'tag3': 'value3' 429 | }; 430 | const paginationKey: TaskItemListPaginationKey = { 431 | id: '011' 432 | }; 433 | const count = 2; 434 | 435 | /* 436 | mocks 437 | */ 438 | const mockedQuery = mockedDocumentClient.query = jest.fn() 439 | // mock 1st page of results for tag1 440 | .mockImplementationOnce(() => mockQueryOutput( 441 | ['value1#task#011', 'value1#task#013', 'value1#task#015', 'value1#task#017'].map(v => ({ sk: v })), { pk: 'tag#tag1', sk: 'value1#task#017' }) 442 | ) 443 | // mock 1st page of results for tag2 444 | .mockImplementationOnce(() => mockQueryOutput( 445 | ['value2#task#013', 'value2#task#017', 'value2#task#021', 'value2#task#025'].map(v => ({ sk: v })), { pk: 'tag#tag2', sk: 'value2#task#025' }) 446 | ) 447 | // mock 1st page of results for tag3 448 | .mockImplementationOnce(() => mockQueryOutput( 449 | ['value3#task#017', 'value3#task#025', 'value3#task#033', 'value3#task#041'].map(v => ({ sk: v })), { pk: 'tag#tag3', sk: 'value3#task#041' }) 450 | ) 451 | // mock 2nd page of results for tag1 452 | .mockImplementationOnce(() => mockQueryOutput( 453 | ['value1#task#019', 'value1#task#021', 'value1#task#023', 'value1#task#025'].map(v => ({ sk: v })), { pk: 'tag#tag1', sk: 'value1#task#025' }) 454 | ); 455 | 456 | // mock retrieving the items where the tags match 457 | const mockedBatchGet = mockedDocumentClient.batchGet = jest.fn() 458 | .mockImplementationOnce(() => mockedBatchGetItemOutput('myTable', [ 459 | { 460 | pk: 'task#017', 461 | sk: 'task#017', 462 | name: 'item17', 463 | description: 'item17 description', 464 | tags: { 465 | 'tag1': 'value1', 466 | 'tag2': 'value2', 467 | 'tag3': 'value3' 468 | } 469 | }, 470 | { 471 | pk: 'task#025', 472 | sk: 'task#025', 473 | name: 'item25', 474 | description: 'item25 description', 475 | tags: { 476 | 'tag1': 'value1', 477 | 'tag2': 'value2', 478 | 'tag3': 'value3' 479 | } 480 | } 481 | ]) 482 | ); 483 | 484 | /* 485 | test 486 | */ 487 | const results = await underTest.process(tags, paginationKey, count); 488 | 489 | /* 490 | assertions 491 | */ 492 | // correct number of results returned 493 | expect(results?.[0]?.length).toBe(2); 494 | 495 | // 1st result is as expected 496 | expect(results?.[0]?.[0]).toStrictEqual({ 497 | id: '017', 498 | name: 'item17', 499 | description: 'item17 description', 500 | tags: { 501 | 'tag1': 'value1', 502 | 'tag2': 'value2', 503 | 'tag3': 'value3' 504 | } 505 | }); 506 | 507 | // 2nd result is as expected 508 | expect(results?.[0]?.[1]).toStrictEqual({ 509 | id: '025', 510 | name: 'item25', 511 | description: 'item25 description', 512 | tags: { 513 | 'tag1': 'value1', 514 | 'tag2': 'value2', 515 | 'tag3': 'value3' 516 | } 517 | }); 518 | 519 | // pagination returned is as expected 520 | const expectedPaginationKey: TaskItemListPaginationKey = { 521 | id: '025' 522 | }; 523 | expect(results[1]).toStrictEqual(expectedPaginationKey); 524 | 525 | // 1st database call should be `query` for page 1 of tag 1 items 526 | const expectedQueryInput1: DocumentClient.QueryInput = { 527 | TableName: 'myTable', 528 | KeyConditionExpression: `#hash = :hash AND begins_with(#sort,:sort)`, 529 | ExpressionAttributeNames: { 530 | '#hash': 'pk', 531 | '#sort': 'sk', 532 | }, 533 | ExpressionAttributeValues: { 534 | ':hash': 'tag#tag1', 535 | ':sort': 'value1#task#', 536 | }, 537 | ExclusiveStartKey: { 538 | pk: 'tag#tag1', 539 | sk: 'value1#task#011' 540 | }, 541 | Limit: count 542 | }; 543 | expect(mockedQuery.mock.calls[0][0]).toStrictEqual(expectedQueryInput1); 544 | 545 | // 2nd database call should be `query` for page 1 of tag 2 items 546 | const expectedQueryInput2: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 547 | expectedQueryInput2.ExpressionAttributeValues = { 548 | ':hash': 'tag#tag2', 549 | ':sort': 'value2#task#', 550 | }; 551 | expectedQueryInput2.ExclusiveStartKey = { 552 | pk: 'tag#tag2', 553 | sk: 'value2#task#011' 554 | } 555 | expect(mockedQuery.mock.calls[1][0]).toStrictEqual(expectedQueryInput2); 556 | 557 | // 3rd database call should be `query` for page 1 of tag 3 items 558 | const expectedQueryInput3: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 559 | expectedQueryInput3.ExpressionAttributeValues = { 560 | ':hash': 'tag#tag3', 561 | ':sort': 'value3#task#', 562 | }; 563 | expectedQueryInput3.ExclusiveStartKey = { 564 | pk: 'tag#tag3', 565 | sk: 'value3#task#011' 566 | } 567 | expect(mockedQuery.mock.calls[2][0]).toStrictEqual(expectedQueryInput3); 568 | 569 | // 4th database call should be `query` for page 2 of tag 1 items 570 | const expectedQueryInput4: DocumentClient.QueryInput = Object.assign({}, expectedQueryInput1); 571 | expectedQueryInput4.ExclusiveStartKey = { 572 | pk: 'tag#tag1', 573 | sk: 'value1#task#017' 574 | }; 575 | expect(mockedQuery.mock.calls[3][0]).toStrictEqual(expectedQueryInput4); 576 | 577 | // 5th database call should be `batchget` for the matching items 578 | const expectedBatchGet: DocumentClient.BatchGetItemInput = { 579 | RequestItems: { 580 | 'myTable': { 581 | Keys: [ 582 | { pk: 'task#017', sk: 'task#017' }, 583 | { pk: 'task#025', sk: 'task#025' }, 584 | ] 585 | } 586 | } 587 | }; 588 | expect(mockedBatchGet.mock.calls[0][0]).toStrictEqual(expectedBatchGet); 589 | }); 590 | 591 | it('listing items with no tags - happy path', async () => { 592 | 593 | /* 594 | mocks 595 | */ 596 | const mockedQuery = mockedDocumentClient.query = jest.fn() 597 | // mock 1st page of results 598 | .mockImplementationOnce(() => mockQueryOutput([ 599 | { 600 | pk: 'task#001', 601 | sk: 'task#001', 602 | name: 'item1', 603 | description: 'item1 description', 604 | tags: { 605 | tag1: 'value1' 606 | }, 607 | }, 608 | { 609 | pk: 'task#002', 610 | sk: 'task#002', 611 | name: 'item2', 612 | description: 'item2 description', 613 | }]) 614 | ); 615 | 616 | /* 617 | test 618 | */ 619 | const results = await underTest.process(); 620 | 621 | /* 622 | assertions 623 | */ 624 | // correct number of results returned 625 | expect(results?.[0]?.length).toBe(2); 626 | 627 | // 1st result is as expected 628 | expect(results?.[0]?.[0]).toStrictEqual({ 629 | id: '001', 630 | name: 'item1', 631 | description: 'item1 description', 632 | tags: { 633 | tag1: 'value1' 634 | }, 635 | }); 636 | 637 | // 2nd result is as expected 638 | expect(results?.[0]?.[1]).toStrictEqual({ 639 | id: '002', 640 | name: 'item2', 641 | description: 'item2 description', 642 | tags: undefined, 643 | }); 644 | 645 | // should be no pagination returned 646 | expect(results[1]).toBeUndefined(); 647 | 648 | // 1st database call should be `query` for page 1 of items 649 | const expectedQueryInput1: DocumentClient.QueryInput = { 650 | TableName: 'myTable', 651 | IndexName: 'siKey1-sk-index', 652 | KeyConditionExpression: `#hash = :hash`, 653 | ExpressionAttributeNames: { 654 | '#hash': 'siKey1', 655 | }, 656 | ExpressionAttributeValues: { 657 | ':hash': 'task', 658 | }, 659 | Select: 'ALL_ATTRIBUTES', 660 | ExclusiveStartKey: undefined, 661 | Limit: 20 662 | }; 663 | expect(mockedQuery.mock.calls[0][0]).toStrictEqual(expectedQueryInput1); 664 | 665 | }); 666 | 667 | it('listing items with no tags with pagination', async () => { 668 | 669 | /* 670 | input data 671 | */ 672 | const tags: Tags = undefined; 673 | const paginationKey: TaskItemListPaginationKey = { 674 | id: '003' 675 | }; 676 | const count = 2; 677 | 678 | /* 679 | mocks 680 | */ 681 | const mockedQuery = mockedDocumentClient.query = jest.fn() 682 | // mock 1st page of results 683 | .mockImplementationOnce(() => mockQueryOutput([ 684 | { 685 | pk: 'task#004', 686 | sk: 'task#004', 687 | name: 'item4', 688 | description: 'item4 description', 689 | tags: { 690 | tag1: 'value1' 691 | }, 692 | }, 693 | { 694 | pk: 'task#005', 695 | sk: 'task#005', 696 | name: 'item5', 697 | description: 'item5 description', 698 | }], 699 | { 700 | pk: 'task#005', 701 | sk: 'task#005', 702 | siKey1: 'task', 703 | }) 704 | ); 705 | 706 | /* 707 | test 708 | */ 709 | const results = await underTest.process(tags, paginationKey, count); 710 | 711 | /* 712 | assertions 713 | */ 714 | // correct number of results returned 715 | expect(results?.[0]?.length).toBe(2); 716 | 717 | // 1st result is as expected 718 | expect(results?.[0]?.[0]).toStrictEqual({ 719 | id: '004', 720 | name: 'item4', 721 | description: 'item4 description', 722 | tags: { 723 | tag1: 'value1' 724 | }, 725 | }); 726 | 727 | // 2nd result is as expected 728 | expect(results?.[0]?.[1]).toStrictEqual({ 729 | id: '005', 730 | name: 'item5', 731 | description: 'item5 description', 732 | tags: undefined, 733 | }); 734 | 735 | // pagination returned is as expected 736 | const expectedPaginationKey: TaskItemListPaginationKey = { 737 | id: '005' 738 | }; 739 | expect(results[1]).toStrictEqual(expectedPaginationKey); 740 | 741 | // 1st database call should be `query` for page 1 of items 742 | const expectedQueryInput1: DocumentClient.QueryInput = { 743 | TableName: 'myTable', 744 | IndexName: 'siKey1-sk-index', 745 | KeyConditionExpression: `#hash = :hash`, 746 | ExpressionAttributeNames: { 747 | '#hash': 'siKey1', 748 | }, 749 | ExpressionAttributeValues: { 750 | ':hash': 'task', 751 | }, 752 | Select: 'ALL_ATTRIBUTES', 753 | ExclusiveStartKey: { 754 | pk: 'task#003', 755 | sk: 'task#003', 756 | siKey1: 'task', 757 | }, 758 | Limit: count 759 | }; 760 | expect(mockedQuery.mock.calls[0][0]).toStrictEqual(expectedQueryInput1); 761 | 762 | }); 763 | 764 | function mockQueryOutput(items: unknown[], lastEvaluatedKey?: DocumentClient.Key): { promise: () => DocumentClient.QueryOutput } { 765 | const mocked: DocumentClient.QueryOutput = { 766 | Items: items, 767 | LastEvaluatedKey: lastEvaluatedKey, 768 | Count: items?.length ?? 0 769 | }; 770 | return { 771 | promise: () => mocked 772 | }; 773 | } 774 | 775 | function mockedBatchGetItemOutput(tableName: string, items: unknown[]): { promise: () => DocumentClient.BatchGetItemOutput } { 776 | const mocked: DocumentClient.BatchGetItemOutput = { 777 | Responses: { 778 | [tableName]: items 779 | } 780 | }; 781 | return { 782 | promise: () => mocked 783 | }; 784 | } 785 | }); 786 | -------------------------------------------------------------------------------- /src/lambda/list.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | // SPDX-License-Identifier: MIT-0 4 | */ 5 | import { DocumentClient } from 'aws-sdk/clients/dynamodb'; 6 | 7 | import { DynamoDbUtils } from '../utils/dynamoDb.util'; 8 | import { 9 | Tags, TaskItem, TaskItemIdListPaginationKey, TaskItemListPaginationKey 10 | } from './models'; 11 | 12 | /** 13 | * The lambda handler function to list existing task items, optionally filtering by tags 14 | */ 15 | export class ListService { 16 | 17 | private readonly MAX_LIST_RESULTS = 20; 18 | private readonly dynamoDbUtils: DynamoDbUtils; 19 | 20 | public constructor ( 21 | private dc: DocumentClient, 22 | private tableName: string 23 | ) { 24 | this.dynamoDbUtils = new DynamoDbUtils(dc); 25 | } 26 | 27 | public async process(tags?: Tags, paginationKey?: TaskItemListPaginationKey, count?: number): Promise<[TaskItem[], TaskItemListPaginationKey]> { 28 | 29 | let results: [TaskItem[], TaskItemListPaginationKey] = [undefined, undefined]; 30 | 31 | // if tags have been provided as a filter then we need to search filtering by tags first, but if not then we can just retrieve the task items 32 | if (Object.keys(tags??{}).length > 0) { 33 | // first retrieve the list of task ids that match all the tags 34 | const [taskIds, nextToken] = await this.listIds(tags, paginationKey, count); 35 | if ((taskIds?.length ?? 0) > 0) { 36 | // next retrieve the actual task items 37 | const items = await this.getItemsFromDb(taskIds); 38 | results = [items, nextToken]; 39 | } 40 | } else { 41 | results = await this.listItemsFromDb(paginationKey, count); 42 | } 43 | 44 | return results; 45 | } 46 | 47 | private async listIds(tags: Tags, paginationKey?: TaskItemListPaginationKey, count = this.MAX_LIST_RESULTS): Promise<[string[], TaskItemListPaginationKey]> { 48 | 49 | if (count) { 50 | count = Number(count); 51 | } 52 | 53 | // convert tags map to arrays to make referencing them later easier 54 | const tagKeys = Object.keys(tags); 55 | const tagValues = Object.values(tags); 56 | const tagCount = tagKeys.length; 57 | 58 | // retrieve the first page of results for each tag 59 | const resultsForTagsFutures: Promise<[string[], TaskItemListPaginationKey]>[] = new Array(tagCount); 60 | for (let tagIndex = 0; tagIndex < tagCount; tagIndex++) { 61 | const tagPaginationKey: TaskItemIdListPaginationKey = { 62 | id: paginationKey?.id, 63 | tagName: tagKeys[tagIndex], 64 | tagValue: tagValues[tagIndex], 65 | }; 66 | resultsForTagsFutures[tagIndex] = this.listIdsFromDbUsingTags(tagKeys[tagIndex], tagValues[tagIndex], tagPaginationKey, count); 67 | } 68 | const resultsForTags = await Promise.all(resultsForTagsFutures); 69 | const idsForTags = resultsForTags.map(([ids, _paginationKey]) => ids); 70 | 71 | // if any of the initial results are empty, then we can exit immediately as there are no common matches across all requested tags 72 | for (let tagIndex = 0; tagIndex < tagCount; tagIndex++) { 73 | if ((idsForTags[tagIndex]?.length ?? 0) === 0) { 74 | return [undefined, undefined]; 75 | } 76 | } 77 | 78 | // this inline function will populate new pages of TaskItem ids for a specific tag 79 | const getNextPageOfResults = async (tagIndex: number): Promise => { 80 | const paginationKey = resultsForTags[tagIndex]?.[1]; 81 | if (paginationKey === undefined) { 82 | // no more to process 83 | return false; 84 | } 85 | resultsForTags[tagIndex] = await this.listIdsFromDbUsingTags(tagKeys[tagIndex], tagValues[tagIndex], paginationKey, count); 86 | if ((resultsForTags[tagIndex]?.[0]?.length ?? 0) === 0) { 87 | // no more to process 88 | return false; 89 | } else { 90 | // store the new page of tags, and reset its pointer 91 | idsForTags[tagIndex] = resultsForTags[tagIndex]?.[0]; 92 | listPointers[tagIndex] = 0; 93 | return true; 94 | } 95 | } 96 | 97 | // this inline function will retrieve the next item id for a specific tag from the returned results 98 | const getNextItemIdFromResults = async (tagIndex: number): Promise => { 99 | let tagTaskItemId = idsForTags[tagIndex][listPointers[tagIndex]]; 100 | if (tagTaskItemId === undefined) { 101 | const hasMoreResults = await getNextPageOfResults(tagIndex); 102 | if (hasMoreResults) { 103 | tagTaskItemId = idsForTags[tagIndex][listPointers[tagIndex]]; 104 | } 105 | } 106 | return tagTaskItemId; 107 | } 108 | 109 | // process each list of TaskItemIds per tag, saving where the TaskItemId is found across all tags 110 | const matchedItemIds: string[] = []; 111 | const listPointers = new Array(tagCount).fill(0); 112 | const lastTagIndex = tagCount - 1; 113 | let keepGoing = true; 114 | while (keepGoing && matchedItemIds.length < count) { 115 | for (let tagIndex = 0; tagIndex < tagCount; tagIndex++) { 116 | const currentTagTaskItemId = await getNextItemIdFromResults(tagIndex) 117 | if (currentTagTaskItemId === undefined) { 118 | // no more results, so we can stop searching 119 | keepGoing = false; 120 | break; 121 | } 122 | // if we reached the last tag index, it means we found a match across all tags 123 | if (tagIndex === lastTagIndex) { 124 | // add the matched id to the result 125 | matchedItemIds.push(currentTagTaskItemId); 126 | // increment all the pointers to reference the next result for each tag 127 | listPointers.forEach((_value, index) => listPointers[index]++); 128 | } else { 129 | // check for matching TaskItemIds between this and the next tag to be compared 130 | const nextTagIndex = tagIndex + 1; 131 | const nextTagTaskItemId = await getNextItemIdFromResults(nextTagIndex) 132 | if (nextTagTaskItemId === undefined) { 133 | // no more results, so we can stop searching 134 | keepGoing = false; 135 | break; 136 | } 137 | 138 | if (currentTagTaskItemId === nextTagTaskItemId) { 139 | // we have a match across the tag pair being checked, so lets move onto checking the next tag pair 140 | continue; 141 | } else if (currentTagTaskItemId < nextTagTaskItemId) { 142 | // this tag has a lower TaskItem id, therefore increment this tags index, then restart the matching 143 | listPointers[tagIndex]++; 144 | break; 145 | } else { 146 | // the next tag has a lower TaskItem id, therefore increment the next tags index, then restart the matching 147 | listPointers[nextTagIndex]++; 148 | break; 149 | } 150 | } 151 | } 152 | } 153 | 154 | let nextToken: TaskItemListPaginationKey; 155 | if (matchedItemIds.length === count) { 156 | nextToken = { 157 | id: matchedItemIds[count - 1], 158 | } 159 | } 160 | 161 | const result: [string[], TaskItemListPaginationKey] = [matchedItemIds, nextToken]; 162 | return result; 163 | 164 | } 165 | 166 | private async listIdsFromDbUsingTags(tagName: string, tagValue: string, exclusiveStart?: TaskItemIdListPaginationKey, count?: number): Promise<[string[], TaskItemIdListPaginationKey]> { 167 | 168 | let exclusiveStartKey: DocumentClient.Key; 169 | if (exclusiveStart?.id) { 170 | exclusiveStartKey = { 171 | pk: `tag#${exclusiveStart.tagName}`, 172 | sk: `${exclusiveStart.tagValue}#task#${exclusiveStart.id}`, 173 | } 174 | } 175 | 176 | const params: DocumentClient.QueryInput = { 177 | TableName: this.tableName, 178 | KeyConditionExpression: `#hash = :hash AND begins_with(#sort,:sort)`, 179 | ExpressionAttributeNames: { 180 | '#hash': 'pk', 181 | '#sort': 'sk', 182 | }, 183 | ExpressionAttributeValues: { 184 | ':hash': `tag#${tagName}`, 185 | ':sort': `${tagValue}#task#`, 186 | }, 187 | ExclusiveStartKey: exclusiveStartKey, 188 | Limit: count ?? this.MAX_LIST_RESULTS, 189 | }; 190 | 191 | const results = await this.dc.query(params).promise(); 192 | if ((results?.Count ?? 0) === 0) { 193 | return [undefined, undefined]; 194 | } 195 | 196 | const taskIds: string[] = []; 197 | for (const i of results.Items) { 198 | taskIds.push(i.sk.split('#')[2]); 199 | } 200 | 201 | let paginationKey: TaskItemIdListPaginationKey; 202 | if (results.LastEvaluatedKey) { 203 | paginationKey = { 204 | tagName: results.LastEvaluatedKey.pk.split('#')[1], 205 | tagValue: results.LastEvaluatedKey.sk.split('#')[0], 206 | id: results.LastEvaluatedKey.sk.split('#')[2], 207 | } 208 | } 209 | const response: [string[], TaskItemIdListPaginationKey] = [taskIds, paginationKey]; 210 | return response; 211 | } 212 | 213 | private async getItemsFromDb(taskIds: string[]): Promise { 214 | 215 | const params: DocumentClient.BatchGetItemInput = { 216 | RequestItems: {} 217 | }; 218 | params.RequestItems[this.tableName] = { 219 | Keys: taskIds.map(id => ({ 220 | pk: `task#${id}`, 221 | sk: `task#${id}`, 222 | })) 223 | }; 224 | 225 | const response = await this.dynamoDbUtils.batchGetAll(params); 226 | if (response?.Responses?.[this.tableName] == undefined) { 227 | return []; 228 | } 229 | const items = this.assembleItems(response.Responses[this.tableName]); 230 | 231 | return items; 232 | } 233 | 234 | private async listItemsFromDb(exclusiveStart?: TaskItemListPaginationKey, count?: number): Promise<[TaskItem[], TaskItemListPaginationKey]> { 235 | 236 | let exclusiveStartKey: DocumentClient.Key; 237 | if (exclusiveStart?.id) { 238 | const lasttaskId = `task#${exclusiveStart.id}`; 239 | exclusiveStartKey = { 240 | pk: lasttaskId, 241 | sk: lasttaskId, 242 | siKey1: 'task', 243 | } 244 | } 245 | 246 | const params: DocumentClient.QueryInput = { 247 | TableName: this.tableName, 248 | IndexName: 'siKey1-sk-index', 249 | KeyConditionExpression: `#hash = :hash`, 250 | ExpressionAttributeNames: { 251 | '#hash': 'siKey1' 252 | }, 253 | ExpressionAttributeValues: { 254 | ':hash': 'task' 255 | }, 256 | Select: 'ALL_ATTRIBUTES', 257 | ExclusiveStartKey: exclusiveStartKey, 258 | Limit: count ?? this.MAX_LIST_RESULTS 259 | }; 260 | 261 | const results = await this.dc.query(params).promise(); 262 | if ((results?.Count ?? 0) === 0) { 263 | return [undefined, undefined]; 264 | } 265 | 266 | const items = this.assembleItems(results.Items); 267 | 268 | let paginationKey: TaskItemListPaginationKey; 269 | if (results.LastEvaluatedKey) { 270 | const lastEvaluatedtaskId = results.LastEvaluatedKey.pk.split('#')[1]; 271 | paginationKey = { 272 | id: lastEvaluatedtaskId, 273 | } 274 | } 275 | const response: [TaskItem[], TaskItemListPaginationKey] = [items, paginationKey]; 276 | return response; 277 | } 278 | 279 | private assembleItems(items: DocumentClient.ItemList): TaskItem[] { 280 | const list: TaskItem[] = []; 281 | for (const attrs of items) { 282 | const r: TaskItem = { 283 | id: attrs.pk.split('#')[1], 284 | name: attrs.name, 285 | description: attrs.description, 286 | tags: attrs.tags, 287 | }; 288 | list.push(r); 289 | } 290 | return list; 291 | } 292 | } 293 | -------------------------------------------------------------------------------- /src/lambda/models.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | // SPDX-License-Identifier: MIT-0 4 | */ 5 | export interface TaskItem { 6 | id?: string; 7 | name: string; 8 | description: string; 9 | tags?: Tags; 10 | } 11 | 12 | export type Tags = { [key: string]: Tag }; 13 | export type Tag = string; 14 | 15 | export interface TaskListItem { 16 | items: TaskItem[]; 17 | pagination?: { 18 | nextToken?: string; 19 | count?: number; 20 | } 21 | } 22 | 23 | export interface TaskItemListPaginationKey { 24 | id:string; 25 | } 26 | 27 | export interface TaskItemIdListPaginationKey { 28 | tagName?:string; 29 | tagValue?:string; 30 | id:string; 31 | } -------------------------------------------------------------------------------- /src/utils/dynamoDb.util.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | // SPDX-License-Identifier: MIT-0 4 | */ 5 | 6 | import { DocumentClient } from 'aws-sdk/clients/dynamodb'; 7 | 8 | export class DynamoDbUtils { 9 | 10 | private readonly MAX_RETRIES=3; 11 | private readonly DEFAULT_MAX_WRITE_BATCH_SIZE=25; 12 | private readonly DEFAULT_MAX_GET_BATCH_SIZE=100; 13 | 14 | public constructor(private dc: DocumentClient) { 15 | } 16 | 17 | public hasUnprocessedItems(result:DocumentClient.BatchWriteItemOutput):boolean { 18 | const has = result!==undefined && result.UnprocessedItems!==undefined; 19 | return has; 20 | } 21 | 22 | public async batchWriteAll(params:DocumentClient.BatchWriteItemInput, attempt=1) : Promise { 23 | 24 | if (attempt>this.MAX_RETRIES) { 25 | return params.RequestItems; 26 | } 27 | 28 | // dynamodb max batch size is 25 items, therefore split into smaller chunks if needed... 29 | const chunks = this.splitBatchWriteIntoChunks(params); 30 | 31 | // now process each chunk, including retries on failed intems... 32 | while(chunks.length) { 33 | const chunk = chunks.shift(); 34 | const response = await this.dc.batchWrite(chunk).promise(); 35 | if (response.UnprocessedItems!==undefined && Object.keys(response.UnprocessedItems).length>0) { 36 | const retryParams: DocumentClient.BatchWriteItemInput = { 37 | RequestItems: response.UnprocessedItems 38 | }; 39 | const retryResponse = await this.batchWriteAll(retryParams, attempt++); 40 | if (retryResponse.UnprocessedItems!==undefined && Object.keys(retryResponse.UnprocessedItems).length>0) { 41 | // even after max retries we have failed items, therefore return all unprocessed items 42 | return this.joinChunksIntoOutputBatchWrite(retryResponse, chunks); 43 | } 44 | } 45 | } 46 | 47 | return undefined; 48 | 49 | } 50 | 51 | public async batchGetAll(params:DocumentClient.BatchGetItemInput, attempt=1) : Promise { 52 | 53 | if (attempt>this.MAX_RETRIES) { 54 | return params.RequestItems; 55 | } 56 | 57 | // dynamodb max read batch size is 100 items, therefore split into smaller chunks if needed... 58 | const chunks = this.splitBatchGetIntoChunks(params); 59 | let response:DocumentClient.BatchGetItemOutput = {Responses: {}}; 60 | 61 | // now process each chunk, including retries on failed items... 62 | while(chunks.length) { 63 | const chunk = chunks.shift(); 64 | const r = await this.dc.batchGet(chunk).promise(); 65 | response = this.mergeBatchGetOutput(response, {Responses: r.Responses}); 66 | if (r.UnprocessedKeys!==undefined && Object.keys(r.UnprocessedKeys).length>0) { 67 | const retryParams: DocumentClient.BatchGetItemInput = { 68 | RequestItems: r.UnprocessedKeys 69 | }; 70 | const retryResponse = await this.batchGetAll(retryParams, attempt++); 71 | response = this.mergeBatchGetOutput(response, {Responses: retryResponse.Responses}); 72 | } 73 | } 74 | 75 | return response; 76 | } 77 | 78 | private splitBatchWriteIntoChunks(batch:DocumentClient.BatchWriteItemInput, maxBatchSize?:number) : DocumentClient.BatchWriteItemInput[] { 79 | 80 | if (maxBatchSize===undefined) { 81 | maxBatchSize=this.DEFAULT_MAX_WRITE_BATCH_SIZE; 82 | } 83 | 84 | // dynamodb max batch size is max 25 items, therefore split into smaller chunks if needed... 85 | let itemCount=0; 86 | Object.keys(batch.RequestItems).forEach(k=> itemCount+=batch.RequestItems[k].length); 87 | 88 | const chunks:DocumentClient.BatchWriteItemInput[]= []; 89 | if (itemCount>maxBatchSize) { 90 | let chunkSize=0; 91 | let chunk:DocumentClient.BatchWriteItemInput; 92 | Object.keys(batch.RequestItems).forEach(table=> { 93 | if (chunk===undefined) { 94 | chunk=this.newBatchWriteItemInput(table); 95 | } else { 96 | chunk.RequestItems[table]= []; 97 | } 98 | batch.RequestItems[table].forEach(item=> { 99 | if (chunkSize>=maxBatchSize) { 100 | // we've exceeded the max batch size, therefore save this and start with a new one 101 | chunks.push(chunk); 102 | chunk=this.newBatchWriteItemInput(table); 103 | chunkSize=0; 104 | } 105 | // add it to the current chunk 106 | chunk.RequestItems[table].push(item); 107 | chunkSize++; 108 | }); 109 | }); 110 | chunks.push(chunk); 111 | 112 | } else { 113 | chunks.push(batch); 114 | } 115 | 116 | return chunks; 117 | } 118 | 119 | private splitBatchGetIntoChunks(batch:DocumentClient.BatchGetItemInput, maxBatchSize?:number) : DocumentClient.BatchGetItemInput[] { 120 | 121 | if (maxBatchSize===undefined) { 122 | maxBatchSize=this.DEFAULT_MAX_GET_BATCH_SIZE; 123 | } 124 | 125 | // dynamodb max get batch size is max 100 items, therefore split into smaller chunks if needed... 126 | let itemCount=0; 127 | Object.keys(batch.RequestItems).forEach(k=> itemCount+=batch.RequestItems[k].Keys.length); 128 | 129 | const chunks:DocumentClient.BatchGetItemInput[]= []; 130 | if (itemCount>maxBatchSize) { 131 | let chunkSize=0; 132 | let chunk:DocumentClient.BatchGetItemInput; 133 | Object.keys(batch.RequestItems).forEach(table=> { 134 | if (chunk===undefined) { 135 | chunk=this.newBatchGetItemInput(table); 136 | } else { 137 | chunk.RequestItems[table]= {Keys:[]}; 138 | } 139 | batch.RequestItems[table].Keys.forEach(item=> { 140 | if (chunkSize>=maxBatchSize) { 141 | // we've exceeded the max batch size, therefore save this and start with a new one 142 | chunks.push(chunk); 143 | chunk=this.newBatchGetItemInput(table); 144 | chunkSize=0; 145 | } 146 | // add it to the current chunk 147 | chunk.RequestItems[table].Keys.push(item); 148 | chunkSize++; 149 | }); 150 | }); 151 | chunks.push(chunk); 152 | 153 | } else { 154 | chunks.push(batch); 155 | } 156 | 157 | return chunks; 158 | } 159 | 160 | public test___splitBatchWriteIntoChunks(params:DocumentClient.BatchWriteItemInput, maxBatchSize?:number) : DocumentClient.BatchWriteItemInput[] { 161 | return this.splitBatchWriteIntoChunks(params, maxBatchSize); 162 | } 163 | 164 | private joinChunksIntoOutputBatchWrite(unprocessed:DocumentClient.BatchWriteItemOutput, remaining:DocumentClient.BatchWriteItemInput[]) : DocumentClient.BatchWriteItemOutput { 165 | 166 | remaining.forEach(chunk=> { 167 | Object.keys(chunk.RequestItems).forEach(table=> { 168 | if (unprocessed.UnprocessedItems[table]===undefined) { 169 | unprocessed.UnprocessedItems[table]= []; 170 | } 171 | unprocessed.UnprocessedItems[table].push(...chunk.RequestItems[table]); 172 | }); 173 | }); 174 | 175 | return unprocessed; 176 | } 177 | 178 | private mergeBatchGetOutput(response:DocumentClient.BatchGetItemOutput, toMerge:DocumentClient.BatchGetItemOutput) : DocumentClient.BatchGetItemOutput { 179 | 180 | if (toMerge.Responses) { 181 | Object.keys(toMerge.Responses).forEach(table=> { 182 | if (response.Responses[table]===undefined) { 183 | response.Responses[table]= []; 184 | } 185 | response.Responses[table].push(...toMerge.Responses[table]); 186 | }); 187 | } 188 | 189 | if (toMerge.UnprocessedKeys) { 190 | Object.keys(toMerge.UnprocessedKeys).forEach(table=> { 191 | if (response.UnprocessedKeys[table]===undefined) { 192 | response.UnprocessedKeys[table]= {Keys:[]}; 193 | } 194 | response.UnprocessedKeys[table].Keys.push(...toMerge.UnprocessedKeys[table].Keys); 195 | }); 196 | 197 | } 198 | 199 | return response; 200 | } 201 | 202 | public test___joinChunksIntoOutputBatchWrite(unprocessed:DocumentClient.BatchWriteItemOutput, remaining:DocumentClient.BatchWriteItemInput[]) : DocumentClient.BatchWriteItemOutput { 203 | return this.joinChunksIntoOutputBatchWrite(unprocessed, remaining); 204 | } 205 | 206 | private newBatchWriteItemInput(table?:string) : DocumentClient.BatchWriteItemInput { 207 | const r:DocumentClient.BatchWriteItemInput = { 208 | RequestItems: {} 209 | }; 210 | if (table!==undefined) { 211 | r.RequestItems[table]= []; 212 | } 213 | return r; 214 | } 215 | 216 | private newBatchGetItemInput(table?:string) : DocumentClient.BatchGetItemInput { 217 | const r:DocumentClient.BatchGetItemInput = { 218 | RequestItems: {} 219 | }; 220 | if (table!==undefined) { 221 | r.RequestItems[table]= { 222 | Keys: [] 223 | }; 224 | } 225 | return r; 226 | } 227 | 228 | } 229 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "dist", 4 | "rootDir": "src", 5 | "module": "commonjs", 6 | "target": "es2019", 7 | "allowSyntheticDefaultImports": true, 8 | "allowUnreachableCode": false, 9 | "allowUnusedLabels": false, 10 | "declaration": true, 11 | "declarationMap": true, 12 | "emitDecoratorMetadata": true, 13 | "esModuleInterop": true, 14 | "experimentalDecorators": true, 15 | "noImplicitAny": true, 16 | "noImplicitReturns": true, 17 | "noImplicitUseStrict": false, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "pretty": true, 21 | "removeComments": true, 22 | "skipLibCheck": true, 23 | "sourceMap": true, 24 | "suppressImplicitAnyIndexErrors": true 25 | }, 26 | "include": [ 27 | "src" 28 | ], 29 | "exclude": [ 30 | "node_modules", 31 | "dist", 32 | ".vscode", 33 | ".git", 34 | ".history", 35 | "./**/__mocks__/*.ts", 36 | "cdk.out" 37 | ] 38 | } --------------------------------------------------------------------------------