├── .gitignore ├── .prettierrc.json ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── VERSION ├── config.json ├── config ├── coreConstant.js ├── dynamoFactory.js └── error │ ├── general.json │ └── param.json ├── index.js ├── lib ├── autoScale │ ├── Base.js │ └── helper.js ├── dynamodb │ └── base.js ├── formatter │ └── response.js ├── logger │ └── customConsoleLogger.js ├── models │ ├── dynamodb │ │ └── Base.js │ └── shardHelper.js └── util.js ├── package.json ├── services ├── autoScale │ ├── Base.js │ └── api.js └── dynamodb │ ├── Base.js │ ├── BatchGet.js │ ├── BatchWrite.js │ ├── CreateTableMigration.js │ ├── RetryQuery.js │ ├── TableExist.js │ ├── WaitFor.js │ └── api.js └── tests └── mocha └── services ├── auto_scale ├── create_table_migration.js ├── delete_scaling_policy.js ├── deregister_scalable_target.js ├── describe_scalable_targets.js ├── describe_scaling_policies.js ├── helper.js ├── put_scaling_policy.js ├── register_scalable_target.js └── update_continuous_backup.js ├── constants.js └── dynamodb ├── batch_get.js ├── batch_write.js ├── check_table_exist.js ├── create_table.js ├── delete_item.js ├── delete_table.js ├── describe_table.js ├── helper.js ├── list_tables.js ├── put_item.js ├── query.js ├── scan.js ├── testdata └── batch_get_write_data.js ├── update_item.js └── update_table.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore tmp files and folders 2 | /tmp/* 3 | !/log/.keep 4 | !/tmp/.keep 5 | /.idea 6 | /.idea/* 7 | 8 | package-lock.json 9 | 10 | # Runtime data 11 | pids 12 | *.pid 13 | *.seed 14 | 15 | # Directory for instrumented libs generated by jscoverage/JSCover 16 | lib-cov 17 | 18 | # Coverage directory used by tools like istanbul 19 | coverage 20 | 21 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 22 | .grunt 23 | 24 | # node-waf configuration 25 | .lock-wscript 26 | 27 | # Compiled binary addons (http://nodejs.org/api/addons.html) 28 | build/Release 29 | 30 | # Dependency directory 31 | # https://docs.npmjs.com/cli/shrinkwrap#caveats 32 | node_modules 33 | 34 | tests/scripts/st-poa/ 35 | 36 | tests/scripts/poa-genesis.json 37 | 38 | tests/scripts/pw 39 | 40 | 41 | # Dev Folder 42 | .dev 43 | 44 | # Debug log from npm 45 | npm-debug.log 46 | .vscode 47 | 48 | .DS_Store 49 | 50 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 120, 3 | "tabWidth": 2, 4 | "useTabs": false, 5 | "semi": true, 6 | "singleQuote": true, 7 | "trailingComma": "none", 8 | "bracketSpacing": true, 9 | "arrowParens": "always", 10 | "parser": "flow", 11 | "proseWrap": "preserve" 12 | } 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | language: node_js 3 | sudo: required 4 | branches: 5 | only: 6 | - master 7 | - develop 8 | - /^release-.*/ 9 | notifications: 10 | email: 11 | recipients: 12 | - ci.report@ost.com 13 | on_success: always 14 | on_failure: always 15 | env: 16 | global: 17 | - OS_CACHING_ENGINE=memcached 18 | - OST_MEMCACHE_SERVERS=127.0.0.1:11211 19 | node_js: 20 | - "12" 21 | - "10" 22 | services: 23 | - memcached 24 | before_install: 25 | - sudo apt-get update 26 | - sudo apt-get install nodejs 27 | - sudo apt-get install npm 28 | install: 29 | - npm install 30 | before_script: 31 | - mkdir -p dynamodb 32 | - wget https://s3.ap-south-1.amazonaws.com/dynamodb-local-mumbai/dynamodb_local_latest.zip -O ./dynamodb/dynamodb_local_latest.zip 33 | - unzip ./dynamodb/dynamodb_local_latest.zip 34 | - mkdir -p dynamodb_instance_2 35 | - nohup java -Djava.library.path=./dynamodb/DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb & 36 | - nohup java -Djava.library.path=./dynamodb/DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb -dbPath ./dynamodb_instance_2 -port 8001 & 37 | - sleep 30 38 | script: 39 | - node_modules/mocha/bin/mocha tests/mocha/services/dynamodb/* --timeout 10000 --exit 40 | after_script: 41 | - kill $(ps aux | grep 'DynamoDBLocal.jar' | awk '{print $2}') 42 | - rm -rf dynamodb 43 | - rm -rf dynamodb_instance_2 44 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Storage v1.0.4 2 | - Upgraded node version to 10.x 3 | - Version bump for dependencies. 4 | 5 | ## Storage v1.0.3 6 | - Integrated with new Instance Composer. 7 | - Migrated to ES6. 8 | - Migrated repository from OpenST Foundation to OST organization and renamed it. 9 | - Removed TokenBalance Model and Cache. 10 | 11 | ## Storage v1.0.2 12 | - Common style guide followed across all OST repos using prettier ([Storage#21](https://github.com/ostdotcom/storage/issues/21)) 13 | - Environment variables were a hard dependency. These lead to problems when multiple instances of OST Storage are needed, 14 | each having connection to different cache instances (for example). Thus making scaling not possible. Instead of reading 15 | the environment variables, we now depend on config object passed in the OST Storage constructor. Thus 2 objects of 16 | OST Storage can have totally different config strategies and thus will not interfere amongst each other. 17 | ([Storage#20](https://github.com/ostdotcom/storage/issues/20)) 18 | - Integrated use of Amazon DynamoDB Accelerator (DAX) to speed up certain actions ([Storage#18](https://github.com/ostdotcom/storage/issues/18)) 19 | - Exposed models/dynamodb/base. It is the base class for all models which use sharded tables. 20 | - Restructured model directory. 21 | - Updated versions for dependencies to resolve package vulnerabilities. 22 | - Available shards and managed shards tables are deprecated. 23 | 24 | ## Storage v1.0.0 25 | - OST Storage contains storage and sharding related services. 26 | - Wrapper services over Dynamo DB AWS SDK. 27 | - Auto Scale services to scale read/write capacity of DynamoDB tables. 28 | - Cache layer on top of Shard management services. 29 | - Model layer for token_balances and transaction_logs to support respective queries to DynamoDB. 30 | - Cache layer on top of token_balances. 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.0.4 2 | -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "ddbTablePrefix": "de_ma_", 3 | "cache": { 4 | "engine": "memcached", 5 | "servers": [ 6 | "127.0.0.1:11211" 7 | ], 8 | "defaultTtl": 36000 9 | }, 10 | "storage": { 11 | "endpoint": "http://localhost:8000", 12 | "region": "localhost", 13 | "apiVersion": "2012-08-10", 14 | "apiKey": "X", 15 | "apiSecret": "X", 16 | "enableSsl": "0", 17 | "enableLogging": "0", 18 | "enableAutoscaling": "0", 19 | "enableDax": "0", 20 | "maxRetryCount": 10, 21 | "autoScaling": { 22 | "endpoint": "http://localhost:8000", 23 | "region": "localhost", 24 | "apiKey": "X", 25 | "apiSecret": "X", 26 | "apiVersion": "2012-08-10", 27 | "enableSsl": "0" 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /config/coreConstant.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * Load all the core constants from the environment variables OR define them as literals here and export them. 5 | * 6 | * @module config/coreConstant 7 | * 8 | */ 9 | 10 | const rootPrefix = '..', 11 | paramErrorConfig = require(rootPrefix + '/config/error/param'), 12 | apiErrorConfig = require(rootPrefix + '/config/error/general'); 13 | 14 | /** 15 | * Constructor for core constants 16 | * 17 | * @constructor 18 | */ 19 | const CoreConstant = function() { 20 | const oThis = this; 21 | 22 | // Generic env variable across NPM packages 23 | oThis.DEBUG_ENABLED = process.env.OST_DEBUG_ENABLED; 24 | 25 | oThis.DAX_API_VERSION = ''; 26 | oThis.DAX_ACCESS_KEY_ID = ''; 27 | oThis.DAX_SECRET_ACCESS_KEY = ''; 28 | oThis.DAX_REGION = ''; 29 | oThis.DAX_ENDPOINT = ''; 30 | oThis.DAX_SSL_ENABLED = ''; 31 | }; 32 | 33 | CoreConstant.prototype = { 34 | // Generic env variable across NPM packages 35 | DEBUG_ENABLED: null, 36 | 37 | ERROR_CONFIG: { 38 | param_error_config: paramErrorConfig, 39 | api_error_config: apiErrorConfig 40 | }, 41 | 42 | get icNameSpace() { 43 | return 'OSTStorage'; 44 | }, 45 | 46 | // in ms fixed time after which we would retry 47 | fixedRetryAfterTime: function() { 48 | return 25; 49 | }, 50 | 51 | // in ms variable time (which is incremented after every iteration) which we would retry 52 | variableRetryAfterTime: function() { 53 | return 25; 54 | }, 55 | 56 | // default retry count 57 | defaultRetryCount: function() { 58 | return 50; 59 | } 60 | }; 61 | 62 | module.exports = new CoreConstant(); 63 | -------------------------------------------------------------------------------- /config/dynamoFactory.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | //Load external files 4 | require('http').globalAgent.keepAlive = true; 5 | 6 | const rootPrefix = '..'; 7 | const AWS = require('aws-sdk'), 8 | AWSDaxClient = require('amazon-dax-client'), 9 | OSTBase = require('@ostdotcom/base'), 10 | coreConstant = require(rootPrefix + '/config/coreConstant'), 11 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 12 | util = require(rootPrefix + '/lib/util'); 13 | 14 | const InstanceComposer = OSTBase.InstanceComposer; 15 | 16 | AWS.config.httpOptions.keepAlive = true; 17 | AWS.config.httpOptions.disableProgressEvents = false; 18 | 19 | /** 20 | * Constructor for DynamoDB Config 21 | * 22 | * @constructor 23 | */ 24 | const DynamoConfigFactory = function(configStrategies, instanceComposer) {}; 25 | 26 | DynamoConfigFactory.prototype = { 27 | /** 28 | * Type Raw 29 | * 30 | * @constant {string} 31 | * 32 | */ 33 | raw: 'raw', 34 | 35 | /** 36 | * Type DocumentClient 37 | * 38 | * @constant {string} 39 | * 40 | */ 41 | dax: 'dax', 42 | 43 | connectionParams: {}, 44 | 45 | /** 46 | * Get provider 47 | * 48 | * @param {string} preferredEndpoint - type of service, either raw or dax 49 | * @returns {object} - DynamoDB/Dax connection object 50 | * 51 | */ 52 | getProvider: async function(preferredEndpoint) { 53 | const oThis = this; 54 | 55 | let configStrategies = oThis.ic().configStrategy; 56 | 57 | if (configStrategies.storage.enableDax == 1 && preferredEndpoint === oThis.dax) { 58 | return await oThis.createDaxObject({ 59 | apiVersion: configStrategies.DAX_API_VERSION, 60 | accessKeyId: configStrategies.DAX_ACCESS_KEY_ID, 61 | secretAccessKey: configStrategies.DAX_SECRET_ACCESS_KEY, 62 | region: configStrategies.DAX_REGION, 63 | endpoint: configStrategies.DAX_ENDPOINT, 64 | sslEnabled: configStrategies.DAX_SSL_ENABLED == 1, 65 | logger: configStrategies.storage.enableLogging == 1 ? console : '' 66 | }); 67 | } else { 68 | return await oThis.createRawObject({ 69 | apiVersion: configStrategies.storage.apiVersion, 70 | accessKeyId: configStrategies.storage.apiKey, 71 | secretAccessKey: configStrategies.storage.apiSecret, 72 | region: configStrategies.storage.region, 73 | endpoint: configStrategies.storage.endpoint, 74 | sslEnabled: configStrategies.storage.enableSsl == 1, 75 | logger: configStrategies.storage.enableLogging == 1 ? console : '', 76 | retryDelayOptions: { 77 | customBackoff: function(retryCount) { 78 | logger.debug(`DDB Retry customBackoff called for retryCount: ${retryCount}`); 79 | return coreConstant.fixedRetryAfterTime() + retryCount * coreConstant.variableRetryAfterTime(); 80 | } 81 | }, 82 | maxRetries: util.isVarNull(configStrategies.storage.maxRetryCount) 83 | ? coreConstant.defaultRetryCount() 84 | : configStrategies.storage.maxRetryCount 85 | }); 86 | } 87 | }, 88 | 89 | // apiVersion-accessKeyId-region-endpoint-sslEnabled 90 | 91 | createRawObject: async function(connectionParams) { 92 | return await new AWS.DynamoDB(connectionParams); 93 | }, 94 | 95 | createDaxObject: async function(connectionParams) { 96 | return await new AWSDaxClient(connectionParams); 97 | } 98 | }; 99 | 100 | InstanceComposer.registerAsObject(DynamoConfigFactory, coreConstant.icNameSpace, 'dynamoConfigFactory', true); 101 | 102 | module.exports = {}; 103 | -------------------------------------------------------------------------------- /config/error/general.json: -------------------------------------------------------------------------------- 1 | { 2 | "exception": { 3 | "http_code": "500", 4 | "code": "INTERNAL_SERVER_ERROR", 5 | "message": "Something went wrong exception." 6 | }, 7 | "ddb_method_call_error":{ 8 | "http_code": "422", 9 | "code": "UNPROCESSABLE_ENTITY", 10 | "message": "DynamoDB method was called successfully but returned error" 11 | }, 12 | "auto_scale_method_call_error":{ 13 | "http_code": "422", 14 | "code": "UNPROCESSABLE_ENTITY", 15 | "message": "Auto Scale method was called successfully but returned error" 16 | }, 17 | "invalid_db_instance": { 18 | "http_code": "422", 19 | "code": "UNPROCESSABLE_ENTITY", 20 | "message": "DB instance is not defined" 21 | }, 22 | "invalid_method_ref": { 23 | "http_code": "422", 24 | "code": "UNPROCESSABLE_ENTITY", 25 | "message": "Method ref is not defined" 26 | }, 27 | "invalid_auto_scale_instance": { 28 | "http_code": "422", 29 | "code": "UNPROCESSABLE_ENTITY", 30 | "message": "Auto scale instance is not provided" 31 | }, 32 | "invalid_wait_for_method": { 33 | "http_code": "422", 34 | "code": "UNPROCESSABLE_ENTITY", 35 | "message": "Method to wait for is not defined" 36 | }, 37 | "invalid_table_name": { 38 | "http_code": "422", 39 | "code": "UNPROCESSABLE_ENTITY", 40 | "message": "Table name is not defined" 41 | }, 42 | "invalid_create_table_config": { 43 | "http_code": "422", 44 | "code": "UNPROCESSABLE_ENTITY", 45 | "message": "Create table config is not defined" 46 | }, 47 | "invalid_auto_scale_object": { 48 | "http_code": "422", 49 | "code": "UNPROCESSABLE_ENTITY", 50 | "message": "Provided auto scale object is not of Auto Scale Service" 51 | }, 52 | "invalid_auto_scale_config": { 53 | "http_code": "422", 54 | "code": "UNPROCESSABLE_ENTITY", 55 | "message": "Auto scale config is not defined" 56 | }, 57 | "invalid_register_scalable_target_write": { 58 | "http_code": "422", 59 | "code": "UNPROCESSABLE_ENTITY", 60 | "message": "Register Scalable Target Write config is not defined" 61 | }, 62 | "invalid_register_scalable_target_read": { 63 | "http_code": "422", 64 | "code": "UNPROCESSABLE_ENTITY", 65 | "message": "Register Scalable Target Read config is not defined" 66 | }, 67 | "invalid_put_scaling_policy_write": { 68 | "http_code": "422", 69 | "code": "UNPROCESSABLE_ENTITY", 70 | "message": "Put Scaling Policy Write config is not defined" 71 | }, 72 | "invalid_put_scaling_policy_read": { 73 | "http_code": "422", 74 | "code": "UNPROCESSABLE_ENTITY", 75 | "message": "Put Scaling Policy Read config is not defined" 76 | }, 77 | "invalid_method_name": { 78 | "http_code": "422", 79 | "code": "UNPROCESSABLE_ENTITY", 80 | "message": "Method name is not defined" 81 | }, 82 | "invalid_ddb_object": { 83 | "http_code": "422", 84 | "code": "UNPROCESSABLE_ENTITY", 85 | "message": "DDB object is not defined" 86 | }, 87 | "invalid_params": { 88 | "http_code": "422", 89 | "code": "UNPROCESSABLE_ENTITY", 90 | "message": "params are not defined" 91 | }, 92 | "invalid_ddb_api_object": { 93 | "http_code": "422", 94 | "code": "UNPROCESSABLE_ENTITY", 95 | "message": "DDB api object is not defined" 96 | }, 97 | "invalid_shard_type": { 98 | "http_code": "422", 99 | "code": "UNPROCESSABLE_ENTITY", 100 | "message": "Shard type is not valid" 101 | }, 102 | "invalid_shard_name": { 103 | "http_code": "422", 104 | "code": "UNPROCESSABLE_ENTITY", 105 | "message": "Shard name is not defined" 106 | }, 107 | "invalid_entity_type": { 108 | "http_code": "422", 109 | "code": "UNPROCESSABLE_ENTITY", 110 | "message": "Entity type is not valid" 111 | }, 112 | "invalid_shard_array": { 113 | "http_code": "422", 114 | "code": "UNPROCESSABLE_ENTITY", 115 | "message": "Shard name array is not valid" 116 | }, 117 | "invalid_ids_array": { 118 | "http_code": "422", 119 | "code": "UNPROCESSABLE_ENTITY", 120 | "message": "Ids array is not valid" 121 | }, 122 | "exceeded_batch_limit": { 123 | "http_code": "422", 124 | "code": "UNPROCESSABLE_ENTITY", 125 | "message": "Batch limit is exceeded" 126 | }, 127 | "invalid_allocation_type": { 128 | "http_code": "422", 129 | "code": "UNPROCESSABLE_ENTITY", 130 | "message": "Allocation type is not valid" 131 | }, 132 | "invalid_shard_identifier": { 133 | "http_code": "422", 134 | "code": "UNPROCESSABLE_ENTITY", 135 | "message": "Shard identifier is not defined" 136 | }, 137 | "invalid_force_allocation": { 138 | "http_code": "422", 139 | "code": "UNPROCESSABLE_ENTITY", 140 | "message": "force_allocation flag is not valid" 141 | }, 142 | "unhandled_catch_response": { 143 | "http_code": "500", 144 | "code": "INTERNAL_SERVER_ERROR", 145 | "message": "Something went wrong." 146 | }, 147 | "ddb_batch_write_failed": { 148 | "http_code": "422", 149 | "code": "UNPROCESSABLE_ENTITY", 150 | "message": "DDB Batch PUT failed for some or all entries." 151 | }, 152 | "ddb_batch_get_failed": { 153 | "http_code": "422", 154 | "code": "UNPROCESSABLE_ENTITY", 155 | "message": "DDB Batch GET failed for some or all entries." 156 | } 157 | } -------------------------------------------------------------------------------- /config/error/param.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Index File for OST Storage 3 | */ 4 | 5 | 'use strict'; 6 | 7 | const rootPrefix = '.', 8 | version = require(rootPrefix + '/package.json').version, 9 | OSTBase = require('@ostdotcom/base'), 10 | coreConstant = require(rootPrefix + '/config/coreConstant'); 11 | 12 | const InstanceComposer = OSTBase.InstanceComposer; 13 | 14 | require(rootPrefix + '/services/dynamodb/api'); 15 | require(rootPrefix + '/services/autoScale/api'); 16 | require(rootPrefix + '/lib/models/shardHelper'); 17 | 18 | const OSTStorage = function(configStrategy) { 19 | const oThis = this, 20 | instanceComposer = (oThis.ic = new InstanceComposer(configStrategy)), 21 | DynamodbShardHelper = instanceComposer.getShadowedClassFor(coreConstant.icNameSpace, 'DynamodbShardHelper'), 22 | dynamoDBApiService = instanceComposer.getInstanceFor(coreConstant.icNameSpace, 'dynamoDBApiService'), 23 | autoScaleApiService = instanceComposer.getInstanceFor(coreConstant.icNameSpace, 'autoScaleApiService'); 24 | 25 | if (!configStrategy) { 26 | throw 'Mandatory argument configStrategy missing'; 27 | } 28 | 29 | oThis.version = version; 30 | 31 | const model = (oThis.model = {}); 32 | model.DynamodbShardHelper = DynamodbShardHelper; 33 | 34 | oThis.dynamoDBService = dynamoDBApiService; 35 | oThis.autoScalingService = autoScaleApiService; 36 | }; 37 | 38 | const getInstanceKey = function(configStrategy) { 39 | return [ 40 | configStrategy.storage.apiVersion, 41 | configStrategy.storage.apiKey, 42 | configStrategy.storage.region, 43 | configStrategy.storage.endpoint, 44 | configStrategy.storage.enableSsl, 45 | 46 | configStrategy.storage.autoScaling.apiVersion, 47 | configStrategy.storage.autoScaling.apiKey, 48 | configStrategy.storage.autoScaling.region, 49 | configStrategy.storage.autoScaling.endpoint, 50 | configStrategy.storage.autoScaling.enableSsl 51 | ].join('-'); 52 | }; 53 | 54 | const instanceMap = {}; 55 | 56 | const Factory = function() {}; 57 | 58 | Factory.prototype = { 59 | getInstance: function(configStrategy) { 60 | // check if instance already present 61 | let instanceKey = getInstanceKey(configStrategy), 62 | _instance = instanceMap[instanceKey]; 63 | 64 | if (!_instance) { 65 | _instance = new OSTStorage(configStrategy); 66 | instanceMap[instanceKey] = _instance; 67 | } 68 | 69 | return _instance; 70 | } 71 | }; 72 | 73 | const factory = new Factory(); 74 | OSTStorage.getInstance = function() { 75 | return factory.getInstance.apply(factory, arguments); 76 | }; 77 | 78 | module.exports = OSTStorage; 79 | -------------------------------------------------------------------------------- /lib/autoScale/Base.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * DynamoDB Auto scale Library Base class 5 | * 6 | * @module lib/autoScale/Base 7 | * 8 | */ 9 | 10 | //Load external files 11 | 12 | require('http').globalAgent.keepAlive = true; 13 | 14 | const AWS = require('aws-sdk'); 15 | AWS.config.httpOptions.keepAlive = true; 16 | AWS.config.httpOptions.disableProgressEvents = false; 17 | 18 | const rootPrefix = '../..', 19 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 20 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 21 | OSTBase = require('@ostdotcom/base'), 22 | coreConstant = require(rootPrefix + '/config/coreConstant'); 23 | 24 | const InstanceComposer = OSTBase.InstanceComposer; 25 | 26 | /** 27 | * Constructor for base class 28 | * 29 | * @constructor 30 | */ 31 | const LibAutoScaleBase = function() { 32 | const oThis = this, 33 | configStrategy = oThis.ic().configStrategy; 34 | 35 | const autoscaleParams = { 36 | apiVersion: configStrategy.storage.autoScaling.apiVersion, 37 | accessKeyId: configStrategy.storage.autoScaling.apiKey, 38 | secretAccessKey: configStrategy.storage.autoScaling.apiSecret, 39 | region: configStrategy.storage.autoScaling.region, 40 | endpoint: configStrategy.storage.autoScaling.endpoint, 41 | sslEnabled: configStrategy.storage.autoScaling.enableSsl == 1, 42 | logger: configStrategy.storage.enableLogging == 1 ? console : '' 43 | }; 44 | 45 | oThis.autoScaling = new AWS.ApplicationAutoScaling(autoscaleParams); 46 | }; 47 | 48 | LibAutoScaleBase.prototype = { 49 | /** 50 | * Call dynamoDB methods 51 | * 52 | * @params {String} method - method name 53 | * @params {Object} params - Parameters 54 | * 55 | * @return {Promise} 56 | * 57 | */ 58 | call: function(method, ...methodArgs) { 59 | const oThis = this, 60 | autoScalingInstance = oThis.autoScaling, 61 | methodRef = autoScalingInstance[method]; 62 | // return promise 63 | return new Promise(function(onResolve) { 64 | try { 65 | // validate if the autoScaling instance is available 66 | if (!autoScalingInstance) 67 | return onResolve( 68 | responseHelper.error({ 69 | internal_error_identifier: 'l_as_b_call_1', 70 | api_error_identifier: 'invalid_auto_scale_instance', 71 | debug_options: {}, 72 | error_config: coreConstant.ERROR_CONFIG 73 | }) 74 | ); 75 | 76 | // validate if the method is available 77 | if (!methodRef) 78 | return onResolve( 79 | responseHelper.error({ 80 | internal_error_identifier: 'l_as_b_call_2', 81 | api_error_identifier: 'invalid_method_ref', 82 | debug_options: {}, 83 | error_config: coreConstant.ERROR_CONFIG 84 | }) 85 | ); 86 | 87 | methodArgs.push(function(err, data) { 88 | if (err) { 89 | logger.error('Error from AutoScaling ', err); 90 | return onResolve( 91 | responseHelper.error({ 92 | internal_error_identifier: 'l_as_b_call_3', 93 | api_error_identifier: 'auto_scale_method_call_error', 94 | debug_options: { error: err.stack }, 95 | error_config: coreConstant.ERROR_CONFIG 96 | }) 97 | ); 98 | } else { 99 | logger.debug(data); // successful response 100 | return onResolve(responseHelper.successWithData(data)); 101 | } 102 | }); 103 | 104 | methodRef.apply(autoScalingInstance, methodArgs); 105 | } catch (err) { 106 | logger.error('lib/autoScale/Base.js:call inside catch ', err); 107 | return onResolve( 108 | responseHelper.error({ 109 | internal_error_identifier: 'l_as_b_call_4', 110 | api_error_identifier: 'exception', 111 | debug_options: { error: err.stack }, 112 | error_config: coreConstant.ERROR_CONFIG 113 | }) 114 | ); 115 | } 116 | }); 117 | } 118 | }; 119 | 120 | InstanceComposer.registerAsShadowableClass(LibAutoScaleBase, coreConstant.icNameSpace, 'LibAutoScaleBase'); 121 | 122 | module.exports = {}; 123 | -------------------------------------------------------------------------------- /lib/autoScale/helper.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const rootPrefix = '../..', 4 | util = require(rootPrefix + '/lib/util'), 5 | OSTBase = require('@ostdotcom/base'), 6 | coreConstant = require(rootPrefix + '/config/coreConstant'); 7 | 8 | const InstanceComposer = OSTBase.InstanceComposer; 9 | 10 | const AutoScaleHelper = function() {}; 11 | 12 | AutoScaleHelper.prototype = { 13 | readCapacityScalableDimension: 'dynamodb:table:ReadCapacityUnits', 14 | writeCapacityScalableDimension: 'dynamodb:table:WriteCapacityUnits', 15 | indexReadCapacityScalableDimension: 'dynamodb:index:ReadCapacityUnits', 16 | indexWriteCapacityScalableDimenstion: 'dynamodb:index:WriteCapacityUnits', 17 | 18 | readMetricType: 'DynamoDBReadCapacityUtilization', 19 | writeMetricType: 'DynamoDBWriteCapacityUtilization', 20 | 21 | /** 22 | * Create resource Id 23 | * @param tableName 24 | * @return {string} 25 | */ 26 | createResourceId: function(tableName) { 27 | return 'table/' + tableName; 28 | }, 29 | 30 | /** 31 | * Create Index resource Id 32 | * @param tableName 33 | * @param indexName 34 | * @return {string} 35 | */ 36 | createIndexResourceId: function(tableName, indexName) { 37 | const oThis = this; 38 | 39 | return oThis.createResourceId(tableName) + '/index/' + indexName; 40 | }, 41 | 42 | /** 43 | * To create Scalable Target Params 44 | * @param resourceId 45 | * @param scalableDimension 46 | * @param minCapacityValue 47 | * @param maxCapacityValue 48 | * @return {Object} 49 | */ 50 | createScalableTargetParams: function(resourceId, scalableDimension, minCapacityValue, maxCapacityValue) { 51 | return { 52 | ResourceId: resourceId /* required */, 53 | ScalableDimension: scalableDimension, 54 | ServiceNamespace: 'dynamodb' /* required */, 55 | MaxCapacity: maxCapacityValue, 56 | MinCapacity: minCapacityValue, 57 | RoleARN: 'wrongArn' 58 | }; 59 | }, 60 | 61 | /** 62 | * To create Scaling Policy Params 63 | * @param tableName 64 | * @param resourceId 65 | * @param scalableDimension 66 | * @param predefinedMetricType 67 | * @param scaleOutCoolDown 68 | * @param scaleInCoolDown 69 | * @param targetValue 70 | * @return {Object} 71 | */ 72 | createPolicyParams: function( 73 | tableName, 74 | resourceId, 75 | scalableDimension, 76 | predefinedMetricType, 77 | scaleOutCoolDown, 78 | scaleInCoolDown, 79 | targetValue 80 | ) { 81 | return { 82 | ServiceNamespace: 'dynamodb', 83 | ResourceId: resourceId, 84 | ScalableDimension: scalableDimension, 85 | PolicyName: tableName + '-scaling-policy', 86 | PolicyType: 'TargetTrackingScaling', 87 | TargetTrackingScalingPolicyConfiguration: { 88 | PredefinedMetricSpecification: { 89 | PredefinedMetricType: predefinedMetricType 90 | }, 91 | ScaleOutCooldown: scaleOutCoolDown, // seconds 92 | ScaleInCooldown: scaleInCoolDown, // seconds 93 | TargetValue: targetValue 94 | } 95 | }; 96 | } 97 | }; 98 | 99 | InstanceComposer.registerAsObject(AutoScaleHelper, coreConstant.icNameSpace, 'autoScaleHelper', true); 100 | 101 | module.exports = AutoScaleHelper; 102 | -------------------------------------------------------------------------------- /lib/dynamodb/base.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * DynamoDB Libraries Base class 5 | * 6 | * @module lib/dynamodb/base 7 | * 8 | */ 9 | 10 | //Load external files 11 | require('http').globalAgent.keepAlive = true; 12 | 13 | const AWS = require('aws-sdk'); 14 | AWS.config.httpOptions.keepAlive = true; 15 | AWS.config.httpOptions.disableProgressEvents = false; 16 | 17 | const rootPrefix = '../..', 18 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 19 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 20 | OSTBase = require('@ostdotcom/base'), 21 | coreConstant = require(rootPrefix + '/config/coreConstant'); 22 | 23 | const InstanceComposer = OSTBase.InstanceComposer; 24 | 25 | require(rootPrefix + '/config/dynamoFactory'); 26 | 27 | /** 28 | * Constructor for base class 29 | * 30 | * @params {Object} params - DynamoDB configurations params 31 | * 32 | * @constructor 33 | */ 34 | const LibDynamoDBBase = function() {}; 35 | 36 | LibDynamoDBBase.prototype = { 37 | /** 38 | * Query on DynamoDB . 39 | * 40 | * @param methodName 41 | * @param preferredEndPoint - If the function is Dax supported or raw supported 42 | * @param params 43 | * @returns {Promise<*|Promise>} 44 | */ 45 | 46 | queryDdb: async function(methodName, preferredEndPoint, ...params) { 47 | const oThis = this, 48 | dynamoFactory = oThis.ic().getInstanceFor(coreConstant.icNameSpace, 'dynamoConfigFactory'); 49 | // Last parameter is the serviceType passed from every method. 50 | //let serviceType = params.splice(-1, 1); // Removes the last element from the params and modifies params. 51 | const dbInstance = await dynamoFactory.getProvider(preferredEndPoint); 52 | return oThis.callGeneric(dbInstance, methodName, params); 53 | }, 54 | 55 | /** 56 | * Call dynamoDB methods 57 | * 58 | * @params {String} method - method name 59 | * @params {object} params - Parameters 60 | * 61 | * @return {Promise} 62 | * 63 | */ 64 | 65 | callGeneric: function(dbInstance, method, methodArgs) { 66 | const oThis = this, 67 | methodRef = dbInstance[method]; 68 | // return promise 69 | return new Promise(function(onResolve, onReject) { 70 | let preQuery = Date.now(); 71 | try { 72 | // validate if the DB instance is available 73 | if (!dbInstance) 74 | return onResolve( 75 | responseHelper.error({ 76 | internal_error_identifier: 'l_dy_b_call_1', 77 | api_error_identifier: 'invalid_db_instance', 78 | debug_options: {}, 79 | error_config: coreConstant.ERROR_CONFIG 80 | }) 81 | ); 82 | 83 | // validate if the method is available 84 | if (!methodRef) 85 | return onResolve( 86 | responseHelper.error({ 87 | internal_error_identifier: 'l_dy_b_call_2', 88 | api_error_identifier: 'invalid_method_ref', 89 | debug_options: {}, 90 | error_config: coreConstant.ERROR_CONFIG 91 | }) 92 | ); 93 | 94 | methodArgs.push(function(err, data) { 95 | if (err) { 96 | return onResolve( 97 | responseHelper.error({ 98 | internal_error_identifier: `l_dy_b_call_3:${err.code}`, 99 | api_error_identifier: 'ddb_method_call_error', 100 | debug_options: { method: method, methodArgs: methodArgs, error: err }, 101 | error_config: coreConstant.ERROR_CONFIG 102 | }) 103 | ); 104 | } else { 105 | logger.debug('(' + (Date.now() - preQuery) + ' ms) DDB Call Completed: Method: ', method, methodArgs); 106 | return onResolve(responseHelper.successWithData(data)); 107 | } 108 | }); 109 | 110 | methodRef.apply(dbInstance, methodArgs); 111 | } catch (err) { 112 | logger.error('lib/dynamodb/base.js:call inside catch ', err); 113 | return onResolve( 114 | responseHelper.error({ 115 | internal_error_identifier: 'l_dy_b_call_4', 116 | api_error_identifier: 'exception', 117 | debug_options: { method: method, methodArgs: methodArgs, error: err.stack }, 118 | error_config: coreConstant.ERROR_CONFIG 119 | }) 120 | ); 121 | } 122 | }); 123 | } 124 | }; 125 | 126 | InstanceComposer.registerAsObject(LibDynamoDBBase, coreConstant.icNameSpace, 'libDynamoDBBase', true); 127 | 128 | module.exports = {}; 129 | -------------------------------------------------------------------------------- /lib/formatter/response.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /* 4 | * Restful API response formatter 5 | */ 6 | 7 | const OSTBase = require('@ostdotcom/base'), 8 | responseHelper = new OSTBase.responseHelper({ 9 | moduleName: 'OSTStorage' 10 | }); 11 | 12 | module.exports = responseHelper; 13 | -------------------------------------------------------------------------------- /lib/logger/customConsoleLogger.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /* 4 | * Custom Console log methods. 5 | * 6 | */ 7 | const OSTBase = require('@ostdotcom/base'), 8 | Logger = OSTBase.Logger; 9 | 10 | const rootPrefix = '../..', 11 | coreConstant = require(rootPrefix + '/config/coreConstant'), 12 | loggerLevel = 1 === Number(coreConstant.DEBUG_ENABLED) ? Logger.LOG_LEVELS.DEBUG : Logger.LOG_LEVELS.INFO; 13 | 14 | module.exports = new Logger('OSTStorage', loggerLevel); 15 | -------------------------------------------------------------------------------- /lib/models/dynamodb/Base.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Base Model 3 | * 4 | * This is the base class for all models which use sharded tables. 5 | * 6 | * @module lib/models/dynamodb/Base 7 | * 8 | */ 9 | 10 | const rootPrefix = '../../..', 11 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 12 | OSTBase = require('@ostdotcom/base'), 13 | coreConstant = require(rootPrefix + '/config/coreConstant'); 14 | 15 | const InstanceComposer = OSTBase.InstanceComposer; 16 | 17 | require(rootPrefix + '/lib/autoScale/helper'); 18 | require(rootPrefix + '/services/dynamodb/api'); 19 | 20 | /** 21 | * Base Model Constructor 22 | * 23 | * @constructor 24 | */ 25 | const BaseDynamodbModel = function() {}; 26 | 27 | BaseDynamodbModel.prototype = { 28 | /** 29 | * Create shard 30 | * 31 | * @return {Promise} 32 | */ 33 | createShard: async function() { 34 | const oThis = this, 35 | ddbServiceObj = oThis.ic().getInstanceFor(coreConstant.icNameSpace, 'dynamoDBApiService'); 36 | 37 | let schema = oThis._createTableParams(oThis.shardName); 38 | 39 | let createParams = { 40 | createTableConfig: schema, 41 | autoScalingConfig: oThis._getAutoScalingParams(oThis.shardName) 42 | }; 43 | 44 | let createTableResponse = await ddbServiceObj.createTableMigration(createParams); 45 | if (createTableResponse.isFailure()) return Promise.reject(createTableResponse); 46 | 47 | return responseHelper.successWithData({}); 48 | }, 49 | 50 | /** 51 | * get auto scaling params 52 | * 53 | * @return {Object} 54 | */ 55 | _getAutoScalingParams: function(tableName) { 56 | const oThis = this, 57 | autoScaleHelper = oThis.ic().getInstanceFor(coreConstant.icNameSpace, 'autoScaleHelper'), 58 | resourceId = autoScaleHelper.createResourceId(tableName), 59 | gsiArray = oThis._createTableParams(tableName).GlobalSecondaryIndexes || []; 60 | 61 | let autoScalingConfig = {}; 62 | 63 | autoScalingConfig.registerScalableTargetWrite = autoScaleHelper.createScalableTargetParams( 64 | resourceId, 65 | autoScaleHelper.writeCapacityScalableDimension, 66 | 10, 67 | 500 68 | ); 69 | 70 | autoScalingConfig.registerScalableTargetRead = autoScaleHelper.createScalableTargetParams( 71 | resourceId, 72 | autoScaleHelper.readCapacityScalableDimension, 73 | 10, 74 | 500 75 | ); 76 | 77 | autoScalingConfig.putScalingPolicyWrite = autoScaleHelper.createPolicyParams( 78 | tableName, 79 | resourceId, 80 | autoScaleHelper.writeCapacityScalableDimension, 81 | autoScaleHelper.writeMetricType, 82 | 1, 83 | 10, 84 | 50.0 85 | ); 86 | 87 | autoScalingConfig.putScalingPolicyRead = autoScaleHelper.createPolicyParams( 88 | tableName, 89 | resourceId, 90 | autoScaleHelper.readCapacityScalableDimension, 91 | autoScaleHelper.readMetricType, 92 | 1, 93 | 10, 94 | 50.0 95 | ); 96 | 97 | autoScalingConfig.globalSecondaryIndex = {}; 98 | 99 | for (let index = 0; index < gsiArray.length; index++) { 100 | let gsiIndexName = gsiArray[index].IndexName, 101 | indexResourceId = autoScaleHelper.createIndexResourceId(tableName, gsiIndexName); 102 | 103 | autoScalingConfig.globalSecondaryIndex[gsiIndexName] = {}; 104 | 105 | autoScalingConfig.globalSecondaryIndex[ 106 | gsiIndexName 107 | ].registerScalableTargetWrite = autoScaleHelper.createScalableTargetParams( 108 | indexResourceId, 109 | autoScaleHelper.indexWriteCapacityScalableDimenstion, 110 | 5, 111 | 500 112 | ); 113 | 114 | autoScalingConfig.globalSecondaryIndex[ 115 | gsiIndexName 116 | ].registerScalableTargetRead = autoScaleHelper.createScalableTargetParams( 117 | indexResourceId, 118 | autoScaleHelper.indexReadCapacityScalableDimension, 119 | 5, 120 | 500 121 | ); 122 | 123 | autoScalingConfig.globalSecondaryIndex[gsiIndexName].putScalingPolicyWrite = autoScaleHelper.createPolicyParams( 124 | tableName, 125 | indexResourceId, 126 | autoScaleHelper.indexWriteCapacityScalableDimenstion, 127 | autoScaleHelper.writeMetricType, 128 | 1, 129 | 10, 130 | 50.0 131 | ); 132 | 133 | autoScalingConfig.globalSecondaryIndex[gsiIndexName].putScalingPolicyRead = autoScaleHelper.createPolicyParams( 134 | tableName, 135 | indexResourceId, 136 | autoScaleHelper.indexReadCapacityScalableDimension, 137 | autoScaleHelper.readMetricType, 138 | 1, 139 | 10, 140 | 50.0 141 | ); 142 | } 143 | return autoScalingConfig; 144 | }, 145 | 146 | /** 147 | * Handles logic of shorting input param keys 148 | * 149 | * @private 150 | * @param longName - long name of key 151 | * 152 | * @return {String} 153 | */ 154 | shortNameFor: function(longName) { 155 | const oThis = this; 156 | return oThis.longToShortNamesMap[longName]; 157 | }, 158 | 159 | /** 160 | * Handles logic of shorting input param keys 161 | * 162 | * @private 163 | * @param shortName - long name of key 164 | * 165 | * @return {String} 166 | */ 167 | longNameFor: function(shortName) { 168 | const oThis = this; 169 | return oThis.shortToLongNamesMap[shortName]; 170 | } 171 | }; 172 | 173 | module.exports = BaseDynamodbModel; 174 | -------------------------------------------------------------------------------- /lib/models/shardHelper.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Shard Creation Helper 3 | * 4 | * @module lib/models/shardHelper 5 | * 6 | */ 7 | 8 | const rootPrefix = '../..', 9 | BaseDynamodbModel = require(rootPrefix + '/lib/models/dynamodb/Base'), 10 | OSTBase = require('@ostdotcom/base'), 11 | coreConstant = require(rootPrefix + '/config/coreConstant'); 12 | 13 | const InstanceComposer = OSTBase.InstanceComposer; 14 | 15 | /** 16 | * Shard Helper 17 | * 18 | * @augments BaseDynamodbModel 19 | * 20 | * @constructor 21 | */ 22 | const DynamodbShardHelper = function(params) { 23 | const oThis = this; 24 | 25 | oThis.tableSchema = params.table_schema; 26 | oThis.shardName = params.shard_name; 27 | 28 | BaseDynamodbModel.call(oThis); 29 | }; 30 | 31 | DynamodbShardHelper.prototype = Object.create(BaseDynamodbModel.prototype); 32 | 33 | const shardHelperSpecificPrototype = { 34 | /** 35 | * Create table params 36 | * 37 | * @return {object} 38 | */ 39 | _createTableParams: function(shardName) { 40 | const oThis = this; 41 | 42 | return oThis.tableSchema; 43 | } 44 | }; 45 | 46 | Object.assign(DynamodbShardHelper.prototype, shardHelperSpecificPrototype); 47 | 48 | InstanceComposer.registerAsShadowableClass(DynamodbShardHelper, coreConstant.icNameSpace, 'DynamodbShardHelper'); 49 | 50 | module.exports = DynamodbShardHelper; 51 | -------------------------------------------------------------------------------- /lib/util.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Utils = function() {}; 4 | 5 | Utils.prototype = { 6 | constructor: Utils, 7 | 8 | formatDbDate: function(dateObj) { 9 | function pad(n) { 10 | return n < 10 ? '0' + n : n; 11 | } 12 | 13 | return ( 14 | dateObj.getFullYear() + 15 | '-' + 16 | pad(dateObj.getMonth() + 1) + 17 | '-' + 18 | pad(dateObj.getDate()) + 19 | ' ' + 20 | pad(dateObj.getHours()) + 21 | ':' + 22 | pad(dateObj.getMinutes()) + 23 | ':' + 24 | pad(dateObj.getSeconds()) 25 | ); 26 | }, 27 | 28 | invert: function(json) { 29 | let ret = {}; 30 | for (let key in json) { 31 | ret[json[key]] = key; 32 | } 33 | return ret; 34 | }, 35 | 36 | clone: function(obj) { 37 | return JSON.parse(JSON.stringify(obj)); 38 | }, 39 | 40 | isVarNull(variable) { 41 | return typeof variable === 'undefined' || variable == null; 42 | } 43 | }; 44 | 45 | module.exports = new Utils(); 46 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@ostdotcom/storage", 3 | "version": "1.0.4", 4 | "description": "OST storage provides data storage and sharding services.", 5 | "main": "index.js", 6 | "scripts": { 7 | "pre-commit": "lint-staged" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/ostdotcom/storage.git" 12 | }, 13 | "keywords": [ 14 | "OpenST", 15 | "OST", 16 | "Simple Token", 17 | "OST Storage", 18 | "DynamoDB" 19 | ], 20 | "author": "OST.com Inc.", 21 | "license": "Apache-2.0", 22 | "bugs": { 23 | "url": "https://github.com/ostdotcom/storage/issues" 24 | }, 25 | "homepage": "https://github.com/ostdotcom/storage#readme", 26 | "dependencies": { 27 | "@ostdotcom/base": "^2.0.0", 28 | "aws-sdk": "2.263.1", 29 | "amazon-dax-client": "1.1.3", 30 | "bignumber.js": "4.1.0" 31 | }, 32 | "devDependencies": { 33 | "chai": "4.2.0", 34 | "ink-docstrap": "1.3.2", 35 | "lint-staged": "8.0.3", 36 | "mocha": "5.2.0", 37 | "pre-commit": "1.2.2", 38 | "prettier": "1.14.3" 39 | }, 40 | "pre-commit": [ 41 | "pre-commit" 42 | ], 43 | "lint-staged": { 44 | "*.js": [ 45 | "prettier --write --config .prettierrc.json", 46 | "git add" 47 | ] 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /services/autoScale/Base.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * AutoScale service base class 5 | * 6 | * @module services/autoScale/Base 7 | * 8 | */ 9 | const rootPrefix = '../..', 10 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 11 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 12 | OSTBase = require('@ostdotcom/base'), 13 | coreConstant = require(rootPrefix + '/config/coreConstant'); 14 | 15 | const InstanceComposer = OSTBase.InstanceComposer; 16 | 17 | require(rootPrefix + '/lib/autoScale/Base'); 18 | 19 | /** 20 | * Constructor for base service class 21 | * 22 | * @param {string} methodName - AutoScale method name 23 | * @param {object} params - AutoScale method params 24 | * 25 | * @constructor 26 | */ 27 | const AutoScaleServicesBase = function(methodName, params) { 28 | const oThis = this; 29 | logger.debug('=======AutoScale.Base.params======='); 30 | logger.debug('\nmethodName: ' + methodName, '\nparams: ' + params); 31 | oThis.params = params; 32 | oThis.methodName = methodName; 33 | }; 34 | 35 | AutoScaleServicesBase.prototype = { 36 | /** 37 | * Perform method 38 | * 39 | * @return {Promise} 40 | * 41 | */ 42 | perform: async function() { 43 | const oThis = this; 44 | 45 | return oThis.asyncPerform().catch(function(err) { 46 | logger.error('services/autoScale/Base.js:perform inside catch ', err); 47 | return responseHelper.error({ 48 | internal_error_identifier: 's_as_b_perform_1', 49 | api_error_identifier: 'exception', 50 | debug_options: { message: err.message }, 51 | error_config: coreConstant.ERROR_CONFIG 52 | }); 53 | }); 54 | }, 55 | 56 | /** 57 | * Async Perform 58 | * 59 | * @return {Promise<*>} 60 | */ 61 | asyncPerform: async function() { 62 | const oThis = this; 63 | 64 | let r = oThis.validateParams(); 65 | logger.debug('=======AutoScale.Base.validateParams.result======='); 66 | logger.debug(r); 67 | if (r.isFailure()) return r; 68 | 69 | r = oThis.executeAutoScaleRequest(); 70 | logger.debug('=======AutoScale.Base.executeAutoScaleRequest.result======='); 71 | logger.debug(r); 72 | return r; 73 | }, 74 | 75 | /** 76 | * Validation of params 77 | * 78 | * @return {result} 79 | * 80 | */ 81 | validateParams: function() { 82 | const oThis = this; 83 | 84 | // validate if the method is available 85 | if (!oThis.methodName) 86 | return responseHelper.error({ 87 | internal_error_identifier: 'l_as_b_validateParams_1', 88 | api_error_identifier: 'invalid_method_name', 89 | debug_options: {}, 90 | error_config: coreConstant.ERROR_CONFIG 91 | }); 92 | 93 | if (!oThis.params) 94 | return responseHelper.error({ 95 | internal_error_identifier: 'l_as_b_validateParams_3', 96 | api_error_identifier: 'invalid_params', 97 | debug_options: {}, 98 | error_config: coreConstant.ERROR_CONFIG 99 | }); 100 | 101 | return responseHelper.successWithData({}); 102 | }, 103 | 104 | /** 105 | * Execute AutoScale request 106 | * 107 | * @return {promise} 108 | * 109 | */ 110 | executeAutoScaleRequest: async function() { 111 | const oThis = this, 112 | ASBase = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'LibAutoScaleBase'), 113 | autoScaleObject = new ASBase(), 114 | r = await autoScaleObject.call(oThis.methodName, oThis.params); 115 | 116 | logger.debug('=======Base.perform.result======='); 117 | logger.debug(r); 118 | return r; 119 | } 120 | }; 121 | 122 | InstanceComposer.registerAsShadowableClass(AutoScaleServicesBase, coreConstant.icNameSpace, 'AutoScaleServicesBase'); 123 | 124 | module.exports = AutoScaleServicesBase; 125 | -------------------------------------------------------------------------------- /services/autoScale/api.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * AutoScale service api 5 | * 6 | * @module services/autoScale/api 7 | * 8 | */ 9 | 10 | const rootPrefix = '../..', 11 | OSTBase = require('@ostdotcom/base'), 12 | coreConstant = require(rootPrefix + '/config/coreConstant'); 13 | 14 | const InstanceComposer = OSTBase.InstanceComposer; 15 | 16 | require(rootPrefix + '/services/autoScale/Base'); 17 | 18 | /** 19 | * Constructor for AutoScale api service class 20 | * 21 | * @params {Object} params - AutoScale connection configurations 22 | * 23 | * @constructor 24 | */ 25 | const AutoScaleApiService = function() {}; 26 | 27 | AutoScaleApiService.prototype = { 28 | /** 29 | * Register scalable Target 30 | * 31 | * @param {Object} params - Parameters 32 | * 33 | * @return {*} 34 | */ 35 | registerScalableTarget: function(params) { 36 | const oThis = this, 37 | ASServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'AutoScaleServicesBase'), 38 | createAutoScalingGroup = new ASServiceBaseKlass('registerScalableTarget', params); 39 | return createAutoScalingGroup.perform(); 40 | }, 41 | 42 | /** 43 | * Put Scaling Policy 44 | * 45 | * @param {Object} params - Parameters 46 | * 47 | * @return {*} 48 | */ 49 | putScalingPolicy: function(params) { 50 | const oThis = this, 51 | ASServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'AutoScaleServicesBase'), 52 | createAutoScalingGroup = new ASServiceBaseKlass('putScalingPolicy', params); 53 | return createAutoScalingGroup.perform(); 54 | }, 55 | 56 | /** 57 | * Delete Scaling policy 58 | * 59 | * @param {Object} params - Parameters 60 | * 61 | * @return {*} 62 | */ 63 | deleteScalingPolicy: function(params) { 64 | const oThis = this, 65 | ASServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'AutoScaleServicesBase'), 66 | createAutoScalingGroup = new ASServiceBaseKlass('deleteScalingPolicy', params); 67 | return createAutoScalingGroup.perform(); 68 | }, 69 | 70 | /** 71 | * De Register Scalable Target 72 | * 73 | * @param {Object} params - Parameters 74 | * 75 | * @return {*} 76 | */ 77 | deregisterScalableTarget: function(params) { 78 | const oThis = this, 79 | ASServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'AutoScaleServicesBase'), 80 | createAutoScalingGroup = new ASServiceBaseKlass('deregisterScalableTarget', params); 81 | return createAutoScalingGroup.perform(); 82 | }, 83 | 84 | /** 85 | * Describe Scalable Targets 86 | * 87 | * @param {Object} params - Parameters 88 | * 89 | * @return {*} 90 | */ 91 | describeScalableTargets: function(params) { 92 | const oThis = this, 93 | ASServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'AutoScaleServicesBase'), 94 | createAutoScalingGroup = new ASServiceBaseKlass('describeScalableTargets', params); 95 | return createAutoScalingGroup.perform(); 96 | }, 97 | 98 | /** 99 | * Describe scaling policies 100 | * 101 | * @param {Object} params - Parameters 102 | * 103 | * @return {*} 104 | */ 105 | describeScalingPolicies: function(params) { 106 | const oThis = this, 107 | ASServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'AutoScaleServicesBase'), 108 | createAutoScalingGroup = new ASServiceBaseKlass('describeScalingPolicies', params); 109 | return createAutoScalingGroup.perform(); 110 | } 111 | }; 112 | 113 | AutoScaleApiService.prototype.constructor = AutoScaleApiService; 114 | 115 | InstanceComposer.registerAsObject(AutoScaleApiService, coreConstant.icNameSpace, 'autoScaleApiService', true); 116 | 117 | module.exports = AutoScaleApiService; 118 | -------------------------------------------------------------------------------- /services/dynamodb/Base.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * DynamoDB service base class 5 | * 6 | * @module services/dynamodb/Base 7 | * 8 | */ 9 | 10 | const rootPrefix = '../..', 11 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 12 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 13 | OSTBase = require('@ostdotcom/base'), 14 | coreConstant = require(rootPrefix + '/config/coreConstant'); 15 | 16 | const InstanceComposer = OSTBase.InstanceComposer; 17 | 18 | require(rootPrefix + '/lib/dynamodb/base'); 19 | 20 | /** 21 | * Constructor for base service class 22 | * 23 | * @param {String} methodName - DDB method name 24 | * @param {Object} params - DDB method params 25 | * @param {String} serviceType - type of service supported 26 | * 27 | * @constructor 28 | */ 29 | const DDBServiceBase = function(methodName, params, serviceType) { 30 | const oThis = this; 31 | 32 | oThis.params = params; 33 | oThis.methodName = methodName; 34 | oThis.serviceType = serviceType; 35 | }; 36 | 37 | DDBServiceBase.prototype = { 38 | /** 39 | * Perform method 40 | * 41 | * @return {Promise} 42 | * 43 | */ 44 | perform: async function() { 45 | const oThis = this; 46 | 47 | return oThis.asyncPerform().catch(function(err) { 48 | logger.error('services/dynamodb/Base.js:perform inside catch ', err); 49 | return responseHelper.error({ 50 | internal_error_identifier: 's_dy_b_perform_1', 51 | api_error_identifier: 'exception', 52 | debug_options: { error: err.stack }, 53 | error_config: coreConstant.ERROR_CONFIG 54 | }); 55 | }); 56 | }, 57 | 58 | /** 59 | * Async Perform 60 | * 61 | * @return {Promise<*>} 62 | */ 63 | asyncPerform: async function() { 64 | const oThis = this; 65 | 66 | let r = oThis.validateParams(); 67 | 68 | if (r.isFailure()) return r; 69 | 70 | r = await oThis.executeDdbRequest(); 71 | 72 | return r; 73 | }, 74 | 75 | /** 76 | * Validation of params 77 | * 78 | * @return {result} 79 | * 80 | */ 81 | validateParams: function() { 82 | const oThis = this; 83 | 84 | if (!oThis.methodName) { 85 | return responseHelper.error({ 86 | internal_error_identifier: 'l_dy_b_validateParams_1', 87 | api_error_identifier: 'invalid_method_name', 88 | debug_options: {}, 89 | error_config: coreConstant.ERROR_CONFIG 90 | }); 91 | } 92 | 93 | if (!oThis.params) { 94 | return responseHelper.error({ 95 | internal_error_identifier: 'l_dy_b_validateParams_3', 96 | api_error_identifier: 'invalid_params', 97 | debug_options: {}, 98 | error_config: coreConstant.ERROR_CONFIG 99 | }); 100 | } 101 | 102 | return responseHelper.successWithData({}); 103 | }, 104 | 105 | /** 106 | * Execute dynamoDB request 107 | * 108 | * @return {promise} 109 | * 110 | */ 111 | executeDdbRequest: async function() { 112 | const oThis = this; 113 | // Last parameter is service type (dax or dynamoDB) 114 | return await oThis 115 | .ic() 116 | .getInstanceFor(coreConstant.icNameSpace, 'libDynamoDBBase') 117 | .queryDdb(oThis.methodName, oThis.serviceType, oThis.params); 118 | } 119 | }; 120 | 121 | InstanceComposer.registerAsShadowableClass(DDBServiceBase, coreConstant.icNameSpace, 'DDBServiceBase'); 122 | 123 | module.exports = DDBServiceBase; 124 | -------------------------------------------------------------------------------- /services/dynamodb/BatchGet.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * DynamoDB Batch Write with retry count 5 | * 6 | * @module services/dynamodb/BatchGet 7 | * 8 | */ 9 | 10 | const rootPrefix = '../..', 11 | base = require(rootPrefix + '/services/dynamodb/Base'), 12 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 13 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 14 | OSTBase = require('@ostdotcom/base'), 15 | coreConstant = require(rootPrefix + '/config/coreConstant'); 16 | 17 | const InstanceComposer = OSTBase.InstanceComposer; 18 | 19 | require(rootPrefix + '/lib/dynamodb/base'); 20 | 21 | /** 22 | * Constructor for batch write item service class 23 | * @param {Object} params - Parameters 24 | * @param {Integer} unprocessed_keys_retry_count - retry count for unprocessed keys (optional) 25 | * @param {String} serviceType - type of service supported 26 | * 27 | * @constructor 28 | */ 29 | const DDBServiceBatchGetItem = function(params, unprocessed_keys_retry_count, serviceType) { 30 | const oThis = this; 31 | oThis.serviceType = serviceType; 32 | 33 | let configStrategies = oThis.ic().configStrategy; 34 | oThis.unprocessedKeysRetryCount = 35 | unprocessed_keys_retry_count || configStrategies.storage.maxRetryCount || coreConstant.defaultRetryCount(); 36 | 37 | base.call(oThis, 'batchGetItem', params, oThis.serviceType); 38 | }; 39 | 40 | DDBServiceBatchGetItem.prototype = Object.create(base.prototype); 41 | 42 | const batchGetPrototype = { 43 | /** 44 | * Validation of params 45 | * 46 | * @return {*} 47 | */ 48 | validateParams: function() { 49 | const oThis = this, 50 | validationResponse = base.prototype.validateParams.call(oThis); 51 | 52 | if (validationResponse.isFailure()) return validationResponse; 53 | 54 | return responseHelper.successWithData({}); 55 | }, 56 | 57 | /** 58 | * Execute dynamoDB request 59 | * 60 | * @return {promise} 61 | * 62 | */ 63 | executeDdbRequest: async function() { 64 | const oThis = this; 65 | 66 | try { 67 | let batchGetParams = oThis.params, 68 | waitTime = 0, 69 | constantTimeFactor = coreConstant.fixedRetryAfterTime(), 70 | variableTimeFactor = coreConstant.variableRetryAfterTime(), 71 | localResponse, 72 | globalResponse, 73 | attemptNo = 1, 74 | unprocessedKeys, 75 | unprocessedKeysLength; 76 | 77 | while (true) { 78 | logger.debug('executeDdbRequest batch_get attemptNo ', attemptNo); 79 | 80 | localResponse = await oThis.batchGetItemAfterWait(batchGetParams, waitTime); 81 | 82 | if (!localResponse.isSuccess()) { 83 | if ( 84 | localResponse.internalErrorCode.includes('ResourceNotFoundException') || 85 | localResponse.internalErrorCode.includes('ProvisionedThroughputExceededException') 86 | ) { 87 | logger.error( 88 | `services/dynamodb/BatchGet.js:executeDdbRequest, ${localResponse.internalErrorCode} : attemptNo: `, 89 | attemptNo 90 | ); 91 | localResponse.data['UnprocessedKeys'] = batchGetParams['RequestItems']; 92 | } else { 93 | return responseHelper.error({ 94 | internal_error_identifier: 's_dy_bw_executeDdbRequest_1', 95 | api_error_identifier: 'exception', 96 | debug_options: { error: localResponse.toHash() }, 97 | error_config: coreConstant.ERROR_CONFIG 98 | }); 99 | } 100 | } 101 | 102 | if (!globalResponse) { 103 | globalResponse = localResponse; 104 | } else { 105 | // append response of each successful (partial/complete) attempt to globalresponse 106 | let localResponses = localResponse.data.Responses, 107 | globalResponses = globalResponse.data.Responses; 108 | for (let tableName in localResponses) { 109 | if (globalResponses.hasOwnProperty(tableName)) { 110 | globalResponses[tableName] = globalResponses[tableName].concat(localResponses[tableName]); 111 | } else { 112 | globalResponses[tableName] = localResponses[tableName]; 113 | } 114 | } 115 | } 116 | 117 | unprocessedKeys = localResponse.data['UnprocessedKeys']; 118 | unprocessedKeysLength = 0; 119 | 120 | for (let tableName in unprocessedKeys) { 121 | if (unprocessedKeys.hasOwnProperty(tableName)) { 122 | unprocessedKeysLength += unprocessedKeys[tableName]['Keys'].length; 123 | logger.warn( 124 | 'DynamoDB BATCH_GET ATTEMPT_FAILED TableName :', 125 | tableName, 126 | ' unprocessedItemsCount: ', 127 | unprocessedKeysLength, 128 | ' keys count: ', 129 | batchGetParams.RequestItems[tableName]['Keys'].length, 130 | ' attemptNo ', 131 | attemptNo 132 | ); 133 | if (oThis.unprocessedKeysRetryCount) { 134 | logger.info('Retry will be attempted.'); 135 | } 136 | } 137 | } 138 | 139 | // Break the loop if unprocessedItems get empty or retry count exceeds the given limit 140 | if (unprocessedKeysLength === 0 || oThis.unprocessedKeysRetryCount === 0) { 141 | globalResponse.data.UnprocessedKeys = unprocessedKeys; 142 | break; 143 | } 144 | 145 | //Create new batchWriteParams of unprocessedItems 146 | batchGetParams = { RequestItems: unprocessedKeys }; 147 | 148 | //adjust retry variables 149 | waitTime = constantTimeFactor + attemptNo * variableTimeFactor; 150 | attemptNo += 1; 151 | oThis.unprocessedKeysRetryCount -= 1; 152 | } 153 | 154 | for (let tableName in unprocessedKeys) { 155 | if (unprocessedKeys.hasOwnProperty(tableName)) { 156 | logger.error( 157 | 'DynamoDB BATCH_GET ALL_ATTEMPTS_FAILED TableName :', 158 | tableName, 159 | ' unprocessedItemsCount: ', 160 | unprocessedKeysLength, 161 | ' attempts Failed ', 162 | attemptNo 163 | ); 164 | } 165 | } 166 | 167 | return globalResponse; 168 | } catch (err) { 169 | logger.error('services/dynamodb/BatchGet.js:executeDdbRequest inside catch ', err); 170 | return responseHelper.error({ 171 | internal_error_identifier: 's_dy_bw_executeDdbRequest_1', 172 | api_error_identifier: 'exception', 173 | debug_options: { error: err.message }, 174 | error_config: coreConstant.ERROR_CONFIG 175 | }); 176 | } 177 | }, 178 | 179 | /** 180 | * Batch get Item after waiting for given time 181 | * @param {Object} batchGetKeys - Batch get keys 182 | * @param {Integer} waitTime - wait time in milliseconds 183 | * @return {Promise} 184 | */ 185 | batchGetItemAfterWait: async function(batchGetKeys, waitTime) { 186 | const oThis = this; 187 | 188 | return new Promise(function(resolve) { 189 | setTimeout(async function() { 190 | let r = await oThis 191 | .ic() 192 | .getInstanceFor(coreConstant.icNameSpace, 'libDynamoDBBase') 193 | .queryDdb(oThis.methodName, oThis.serviceType, batchGetKeys); 194 | resolve(r); 195 | }, waitTime); 196 | }); 197 | } 198 | }; 199 | 200 | Object.assign(DDBServiceBatchGetItem.prototype, batchGetPrototype); 201 | DDBServiceBatchGetItem.prototype.constructor = batchGetPrototype; 202 | 203 | InstanceComposer.registerAsShadowableClass(DDBServiceBatchGetItem, coreConstant.icNameSpace, 'DDBServiceBatchGetItem'); 204 | 205 | module.exports = DDBServiceBatchGetItem; 206 | -------------------------------------------------------------------------------- /services/dynamodb/BatchWrite.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /** 3 | * DynamoDB Batch Write with retry count 4 | * 5 | * @module services/dynamodb/BatchWrite 6 | */ 7 | const OSTBase = require('@ostdotcom/base'), 8 | InstanceComposer = OSTBase.InstanceComposer; 9 | 10 | const rootPrefix = '../..', 11 | base = require(rootPrefix + '/services/dynamodb/Base'), 12 | coreConstant = require(rootPrefix + '/config/coreConstant'), 13 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 14 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'); 15 | 16 | /** 17 | * Constructor for batch write item service class 18 | * @param {Object} params - Parameters 19 | * @param {Integer} unprocessed_items_retry_count - retry count for unprocessed items (optional) 20 | * @param {String} serviceType - type of service supported 21 | * 22 | * @constructor 23 | */ 24 | const DDBServiceBatchWriteItem = function(params, unprocessed_items_retry_count, serviceType) { 25 | const oThis = this; 26 | 27 | oThis.serviceType = serviceType; 28 | 29 | let configStrategies = oThis.ic().configStrategy; 30 | oThis.unprocessedItemsRetryCount = 31 | unprocessed_items_retry_count || configStrategies.storage.maxRetryCount || coreConstant.defaultRetryCount(); 32 | 33 | base.call(oThis, 'batchWriteItem', params, oThis.serviceType); 34 | }; 35 | 36 | DDBServiceBatchWriteItem.prototype = Object.create(base.prototype); 37 | 38 | const batchWritePrototype = { 39 | /** 40 | * Validation of params 41 | * 42 | * @return {*} 43 | */ 44 | validateParams: function() { 45 | const oThis = this, 46 | validationResponse = base.prototype.validateParams.call(oThis); 47 | if (validationResponse.isFailure()) return validationResponse; 48 | 49 | return responseHelper.successWithData({}); 50 | }, 51 | 52 | /** 53 | * Execute dynamoDB request 54 | * 55 | * @return {promise} 56 | * 57 | */ 58 | executeDdbRequest: async function() { 59 | const oThis = this; 60 | 61 | try { 62 | let batchWriteParams = oThis.params, 63 | waitTime = 0, 64 | constantTimeFactor = coreConstant.fixedRetryAfterTime(), 65 | variableTimeFactor = coreConstant.variableRetryAfterTime(), 66 | response, 67 | attemptNo = 1, 68 | unprocessedItems, 69 | unprocessedItemsLength; 70 | 71 | while (true) { 72 | logger.debug('executeDdbRequest attemptNo ', attemptNo); 73 | 74 | response = await oThis.batchWriteItemAfterWait(batchWriteParams, waitTime); 75 | 76 | if (!response.isSuccess()) { 77 | if ( 78 | response.internalErrorCode.includes('ResourceNotFoundException') || 79 | response.internalErrorCode.includes('ProvisionedThroughputExceededException') 80 | ) { 81 | logger.error( 82 | `services/dynamodb/BatchWrite.js:executeDdbRequest, ${response.internalErrorCode} : attemptNo: `, 83 | attemptNo 84 | ); 85 | response.data['UnprocessedItems'] = batchWriteParams['RequestItems']; 86 | } else { 87 | return responseHelper.error({ 88 | internal_error_identifier: 's_dy_bw_executeDdbRequest_1', 89 | api_error_identifier: 'exception', 90 | debug_options: { error: response.toHash() }, 91 | error_config: coreConstant.ERROR_CONFIG 92 | }); 93 | } 94 | } 95 | 96 | unprocessedItems = response.data['UnprocessedItems']; 97 | unprocessedItemsLength = 0; 98 | 99 | for (let tableName in unprocessedItems) { 100 | if (unprocessedItems.hasOwnProperty(tableName)) { 101 | unprocessedItemsLength += unprocessedItems[tableName].length; 102 | logger.warn( 103 | 'DynamoDB BATCH_WRITE ATTEMPT_FAILED TableName :', 104 | tableName, 105 | ' unprocessedItemsCount: ', 106 | unprocessedItemsLength, 107 | ' items count: ', 108 | batchWriteParams.RequestItems[tableName].length, 109 | ' attemptNo ', 110 | attemptNo 111 | ); 112 | if (oThis.unprocessedItemsRetryCount) { 113 | logger.info('Retry will be attempted.'); 114 | } 115 | } 116 | } 117 | 118 | // Break the loop if unprocessedItems get empty or retry count exceeds the given limit 119 | if (unprocessedItemsLength === 0 || oThis.unprocessedItemsRetryCount === 0) { 120 | break; 121 | } 122 | 123 | //Create new batchWriteParams of unprocessedItems 124 | batchWriteParams = { RequestItems: unprocessedItems }; 125 | 126 | //adjust retry variables 127 | waitTime = constantTimeFactor + attemptNo * variableTimeFactor; 128 | attemptNo += 1; 129 | oThis.unprocessedItemsRetryCount -= 1; 130 | } 131 | 132 | for (let tableName in unprocessedItems) { 133 | if (unprocessedItems.hasOwnProperty(tableName)) { 134 | logger.error( 135 | 'DynamoDB BATCH_WRITE ALL_ATTEMPTS_FAILED TableName :', 136 | tableName, 137 | ' unprocessedItemsCount: ', 138 | unprocessedItemsLength, 139 | ' attempts Failed ', 140 | attemptNo 141 | ); 142 | } 143 | } 144 | 145 | return response; 146 | } catch (err) { 147 | logger.error('services/dynamodb/BatchWrite.js:executeDdbRequest inside catch ', err); 148 | return responseHelper.error({ 149 | internal_error_identifier: 's_dy_bw_executeDdbRequest_1', 150 | api_error_identifier: 'exception', 151 | debug_options: { error: err.message }, 152 | error_config: coreConstant.ERROR_CONFIG 153 | }); 154 | } 155 | }, 156 | 157 | /** 158 | * Batch write Item with wait time 159 | * @param {Object} batchWriteParams - Batch write params 160 | * @param {Integer} waitTime - wait time in milliseconds 161 | * @return {Promise} 162 | */ 163 | batchWriteItemAfterWait: async function(batchWriteParams, waitTime) { 164 | const oThis = this; 165 | 166 | return new Promise(function(resolve) { 167 | setTimeout(async function() { 168 | let r = await oThis 169 | .ic() 170 | .getInstanceFor(coreConstant.icNameSpace, 'libDynamoDBBase') 171 | .queryDdb(oThis.methodName, oThis.serviceType, batchWriteParams); 172 | resolve(r); 173 | }, waitTime); 174 | }); 175 | } 176 | }; 177 | 178 | Object.assign(DDBServiceBatchWriteItem.prototype, batchWritePrototype); 179 | DDBServiceBatchWriteItem.prototype.constructor = batchWritePrototype; 180 | 181 | InstanceComposer.registerAsShadowableClass( 182 | DDBServiceBatchWriteItem, 183 | coreConstant.icNameSpace, 184 | 'DDBServiceBatchWriteItem' 185 | ); 186 | 187 | module.exports = DDBServiceBatchWriteItem; 188 | -------------------------------------------------------------------------------- /services/dynamodb/CreateTableMigration.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * DynamoDB create table migration having multiple services 5 | * 1. Create table 6 | * 2. Check active table status 7 | * 2. Enable continuous back up 8 | * 3. Enable auto scaling 9 | * 10 | * @module services/dynamodb/CreateTableMigration 11 | * 12 | */ 13 | 14 | const rootPrefix = '../..', 15 | DDBServiceBaseKlass = require(rootPrefix + '/services/dynamodb/Base'), 16 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 17 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 18 | OSTBase = require('@ostdotcom/base'), 19 | coreConstant = require(rootPrefix + '/config/coreConstant'); 20 | 21 | const InstanceComposer = OSTBase.InstanceComposer; 22 | 23 | require(rootPrefix + '/services/autoScale/api'); 24 | 25 | /** 26 | * Constructor for TableExist service class 27 | * 28 | * @params {Object} autoScalingObject - auto scaling Object 29 | * @params {Object} params - params 30 | * @params {Object} params.createTableConfig - create table configurations 31 | * @params {Object} params.autoScalingConfig - scaling params 32 | * @params {Object} params.autoScalingConfig.registerScalableTargetWrite - register Scalable Target write configurations 33 | * @params {Object} params.autoScalingConfig.registerScalableTargetRead - register Scalable Target read configurations 34 | * @params {Object} params.autoScalingConfig.putScalingPolicyWrite- Put scaling policy write configurations 35 | * @params {Object} params.autoScalingConfig.putScalingPolicyRead - Put scaling policy read configurations 36 | * @params {Object} params.autoScalingConfig.globalSecondaryIndex - Auto Scaling configuration of Global Secondary Indexes 37 | * 38 | * @constructor 39 | */ 40 | const DDBServiceCreateTableMigration = function(params, serviceType) { 41 | const oThis = this; 42 | oThis.autoScalingObject = oThis.ic().getInstanceFor(coreConstant.icNameSpace, 'autoScaleApiService'); 43 | oThis.createTableConfig = params.createTableConfig; 44 | //oThis.updateContinuousBackupConfig = params.updateContinuousBackupConfig; 45 | oThis.autoScalingConfig = params.autoScalingConfig; 46 | logger.debug( 47 | '\nparams.createTableConfig', 48 | params.createTableConfig, 49 | '\nparams.autoScalingConfig', 50 | params.autoScalingConfig 51 | ); 52 | oThis.shouldAutoScale = !!oThis.autoScalingObject; 53 | oThis.serviceType = serviceType; 54 | 55 | DDBServiceBaseKlass.call(oThis, 'createTableMigration', params, serviceType); 56 | }; 57 | 58 | DDBServiceCreateTableMigration.prototype = Object.create(DDBServiceBaseKlass.prototype); 59 | 60 | const CreateTableMigrationPrototype = { 61 | /** 62 | * Validation of params 63 | * 64 | * @return {result} 65 | * 66 | */ 67 | validateParams: function() { 68 | const oThis = this, 69 | configStrategy = oThis.ic().configStrategy, 70 | baseValidationResponse = DDBServiceBaseKlass.prototype.validateParams.call(oThis); 71 | if (baseValidationResponse.isFailure()) return baseValidationResponse; 72 | 73 | if (!oThis.params.createTableConfig) { 74 | return responseHelper.error({ 75 | internal_error_identifier: 'l_dy_ctm_validateParams_2', 76 | api_error_identifier: 'invalid_create_table_config', 77 | debug_options: {}, 78 | error_config: coreConstant.ERROR_CONFIG 79 | }); 80 | } 81 | 82 | // if (!oThis.params.updateContinuousBackupConfig) { 83 | // return responseHelper.error('l_dy_ctm_validateParams_3', 'updateContinuousBackupConfig config is mandatory'); 84 | // } 85 | 86 | if (configStrategy.storage.enableAutoscaling == 1) { 87 | if (oThis.autoScalingObject.constructor.name !== 'AutoScaleApiService') { 88 | return responseHelper.error({ 89 | internal_error_identifier: 'l_dy_ctm_validateParams_1', 90 | api_error_identifier: 'invalid_auto_scale_object', 91 | debug_options: {}, 92 | error_config: coreConstant.ERROR_CONFIG 93 | }); 94 | } 95 | 96 | if (!oThis.params.autoScalingConfig) { 97 | return responseHelper.error({ 98 | internal_error_identifier: 'l_dy_ctm_validateParams_4', 99 | api_error_identifier: 'invalid_auto_scale_config', 100 | debug_options: {}, 101 | error_config: coreConstant.ERROR_CONFIG 102 | }); 103 | } 104 | 105 | if (!oThis.params.autoScalingConfig.registerScalableTargetWrite) { 106 | return responseHelper.error({ 107 | internal_error_identifier: 'l_dy_ctm_validateParams_5', 108 | api_error_identifier: 'invalid_register_scalable_target_write', 109 | debug_options: {}, 110 | error_config: coreConstant.ERROR_CONFIG 111 | }); 112 | } 113 | 114 | if (!oThis.params.autoScalingConfig.registerScalableTargetRead) { 115 | return responseHelper.error({ 116 | internal_error_identifier: 'l_dy_ctm_validateParams_6', 117 | api_error_identifier: 'invalid_register_scalable_target_read', 118 | debug_options: {}, 119 | error_config: coreConstant.ERROR_CONFIG 120 | }); 121 | } 122 | 123 | if (!oThis.params.autoScalingConfig.putScalingPolicyWrite) { 124 | return responseHelper.error({ 125 | internal_error_identifier: 'l_dy_ctm_validateParams_7', 126 | api_error_identifier: 'invalid_put_scaling_policy_write', 127 | debug_options: {}, 128 | error_config: coreConstant.ERROR_CONFIG 129 | }); 130 | } 131 | 132 | if (!oThis.params.autoScalingConfig.putScalingPolicyRead) { 133 | return responseHelper.error({ 134 | internal_error_identifier: 'l_dy_ctm_validateParams_8', 135 | api_error_identifier: 'invalid_put_scaling_policy_read', 136 | debug_options: {}, 137 | error_config: coreConstant.ERROR_CONFIG 138 | }); 139 | } 140 | } else { 141 | logger.warn('AutoScale Object is not provided. Auto Scaling will not be done for the same'); 142 | } 143 | 144 | return responseHelper.successWithData({}); 145 | }, 146 | 147 | /** 148 | * run create table migration 149 | * 150 | * @params {Object} params - Parameters 151 | * 152 | * @return {Promise} true/false 153 | * 154 | */ 155 | // TODO Refactor to small methods 156 | executeDdbRequest: function() { 157 | const oThis = this, 158 | ddbObject = oThis.ic().getInstanceFor(coreConstant.icNameSpace, 'libDynamoDBBase'), 159 | configStrategy = oThis.ic().configStrategy; 160 | 161 | return new Promise(async function(onResolve) { 162 | logger.info('Creating table..'); 163 | const createTableResponse = await ddbObject.queryDdb('createTable', oThis.serviceType, oThis.createTableConfig); 164 | if (createTableResponse.isFailure()) { 165 | return onResolve(createTableResponse); 166 | } 167 | 168 | const roleARN = createTableResponse.data.TableDescription.TableArn, 169 | gsiArray = createTableResponse.data.TableDescription.GlobalSecondaryIndexes || [], 170 | tableName = oThis.createTableConfig.TableName, 171 | waitForTableExistsParams = { TableName: tableName }; 172 | logger.debug('Table arn :', roleARN); 173 | 174 | logger.info('Waiting for table creation..'); 175 | const waitForTableExistsResponse = await ddbObject.queryDdb( 176 | 'waitFor', 177 | oThis.serviceType, 178 | 'tableExists', 179 | waitForTableExistsParams 180 | ); 181 | if (waitForTableExistsResponse.isFailure()) { 182 | return onResolve(waitForTableExistsResponse); 183 | } 184 | logger.info(tableName + ' Table created..'); 185 | 186 | if (configStrategy.storage.enableAutoscaling == 1) { 187 | oThis.autoScalingConfig.registerScalableTargetWrite.RoleARN = roleARN; 188 | oThis.autoScalingConfig.registerScalableTargetRead.RoleARN = roleARN; 189 | 190 | logger.info('Register auto scaling read/write target started..'); 191 | let registerAutoScalePromiseArray = [], 192 | putAutoScalePolicyArray = []; 193 | // registerAutoScale for table 194 | registerAutoScalePromiseArray.push( 195 | oThis.autoScalingObject.registerScalableTarget(oThis.autoScalingConfig.registerScalableTargetWrite) 196 | ); 197 | registerAutoScalePromiseArray.push( 198 | oThis.autoScalingObject.registerScalableTarget(oThis.autoScalingConfig.registerScalableTargetRead) 199 | ); 200 | 201 | // registerAutoScale for index 202 | for (let index = 0; index < gsiArray.length; index++) { 203 | let gsi = gsiArray[index], 204 | indexName = gsi.IndexName, 205 | indexArn = gsi.IndexArn, 206 | gsiParamObject = oThis.autoScalingConfig.globalSecondaryIndex[indexName]; 207 | 208 | // Ignore if one of GSI auto scale config is not passed 209 | // In that case default read/write of GSI capacity will be used 210 | if (!gsiParamObject) continue; 211 | 212 | gsiParamObject.registerScalableTargetWrite.RoleARN = indexArn; 213 | gsiParamObject.registerScalableTargetRead.RoleARN = indexArn; 214 | 215 | registerAutoScalePromiseArray.push( 216 | oThis.autoScalingObject.registerScalableTarget(gsiParamObject.registerScalableTargetWrite) 217 | ); 218 | registerAutoScalePromiseArray.push( 219 | oThis.autoScalingObject.registerScalableTarget(gsiParamObject.registerScalableTargetRead) 220 | ); 221 | } 222 | 223 | const registerAutoScalePromiseResponse = await Promise.all(registerAutoScalePromiseArray); 224 | 225 | for (let index = 0; index < registerAutoScalePromiseResponse.length; index++) { 226 | if (registerAutoScalePromiseResponse[index].isFailure()) { 227 | return onResolve(registerAutoScalePromiseResponse[index]); 228 | } 229 | } 230 | logger.info('Register auto scaling read/write target done.'); 231 | 232 | logger.info('Putting auto scale read/write policy..'); 233 | // putAutoScalePolicy For Table 234 | putAutoScalePolicyArray.push( 235 | oThis.autoScalingObject.putScalingPolicy(oThis.autoScalingConfig.putScalingPolicyWrite) 236 | ); 237 | putAutoScalePolicyArray.push( 238 | oThis.autoScalingObject.putScalingPolicy(oThis.autoScalingConfig.putScalingPolicyRead) 239 | ); 240 | 241 | // putAutoScalePolicy For index 242 | for (let index = 0; index < gsiArray.length; index++) { 243 | let gsi = gsiArray[index], 244 | indexName = gsi.IndexName, 245 | gsiParamObject = oThis.autoScalingConfig.globalSecondaryIndex[indexName]; 246 | 247 | // Ignore if one of GSI auto scale config is not passed 248 | // In that case default read/write of GSI capacity will be used 249 | if (!gsiParamObject) continue; 250 | 251 | putAutoScalePolicyArray.push(oThis.autoScalingObject.putScalingPolicy(gsiParamObject.putScalingPolicyWrite)); 252 | putAutoScalePolicyArray.push(oThis.autoScalingObject.putScalingPolicy(gsiParamObject.putScalingPolicyRead)); 253 | } 254 | 255 | const putAutoScalePolicyPromiseResponse = await Promise.all(putAutoScalePolicyArray); 256 | 257 | for (let index = 0; index < putAutoScalePolicyPromiseResponse.length; index++) { 258 | if (putAutoScalePolicyPromiseResponse[index].isFailure()) { 259 | return onResolve(putAutoScalePolicyPromiseResponse[index]); 260 | } 261 | } 262 | 263 | logger.info('Putting auto scale read/write policy done.'); 264 | } 265 | 266 | const describeTableParams = { TableName: tableName }, 267 | describeTableResponse = await ddbObject.queryDdb('describeTable', oThis.serviceType, describeTableParams); 268 | 269 | onResolve(describeTableResponse); 270 | }); 271 | } 272 | }; 273 | 274 | Object.assign(DDBServiceCreateTableMigration.prototype, CreateTableMigrationPrototype); 275 | DDBServiceCreateTableMigration.prototype.constructor = DDBServiceCreateTableMigration; 276 | 277 | InstanceComposer.registerAsShadowableClass( 278 | DDBServiceCreateTableMigration, 279 | coreConstant.icNameSpace, 280 | 'DDBServiceCreateTableMigration' 281 | ); 282 | 283 | module.exports = DDBServiceCreateTableMigration; 284 | -------------------------------------------------------------------------------- /services/dynamodb/RetryQuery.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const rootPrefix = '../..', 4 | base = require(rootPrefix + '/services/dynamodb/Base'), 5 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 6 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 7 | OSTBase = require('@ostdotcom/base'), 8 | coreConstant = require(rootPrefix + '/config/coreConstant'); 9 | 10 | const InstanceComposer = OSTBase.InstanceComposer; 11 | 12 | /** 13 | * Constructor for RetryQuery service class 14 | * 15 | * @param {Object} params - Parameters 16 | * @param {String} queryType - Type of Query (PutItem, UpdateItem, DeleteItem, Query, Scan) 17 | * @param {Number} retryCount - Retry count for ProvisionedThroughputExceededException exception (optional) 18 | * @param {String} serviceType - type of service supported 19 | * 20 | * @constructor 21 | */ 22 | const DDBServiceRetryQuery = function(params, queryType, retryCount, serviceType) { 23 | const oThis = this; 24 | oThis.serviceType = serviceType; 25 | if (retryCount) { 26 | oThis.attemptToPerformCount = retryCount + 1; 27 | } else { 28 | let configStrategies = oThis.ic().configStrategy; 29 | oThis.attemptToPerformCount = configStrategies.storage.maxRetryCount || coreConstant.defaultRetryCount(); 30 | } 31 | oThis.queryType = queryType; 32 | 33 | base.call(oThis, oThis.queryType, params, serviceType); 34 | }; 35 | 36 | DDBServiceRetryQuery.prototype = Object.create(base.prototype); 37 | 38 | const retryQueryPrototype = { 39 | /** 40 | * Execute dynamoDB request 41 | * 42 | * @return {promise} 43 | * 44 | */ 45 | executeDdbRequest: async function() { 46 | const oThis = this; 47 | 48 | try { 49 | let waitTime = 0, 50 | constantTimeFactor = coreConstant.fixedRetryAfterTime(), 51 | variableTimeFactor = coreConstant.variableRetryAfterTime(), 52 | response, 53 | attemptNo = 1; 54 | 55 | while (attemptNo <= oThis.attemptToPerformCount) { 56 | logger.debug(`dynamodb ${oThis.queryType} attemptNo : ${attemptNo}`); 57 | 58 | response = await oThis.queryAfterWait(oThis.params, waitTime); 59 | 60 | // if success or if error was any other than was ResourceNotFoundException return 61 | // NOTE: Except batch requests, all other retries are already handled by AWS SDK 62 | if (response.isSuccess() || !response.internalErrorCode.includes('ResourceNotFoundException')) { 63 | return response; 64 | } 65 | 66 | logger.warn( 67 | 'DynamoDB ATTEMPT_FAILED TableName: ', 68 | oThis.params.TableName, 69 | 'Query Type: ', 70 | oThis.queryType, 71 | 'attemptNo: ', 72 | attemptNo 73 | ); 74 | 75 | //adjust retry variables 76 | waitTime = constantTimeFactor + attemptNo * variableTimeFactor; 77 | attemptNo += 1; 78 | } 79 | 80 | logger.error( 81 | 'DynamoDB ALL_ATTEMPTS_FAILED TableName: ', 82 | oThis.params.TableName, 83 | 'Query Type: ', 84 | oThis.queryType, 85 | 'attemptToPerformCount: ', 86 | oThis.attemptToPerformCount 87 | ); 88 | 89 | return response; 90 | } catch (err) { 91 | logger.error('services/dynamodb/RetryQuery.js:executeDdbRequest inside catch ', err); 92 | return responseHelper.error({ 93 | internal_error_identifier: 's_dy_ui_executeDdbRequest_1', 94 | api_error_identifier: 'exception', 95 | debug_options: { error: err.message }, 96 | error_config: coreConstant.ERROR_CONFIG 97 | }); 98 | } 99 | }, 100 | 101 | /** 102 | * Query DDB after wait time 103 | * 104 | * @param {Object} queryParams - Query params 105 | * @param {Number} waitTime - wait time in milliseconds 106 | * 107 | * @return {Promise} 108 | */ 109 | queryAfterWait: async function(queryParams, waitTime) { 110 | const oThis = this; 111 | 112 | return new Promise(function(resolve) { 113 | setTimeout(async function() { 114 | let r = await oThis 115 | .ic() 116 | .getInstanceFor(coreConstant.icNameSpace, 'libDynamoDBBase') 117 | .queryDdb(oThis.methodName, oThis.serviceType, queryParams); 118 | resolve(r); 119 | }, waitTime); 120 | }); 121 | } 122 | }; 123 | 124 | Object.assign(DDBServiceRetryQuery.prototype, retryQueryPrototype); 125 | DDBServiceRetryQuery.prototype.constructor = retryQueryPrototype; 126 | 127 | InstanceComposer.registerAsShadowableClass(DDBServiceRetryQuery, coreConstant.icNameSpace, 'DDBServiceRetryQuery'); 128 | 129 | module.exports = DDBServiceRetryQuery; 130 | -------------------------------------------------------------------------------- /services/dynamodb/TableExist.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * DynamoDB service api 5 | * 6 | * @module services/dynamodb/TableExist 7 | * 8 | */ 9 | 10 | const rootPrefix = '../..', 11 | DDBServiceBaseKlass = require(rootPrefix + '/services/dynamodb/Base'), 12 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 13 | OSTBase = require('@ostdotcom/base'), 14 | coreConstant = require(rootPrefix + '/config/coreConstant'); 15 | 16 | const InstanceComposer = OSTBase.InstanceComposer; 17 | 18 | /** 19 | * Constructor for TableExist service class 20 | * 21 | * @params {Object} params - TableExist configurations 22 | * @params {String} TableName - name of table 23 | * 24 | * @constructor 25 | */ 26 | const DDBServiceTableExist = function(params, serviceType) { 27 | const oThis = this; 28 | 29 | DDBServiceBaseKlass.call(oThis, 'describeTable', params, serviceType); 30 | }; 31 | 32 | DDBServiceTableExist.prototype = Object.create(DDBServiceBaseKlass.prototype); 33 | 34 | const TableExistPrototype = { 35 | /** 36 | * Validation of params 37 | * 38 | * @return {result} 39 | * 40 | */ 41 | validateParams: function() { 42 | const oThis = this, 43 | baseValidationResponse = DDBServiceBaseKlass.prototype.validateParams.call(oThis); 44 | if (baseValidationResponse.isFailure()) return baseValidationResponse; 45 | 46 | if (!oThis.params.TableName) 47 | return responseHelper.error({ 48 | internal_error_identifier: 'l_dy_te_validateParams_1', 49 | api_error_identifier: 'invalid_table_name', 50 | debug_options: {}, 51 | error_config: coreConstant.ERROR_CONFIG 52 | }); 53 | 54 | return responseHelper.successWithData({}); 55 | }, 56 | 57 | /** 58 | * Check if Table exists using describe table 59 | * 60 | * @params {object} params - Parameters 61 | * 62 | * @return {Promise} true/false 63 | * 64 | */ 65 | executeDdbRequest: function() { 66 | const oThis = this; 67 | return new Promise(async function(onResolve) { 68 | const describeTableResponse = await oThis 69 | .ic() 70 | .getInstanceFor(coreConstant.icNameSpace, 'libDynamoDBBase') 71 | .queryDdb('describeTable', 'raw', oThis.params); 72 | if (describeTableResponse.isFailure()) { 73 | return onResolve(responseHelper.successWithData({ response: false, status: 'DELETED' })); 74 | } 75 | const tableStatus = describeTableResponse.data.Table.TableStatus || ''; 76 | return onResolve(responseHelper.successWithData({ response: tableStatus === 'ACTIVE', status: tableStatus })); 77 | }); 78 | } 79 | }; 80 | 81 | Object.assign(DDBServiceTableExist.prototype, TableExistPrototype); 82 | DDBServiceTableExist.prototype.constructor = DDBServiceTableExist; 83 | 84 | InstanceComposer.registerAsShadowableClass(DDBServiceTableExist, coreConstant.icNameSpace, 'DDBServiceTableExist'); 85 | 86 | module.exports = DDBServiceTableExist; 87 | -------------------------------------------------------------------------------- /services/dynamodb/WaitFor.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * DynamoDB wait for service 5 | * 6 | * @module services/dynamodb/WaitFor 7 | * 8 | */ 9 | 10 | const rootPrefix = '../..', 11 | base = require(rootPrefix + '/services/dynamodb/Base'), 12 | responseHelper = require(rootPrefix + '/lib/formatter/response'), 13 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 14 | OSTBase = require('@ostdotcom/base'), 15 | coreConstant = require(rootPrefix + '/config/coreConstant'); 16 | 17 | const InstanceComposer = OSTBase.InstanceComposer; 18 | 19 | /** 20 | * Constructor for wait for service class 21 | * @param {Object} params - Parameters 22 | * @param {String} waitForMethod - wait for method 23 | * @param {String} serviceType - type of service supported 24 | * 25 | * @constructor 26 | */ 27 | const WaitFor = function(waitForMethod, params, serviceType) { 28 | const oThis = this; 29 | oThis.waitForMethod = waitForMethod; 30 | base.call(oThis, 'waitFor', params, serviceType); 31 | }; 32 | 33 | WaitFor.prototype = Object.create(base.prototype); 34 | 35 | const waitForPrototype = { 36 | /** 37 | * Validation of params 38 | * 39 | * @return {*} 40 | */ 41 | validateParams: function() { 42 | const oThis = this, 43 | validationResponse = base.prototype.validateParams.call(oThis); 44 | if (validationResponse.isFailure()) return validationResponse; 45 | 46 | if (!oThis.waitForMethod) 47 | return responseHelper.error({ 48 | internal_error_identifier: 'l_dy_wf_validateParams_1', 49 | api_error_identifier: 'invalid_wait_for_method', 50 | debug_options: {}, 51 | error_config: coreConstant.ERROR_CONFIG 52 | }); 53 | 54 | return responseHelper.successWithData({}); 55 | }, 56 | 57 | /** 58 | * Execute dynamoDB request 59 | * 60 | * @return {promise} 61 | * 62 | */ 63 | executeDdbRequest: async function() { 64 | const oThis = this; 65 | 66 | try { 67 | const r = await oThis 68 | .ic() 69 | .getInstanceFor(coreConstant.icNameSpace, 'libDynamoDBBase') 70 | .queryDdb(oThis.methodName, 'raw', oThis.waitForMethod, oThis.params); 71 | return r; 72 | } catch (err) { 73 | logger.error('services/dynamodb/Base.js:executeDdbRequest inside catch ', err); 74 | return responseHelper.error({ 75 | internal_error_identifier: 's_dy_b_executeDdbRequest_1', 76 | api_error_identifier: 'exception', 77 | debug_options: { error: err.message }, 78 | error_config: coreConstant.ERROR_CONFIG 79 | }); 80 | } 81 | } 82 | }; 83 | 84 | Object.assign(WaitFor.prototype, waitForPrototype); 85 | WaitFor.prototype.constructor = waitForPrototype; 86 | 87 | InstanceComposer.registerAsShadowableClass(WaitFor, coreConstant.icNameSpace, 'DDBServiceWaitFor'); 88 | 89 | module.exports = WaitFor; 90 | -------------------------------------------------------------------------------- /services/dynamodb/api.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * DynamoDB service api 5 | * 6 | * @module services/dynamodb/api 7 | * 8 | */ 9 | 10 | const rootPrefix = '../..', 11 | OSTBase = require('@ostdotcom/base'), 12 | coreConstant = require(rootPrefix + '/config/coreConstant'); 13 | 14 | const InstanceComposer = OSTBase.InstanceComposer; 15 | 16 | require(rootPrefix + '/lib/dynamodb/base'); 17 | require(rootPrefix + '/services/dynamodb/Base'); 18 | require(rootPrefix + '/services/dynamodb/TableExist'); 19 | require(rootPrefix + '/services/dynamodb/WaitFor'); 20 | require(rootPrefix + '/services/dynamodb/CreateTableMigration'); 21 | require(rootPrefix + '/services/dynamodb/BatchGet'); 22 | require(rootPrefix + '/services/dynamodb/BatchWrite'); 23 | require(rootPrefix + '/services/dynamodb/RetryQuery'); 24 | 25 | /** 26 | * Constructor for DynamoDB api service class 27 | * 28 | * @params {Object} params - DynamoDB connection configurations 29 | * 30 | * @constructor 31 | */ 32 | const DynamoDBApiService = function() {}; 33 | 34 | DynamoDBApiService.prototype = { 35 | /** 36 | * Create table 37 | * 38 | * @params {Object} params - Parameters 39 | * 40 | * @return {promise} 41 | * 42 | */ 43 | createTable: function(params) { 44 | const oThis = this, 45 | DDBServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceBase'), 46 | createTableObject = new DDBServiceBaseKlass('createTable', params, 'raw'); 47 | return createTableObject.perform(); 48 | }, 49 | 50 | /** 51 | * Run table migration with added features 52 | * 1. active status check 53 | * 2. enabling continuous back up 54 | * 3. enabling auto scaling 55 | * 56 | * @params {Object} autoScaleObject - Auto Scaling Object to configure table 57 | * @params {Object} params - Params as JSON object having further params 58 | * @params {Object} params.createTableConfig - Create table configurations params as JSON object 59 | * @params {Object} params.updateContinuousBackupConfig - Update Continuous Backup configurations params as JSON object 60 | * @params {Object} params.autoScalingConfig - Auto scaling params as JSON Object having further params as JSON object 61 | * @params {Object} params.autoScalingConfig.registerScalableTargetWrite - Register Scalable Target write configurations params as JSON object 62 | * @params {Object} params.autoScalingConfig.registerScalableTargetRead - Register Scalable Target read configurations params as JSON object 63 | * @params {Object} params.autoScalingConfig.putScalingPolicyWrite- Put scaling policy write configurations params as JSON object 64 | * @params {Object} params.autoScalingConfig.putScalingPolicyRead - Put scaling policy read configurations params as JSON object 65 | * @return {promise} 66 | * 67 | */ 68 | createTableMigration: function(params) { 69 | const oThis = this, 70 | CreateTableMigrationServiceKlass = oThis 71 | .ic() 72 | .getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceCreateTableMigration'), 73 | createTableMigrationObject = new CreateTableMigrationServiceKlass(params, 'raw'); 74 | return createTableMigrationObject.perform(); 75 | }, 76 | 77 | /** 78 | * Update table 79 | * 80 | * @params {Object} params - Params as per dynamo db updateTable api params 81 | * 82 | * @return {promise} 83 | * 84 | */ 85 | updateTable: function(params) { 86 | const oThis = this, 87 | DDBServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceBase'), 88 | updateTableObject = new DDBServiceBaseKlass('updateTable', params, 'raw'); 89 | return updateTableObject.perform(); 90 | }, 91 | 92 | /** 93 | * Describe table 94 | * 95 | * @params {Object} params - Params as per dynamo db describeTable api params 96 | * 97 | * @return {promise} 98 | * 99 | */ 100 | describeTable: function(params) { 101 | const oThis = this, 102 | DDBServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceBase'), 103 | describeTableObject = new DDBServiceBaseKlass('describeTable', params, 'raw'); 104 | return describeTableObject.perform(); 105 | }, 106 | 107 | /** 108 | * List table 109 | * 110 | * @params {Object} params - Params as per dynamo db listTables api params 111 | * 112 | * @return {promise} 113 | * 114 | */ 115 | listTables: function(params) { 116 | const oThis = this, 117 | DDBServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceBase'), 118 | listTablesObject = new DDBServiceBaseKlass('listTables', params, 'raw'); 119 | return listTablesObject.perform(); 120 | }, 121 | 122 | /** 123 | * Enables or disables point in time recovery for the specified table 124 | * 125 | * @params {Object} params - Params as per dynamo db updateContinuousBackup api params 126 | * 127 | * @return {promise} 128 | * 129 | */ 130 | updateContinuousBackups: function(params) { 131 | const oThis = this, 132 | DDBServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceBase'), 133 | updateContinuousBackupObject = new DDBServiceBaseKlass('updateContinuousBackups', params, 'raw'); 134 | return updateContinuousBackupObject.perform(); 135 | }, 136 | 137 | /** 138 | * Delete table 139 | * 140 | * @params {Object} params - Params as per dynamo db deleteTable api params 141 | * 142 | * @return {promise} 143 | * 144 | */ 145 | deleteTable: function(params) { 146 | const oThis = this, 147 | DDBServiceBaseKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceBase'), 148 | deleteTableObject = new DDBServiceBaseKlass('deleteTable', params, 'raw'); 149 | return deleteTableObject.perform(); 150 | }, 151 | 152 | /** 153 | * Batch get 154 | * 155 | * @params {Object} params - Params as per dynamo db batchGetItem api params 156 | * @params {Integer} unprocessedKeysRetryCount - Retry count for unprocessed keys 157 | * 158 | * @return {promise} 159 | * 160 | */ 161 | batchGetItem: function(params, unprocessedKeysRetryCount) { 162 | const oThis = this, 163 | BatchGetItemKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceBatchGetItem'), 164 | bathGetObject = new BatchGetItemKlass(params, unprocessedKeysRetryCount, 'dax'); 165 | return bathGetObject.perform(); 166 | }, 167 | 168 | /** 169 | * Batch write 170 | * 171 | * @params {Object} params - Params as per dynamo db batchWriteItem api params 172 | * @params {Integer} unprocessedItemsRetryCount - Retry count for unprocessed Items 173 | * 174 | * @return {promise} 175 | * 176 | */ 177 | batchWriteItem: function(params, unprocessedItemsRetryCount) { 178 | const oThis = this, 179 | BatchWriteItemKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceBatchWriteItem'), 180 | batchWriteObject = new BatchWriteItemKlass(params, unprocessedItemsRetryCount, 'dax'); 181 | return batchWriteObject.perform(); 182 | }, 183 | 184 | /** 185 | * Query dynamodb 186 | * 187 | * @params {Object} params - Params as per dynamo db query api params 188 | * 189 | * @return {promise} 190 | * 191 | */ 192 | query: function(params, retryCount) { 193 | const oThis = this, 194 | retryQueryKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceRetryQuery'), 195 | queryObject = new retryQueryKlass(params, 'query', retryCount, 'dax'); 196 | return queryObject.perform(); 197 | }, 198 | 199 | /** 200 | * Scan 201 | * 202 | * @params {Object} params - Params as per dynamo db scan api params 203 | * 204 | * @return {promise} 205 | * 206 | */ 207 | scan: function(params, retryCount) { 208 | const oThis = this, 209 | retryQueryKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceRetryQuery'), 210 | scanObject = new retryQueryKlass(params, 'scan', retryCount, 'dax'); 211 | return scanObject.perform(); 212 | }, 213 | 214 | /** 215 | * Put item 216 | * 217 | * @params {Object} params - Params as per dynamo db putItem api params 218 | * 219 | * @return {promise} 220 | * 221 | */ 222 | putItem: function(params, retryCount) { 223 | const oThis = this, 224 | retryQueryKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceRetryQuery'), 225 | putItemObject = new retryQueryKlass(params, 'putItem', retryCount, 'dax'); 226 | return putItemObject.perform(); 227 | }, 228 | 229 | /** 230 | * Update item 231 | * 232 | * @params {Object} params - Params as per dynamo db updateItem api params 233 | * @params {Integer} retryCount - Retry count for ProvisionedThroughputExceededException exception 234 | * 235 | * @return {promise} 236 | * 237 | */ 238 | updateItem: function(params, retryCount) { 239 | const oThis = this, 240 | retryQueryKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceRetryQuery'), 241 | updateItemObject = new retryQueryKlass(params, 'updateItem', retryCount, 'dax'); 242 | return updateItemObject.perform(); 243 | }, 244 | 245 | /** 246 | * Delete item 247 | * 248 | * @params {Object} params - Params as per dynamo db deleteItem api params 249 | * 250 | * @return {promise} 251 | * 252 | */ 253 | deleteItem: function(params, retryCount) { 254 | const oThis = this, 255 | retryQueryKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceRetryQuery'), 256 | deleteItemObject = new retryQueryKlass(params, 'deleteItem', retryCount, 'dax'); 257 | return deleteItemObject.perform(); 258 | }, 259 | 260 | /** 261 | * Check Table exists 262 | * 263 | * @params {Object} params - Params as per dynamo db tableExists api params 264 | * 265 | * @return {promise} 266 | * 267 | */ 268 | tableExistsUsingWaitFor: function(params) { 269 | const oThis = this, 270 | WaitForServiceKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceWaitFor'), 271 | tableExistsObject = new WaitForServiceKlass('tableExists', params, 'raw'); 272 | return tableExistsObject.perform(); 273 | }, 274 | 275 | /** 276 | * Check Table not exists 277 | * 278 | * @params {Object} params - Params as per dynamo db tableNotExists api params 279 | * 280 | * @return {promise} 281 | * 282 | */ 283 | tableNotExistsUsingWaitFor: function(params) { 284 | const oThis = this, 285 | WaitForServiceKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceWaitFor'), 286 | tableExistsObject = new WaitForServiceKlass('tableNotExists', params, 'raw'); 287 | return tableExistsObject.perform(); 288 | }, 289 | 290 | /** 291 | * Check if Table exists using describe table 292 | * 293 | * @params {Object} params - Params as per dynamo db tableExists api params 294 | * 295 | * @return {promise} 296 | * 297 | */ 298 | checkTableExist: function(params) { 299 | const oThis = this, 300 | TableExistServiceApiKlass = oThis.ic().getShadowedClassFor(coreConstant.icNameSpace, 'DDBServiceTableExist'), 301 | tableExistObject = new TableExistServiceApiKlass(params, 'raw'); 302 | return tableExistObject.perform(); 303 | } 304 | }; 305 | 306 | DynamoDBApiService.prototype.constructor = DynamoDBApiService; 307 | 308 | InstanceComposer.registerAsObject(DynamoDBApiService, coreConstant.icNameSpace, 'dynamoDBApiService', true); 309 | 310 | module.exports = DynamoDBApiService; 311 | -------------------------------------------------------------------------------- /tests/mocha/services/auto_scale/create_table_migration.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | const rootPrefix = '../../../..', 5 | OStStorage = require(rootPrefix + '/index'), 6 | coreConstant = require(rootPrefix + '/config/coreConstant'), 7 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 8 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 9 | helper = require(rootPrefix + '/tests/mocha/services/auto_scale/helper'); 10 | 11 | describe('Create Table', function() { 12 | let dynamodbApiObject = null; 13 | let autoScaleObj = null; 14 | 15 | before(async function() { 16 | this.timeout(1000000); 17 | // create ostStorage 18 | const ostStorage = OStStorage.getInstance(testConstants.CONFIG_STRATEGIES); 19 | dynamodbApiObject = ostStorage.dynamoDBService; 20 | autoScaleObj = ostStorage.ic.getInstanceFor(coreConstant.icNameSpace, 'autoScaleApiService'); 21 | 22 | const params = { 23 | TableName: testConstants.dummyTestTableName 24 | }; 25 | 26 | const checkTableExistsResponse = await dynamodbApiObject.checkTableExist(params); 27 | 28 | if (checkTableExistsResponse.data.response === true) { 29 | logger.log(testConstants.dummyTestTableName, 'Table exists . Deleting it....'); 30 | await helper.deleteTable(dynamodbApiObject, params, true); 31 | 32 | logger.info('Waiting for table to get deleted'); 33 | await helper.waitForTableToGetDeleted(dynamodbApiObject, params); 34 | logger.info('Table got deleted'); 35 | } else { 36 | logger.log(testConstants.dummyTestTableName, 'Table does not exist.'); 37 | } 38 | }); 39 | 40 | it('should create table successfully', async function() { 41 | this.timeout(1000000); 42 | // build create table params 43 | const response = await helper.createTableMigration(dynamodbApiObject, autoScaleObj); 44 | assert.isTrue(response.isSuccess(), 'createTableMigration failed'); 45 | }); 46 | 47 | after(async function() { 48 | this.timeout(100000); 49 | const params = { 50 | TableName: testConstants.dummyTestTableName 51 | }; 52 | await helper.deleteTable(dynamodbApiObject, params, true); 53 | 54 | logger.debug('Create Table Mocha Tests Complete'); 55 | 56 | logger.log('Waiting for Table get deleted...............'); 57 | await helper.waitForTableToGetDeleted(dynamodbApiObject, params); 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /tests/mocha/services/auto_scale/delete_scaling_policy.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // Load external packages 4 | const Chai = require('chai'), 5 | assert = Chai.assert; 6 | 7 | // Load dependencies package 8 | const rootPrefix = '../../../..', 9 | OSTStorage = require(rootPrefix + '/index'), 10 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 11 | coreConstant = require(rootPrefix + '/config/coreConstant'), 12 | helper = require(rootPrefix + '/tests/mocha/services/auto_scale/helper'), 13 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'); 14 | 15 | const ostStorage = OSTStorage.getInstance(testConstants.CONFIG_STRATEGIES), 16 | autoScaleObj = ostStorage.ic.getInstanceFor(coreConstant.icNameSpace, 'autoScaleApiService'), 17 | dynamodbApiObject = ostStorage.dynamoDBService; 18 | 19 | let resourceId = 'table/' + testConstants.dummyTestTableName, 20 | roleARN = null; 21 | 22 | const createTestCasesForOptions = function(optionsDesc, options, toAssert) { 23 | optionsDesc = optionsDesc || ''; 24 | 25 | options = options || { invalid_policy_name: false }; 26 | 27 | it(optionsDesc, async function() { 28 | this.timeout(100000); 29 | 30 | let policyName = testConstants.dummyTestTableName + '-scaling-policy'; 31 | if (options.invalid_policy_name) { 32 | policyName = 'invalidPolicyId'; 33 | } else { 34 | const scalableTargetParams = { 35 | ResourceId: resourceId /* required */, 36 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 37 | ServiceNamespace: 'dynamodb' /* required */, 38 | MaxCapacity: 15, 39 | MinCapacity: 1, 40 | RoleARN: roleARN 41 | }; 42 | const registerScalableTargetResponse = await autoScaleObj.registerScalableTarget(scalableTargetParams); 43 | assert.equal(registerScalableTargetResponse.isSuccess(), true, 'registerScalableTarget failed'); 44 | 45 | const scalingPolicy = { 46 | ServiceNamespace: 'dynamodb', 47 | ResourceId: 'table/' + testConstants.dummyTestTableName, 48 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 49 | PolicyName: policyName, 50 | PolicyType: 'TargetTrackingScaling', 51 | TargetTrackingScalingPolicyConfiguration: { 52 | PredefinedMetricSpecification: { 53 | PredefinedMetricType: 'DynamoDBWriteCapacityUtilization' 54 | }, 55 | ScaleOutCooldown: 60, 56 | ScaleInCooldown: 60, 57 | TargetValue: 70.0 58 | } 59 | }; 60 | const putScalingPolicyResponse = await autoScaleObj.putScalingPolicy(scalingPolicy); 61 | assert.equal(putScalingPolicyResponse.isSuccess(), true, 'putScalingPolicy failed'); 62 | } 63 | 64 | const params = { 65 | PolicyName: testConstants.dummyTestTableName + '-scaling-policy', 66 | ResourceId: 'table/' + testConstants.dummyTestTableName, 67 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 68 | ServiceNamespace: 'dynamodb' 69 | }; 70 | 71 | const response = await autoScaleObj.deleteScalingPolicy(params); 72 | 73 | logger.log(response); 74 | assert.equal(response.isSuccess(), toAssert, 'describeScalingPolicies failed'); 75 | }); 76 | }; 77 | 78 | describe('services/autoScale/api#deleteScalingPolicy', function() { 79 | before(async function() { 80 | this.timeout(1000000); 81 | 82 | const returnObject = await helper.createTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 83 | roleARN = returnObject.role_arn; 84 | }); 85 | 86 | createTestCasesForOptions('Delete scaling policy happy case', null, true); 87 | 88 | createTestCasesForOptions('Delete scaling policy having policy name case', { invalid_policy_name: true }, false); 89 | 90 | after(async function() { 91 | this.timeout(1000000); 92 | await helper.cleanTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 93 | }); 94 | }); 95 | -------------------------------------------------------------------------------- /tests/mocha/services/auto_scale/deregister_scalable_target.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // Load external packages 4 | const Chai = require('chai'), 5 | assert = Chai.assert; 6 | 7 | // Load dependencies package 8 | const rootPrefix = '../../../..', 9 | OSTStorage = require(rootPrefix + '/index'), 10 | coreConstant = require(rootPrefix + '/config/coreConstant'), 11 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 12 | helper = require(rootPrefix + '/tests/mocha/services/auto_scale/helper'), 13 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'); 14 | 15 | const ostStorage = OSTStorage.getInstance(testConstants.CONFIG_STRATEGIES), 16 | autoScaleObj = ostStorage.ic.getInstanceFor(coreConstant.icNameSpace, 'autoScaleApiService'), 17 | dynamodbApiObject = ostStorage.dynamoDBService; 18 | 19 | let resourceId = 'table/' + testConstants.dummyTestTableName, 20 | roleARN = null; 21 | 22 | const createTestCasesForOptions = function(optionsDesc, options, toAssert) { 23 | optionsDesc = optionsDesc || ''; 24 | 25 | options = options || { invalid_service_name: false }; 26 | 27 | it(optionsDesc, async function() { 28 | this.timeout(100000); 29 | 30 | let resId = resourceId; 31 | if (options.invalid_service_id) { 32 | resId = 'invalidResId'; 33 | } else { 34 | const scalableTargetParams = { 35 | ResourceId: resId /* required */, 36 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 37 | ServiceNamespace: 'dynamodb' /* required */, 38 | MaxCapacity: 15, 39 | MinCapacity: 1, 40 | RoleARN: roleARN 41 | }; 42 | const registerScalableTargetResponse = await autoScaleObj.registerScalableTarget(scalableTargetParams); 43 | assert.equal(registerScalableTargetResponse.isSuccess(), true, 'registerScalableTarget failed'); 44 | } 45 | const params = { 46 | ResourceId: resId, 47 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 48 | ServiceNamespace: 'dynamodb' 49 | }; 50 | const response = await autoScaleObj.deregisterScalableTarget(params); 51 | 52 | logger.log(response); 53 | assert.equal(response.isSuccess(), toAssert, 'deregisterScalableTarget failed'); 54 | }); 55 | }; 56 | 57 | describe('services/autoScale/api#deregisterScalableTarget', function() { 58 | before(async function() { 59 | this.timeout(1000000); 60 | 61 | const returnObject = await helper.createTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 62 | roleARN = returnObject.role_arn; 63 | }); 64 | 65 | createTestCasesForOptions('DeRegister scalable target happy case', null, true); 66 | 67 | createTestCasesForOptions( 68 | 'DeRegister scalable target having invalid res id case', 69 | { invalid_service_id: true }, 70 | false 71 | ); 72 | 73 | after(async function() { 74 | this.timeout(1000000); 75 | await helper.cleanTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 76 | }); 77 | }); 78 | -------------------------------------------------------------------------------- /tests/mocha/services/auto_scale/describe_scalable_targets.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // Load external packages 4 | const Chai = require('chai'), 5 | assert = Chai.assert; 6 | 7 | // Load dependencies package 8 | const rootPrefix = '../../../..', 9 | OSTStorage = require(rootPrefix + '/index'), 10 | coreConstant = require(rootPrefix + '/config/coreConstant'), 11 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 12 | helper = require(rootPrefix + '/tests/mocha/services/auto_scale/helper'), 13 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'); 14 | 15 | const ostStorage = OSTStorage.getInstance(testConstants.CONFIG_STRATEGIES), 16 | autoScaleObj = ostStorage.ic.getInstanceFor(coreConstant.icNameSpace, 'autoScaleApiService'), 17 | dynamodbApiObject = ostStorage.dynamoDBService; 18 | 19 | let resourceId = 'table/' + testConstants.dummyTestTableName, 20 | roleARN = null; 21 | 22 | const createTestCasesForOptions = function(optionsDesc, options, toAssert) { 23 | optionsDesc = optionsDesc || ''; 24 | 25 | options = options || { invalid_service_name: false }; 26 | 27 | it(optionsDesc, async function() { 28 | this.timeout(100000); 29 | 30 | let serviceNameSpace = 'dynamodb'; 31 | if (options.invalid_service_name) { 32 | serviceNameSpace = 'invalidResId'; 33 | } 34 | 35 | const scalableTargetParams = { 36 | ResourceId: resourceId /* required */, 37 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 38 | ServiceNamespace: 'dynamodb' /* required */, 39 | MaxCapacity: 15, 40 | MinCapacity: 1, 41 | RoleARN: roleARN 42 | }; 43 | const registerScalableTargetResponse = await autoScaleObj.registerScalableTarget(scalableTargetParams); 44 | assert.equal(registerScalableTargetResponse.isSuccess(), true, 'registerScalableTarget failed'); 45 | 46 | const params = { 47 | ServiceNamespace: serviceNameSpace 48 | }; 49 | 50 | const response = await autoScaleObj.describeScalableTargets(params); 51 | 52 | logger.log(response); 53 | assert.equal(response.isSuccess(), toAssert, 'describeScalableTargets failed'); 54 | }); 55 | }; 56 | 57 | describe('services/autoScale/api#describeScalableTargets', function() { 58 | before(async function() { 59 | this.timeout(1000000); 60 | 61 | const returnObject = await helper.createTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 62 | roleARN = returnObject.role_arn; 63 | }); 64 | 65 | createTestCasesForOptions('Describe scalable targets happy case', null, true); 66 | 67 | createTestCasesForOptions( 68 | 'Describe scalable targets having invalid service name case', 69 | { invalid_service_name: true }, 70 | false 71 | ); 72 | 73 | after(async function() { 74 | this.timeout(1000000); 75 | await helper.cleanTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 76 | }); 77 | }); 78 | -------------------------------------------------------------------------------- /tests/mocha/services/auto_scale/describe_scaling_policies.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // Load external packages 4 | const Chai = require('chai'), 5 | assert = Chai.assert; 6 | 7 | // Load dependencies package 8 | const rootPrefix = '../../../..', 9 | OSTStorage = require(rootPrefix + '/index'), 10 | coreConstant = require(rootPrefix + '/config/coreConstant'), 11 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 12 | helper = require(rootPrefix + '/tests/mocha/services/auto_scale/helper'), 13 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'); 14 | 15 | const ostStorage = OSTStorage.getInstance(testConstants.CONFIG_STRATEGIES), 16 | autoScaleObj = ostStorage.ic.getInstanceFor(coreConstant.icNameSpace, 'autoScaleApiService'), 17 | dynamodbApiObject = ostStorage.dynamoDBService; 18 | 19 | let resourceId = 'table/' + testConstants.dummyTestTableName, 20 | roleARN = null; 21 | 22 | const createTestCasesForOptions = function(optionsDesc, options, toAssert) { 23 | optionsDesc = optionsDesc || ''; 24 | 25 | options = options || { invalid_service_name: false }; 26 | 27 | it(optionsDesc, async function() { 28 | this.timeout(100000); 29 | 30 | let serviceNameSpace = 'dynamodb'; 31 | if (options.invalid_service_name) { 32 | serviceNameSpace = 'invalidResId'; 33 | } 34 | 35 | const scalingPolicy = { 36 | ServiceNamespace: 'dynamodb', 37 | ResourceId: 'table/' + testConstants.dummyTestTableName, 38 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 39 | PolicyName: testConstants.dummyTestTableName + '-scaling-policy', 40 | PolicyType: 'TargetTrackingScaling', 41 | TargetTrackingScalingPolicyConfiguration: { 42 | PredefinedMetricSpecification: { 43 | PredefinedMetricType: 'DynamoDBWriteCapacityUtilization' 44 | }, 45 | ScaleOutCooldown: 60, 46 | ScaleInCooldown: 60, 47 | TargetValue: 70.0 48 | } 49 | }; 50 | const putScalingPolicyResponse = await autoScaleObj.putScalingPolicy(scalingPolicy); 51 | assert.equal(putScalingPolicyResponse.isSuccess(), true, 'putScalingPolicy failed'); 52 | 53 | const params = { 54 | ServiceNamespace: serviceNameSpace 55 | }; 56 | 57 | const response = await autoScaleObj.describeScalingPolicies(params); 58 | 59 | logger.log(response); 60 | assert.equal(response.isSuccess(), toAssert, 'describeScalingPolicies failed'); 61 | }); 62 | }; 63 | 64 | describe('services/autoScale/api#describeScalingPolicies', function() { 65 | before(async function() { 66 | this.timeout(1000000); 67 | 68 | const returnObject = await helper.createTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 69 | roleARN = returnObject.role_arn; 70 | }); 71 | 72 | createTestCasesForOptions('Describe scaling policy happy case', null, true); 73 | 74 | createTestCasesForOptions('Describe scaling policy having invalid name case', { invalid_service_name: true }, false); 75 | 76 | after(async function() { 77 | this.timeout(1000000); 78 | await helper.cleanTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 79 | }); 80 | }); 81 | -------------------------------------------------------------------------------- /tests/mocha/services/auto_scale/helper.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const chai = require('chai'), 4 | assert = chai.assert; 5 | 6 | const rootPrefix = '../../../..', 7 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 8 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 9 | autoScaleHelper = require(rootPrefix + '/lib/autoScale/helper'); 10 | 11 | /** 12 | * Constructor for helper class 13 | * 14 | * @constructor 15 | */ 16 | const helper = function() {}; 17 | 18 | helper.prototype = { 19 | /** 20 | * To wait till response 21 | * @param dynamoDbApiObject 22 | * @param func 23 | * @param params 24 | * @param toAssert 25 | * @param retries 26 | * @return {Promise} 27 | */ 28 | waitTillTableStatusProvided: async function(dynamoDbApiObject, func, params, toAssert, retries) { 29 | const oThis = this, 30 | WAIT = retries ? retries : 30; 31 | let count = WAIT; 32 | let response = null; 33 | while (count > 0) { 34 | response = await oThis.waitTillResponse(dynamoDbApiObject, func, params); 35 | 36 | assert.isTrue(response.isSuccess(), 'Get status table failed'); 37 | 38 | if (response.data.status === toAssert) { 39 | return response; 40 | } 41 | count -= 1; 42 | } 43 | return response; 44 | }, 45 | 46 | /** 47 | * wait till response 48 | * @param dynamodbApiObject 49 | * @param func 50 | * @param params 51 | * @return {Promise} 52 | */ 53 | waitTillResponse: async function(dynamodbApiObject, func, params) { 54 | return new Promise(function(resolve) { 55 | setTimeout(async function() { 56 | let response = await func.call(dynamodbApiObject, params); 57 | resolve(response); 58 | }, 5000); 59 | }); 60 | }, 61 | 62 | /** 63 | * Delete Table Helper method 64 | * 65 | * @params {object} dynamodbApiObject - DynamoDB Api object 66 | * @params {object} params - batch get params 67 | * @params {object} isResultSuccess - expected result 68 | * 69 | * @return {result} 70 | * 71 | */ 72 | deleteTable: async function(dynamodbApiObject, params, isResultSuccess) { 73 | const deleteTableResponse = await dynamodbApiObject.deleteTable(params); 74 | 75 | if (isResultSuccess === true) { 76 | assert.equal(deleteTableResponse.isSuccess(), true); 77 | logger.debug('deleteTableResponse.data.TableDescription', deleteTableResponse.data.TableDescription); 78 | assert.exists(deleteTableResponse.data.TableDescription, params.TableName); 79 | } else { 80 | assert.equal(deleteTableResponse.isSuccess(), false); 81 | } 82 | 83 | return deleteTableResponse; 84 | }, 85 | 86 | /** 87 | * Create Table Helper method 88 | * 89 | * @params {object} dynamodbApiObject - DynamoDB Api object 90 | * @params {object} params - batch get params 91 | * @params {object} isResultSuccess - expected result 92 | * 93 | * @return {result} 94 | * 95 | */ 96 | createTable: async function(dynamodbApiObject, params, isResultSuccess) { 97 | const createTableResponse = await dynamodbApiObject.createTable(params); 98 | 99 | if (isResultSuccess) { 100 | assert.equal(createTableResponse.isSuccess(), true); 101 | assert.exists(createTableResponse.data.TableDescription, params.TableName); 102 | } else { 103 | assert.equal(createTableResponse.isSuccess(), false, 'createTable: successfull, should fail for this case'); 104 | } 105 | return createTableResponse; 106 | }, 107 | 108 | /** 109 | * Wait for table to get deleted 110 | * 111 | * @param dynamodbApiObject 112 | * @param params 113 | */ 114 | waitForTableToGetDeleted: async function(dynamodbApiObject, params) { 115 | const oThis = this; 116 | const response = await dynamodbApiObject.tableNotExistsUsingWaitFor(params); 117 | 118 | assert.isTrue(response.isSuccess(), 'tableNotExists failed'); 119 | }, 120 | 121 | /** 122 | * Wait for table to get created 123 | * 124 | * @param dynamodbApiObject 125 | * @param params 126 | */ 127 | waitForTableToGetCreated: async function(dynamodbApiObject, params) { 128 | const oThis = this; 129 | const response = await dynamodbApiObject.tableExistsUsingWaitFor(params); 130 | 131 | assert.isTrue(response.isSuccess(), 'tableExists failed'); 132 | }, 133 | 134 | /** 135 | * Delete Scaling Policy 136 | * 137 | * @param autoScaleObject 138 | * @param params 139 | * @return {Promise} 140 | */ 141 | deleteScalingPolicy: async function(autoScaleObject, params) { 142 | const oThis = this; 143 | const response = await autoScaleObject.deleteScalingPolicy(params); 144 | 145 | assert.isTrue(response.isSuccess(), 'deleteScalingPolicy failed'); 146 | }, 147 | 148 | /** 149 | * De Register Scalable Target 150 | * 151 | * @param autoScaleObject 152 | * @param params 153 | * @return {Promise} 154 | */ 155 | deregisterScalableTarget: async function(autoScaleObject, params) { 156 | const oThis = this; 157 | const response = await autoScaleObject.deregisterScalableTarget(params); 158 | 159 | assert.isTrue(response.isSuccess(), 'deregisterScalableTarget failed'); 160 | }, 161 | 162 | /** 163 | * Create test case env 164 | * 165 | * @return {Promise} 166 | */ 167 | createTestCaseEnvironment: async function(dynamodbApiObject, autoScaleObj) { 168 | const oThis = this, 169 | params = { 170 | TableName: testConstants.dummyTestTableName 171 | }; 172 | 173 | const checkTableExistsResponse1 = await dynamodbApiObject.checkTableExist(params); 174 | 175 | if (checkTableExistsResponse1.data.response === true) { 176 | logger.log(testConstants.dummyTestTableName, 'Table exists . Deleting it....'); 177 | await oThis.deleteTable(dynamodbApiObject, params, true); 178 | 179 | logger.info('Waiting for table to get deleted'); 180 | await oThis.waitForTableToGetDeleted(dynamodbApiObject, params); 181 | logger.info('Table got deleted'); 182 | } else { 183 | logger.log(testConstants.dummyTestTableName, 'Table does not exist.'); 184 | } 185 | 186 | logger.info('Creating table'); 187 | const createTableResponse = await oThis.createTable(dynamodbApiObject, oThis.getCreateTableParams(), true); 188 | 189 | const roleARN = createTableResponse.data.TableDescription.TableArn; 190 | logger.log('Table arn :', roleARN); 191 | 192 | logger.info('Waiting for table to get created.............'); 193 | await oThis.waitForTableToGetCreated(dynamodbApiObject, params); 194 | 195 | logger.info('Table is active'); 196 | 197 | return { role_arn: roleARN }; 198 | }, 199 | 200 | cleanTestCaseEnvironment: async function(dynamodbApiObject, autoScaleObj) { 201 | const oThis = this, 202 | params = { 203 | TableName: testConstants.dummyTestTableName 204 | }; 205 | 206 | const deleteTableResponse = await oThis.deleteTable(dynamodbApiObject, params, true); 207 | if (deleteTableResponse.isFailure()) { 208 | assert.fail('Not able to delete table'); 209 | } 210 | 211 | logger.log('Waiting for Table get deleted...............'); 212 | await oThis.waitForTableToGetDeleted(dynamodbApiObject, params); 213 | 214 | logger.log('Table got deleted'); 215 | }, 216 | 217 | /** 218 | * Create table migration 219 | * @param dynamodbApiObject 220 | * @param autoScaleObj 221 | * @return {Promise<*|Promise<*>|promise>} 222 | */ 223 | createTableMigration: async function(dynamodbApiObject, autoScaleObj) { 224 | const oThis = this, 225 | params = {}, 226 | tableName = testConstants.dummyTestTableName, 227 | resourceId = autoScaleHelper.createResourceId(testConstants.dummyTestTableName); 228 | 229 | params.createTableConfig = oThis.getCreateTableParams(testConstants.dummyTestTableName); 230 | 231 | params.updateContinuousBackupConfig = { 232 | PointInTimeRecoverySpecification: { 233 | /* required */ 234 | PointInTimeRecoveryEnabled: true || false /* required */ 235 | }, 236 | TableName: testConstants.dummyTestTableName /* required */ 237 | }; 238 | 239 | const autoScalingConfig = {}, 240 | gsiArray = params.createTableConfig.GlobalSecondaryIndexes || []; 241 | 242 | autoScalingConfig.registerScalableTargetWrite = autoScaleHelper.createScalableTargetParams( 243 | resourceId, 244 | autoScaleHelper.writeCapacityScalableDimension, 245 | 1, 246 | 50 247 | ); 248 | 249 | autoScalingConfig.registerScalableTargetRead = autoScaleHelper.createScalableTargetParams( 250 | resourceId, 251 | autoScaleHelper.readCapacityScalableDimension, 252 | 1, 253 | 50 254 | ); 255 | 256 | autoScalingConfig.putScalingPolicyWrite = autoScaleHelper.createPolicyParams( 257 | tableName, 258 | resourceId, 259 | autoScaleHelper.writeCapacityScalableDimension, 260 | autoScaleHelper.writeMetricType, 261 | 1, 262 | 1, 263 | 50.0 264 | ); 265 | 266 | autoScalingConfig.putScalingPolicyRead = autoScaleHelper.createPolicyParams( 267 | tableName, 268 | resourceId, 269 | autoScaleHelper.readCapacityScalableDimension, 270 | autoScaleHelper.readMetricType, 271 | 1, 272 | 1, 273 | 50.0 274 | ); 275 | 276 | autoScalingConfig.globalSecondaryIndex = {}; 277 | 278 | for (let index = 0; index < gsiArray.length; index++) { 279 | let gsiIndexName = gsiArray[index].IndexName, 280 | indexResourceId = autoScaleHelper.createIndexResourceId(tableName, gsiIndexName); 281 | 282 | autoScalingConfig.globalSecondaryIndex[gsiIndexName] = {}; 283 | 284 | autoScalingConfig.globalSecondaryIndex[ 285 | gsiIndexName 286 | ].registerScalableTargetWrite = autoScaleHelper.createScalableTargetParams( 287 | indexResourceId, 288 | autoScaleHelper.indexWriteCapacityScalableDimenstion, 289 | 1, 290 | 20 291 | ); 292 | 293 | autoScalingConfig.globalSecondaryIndex[ 294 | gsiIndexName 295 | ].registerScalableTargetRead = autoScaleHelper.createScalableTargetParams( 296 | indexResourceId, 297 | autoScaleHelper.indexReadCapacityScalableDimension, 298 | 1, 299 | 20 300 | ); 301 | 302 | autoScalingConfig.globalSecondaryIndex[gsiIndexName].putScalingPolicyWrite = autoScaleHelper.createPolicyParams( 303 | tableName, 304 | indexResourceId, 305 | autoScaleHelper.indexWriteCapacityScalableDimenstion, 306 | autoScaleHelper.writeMetricType, 307 | 1, 308 | 1, 309 | 70.0 310 | ); 311 | 312 | autoScalingConfig.globalSecondaryIndex[gsiIndexName].putScalingPolicyRead = autoScaleHelper.createPolicyParams( 313 | tableName, 314 | indexResourceId, 315 | autoScaleHelper.indexReadCapacityScalableDimension, 316 | autoScaleHelper.readMetricType, 317 | 1, 318 | 1, 319 | 70.0 320 | ); 321 | } 322 | 323 | params.autoScalingConfig = autoScalingConfig; 324 | return dynamodbApiObject.createTableMigration(autoScaleObj, params); 325 | }, 326 | 327 | /** 328 | * Get crate table params 329 | * 330 | * @param tableName 331 | * @return {{TableName: *|string, KeySchema: *[], AttributeDefinitions: *[], ProvisionedThroughput: {ReadCapacityUnits: number, WriteCapacityUnits: number}, GlobalSecondaryIndexes: *[], SSESpecification: {Enabled: boolean}}} 332 | */ 333 | getCreateTableParams: function(tableName) { 334 | tableName = tableName || testConstants.dummyTestTableName; 335 | return { 336 | TableName: tableName, 337 | KeySchema: [ 338 | { 339 | AttributeName: 'tuid', 340 | KeyType: 'HASH' 341 | }, //Partition key 342 | { 343 | AttributeName: 'cid', 344 | KeyType: 'RANGE' 345 | } //Sort key 346 | ], 347 | AttributeDefinitions: [ 348 | { AttributeName: 'tuid', AttributeType: 'S' }, 349 | { AttributeName: 'cid', AttributeType: 'N' }, 350 | { AttributeName: 'thash', AttributeType: 'S' } 351 | ], 352 | ProvisionedThroughput: { 353 | ReadCapacityUnits: 1, 354 | WriteCapacityUnits: 1 355 | }, 356 | GlobalSecondaryIndexes: [ 357 | { 358 | IndexName: 'thash_global_secondary_index', 359 | KeySchema: [ 360 | { 361 | AttributeName: 'thash', 362 | KeyType: 'HASH' 363 | } 364 | ], 365 | Projection: { 366 | ProjectionType: 'KEYS_ONLY' 367 | }, 368 | ProvisionedThroughput: { 369 | ReadCapacityUnits: 1, 370 | WriteCapacityUnits: 1 371 | } 372 | } 373 | ], 374 | SSESpecification: { 375 | Enabled: false 376 | } 377 | }; 378 | }, 379 | 380 | /** 381 | * Get crate table params 382 | * 383 | * @param tableName 384 | * @return {{TableName: *|string, KeySchema: *[], AttributeDefinitions: *[], ProvisionedThroughput: {ReadCapacityUnits: number, WriteCapacityUnits: number}, GlobalSecondaryIndexes: *[], SSESpecification: {Enabled: boolean}}} 385 | */ 386 | getCreateTableParamsWithoutSecondaryIndex: function(tableName) { 387 | tableName = tableName || testConstants.dummyTestTableName; 388 | return { 389 | TableName: tableName, 390 | KeySchema: [ 391 | { 392 | AttributeName: 'tuid', 393 | KeyType: 'HASH' 394 | }, //Partition key 395 | { 396 | AttributeName: 'cid', 397 | KeyType: 'RANGE' 398 | } //Sort key 399 | ], 400 | AttributeDefinitions: [ 401 | { AttributeName: 'tuid', AttributeType: 'S' }, 402 | { AttributeName: 'cid', AttributeType: 'N' } 403 | ], 404 | ProvisionedThroughput: { 405 | ReadCapacityUnits: 1, 406 | WriteCapacityUnits: 1 407 | }, 408 | SSESpecification: { 409 | Enabled: false 410 | } 411 | }; 412 | } 413 | }; 414 | 415 | module.exports = new helper(); 416 | -------------------------------------------------------------------------------- /tests/mocha/services/auto_scale/put_scaling_policy.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // Load external packages 4 | const Chai = require('chai'), 5 | assert = Chai.assert; 6 | 7 | // Load dependencies package 8 | const rootPrefix = '../../../..', 9 | OSTStorage = require(rootPrefix + '/index'), 10 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 11 | coreConstant = require(rootPrefix + '/config/coreConstant'), 12 | helper = require(rootPrefix + '/tests/mocha/services/auto_scale/helper'), 13 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'); 14 | 15 | const ostStorage = OSTStorage.getInstance(testConstants.CONFIG_STRATEGIES), 16 | autoScaleObj = ostStorage.ic.getInstanceFor(coreConstant.icNameSpace, 'autoScaleApiService'), 17 | dynamodbApiObject = ostStorage.dynamoDBService; 18 | 19 | let resourceId = 'table/' + testConstants.dummyTestTableName, 20 | roleARN = null; 21 | 22 | const createTestCasesForOptions = function(optionsDesc, options, toAssert) { 23 | optionsDesc = optionsDesc || ''; 24 | 25 | options = options || { invalidResId: false }; 26 | 27 | it(optionsDesc, async function() { 28 | this.timeout(100000); 29 | 30 | let resId = 'table/' + testConstants.dummyTestTableName; 31 | if (options.invalidResId) { 32 | resId = 'invalidResId'; 33 | } 34 | 35 | const scalableTargetParams = { 36 | ResourceId: resourceId /* required */, 37 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 38 | ServiceNamespace: 'dynamodb' /* required */, 39 | MaxCapacity: 15, 40 | MinCapacity: 1, 41 | RoleARN: roleARN 42 | }; 43 | const registerScalableTargetResponse = await autoScaleObj.registerScalableTarget(scalableTargetParams); 44 | assert.equal(registerScalableTargetResponse.isSuccess(), true, 'registerScalableTarget failed'); 45 | 46 | let scalingPolicy = null; 47 | if (options.stepScaling) { 48 | scalingPolicy = { 49 | PolicyName: testConstants.dummyTestTableName + '-scaling-policy', 50 | PolicyType: 'StepScaling', 51 | ResourceId: resId, 52 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 53 | ServiceNamespace: 'dynamodb', 54 | StepScalingPolicyConfiguration: { 55 | AdjustmentType: 'PercentChangeInCapacity', 56 | Cooldown: 60, 57 | StepAdjustments: [ 58 | { 59 | MetricIntervalLowerBound: 0, 60 | ScalingAdjustment: 80 61 | } 62 | ] 63 | } 64 | }; 65 | } else { 66 | scalingPolicy = { 67 | ServiceNamespace: 'dynamodb', 68 | ResourceId: resId, 69 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 70 | PolicyName: testConstants.dummyTestTableName + '-scaling-policy', 71 | PolicyType: 'TargetTrackingScaling', 72 | TargetTrackingScalingPolicyConfiguration: { 73 | PredefinedMetricSpecification: { 74 | PredefinedMetricType: 'DynamoDBWriteCapacityUtilization' 75 | }, 76 | ScaleOutCooldown: 60, 77 | ScaleInCooldown: 60, 78 | TargetValue: 70.0 79 | } 80 | }; 81 | } 82 | const response = await autoScaleObj.putScalingPolicy(scalingPolicy); 83 | 84 | logger.log(response); 85 | assert.equal(response.isSuccess(), toAssert, 'put Scaling policy failed'); 86 | }); 87 | }; 88 | 89 | describe('services/autoScale/api#putScalingPolicy', function() { 90 | before(async function() { 91 | this.timeout(1000000); 92 | 93 | const returnObject = await helper.createTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 94 | roleARN = returnObject.role_arn; 95 | }); 96 | 97 | createTestCasesForOptions('Put scaling policy happy case', null, true); 98 | 99 | createTestCasesForOptions('Put scaling policy invalid resource Id case', { invalidResId: true }, false); 100 | 101 | // TODO test case for step scaling 102 | //createTestCasesForOptions("Put scaling policy having step scaling ", {stepScaling : true}, true); 103 | 104 | after(async function() { 105 | this.timeout(1000000); 106 | await helper.cleanTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 107 | }); 108 | }); 109 | -------------------------------------------------------------------------------- /tests/mocha/services/auto_scale/register_scalable_target.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // Load external packages 4 | const Chai = require('chai'), 5 | assert = Chai.assert; 6 | 7 | // Load dependencies package 8 | const rootPrefix = '../../../..', 9 | OSTStorage = require(rootPrefix + '/index'), 10 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 11 | coreConstant = require(rootPrefix + '/config/coreConstant'), 12 | helper = require(rootPrefix + '/tests/mocha/services/auto_scale/helper'), 13 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'); 14 | 15 | const ostStorage = OSTStorage.getInstance(testConstants.CONFIG_STRATEGIES), 16 | autoScaleObj = ostStorage.ic.getInstanceFor(coreConstant.icNameSpace, 'autoScaleApiService'), 17 | dynamodbApiObject = ostStorage.dynamoDBService; 18 | 19 | let resourceId = 'table/' + testConstants.dummyTestTableName, 20 | roleARN = null; 21 | 22 | const createTestCasesForOptions = function(optionsDesc, options, toAssert) { 23 | optionsDesc = optionsDesc || ''; 24 | 25 | options = options || { invalidARN: false }; 26 | it(optionsDesc, async function() { 27 | this.timeout(100000); 28 | let arn = roleARN; 29 | if (options.invalidARN) { 30 | arn = 'invalidArn'; 31 | } 32 | const params = { 33 | ResourceId: resourceId /* required */, 34 | ScalableDimension: 'dynamodb:table:WriteCapacityUnits', 35 | ServiceNamespace: 'dynamodb' /* required */, 36 | MaxCapacity: 15, 37 | MinCapacity: 1, 38 | RoleARN: arn 39 | }; 40 | const response = await autoScaleObj.registerScalableTarget(params); 41 | logger.log(response); 42 | assert.equal(response.isSuccess(), toAssert, 'Not able to register Scalable Target'); 43 | }); 44 | }; 45 | 46 | describe('services/autoScale/api#registerScalableTarget', function() { 47 | before(async function() { 48 | this.timeout(1000000); 49 | 50 | const returnObject = await helper.createTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 51 | roleARN = returnObject.role_arn; 52 | }); 53 | 54 | createTestCasesForOptions('Register scalable target happy case', null, true); 55 | 56 | createTestCasesForOptions('Register scalable target with wrong arn', { invalidARN: true }, false); 57 | 58 | after(async function() { 59 | this.timeout(1000000); 60 | 61 | await helper.cleanTestCaseEnvironment(dynamodbApiObject, autoScaleObj); 62 | }); 63 | }); 64 | -------------------------------------------------------------------------------- /tests/mocha/services/auto_scale/update_continuous_backup.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | const rootPrefix = '../../../..', 5 | OSTStorage = require(rootPrefix + '/index'), 6 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 7 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 8 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'), 9 | autoScaleHelper = require(rootPrefix + '/tests/mocha/services/auto_scale/helper'); 10 | 11 | describe('Create Table', function() { 12 | let ostStorage = null; 13 | let dynamodbApiObject = null; 14 | 15 | before(async function() { 16 | // create ostStorage 17 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 18 | dynamodbApiObject = ostStorage.dynamoDBService; 19 | }); 20 | 21 | it('should delete table successfully if exists', async function() { 22 | this.timeout(100000); 23 | const params = { 24 | TableName: testConstants.dummyTestTableName 25 | }; 26 | const checkTableExistsResponse = await dynamodbApiObject.checkTableExist(params); 27 | if (checkTableExistsResponse.data.response === true) { 28 | await autoScaleHelper.cleanTestCaseEnvironment(dynamodbApiObject, null); 29 | } 30 | }); 31 | 32 | it('should create table successfully', async function() { 33 | this.timeout(100000); 34 | const returnObject = await autoScaleHelper.createTestCaseEnvironment(dynamodbApiObject, null); 35 | }); 36 | 37 | it('should enable update continuous backup successfully', async function() { 38 | // build update continuous params 39 | const enableContinousBackupParams = { 40 | TableName: testConstants.dummyTestTableName, 41 | PointInTimeRecoverySpecification: { 42 | PointInTimeRecoveryEnabled: true 43 | } 44 | }; 45 | await helper.updateContinuousBackup(dynamodbApiObject, enableContinousBackupParams, true); 46 | }); 47 | 48 | it('should fail enable update continuous backup if table name not present', async function() { 49 | // build update continuous params 50 | const enableContinousBackupParams = { 51 | PointInTimeRecoverySpecification: { 52 | PointInTimeRecoveryEnabled: true 53 | } 54 | }; 55 | await helper.updateContinuousBackup(dynamodbApiObject, enableContinousBackupParams, false); 56 | }); 57 | 58 | after(async function() { 59 | this.timeout(1000000); 60 | await autoScaleHelper.cleanTestCaseEnvironment(dynamodbApiObject, null); 61 | logger.debug('Create Table Mocha Tests Complete'); 62 | }); 63 | }); 64 | -------------------------------------------------------------------------------- /tests/mocha/services/constants.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * Load all the constants from the mocha tests 5 | * 6 | * @module tests/mocha/services/dynamodb/constants 7 | * 8 | */ 9 | 10 | /** 11 | * Constructor for mocha test constants 12 | * 13 | * @constructor 14 | */ 15 | const MochaTestConstants = function() {}; 16 | 17 | MochaTestConstants.prototype = { 18 | CONFIG_STRATEGIES: { 19 | ddbTablePrefix: 'de_ma_', 20 | cache: { 21 | engine: 'memcached', 22 | servers: ['127.0.0.1:11211'], 23 | defaultTtl: 36000 24 | }, 25 | storage: { 26 | endpoint: 'http://localhost:8000', 27 | region: 'localhost', 28 | apiVersion: '2012-08-10', 29 | apiKey: 'X', 30 | apiSecret: 'X', 31 | enableSsl: '0', 32 | enableLogging: '0', 33 | enableAutoscaling: '0', 34 | enableDax: '0', 35 | maxRetryCount: 1, 36 | autoScaling: { 37 | endpoint: 'http://localhost:8000', 38 | region: 'localhost', 39 | apiKey: 'X', 40 | apiSecret: 'X', 41 | apiVersion: '2012-08-10', 42 | enableSsl: '0', 43 | enableLogging: '0' 44 | } 45 | } 46 | }, 47 | 48 | CONFIG_STRATEGIES_2: { 49 | ddbTablePrefix: 'de_ma_', 50 | cache: { 51 | engine: 'memcached', 52 | servers: ['127.0.0.1:11211'], 53 | defaultTtl: 36000 54 | }, 55 | storage: { 56 | endpoint: 'http://localhost:8001', 57 | region: 'localhost', 58 | apiVersion: '2012-08-10', 59 | apiKey: 'X', 60 | apiSecret: 'X', 61 | enableSsl: '0', 62 | enableLogging: '1', 63 | enableAutoscaling: '0', 64 | enableDax: '0', 65 | maxRetryCount: 1, 66 | autoScaling: { 67 | endpoint: 'http://localhost:8001', 68 | region: 'localhost', 69 | apiKey: 'X', 70 | apiSecret: 'X', 71 | apiVersion: '2016-02-06', 72 | enableSsl: '0', 73 | enableLogging: '0' 74 | } 75 | } 76 | }, 77 | 78 | dummyTestTableName: 'shard_00001_dummy_table' 79 | }; 80 | 81 | module.exports = new MochaTestConstants(); 82 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/batch_get.js: -------------------------------------------------------------------------------- 1 | /* global describe, it */ 2 | 3 | const rootPrefix = '../../../..', 4 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 5 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'), 6 | testDataSource = require(rootPrefix + '/tests/mocha/services/dynamodb/testdata/batch_get_write_data'); 7 | 8 | function performTest(ddbServiceObject) { 9 | describe('Batch get', function() { 10 | before(async function() { 11 | this.timeout(100000); 12 | 13 | // check if table exists 14 | const checkTableExistsResponse = await ddbServiceObject.checkTableExist(testDataSource.DELETE_TABLE_DATA); 15 | if (checkTableExistsResponse.data.response === true) { 16 | // delete if table exists 17 | await helper.deleteTable(ddbServiceObject, testDataSource.DELETE_TABLE_DATA, true); 18 | } 19 | 20 | // create table for the test 21 | await helper.createTable(ddbServiceObject, testDataSource.CREATE_TABLE_DATA, true); 22 | 23 | // populate test data 24 | const batchWriteParams = testDataSource.getBatchWriteDataBasedOnParam(4); 25 | await helper.performBatchWriteTest(ddbServiceObject, batchWriteParams, true); 26 | }); 27 | 28 | it('batch get happy case', async function() { 29 | this.timeout(100000); 30 | const batchGetParams = { 31 | RequestItems: { 32 | [testConstants.dummyTestTableName]: { 33 | Keys: [ 34 | { 35 | tuid: { 36 | S: 'tuid_1' 37 | }, 38 | cid: { 39 | N: '1' 40 | } 41 | }, 42 | { 43 | tuid: { 44 | S: 'tuid_2' 45 | }, 46 | cid: { 47 | N: '2' 48 | } 49 | }, 50 | { 51 | tuid: { 52 | S: 'tuid_3' 53 | }, 54 | cid: { 55 | N: '3' 56 | } 57 | } 58 | ] 59 | } 60 | } 61 | }; 62 | let returnCount = 3; 63 | await helper.performBatchGetTest(ddbServiceObject, batchGetParams, true, returnCount); 64 | }); 65 | 66 | it('batch get partial valid cases', async function() { 67 | this.timeout(100000); 68 | const batchGetParams = { 69 | RequestItems: { 70 | [testConstants.dummyTestTableName]: { 71 | Keys: [ 72 | { 73 | tuid: { 74 | S: 'tuid_1' 75 | }, 76 | cid: { 77 | N: '1' 78 | } 79 | }, 80 | { 81 | tuid: { 82 | S: 'tuid_2' 83 | }, 84 | cid: { 85 | N: '2' 86 | } 87 | }, 88 | { 89 | tuid: { 90 | S: 'tuid_5' 91 | }, 92 | cid: { 93 | N: '5' 94 | } 95 | } 96 | ] 97 | } 98 | } 99 | }; 100 | let returnCount = 2; 101 | await helper.performBatchGetTest(ddbServiceObject, batchGetParams, true, returnCount); 102 | }); 103 | 104 | it('batch get zero keys', async function() { 105 | this.timeout(100000); 106 | const batchGetParams = { 107 | RequestItems: { 108 | [testConstants.dummyTestTableName]: { 109 | Keys: [] 110 | } 111 | } 112 | }; 113 | let returnCount = 0; 114 | await helper.performBatchGetTest(ddbServiceObject, batchGetParams, false, returnCount); 115 | }); 116 | 117 | it('batch get none key match keys', async function() { 118 | this.timeout(100000); 119 | const batchGetParams = { 120 | RequestItems: { 121 | [testConstants.dummyTestTableName]: { 122 | Keys: [ 123 | { 124 | tuid: { 125 | S: 'tuid_5' 126 | }, 127 | cid: { 128 | N: '5' 129 | } 130 | } 131 | ] 132 | } 133 | } 134 | }; 135 | let returnCount = 0; 136 | await helper.performBatchGetTest(ddbServiceObject, batchGetParams, true, returnCount); 137 | }); 138 | 139 | after(function() { 140 | // runs after all tests in this block 141 | console.log('after function called'); 142 | }); 143 | }); 144 | } 145 | 146 | function performMultipleTest(ddbServiceObject1, ddbServiceObject2) { 147 | describe('Batch Multiple Get', function() { 148 | before(async function() { 149 | this.timeout(100000); 150 | 151 | // check if table exists 152 | const checkTableExistsResponse1 = await ddbServiceObject1.checkTableExist(testDataSource.DELETE_TABLE_DATA); 153 | const checkTableExistsResponse2 = await ddbServiceObject2.checkTableExist(testDataSource.DELETE_TABLE_DATA); 154 | if (checkTableExistsResponse1.data.response === true) { 155 | // delete if table exists 156 | await helper.deleteTable(ddbServiceObject1, testDataSource.DELETE_TABLE_DATA, true); 157 | } 158 | if (checkTableExistsResponse2.data.response === true) { 159 | // delete if table exists 160 | await helper.deleteTable(ddbServiceObject2, testDataSource.DELETE_TABLE_DATA, true); 161 | } 162 | 163 | // create table for the test 164 | await helper.createTable(ddbServiceObject1, testDataSource.CREATE_TABLE_DATA, true); 165 | await helper.createTable(ddbServiceObject2, testDataSource.CREATE_TABLE_DATA, true); 166 | 167 | // populate test data 168 | const batchWriteParams1 = testDataSource.getBatchWriteDataBasedOnParam(4); 169 | const batchWriteParams2 = testDataSource.getBatchWriteDataBasedOnParam_2(4); 170 | await helper.performBatchWriteTest(ddbServiceObject1, batchWriteParams1, true); 171 | await helper.performBatchWriteTest(ddbServiceObject2, batchWriteParams2, true); 172 | }); 173 | 174 | it('gets data from different dynamodb instances', async function() { 175 | this.timeout(100000); 176 | const batchGetParams1 = { 177 | RequestItems: { 178 | [testConstants.dummyTestTableName]: { 179 | Keys: [ 180 | { 181 | tuid: { 182 | S: 'tuid_0' 183 | }, 184 | cid: { 185 | N: '0' 186 | } 187 | }, 188 | { 189 | tuid: { 190 | S: 'tuid_1' 191 | }, 192 | cid: { 193 | N: '1' 194 | } 195 | }, 196 | { 197 | tuid: { 198 | S: 'tuid_2' 199 | }, 200 | cid: { 201 | N: '2' 202 | } 203 | }, 204 | { 205 | tuid: { 206 | S: 'tuid_3' 207 | }, 208 | cid: { 209 | N: '3' 210 | } 211 | } 212 | ] 213 | } 214 | } 215 | }; 216 | 217 | const batchGetParams2 = { 218 | RequestItems: { 219 | [testConstants.dummyTestTableName]: { 220 | Keys: [ 221 | { 222 | tuid: { 223 | S: 'tuid_4' 224 | }, 225 | cid: { 226 | N: '4' 227 | } 228 | }, 229 | { 230 | tuid: { 231 | S: 'tuid_5' 232 | }, 233 | cid: { 234 | N: '5' 235 | } 236 | }, 237 | { 238 | tuid: { 239 | S: 'tuid_6' 240 | }, 241 | cid: { 242 | N: '6' 243 | } 244 | }, 245 | { 246 | tuid: { 247 | S: 'tuid_7' 248 | }, 249 | cid: { 250 | N: '7' 251 | } 252 | } 253 | ] 254 | } 255 | } 256 | }; 257 | let returnCount = 4; 258 | await helper.performBatchGetTest(ddbServiceObject1, batchGetParams1, true, returnCount); 259 | await helper.performBatchGetTest(ddbServiceObject2, batchGetParams2, true, returnCount); 260 | }); 261 | 262 | it('fails as data is fetched from wrong dynamodb instances', async function() { 263 | this.timeout(100000); 264 | const batchGetParams1 = { 265 | RequestItems: { 266 | [testConstants.dummyTestTableName]: { 267 | Keys: [ 268 | { 269 | tuid: { 270 | S: 'tuid_4' 271 | }, 272 | cid: { 273 | N: '4' 274 | } 275 | } 276 | ] 277 | } 278 | } 279 | }; 280 | 281 | const batchGetParams2 = { 282 | RequestItems: { 283 | [testConstants.dummyTestTableName]: { 284 | Keys: [ 285 | { 286 | tuid: { 287 | S: 'tuid_0' 288 | }, 289 | cid: { 290 | N: '0' 291 | } 292 | } 293 | ] 294 | } 295 | } 296 | }; 297 | let returnCount = 0; 298 | await helper.performBatchGetTest(ddbServiceObject1, batchGetParams1, true, returnCount); 299 | await helper.performBatchGetTest(ddbServiceObject2, batchGetParams2, true, returnCount); 300 | }); 301 | }); 302 | } 303 | 304 | ostStorage1 = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 305 | ddb_service1 = ostStorage1.dynamoDBService; 306 | 307 | ostStorage2 = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES_2); 308 | ddb_service2 = ostStorage2.dynamoDBService; 309 | 310 | performTest(ddb_service1); 311 | performMultipleTest(ddb_service1, ddb_service2); 312 | 313 | // mocha tests/mocha/services/dynamodb/ 314 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/check_table_exist.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | //Load external files 5 | const rootPrefix = '../../../..', 6 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 7 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 8 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 9 | 10 | describe('Check table exists', function() { 11 | let ostStorage = null; 12 | 13 | before(async function() { 14 | // create dynamoDbApiObject 15 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 16 | 17 | // put item 18 | const createTableParams = { 19 | TableName: testConstants.dummyTestTableName, 20 | KeySchema: [ 21 | { 22 | AttributeName: 'tuid', 23 | KeyType: 'HASH' 24 | }, //Partition key 25 | { 26 | AttributeName: 'cid', 27 | KeyType: 'RANGE' 28 | } //Sort key 29 | ], 30 | AttributeDefinitions: [ 31 | { AttributeName: 'tuid', AttributeType: 'S' }, 32 | { AttributeName: 'cid', AttributeType: 'N' } 33 | ], 34 | ProvisionedThroughput: { 35 | ReadCapacityUnits: 1, 36 | WriteCapacityUnits: 1 37 | } 38 | }; 39 | ddb_service = ostStorage.dynamoDBService; 40 | await helper.createTable(ddb_service, createTableParams, true); 41 | }); 42 | 43 | it('check table exist successfully', async function() { 44 | const response = await ddb_service.checkTableExist({ TableName: testConstants.dummyTestTableName }); 45 | assert.equal(response.isSuccess(), true, 'check table exist failed'); 46 | }); 47 | 48 | it('check table exist unsuccessfully', async function() { 49 | const response = await ddb_service.checkTableExist({ TableName: 'unKnown_Table' }); 50 | assert.equal(response.isSuccess(), true, 'check table exist failed'); 51 | }); 52 | 53 | after(async function() { 54 | const deleteTableParams = { 55 | TableName: testConstants.dummyTestTableName 56 | }; 57 | await helper.deleteTable(ddb_service, deleteTableParams, true); 58 | logger.debug('Update Table Mocha Tests Complete'); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/create_table.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | const rootPrefix = '../../../..', 5 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 6 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 7 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 8 | 9 | describe('Create Table', function() { 10 | let ostStorage = null; 11 | 12 | before(async function() { 13 | // get ostStorage 14 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 15 | ddb_service = ostStorage.dynamoDBService; 16 | }); 17 | 18 | it('should delete table successfully if exists', async function() { 19 | const params = { 20 | TableName: testConstants.dummyTestTableName 21 | }; 22 | const checkTableExistsResponse = await ddb_service.checkTableExist(params); 23 | if (checkTableExistsResponse.data.response === true) { 24 | await helper.deleteTable(ddb_service, params, true); 25 | } 26 | }); 27 | 28 | it('should create table successfully', async function() { 29 | // build create table params 30 | const createTableParams = { 31 | TableName: testConstants.dummyTestTableName, 32 | KeySchema: [ 33 | { 34 | AttributeName: 'tuid', 35 | KeyType: 'HASH' 36 | }, //Partition key 37 | { 38 | AttributeName: 'cid', 39 | KeyType: 'RANGE' 40 | } //Sort key 41 | ], 42 | AttributeDefinitions: [ 43 | { AttributeName: 'tuid', AttributeType: 'S' }, 44 | { AttributeName: 'cid', AttributeType: 'N' }, 45 | { AttributeName: 'thash', AttributeType: 'S' } 46 | ], 47 | ProvisionedThroughput: { 48 | ReadCapacityUnits: 5, 49 | WriteCapacityUnits: 5 50 | }, 51 | GlobalSecondaryIndexes: [ 52 | { 53 | IndexName: 'thash_global_secondary_index', 54 | KeySchema: [ 55 | { 56 | AttributeName: 'thash', 57 | KeyType: 'HASH' 58 | } 59 | ], 60 | Projection: { 61 | ProjectionType: 'KEYS_ONLY' 62 | }, 63 | ProvisionedThroughput: { 64 | ReadCapacityUnits: 1, 65 | WriteCapacityUnits: 1 66 | } 67 | } 68 | ], 69 | SSESpecification: { 70 | Enabled: false 71 | } 72 | }; 73 | await helper.createTable(ddb_service, createTableParams, true); 74 | }); 75 | 76 | it('create table should fail when table name is not passed', async function() { 77 | // build create table params 78 | const createTableParams = { 79 | KeySchema: [ 80 | { 81 | AttributeName: 'tuid', 82 | KeyType: 'HASH' 83 | }, //Partition key 84 | { 85 | AttributeName: 'cid', 86 | KeyType: 'RANGE' 87 | } //Sort key 88 | ], 89 | AttributeDefinitions: [ 90 | { AttributeName: 'tuid', AttributeType: 'S' }, 91 | { AttributeName: 'cid', AttributeType: 'N' }, 92 | { AttributeName: 'thash', AttributeType: 'S' } 93 | ], 94 | ProvisionedThroughput: { 95 | ReadCapacityUnits: 5, 96 | WriteCapacityUnits: 5 97 | }, 98 | GlobalSecondaryIndexes: [ 99 | { 100 | IndexName: 'thash_global_secondary_index', 101 | KeySchema: [ 102 | { 103 | AttributeName: 'thash', 104 | KeyType: 'HASH' 105 | } 106 | ], 107 | Projection: { 108 | ProjectionType: 'KEYS_ONLY' 109 | }, 110 | ProvisionedThroughput: { 111 | ReadCapacityUnits: 1, 112 | WriteCapacityUnits: 1 113 | } 114 | } 115 | ], 116 | SSESpecification: { 117 | Enabled: false 118 | } 119 | }; 120 | await helper.createTable(ddb_service, createTableParams, false); 121 | }); 122 | 123 | // it('should enable continous backup successfully', async function () { 124 | // // build create table params 125 | // const enableContinousBackupParams = { 126 | // TableName: testConstants.dummyTestTableName, 127 | // PointInTimeRecoverySpecification: { 128 | // PointInTimeRecoveryEnabled: true 129 | // } 130 | // }; 131 | // await helper.updateContinuousBackup(dynamodbApiObject, enableContinousBackupParams); 132 | // }); 133 | 134 | after(async function() { 135 | const params = { 136 | TableName: testConstants.dummyTestTableName 137 | }; 138 | await helper.deleteTable(ddb_service, params, true); 139 | 140 | logger.debug('Create Table Mocha Tests Complete'); 141 | }); 142 | }); 143 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/delete_item.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | //Load external files 5 | const rootPrefix = '../../../..', 6 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 7 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 8 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 9 | 10 | describe('Delete Item', function() { 11 | let ostStorage = null; 12 | 13 | before(async function() { 14 | // get ostStorage 15 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 16 | ddb_service = ostStorage.dynamoDBService; 17 | 18 | // put item 19 | const createTableParams = { 20 | TableName: testConstants.dummyTestTableName, 21 | KeySchema: [ 22 | { 23 | AttributeName: 'tuid', 24 | KeyType: 'HASH' 25 | }, //Partition key 26 | { 27 | AttributeName: 'cid', 28 | KeyType: 'RANGE' 29 | } //Sort key 30 | ], 31 | AttributeDefinitions: [ 32 | { AttributeName: 'tuid', AttributeType: 'S' }, 33 | { AttributeName: 'cid', AttributeType: 'N' } 34 | ], 35 | ProvisionedThroughput: { 36 | ReadCapacityUnits: 1, 37 | WriteCapacityUnits: 1 38 | } 39 | }; 40 | await helper.createTable(ddb_service, createTableParams, true); 41 | }); 42 | 43 | it('should delete item successfully', async function() { 44 | const insertItemParams = { 45 | TableName: testConstants.dummyTestTableName, 46 | Item: { 47 | tuid: { S: 'shardTableName' }, 48 | cid: { N: '2' }, 49 | C: { S: String(new Date().getTime()) }, 50 | U: { S: String(new Date().getTime()) } 51 | } 52 | }; 53 | await helper.putItem(ddb_service, insertItemParams, true); 54 | 55 | const deleteItemParams = { 56 | TableName: testConstants.dummyTestTableName, 57 | Key: { 58 | tuid: { 59 | S: 'shardTableName' 60 | }, 61 | cid: { 62 | N: '2' 63 | } 64 | } 65 | }; 66 | await helper.deleteItem(ddb_service, deleteItemParams, true); 67 | }); 68 | 69 | it('should delete item successfully with unknown key', async function() { 70 | const insertItemParams = { 71 | TableName: testConstants.dummyTestTableName, 72 | Item: { 73 | tuid: { S: 'shardTableName' }, 74 | cid: { N: '2' }, 75 | C: { S: String(new Date().getTime()) }, 76 | U: { S: String(new Date().getTime()) } 77 | } 78 | }; 79 | await helper.putItem(ddb_service, insertItemParams, true); 80 | 81 | const deleteItemParams = { 82 | TableName: testConstants.dummyTestTableName, 83 | Key: { 84 | tuid: { 85 | S: 'shardTable' 86 | }, 87 | cid: { 88 | N: '2' 89 | } 90 | } 91 | }; 92 | await helper.deleteItem(ddb_service, deleteItemParams, true); 93 | }); 94 | 95 | it('should delete item unsuccessfully with invalid table name', async function() { 96 | const deleteItemParams = { 97 | TableName: 'InvalidTableName', 98 | Key: { 99 | tuid: { 100 | S: 'shardTableName' 101 | }, 102 | cid: { 103 | N: '2' 104 | } 105 | } 106 | }; 107 | await helper.deleteItem(ddb_service, deleteItemParams, false); 108 | }); 109 | 110 | after(async function() { 111 | const deleteTableParams = { 112 | TableName: testConstants.dummyTestTableName 113 | }; 114 | await helper.deleteTable(ddb_service, deleteTableParams, true); 115 | logger.debug('Update Table Mocha Tests Complete'); 116 | }); 117 | }); 118 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/delete_table.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | const rootPrefix = '../../../..', 5 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 6 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 7 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 8 | 9 | describe('Delete Table', function() { 10 | let ostStorage = null; 11 | 12 | before(async function() { 13 | // get ostStorage 14 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 15 | 16 | ddb_service = ostStorage.dynamoDBService; 17 | }); 18 | 19 | it('should create table successfully', async function() { 20 | // build create table params 21 | const createTableParams = { 22 | TableName: testConstants.dummyTestTableName, 23 | KeySchema: [ 24 | { 25 | AttributeName: 'tuid', 26 | KeyType: 'HASH' 27 | }, //Partition key 28 | { 29 | AttributeName: 'cid', 30 | KeyType: 'RANGE' 31 | } //Sort key 32 | ], 33 | AttributeDefinitions: [ 34 | { AttributeName: 'tuid', AttributeType: 'S' }, 35 | { AttributeName: 'cid', AttributeType: 'N' }, 36 | { AttributeName: 'thash', AttributeType: 'S' } 37 | ], 38 | ProvisionedThroughput: { 39 | ReadCapacityUnits: 5, 40 | WriteCapacityUnits: 5 41 | }, 42 | GlobalSecondaryIndexes: [ 43 | { 44 | IndexName: 'thash_global_secondary_index', 45 | KeySchema: [ 46 | { 47 | AttributeName: 'thash', 48 | KeyType: 'HASH' 49 | } 50 | ], 51 | Projection: { 52 | ProjectionType: 'KEYS_ONLY' 53 | }, 54 | ProvisionedThroughput: { 55 | ReadCapacityUnits: 1, 56 | WriteCapacityUnits: 1 57 | } 58 | } 59 | ], 60 | SSESpecification: { 61 | Enabled: false 62 | } 63 | }; 64 | await helper.createTable(ddb_service, createTableParams, true); 65 | }); 66 | 67 | it('should delete table successfully', async function() { 68 | // build delete table params 69 | const deleteTableParams = { 70 | TableName: testConstants.dummyTestTableName 71 | }; 72 | 73 | await helper.deleteTable(ddb_service, deleteTableParams, true); 74 | }); 75 | 76 | it('should fail when table name is not passed', async function() { 77 | // build delete table params 78 | const deleteTableParams = { 79 | TableName: testConstants.dummyTestTableName 80 | }; 81 | 82 | await helper.deleteTable(ddb_service, deleteTableParams, false); 83 | }); 84 | 85 | after(function() { 86 | logger.debug('Delete Table Mocha Tests Complete'); 87 | }); 88 | }); 89 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/describe_table.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | const rootPrefix = '../../../..', 5 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 6 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 7 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 8 | 9 | describe('Describe Dynamodb Table', function() { 10 | let ostStorage = null; 11 | 12 | before(async function() { 13 | // get ostStorage 14 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 15 | ddb_service = ostStorage.dynamoDBService; 16 | }); 17 | 18 | it('should create table successfully', async function() { 19 | // build create table params 20 | const createTableParams = { 21 | TableName: testConstants.dummyTestTableName, 22 | KeySchema: [ 23 | { 24 | AttributeName: 'tuid', 25 | KeyType: 'HASH' 26 | }, //Partition key 27 | { 28 | AttributeName: 'cid', 29 | KeyType: 'RANGE' 30 | } //Sort key 31 | ], 32 | AttributeDefinitions: [ 33 | { AttributeName: 'tuid', AttributeType: 'S' }, 34 | { AttributeName: 'cid', AttributeType: 'N' }, 35 | { AttributeName: 'thash', AttributeType: 'S' } 36 | ], 37 | ProvisionedThroughput: { 38 | ReadCapacityUnits: 5, 39 | WriteCapacityUnits: 5 40 | }, 41 | GlobalSecondaryIndexes: [ 42 | { 43 | IndexName: 'thash_global_secondary_index', 44 | KeySchema: [ 45 | { 46 | AttributeName: 'thash', 47 | KeyType: 'HASH' 48 | } 49 | ], 50 | Projection: { 51 | ProjectionType: 'KEYS_ONLY' 52 | }, 53 | ProvisionedThroughput: { 54 | ReadCapacityUnits: 1, 55 | WriteCapacityUnits: 1 56 | } 57 | } 58 | ], 59 | SSESpecification: { 60 | Enabled: false 61 | } 62 | }; 63 | await helper.createTable(ddb_service, createTableParams, true); 64 | }); 65 | 66 | it('should describe table successfully', async function() { 67 | const describeTableParams = { 68 | TableName: testConstants.dummyTestTableName 69 | }; 70 | await helper.describeTable(ddb_service, describeTableParams, true); 71 | }); 72 | 73 | it('should fail when table name is not passed', async function() { 74 | const describeTableParams = {}; 75 | await helper.describeTable(ddb_service, describeTableParams, false); 76 | }); 77 | 78 | after(async function() { 79 | // build delete table params 80 | const deleteTableParams = { 81 | TableName: testConstants.dummyTestTableName 82 | }; 83 | 84 | await helper.deleteTable(ddb_service, deleteTableParams, true); 85 | logger.debug('Update Table Mocha Tests Complete'); 86 | }); 87 | }); 88 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/helper.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const chai = require('chai'), 4 | assert = chai.assert; 5 | 6 | const rootPrefix = '../../../..', 7 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 8 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 9 | OSTStorage = require(rootPrefix + '/index'); 10 | 11 | /** 12 | * Constructor for helper class 13 | * 14 | * @constructor 15 | */ 16 | const helper = function() {}; 17 | 18 | helper.prototype = { 19 | /** 20 | * Validate OST Storage Object 21 | * 22 | * @params {object} configStrategy - configuration 23 | * 24 | * @return {result} 25 | * 26 | */ 27 | validateOstStorageObject: function(configStrategy) { 28 | // validate if the dynamodb configuration is available 29 | assert.exists(configStrategy, 'configStrategy is neither `null` nor `undefined`'); 30 | 31 | // create dynamoDBApi object 32 | const ostStorage = OSTStorage.getInstance(configStrategy); 33 | assert.exists(ostStorage, 'ostStorage is not created'); 34 | assert.equal(typeof ostStorage, 'object'); 35 | assert.equal(ostStorage.constructor.name, 'OSTStorage'); 36 | 37 | return ostStorage; 38 | }, 39 | 40 | /** 41 | * Create Table Helper method 42 | * 43 | * @params {object} ostStorage - DynamoDB Api object 44 | * @params {object} params - batch get params 45 | * @params {object} isResultSuccess - expected result 46 | * 47 | * @return {result} 48 | * 49 | */ 50 | createTable: async function(ostStorage, params, isResultSuccess) { 51 | const createTableResponse = await ostStorage.createTable(params); 52 | 53 | if (isResultSuccess) { 54 | assert.equal(createTableResponse.isSuccess(), true); 55 | assert.exists(createTableResponse.data.TableDescription, params.TableName); 56 | // logger.info("Waiting for table to get created............."); 57 | // await autoScaleHelper.waitForTableToGetCreated(ostStorage, params); 58 | // logger.info("Table is active"); 59 | } else { 60 | assert.equal(createTableResponse.isSuccess(), false, 'createTable: successfull, should fail for this case'); 61 | } 62 | return createTableResponse; 63 | }, 64 | 65 | /** 66 | * Delete Table Helper method 67 | * 68 | * @params {object} ostStorage - DynamoDB Api object 69 | * @params {object} params - batch get params 70 | * @params {object} isResultSuccess - expected result 71 | * 72 | * @return {result} 73 | * 74 | */ 75 | deleteTable: async function(ostStorage, params, isResultSuccess) { 76 | const deleteTableResponse = await ostStorage.deleteTable(params); 77 | 78 | if (isResultSuccess === true) { 79 | assert.equal(deleteTableResponse.isSuccess(), true); 80 | logger.debug('deleteTableResponse.data.TableDescription', deleteTableResponse.data.TableDescription); 81 | assert.exists(deleteTableResponse.data.TableDescription, params.TableName); 82 | // logger.info("Waiting for table to get deleted"); 83 | // await autoScaleHelper.waitForTableToGetDeleted(ostStorage, params); 84 | // logger.info("Table got deleted") 85 | } else { 86 | assert.equal(deleteTableResponse.isSuccess(), false); 87 | } 88 | 89 | return deleteTableResponse; 90 | }, 91 | 92 | /** 93 | * Update Continuous Backup Table Helper method 94 | * 95 | * @params {object} ostStorage - DynamoDB Api object 96 | * @params {object} params - batch get params 97 | * @params {object} isResultSuccess - expected result 98 | * 99 | * @return {result} 100 | * 101 | */ 102 | updateContinuousBackup: async function(ostStorage, params, isResultSuccess) { 103 | const enableContinousBackupResponse = await ostStorage.updateContinuousBackups(params); 104 | if (isResultSuccess === true) { 105 | assert.equal(enableContinousBackupResponse.isSuccess(), true); 106 | assert.equal(enableContinousBackupResponse.data.ContinuousBackupsStatus, 'ENABLED'); 107 | } else { 108 | assert.equal(updateTableResponse.isSuccess(), false); 109 | } 110 | return enableContinousBackupResponse; 111 | }, 112 | 113 | /** 114 | * Update Table Helper method 115 | * 116 | * @params {object} ostStorage - DynamoDB Api object 117 | * @params {object} params - batch get params 118 | * @params {object} isResultSuccess - expected result 119 | * 120 | * @return {result} 121 | * 122 | */ 123 | updateTable: async function(ostStorage, params, isResultSuccess) { 124 | const updateTableResponse = await ostStorage.updateTable(params); 125 | if (isResultSuccess === true) { 126 | assert.equal(updateTableResponse.isSuccess(), true); 127 | assert.exists(updateTableResponse.data.TableDescription, params.TableName); 128 | } else { 129 | assert.equal(updateTableResponse.isSuccess(), false); 130 | } 131 | return updateTableResponse; 132 | }, 133 | 134 | /** 135 | * Describe Table Helper method 136 | * 137 | * @params {object} ostStorage - DynamoDB Api object 138 | * @params {object} params - batch get params 139 | * @params {object} isResultSuccess - expected result 140 | * 141 | * @return {result} 142 | * 143 | */ 144 | describeTable: async function(ostStorage, params, isResultSuccess) { 145 | const describeTableResponse = await ostStorage.describeTable(params); 146 | if (isResultSuccess === true) { 147 | assert.equal(describeTableResponse.isSuccess(), true); 148 | assert.exists(describeTableResponse.data.Table.TableName, params.TableName); 149 | } else { 150 | assert.equal(describeTableResponse.isSuccess(), false); 151 | } 152 | 153 | return describeTableResponse; 154 | }, 155 | 156 | /** 157 | * List Tables Helper method 158 | * 159 | * @params {object} ostStorage - DynamoDB Api object 160 | * @params {object} params - batch get params 161 | * @params {object} isResultSuccess - expected result 162 | * 163 | * @return {result} 164 | * 165 | */ 166 | listTables: async function(ostStorage, params, isResultSuccess) { 167 | const listTablesResponse = await ostStorage.listTables(params); 168 | if (isResultSuccess === true) { 169 | assert.equal(listTablesResponse.isSuccess(), true); 170 | assert.include(listTablesResponse.data.TableNames, testConstants.dummyTestTableName); 171 | } else { 172 | assert.equal(listTablesResponse.isSuccess(), false); 173 | } 174 | 175 | return listTablesResponse; 176 | }, 177 | 178 | /** 179 | * Perform batch get 180 | * 181 | * @params {object} ostStorage - DynamoDB Api object 182 | * @params {object} params - batch get params 183 | * @params {object} isResultSuccess - expected result 184 | * @params {number} resultCount - Result Count 185 | * 186 | * @return {result} 187 | * 188 | */ 189 | performBatchGetTest: async function(ostStorage, params, isResultSuccess, resultCount) { 190 | assert.exists(ostStorage, 'dynamoDBApiRef is neither `null` nor `undefined`'); 191 | assert.exists(params, 'params is neither `null` nor `undefined`'); 192 | 193 | // call batch get 194 | const batchGetResponse = await ostStorage.batchGetItem(params); 195 | 196 | // validate if the table is created 197 | assert.equal(batchGetResponse.isSuccess(), isResultSuccess, 'batch get failed'); 198 | 199 | if (isResultSuccess) { 200 | // validate batchGet output count 201 | assert.equal( 202 | batchGetResponse.data.Responses[testConstants.dummyTestTableName].length, 203 | resultCount, 204 | 'Result count is not equal' 205 | ); 206 | 207 | // validate return output is object or not 208 | let returnObject = batchGetResponse.data.Responses[testConstants.dummyTestTableName]; 209 | if (resultCount) { 210 | assert.typeOf(returnObject[0], 'object'); 211 | } 212 | } 213 | 214 | // return the response 215 | return batchGetResponse; 216 | }, 217 | 218 | /** 219 | * Perform batch write 220 | * 221 | * @params {object} ostStorage - DynamoDB Api object 222 | * @params {object} params - batch write params 223 | * @params {object} isResultSuccess - expected result 224 | * 225 | * @return {result} 226 | * 227 | */ 228 | performBatchWriteTest: async function(ostStorage, params, isResultSuccess) { 229 | assert.exists(ostStorage, 'dynamoDBApiRef is neither `null` nor `undefined`'); 230 | assert.exists(params, 'params is neither `null` nor `undefined`'); 231 | 232 | // call batch get 233 | const batchWriteResponse = await ostStorage.batchWriteItem(params); 234 | 235 | // validate if the table is created 236 | assert.equal(batchWriteResponse.isSuccess(), isResultSuccess, 'batch write failed'); 237 | 238 | // return the response 239 | return batchWriteResponse; 240 | }, 241 | 242 | /** 243 | * put Item 244 | * @param ostStorage 245 | * @param params 246 | * @param isResultSuccess 247 | * @return {Promise<*|result|DynamoDB.PutItemOutput>} 248 | */ 249 | putItem: async function(ostStorage, params, isResultSuccess) { 250 | assert.exists(ostStorage, 'dynamoDBApiRef is neither `null` nor `undefined`'); 251 | assert.exists(params, 'params is neither `null` nor `undefined`'); 252 | 253 | //call put Item 254 | const putItemResponse = await ostStorage.putItem(params); 255 | 256 | // validate if the insertion is successful or not 257 | assert.equal(putItemResponse.isSuccess(), isResultSuccess, 'put item failed'); 258 | 259 | return putItemResponse; 260 | }, 261 | 262 | /** 263 | * Delete Item 264 | * @param ostStorage 265 | * @param params 266 | * @param isResultSuccess 267 | * @return {Promise<*|result|DynamoDB.PutItemOutput>} 268 | */ 269 | deleteItem: async function(ostStorage, params, isResultSuccess) { 270 | assert.exists(ostStorage, 'dynamoDBApiRef is neither `null` nor `undefined`'); 271 | assert.exists(params, 'params is neither `null` nor `undefined`'); 272 | 273 | //call put Item 274 | const deleteItemResponse = await ostStorage.deleteItem(params); 275 | 276 | // validate if the delete is successful or not 277 | assert.equal(deleteItemResponse.isSuccess(), isResultSuccess, 'delete item failed'); 278 | 279 | return deleteItemResponse; 280 | }, 281 | 282 | /** 283 | * Update Item 284 | * @param ostStorage 285 | * @param params 286 | * @param isResultSuccess 287 | * @return {Promise<*|DynamoDB.DeleteItemOutput|result>} 288 | */ 289 | updateItem: async function(ostStorage, params, isResultSuccess) { 290 | assert.exists(ostStorage, 'dynamoDBApiRef is neither `null` nor `undefined`'); 291 | assert.exists(params, 'params is neither `null` nor `undefined`'); 292 | 293 | //call put Item 294 | const updateItemResponse = await ostStorage.updateItem(params); 295 | 296 | // validate if the update is successful or not 297 | assert.equal(updateItemResponse.isSuccess(), isResultSuccess, 'update item failed'); 298 | 299 | return updateItemResponse; 300 | }, 301 | 302 | /** 303 | * query test helper method 304 | * @param ostStorage 305 | * @param params 306 | * @param isResultSuccess 307 | * @param resultCount 308 | * @return {Promise<*>} 309 | */ 310 | query: async function(ostStorage, params, isResultSuccess, resultCount) { 311 | assert.exists(ostStorage, 'dynamoDBApiRef is neither `null` nor `undefined`'); 312 | assert.exists(params, 'params is neither `null` nor `undefined`'); 313 | 314 | //call query 315 | const queryResponse = await ostStorage.query(params); 316 | 317 | // validate if the query is successful or not 318 | assert.equal(queryResponse.isSuccess(), isResultSuccess, 'query failed'); 319 | 320 | if (isResultSuccess) { 321 | // validate query output count 322 | assert.equal(queryResponse.data.Count, resultCount, 'Result count is not equal'); 323 | 324 | // validate return output is object or not 325 | if (resultCount) { 326 | assert.typeOf(queryResponse.data.Items[0], 'object'); 327 | } 328 | } 329 | 330 | return queryResponse; 331 | }, 332 | 333 | /** 334 | * scan test helper method 335 | * @param ostStorage 336 | * @param params 337 | * @param isResultSuccess 338 | * @param resultCount 339 | * @return {Promise<*|DocumentClient.ScanOutput|result|DynamoDB.ScanOutput>} 340 | */ 341 | scan: async function(ostStorage, params, isResultSuccess, resultCount) { 342 | assert.exists(ostStorage, 'dynamoDBApiRef is neither `null` nor `undefined`'); 343 | assert.exists(params, 'params is neither `null` nor `undefined`'); 344 | 345 | //call scan 346 | const scanResponse = await ostStorage.scan(params); 347 | 348 | // validate if the scan is successful or not 349 | assert.equal(scanResponse.isSuccess(), isResultSuccess, 'scan failed'); 350 | 351 | if (isResultSuccess) { 352 | // validate scan output count 353 | assert.equal(scanResponse.data.Count, resultCount, 'Result count is not equal'); 354 | 355 | // validate return output is object or not 356 | if (resultCount) { 357 | assert.typeOf(scanResponse.data.Items[0], 'object'); 358 | } 359 | } 360 | 361 | return scanResponse; 362 | }, 363 | 364 | /** 365 | * To wait till response 366 | * @param ostStorage 367 | * @param func 368 | * @param params 369 | * @param toAssert 370 | * @param retries 371 | * @return {Promise} 372 | */ 373 | waitTillTableStatusProvided: async function(ostStorage, func, params, toAssert, retries) { 374 | const oThis = this, 375 | WAIT = retries ? retries : 30; 376 | let count = WAIT; 377 | let response = null; 378 | while (count > 0) { 379 | response = await oThis.waitTillResponse(ostStorage, func, params); 380 | count -= 1; 381 | } 382 | }, 383 | 384 | waitTillResponse: async function(ostStorage, func, params) { 385 | return new Promise(function(resolve) { 386 | setTimeout(async function() { 387 | let response = await func.call(ostStorage, params); 388 | resolve(response); 389 | }, 1000); 390 | }); 391 | } 392 | }; 393 | 394 | module.exports = new helper(); 395 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/list_tables.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | const rootPrefix = '../../../..', 5 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 6 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 7 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 8 | 9 | describe('List Tables', function() { 10 | let ostStorage = null; 11 | 12 | before(async function() { 13 | // get ostStorage 14 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 15 | ddb_service = ostStorage.dynamoDBService; 16 | }); 17 | 18 | it('should create table successfully', async function() { 19 | // build create table params 20 | const createTableParams = { 21 | TableName: testConstants.dummyTestTableName, 22 | KeySchema: [ 23 | { 24 | AttributeName: 'tuid', 25 | KeyType: 'HASH' 26 | }, //Partition key 27 | { 28 | AttributeName: 'cid', 29 | KeyType: 'RANGE' 30 | } //Sort key 31 | ], 32 | AttributeDefinitions: [ 33 | { AttributeName: 'tuid', AttributeType: 'S' }, 34 | { AttributeName: 'cid', AttributeType: 'N' }, 35 | { AttributeName: 'thash', AttributeType: 'S' } 36 | ], 37 | ProvisionedThroughput: { 38 | ReadCapacityUnits: 5, 39 | WriteCapacityUnits: 5 40 | }, 41 | GlobalSecondaryIndexes: [ 42 | { 43 | IndexName: 'thash_global_secondary_index', 44 | KeySchema: [ 45 | { 46 | AttributeName: 'thash', 47 | KeyType: 'HASH' 48 | } 49 | ], 50 | Projection: { 51 | ProjectionType: 'KEYS_ONLY' 52 | }, 53 | ProvisionedThroughput: { 54 | ReadCapacityUnits: 1, 55 | WriteCapacityUnits: 1 56 | } 57 | } 58 | ], 59 | SSESpecification: { 60 | Enabled: false 61 | } 62 | }; 63 | await helper.createTable(ddb_service, createTableParams, true); 64 | }); 65 | 66 | it('should list table successfully', async function() { 67 | // build create table params 68 | const listTablesParams = {}; 69 | await helper.listTables(ddb_service, listTablesParams, true); 70 | }); 71 | 72 | it('should fail when table name is passed in parameter', async function() { 73 | // build create table params 74 | const listTablesParams = { 75 | TableName: testConstants.dummyTestTableName 76 | }; 77 | await helper.listTables(ddb_service, listTablesParams, false); 78 | }); 79 | 80 | after(async function() { 81 | const deleteTableParams = { 82 | TableName: testConstants.dummyTestTableName 83 | }; 84 | 85 | await helper.deleteTable(ddb_service, deleteTableParams, true); 86 | logger.debug('List Tables Mocha Tests Complete'); 87 | }); 88 | }); 89 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/put_item.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | //Load external files 5 | const rootPrefix = '../../../..', 6 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 7 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 8 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 9 | 10 | describe('Delete Table', function() { 11 | let ostStorage = null; 12 | 13 | before(async function() { 14 | // get ostStorage 15 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 16 | ddb_service = ostStorage.dynamoDBService; 17 | 18 | // put item 19 | const createTableParams = { 20 | TableName: testConstants.dummyTestTableName, 21 | KeySchema: [ 22 | { 23 | AttributeName: 'tuid', 24 | KeyType: 'HASH' 25 | }, //Partition key 26 | { 27 | AttributeName: 'cid', 28 | KeyType: 'RANGE' 29 | } //Sort key 30 | ], 31 | AttributeDefinitions: [ 32 | { AttributeName: 'tuid', AttributeType: 'S' }, 33 | { AttributeName: 'cid', AttributeType: 'N' } 34 | ], 35 | ProvisionedThroughput: { 36 | ReadCapacityUnits: 1, 37 | WriteCapacityUnits: 1 38 | } 39 | }; 40 | await helper.createTable(ddb_service, createTableParams, true); 41 | }); 42 | 43 | it('should put item successfully', async function() { 44 | const insertItemParams = { 45 | TableName: testConstants.dummyTestTableName, 46 | Item: { 47 | tuid: { S: 'shardTableName' }, 48 | cid: { N: '2' }, 49 | C: { S: String(new Date().getTime()) }, 50 | U: { S: String(new Date().getTime()) } 51 | } 52 | }; 53 | await helper.putItem(ddb_service, insertItemParams, true); 54 | }); 55 | 56 | it('should put item with invalid datatype', async function() { 57 | const insertItemParams = { 58 | TableName: testConstants.dummyTestTableName, 59 | Item: { 60 | tuid: { S: 'shardTableName' }, 61 | cid: { S: '2' }, 62 | C: { S: String(new Date().getTime()) }, 63 | U: { S: String(new Date().getTime()) } 64 | } 65 | }; 66 | await helper.putItem(ddb_service, insertItemParams, false); 67 | }); 68 | 69 | after(async function() { 70 | const deleteTableParams = { 71 | TableName: testConstants.dummyTestTableName 72 | }; 73 | await helper.deleteTable(ddb_service, deleteTableParams, true); 74 | logger.debug('Update Table Mocha Tests Complete'); 75 | }); 76 | }); 77 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/query.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | //Load external files 5 | const rootPrefix = '../../../..', 6 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 7 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 8 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 9 | 10 | describe('Query Table', function() { 11 | let ostStorage = null; 12 | 13 | before(async function() { 14 | // get ostStorage 15 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 16 | ddb_service = ostStorage.dynamoDBService; 17 | 18 | // put item 19 | const createTableParams = { 20 | TableName: testConstants.dummyTestTableName, 21 | KeySchema: [ 22 | { 23 | AttributeName: 'tuid', 24 | KeyType: 'HASH' 25 | }, //Partition key 26 | { 27 | AttributeName: 'cid', 28 | KeyType: 'RANGE' 29 | } //Sort key 30 | ], 31 | AttributeDefinitions: [ 32 | { AttributeName: 'tuid', AttributeType: 'S' }, 33 | { AttributeName: 'cid', AttributeType: 'N' } 34 | ], 35 | ProvisionedThroughput: { 36 | ReadCapacityUnits: 1, 37 | WriteCapacityUnits: 1 38 | } 39 | }; 40 | await helper.createTable(ddb_service, createTableParams, true); 41 | 42 | const insertItemParams = { 43 | TableName: testConstants.dummyTestTableName, 44 | Item: { 45 | tuid: { S: 'shardTableName' }, 46 | cid: { N: '2' }, 47 | C: { S: String(new Date().getTime()) }, 48 | U: { S: String(new Date().getTime()) } 49 | } 50 | }; 51 | await helper.putItem(ddb_service, insertItemParams, true); 52 | }); 53 | 54 | it('query table for item successfully', async function() { 55 | const queryParams = { 56 | TableName: testConstants.dummyTestTableName, 57 | ExpressionAttributeValues: { 58 | ':v1': { 59 | S: 'shardTableName' 60 | }, 61 | ':v2': { 62 | N: '2' 63 | } 64 | }, 65 | KeyConditionExpression: '#id = :v1 AND #cid = :v2', 66 | ExpressionAttributeNames: { '#id': 'tuid', '#cid': 'cid' } 67 | }; 68 | const resultCount = 1; 69 | const response = await helper.query(ddb_service, queryParams, true, resultCount); 70 | }); 71 | 72 | it('query table for item with invalid key successfully', async function() { 73 | const queryParams = { 74 | TableName: testConstants.dummyTestTableName, 75 | ExpressionAttributeValues: { 76 | ':v1': { 77 | S: 'shardTable' 78 | }, 79 | ':v2': { 80 | N: '2' 81 | } 82 | }, 83 | KeyConditionExpression: '#id = :v1 AND #cid = :v2', 84 | ExpressionAttributeNames: { '#id': 'tuid', '#cid': 'cid' } 85 | }; 86 | 87 | const resultCount = 0; 88 | const response = await helper.query(ddb_service, queryParams, true, resultCount); 89 | }); 90 | 91 | it('query table for item with key only without using sort key unsuccessfully', async function() { 92 | const queryParams = { 93 | TableName: testConstants.dummyTestTableName, 94 | ExpressionAttributeValues: { 95 | ':v1': { 96 | S: 'shardTable' 97 | } 98 | }, 99 | KeyConditionExpression: '#id = :v1', 100 | ExpressionAttributeNames: { '#id': 'tuid' } 101 | }; 102 | 103 | const resultCount = 0; 104 | const response = await helper.query(ddb_service, queryParams, true, resultCount); 105 | }); 106 | 107 | it('query table for item with invalid table name unsuccessfully', async function() { 108 | const queryParams = { 109 | TableName: 'invalidTable', 110 | ExpressionAttributeValues: { 111 | ':v1': { 112 | S: 'shardTable' 113 | }, 114 | ':v2': { 115 | N: '2' 116 | } 117 | }, 118 | KeyConditionExpression: '#id = :v1 AND #cid = :v2', 119 | ExpressionAttributeNames: { '#id': 'tuid', '#cid': 'cid' } 120 | }; 121 | 122 | const resultCount = 0; 123 | const response = await helper.query(ddb_service, queryParams, false, resultCount); 124 | }); 125 | 126 | after(async function() { 127 | const deleteTableParams = { 128 | TableName: testConstants.dummyTestTableName 129 | }; 130 | await helper.deleteTable(ddb_service, deleteTableParams, true); 131 | logger.debug('Update Table Mocha Tests Complete'); 132 | }); 133 | }); 134 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/scan.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | //Load external files 5 | const rootPrefix = '../../../..', 6 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 7 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 8 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 9 | 10 | describe('Scan Table', function() { 11 | let ostStorage = null; 12 | 13 | before(async function() { 14 | // get ostStorage 15 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 16 | ddb_service = ostStorage.dynamoDBService; 17 | 18 | // put item 19 | const createTableParams = { 20 | TableName: testConstants.dummyTestTableName, 21 | KeySchema: [ 22 | { 23 | AttributeName: 'tuid', 24 | KeyType: 'HASH' 25 | }, //Partition key 26 | { 27 | AttributeName: 'cid', 28 | KeyType: 'RANGE' 29 | } //Sort key 30 | ], 31 | AttributeDefinitions: [ 32 | { AttributeName: 'tuid', AttributeType: 'S' }, 33 | { AttributeName: 'cid', AttributeType: 'N' } 34 | ], 35 | ProvisionedThroughput: { 36 | ReadCapacityUnits: 1, 37 | WriteCapacityUnits: 1 38 | } 39 | }; 40 | await helper.createTable(ddb_service, createTableParams, true); 41 | 42 | const insertItem1Params = { 43 | TableName: testConstants.dummyTestTableName, 44 | Item: { 45 | tuid: { S: 'shardTableName1' }, 46 | cid: { N: '1' }, 47 | C: { S: String(new Date().getTime()) }, 48 | U: { S: String(new Date().getTime()) } 49 | } 50 | }; 51 | await helper.putItem(ddb_service, insertItem1Params, true); 52 | 53 | const insertItem2Params = { 54 | TableName: testConstants.dummyTestTableName, 55 | Item: { 56 | tuid: { S: 'shardTableName2' }, 57 | cid: { N: '2' }, 58 | C: { S: String(new Date().getTime()) }, 59 | U: { S: String(new Date().getTime()) } 60 | } 61 | }; 62 | await helper.putItem(ddb_service, insertItem2Params, true); 63 | }); 64 | 65 | it('scan table for items successfully', async function() { 66 | const queryParams = { 67 | TableName: testConstants.dummyTestTableName, 68 | ExpressionAttributeValues: { 69 | ':v1': { 70 | S: 'shardTableName1' 71 | }, 72 | ':v2': { 73 | N: '1' 74 | } 75 | }, 76 | FilterExpression: '#id = :v1 AND #cid = :v2', 77 | ExpressionAttributeNames: { '#id': 'tuid', '#cid': 'cid' } 78 | }; 79 | 80 | const resultCount = 1; 81 | const response = await helper.scan(ddb_service, queryParams, true, resultCount); 82 | }); 83 | 84 | it('scan table for item with invalid key successfully', async function() { 85 | const queryParams = { 86 | TableName: testConstants.dummyTestTableName, 87 | ExpressionAttributeValues: { 88 | ':v1': { 89 | S: 'shardTableNae1' 90 | }, 91 | ':v2': { 92 | N: '1' 93 | } 94 | }, 95 | FilterExpression: '#id = :v1 AND #cid = :v2', 96 | ExpressionAttributeNames: { '#id': 'tuid', '#cid': 'cid' } 97 | }; 98 | 99 | const resultCount = 0; 100 | const response = await helper.scan(ddb_service, queryParams, true, resultCount); 101 | }); 102 | 103 | it('scan table for item with key only without using sort key successfully', async function() { 104 | const queryParams = { 105 | TableName: testConstants.dummyTestTableName, 106 | ExpressionAttributeValues: { 107 | ':v1': { 108 | S: 'shardTableName1' 109 | } 110 | }, 111 | FilterExpression: '#id = :v1', 112 | ExpressionAttributeNames: { '#id': 'tuid' } 113 | }; 114 | 115 | const resultCount = 1; 116 | const response = await helper.scan(ddb_service, queryParams, true, resultCount); 117 | }); 118 | 119 | it('scan table for item with invalid table name unsuccessfully', async function() { 120 | const queryParams = { 121 | TableName: 'invalidTable', 122 | ExpressionAttributeValues: { 123 | ':v1': { 124 | S: 'shardTableName1' 125 | }, 126 | ':v2': { 127 | N: '1' 128 | } 129 | }, 130 | FilterExpression: '#id = :v1 AND #cid = :v2', 131 | ExpressionAttributeNames: { '#id': 'tuid', '#cid': 'cid' } 132 | }; 133 | 134 | const resultCount = 0; 135 | const response = await helper.scan(ddb_service, queryParams, false, resultCount); 136 | }); 137 | 138 | after(async function() { 139 | const deleteTableParams = { 140 | TableName: testConstants.dummyTestTableName 141 | }; 142 | await helper.deleteTable(ddb_service, deleteTableParams, true); 143 | logger.debug('Update Table Mocha Tests Complete'); 144 | }); 145 | }); 146 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/testdata/batch_get_write_data.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * Load all the test data for batch write and batch get 5 | * 6 | * @module tests/mocha/services/dynamodb/testdata/batch_get_write_data 7 | * 8 | */ 9 | 10 | const rootPrefix = '../../../../..', 11 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'); 12 | 13 | /** 14 | * Constructor for test data 15 | * 16 | * @constructor 17 | */ 18 | const TestData = function() {}; 19 | 20 | var cid = 0; 21 | var tuid = `tuid_${cid}`; 22 | var thash = `thash${cid}`; 23 | 24 | const tableName = testConstants.dummyTestTableName; 25 | 26 | TestData.prototype = { 27 | TABLE_NAME: tableName, 28 | 29 | /** 30 | * Create table data 31 | * 32 | * @constant {object} 33 | * 34 | */ 35 | CREATE_TABLE_DATA: { 36 | TableName: tableName, 37 | KeySchema: [ 38 | { 39 | AttributeName: 'tuid', 40 | KeyType: 'HASH' 41 | }, //Partition key 42 | { 43 | AttributeName: 'cid', 44 | KeyType: 'RANGE' 45 | } //Sort key 46 | ], 47 | AttributeDefinitions: [ 48 | { AttributeName: 'tuid', AttributeType: 'S' }, 49 | { AttributeName: 'cid', AttributeType: 'N' }, 50 | { AttributeName: 'thash', AttributeType: 'S' } 51 | ], 52 | ProvisionedThroughput: { 53 | ReadCapacityUnits: 5, 54 | WriteCapacityUnits: 5 55 | }, 56 | GlobalSecondaryIndexes: [ 57 | { 58 | IndexName: 'thash_global_secondary_index', 59 | KeySchema: [ 60 | { 61 | AttributeName: 'thash', 62 | KeyType: 'HASH' 63 | } 64 | ], 65 | Projection: { 66 | ProjectionType: 'KEYS_ONLY' 67 | }, 68 | ProvisionedThroughput: { 69 | ReadCapacityUnits: 1, 70 | WriteCapacityUnits: 1 71 | } 72 | } 73 | ], 74 | SSESpecification: { 75 | Enabled: false 76 | } 77 | }, 78 | 79 | DELETE_TABLE_DATA: { 80 | TableName: tableName 81 | }, 82 | 83 | getBatchWriteData: function(numberOfItems) { 84 | const data = []; 85 | for (var i = 0; i < numberOfItems; i++) { 86 | cid++; 87 | tuid = `tuid_${cid}`; 88 | thash = `thash${cid}`; 89 | 90 | let item = {}; 91 | item.tuid = { 92 | S: tuid 93 | }; 94 | item.cid = { 95 | N: `${cid}` 96 | }; 97 | item.thash = { 98 | S: thash 99 | }; 100 | 101 | data.push({ PutRequest: { Item: item } }); 102 | } 103 | 104 | const requestItems = {}; 105 | requestItems[tableName] = data; 106 | return { RequestItems: requestItems }; 107 | }, 108 | 109 | getBatchWriteLargeData: function(numberOfItems) { 110 | const data = []; 111 | for (var i = 0; i < numberOfItems; i++) { 112 | cid++; 113 | tuid = `tuid_${cid}`; 114 | thash = `thash${cid}`; 115 | 116 | let item = {}; 117 | item.tuid = { 118 | S: tuid 119 | }; 120 | item.cid = { 121 | N: `${cid}` 122 | }; 123 | item.thash = { 124 | S: thash 125 | }; 126 | 127 | data.push({ PutRequest: { Item: item } }); 128 | } 129 | 130 | const requestItems = {}; 131 | requestItems[tableName] = data; 132 | return { RequestItems: requestItems }; 133 | }, 134 | 135 | getBatchWriteDataBasedOnParam: function(numberOfItems) { 136 | const data = []; 137 | for (var i = 0; i < numberOfItems; i++) { 138 | cid = i; 139 | tuid = `tuid_${cid}`; 140 | thash = `thash${cid}`; 141 | 142 | let item = {}; 143 | item.tuid = { 144 | S: tuid 145 | }; 146 | item.cid = { 147 | N: `${cid}` 148 | }; 149 | item.thash = { 150 | S: thash 151 | }; 152 | 153 | data.push({ PutRequest: { Item: item } }); 154 | } 155 | 156 | const requestItems = {}; 157 | requestItems[tableName] = data; 158 | return { RequestItems: requestItems }; 159 | }, 160 | 161 | getBatchWriteDataBasedOnParam_2: function(numberOfItems) { 162 | const data = []; 163 | for (var i = 0; i < numberOfItems; i++) { 164 | cid = i + 4; 165 | tuid = `tuid_${cid}`; 166 | thash = `thash${cid}`; 167 | 168 | let item = {}; 169 | item.tuid = { 170 | S: tuid 171 | }; 172 | item.cid = { 173 | N: `${cid}` 174 | }; 175 | item.thash = { 176 | S: thash 177 | }; 178 | 179 | data.push({ PutRequest: { Item: item } }); 180 | } 181 | 182 | const requestItems = {}; 183 | requestItems[tableName] = data; 184 | return { RequestItems: requestItems }; 185 | } 186 | }; 187 | 188 | module.exports = new TestData(); 189 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/update_item.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | //Load external files 5 | const rootPrefix = '../../../..', 6 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 7 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 8 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 9 | 10 | describe('Update Item in Table', function() { 11 | let ostStorage = null; 12 | 13 | before(async function() { 14 | // get ostStorage 15 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 16 | ddb_service = ostStorage.dynamoDBService; 17 | 18 | // put item 19 | const createTableParams = { 20 | TableName: testConstants.dummyTestTableName, 21 | KeySchema: [ 22 | { 23 | AttributeName: 'tuid', 24 | KeyType: 'HASH' 25 | }, //Partition key 26 | { 27 | AttributeName: 'cid', 28 | KeyType: 'RANGE' 29 | } //Sort key 30 | ], 31 | AttributeDefinitions: [ 32 | { AttributeName: 'tuid', AttributeType: 'S' }, 33 | { AttributeName: 'cid', AttributeType: 'N' } 34 | ], 35 | ProvisionedThroughput: { 36 | ReadCapacityUnits: 1, 37 | WriteCapacityUnits: 1 38 | } 39 | }; 40 | await helper.createTable(ddb_service, createTableParams, true); 41 | 42 | const insertItemParams = { 43 | TableName: testConstants.dummyTestTableName, 44 | Item: { 45 | tuid: { S: 'shardTableName' }, 46 | cid: { N: '2' }, 47 | C: { S: String(new Date().getTime()) }, 48 | U: { S: String(new Date().getTime()) } 49 | } 50 | }; 51 | await helper.putItem(ddb_service, insertItemParams, true); 52 | }); 53 | 54 | it('should update item successfully', async function() { 55 | const updateItemParam = { 56 | ExpressionAttributeNames: { 57 | '#c': 'C' 58 | }, 59 | ExpressionAttributeValues: { 60 | ':t': { 61 | S: '2342' 62 | } 63 | }, 64 | Key: { 65 | tuid: { 66 | S: 'shardTableName' 67 | }, 68 | cid: { 69 | N: '2' 70 | } 71 | }, 72 | ReturnValues: 'ALL_NEW', 73 | TableName: testConstants.dummyTestTableName, 74 | UpdateExpression: 'SET #c = :t' 75 | }; 76 | 77 | await helper.updateItem(ddb_service, updateItemParam, true); 78 | }); 79 | 80 | it('update item should be unsuccessfully when key type is invalid', async function() { 81 | const updateItemParam = { 82 | ExpressionAttributeNames: { 83 | '#c': 'C' 84 | }, 85 | ExpressionAttributeValues: { 86 | ':t': { 87 | C: '2342' 88 | } 89 | }, 90 | Key: { 91 | tuid: { 92 | S: 'shardTableName' 93 | }, 94 | cid: { 95 | S: '2' 96 | } 97 | }, 98 | ReturnValues: 'ALL_NEW', 99 | TableName: testConstants.dummyTestTableName, 100 | UpdateExpression: 'SET #c = :t' 101 | }; 102 | 103 | await helper.updateItem(ddb_service, updateItemParam, false); 104 | }); 105 | 106 | after(async function() { 107 | const deleteTableParams = { 108 | TableName: testConstants.dummyTestTableName 109 | }; 110 | await helper.deleteTable(ddb_service, deleteTableParams, true); 111 | logger.debug('Update Table Mocha Tests Complete'); 112 | }); 113 | }); 114 | -------------------------------------------------------------------------------- /tests/mocha/services/dynamodb/update_table.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'), 2 | assert = chai.assert; 3 | 4 | const rootPrefix = '../../../..', 5 | testConstants = require(rootPrefix + '/tests/mocha/services/constants'), 6 | logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), 7 | helper = require(rootPrefix + '/tests/mocha/services/dynamodb/helper'); 8 | 9 | describe('Delete Table', function() { 10 | let ostStorage = null; 11 | 12 | before(async function() { 13 | // get ostStorage 14 | ostStorage = helper.validateOstStorageObject(testConstants.CONFIG_STRATEGIES); 15 | ddb_service = ostStorage.dynamoDBService; 16 | }); 17 | 18 | it('should create table successfully', async function() { 19 | // build create table params 20 | const createTableParams = { 21 | TableName: testConstants.dummyTestTableName, 22 | KeySchema: [ 23 | { 24 | AttributeName: 'tuid', 25 | KeyType: 'HASH' 26 | }, //Partition key 27 | { 28 | AttributeName: 'cid', 29 | KeyType: 'RANGE' 30 | } //Sort key 31 | ], 32 | AttributeDefinitions: [ 33 | { AttributeName: 'tuid', AttributeType: 'S' }, 34 | { AttributeName: 'cid', AttributeType: 'N' } 35 | ], 36 | ProvisionedThroughput: { 37 | ReadCapacityUnits: 1, 38 | WriteCapacityUnits: 1 39 | }, 40 | SSESpecification: { 41 | Enabled: false 42 | } 43 | }; 44 | await helper.createTable(ddb_service, createTableParams, true); 45 | }); 46 | 47 | it('should update table successfully', async function() { 48 | // build create table params 49 | const updateTableParams = { 50 | TableName: testConstants.dummyTestTableName, 51 | ProvisionedThroughput: { 52 | ReadCapacityUnits: 5, 53 | WriteCapacityUnits: 5 54 | } 55 | }; 56 | await helper.updateTable(ddb_service, updateTableParams, true); 57 | }); 58 | 59 | it('should fail when no update config data is passed', async function() { 60 | // build create table params 61 | const updateTableParams = { 62 | TableName: testConstants.dummyTestTableName 63 | }; 64 | await helper.updateTable(ddb_service, updateTableParams, false); 65 | }); 66 | 67 | after(async function() { 68 | const deleteTableParams = { 69 | TableName: testConstants.dummyTestTableName 70 | }; 71 | await helper.deleteTable(ddb_service, deleteTableParams, true); 72 | logger.debug('Update Table Mocha Tests Complete'); 73 | }); 74 | }); 75 | --------------------------------------------------------------------------------