├── .babelrc ├── .eslintrc ├── .flowconfig ├── .gitignore ├── LICENSE ├── README.md ├── flow ├── aws-sdk.js ├── invariant.js ├── measured.js └── warning.js ├── gulpfile.js ├── make-webpack-config.js ├── package-lock.json ├── package.json ├── scripts └── start.js ├── src ├── App.js ├── CapacityCalculator.js ├── Global.js ├── Index.js ├── Provisioner.js ├── aws │ ├── CloudWatch.js │ └── DynamoDB.js ├── capacity │ └── CapacityCalculatorBase.js ├── configuration │ ├── ClimbingProvisioner.json │ ├── DefaultProvisioner.json │ ├── FixedProvisioner.json │ └── Region.json ├── flow │ └── FlowTypes.js ├── provisioning │ ├── ProvisionerBase.js │ ├── ProvisionerConfigurableBase.js │ └── ProvisionerLogging.js └── utils │ ├── CostEstimation.js │ ├── Delay.js │ ├── RateLimitedDecrement.js │ ├── Stats.js │ └── Throughput.js ├── webpack-dev.config.js ├── webpack-prod.config.js └── webpack.config.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["react-native"] 3 | } 4 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "babel-eslint", 3 | 4 | "plugins": [ 5 | "babel" 6 | ], 7 | 8 | "env": { 9 | "es6": true, 10 | "node": true 11 | }, 12 | 13 | "ecmaFeatures": { 14 | "arrowFunctions": true, 15 | "binaryLiterals": true, 16 | "blockBindings": true, 17 | "classes": true, 18 | "defaultParams": true, 19 | "destructuring": true, 20 | "experimentalObjectRestSpread": true, 21 | "forOf": true, 22 | "generators": true, 23 | "globalReturn": true, 24 | "jsx": true, 25 | "modules": true, 26 | "objectLiteralComputedProperties": true, 27 | "objectLiteralDuplicateProperties": true, 28 | "objectLiteralShorthandMethods": true, 29 | "objectLiteralShorthandProperties": true, 30 | "octalLiterals": true, 31 | "regexUFlag": true, 32 | "regexYFlag": true, 33 | "restParams": true, 34 | "spread": true, 35 | "superInFunctions": true, 36 | "templateStrings": true, 37 | "unicodeCodePointEscapes": true 38 | }, 39 | 40 | "rules": { 41 | "babel/arrow-parens": [2, "as-needed"], 42 | 43 | "array-bracket-spacing": [2, "always"], 44 | "arrow-spacing": 2, 45 | "block-scoped-var": 0, 46 | "brace-style": [2, "1tbs", {"allowSingleLine": true}], 47 | "callback-return": 2, 48 | "camelcase": [2, {"properties": "always"}], 49 | "comma-dangle": 0, 50 | "comma-spacing": 0, 51 | "comma-style": [2, "last"], 52 | "complexity": 0, 53 | "computed-property-spacing": [2, "never"], 54 | "consistent-return": 0, 55 | "consistent-this": 0, 56 | "curly": [2, "all"], 57 | "default-case": 0, 58 | "dot-location": [2, "property"], 59 | "dot-notation": 0, 60 | "eol-last": 2, 61 | "eqeqeq": [2, "allow-null"], 62 | "func-names": 0, 63 | "func-style": 0, 64 | "generator-star-spacing": [0, {"before": true, "after": false}], 65 | "guard-for-in": 2, 66 | "handle-callback-err": [2, "error"], 67 | "id-length": 0, 68 | "id-match": [2, "^(?:_?[a-zA-Z0-9]*)|[_A-Z0-9]+$"], 69 | "indent": [2, 2, {"SwitchCase": 1}], 70 | "init-declarations": 0, 71 | "key-spacing": [2, {"beforeColon": false, "afterColon": true}], 72 | "linebreak-style": 2, 73 | "lines-around-comment": 0, 74 | "max-depth": 0, 75 | "max-len": [2, 100, 4], 76 | "max-nested-callbacks": 0, 77 | "max-params": 0, 78 | "max-statements": 0, 79 | "new-cap": 0, 80 | "new-parens": 2, 81 | "newline-after-var": 0, 82 | "no-alert": 2, 83 | "no-array-constructor": 2, 84 | "no-bitwise": 0, 85 | "no-caller": 2, 86 | "no-catch-shadow": 0, 87 | "no-class-assign": 2, 88 | "no-cond-assign": 2, 89 | "no-console": 1, 90 | "no-const-assign": 2, 91 | "no-constant-condition": 2, 92 | "no-continue": 0, 93 | "no-control-regex": 0, 94 | "no-debugger": 1, 95 | "no-delete-var": 2, 96 | "no-div-regex": 2, 97 | "no-dupe-args": 2, 98 | "no-dupe-keys": 2, 99 | "no-duplicate-case": 2, 100 | "no-else-return": 2, 101 | "no-empty": 2, 102 | "no-empty-character-class": 2, 103 | // "no-empty-label": 2, 104 | "no-eq-null": 0, 105 | "no-eval": 2, 106 | "no-ex-assign": 2, 107 | "no-extend-native": 2, 108 | "no-extra-bind": 2, 109 | "no-extra-boolean-cast": 2, 110 | "no-extra-parens": 0, 111 | "no-extra-semi": 2, 112 | "no-fallthrough": 2, 113 | "no-floating-decimal": 2, 114 | "no-func-assign": 2, 115 | "no-implicit-coercion": 2, 116 | "no-implied-eval": 2, 117 | "no-inline-comments": 0, 118 | "no-inner-declarations": [2, "functions"], 119 | "no-invalid-regexp": 2, 120 | "no-invalid-this": 0, 121 | "no-irregular-whitespace": 2, 122 | "no-iterator": 2, 123 | "no-label-var": 2, 124 | "no-labels": 0, 125 | "no-lone-blocks": 2, 126 | "no-lonely-if": 2, 127 | "no-loop-func": 0, 128 | "no-mixed-requires": [2, true], 129 | "no-mixed-spaces-and-tabs": 2, 130 | "no-multi-spaces": 2, 131 | "no-multi-str": 2, 132 | "no-multiple-empty-lines": 0, 133 | "no-native-reassign": 0, 134 | "no-negated-in-lhs": 2, 135 | "no-nested-ternary": 0, 136 | "no-new": 2, 137 | "no-new-func": 0, 138 | "no-new-object": 2, 139 | "no-new-require": 2, 140 | "no-new-wrappers": 2, 141 | "no-obj-calls": 2, 142 | "no-octal": 2, 143 | "no-octal-escape": 2, 144 | "no-param-reassign": 2, 145 | "no-path-concat": 2, 146 | "no-plusplus": 0, 147 | "no-process-env": 0, 148 | "no-process-exit": 0, 149 | "no-proto": 2, 150 | "no-redeclare": 2, 151 | "no-regex-spaces": 2, 152 | "no-restricted-modules": 0, 153 | "no-return-assign": 2, 154 | "no-script-url": 2, 155 | "no-self-compare": 0, 156 | "no-sequences": 2, 157 | "no-shadow": 2, 158 | "no-shadow-restricted-names": 2, 159 | "no-spaced-func": 2, 160 | "no-sparse-arrays": 2, 161 | "no-sync": 2, 162 | "no-ternary": 0, 163 | "no-this-before-super": 2, 164 | "no-throw-literal": 2, 165 | "no-trailing-spaces": 2, 166 | "no-undef": 2, 167 | "no-undef-init": 2, 168 | "no-undefined": 0, 169 | "no-underscore-dangle": 0, 170 | "no-unexpected-multiline": 2, 171 | "no-unneeded-ternary": 2, 172 | "no-unreachable": 2, 173 | "no-unused-expressions": 2, 174 | "no-unused-vars": [2, {"vars": "all", "args": "after-used"}], 175 | "no-use-before-define": 0, 176 | "no-useless-call": 2, 177 | "no-var": 0, 178 | "no-void": 2, 179 | "no-warning-comments": 0, 180 | "no-with": 2, 181 | "object-curly-spacing": [0, "always"], 182 | "object-shorthand": [2, "always"], 183 | "one-var": [2, "never"], 184 | "operator-assignment": [2, "always"], 185 | "operator-linebreak": [2, "after"], 186 | "padded-blocks": 0, 187 | "prefer-const": 0, 188 | "prefer-reflect": 0, 189 | "prefer-spread": 0, 190 | "quote-props": [2, "as-needed"], 191 | "quotes": [2, "single"], 192 | "radix": 2, 193 | "require-yield": 2, 194 | "semi": [2, "always"], 195 | "semi-spacing": [2, {"before": false, "after": true}], 196 | "sort-vars": 0, 197 | // "space-after-keywords": [2, "always"], 198 | "space-before-blocks": [2, "always"], 199 | "space-before-function-paren": [2, {"anonymous": "always", "named": "never"}], 200 | "space-in-parens": 0, 201 | "space-infix-ops": [2, {"int32Hint": false}], 202 | // "space-return-throw-case": 2, 203 | "space-unary-ops": [2, {"words": true, "nonwords": false}], 204 | "spaced-comment": [2, "always"], 205 | "strict": 0, 206 | "use-isnan": 2, 207 | "valid-jsdoc": 0, 208 | "valid-typeof": 2, 209 | "vars-on-top": 0, 210 | "wrap-iife": 2, 211 | "wrap-regex": 0, 212 | "yoda": [2, "never", {"exceptRange": true}] 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /.flowconfig: -------------------------------------------------------------------------------- 1 | [ignore] 2 | .*/lib/.* 3 | .*/dist/.* 4 | .*/coverage/.* 5 | .*/resources/.* 6 | 7 | [include] 8 | 9 | [libs] 10 | ./flow 11 | 12 | [options] 13 | esproposal.class_static_fields=enable 14 | esproposal.class_instance_fields=enable 15 | munge_underscores=true 16 | suppress_comment= \\(.\\|\n\\)*\\$FlowIgnore 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 18 | .grunt 19 | 20 | # node-waf configuration 21 | .lock-wscript 22 | 23 | # Compiled binary addons (http://nodejs.org/api/addons.html) 24 | build/Release 25 | 26 | # Dependency directory 27 | node_modules 28 | 29 | # Optional npm cache directory 30 | .npm 31 | 32 | # Optional REPL history 33 | .node_repl_history 34 | 35 | dist/ 36 | dist.zip 37 | build/ 38 | config.env.production 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 for dynamodb-lambda-autoscale 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dynamodb-lambda-autoscale 2 | **Autoscale AWS DynamoDB using an AWS Lambda function** 3 | 4 | + 5 minute setup process 5 | + Serverless design 6 | + Flexible code over configuration style 7 | + Autoscale table and global secondary indexes 8 | + Autoscale multiple tables 9 | + Autoscale by fixed settings 10 | + Autoscale by provisioned capacity utilisation 11 | + Autoscale by throttled event metrics 12 | + Optimised for large spikes in usage and hotkey issues by incorporating throttled event metrics 13 | + Optimised performance using concurrent queries 14 | + RateLimitedDecrement as imposed by AWS 15 | + Statistics via 'measured' 16 | + AWS credential configuration via 'dotenv' 17 | + Optimised lambda package via 'webpack' 18 | + ES7 code 19 | + 100% [Flow](https://flowtype.org/) static type checking coverage 20 | 21 | ## Disclaimer 22 | 23 | Any reliance you place on dynamodb-lambda-autoscale is strictly at your own 24 | risk. 25 | 26 | In no event will we be liable for any loss or damage including without 27 | limitation, indirect or consequential loss or damage, or any loss or damage 28 | whatsoever arising from loss of data or profits arising out of, or in 29 | connection with, the use of this code. 30 | 31 | ## Getting started 32 | 33 | Note: dynamodb-lambda-autoscale uses [Flow](https://flowtype.org/) extensively for static type 34 | checking, we highly recommend you use [Nuclide](https://nuclide.io/) when making modification to code / 35 | configuration. Please see the respective websites for advantages / reasons. 36 | 37 | 1. Build and package the code 38 | 1. Fork the repo 39 | 2. Clone your fork 40 | 3. Create a new file in the root folder called 'config.env.production' 41 | 4. Put your AWS credentials into the file in the following format, only if you want to run a local test (not needed for lambda) 42 | 43 | ```javascript 44 | AWS_ACCESS_KEY_ID="###################" 45 | AWS_SECRET_ACCESS_KEY="###############" 46 | ``` 47 | 48 | 5. Update [Region.json](./src/configuration/Region.json) to match the region of your DynamoDB instance 49 | 6. Run 'npm install' 50 | 7. Run 'npm run build' 51 | 8. Verify this has created a 'dist.zip' file 52 | 9. Optionally, run a local test by running 'npm run start' 53 | 54 | ## Running on AWS Lambda 55 | 56 | 1. Follow the steps in 'Running locally' 57 | 2. Create an AWS Policy and Role 58 | 1. Create a policy called 'DynamoDBLambdaAutoscale' 59 | 2. Use the following content to give access to dynamoDB, cloudwatch and lambda logging 60 | 61 | ```javascript 62 | { 63 | "Version": "2012-10-17", 64 | "Statement": [ 65 | { 66 | "Action": [ 67 | "dynamodb:ListTables", 68 | "dynamodb:DescribeTable", 69 | "dynamodb:UpdateTable", 70 | "cloudwatch:GetMetricStatistics", 71 | "logs:CreateLogGroup", 72 | "logs:CreateLogStream", 73 | "logs:PutLogEvents" 74 | ], 75 | "Effect": "Allow", 76 | "Resource": "*" 77 | } 78 | ] 79 | } 80 | ``` 81 | 82 | 3. Create a role called 'DynamoDBLambdaAutoscale' 83 | 4. Attach the newly created policy to the role 84 | 3. Create a AWS Lambda function 85 | 1. Skip the pre defined functions step 86 | 2. Set the name to 'DynamoDBLambdaAutoscale' 87 | 3. Set the runtime to 'Node.js 4.3' 88 | 4. Select upload a zip file and select 'dist.zip' which you created earlier 89 | 5. Set the handler to 'index.handler' 90 | 6. Set the Role to 'DynamoDBLambdaAutoscale' 91 | 7. Set the Memory to the lowest value initially but test different values at a later date to see how it affects performance 92 | 8. Set the Timeout to approximately 5 seconds (higher or lower depending on the amount of tables you have and the selected memory setting) 93 | 9. Once the function is created, attach a 'scheduled event' event source and make it run every minute. Event Sources > Add Event Source > Event Type = Cloudwatch Events - Schedule. Set the name to 'DynamoDBLambdaAutoscale' and the schedule expression to 'rate(1 minute)' 94 | 95 | ## Configuration 96 | 97 | The default setup in the [Provisioner.js](./src/Provisioner.js) allows for a quick no touch setup. 98 | A breakdown of the configuration behaviour is as follows: 99 | - AWS region is set to 'us-east-1' via [Region.json](./src/configuration/Region.json) configuration 100 | - Autoscales all tables and indexes 101 | - Autoscaling 'Strategy' settings are defined in [DefaultProvisioner.json](./src/configuration/DefaultProvisioner.json) and are as follows 102 | - Separate 'Read' and 'Write' capacity adjustment strategies 103 | - Separate asymmetric 'Increment' and 'Decrement' capacity adjustment strategies 104 | - Read/Write provisioned capacity increased 105 | - when capacity utilisation > 75% or throttled events in the last minute > 25 106 | - by 3 + (0.7 * throttled events) units or by 30% + (0.7 * throttled events) of provisioned value or to 130% of the current consumed capacity, which ever is the greater 107 | - with hard min/max limits of 1 and 100 respectively 108 | - Read/Write provisioned capacity decreased 109 | - when capacity utilisation < 30% AND 110 | - when at least 60 minutes have passed since the last increment AND 111 | - when at least 60 minutes have passed since the last decrement AND 112 | - when the adjustment will be at least 5 units AND 113 | - when we are allowed to utilise 1 of our 4 AWS enforced decrements 114 | - to the consumed throughput value 115 | - with hard min/max limits of 1 and 100 respectively 116 | 117 | ## Strategy Settings 118 | 119 | The strategy settings described above uses a simple schema which applies to both Read/Write and to 120 | both the Increment/Decrement. Using the options below many different strategies can be constructed: 121 | - ReadCapacity.Min : (Optional) Define a minimum allowed capacity, otherwise 1 122 | - ReadCapacity.Max : (Optional) Define a maximum allowed capacity, otherwise unlimited 123 | - ReadCapacity.Increment : (Optional) Defined an increment strategy 124 | - ReadCapacity.Increment.When : (Required) Define when capacity should be incremented 125 | - ReadCapacity.Increment.When.ThrottledEventsPerMinuteIsAbove : (Optional) Define a threshold at which throttled events trigger an increment 126 | - ReadCapacity.Increment.When.UtilisationIsAbovePercent : (Optional) Define a percentage utilisation upper threshold at which capacity is subject to recalculation 127 | - ReadCapacity.Increment.When.UtilisationIsBelowPercent : (Optional) Define a percentage utilisation lower threshold at which capacity is subject to recalculation, possible but non sensical for increments however. 128 | - ReadCapacity.Increment.When.AfterLastIncrementMinutes : (Optional) Define a grace period based off the previous increment in which capacity adjustments should not occur 129 | - ReadCapacity.Increment.When.AfterLastDecrementMinutes : (Optional) Define a grace period based off the previous decrement in which capacity adjustments should not occur 130 | - ReadCapacity.Increment.When.UnitAdjustmentGreaterThan : (Optional) Define a minimum unit adjustment so that only capacity adjustments of a certain size are allowed 131 | - ReadCapacity.Increment.By : (Optional) Define a 'relative' value to change the capacity by 132 | - ReadCapacity.Increment.By.ConsumedPercent : (Optional) Define a 'relative' percentage adjustment based on the current ConsumedCapacity 133 | - ReadCapacity.Increment.By.ProvisionedPercent : (Optional) Define a 'relative' percentage adjustment based on the current ProvisionedCapacity 134 | - ReadCapacity.Increment.By.Units : (Optional) Define a 'relative' unit adjustment 135 | - ReadCapacity.Increment.By.ThrottledEventsWithMultiplier : (Optional) Define a 'multiple' of the throttled events in the last minute which are added to all other 'By' unit adjustments 136 | - ReadCapacity.Increment.To : (Optional) Define an 'absolute' value to change the capacity to 137 | - ReadCapacity.Increment.To.ConsumedPercent : (Optional) Define an 'absolute' percentage adjustment based on the current ConsumedCapacity 138 | - ReadCapacity.Increment.To.ProvisionedPercent : (Optional) Define an 'absolute' percentage adjustment based on the current ProvisionedCapacity 139 | - ReadCapacity.Increment.To.Units : (Optional) Define an 'absolute' unit adjustment 140 | 141 | A sample of the strategy setting json is... 142 | ```javascript 143 | { 144 | "ReadCapacity": { 145 | "Min": 1, 146 | "Max": 100, 147 | "Increment": { 148 | "When": { 149 | "UtilisationIsAbovePercent": 75, 150 | "ThrottledEventsPerMinuteIsAbove": 25 151 | }, 152 | "By": { 153 | "Units": 3, 154 | "ProvisionedPercent": 30, 155 | "ThrottledEventsWithMultiplier": 0.7 156 | }, 157 | "To": { 158 | "ConsumedPercent": 130 159 | } 160 | }, 161 | "Decrement": { 162 | "When": { 163 | "UtilisationIsBelowPercent": 30, 164 | "AfterLastIncrementMinutes": 60, 165 | "AfterLastDecrementMinutes": 60, 166 | "UnitAdjustmentGreaterThan": 5 167 | }, 168 | "To": { 169 | "ConsumedPercent": 100 170 | } 171 | } 172 | }, 173 | "WriteCapacity": { 174 | "Min": 1, 175 | "Max": 100, 176 | "Increment": { 177 | "When": { 178 | "UtilisationIsAbovePercent": 75, 179 | "ThrottledEventsPerMinuteIsAbove": 25 180 | }, 181 | "By": { 182 | "Units": 3, 183 | "ProvisionedPercent": 30, 184 | "ThrottledEventsWithMultiplier": 0.7 185 | }, 186 | "To": { 187 | "ConsumedPercent": 130 188 | } 189 | }, 190 | "Decrement": { 191 | "When": { 192 | "UtilisationIsBelowPercent": 30, 193 | "AfterLastIncrementMinutes": 60, 194 | "AfterLastDecrementMinutes": 60, 195 | "UnitAdjustmentGreaterThan": 5 196 | }, 197 | "To": { 198 | "ConsumedPercent": 100 199 | } 200 | } 201 | } 202 | } 203 | ``` 204 | 205 | ## Advanced Configuration 206 | 207 | This project takes a 'React' style code first approach over declarative configuration traditionally 208 | used by other autoscaling community projects. Rather than being limited to a structured 209 | configuration file or even the 'strategy' settings above you have the option to extend the [ProvisionerBase.js](./src/provisioning/ProvisionerBase.js) 210 | abstract base class for yourself and programmatically implement any desired logic. 211 | 212 | The following three functions are all that is required to complete the provisioning functionality. 213 | As per the 'React' style, only actual updates to the ProvisionedCapacity will be sent to AWS. 214 | 215 | ```javascript 216 | getDynamoDBRegion(): string { 217 | // Return the AWS region as a string 218 | } 219 | 220 | async getTableNamesAsync(): Promise { 221 | // Return the table names to apply autoscaling to as a string array promise 222 | } 223 | 224 | async getTableUpdateAsync( 225 | tableDescription: TableDescription, 226 | tableConsumedCapacityDescription: TableConsumedCapacityDescription): 227 | Promise { 228 | // Given an AWS DynamoDB TableDescription and AWS CloudWatch ConsumedCapacity metrics 229 | // return an AWS DynamoDB UpdateTable request 230 | } 231 | ``` 232 | [DescribeTable.ResponseSyntax](http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DescribeTable.html#API_DescribeTable_ResponseSyntax) 233 | [UpdateTable.RequestSyntax](http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateTable.html#API_UpdateTable_RequestSyntax) 234 | 235 | Flexibility is great, but implementing all the logic required for a robust autoscaling 236 | strategy isn't something everyone wants to do. Hence, the default 'Provisioner' builds upon the base 237 | class in a layered approach. The layers are as follows: 238 | - [Provisioner.js](./src/Provisioner.js) concrete implementation which provides very robust autoscaling logic which can be manipulated with a 'strategy' settings json object 239 | - [ProvisionerConfigurableBase.js](./src/provisioning/ProvisionerConfigurableBase.js) abstract base class which breaks out the 'getTableUpdateAsync' function into more manageable abstract methods 240 | - [ProvisionerBase.js](./src/provisioning/ProvisionerBase.js) the root abstract base class which defines the minimum contract 241 | 242 | ## Throttled Events 243 | Throttled events are now taken into account as part of the provisioning calculation. A multiple of the events can be added to the existing calculation so that both large spikes in usage and hot key issues are both dealt with. 244 | 245 | ## Rate Limited Decrement 246 | 247 | AWS only allows 4 table decrements in a calendar day. To account for this we have included 248 | an algorithm which segments the remaining time to midnight by the amount of decrements we have left. 249 | This logic allows us to utilise each 4 decrements as efficiently as possible. The increments on the 250 | other hand are unlimited, so the algorithm follows a unique 'sawtooth' profile, dropping the 251 | provisioned capacity all the way down to the consumed throughput rather than gradually. Please see 252 | [RateLimitedDecrement.js](./src/utils/RateLimitedDecrement.js) for full implementation. 253 | 254 | ## Capacity Calculation 255 | 256 | As well as implementing the correct Provisioning logic it is also important to calculate the 257 | ConsumedCapacity for the current point in time. We have provided a default algorithm in 258 | [CapacityCalculator.js](./src/CapacityCalculator.js) which should be good enough for most purposes 259 | but it could be swapped out with perhaps an improved version. The newer version could potentially 260 | take a series of data points and plot a linear regression line through them for example. 261 | 262 | ## Dependencies 263 | 264 | This project has the following main dependencies (n.b. all third party dependencies are compiled 265 | into a single javascript file before being zipped and uploaded to lambda): 266 | + aws-sdk - Access to AWS services 267 | + dotenv - Environment variable configuration useful for lambda 268 | + measured - Statistics gathering 269 | 270 | ## Licensing 271 | 272 | The source code is licensed under the MIT license found in the 273 | [LICENSE](LICENSE) file in the root directory of this source tree. 274 | -------------------------------------------------------------------------------- /flow/aws-sdk.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | declare module 'aws-sdk' { 3 | declare class DynamoDB { 4 | constructor(dynamoDBConfig: DynamoDBConfig): void; 5 | 6 | listTables(params: ?ListTablesRequest, callback: ?(err: ?Error, 7 | data: ListTablesResponse) => void): PromiseRequest; 8 | 9 | deleteTable(params: ?DeleteTableRequest,callback: ?(err: ?Error, 10 | data: DeleteTableResponse) => void): PromiseRequest; 11 | 12 | createTable(params: ?CreateTableRequest, callback: ?(err: ?Error, 13 | data: CreateTableResponse) => void): PromiseRequest; 14 | 15 | describeTable(params: ?DescribeTableRequest, callback: ?(err: ?Error, 16 | data: DescribeTableResponse) => void): PromiseRequest; 17 | 18 | updateTable(params: ?UpdateTableRequest, callback: ?(err: ?Error, 19 | data: UpdateTableResponse) => void): PromiseRequest; 20 | 21 | scan(params: ?ScanRequest, callback: ?(err: ?Error, 22 | data: ScanQueryResponse) => void): PromiseRequest; 23 | 24 | query(params: ?QueryRequest, callback: ?(err: ?Error, 25 | data: ScanQueryResponse) => void): PromiseRequest; 26 | 27 | putItem(params: ?PutItemRequest, callback: ?(err: ?Error, 28 | data: PutItemResponse) => void): PromiseRequest; 29 | 30 | getItem(params: ?GetItemRequest, callback: ?(err: ?Error, 31 | data: GetItemResponse) => void): PromiseRequest; 32 | 33 | batchGetItem(params: ?BatchGetItemRequest, callback: ?(err: ?Error, 34 | data: BatchGetItemResponse) => void): PromiseRequest; 35 | 36 | batchWriteItem(params: ?BatchWriteItemRequest, callback: ?(err: ?Error, 37 | data: BatchWriteItemResponse) => void): PromiseRequest; 38 | 39 | deleteItem(params: ?DeleteItemRequest, callback: ?(err: ?Error, 40 | data: DeleteItemResponse) => void): PromiseRequest; 41 | 42 | updateItem(params: ?UpdateItemRequest, callback: ?(err: ?Error, 43 | data: UpdateItemResponse) => void): PromiseRequest; 44 | } 45 | 46 | declare class CloudWatch { 47 | getMetricStatistics(params: ?GetMetricStatisticsRequest, callback: ?(err: ?Error, 48 | data: GetMetricStatisticsResponse) => void): PromiseRequest; 49 | } 50 | 51 | declare class PromiseRequest { 52 | promise(): Promise; 53 | } 54 | 55 | declare type ListTablesRequest = { 56 | ExclusiveStartTableName?: string, 57 | Limit?: number 58 | }; 59 | 60 | declare type ListTablesResponse = { 61 | LastEvaluatedTableName?: string, 62 | TableNames: string[] 63 | }; 64 | 65 | declare type DeleteTableRequest = { 66 | TableName: string, 67 | }; 68 | 69 | declare type DeleteTableResponse = { 70 | TableDescription: TableDescription, 71 | }; 72 | 73 | declare type CreateTableRequest = { 74 | AttributeDefinitions: AttributeDefinition[], 75 | KeySchema: KeyDefinition[], 76 | ProvisionedThroughput: ProvisionedThroughput, 77 | TableName: string, 78 | GlobalSecondaryIndexes: GlobalSecondaryIndex[], 79 | LocalSecondaryIndexes: LocalSecondaryIndex[], 80 | StreamSpecification: StreamSpecification, 81 | }; 82 | 83 | declare type CreateTableResponse = { 84 | TableDescription: TableDescription, 85 | }; 86 | 87 | declare type ScanRequest = { 88 | AttributesToGet: string[], 89 | ConditionalOperator: string, 90 | ConsistentRead: boolean, 91 | ExclusiveStartKey: any, 92 | ExpressionAttributeNames: any, 93 | ExpressionAttributeValues: any, 94 | FilterExpression: string, 95 | IndexName: string, 96 | Limit: number, 97 | ProjectionExpression: string, 98 | ReturnConsumedCapacity: string, 99 | ScanFilter: any, 100 | Segment: number, 101 | Select: string, 102 | TableName: string, 103 | TotalSegments: number 104 | }; 105 | 106 | declare type TableConsumedCapacity = { 107 | CapacityUnits: number, 108 | }; 109 | 110 | declare type ConsumedCapacity = { 111 | CapacityUnits: number, 112 | GlobalSecondaryIndexes: any, 113 | LocalSecondaryIndexes: any, 114 | Table: TableConsumedCapacity, 115 | TableName: string 116 | }; 117 | 118 | declare type ScanQueryResponse = { 119 | ConsumedCapacity: ConsumedCapacity, 120 | Count: number, 121 | Items: any[], 122 | LastEvaluatedKey: any, 123 | ScannedCount: number, 124 | }; 125 | 126 | declare type QueryRequest = { 127 | AttributesToGet: string[], 128 | ConditionalOperator: string, 129 | ConsistentRead: boolean, 130 | ExclusiveStartKey: any, 131 | ExpressionAttributeNames: any, 132 | ExpressionAttributeValues: any, 133 | FilterExpression: string, 134 | IndexName: string, 135 | KeyConditionExpression: string, 136 | KeyConditions: any, 137 | Limit: number, 138 | ProjectionExpression: string, 139 | QueryFilter: any, 140 | ReturnConsumedCapacity: string, 141 | ScanIndexForward: boolean, 142 | Select: string, 143 | TableName: string, 144 | }; 145 | 146 | declare type PutItemRequest = { 147 | ConditionalOperator: string, 148 | ConditionExpression: string, 149 | Expected: any, 150 | ExpressionAttributeNames: string, 151 | ExpressionAttributeValues: any, 152 | Item: any, 153 | ReturnConsumedCapacity: string, 154 | ReturnItemCollectionMetrics: string, 155 | ReturnValues: string, 156 | TableName: string, 157 | }; 158 | 159 | declare type PutItemResponse = { 160 | Attributes: any, 161 | ConsumedCapacity: ConsumedCapacity, 162 | ItemCollectionMetrics: any, 163 | }; 164 | 165 | declare type GetItemRequest = { 166 | AttributesToGet: string[], 167 | ConsistentRead: boolean, 168 | ExpressionAttributeNames: any, 169 | Key: any, 170 | ProjectionExpression: string, 171 | ReturnConsumedCapacity: string, 172 | TableName: string, 173 | }; 174 | 175 | declare type GetItemResponse = { 176 | ConsumedCapacity: ConsumedCapacity, 177 | Item: any, 178 | }; 179 | 180 | declare type BatchGetItemRequest = { 181 | RequestItems: any, 182 | ReturnConsumedCapacity: string, 183 | }; 184 | 185 | declare type BatchGetItemResponse = { 186 | ConsumedCapacity: ConsumedCapacity[], 187 | Responses: any, 188 | UnprocessedKeys: any, 189 | }; 190 | 191 | declare type BatchWriteItemRequest = { 192 | RequestItems: any, 193 | ReturnConsumedCapacity: string, 194 | ReturnItemCollectionMetrics: string, 195 | }; 196 | 197 | declare type BatchWriteItemResponse = { 198 | ConsumedCapacity: ConsumedCapacity[], 199 | ItemCollectionMetrics: ItemCollectionMetrics, 200 | UnprocessedKeys: any, 201 | }; 202 | 203 | declare type DeleteItemRequest = { 204 | ConditionalOperator: string, 205 | ConditionExpression: string, 206 | Expected: any, 207 | ExpressionAttributeNames: any, 208 | ExpressionAttributeValues: any, 209 | Key: any, 210 | ReturnConsumedCapacity: string, 211 | ReturnItemCollectionMetrics: string, 212 | ReturnValues: string, 213 | TableName: string, 214 | }; 215 | 216 | declare type DeleteItemResponse = { 217 | Attributes: any, 218 | ConsumedCapacity: ConsumedCapacity, 219 | ItemCollectionMetrics: ItemCollectionMetrics, 220 | }; 221 | 222 | declare type UpdateItemRequest = { 223 | AttributeUpdates: any, 224 | ConditionalOperator: string, 225 | ConditionExpression: string, 226 | Expected: any, 227 | ExpressionAttributeNames: any, 228 | ExpressionAttributeValues: any, 229 | ReturnConsumedCapacity: string, 230 | ReturnItemCollectionMetrics: string, 231 | ReturnValues: string, 232 | TableName: string, 233 | UpdateExpression: string 234 | }; 235 | 236 | declare type UpdateItemResponse = { 237 | Attributes: any, 238 | ConsumedCapacity: ConsumedCapacity, 239 | ItemCollectionMetrics: ItemCollectionMetrics, 240 | }; 241 | 242 | declare type ItemCollectionMetrics = { 243 | ItemCollectionKey: any, 244 | SizeEstimateRangeGB: number[], 245 | }; 246 | 247 | declare type DynamoDBConfig = { 248 | apiVersion: string, 249 | region: string, 250 | dynamoDbCrc32: boolean 251 | }; 252 | 253 | declare type DynamoDBAttributeDefinition = { 254 | AttributeName: string, 255 | AttributeType: string 256 | }; 257 | 258 | declare type DynamoDBKeySchema = { 259 | AttributeName: string, 260 | KeyType: string 261 | }; 262 | 263 | declare type DynamoDBTable = { 264 | TableName: string, 265 | AttributeDefinitions: DynamoDBAttributeDefinition[], 266 | KeySchema: DynamoDBKeySchema[], 267 | GlobalSecondaryIndexes?: DynamoDBGlobalSecondaryIndex[], 268 | LocalSecondaryIndexes?: DynamoDBLocalSecondaryIndex[], 269 | ProvisionedThroughput: DynamoDBProvisionedThroughput, 270 | StreamSpecification?: DynamoDBStreamSpecification 271 | }; 272 | 273 | declare type DynamoDBGlobalSecondaryIndex = { 274 | IndexName: string, 275 | KeySchema: DynamoDBKeySchema [], 276 | Projection: DynamoDBProjection, 277 | ProvisionedThroughput: DynamoDBProvisionedThroughput 278 | }; 279 | 280 | declare type DynamoDBLocalSecondaryIndex = { 281 | IndexName: string, 282 | KeySchema: DynamoDBKeySchema [], 283 | Projection: DynamoDBProjection, 284 | }; 285 | 286 | declare type DynamoDBProjection = { 287 | NonKeyAttributes?: string[], 288 | ProjectionType: string 289 | }; 290 | 291 | declare type DynamoDBProvisionedThroughput = { 292 | ReadCapacityUnits: number, 293 | WriteCapacityUnits: number 294 | }; 295 | 296 | declare type DynamoDBStreamSpecification = { 297 | StreamEnabled: boolean, 298 | StreamViewType: string 299 | }; 300 | 301 | declare type DynamoDBSchema = { 302 | tables: DynamoDBTable[] 303 | }; 304 | 305 | // DynamoDB 306 | declare type DynamoDBOptions = { 307 | apiVersion: string, 308 | region: string, 309 | dynamoDbCrc32: boolean, 310 | httpOptions: HTTPOptions, 311 | }; 312 | 313 | declare type HTTPOptions = { 314 | timeout: number, 315 | }; 316 | 317 | declare type AttributeDefinition = { 318 | AttributeName: string, 319 | AttributeType: string, 320 | }; 321 | 322 | declare type KeyDefinition = { 323 | AttributeName: string, 324 | KeyType: string, 325 | }; 326 | 327 | declare type Projection = { 328 | NonKeyAttributes: string[], 329 | ProjectionType: string, 330 | }; 331 | 332 | declare type ProvisionedThroughput = { 333 | LastDecreaseDateTime: string, 334 | LastIncreaseDateTime: string, 335 | NumberOfDecreasesToday: number, 336 | ReadCapacityUnits: number, 337 | WriteCapacityUnits: number, 338 | }; 339 | 340 | declare type Throughput = { 341 | ReadCapacityUnits: number, 342 | WriteCapacityUnits: number, 343 | }; 344 | 345 | declare type GlobalSecondaryIndex = { 346 | Backfilling: boolean, 347 | IndexArn: string, 348 | IndexName: string, 349 | IndexSizeBytes: number, 350 | IndexStatus: string, 351 | ItemCount: number, 352 | KeySchema: KeyDefinition[], 353 | Projection: Projection, 354 | ProvisionedThroughput: ProvisionedThroughput, 355 | }; 356 | 357 | declare type LocalSecondaryIndex = { 358 | IndexArn: string, 359 | IndexName: string, 360 | IndexSizeBytes: number, 361 | ItemCount: number, 362 | KeySchema: KeyDefinition[], 363 | Projection: Projection, 364 | }; 365 | 366 | declare type StreamSpecification = { 367 | StreamEnabled: boolean, 368 | StreamViewType: string, 369 | }; 370 | 371 | declare type TableDescription = { 372 | AttributeDefinitions: AttributeDefinition[], 373 | CreationDateTime: number, 374 | GlobalSecondaryIndexes: GlobalSecondaryIndex[], 375 | ItemCount: number, 376 | KeySchema: KeyDefinition[], 377 | LatestStreamArn: string, 378 | LatestStreamLabel: string, 379 | LocalSecondaryIndexes: LocalSecondaryIndex[], 380 | ProvisionedThroughput: ProvisionedThroughput, 381 | StreamSpecification: StreamSpecification, 382 | TableArn: string, 383 | TableName: string, 384 | TableSizeBytes: number, 385 | TableStatus: string 386 | }; 387 | 388 | declare type DescribeTableRequest = { 389 | TableName: string, 390 | }; 391 | 392 | declare type DescribeTableResponse = { 393 | Table: TableDescription, 394 | }; 395 | 396 | declare type GlobalSecondaryIndexUpdateCreate = { 397 | IndexName: string, 398 | KeySchema: KeyDefinition[], 399 | Projection: Projection, 400 | ProvisionedThroughput: Throughput, 401 | }; 402 | 403 | declare type GlobalSecondaryIndexUpdateDelete = { 404 | IndexName: string, 405 | }; 406 | 407 | declare type GlobalSecondaryIndexUpdateUpdate = { 408 | IndexName: string, 409 | ProvisionedThroughput: Throughput, 410 | }; 411 | 412 | declare type GlobalSecondaryIndexUpdate = { 413 | Create?: GlobalSecondaryIndexUpdateCreate, 414 | Delete?: GlobalSecondaryIndexUpdateDelete, 415 | Update?: GlobalSecondaryIndexUpdateUpdate, 416 | }; 417 | 418 | declare type UpdateTableRequest = { 419 | AttributeDefinitions?: AttributeDefinition[], 420 | GlobalSecondaryIndexUpdates? : GlobalSecondaryIndexUpdate[], 421 | ProvisionedThroughput?: Throughput, 422 | StreamSpecification?: StreamSpecification, 423 | TableName: string 424 | }; 425 | 426 | declare type UpdateTableResponse = { 427 | TableDescription: TableDescription, 428 | }; 429 | 430 | // CloudWatch 431 | declare type CloudWatchOptions = { 432 | apiVersion: string, 433 | region: string, 434 | httpOptions: HTTPOptions, 435 | }; 436 | 437 | declare type GetMetricStatisticsResponse = { 438 | ResponseMetadata: ResponseMetadata, 439 | Label: string, 440 | Datapoints: Datapoint[], 441 | }; 442 | 443 | declare type Dimension = { 444 | Name: string, 445 | Value: string, 446 | }; 447 | 448 | declare type GetMetricStatisticsRequest = { 449 | Namespace: string, 450 | MetricName: string, 451 | Dimensions: Dimension[], 452 | StartTime: Date, 453 | EndTime: Date, 454 | Period: number, 455 | Statistics: string[], 456 | Unit: string, 457 | }; 458 | 459 | declare type ResponseMetadata = { 460 | RequestId: string, 461 | }; 462 | 463 | declare type Datapoint = { 464 | Timestamp: string, 465 | Average: number, 466 | Sum: number, 467 | Unit: string, 468 | }; 469 | } 470 | -------------------------------------------------------------------------------- /flow/invariant.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | declare module 'invariant' { 3 | declare class Invariant { 4 | (condition: boolean, message: string): any; 5 | } 6 | declare var exports: Invariant; 7 | } 8 | -------------------------------------------------------------------------------- /flow/measured.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | declare module 'measured' { 3 | declare class MeasuredCollection { 4 | _metrics: any; 5 | 6 | timer(name: string): MeasuredTimer; 7 | 8 | counter(name: string): MeasuredCounter; 9 | 10 | toJSON(): any; 11 | } 12 | 13 | declare class MeasuredTimer { 14 | start(): Stopwatch; 15 | } 16 | 17 | declare class MeasuredCounter { 18 | inc(value: number): void; 19 | } 20 | 21 | declare class Stopwatch { 22 | end(): void; 23 | } 24 | 25 | declare function createCollection(): MeasuredCollection; 26 | } 27 | -------------------------------------------------------------------------------- /flow/warning.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | declare module 'warning' { 3 | declare var exports: (shouldBeTrue: bool, warning: string) => void; 4 | } 5 | -------------------------------------------------------------------------------- /gulpfile.js: -------------------------------------------------------------------------------- 1 | var gulp = require("gulp"); 2 | var del = require('del'); 3 | var rename = require('gulp-rename'); 4 | var install = require('gulp-install'); 5 | var zip = require('gulp-zip'); 6 | var uglify = require('gulp-uglify'); 7 | var AWS = require('aws-sdk'); 8 | var fs = require('fs'); 9 | var runSequence = require('run-sequence'); 10 | var webpack = require('webpack-stream'); 11 | 12 | // First we need to clean out the dist folder and remove the compiled zip file. 13 | gulp.task('clean', function(cb) { 14 | del('./dist'); 15 | cb(); 16 | }); 17 | 18 | gulp.task("webpack", function () { 19 | return gulp.src('src/Index.js') 20 | .pipe(webpack( require('./webpack-dev.config.js') )) 21 | .pipe(gulp.dest('dist/')); 22 | }); 23 | 24 | // The js task could be replaced with gulp-coffee as desired. 25 | gulp.task("js", function () { 26 | return gulp 27 | .src("dist/index.js") 28 | .pipe(gulp.dest("dist/")); 29 | }); 30 | 31 | // Here we want to install npm packages to dist, ignoring devDependencies. 32 | gulp.task('npm', function() { 33 | return gulp 34 | .src('./package.json') 35 | .pipe(gulp.dest('./dist/')) 36 | .pipe(install({production: true})); 37 | }); 38 | 39 | // Next copy over environment variables managed outside of source control. 40 | gulp.task('env', function() { 41 | return gulp 42 | .src('./config.env.production') 43 | .pipe(rename('config.env')) 44 | .pipe(gulp.dest('./dist')); 45 | }); 46 | 47 | // Now the dist directory is ready to go. Zip it. 48 | gulp.task('zip', function() { 49 | return gulp 50 | .src(['dist/**/*', '!dist/package.json', 'dist/.*']) 51 | .pipe(zip('dist.zip')) 52 | .pipe(gulp.dest('./')); 53 | }); 54 | 55 | // Per the gulp guidelines, we do not need a plugin for something that can be 56 | // done easily with an existing node module. #CodeOverConfig 57 | // 58 | // Note: This presumes that AWS.config already has credentials. This will be 59 | // the case if you have installed and configured the AWS CLI. 60 | // 61 | // See http://aws.amazon.com/sdk-for-node-js/ 62 | gulp.task('upload', function() { 63 | 64 | // TODO: This should probably pull from package.json 65 | AWS.config.region = 'us-east-1'; 66 | var lambda = new AWS.Lambda(); 67 | var functionName = 'video-events'; 68 | 69 | lambda.getFunction({FunctionName: functionName}, function(err, data) { 70 | if (err) { 71 | if (err.statusCode === 404) { 72 | var warning = 'Unable to find lambda function ' + deploy_function + '. ' 73 | warning += 'Verify the lambda function name and AWS region are correct.' 74 | gutil.log(warning); 75 | } else { 76 | var warning = 'AWS API request failed. ' 77 | warning += 'Check your AWS credentials and permissions.' 78 | gutil.log(warning); 79 | } 80 | } 81 | 82 | // This is a bit silly, simply because these five parameters are required. 83 | var current = data.Configuration; 84 | var params = { 85 | FunctionName: functionName, 86 | Handler: current.Handler, 87 | Mode: current.Mode, 88 | Role: current.Role, 89 | Runtime: current.Runtime 90 | }; 91 | 92 | fs.readFile('./dist.zip', function(err, data) { 93 | params['FunctionZip'] = data; 94 | lambda.uploadFunction(params, function(err, data) { 95 | if (err) { 96 | var warning = 'Package upload failed. ' 97 | warning += 'Check your iam:PassRole permissions.' 98 | gutil.log(warning); 99 | } 100 | }); 101 | }); 102 | }); 103 | }); 104 | 105 | // The key to deploying as a single command is to manage the sequence of events. 106 | gulp.task('dist', function(cb) { 107 | return runSequence( 108 | ['clean'], 109 | ['webpack'], 110 | ['js', 'npm', 'env'], 111 | ['zip'], 112 | // ['upload'], 113 | function (err) { 114 | //if any error happened in the previous tasks, exit with a code > 0 115 | if (err) { 116 | cb(err); 117 | var exitCode = 2; 118 | console.log('[ERROR] gulp build task failed', err); 119 | console.log('[FAIL] gulp build task failed - exiting with code ' + exitCode); 120 | return process.exit(exitCode); 121 | } 122 | else { 123 | return cb(); 124 | } 125 | } 126 | ); 127 | }); 128 | -------------------------------------------------------------------------------- /make-webpack-config.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var webpack = require('webpack'); 3 | var StatsPlugin = require('stats-webpack-plugin'); 4 | var ContextReplacementPlugin = require('webpack/lib/ContextReplacementPlugin'); 5 | 6 | module.exports = function (options) { 7 | 8 | var entry = { index: './src/Index.js' }; 9 | var loaders = [ 10 | { test: /\.jsx$/, loader: options.hotComponents ? 11 | [ 'react-hot-loader', 'babel-loader?stage=0' ] : 'babel-loader?stage=0'}, 12 | { test: /\.js$/, loader: 'babel-loader', include: path.join(__dirname, 'src')}, 13 | { test: /\.json$/, loader: 'json-loader' }, 14 | { test: /\.md|markdown$/, loader: 'markdown-loader'} 15 | ]; 16 | 17 | var alias = {}; 18 | var aliasLoader = {}; 19 | var externals = [ 20 | { 'aws-sdk': 'commonjs aws-sdk' } // This is already available on the lambda server 21 | ]; 22 | var modulesDirectories = [ 'node_modules', 'web_modules' ]; 23 | var extensions = [ '', '.web.js', '.js', '.jsx', '.json' ]; 24 | var root = __dirname; 25 | 26 | var publicPath = options.devServer ? 'http://localhost:2992/assets/' : '/assets/'; 27 | 28 | var output = { 29 | path: path.join(__dirname, 'dist'), 30 | publicPath, 31 | filename: '[name].js', 32 | chunkFilename: options.devServer ? '[id].js' : '[name].js', 33 | sourceMapFilename: 'debugging/[file].map', 34 | pathinfo: options.debug, 35 | libraryTarget: 'umd' 36 | }; 37 | 38 | var excludeFromStats = [ 39 | /node_modules[\\\/]react(-router)?[\\\/]/, 40 | /node_modules[\\\/]items-store[\\\/]/ 41 | ]; 42 | 43 | var plugins = [ 44 | new StatsPlugin(path.join(__dirname, 'build', 'stats.json'), { 45 | chunkModules: true, 46 | exclude: excludeFromStats 47 | }), 48 | new ContextReplacementPlugin(/moment\.js[\/\\]lang$/, /^\.\/(de|pl)$/) 49 | ]; 50 | 51 | if(options.minimize) { 52 | plugins.push( 53 | new webpack.optimize.UglifyJsPlugin({ 54 | compressor: { 55 | warnings: false 56 | } 57 | }), 58 | new webpack.optimize.DedupePlugin() 59 | ); 60 | plugins.push( 61 | new webpack.DefinePlugin({'process.env': { NODE_ENV: JSON.stringify('production')}}), 62 | new webpack.NoErrorsPlugin() 63 | ); 64 | } 65 | 66 | return { 67 | entry, 68 | output, 69 | target: 'node', 70 | module: { loaders }, 71 | devtool: options.devtool, 72 | debug: options.debug, 73 | resolveLoader: { 74 | root: path.join(__dirname, 'node_modules'), 75 | alias: aliasLoader 76 | }, 77 | externals, 78 | resolve: { 79 | root, 80 | modulesDirectories, 81 | extensions, 82 | alias 83 | }, 84 | plugins, 85 | devServer: { 86 | stats: { 87 | cached: false, 88 | exclude: excludeFromStats 89 | } 90 | } 91 | }; 92 | }; 93 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamodb-lambda-autoscale", 3 | "version": "0.3.0", 4 | "description": "Autoscale DynamoDB provisioned capacity using AWS Lambda", 5 | "contributors": [ 6 | "Thomas Mitchell " 7 | ], 8 | "license": "MIT", 9 | "repository": { 10 | "type": "git", 11 | "url": "http://github.com/channl/dynamodb-lambda-autoscale.git" 12 | }, 13 | "main": "dist/index.js", 14 | "options": { 15 | "mocha": "--require resources/mocha-bootload src/**/__tests__/**/*.js" 16 | }, 17 | "scripts": { 18 | "localtest": "babel-node ./scripts/localtest.js", 19 | "test": "npm run lint && npm run check", 20 | "lint": "eslint src", 21 | "check": "flow check", 22 | "build": "gulp dist", 23 | "start": "node ./scripts/start.js", 24 | "debug": "node --inspect --debug-brk ./scripts/start.js" 25 | }, 26 | "dependencies": {}, 27 | "devDependencies": { 28 | "async": "^2.0.0", 29 | "aws-sdk": "2.6.9", 30 | "babel": "^6.5.2", 31 | "babel-cli": "^6.7.7", 32 | "babel-eslint": "^6.0.4", 33 | "babel-loader": "^6.2.4", 34 | "babel-polyfill": "^6.7.4", 35 | "babel-preset-react-native": "^1.5.7", 36 | "del": "^2.2.0", 37 | "dotenv": "^2.0.0", 38 | "eslint": "^2.9.0", 39 | "eslint-plugin-babel": "^3.2.0", 40 | "gulp": "^3.9.1", 41 | "gulp-install": "^0.6.0", 42 | "gulp-rename": "^1.2.2", 43 | "gulp-uglify": "^1.5.3", 44 | "gulp-zip": "^3.2.0", 45 | "invariant": "^2.2.1", 46 | "json-loader": "^0.5.4", 47 | "measured": "^1.1.0", 48 | "run-sequence": "^1.1.5", 49 | "stats-webpack-plugin": "^0.3.1", 50 | "warning": "^2.1.0", 51 | "webpack": "^1.13.0", 52 | "webpack-stream": "^3.2.0" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /scripts/start.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | 3 | try { 4 | var lambda = require('../dist/index.js'); 5 | 6 | process.chdir('./dist'); 7 | 8 | var context = { 9 | succeed: data => { 10 | try { 11 | if (data) { 12 | console.log(JSON.stringify(data)); 13 | } 14 | } catch (e) { 15 | console.error(e.stack); 16 | } 17 | }, 18 | fail: e => { 19 | console.error(e.stack); 20 | } 21 | }; 22 | 23 | var event = { 24 | json: { padding: 2 } 25 | }; 26 | 27 | lambda.handler(event, context); 28 | 29 | } catch (e) { 30 | console.log(e.stack); 31 | console.error(e); 32 | } 33 | -------------------------------------------------------------------------------- /src/App.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import Provisioner from './Provisioner'; 3 | import Stats from './utils/Stats'; 4 | import CostEstimation from './utils/CostEstimation'; 5 | import Throughput from './utils/Throughput'; 6 | import CapacityCalculator from './CapacityCalculator'; 7 | import { json, stats, log, invariant } from './Global'; 8 | import type { UpdateTableRequest } from 'aws-sdk'; 9 | 10 | export default class App { 11 | _provisioner: Provisioner; 12 | _capacityCalculator: CapacityCalculator; 13 | 14 | constructor() { 15 | this._provisioner = new Provisioner(); 16 | this._capacityCalculator = new CapacityCalculator(); 17 | } 18 | 19 | async runAsync(event: any, context: any): Promise { 20 | invariant(event != null, 'The argument \'event\' was null'); 21 | invariant(context != null, 'The argument \'context\' was null'); 22 | 23 | let sw = stats.timer('Index.handler').start(); 24 | 25 | // In local mode the json padding can be overridden 26 | if (event.json && event.json.padding) { 27 | json.padding = event.json.padding; 28 | } 29 | 30 | log('Getting table names'); 31 | let tableNames = await this._provisioner.getTableNamesAsync(); 32 | 33 | log('Getting table details'); 34 | let tableDetails = await this._getTableDetailsAsync(tableNames); 35 | 36 | log('Getting required table update requests'); 37 | let tableUpdateRequests = this._getTableUpdateRequests(tableDetails); 38 | 39 | if (tableUpdateRequests.length > 0) { 40 | log('Updating tables'); 41 | await this._updateTablesAsync(tableUpdateRequests); 42 | log('Updated tables'); 43 | } else { 44 | log('No table updates required'); 45 | } 46 | 47 | sw.end(); 48 | this._logMetrics(tableDetails); 49 | 50 | // Return an empty response 51 | if (context) { 52 | context.succeed(null); 53 | } 54 | } 55 | 56 | async _getTableDetailsAsync(tableNames: string[]): Promise { 57 | invariant(tableNames instanceof Array, 'The argument \'tableNames\' was not an array'); 58 | 59 | let tasks = tableNames.map(name => this._getTableDetailAsync(name)); 60 | return await Promise.all(tasks); 61 | } 62 | 63 | async _getTableDetailAsync(tableName: string): Promise { 64 | invariant(typeof tableName === 'string', 'The argument \'tableName\' was not a string'); 65 | 66 | log('Getting table description', tableName); 67 | let describeTableResponse = await this._provisioner.db 68 | .describeTableAsync({TableName: tableName}); 69 | 70 | let tableDescription = describeTableResponse.Table; 71 | 72 | log('Getting table consumed capacity description', tableName); 73 | let consumedCapacityTableDescription = await this._capacityCalculator 74 | .describeTableConsumedCapacityAsync(tableDescription); 75 | 76 | log('Getting table update request', tableName); 77 | let tableUpdateRequest = await this._provisioner.getTableUpdateAsync(tableDescription, 78 | consumedCapacityTableDescription); 79 | 80 | // Log the monthlyEstimatedCost 81 | let totalTableProvisionedThroughput = Throughput 82 | .getTotalTableProvisionedThroughput(tableDescription); 83 | 84 | let monthlyEstimatedCost = CostEstimation 85 | .getMonthlyEstimatedTableCost(totalTableProvisionedThroughput); 86 | 87 | stats 88 | .counter('DynamoDB.monthlyEstimatedCost') 89 | .inc(monthlyEstimatedCost); 90 | 91 | let result = { 92 | tableName, 93 | tableDescription, 94 | consumedCapacityTableDescription, 95 | tableUpdateRequest, 96 | totalTableProvisionedThroughput, 97 | monthlyEstimatedCost, 98 | }; 99 | 100 | return result; 101 | } 102 | 103 | async _updateTablesAsync(tableUpdateRequests: UpdateTableRequest[]): Promise { 104 | invariant(tableUpdateRequests instanceof Array, 105 | 'The argument \'tableUpdateRequests\' was not an array'); 106 | 107 | // If we are updating more than 10 tables in a single run 108 | // then we must wait until each one has been completed to 109 | // ensure we do not hit the AWS limit of 10 concurrent updates 110 | let isRateLimitedUpdatingRequired = tableUpdateRequests.length > 10; 111 | await Promise.all(tableUpdateRequests.map( 112 | async req => this._updateTableAsync(req, isRateLimitedUpdatingRequired) 113 | )); 114 | } 115 | 116 | async _updateTableAsync(tableUpdateRequest: UpdateTableRequest, 117 | isRateLimitedUpdatingRequired: boolean): Promise { 118 | invariant(tableUpdateRequest != null, 'The argument \'tableUpdateRequest\' was null'); 119 | invariant(typeof isRateLimitedUpdatingRequired === 'boolean', 120 | 'The argument \'isRateLimitedUpdatingRequired\' was not a boolean'); 121 | 122 | log('Updating table', tableUpdateRequest.TableName); 123 | await this._provisioner.db 124 | .updateTableWithRateLimitAsync(tableUpdateRequest, isRateLimitedUpdatingRequired); 125 | 126 | log('Updated table', tableUpdateRequest.TableName); 127 | } 128 | 129 | _getTableUpdateRequests(tableDetails: Object[]): UpdateTableRequest[] { 130 | invariant(tableDetails instanceof Array, 131 | 'The argument \'tableDetails\' was not an array'); 132 | 133 | return tableDetails 134 | .filter(({tableUpdateRequest}) => { return tableUpdateRequest != null; }) 135 | .map(({tableUpdateRequest}) => tableUpdateRequest); 136 | } 137 | 138 | _logMetrics(tableDetails: Object[]) { 139 | invariant(tableDetails instanceof Array, 140 | 'The argument \'tableDetails\' was not an array'); 141 | 142 | // Log stats 143 | let st = new Stats(stats); 144 | let stJSON = st.toJSON(); 145 | st.reset(); 146 | 147 | // Log readable info 148 | let updateRequests = tableDetails.map(i => i.tableUpdateRequest).filter(i => i !== null); 149 | let totalMonthlyEstimatedCost = tableDetails 150 | .reduce((prev, curr) => prev + curr.monthlyEstimatedCost, 0); 151 | let totalProvisionedThroughput = tableDetails.reduce((prev, curr) => { 152 | return { 153 | ReadCapacityUnits: prev.ReadCapacityUnits + 154 | curr.totalTableProvisionedThroughput.ReadCapacityUnits, 155 | WriteCapacityUnits: prev.WriteCapacityUnits + 156 | curr.totalTableProvisionedThroughput.WriteCapacityUnits, 157 | }; 158 | }, {ReadCapacityUnits: 0, WriteCapacityUnits: 0}); 159 | 160 | let indexHandler = stJSON['Index.handler'] != null ? { 161 | mean: stJSON['Index.handler'].histogram.mean 162 | } : undefined; 163 | 164 | let dynamoDBListTablesAsync = stJSON['DynamoDB.listTablesAsync'] != null ? { 165 | mean: stJSON['DynamoDB.listTablesAsync'].histogram.mean, 166 | } : undefined; 167 | 168 | let dynamoDBDescribeTableAsync = stJSON['DynamoDB.describeTableAsync'] != null ? { 169 | mean: stJSON['DynamoDB.describeTableAsync'].histogram.mean, 170 | } : undefined; 171 | 172 | let dynamoDBDescribeTableConsumedCapacityAsync = 173 | stJSON['DynamoDB.describeTableConsumedCapacityAsync'] != null ? 174 | { mean: stJSON['DynamoDB.describeTableConsumedCapacityAsync'].histogram.mean } : 175 | undefined; 176 | 177 | let cloudWatchGetMetricStatisticsAsync = 178 | stJSON['CloudWatch.getMetricStatisticsAsync'] != null ? 179 | { mean: stJSON['CloudWatch.getMetricStatisticsAsync'].histogram.mean } : 180 | undefined; 181 | 182 | let tableUpdates = updateRequests != null ? { count: updateRequests.length } : 183 | undefined; 184 | 185 | log(JSON.stringify({ 186 | 'Index.handler': indexHandler, 187 | 'DynamoDB.listTablesAsync': dynamoDBListTablesAsync, 188 | 'DynamoDB.describeTableAsync': dynamoDBDescribeTableAsync, 189 | 'DynamoDB.describeTableConsumedCapacityAsync': dynamoDBDescribeTableConsumedCapacityAsync, 190 | 'CloudWatch.getMetricStatisticsAsync': cloudWatchGetMetricStatisticsAsync, 191 | TableUpdates: tableUpdates, 192 | TotalProvisionedThroughput: totalProvisionedThroughput, 193 | TotalMonthlyEstimatedCost: totalMonthlyEstimatedCost, 194 | }, null, json.padding)); 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /src/CapacityCalculator.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import { invariant } from './Global'; 3 | import { Region } from './configuration/Region'; 4 | import CapacityCalculatorBase from './capacity/CapacityCalculatorBase'; 5 | import type { GetMetricStatisticsResponse } from 'aws-sdk'; 6 | import type { StatisticSettings } from './flow/FlowTypes'; 7 | 8 | export default class CapacityCalculator extends CapacityCalculatorBase { 9 | 10 | // Get the region 11 | getCloudWatchRegion() { 12 | return Region; 13 | } 14 | 15 | getStatisticSettings(): StatisticSettings { 16 | return { 17 | count: 5, 18 | spanMinutes: 1, 19 | type: 'Sum', 20 | }; 21 | } 22 | 23 | getThrottledEventStatisticSettings(): StatisticSettings { 24 | return { 25 | count: 1, 26 | spanMinutes: 1, 27 | type: 'Sum', 28 | }; 29 | } 30 | 31 | // Gets the projected capacity value based on the cloudwatch datapoints 32 | getProjectedValue(settings: StatisticSettings, data: GetMetricStatisticsResponse) { 33 | invariant(data != null, 'Parameter \'data\' is not set'); 34 | 35 | if (data.Datapoints.length === 0) { 36 | return 0; 37 | } 38 | 39 | // Default algorithm for projecting a good value for the current ConsumedThroughput is: 40 | // 1. Query 5 average readings each spanning a minute 41 | // 2. Select the Max value from those 5 readings 42 | let spanSeconds = settings.spanMinutes * 60; 43 | let averages = data.Datapoints.map(dp => dp.Sum / spanSeconds); 44 | let projectedValue = Math.max(...averages); 45 | return projectedValue; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/Global.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import measured from 'measured'; 3 | import _warning from 'warning'; 4 | import _invariant from 'invariant'; 5 | 6 | export const json = { padding: 0 }; 7 | 8 | export const stats = measured.createCollection(); 9 | 10 | export const log = (...params: any[]) => { 11 | // eslint-disable-next-line no-console 12 | console.log(...params); 13 | }; 14 | 15 | export const warning = (predicateOrValue: any, value: ?any) => { 16 | if (value == null) { 17 | _warning(false, predicateOrValue); 18 | } else { 19 | _warning(predicateOrValue, value); 20 | } 21 | }; 22 | 23 | export const invariant = (predicateOrValue: any, value: ?any) => { 24 | if (value == null) { 25 | _invariant(false, predicateOrValue); 26 | } else { 27 | _invariant(predicateOrValue, value); 28 | } 29 | }; 30 | -------------------------------------------------------------------------------- /src/Index.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | /* eslint-disable */ 3 | // $FlowIgnore 4 | import babelPolyfill from 'babel-polyfill'; 5 | /* eslint-enable */ 6 | // $FlowIgnore 7 | import dotenv from 'dotenv'; 8 | import App from './App'; 9 | import { log } from './Global'; 10 | 11 | log('*** LAMBDA INIT ***'); 12 | export let handler = async (event: any, context: any) => { 13 | try { 14 | dotenv.config({path: 'config.env'}); 15 | 16 | let app = new App(); 17 | log('*** LAMBDA START ***'); 18 | await app.runAsync(event, context); 19 | } catch (e) { 20 | log('*** LAMBDA ERROR ***'); 21 | if (context) { 22 | context.fail(e); 23 | } else { 24 | throw e; 25 | } 26 | } finally { 27 | log('*** LAMBDA FINISH ***'); 28 | } 29 | }; 30 | -------------------------------------------------------------------------------- /src/Provisioner.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | /* eslint-disable max-len */ 3 | import ProvisionerConfigurableBase from './provisioning/ProvisionerConfigurableBase'; 4 | import RateLimitedDecrement from './utils/RateLimitedDecrement'; 5 | import Throughput from './utils/Throughput'; 6 | import ProvisionerLogging from './provisioning/ProvisionerLogging'; 7 | import { Region } from './configuration/Region'; 8 | import DefaultProvisioner from './configuration/DefaultProvisioner'; 9 | import { invariant } from './Global'; 10 | import type { TableProvisionedAndConsumedThroughput, ProvisionerConfig, AdjustmentContext } from './flow/FlowTypes'; 11 | 12 | export default class Provisioner extends ProvisionerConfigurableBase { 13 | 14 | // Get the region 15 | getDynamoDBRegion(): string { 16 | return Region; 17 | } 18 | 19 | // Gets the list of tables which we want to autoscale 20 | async getTableNamesAsync(): Promise { 21 | 22 | // Option 1 - All tables (Default) 23 | return await this.db.listAllTableNamesAsync(); 24 | 25 | // Option 2 - Hardcoded list of tables 26 | // return ['Table1', 'Table2', 'Table3']; 27 | 28 | // Option 3 - DynamoDB / S3 configured list of tables 29 | // return await ...; 30 | } 31 | 32 | // Gets the json settings which control how the specifed table will be autoscaled 33 | // eslint-disable-next-line no-unused-vars 34 | getTableConfig(data: TableProvisionedAndConsumedThroughput): ProvisionerConfig { 35 | 36 | // Option 1 - Default settings for all tables 37 | return DefaultProvisioner; 38 | 39 | // Option 2 - Bespoke table specific settings 40 | // return data.TableName === 'Table1' ? Climbing : Default; 41 | 42 | // Option 3 - DynamoDB / S3 sourced table specific settings 43 | // return await ...; 44 | } 45 | 46 | isReadCapacityIncrementRequired(data: TableProvisionedAndConsumedThroughput): boolean { 47 | invariant(data != null, 'Parameter \'data\' is not set'); 48 | 49 | let config = this.getTableConfig(data); 50 | let adjustmentContext = this.getReadCapacityIncrementAdjustmentContext(data, config); 51 | return this.isCapacityAdjustmentRequired(data, adjustmentContext); 52 | } 53 | 54 | calculateIncrementedReadCapacityValue(data: TableProvisionedAndConsumedThroughput): number { 55 | invariant(data != null, 'Parameter \'data\' is not set'); 56 | 57 | let config = this.getTableConfig(data); 58 | let adjustmentContext = this.getReadCapacityIncrementAdjustmentContext(data, config); 59 | return Throughput.getAdjustedCapacityUnits(adjustmentContext); 60 | } 61 | 62 | isReadCapacityDecrementRequired(data: TableProvisionedAndConsumedThroughput): boolean { 63 | invariant(data != null, 'Parameter \'data\' is not set'); 64 | 65 | let config = this.getTableConfig(data); 66 | let adjustmentContext = this.getReadCapacityDecrementAdjustmentContext(data, config); 67 | return this.isCapacityAdjustmentRequired(data, adjustmentContext); 68 | } 69 | 70 | calculateDecrementedReadCapacityValue(data: TableProvisionedAndConsumedThroughput): number { 71 | invariant(data != null, 'Parameter \'data\' is not set'); 72 | 73 | let config = this.getTableConfig(data); 74 | let adjustmentContext = this.getReadCapacityDecrementAdjustmentContext(data, config); 75 | return Throughput.getAdjustedCapacityUnits(adjustmentContext); 76 | } 77 | 78 | isWriteCapacityIncrementRequired(data: TableProvisionedAndConsumedThroughput): boolean { 79 | invariant(data != null, 'Parameter \'data\' is not set'); 80 | 81 | let config = this.getTableConfig(data); 82 | let adjustmentContext = this.getWriteCapacityIncrementAdjustmentContext(data, config); 83 | return this.isCapacityAdjustmentRequired(data, adjustmentContext); 84 | } 85 | 86 | calculateIncrementedWriteCapacityValue(data: TableProvisionedAndConsumedThroughput): number { 87 | invariant(data != null, 'Parameter \'data\' is not set'); 88 | 89 | let config = this.getTableConfig(data); 90 | let adjustmentContext = this.getWriteCapacityIncrementAdjustmentContext(data, config); 91 | return Throughput.getAdjustedCapacityUnits(adjustmentContext); 92 | } 93 | 94 | isWriteCapacityDecrementRequired(data: TableProvisionedAndConsumedThroughput): boolean { 95 | invariant(data != null, 'Parameter \'data\' is not set'); 96 | 97 | let config = this.getTableConfig(data); 98 | let adjustmentContext = this.getWriteCapacityDecrementAdjustmentContext(data, config); 99 | return this.isCapacityAdjustmentRequired(data, adjustmentContext); 100 | } 101 | 102 | calculateDecrementedWriteCapacityValue(data: TableProvisionedAndConsumedThroughput): number { 103 | invariant(data != null, 'Parameter \'data\' is not set'); 104 | 105 | let config = this.getTableConfig(data); 106 | let adjustmentContext = this.getWriteCapacityDecrementAdjustmentContext(data, config); 107 | return Throughput.getAdjustedCapacityUnits(adjustmentContext); 108 | } 109 | 110 | getReadCapacityIncrementAdjustmentContext(data: TableProvisionedAndConsumedThroughput, config: ProvisionerConfig): AdjustmentContext { 111 | invariant(data != null, 'Argument \'data\' cannot be null'); 112 | invariant(config != null, 'Argument \'config\' cannot be null'); 113 | 114 | let context = { 115 | TableName: data.TableName, 116 | IndexName: data.IndexName, 117 | CapacityType: 'read', 118 | AdjustmentType: 'increment', 119 | ProvisionedValue: data.ProvisionedThroughput.ReadCapacityUnits, 120 | ConsumedValue: data.ConsumedThroughput.ReadCapacityUnits, 121 | ThrottledEvents: data.ThrottledEvents.ThrottledReadEvents, 122 | UtilisationPercent: (data.ConsumedThroughput.ReadCapacityUnits / data.ProvisionedThroughput.ReadCapacityUnits) * 100, 123 | CapacityConfig: config.ReadCapacity, 124 | }; 125 | 126 | if (config.ReadCapacity.Increment != null) { 127 | // $FlowIgnore 128 | context.CapacityAdjustmentConfig = config.ReadCapacity.Increment; 129 | } 130 | 131 | return context; 132 | } 133 | 134 | getReadCapacityDecrementAdjustmentContext(data: TableProvisionedAndConsumedThroughput, config: ProvisionerConfig): AdjustmentContext { 135 | invariant(data != null, 'Argument \'data\' cannot be null'); 136 | invariant(config != null, 'Argument \'config\' cannot be null'); 137 | 138 | let context = { 139 | TableName: data.TableName, 140 | IndexName: data.IndexName, 141 | CapacityType: 'read', 142 | AdjustmentType: 'decrement', 143 | ProvisionedValue: data.ProvisionedThroughput.ReadCapacityUnits, 144 | ConsumedValue: data.ConsumedThroughput.ReadCapacityUnits, 145 | ThrottledEvents: data.ThrottledEvents.ThrottledReadEvents, 146 | UtilisationPercent: (data.ConsumedThroughput.ReadCapacityUnits / data.ProvisionedThroughput.ReadCapacityUnits) * 100, 147 | CapacityConfig: config.ReadCapacity, 148 | }; 149 | 150 | if (config.ReadCapacity.Decrement != null) { 151 | // $FlowIgnore 152 | context.CapacityAdjustmentConfig = config.ReadCapacity.Decrement; 153 | } 154 | 155 | return context; 156 | } 157 | 158 | getWriteCapacityIncrementAdjustmentContext(data: TableProvisionedAndConsumedThroughput, config: ProvisionerConfig): AdjustmentContext { 159 | invariant(data != null, 'Argument \'data\' cannot be null'); 160 | invariant(config != null, 'Argument \'config\' cannot be null'); 161 | 162 | let context = { 163 | TableName: data.TableName, 164 | IndexName: data.IndexName, 165 | CapacityType: 'write', 166 | AdjustmentType: 'increment', 167 | ProvisionedValue: data.ProvisionedThroughput.WriteCapacityUnits, 168 | ConsumedValue: data.ConsumedThroughput.WriteCapacityUnits, 169 | ThrottledEvents: data.ThrottledEvents.ThrottledWriteEvents, 170 | UtilisationPercent: (data.ConsumedThroughput.WriteCapacityUnits / data.ProvisionedThroughput.WriteCapacityUnits) * 100, 171 | CapacityConfig: config.WriteCapacity, 172 | }; 173 | 174 | if (config.WriteCapacity.Increment != null) { 175 | // $FlowIgnore 176 | context.CapacityAdjustmentConfig = config.WriteCapacity.Increment; 177 | } 178 | 179 | return context; 180 | } 181 | 182 | getWriteCapacityDecrementAdjustmentContext(data: TableProvisionedAndConsumedThroughput, config: ProvisionerConfig): AdjustmentContext { 183 | invariant(data != null, 'Argument \'data\' cannot be null'); 184 | invariant(config != null, 'Argument \'config\' cannot be null'); 185 | 186 | let context = { 187 | TableName: data.TableName, 188 | IndexName: data.IndexName, 189 | CapacityType: 'write', 190 | AdjustmentType: 'decrement', 191 | ProvisionedValue: data.ProvisionedThroughput.WriteCapacityUnits, 192 | ConsumedValue: data.ConsumedThroughput.WriteCapacityUnits, 193 | ThrottledEvents: data.ThrottledEvents.ThrottledWriteEvents, 194 | UtilisationPercent: (data.ConsumedThroughput.WriteCapacityUnits / data.ProvisionedThroughput.WriteCapacityUnits) * 100, 195 | CapacityConfig: config.WriteCapacity, 196 | }; 197 | 198 | if (config.WriteCapacity.Decrement != null) { 199 | // $FlowIgnore 200 | context.CapacityAdjustmentConfig = config.WriteCapacity.Decrement; 201 | } 202 | 203 | return context; 204 | } 205 | 206 | isCapacityAdjustmentRequired(data: TableProvisionedAndConsumedThroughput, adjustmentContext: AdjustmentContext): boolean { 207 | 208 | // Determine if an adjustment is wanted 209 | let isProvAboveMax = adjustmentContext.CapacityConfig.Max == null ? false : adjustmentContext.ProvisionedValue > adjustmentContext.CapacityConfig.Max; 210 | let isProvBelowMax = adjustmentContext.CapacityConfig.Max == null ? true : adjustmentContext.ProvisionedValue < adjustmentContext.CapacityConfig.Max; 211 | let isProvBelowMin = adjustmentContext.CapacityConfig.Min == null ? adjustmentContext.ProvisionedValue < 1 : adjustmentContext.ProvisionedValue < adjustmentContext.CapacityConfig.Min; 212 | let isProvAboveMin = adjustmentContext.CapacityConfig.Min == null ? adjustmentContext.ProvisionedValue > 1 : adjustmentContext.ProvisionedValue > adjustmentContext.CapacityConfig.Min; 213 | let isUtilAboveThreshold = this.isAboveThreshold(adjustmentContext); 214 | let isUtilBelowThreshold = this.isBelowThreshold(adjustmentContext); 215 | let isThrottledEventsAboveThreshold = this.isThrottledEventsAboveThreshold(adjustmentContext); 216 | let isAdjustmentWanted = adjustmentContext.AdjustmentType === 'increment' ? 217 | (isProvBelowMin || isUtilAboveThreshold || isUtilBelowThreshold || isThrottledEventsAboveThreshold) && isProvBelowMax : 218 | (isProvAboveMax || isUtilAboveThreshold || isUtilBelowThreshold) && isProvAboveMin; 219 | 220 | // Determine if an adjustment is allowed under the rate limiting rules 221 | let isAfterLastDecreaseGracePeriod = adjustmentContext.CapacityAdjustmentConfig == null || 222 | this.isAfterLastAdjustmentGracePeriod(data.ProvisionedThroughput.LastDecreaseDateTime, 223 | adjustmentContext.CapacityAdjustmentConfig.When.AfterLastDecrementMinutes); 224 | let isAfterLastIncreaseGracePeriod = adjustmentContext.CapacityAdjustmentConfig == null || 225 | this.isAfterLastAdjustmentGracePeriod(data.ProvisionedThroughput.LastIncreaseDateTime, 226 | adjustmentContext.CapacityAdjustmentConfig.When.AfterLastIncrementMinutes); 227 | 228 | let isReadDecrementAllowed = adjustmentContext.AdjustmentType === 'decrement' ? 229 | RateLimitedDecrement.isDecrementAllowed(data, adjustmentContext, d => this.calculateDecrementedReadCapacityValue(d)) : 230 | true; 231 | 232 | let isAdjustmentAllowed = isAfterLastDecreaseGracePeriod && isAfterLastIncreaseGracePeriod && isReadDecrementAllowed; 233 | 234 | // Package up the configuration and the results so that we can produce 235 | // some effective logs 236 | let adjustmentData = { 237 | isAboveMax: isProvAboveMax, 238 | isBelowMin: isProvBelowMin, 239 | isAboveThreshold: isUtilAboveThreshold, 240 | isBelowThreshold: isUtilBelowThreshold, 241 | isAboveThrottledEventThreshold: isThrottledEventsAboveThreshold, 242 | isAfterLastDecreaseGracePeriod, 243 | isAfterLastIncreaseGracePeriod, 244 | isAdjustmentWanted, 245 | isAdjustmentAllowed 246 | }; 247 | 248 | // Log and return result 249 | ProvisionerLogging.isAdjustmentRequiredLog(adjustmentContext, adjustmentData); 250 | return isAdjustmentWanted && isAdjustmentAllowed; 251 | } 252 | 253 | isThrottledEventsAboveThreshold(context: AdjustmentContext): boolean { 254 | invariant(context != null, 'Parameter \'context\' is not set'); 255 | 256 | if (context.CapacityAdjustmentConfig == null || 257 | context.CapacityAdjustmentConfig.When.ThrottledEventsPerMinuteIsAbove == null || 258 | context.AdjustmentType === 'decrement') { 259 | return false; 260 | } 261 | 262 | return context.ThrottledEvents > 263 | context.CapacityAdjustmentConfig.When.ThrottledEventsPerMinuteIsAbove; 264 | } 265 | 266 | isAboveThreshold(context: AdjustmentContext): boolean { 267 | invariant(context != null, 'Parameter \'context\' is not set'); 268 | 269 | if (context.CapacityAdjustmentConfig == null || 270 | context.CapacityAdjustmentConfig.When.UtilisationIsAbovePercent == null) { 271 | return false; 272 | } 273 | 274 | let utilisationPercent = (context.ConsumedValue / context.ProvisionedValue) * 100; 275 | return utilisationPercent > context.CapacityAdjustmentConfig.When.UtilisationIsAbovePercent; 276 | } 277 | 278 | isBelowThreshold(context: AdjustmentContext): boolean { 279 | invariant(context != null, 'Parameter \'context\' is not set'); 280 | 281 | if (context.CapacityAdjustmentConfig == null || 282 | context.CapacityAdjustmentConfig.When.UtilisationIsBelowPercent == null) { 283 | return false; 284 | } 285 | 286 | let utilisationPercent = (context.ConsumedValue / context.ProvisionedValue) * 100; 287 | return utilisationPercent < context.CapacityAdjustmentConfig.When.UtilisationIsBelowPercent; 288 | } 289 | 290 | isAfterLastAdjustmentGracePeriod(lastAdjustmentDateTime: string, afterLastAdjustmentMinutes?: number): boolean { 291 | if (lastAdjustmentDateTime == null || afterLastAdjustmentMinutes == null) { 292 | return true; 293 | } 294 | 295 | let lastDecreaseDateTime = new Date(Date.parse(lastAdjustmentDateTime)); 296 | let thresholdDateTime = new Date(Date.now()); 297 | thresholdDateTime.setMinutes(thresholdDateTime.getMinutes() - (afterLastAdjustmentMinutes)); 298 | return lastDecreaseDateTime < thresholdDateTime; 299 | } 300 | } 301 | -------------------------------------------------------------------------------- /src/aws/CloudWatch.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import AWS from 'aws-sdk'; 3 | import { json, stats, warning, invariant } from '../Global'; 4 | import type { 5 | CloudWatchOptions, 6 | GetMetricStatisticsRequest, 7 | GetMetricStatisticsResponse, 8 | } from 'aws-sdk'; 9 | 10 | export default class CloudWatch { 11 | _cw: AWS.CloudWatch; 12 | 13 | constructor(cloudWatchOptions: CloudWatchOptions) { 14 | invariant(cloudWatchOptions != null, 'Parameter \'cloudWatchOptions\' is not set'); 15 | this._cw = new AWS.CloudWatch(cloudWatchOptions); 16 | } 17 | 18 | static create(region: string): CloudWatch { 19 | var options = { 20 | region, 21 | apiVersion: '2010-08-01', 22 | httpOptions: { timeout: 5000 } 23 | }; 24 | 25 | return new CloudWatch(options); 26 | } 27 | 28 | async getMetricStatisticsAsync(params: GetMetricStatisticsRequest) 29 | : Promise { 30 | let sw = stats.timer('CloudWatch.getMetricStatisticsAsync').start(); 31 | try { 32 | invariant(params != null, 'Parameter \'params\' is not set'); 33 | return await this._cw.getMetricStatistics(params).promise(); 34 | } catch (ex) { 35 | warning(JSON.stringify({ 36 | class: 'CloudWatch', 37 | function: 'getMetricStatisticsAsync', 38 | params 39 | }, null, json.padding)); 40 | throw ex; 41 | } finally { 42 | sw.end(); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/aws/DynamoDB.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import AWS from 'aws-sdk'; 3 | import { json, stats, warning, invariant } from '../Global'; 4 | import Delay from '../utils/Delay'; 5 | import Async from 'async'; 6 | import type { 7 | DynamoDBOptions, 8 | DescribeTableRequest, 9 | DescribeTableResponse, 10 | UpdateTableRequest, 11 | UpdateTableResponse, 12 | ListTablesRequest, 13 | ListTablesResponse, 14 | } from 'aws-sdk'; 15 | 16 | export default class DynamoDB { 17 | _db: AWS.DynamoDB; 18 | _updatePool: Object; 19 | 20 | constructor(dynamoOptions: DynamoDBOptions) { 21 | invariant(dynamoOptions != null, 'Parameter \'dynamoOptions\' is not set'); 22 | this._db = new AWS.DynamoDB(dynamoOptions); 23 | this._updatePool = Async.queue(async (params, callback) => { 24 | let result = await this.updateTableAndWaitAsync(params, true); 25 | callback(result); 26 | }, 10); 27 | } 28 | 29 | static create(region: string): DynamoDB { 30 | var options = { 31 | region, 32 | apiVersion: '2012-08-10', 33 | dynamoDbCrc32: false, 34 | httpOptions: { timeout: 5000 } 35 | }; 36 | 37 | return new DynamoDB(options); 38 | } 39 | 40 | async listTablesAsync(params: ?ListTablesRequest): Promise { 41 | let sw = stats.timer('DynamoDB.listTablesAsync').start(); 42 | try { 43 | return await this._db.listTables(params).promise(); 44 | } catch (ex) { 45 | warning(JSON.stringify({ 46 | class: 'DynamoDB', 47 | function: 'listTablesAsync' 48 | }, null, json.padding)); 49 | throw ex; 50 | } finally { 51 | sw.end(); 52 | } 53 | } 54 | 55 | async listAllTableNamesAsync(): Promise { 56 | let tableNames = []; 57 | let lastTable; 58 | do { 59 | let listTablesResponse = await this.listTablesAsync({ ExclusiveStartTableName: lastTable }); 60 | tableNames = tableNames.concat(listTablesResponse.TableNames); 61 | lastTable = listTablesResponse.LastEvaluatedTableName; 62 | } while (lastTable); 63 | return tableNames; 64 | } 65 | 66 | async describeTableAsync(params: DescribeTableRequest): Promise { 67 | let sw = stats.timer('DynamoDB.describeTableAsync').start(); 68 | try { 69 | invariant(params != null, 'Parameter \'params\' is not set'); 70 | return await this._db.describeTable(params).promise(); 71 | } catch (ex) { 72 | warning(JSON.stringify({ 73 | class: 'DynamoDB', 74 | function: 'describeTableAsync', 75 | params 76 | }, null, json.padding)); 77 | throw ex; 78 | } finally { 79 | sw.end(); 80 | } 81 | } 82 | 83 | async delayUntilTableIsActiveAsync(tableName: string): Promise { 84 | let isActive = false; 85 | let attempt = 0; 86 | do { 87 | let result = await this.describeTableAsync({ TableName: tableName }); 88 | isActive = result.Table.TableStatus === 'ACTIVE'; 89 | if (!isActive) { 90 | await Delay.delayAsync(1000); 91 | attempt++; 92 | } 93 | } while (!isActive && attempt < 10); 94 | } 95 | 96 | updateTableWithRateLimitAsync(params: UpdateTableRequest, 97 | isRateLimited: boolean): Promise { 98 | 99 | if (!isRateLimited) { 100 | return this.updateTableAndWaitAsync(params, isRateLimited); 101 | } 102 | 103 | return new Promise((resolve, reject) => { 104 | let sw = stats.timer('DynamoDB.updateTableAsync').start(); 105 | try { 106 | invariant(params != null, 'Parameter \'params\' is not set'); 107 | this._updatePool.push(params, resolve); 108 | } catch (ex) { 109 | warning(JSON.stringify({ 110 | class: 'DynamoDB', 111 | function: 'updateTableAsync', 112 | params 113 | }, null, json.padding)); 114 | reject(ex); 115 | } finally { 116 | sw.end(); 117 | } 118 | }); 119 | } 120 | 121 | async updateTableAndWaitAsync(params: UpdateTableRequest, 122 | isRateLimited: boolean): Promise { 123 | 124 | let response = await this._db.updateTable(params).promise(); 125 | if (isRateLimited) { 126 | await this.delayUntilTableIsActiveAsync(params.TableName); 127 | } 128 | 129 | return response; 130 | } 131 | 132 | async updateTableAsync(params: UpdateTableRequest): Promise { 133 | let sw = stats.timer('DynamoDB.updateTableAsync').start(); 134 | try { 135 | invariant(params != null, 'Parameter \'params\' is not set'); 136 | return await this._db.updateTable(params).promise(); 137 | } catch (ex) { 138 | warning(JSON.stringify({ 139 | class: 'DynamoDB', 140 | function: 'updateTableAsync', 141 | params 142 | }, null, json.padding)); 143 | throw ex; 144 | } finally { 145 | sw.end(); 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/capacity/CapacityCalculatorBase.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import { json, stats, warning, invariant } from '../Global'; 3 | import CloudWatch from '../aws/CloudWatch'; 4 | import type { 5 | TableConsumedCapacityDescription, 6 | StatisticSettings, 7 | ConsumedCapacityDesc, 8 | } from '../flow/FlowTypes'; 9 | import type { 10 | TableDescription, 11 | GetMetricStatisticsResponse, 12 | Dimension, 13 | } from 'aws-sdk'; 14 | 15 | export default class CapacityCalculatorBase { 16 | cw: CloudWatch; 17 | 18 | constructor() { 19 | this.cw = CloudWatch.create(this.getCloudWatchRegion()); 20 | } 21 | 22 | // Get the region 23 | getCloudWatchRegion(): string { 24 | invariant(false, 'The method \'getCloudWatchRegion\' was not implemented'); 25 | } 26 | 27 | // Gets the settings used to fetch the consumed throughput statistics 28 | getStatisticSettings(): StatisticSettings { 29 | invariant(false, 'The method \'getStatisticSettings\' was not implemented'); 30 | } 31 | 32 | // Gets the settings used to fetch the throttled events statistics 33 | getThrottledEventStatisticSettings(): StatisticSettings { 34 | invariant(false, 'The method \'getThrottledEventStatisticSettings\' was not implemented'); 35 | } 36 | 37 | // Gets the projected capacity value based on the cloudwatch datapoints 38 | // eslint-disable-next-line no-unused-vars 39 | getProjectedValue(settings: StatisticSettings, data: GetMetricStatisticsResponse): number { 40 | invariant(false, 'The method \'getProjectedValue\' was not implemented'); 41 | } 42 | 43 | async describeTableConsumedCapacityAsync(params: TableDescription) 44 | : Promise { 45 | let sw = stats 46 | .timer('DynamoDB.describeTableConsumedCapacityAsync') 47 | .start(); 48 | 49 | try { 50 | invariant(params != null, 'Parameter \'params\' is not set'); 51 | 52 | // Make all the requests concurrently 53 | let tableRead = this.getConsumedCapacityAsync(true, params.TableName, null); 54 | let tableWrite = this.getConsumedCapacityAsync(false, params.TableName, null); 55 | 56 | let gsiReads = (params.GlobalSecondaryIndexes || []) 57 | .map(gsi => this.getConsumedCapacityAsync(true, params.TableName, gsi.IndexName)); 58 | 59 | let gsiWrites = (params.GlobalSecondaryIndexes || []) 60 | .map(gsi => this.getConsumedCapacityAsync(false, params.TableName, gsi.IndexName)); 61 | 62 | let tableTRead = this.getThrottledEventsAsync(true, params.TableName, null); 63 | let tableTWrites = this.getThrottledEventsAsync(false, params.TableName, null); 64 | 65 | let gsiTReads = (params.GlobalSecondaryIndexes || []) 66 | .map(gsi => this.getThrottledEventsAsync(true, params.TableName, gsi.IndexName)); 67 | 68 | let gsiTWrites = (params.GlobalSecondaryIndexes || []) 69 | .map(gsi => this.getThrottledEventsAsync(false, params.TableName, gsi.IndexName)); 70 | 71 | // Await on the results 72 | let tableConsumedRead = await tableRead; 73 | let tableConsumedWrite = await tableWrite; 74 | let gsiConsumedReads = await Promise.all(gsiReads); 75 | let gsiConsumedWrites = await Promise.all(gsiWrites); 76 | 77 | // Await on throttled info 78 | let tableThrottledRead = await tableTRead; 79 | let tableThrottledWrite = await tableTWrites; 80 | let gsiThrottledReads = await Promise.all(gsiTReads); 81 | let gsiThrottledWrites = await Promise.all(gsiTWrites); 82 | 83 | // Format results 84 | let gsis = gsiConsumedReads.map((read, i) => { 85 | let write = gsiConsumedWrites[i]; 86 | let throttledWrite = gsiThrottledWrites[i]; 87 | let throttledRead = gsiThrottledReads[i]; 88 | let gsiIndexName = read.globalSecondaryIndexName; 89 | invariant(gsiIndexName != null, '\'gsiIndexName\' was null'); 90 | return { 91 | IndexName: gsiIndexName, 92 | ConsumedThroughput: { 93 | ReadCapacityUnits: read.value, 94 | WriteCapacityUnits: write.value 95 | }, 96 | ThrottledEvents: { 97 | ThrottledReadEvents: throttledRead, 98 | ThrottledWriteEvents: throttledWrite 99 | } 100 | }; 101 | }); 102 | 103 | return { 104 | TableName: params.TableName, 105 | ConsumedThroughput: { 106 | ReadCapacityUnits: tableConsumedRead.value, 107 | WriteCapacityUnits: tableConsumedWrite.value 108 | }, 109 | ThrottledEvents: { 110 | ThrottledReadEvents: tableThrottledRead, 111 | ThrottledWriteEvents: tableThrottledWrite 112 | }, 113 | GlobalSecondaryIndexes: gsis 114 | }; 115 | } catch (ex) { 116 | warning(JSON.stringify({ 117 | class: 'CapacityCalculator', 118 | function: 'describeTableConsumedCapacityAsync', 119 | params, 120 | }, null, json.padding)); 121 | throw ex; 122 | } finally { 123 | sw.end(); 124 | } 125 | } 126 | 127 | async getConsumedCapacityAsync( 128 | isRead: boolean, tableName: string, globalSecondaryIndexName: ?string): 129 | Promise { 130 | try { 131 | invariant(isRead != null, 'Parameter \'isRead\' is not set'); 132 | invariant(tableName != null, 'Parameter \'tableName\' is not set'); 133 | 134 | let settings = this.getStatisticSettings(); 135 | 136 | let EndTime = new Date(); 137 | let StartTime = new Date(); 138 | StartTime.setTime(EndTime - (60000 * settings.spanMinutes * settings.count)); 139 | let MetricName = isRead ? 'ConsumedReadCapacityUnits' : 'ConsumedWriteCapacityUnits'; 140 | let Dimensions = this.getDimensions(tableName, globalSecondaryIndexName); 141 | let period = (settings.spanMinutes * 60); 142 | let params = { 143 | Namespace: 'AWS/DynamoDB', 144 | MetricName, 145 | Dimensions, 146 | StartTime, 147 | EndTime, 148 | Period: period, 149 | Statistics: [ settings.type ], 150 | Unit: 'Count' 151 | }; 152 | 153 | let statistics = await this.cw.getMetricStatisticsAsync(params); 154 | let value = this.getProjectedValue(settings, statistics); 155 | let result: ConsumedCapacityDesc = { 156 | tableName, 157 | globalSecondaryIndexName, 158 | value 159 | }; 160 | 161 | /* 162 | log(JSON.stringify({ 163 | ...result, 164 | statistics: statistics.Datapoints.map(dp => dp.Sum / (settings.spanMinutes * 60)), 165 | })); 166 | */ 167 | 168 | return result; 169 | } catch (ex) { 170 | warning(JSON.stringify({ 171 | class: 'CapacityCalculator', 172 | function: 'getConsumedCapacityAsync', 173 | isRead, tableName, globalSecondaryIndexName, 174 | }, null, json.padding)); 175 | throw ex; 176 | } 177 | } 178 | 179 | async getThrottledEventsAsync( 180 | isRead: boolean, tableName: string, globalSecondaryIndexName: ?string): 181 | Promise { 182 | try { 183 | invariant(isRead != null, 'Parameter \'isRead\' is not set'); 184 | invariant(tableName != null, 'Parameter \'tableName\' is not set'); 185 | 186 | let settings = this.getThrottledEventStatisticSettings(); 187 | 188 | let EndTime = new Date(); 189 | let StartTime = new Date(); 190 | StartTime.setTime(EndTime - (60000 * settings.spanMinutes * settings.count)); 191 | let MetricName = isRead ? 'ReadThrottleEvents' : 'WriteThrottleEvents'; 192 | let Dimensions = this.getDimensions(tableName, globalSecondaryIndexName); 193 | let period = (settings.spanMinutes * 60); 194 | let params = { 195 | Namespace: 'AWS/DynamoDB', 196 | MetricName, 197 | Dimensions, 198 | StartTime, 199 | EndTime, 200 | Period: period, 201 | Statistics: [ settings.type ], 202 | Unit: 'Count' 203 | }; 204 | 205 | let statistics = await this.cw.getMetricStatisticsAsync(params); 206 | let value = this.getProjectedValue(settings, statistics); 207 | 208 | return value; 209 | } catch (ex) { 210 | warning(JSON.stringify({ 211 | class: 'CapacityCalculator', 212 | function: 'getThrottledEventsAsync', 213 | isRead, tableName, globalSecondaryIndexName, 214 | }, null, json.padding)); 215 | throw ex; 216 | } 217 | } 218 | 219 | getDimensions(tableName: string, globalSecondaryIndexName: ?string): Dimension[] { 220 | if (globalSecondaryIndexName) { 221 | return [ 222 | { Name: 'TableName', Value: tableName}, 223 | { Name: 'GlobalSecondaryIndexName', Value: globalSecondaryIndexName} 224 | ]; 225 | } 226 | 227 | return [ { Name: 'TableName', Value: tableName} ]; 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /src/configuration/ClimbingProvisioner.json: -------------------------------------------------------------------------------- 1 | { 2 | "ReadCapacity": { 3 | "Min": 1, 4 | "Max": 10, 5 | "Increment": { 6 | "When": { 7 | "UtilisationIsAbovePercent": 80 8 | }, 9 | "By": { 10 | "ConsumedPercent": 30, 11 | "Units": 3 12 | } 13 | }, 14 | "Decrement": { 15 | "When": { 16 | "UtilisationIsBelowPercent": 30, 17 | "AfterLastIncrementMinutes": 60, 18 | "AfterLastDecrementMinutes": 60, 19 | "UnitAdjustmentGreaterThan": 5 20 | }, 21 | "To": { 22 | "ConsumedPercent": 100 23 | } 24 | } 25 | }, 26 | "WriteCapacity": { 27 | "Min": 1, 28 | "Max": 10, 29 | "Increment": { 30 | "When": { 31 | "UtilisationIsAbovePercent": 80 32 | }, 33 | "By": { 34 | "ProvisionedPercent": 30, 35 | "Units": 3 36 | } 37 | }, 38 | "Decrement": { 39 | "When": { 40 | "UtilisationIsBelowPercent": 30, 41 | "AfterLastIncrementMinutes": 60, 42 | "AfterLastDecrementMinutes": 60, 43 | "UnitAdjustmentGreaterThan": 5 44 | }, 45 | "To": { 46 | "ConsumedPercent": 100 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/configuration/DefaultProvisioner.json: -------------------------------------------------------------------------------- 1 | { 2 | "ReadCapacity": { 3 | "Min": 1, 4 | "Max": 100, 5 | "Increment": { 6 | "When": { 7 | "UtilisationIsAbovePercent": 75, 8 | "ThrottledEventsPerMinuteIsAbove": 25 9 | }, 10 | "By": { 11 | "Units": 3, 12 | "ProvisionedPercent": 30, 13 | "ThrottledEventsWithMultiplier": 0.7 14 | }, 15 | "To": { 16 | "ConsumedPercent": 130 17 | } 18 | }, 19 | "Decrement": { 20 | "When": { 21 | "UtilisationIsBelowPercent": 30, 22 | "AfterLastIncrementMinutes": 60, 23 | "AfterLastDecrementMinutes": 60, 24 | "UnitAdjustmentGreaterThan": 5 25 | }, 26 | "To": { 27 | "ConsumedPercent": 100 28 | } 29 | } 30 | }, 31 | "WriteCapacity": { 32 | "Min": 1, 33 | "Max": 100, 34 | "Increment": { 35 | "When": { 36 | "UtilisationIsAbovePercent": 75, 37 | "ThrottledEventsPerMinuteIsAbove": 25 38 | }, 39 | "By": { 40 | "Units": 3, 41 | "ProvisionedPercent": 30, 42 | "ThrottledEventsWithMultiplier": 0.7 43 | }, 44 | "To": { 45 | "ConsumedPercent": 130 46 | } 47 | }, 48 | "Decrement": { 49 | "When": { 50 | "UtilisationIsBelowPercent": 30, 51 | "AfterLastIncrementMinutes": 60, 52 | "AfterLastDecrementMinutes": 60, 53 | "UnitAdjustmentGreaterThan": 5 54 | }, 55 | "To": { 56 | "ConsumedPercent": 100 57 | } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/configuration/FixedProvisioner.json: -------------------------------------------------------------------------------- 1 | { 2 | "ReadCapacity": { 3 | "Min": 1, 4 | "Max": 1 5 | }, 6 | "WriteCapacity": { 7 | "Min": 1, 8 | "Max": 1 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/configuration/Region.json: -------------------------------------------------------------------------------- 1 | { 2 | "Region": "us-east-1" 3 | } 4 | -------------------------------------------------------------------------------- /src/flow/FlowTypes.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import type { ProvisionedThroughput, Throughput } from 'aws-sdk'; 3 | 4 | export type ThrottledEventsDescription = { 5 | ThrottledReadEvents: number, 6 | ThrottledWriteEvents: number 7 | } 8 | 9 | export type TableProvisionedAndConsumedThroughput = { 10 | TableName: string, 11 | IndexName?: string, 12 | ProvisionedThroughput: ProvisionedThroughput, 13 | ConsumedThroughput: Throughput, 14 | ThrottledEvents: ThrottledEventsDescription 15 | }; 16 | 17 | export type GlobalSecondaryIndexConsumedThroughput = { 18 | IndexName: string, 19 | ConsumedThroughput: Throughput, 20 | ThrottledEvents: ThrottledEventsDescription, 21 | }; 22 | 23 | export type TableConsumedCapacityDescription = { 24 | TableName: string, 25 | ConsumedThroughput: Throughput, 26 | GlobalSecondaryIndexes: GlobalSecondaryIndexConsumedThroughput[], 27 | ThrottledEvents: ThrottledEventsDescription, 28 | }; 29 | 30 | export type ConsumedCapacityDesc = { 31 | tableName: string, 32 | globalSecondaryIndexName: ?string, 33 | value: number, 34 | }; 35 | 36 | export type ProvisionerConfig = { 37 | ReadCapacity: CapacityConfig, 38 | WriteCapacity: CapacityConfig, 39 | }; 40 | 41 | export type CapacityConfig = { 42 | Min?: number, 43 | Max?: number, 44 | Increment?: CapacityAdjustmentConfig, 45 | Decrement?: CapacityAdjustmentConfig, 46 | }; 47 | 48 | export type CapacityAdjustmentConfig = { 49 | When: WhenConfig, 50 | By?: ByToConfig, 51 | To?: ByToConfig, 52 | }; 53 | 54 | export type WhenConfig = { 55 | UtilisationIsAbovePercent?: number, 56 | UtilisationIsBelowPercent?: number, 57 | ThrottledEventsPerMinuteIsAbove?: number, 58 | AfterLastIncrementMinutes?: number, 59 | AfterLastDecrementMinutes?: number, 60 | UnitAdjustmentGreaterThan?: number, 61 | }; 62 | 63 | export type ByToConfig = { 64 | ConsumedPercent?: number, 65 | ProvisionedPercent?: number, 66 | Units?: number, 67 | ThrottledEventsWithMultiplier?: number, 68 | }; 69 | 70 | export type StatisticSettings = { 71 | count: number, 72 | spanMinutes: number, 73 | type: 'Average' | 'Sum', 74 | }; 75 | 76 | export type AdjustmentContext = { 77 | TableName: string, 78 | IndexName?: string, 79 | CapacityType: 'read' | 'write', 80 | AdjustmentType: 'increment' | 'decrement', 81 | ProvisionedValue: number, 82 | ConsumedValue: number, 83 | ThrottledEvents: number, 84 | UtilisationPercent: number, 85 | CapacityConfig: CapacityConfig, 86 | CapacityAdjustmentConfig?: CapacityAdjustmentConfig, 87 | }; 88 | 89 | export type AdjustmentData = { 90 | isAboveMax: boolean, 91 | isBelowMin: boolean, 92 | isAboveThreshold: boolean, 93 | isBelowThreshold: boolean, 94 | isAboveThrottledEventThreshold: boolean, 95 | isAfterLastDecreaseGracePeriod: boolean, 96 | isAfterLastIncreaseGracePeriod: boolean, 97 | isAdjustmentWanted: boolean, 98 | isAdjustmentAllowed: boolean 99 | }; 100 | -------------------------------------------------------------------------------- /src/provisioning/ProvisionerBase.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | /* eslint-disable no-unused-vars */ 3 | import { invariant } from '../Global'; 4 | import type { TableDescription, UpdateTableRequest } from 'aws-sdk'; 5 | import type { TableConsumedCapacityDescription } from '../flow/FlowTypes'; 6 | import DynamoDB from '../aws/DynamoDB'; 7 | import CloudWatch from '../aws/CloudWatch'; 8 | 9 | export default class ProvisionerBase { 10 | db: DynamoDB; 11 | 12 | constructor() { 13 | this.db = DynamoDB.create(this.getDynamoDBRegion()); 14 | } 15 | 16 | getDynamoDBRegion(): string { 17 | invariant(false, 'The method \'getDynamoDBRegion\' was not implemented'); 18 | } 19 | 20 | async getTableNamesAsync(): Promise { 21 | invariant(false, 'The method \'getTableNamesAsync\' was not implemented'); 22 | } 23 | 24 | async getTableUpdateAsync( 25 | tableDescription: TableDescription, 26 | tableConsumedCapacityDescription: TableConsumedCapacityDescription): 27 | Promise { 28 | invariant(false, 'The method \'getTableUpdateAsync\' was not implemented'); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/provisioning/ProvisionerConfigurableBase.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import { json, warning, invariant } from '../Global'; 3 | import ProvisionerBase from '../provisioning/ProvisionerBase'; 4 | import type { 5 | TableDescription, 6 | GlobalSecondaryIndex, 7 | UpdateTableRequest, 8 | GlobalSecondaryIndexUpdate, 9 | Throughput, 10 | } from 'aws-sdk'; 11 | import type { 12 | TableProvisionedAndConsumedThroughput, 13 | TableConsumedCapacityDescription, 14 | } from '../flow/FlowTypes'; 15 | 16 | export default class ProvisionerConfigurableBase extends ProvisionerBase { 17 | 18 | // eslint-disable-next-line no-unused-vars 19 | isReadCapacityIncrementRequired(data: TableProvisionedAndConsumedThroughput): boolean { 20 | invariant(false, 'The method \'isReadCapacityIncrementRequired\' was not implemented'); 21 | } 22 | 23 | // eslint-disable-next-line no-unused-vars 24 | calculateIncrementedReadCapacityValue(data: TableProvisionedAndConsumedThroughput): number { 25 | invariant(false, 'The method \'calculateIncrementedReadCapacityValue\' was not implemented'); 26 | } 27 | 28 | // eslint-disable-next-line no-unused-vars 29 | isReadCapacityDecrementRequired(data: TableProvisionedAndConsumedThroughput): boolean { 30 | invariant(false, 'The method \'isReadCapacityDecrementRequired\' was not implemented'); 31 | } 32 | 33 | // eslint-disable-next-line no-unused-vars 34 | calculateDecrementedReadCapacityValue(data: TableProvisionedAndConsumedThroughput): number { 35 | invariant(false, 'The method \'calculateDecrementedReadCapacityValue\' was not implemented'); 36 | } 37 | 38 | // eslint-disable-next-line no-unused-vars 39 | isWriteCapacityIncrementRequired(data: TableProvisionedAndConsumedThroughput): boolean { 40 | invariant(false, 'The method \'isWriteCapacityIncrementRequired\' was not implemented'); 41 | } 42 | 43 | // eslint-disable-next-line no-unused-vars 44 | calculateIncrementedWriteCapacityValue(data: TableProvisionedAndConsumedThroughput): number { 45 | invariant(false, 'The method \'calculateIncrementedWriteCapacityValue\' was not implemented'); 46 | } 47 | 48 | // eslint-disable-next-line no-unused-vars 49 | isWriteCapacityDecrementRequired(data: TableProvisionedAndConsumedThroughput): boolean { 50 | invariant(false, 'The method \'isWriteCapacityDecrementRequired\' was not implemented'); 51 | } 52 | 53 | // eslint-disable-next-line no-unused-vars 54 | calculateDecrementedWriteCapacityValue(data: TableProvisionedAndConsumedThroughput): number { 55 | invariant(false, 'The method \'calculateDecrementedWriteCapacityValue\' was not implemented'); 56 | } 57 | 58 | async getTableNamesAsync(): Promise { 59 | invariant(false, 'The method \'getTableNamesAsync\' was not implemented'); 60 | } 61 | 62 | async getTableUpdateAsync(tableDescription: TableDescription, 63 | tableConsumedCapacityDescription: TableConsumedCapacityDescription) : 64 | Promise { 65 | try { 66 | invariant(tableDescription != null, 'Parameter \'tableDescription\' is not set'); 67 | invariant(tableConsumedCapacityDescription != null, 68 | 'Parameter \'tableConsumedCapacityDescription\' is not set'); 69 | 70 | let tableData = { 71 | TableName: tableDescription.TableName, 72 | ProvisionedThroughput: tableDescription.ProvisionedThroughput, 73 | ConsumedThroughput: tableConsumedCapacityDescription.ConsumedThroughput, 74 | ThrottledEvents: tableConsumedCapacityDescription.ThrottledEvents 75 | }; 76 | 77 | let provisionedThroughput = this.getUpdatedProvisionedThroughput(tableData); 78 | 79 | let gsis = tableDescription.GlobalSecondaryIndexes || []; 80 | let globalSecondaryIndexUpdates = gsis 81 | // $FlowIgnore 82 | .map(gsi => this.getGlobalSecondaryIndexUpdate( 83 | tableDescription, tableConsumedCapacityDescription, gsi)) 84 | .filter(i => i !== null); 85 | 86 | // eslint-disable-next-line eqeqeq 87 | if (!provisionedThroughput && (globalSecondaryIndexUpdates == null || 88 | globalSecondaryIndexUpdates.length === 0)) { 89 | return null; 90 | } 91 | 92 | let result: UpdateTableRequest = { 93 | TableName: tableDescription.TableName 94 | }; 95 | 96 | if (provisionedThroughput) { 97 | result.ProvisionedThroughput = provisionedThroughput; 98 | } 99 | 100 | if (globalSecondaryIndexUpdates && globalSecondaryIndexUpdates.length > 0) { 101 | result.GlobalSecondaryIndexUpdates = globalSecondaryIndexUpdates; 102 | } 103 | 104 | return result; 105 | } catch (e) { 106 | warning(JSON.stringify({ 107 | class: 'ConfigurableProvisioner', 108 | function: 'getTableUpdate', 109 | tableDescription, 110 | tableConsumedCapacityDescription 111 | }, null, json.padding)); 112 | throw e; 113 | } 114 | } 115 | 116 | getUpdatedProvisionedThroughput(params: TableProvisionedAndConsumedThroughput) 117 | : ?Throughput { 118 | try { 119 | invariant(params != null, 'Parameter \'params\' is not set'); 120 | 121 | let newProvisionedThroughput = { 122 | ReadCapacityUnits: params.ProvisionedThroughput.ReadCapacityUnits, 123 | WriteCapacityUnits: params.ProvisionedThroughput.WriteCapacityUnits 124 | }; 125 | 126 | // Adjust read capacity 127 | if (this.isReadCapacityIncrementRequired(params)) { 128 | newProvisionedThroughput.ReadCapacityUnits = this 129 | .calculateIncrementedReadCapacityValue(params); 130 | 131 | } else if (this.isReadCapacityDecrementRequired(params)) { 132 | newProvisionedThroughput.ReadCapacityUnits = this 133 | .calculateDecrementedReadCapacityValue(params); 134 | } 135 | 136 | // Adjust write capacity 137 | if (this.isWriteCapacityIncrementRequired(params)) { 138 | newProvisionedThroughput.WriteCapacityUnits = this 139 | .calculateIncrementedWriteCapacityValue(params); 140 | 141 | } else if (this.isWriteCapacityDecrementRequired(params)) { 142 | newProvisionedThroughput.WriteCapacityUnits = this 143 | .calculateDecrementedWriteCapacityValue(params); 144 | } 145 | 146 | if (newProvisionedThroughput.ReadCapacityUnits === 147 | params.ProvisionedThroughput.ReadCapacityUnits && 148 | newProvisionedThroughput.WriteCapacityUnits === 149 | params.ProvisionedThroughput.WriteCapacityUnits) { 150 | return null; 151 | } 152 | 153 | return newProvisionedThroughput; 154 | } catch (e) { 155 | warning(JSON.stringify({ 156 | class: 'ConfigurableProvisioner', 157 | function: 'getUpdatedProvisionedThroughput', params 158 | }, null, json.padding)); 159 | throw e; 160 | } 161 | } 162 | 163 | getGlobalSecondaryIndexUpdate( 164 | tableDescription: TableDescription, 165 | tableConsumedCapacityDescription: TableConsumedCapacityDescription, 166 | gsi: GlobalSecondaryIndex): ?GlobalSecondaryIndexUpdate { 167 | try { 168 | invariant(tableDescription != null, 'Parameter \'tableDescription\' is not set'); 169 | invariant(tableConsumedCapacityDescription != null, 170 | 'Parameter \'tableConsumedCapacityDescription\' is not set'); 171 | invariant(gsi != null, 'Parameter \'gsi\' is not set'); 172 | 173 | let gsicc = tableConsumedCapacityDescription 174 | .GlobalSecondaryIndexes 175 | .find(i => i.IndexName === gsi.IndexName); 176 | 177 | invariant(gsicc != null, 'Specified GSI could not be found'); 178 | let provisionedThroughput = this.getUpdatedProvisionedThroughput({ 179 | TableName: tableDescription.TableName, 180 | IndexName: gsicc.IndexName, 181 | ProvisionedThroughput: gsi.ProvisionedThroughput, 182 | ConsumedThroughput: gsicc.ConsumedThroughput, 183 | ThrottledEvents: gsicc.ThrottledEvents 184 | }); 185 | 186 | // eslint-disable-next-line eqeqeq 187 | if (provisionedThroughput == null) { 188 | return null; 189 | } 190 | 191 | return { 192 | Update: { 193 | IndexName: gsi.IndexName, 194 | ProvisionedThroughput: provisionedThroughput 195 | } 196 | }; 197 | } catch (e) { 198 | warning(JSON.stringify({ 199 | class: 'ConfigurableProvisioner', 200 | function: 'getGlobalSecondaryIndexUpdate', 201 | tableDescription, 202 | tableConsumedCapacityDescription, 203 | gsi 204 | }, null, json.padding)); 205 | throw e; 206 | } 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /src/provisioning/ProvisionerLogging.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import { log } from '../Global'; 3 | import type { 4 | AdjustmentContext, 5 | AdjustmentData, 6 | } from '../flow/FlowTypes'; 7 | 8 | export default class ConfigLogging { 9 | static isAdjustmentRequiredLog( 10 | adjustmentContext: AdjustmentContext, 11 | adjustmentData: AdjustmentData, 12 | ) { 13 | 14 | let logMessage = typeof adjustmentContext.IndexName === 'undefined' ? 15 | adjustmentContext.TableName : adjustmentContext.TableName + '.' + adjustmentContext.IndexName; 16 | logMessage += ' is consuming ' + adjustmentContext.ConsumedValue + ' of ' + 17 | adjustmentContext.ProvisionedValue + ' (' + adjustmentContext.UtilisationPercent + 18 | '%) ' + adjustmentContext.CapacityType + ' capacity units'; 19 | 20 | if (adjustmentContext.CapacityConfig.Max != null && adjustmentData.isAboveMax) { 21 | logMessage += ' and is above max allowed ' + adjustmentContext.CapacityConfig.Max + ' units'; 22 | } 23 | 24 | if (adjustmentContext.CapacityAdjustmentConfig != null && 25 | adjustmentContext.CapacityAdjustmentConfig.When.UtilisationIsAbovePercent != null && 26 | adjustmentData.isAboveThreshold && !adjustmentData.isAboveMax) { 27 | logMessage += ' and is above maximum threshold of ' + 28 | adjustmentContext.CapacityAdjustmentConfig.When.UtilisationIsAbovePercent + '%'; 29 | } 30 | 31 | if (adjustmentContext.CapacityConfig.Min != null && adjustmentData.isBelowMin) { 32 | logMessage += ' and is below the min allowed ' + adjustmentContext.CapacityConfig.Min + 33 | ' units'; 34 | } 35 | 36 | if (adjustmentContext.CapacityAdjustmentConfig != null && 37 | adjustmentContext.CapacityAdjustmentConfig.When.UtilisationIsBelowPercent != null && 38 | adjustmentData.isBelowThreshold && !adjustmentData.isBelowMin) { 39 | logMessage += ' and is below minimum threshold of ' + 40 | adjustmentContext.CapacityAdjustmentConfig.When.UtilisationIsBelowPercent + '%'; 41 | } 42 | 43 | if (adjustmentContext.CapacityAdjustmentConfig != null && 44 | adjustmentContext.CapacityAdjustmentConfig.When.ThrottledEventsPerMinuteIsAbove != null && 45 | adjustmentData.isAboveThrottledEventThreshold) { 46 | logMessage += ' and throttled events per minute is above ' + 47 | adjustmentContext.CapacityAdjustmentConfig.When.ThrottledEventsPerMinuteIsAbove + ' events'; 48 | } 49 | 50 | if (adjustmentData.isAdjustmentWanted) { 51 | logMessage += adjustmentContext.AdjustmentType === 'increment' ? 52 | ' so an increment is WANTED' : ' so a decrement is WANTED'; 53 | if (adjustmentData.isAdjustmentAllowed) { 54 | logMessage += ' and is ALLOWED'; 55 | } else if (!adjustmentData.isAfterLastDecreaseGracePeriod) { 56 | logMessage += ' but is DISALLOWED due to \'AfterLastDecrementMinutes\' grace period'; 57 | } else if (!adjustmentData.isAfterLastIncreaseGracePeriod) { 58 | logMessage += ' but is DISALLOWED due to \'AfterLastIncreaseMinutes\' grace period'; 59 | } else { 60 | logMessage += ' but is DISALLOWED'; 61 | } 62 | } else { 63 | logMessage += adjustmentContext.AdjustmentType === 'increment' ? 64 | ' so an increment is not required' : ' so a decrement is not required'; 65 | } 66 | 67 | log(logMessage); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/utils/CostEstimation.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import { json, warning, invariant } from '../Global'; 3 | import type { 4 | Throughput, 5 | } from 'aws-sdk'; 6 | 7 | export default class CostEstimation { 8 | 9 | static getMonthlyEstimatedTableCost(provisionedThroughput: Throughput) { 10 | try { 11 | invariant(provisionedThroughput != null, 'Parameter \'provisionedThroughput\' is not set'); 12 | 13 | const averageHoursPerMonth = 720; 14 | const readCostPerHour = 0.0065; 15 | const readCostUnits = 50; 16 | const writeCostPerHour = 0.0065; 17 | const writeCostUnits = 10; 18 | 19 | let readCost = provisionedThroughput.ReadCapacityUnits / 20 | readCostUnits * readCostPerHour * averageHoursPerMonth; 21 | 22 | let writeCost = provisionedThroughput.WriteCapacityUnits / 23 | writeCostUnits * writeCostPerHour * averageHoursPerMonth; 24 | 25 | return readCost + writeCost; 26 | } catch (ex) { 27 | warning(JSON.stringify({ 28 | class: 'CostEstimation', 29 | function: 'getMonthlyEstimatedTableCost', 30 | provisionedThroughput 31 | }, null, json.padding)); 32 | throw ex; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/utils/Delay.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import { invariant } from '../Global'; 3 | 4 | export default class Delay { 5 | 6 | static delayAsync(ms: number) { 7 | invariant(typeof ms === 'number', 'Argument \'ms\' is not a number'); 8 | 9 | return new Promise(resolve => { 10 | setTimeout(resolve, ms); 11 | }); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/utils/RateLimitedDecrement.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import { invariant } from '../Global'; 3 | import type { 4 | TableProvisionedAndConsumedThroughput, 5 | AdjustmentContext 6 | } from '../flow/FlowTypes'; 7 | 8 | export default class RateLimitedDecrement { 9 | 10 | static isDecrementAllowed( 11 | data: TableProvisionedAndConsumedThroughput, 12 | adjustmentContext: AdjustmentContext, 13 | calcNewValueFunc: (data: TableProvisionedAndConsumedThroughput) => number) { 14 | 15 | invariant(data != null, 'Parameter \'data\' is not set'); 16 | invariant(adjustmentContext != null, 'Parameter \'adjustmentContext\' is not set'); 17 | invariant(calcNewValueFunc != null, 'Parameter \'calcNewValueFunc\' is not set'); 18 | 19 | if (this.getNextAllowedDecrementDate(data, adjustmentContext) > this.getNowDate()) { 20 | // Disallow if we havent crossed one of four time barriers 21 | return false; 22 | } 23 | 24 | let adjustment = Math.abs(adjustmentContext.ProvisionedValue) - 25 | Math.abs(calcNewValueFunc(data)); 26 | 27 | if (adjustmentContext.CapacityAdjustmentConfig != null && 28 | adjustmentContext.CapacityAdjustmentConfig.When.UnitAdjustmentGreaterThan != null && 29 | adjustment <= adjustmentContext.CapacityAdjustmentConfig.When.UnitAdjustmentGreaterThan && 30 | this.getNowDate().valueOf() < 31 | this.getLastAllowedDecrementDate().valueOf()) { 32 | // Disallow if the adjustment is very small. 33 | // However, if we have crossed the last time 34 | // barrier of the day then we might as well allow it. 35 | return false; 36 | } 37 | 38 | return true; 39 | } 40 | 41 | static getNextAllowedDecrementDate( 42 | data: TableProvisionedAndConsumedThroughput, 43 | adjustmentContext: AdjustmentContext) { 44 | 45 | // Check if we have already had all the decreases we are allowed today 46 | if (data.ProvisionedThroughput.NumberOfDecreasesToday >= 4) { 47 | return this.getTomorrowDate(); 48 | } 49 | 50 | // Get the last decrease or start of day 51 | let lastDecrease = this.parseDate(data.ProvisionedThroughput.LastDecreaseDateTime); 52 | let lastDecrementDate = this.getLastDecrementDate(lastDecrease); 53 | 54 | // Get the next allowed decrement 55 | let lastAllowedDecrementDate = this.getLastAllowedDecrementDate(); 56 | let periodMs = lastAllowedDecrementDate.valueOf() - lastDecrementDate.valueOf(); 57 | let periodMs2 = periodMs / (5 - data.ProvisionedThroughput.NumberOfDecreasesToday); 58 | let nextDecrementDate = this.getLastDecrementDate(lastDecrease); 59 | nextDecrementDate.setMilliseconds(nextDecrementDate.getMilliseconds() + periodMs2); 60 | 61 | // Handle grace periods 62 | let withIncrementGracePeriod = this.parseDate(data.ProvisionedThroughput.LastIncreaseDateTime); 63 | 64 | if (adjustmentContext.CapacityAdjustmentConfig != null && 65 | adjustmentContext.CapacityAdjustmentConfig.When.AfterLastIncrementMinutes != null) { 66 | let incMins = adjustmentContext.CapacityAdjustmentConfig.When.AfterLastIncrementMinutes; 67 | withIncrementGracePeriod.setMinutes(withIncrementGracePeriod.getMinutes() + incMins); 68 | } 69 | 70 | let withDecrementGracePeriod = this.parseDate(data.ProvisionedThroughput.LastDecreaseDateTime); 71 | 72 | if (adjustmentContext.CapacityAdjustmentConfig != null && 73 | adjustmentContext.CapacityAdjustmentConfig.When.AfterLastDecrementMinutes != null) { 74 | let decMins = adjustmentContext.CapacityAdjustmentConfig.When.AfterLastDecrementMinutes; 75 | withDecrementGracePeriod.setMinutes(withDecrementGracePeriod.getMinutes() + decMins); 76 | } 77 | 78 | let result = new Date(Math.max( 79 | nextDecrementDate, withIncrementGracePeriod, withDecrementGracePeriod)); 80 | 81 | return result; 82 | } 83 | 84 | static getNowDate() { 85 | return new Date(Date.now()); 86 | } 87 | 88 | static getTodayDate() { 89 | let value = this.getNowDate(); 90 | value.setHours(0, 0, 0, 0); 91 | return value; 92 | } 93 | 94 | static getTomorrowDate() { 95 | let value = this.getTodayDate(); 96 | value.setDate(value.getDate() + 1); 97 | return value; 98 | } 99 | 100 | static getLastAllowedDecrementDate() { 101 | let value = this.getTodayDate(); 102 | value.setHours(23, 30, 0, 0); 103 | return value; 104 | } 105 | 106 | static getLastDecrementDate(lastDecrease) { 107 | let today = this.getTodayDate(); 108 | return lastDecrease < today ? today : new Date(lastDecrease.valueOf()); 109 | } 110 | 111 | static parseDate(value) { 112 | // eslint-disable-next-line eqeqeq 113 | if (typeof value === 'undefined' || value == null) { 114 | return new Date(-8640000000000000); 115 | } 116 | 117 | return new Date(Date.parse(value)); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/utils/Stats.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | import measured from 'measured'; 3 | 4 | export default class Stats { 5 | _stats: measured.MeasuredCollection; 6 | 7 | constructor(stats: measured.MeasuredCollection) { 8 | this._stats = stats; 9 | } 10 | 11 | reset() { 12 | for(let name in this._stats._metrics) { 13 | if ({}.hasOwnProperty.call(this._stats._metrics, name)) { 14 | let metric = this._stats._metrics[name]; 15 | if (metric.unref) { 16 | metric.unref(); 17 | } 18 | } 19 | } 20 | 21 | this._stats._metrics = {}; 22 | } 23 | 24 | toJSON(): any { 25 | return this._stats.toJSON(); 26 | } 27 | 28 | getSummaries() { 29 | let statsData = this._stats.toJSON(); 30 | let statsSummary = Object 31 | .keys(statsData) 32 | .map(name => { 33 | let mean = this.to2Dec(statsData[name].histogram.mean); 34 | let count = statsData[name].meter.count; 35 | return {name, mean, count}; 36 | }); 37 | 38 | statsSummary.sort((a, b) => { 39 | if (a.mean < b.mean) { 40 | return 1; 41 | } 42 | if (a.mean > b.mean) { 43 | return -1; 44 | } 45 | return 0; 46 | }); 47 | 48 | let nameLen = Math.max.apply(Math, statsSummary.map(i => i.name.length)); 49 | let statsAsStrings = statsSummary.map(s => 50 | this.padRight(s.name, nameLen + 2) + 51 | this.padRight(s.mean + 'ms', 10) + 52 | ' ' + 53 | s.count); 54 | 55 | return statsAsStrings; 56 | } 57 | 58 | padRight(value: string, length: number) { 59 | return value + Array(length - value.length).join(' '); 60 | } 61 | 62 | padLeft(value: string, paddingValue: string) { 63 | return String(paddingValue + value).slice(-paddingValue.length); 64 | } 65 | 66 | to2Dec(value: number) { 67 | return parseFloat(parseFloat(Math.round(value * 100) / 100).toFixed(2)); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/utils/Throughput.js: -------------------------------------------------------------------------------- 1 | /* @flow */ 2 | /* eslint-disable max-len */ 3 | import { json, warning, invariant } from '../Global'; 4 | import type { TableDescription, DynamoDBProvisionedThroughput } from 'aws-sdk'; 5 | import type { TableProvisionedAndConsumedThroughput, AdjustmentContext } from '../flow/FlowTypes'; 6 | 7 | export default class Throughput { 8 | 9 | static getReadCapacityUtilisationPercent(data: TableProvisionedAndConsumedThroughput) { 10 | invariant(data != null, 'Parameter \'data\' is not set'); 11 | 12 | return ( 13 | data.ConsumedThroughput.ReadCapacityUnits / 14 | data.ProvisionedThroughput.ReadCapacityUnits) * 100; 15 | } 16 | 17 | static getWriteCapacityUtilisationPercent(data: TableProvisionedAndConsumedThroughput) { 18 | invariant(data != null, 'Parameter \'data\' is not set'); 19 | 20 | return ( 21 | data.ConsumedThroughput.WriteCapacityUnits / 22 | data.ProvisionedThroughput.WriteCapacityUnits) * 100; 23 | } 24 | 25 | static getAdjustedCapacityUnits(adjustmentContext: AdjustmentContext): number { 26 | invariant(adjustmentContext != null, 'Parameter \'adjustmentContext\' is not set'); 27 | 28 | // If the provisioned units is less than minimum then simply return the minimum allowed 29 | if (adjustmentContext.CapacityConfig.Min != null && 30 | adjustmentContext.ProvisionedValue < adjustmentContext.CapacityConfig.Min) { 31 | return adjustmentContext.CapacityConfig.Min; 32 | } 33 | 34 | // If the provisioned units is greater than maximum then simply return the maximum allowed 35 | if (adjustmentContext.CapacityConfig.Max != null && 36 | adjustmentContext.ProvisionedValue > adjustmentContext.CapacityConfig.Max) { 37 | return adjustmentContext.CapacityConfig.Max; 38 | } 39 | 40 | let direction = adjustmentContext.AdjustmentType === 'increment' ? 1 : -1; 41 | 42 | // Increment 'by' throttled events and configured mutliplier, increments only! 43 | let byTE = (direction === 1 && adjustmentContext.CapacityAdjustmentConfig != null && adjustmentContext.CapacityAdjustmentConfig.By != null && adjustmentContext.CapacityAdjustmentConfig.By.ThrottledEventsWithMultiplier != null) ? 44 | (adjustmentContext.ThrottledEvents * adjustmentContext.CapacityAdjustmentConfig.By.ThrottledEventsWithMultiplier) : 45 | 0; 46 | let byTEVal = adjustmentContext.ProvisionedValue + byTE; 47 | 48 | // Increment 'by' percentage of provisioned 49 | let byP = (adjustmentContext.CapacityAdjustmentConfig != null && adjustmentContext.CapacityAdjustmentConfig.By != null && adjustmentContext.CapacityAdjustmentConfig.By.ProvisionedPercent != null) ? 50 | (((adjustmentContext.ProvisionedValue / 100) * adjustmentContext.CapacityAdjustmentConfig.By.ProvisionedPercent) * direction) : 51 | 0; 52 | let byPVal = adjustmentContext.ProvisionedValue + byP + byTE; 53 | 54 | // Increment 'by' percentage of consumed 55 | let byC = (adjustmentContext.CapacityAdjustmentConfig != null && adjustmentContext.CapacityAdjustmentConfig.By != null && adjustmentContext.CapacityAdjustmentConfig.By.ConsumedPercent != null) ? 56 | (((adjustmentContext.ConsumedValue / 100) * adjustmentContext.CapacityAdjustmentConfig.By.ConsumedPercent) * direction) : 57 | 0; 58 | let byCVal = adjustmentContext.ProvisionedValue + byC + byTE; 59 | 60 | // Increment 'by' unit value 61 | let byU = (adjustmentContext.CapacityAdjustmentConfig != null && adjustmentContext.CapacityAdjustmentConfig.By != null && adjustmentContext.CapacityAdjustmentConfig.By.Units != null) ? 62 | (adjustmentContext.CapacityAdjustmentConfig.By.Units * direction) : 63 | 0; 64 | let byUVal = adjustmentContext.ProvisionedValue + byU + byTE; 65 | 66 | // Increment 'to' percentage of provisioned 67 | let toP = (adjustmentContext.CapacityAdjustmentConfig != null && adjustmentContext.CapacityAdjustmentConfig.To != null && adjustmentContext.CapacityAdjustmentConfig.To.ProvisionedPercent != null) ? 68 | (adjustmentContext.ProvisionedValue / 100) * adjustmentContext.CapacityAdjustmentConfig.To.ProvisionedPercent : 69 | adjustmentContext.ProvisionedValue; 70 | 71 | // Increment 'to' percentage of consumed 72 | let toC = (adjustmentContext.CapacityAdjustmentConfig != null && adjustmentContext.CapacityAdjustmentConfig.To != null && adjustmentContext.CapacityAdjustmentConfig.To.ConsumedPercent != null) ? 73 | (adjustmentContext.ConsumedValue / 100) * adjustmentContext.CapacityAdjustmentConfig.To.ConsumedPercent : 74 | adjustmentContext.ProvisionedValue; 75 | 76 | // Increment 'to' unit value 77 | let toU = (adjustmentContext.CapacityAdjustmentConfig != null && adjustmentContext.CapacityAdjustmentConfig.To != null && adjustmentContext.CapacityAdjustmentConfig.To.Units != null) ? 78 | adjustmentContext.CapacityAdjustmentConfig.To.Units : 79 | adjustmentContext.ProvisionedValue; 80 | 81 | // Select the greatest calculated increment 82 | let newValue = adjustmentContext.AdjustmentType === 'increment' ? 83 | Math.max(byPVal, byCVal, byUVal, byTEVal, toP, toC, toU) : 84 | Math.min(byPVal, byCVal, byUVal, byTEVal, toP, toC, toU); 85 | 86 | // Limit to 'max' if it is specified 87 | if (adjustmentContext.CapacityConfig.Max != null) { 88 | newValue = Math.min(newValue, adjustmentContext.CapacityConfig.Max); 89 | } 90 | 91 | // Limit to 'min' if it is specified 92 | if (adjustmentContext.CapacityConfig.Min != null) { 93 | newValue = Math.max(newValue, adjustmentContext.CapacityConfig.Min, 1); 94 | } 95 | 96 | // Ensure we return a whole number 97 | return Math.round(newValue); 98 | } 99 | 100 | static getTotalTableProvisionedThroughput(params: TableDescription) 101 | : DynamoDBProvisionedThroughput { 102 | try { 103 | invariant(typeof params !== 'undefined', 'Parameter \'params\' is not set'); 104 | 105 | let ReadCapacityUnits = params.ProvisionedThroughput.ReadCapacityUnits; 106 | let WriteCapacityUnits = params.ProvisionedThroughput.WriteCapacityUnits; 107 | 108 | if (params.GlobalSecondaryIndexes) { 109 | ReadCapacityUnits += params.GlobalSecondaryIndexes 110 | .reduce((prev, curr) => 111 | prev + curr.ProvisionedThroughput.ReadCapacityUnits, 0); 112 | 113 | WriteCapacityUnits += params.GlobalSecondaryIndexes 114 | .reduce((prev, curr) => 115 | prev + curr.ProvisionedThroughput.WriteCapacityUnits, 0); 116 | } 117 | 118 | return { 119 | ReadCapacityUnits, 120 | WriteCapacityUnits 121 | }; 122 | } catch (ex) { 123 | warning(JSON.stringify({ 124 | class: 'Throughput', 125 | function: 'getTotalTableProvisionedThroughput', 126 | params 127 | }, null, json.padding)); 128 | throw ex; 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /webpack-dev.config.js: -------------------------------------------------------------------------------- 1 | module.exports = require("./make-webpack-config")({ 2 | devtool: "source-map", 3 | debug: true 4 | }); 5 | -------------------------------------------------------------------------------- /webpack-prod.config.js: -------------------------------------------------------------------------------- 1 | module.exports = require("./make-webpack-config")({ 2 | minimize: true 3 | }); 4 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | module.exports = require("./make-webpack-config")({ 2 | 3 | }); 4 | --------------------------------------------------------------------------------