├── .gitignore ├── download-metrics.js ├── events ├── newAlarm.json └── sns.json ├── functions ├── change_cw_alarm.js ├── cloudwatch.js ├── dynamodb.js ├── generate_head_heavy_load.js ├── generate_linear_load.js ├── load_tester.js └── scale_up_dynamodb.js ├── package.json └── serverless.yml /.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | 9 | .csv -------------------------------------------------------------------------------- /download-metrics.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const co = require('co'); 5 | const Promise = require('bluebird'); 6 | const AWS = require('aws-sdk'); 7 | AWS.config.region = 'us-east-1'; 8 | const cloudwatch = Promise.promisifyAll(new AWS.CloudWatch()); 9 | const Lambda = new AWS.Lambda(); 10 | 11 | const startTime = new Date('2017-07-30T22:45:00.000Z'); 12 | const endTime = new Date('2017-07-31T00:45:00.000Z'); 13 | 14 | const metrics = [ 15 | { namespace: "theburningmonk.com", metricName: "dynamodb_scaling_reqs_count" }, 16 | { namespace: "AWS/DynamoDB", metricName: "ConsumedWriteCapacityUnits" }, 17 | { namespace: "AWS/DynamoDB", metricName: "ProvisionedWriteCapacityUnits" }, 18 | { namespace: "AWS/DynamoDB", metricName: "WriteThrottleEvents" } 19 | ]; 20 | 21 | const tables = [ 22 | // "yc_proposal2_30", 23 | // "yc_proposal2_40", 24 | // "yc_proposal2_50", 25 | // "yc_proposal2_60", 26 | // "yc_proposal2_70", 27 | // "yc_proposal2_80" 28 | "yc_proposal2_top_heavy_30", 29 | "yc_proposal2_top_heavy_40", 30 | "yc_proposal2_top_heavy_50", 31 | "yc_proposal2_top_heavy_60", 32 | "yc_proposal2_top_heavy_70", 33 | "yc_proposal2_top_heavy_80" 34 | ]; 35 | 36 | let getTableMetrics = co.wrap(function* (tableName) { 37 | let getMetrics = co.wrap(function* (namespace, metricName, startTime, endTime) { 38 | let req = { 39 | MetricName: metricName, 40 | Namespace: namespace, 41 | Period: 60, 42 | Dimensions: [ { Name: 'TableName', Value: tableName } ], 43 | Statistics: [ 'Sum' ], 44 | StartTime: startTime, 45 | EndTime: endTime 46 | }; 47 | let resp = yield cloudwatch.getMetricStatisticsAsync(req); 48 | 49 | return resp.Datapoints.map(dp => { 50 | let value = (metricName === 'ProvisionedWriteCapacityUnits') ? dp.Sum * 60 : dp.Sum; 51 | 52 | return { 53 | tableName, 54 | metricName, 55 | timestamp: dp.Timestamp, 56 | value 57 | }; 58 | }); 59 | }); 60 | 61 | let metricsDatum = []; 62 | for (let metric of metrics) { 63 | let datum = yield getMetrics(metric.namespace, metric.metricName, startTime, endTime); 64 | metricsDatum = metricsDatum.concat(datum); 65 | } 66 | 67 | return _.sortBy(metricsDatum, s => `${s.tableName}_${s.metricName}_${s.timestamp}`); 68 | }); 69 | 70 | let run = co.wrap(function* () { 71 | let rows = _.flatten(yield tables.map(tableName => getTableMetrics(tableName))); 72 | for (let row of rows) { 73 | console.log(`${row.tableName},${row.metricName},${row.timestamp},${row.value}`); 74 | } 75 | }); 76 | 77 | run(); -------------------------------------------------------------------------------- /events/newAlarm.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0", 3 | "id": "0220e467-fdd9-4888-b426-30b46283d173", 4 | "detail-type": "AWS API Call via CloudTrail", 5 | "source": "aws.monitoring", 6 | "account": "499014364541", 7 | "time": "2017-07-08T16:53:28Z", 8 | "region": "us-east-1", 9 | "resources": [], 10 | "detail": { 11 | "eventVersion": "1.04", 12 | "userIdentity": { 13 | "type": "AssumedRole", 14 | "principalId": "AROAI6GB7EPYB6G6OGRSA:AutoScaling-ManageAlarms", 15 | "arn": "arn:aws:sts::499014364541:assumed-role/DynamoDBAutoscaleRole/AutoScaling-ManageAlarms", 16 | "accountId": "499014364541", 17 | "accessKeyId": "ASIAJFM3RFSRBXLFO6AA", 18 | "sessionContext": { 19 | "attributes": { 20 | "mfaAuthenticated": "false", 21 | "creationDate": "2017-07-08T16:53:28Z" 22 | }, 23 | "sessionIssuer": { 24 | "type": "Role", 25 | "principalId": "AROAI6GB7EPYB6G6OGRSA", 26 | "arn": "arn:aws:iam::499014364541:role/service-role/DynamoDBAutoscaleRole", 27 | "accountId": "499014364541", 28 | "userName": "DynamoDBAutoscaleRole" 29 | } 30 | }, 31 | "invokedBy": "application-autoscaling.amazonaws.com" 32 | }, 33 | "eventTime": "2017-07-08T16:53:28Z", 34 | "eventSource": "monitoring.amazonaws.com", 35 | "eventName": "PutMetricAlarm", 36 | "awsRegion": "us-east-1", 37 | "sourceIPAddress": "application-autoscaling.amazonaws.com", 38 | "userAgent": "application-autoscaling.amazonaws.com", 39 | "requestParameters": { 40 | "threshold": 1500, 41 | "period": 60, 42 | "metricName": "ConsumedWriteCapacityUnits", 43 | "dimensions": [ 44 | { 45 | "value": "dynamo_scaling_1min", 46 | "name": "TableName" 47 | } 48 | ], 49 | "evaluationPeriods": 5, 50 | "actionsEnabled": true, 51 | "comparisonOperator": "GreaterThanThreshold", 52 | "namespace": "AWS/DynamoDB", 53 | "alarmName": "TargetTracking-table/dynamo_scaling_1min-AlarmHigh-33cdd4a8-df47-4489-b296-f9143e5769e8", 54 | "alarmActions": [ 55 | "arn:aws:autoscaling:us-east-1:499014364541:scalingPolicy:8a923775-de67-4982-b2dc-9f8c6a180cfb:resource/dynamodb/table/dynamo_scaling_1min:policyName/DynamoDBWriteCapacityUtilization:table/dynamo_scaling_1min" 56 | ], 57 | "alarmDescription": "DO NOT EDIT OR DELETE. For TargetTrackingScaling policy arn:aws:autoscaling:us-east-1:499014364541:scalingPolicy:8a923775-de67-4982-b2dc-9f8c6a180cfb:resource/dynamodb/table/dynamo_scaling_1min:policyName/DynamoDBWriteCapacityUtilization:table/dynamo_scaling_1min.", 58 | "statistic": "Sum" 59 | }, 60 | "responseElements": null, 61 | "requestID": "f7805d84-63fd-11e7-928f-f9a6a551af7d", 62 | "eventID": "95557306-2503-4adb-9ce4-5710b6a40aac", 63 | "eventType": "AwsApiCall" 64 | } 65 | } -------------------------------------------------------------------------------- /events/sns.json: -------------------------------------------------------------------------------- 1 | { 2 | "Records": [ 3 | { 4 | "EventSource": "aws:sns", 5 | "EventVersion": "1.0", 6 | "EventSubscriptionArn": "arn:aws:sns:us-east-1:499014364541:scale_up_dynamodb:4dfaf27b-8819-4a16-993c-a3d345e1e8b8", 7 | "Sns": { 8 | "Type": "Notification", 9 | "MessageId": "14c479c0-7ac7-53a8-9347-281b6e3bf2dc", 10 | "TopicArn": "arn:aws:sns:us-east-1:499014364541:scale_up_dynamodb", 11 | "Subject": "ALARM: \"TargetTracking-table/dynamo_scaling_1min-AlarmHigh-33cdd4a8-df4...\" in US East - N. Virginia", 12 | "Message": "{\"AlarmName\":\"TargetTracking-table/dynamo_scaling_1min-AlarmHigh-33cdd4a8-df47-4489-b296-f9143e5769e8\",\"AlarmDescription\":\"DO NOT EDIT OR DELETE. For TargetTrackingScaling policy arn:aws:autoscaling:us-east-1:499014364541:scalingPolicy:8a923775-de67-4982-b2dc-9f8c6a180cfb:resource/dynamodb/table/dynamo_scaling_1min:policyName/DynamoDBWriteCapacityUtilization:table/dynamo_scaling_1min.\",\"AWSAccountId\":\"499014364541\",\"NewStateValue\":\"ALARM\",\"NewStateReason\":\"Threshold Crossed: 1 datapoint [1808.6111111111113 (08/07/17 17:12:00)] was greater than the threshold (1500.0).\",\"StateChangeTime\":\"2017-07-08T17:13:32.930+0000\",\"Region\":\"US East - N. Virginia\",\"OldStateValue\":\"OK\",\"Trigger\":{\"MetricName\":\"dynamodb_scaling_reqs_count\",\"Namespace\":\"theburningmonk.com\",\"StatisticType\":\"Statistic\",\"Statistic\":\"SUM\",\"Unit\":null,\"Dimensions\":[{\"name\":\"TableName\",\"value\":\"dynamo_scaling_1min\"}],\"Period\":60,\"EvaluationPeriods\":1,\"ComparisonOperator\":\"GreaterThanThreshold\",\"Threshold\":1500.0,\"TreatMissingData\":\"\",\"EvaluateLowSampleCountPercentile\":\"\"}}", 13 | "Timestamp": "2017-07-08T17:13:32.963Z", 14 | "SignatureVersion": "1", 15 | "Signature": "DGk915rk+C6MVjnNLnyB5/xzw/bF1bN78A03BnMTzVY4ug3yJ5L4zL2XWVSndnCOHKZWV8pOHqxnC253vuKRYncosoH9VBxWG1aaAkY4nYDQwv5YRmDGd2X4ZERa3Ji2Qcq40tgR5WFaaGZ90rXaztX2QJtXc/nDdAbvrJ6UJHalomIgi7JOetQ+bIb8o2hs9lQhbEadA2BMVT+hApC+TBOuvhgr0n6nQ+MrLn505xhO/yk2Wzs4vj59aNrqr5O21wMEEEscjI67QesIpsv4RZalmt7I9FIa61/y96U2CNMA8EFhbmCFv180ftPYX+bJXcQd1Co5+ctyYk4WCUu+/Q==", 16 | "SigningCertUrl": "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-b95095beb82e8f6a046b3aafc7f4149a.pem", 17 | "UnsubscribeUrl": "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:499014364541:scale_up_dynamodb:4dfaf27b-8819-4a16-993c-a3d345e1e8b8", 18 | "MessageAttributes": {} 19 | } 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /functions/change_cw_alarm.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const AWS = require('aws-sdk'); 4 | const co = require('co'); 5 | const Promise = require('bluebird'); 6 | const _ = require('lodash'); 7 | const cloudwatch = require('./cloudwatch'); 8 | const accountId = process.env.accountId; 9 | const region = process.env.region; 10 | const snsTopic = `arn:aws:sns:${region}:${accountId}:scale_up_dynamodb`; 11 | 12 | let makeProposal2Change = co.wrap(function* (alarmName) { 13 | yield cloudwatch.cloneAndPutMetricAlarm( 14 | alarmName, 15 | x => { 16 | x.Namespace = 'theburningmonk.com'; 17 | x.MetricName = 'dynamodb_scaling_reqs_count'; 18 | x.EvaluationPeriods = 1; 19 | x.AlarmActions = [ snsTopic ] 20 | }); 21 | }); 22 | 23 | let makeProposal1Change = co.wrap(function* (alarmName) { 24 | yield cloudwatch.cloneAndPutMetricAlarm( 25 | alarmName, 26 | x => { 27 | x.EvaluationPeriods = 1; 28 | } 29 | ); 30 | }); 31 | 32 | module.exports.handler = co.wrap(function* (event, context, callback) { 33 | console.log(JSON.stringify(event)); 34 | 35 | let metricName = _.get(event, 'detail.requestParameters.metricName'); 36 | let alarmName = _.get(event, 'detail.requestParameters.alarmName'); 37 | let evaluationPeriods = _.get(event, 'detail.requestParameters.evaluationPeriods'); 38 | 39 | console.log(`metric name : ${metricName}`); 40 | console.log(`alarm name : ${alarmName}`); 41 | console.log(`eval period : ${evaluationPeriods}`); 42 | 43 | // proposal 1 44 | if (/Consumed.*CapacityUnits/.test(metricName) && 45 | alarmName.includes("AlarmHigh") && 46 | alarmName.includes("_proposal1") && 47 | evaluationPeriods !== 1) { 48 | let alarmName = event.detail.requestParameters.alarmName; 49 | 50 | console.log(`[Proposal 1] updating [${alarmName}]...`); 51 | yield makeProposal1Change(alarmName); 52 | console.log("all done"); 53 | } 54 | // proposal 2 55 | else if (/Consumed.*CapacityUnits/.test(metricName) && 56 | alarmName.includes("AlarmHigh") && 57 | alarmName.includes("_proposal2") && 58 | evaluationPeriods !== 1) { 59 | let alarmName = event.detail.requestParameters.alarmName; 60 | 61 | console.log(`[Proposal 2] updating [${alarmName}]...`); 62 | yield makeProposal2Change(alarmName); 63 | console.log("all done"); 64 | } 65 | 66 | callback(null, 'ok'); 67 | }); -------------------------------------------------------------------------------- /functions/cloudwatch.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const co = require('co'); 5 | const AWS = require('aws-sdk'); 6 | const Promise = require('bluebird'); 7 | const cloudwatch = Promise.promisifyAll(new AWS.CloudWatch()); 8 | 9 | let getAlarm = co.wrap(function* (alarmName) { 10 | let req = { 11 | AlarmNames: [ alarmName ], 12 | MaxRecords: 1 13 | }; 14 | let resp = yield cloudwatch.describeAlarmsAsync(req); 15 | let alarm = resp.MetricAlarms[0]; 16 | 17 | delete alarm.AlarmArn; 18 | delete alarm.AlarmConfigurationUpdatedTimestamp; 19 | delete alarm.StateValue; 20 | delete alarm.StateReason; 21 | delete alarm.StateReasonData; 22 | delete alarm.StateUpdatedTimestamp; 23 | 24 | return alarm; 25 | }); 26 | 27 | let cloneAndPutMetricAlarm = co.wrap(function* (alarmName, update) { 28 | let alarm = yield getAlarm(alarmName); 29 | let clone = _.cloneDeep(alarm); 30 | update(clone); 31 | 32 | console.log(JSON.stringify(clone)); 33 | yield cloudwatch.putMetricAlarmAsync(clone); 34 | 35 | console.log(`updated alarm [${clone.AlarmName}]`); 36 | }); 37 | 38 | let putMetric = co.wrap(function* (namespace, metricName, tableName, value) { 39 | let req = { 40 | MetricData: [ 41 | { 42 | MetricName: metricName, 43 | Dimensions: [ 44 | { 45 | Name: 'TableName', 46 | Value: tableName 47 | } 48 | ], 49 | Timestamp: new Date, 50 | Unit: 'Count', 51 | Value: value 52 | } 53 | ], 54 | Namespace: namespace 55 | }; 56 | yield cloudwatch.putMetricDataAsync(req); 57 | }); 58 | 59 | let getLast5MinMetrics = co.wrap(function* (namespace, metricName, tableName) { 60 | let end = new Date(); 61 | let start = new Date(end.getTime() - 5 * 60 * 1000); 62 | 63 | let req = { 64 | EndTime: end, 65 | MetricName: metricName, 66 | Namespace: namespace, 67 | Period: 60, 68 | StartTime: start, 69 | Dimensions: [ 70 | { 71 | Name: 'TableName', 72 | Value: tableName 73 | } 74 | ], 75 | Statistics: [ 'Sum' ] 76 | }; 77 | let resp = yield cloudwatch.getMetricStatisticsAsync(req); 78 | return resp.Datapoints.map(dp => dp.Sum); 79 | }); 80 | 81 | let setAlarmToOK = co.wrap(function* (alarmName) { 82 | let req = { 83 | AlarmName: alarmName, 84 | StateReason: 'handled', 85 | StateValue: 'OK' 86 | }; 87 | yield cloudwatch.setAlarmStateAsync(req) 88 | }); 89 | 90 | module.exports = { 91 | cloneAndPutMetricAlarm, 92 | putMetric, 93 | getLast5MinMetrics, 94 | setAlarmToOK 95 | }; -------------------------------------------------------------------------------- /functions/dynamodb.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const co = require('co'); 4 | const Promise = require('bluebird'); 5 | const AWS = require('aws-sdk'); 6 | const dynamodb = Promise.promisifyAll(new AWS.DynamoDB()); 7 | 8 | let updateThroughput = co.wrap(function* (tableName, newWriteThroughput) { 9 | try { 10 | let req = { 11 | ProvisionedThroughput: { 12 | ReadCapacityUnits: 5, 13 | WriteCapacityUnits: newWriteThroughput 14 | }, 15 | TableName: tableName 16 | }; 17 | yield dynamodb.updateTableAsync(req); 18 | 19 | console.log(`updated table [${tableName}] to [${newWriteThroughput}] write throughput`); 20 | } catch (err) { 21 | console.log(err); 22 | } 23 | }); 24 | 25 | module.exports.updateThroughput = updateThroughput; -------------------------------------------------------------------------------- /functions/generate_head_heavy_load.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const co = require('co'); 4 | const loadTester = require('./load_tester'); 5 | 6 | const peak = 300; 7 | const trough = 25; 8 | const warmup_time = 15 * 60; // 15 mins 9 | const peak_time = 20 * 60; // 20 mins 10 | 11 | 12 | /* traffic pattern (roughly speaking...) 13 | ________ 14 | / --------________ 15 | / --------________ 16 | / --------________ 17 | _____/ --------________ 18 | 19 | */ 20 | function tickToCount(n) { 21 | if (n <= warmup_time) { 22 | return trough; // holding pattern for the first 15 mins 23 | } else if (n <= peak_time) { 24 | let dn = (peak - trough) / (peak_time - warmup_time); 25 | return trough + (n - warmup_time) * dn; // then aggressive spike to 300 ops/s for the next 5 mins 26 | } else { 27 | // slowly decreases at 3 ops/s per min 28 | let dm = 3 / 60; 29 | let m = n - peak_time; 30 | 31 | return peak - (dm * m); 32 | } 33 | } 34 | 35 | module.exports.handler = co.wrap(function* (input, context, callback) { 36 | yield loadTester(input, context, callback, tickToCount); 37 | }); -------------------------------------------------------------------------------- /functions/generate_linear_load.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const co = require('co'); 4 | const loadTester = require('./load_tester'); 5 | 6 | const peak = 300; 7 | const trough = 25; 8 | const warmup_time = 15 * 60; // 15 mins 9 | const peak_time = 60 * 60; // 60 mins 10 | const holding_time = 75 * 60; // 75 mins 11 | // ps. to go from 25 to 300 over 45 mins equates to ~6 ops/s increase per min 12 | 13 | /* traffic pattern (roughly speaking...) 14 | 15 | ___------------_ 16 | ___--- -_ 17 | ___--- -_ 18 | _____--- -_________ 19 | 20 | */ 21 | function tickToCount(n) { 22 | if (n <= warmup_time) { // holding pattern for the first 15 mins 23 | return trough; 24 | } else if (n <= peak_time) { // then gradually ramp up to peak load 25 | let dn = (peak - trough) / (peak_time - warmup_time); 26 | return trough + (n - warmup_time) * dn; 27 | } else if (n <= holding_time) { // then gradually ramp down to trough 28 | let dm = (peak - trough) / (holding_time - peak_time); 29 | let m = n - peak_time; 30 | 31 | return peak - (dm * m); 32 | } else { 33 | return trough; 34 | } 35 | } 36 | 37 | module.exports.handler = co.wrap(function* (input, context, callback) { 38 | yield loadTester(input, context, callback, tickToCount); 39 | }); -------------------------------------------------------------------------------- /functions/load_tester.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const co = require('co'); 4 | const AWS = require('aws-sdk'); 5 | const lambda = new AWS.Lambda(); 6 | const Promise = require('bluebird'); 7 | const _ = require('lodash'); 8 | const dynamodb = Promise.promisifyAll(new AWS.DynamoDB.DocumentClient()); 9 | const uuidv4 = require('uuid/v4'); 10 | const cloudwatch = require('./cloudwatch'); 11 | 12 | let putItems = co.wrap(function* (tableName, count) { 13 | let batchWrite = function* (batch) { 14 | try { 15 | let req = { RequestItems: {} }; 16 | req.RequestItems[tableName] = batch; 17 | 18 | console.log(`saving batch of [${batch.length}]`); 19 | 20 | yield dynamodb.batchWriteAsync(req); 21 | } catch (exn) { 22 | console.log(exn); 23 | } 24 | }; 25 | 26 | let start = new Date().getTime(); 27 | 28 | let items = _.range(0, count).map(n => { 29 | return { 30 | PutRequest: { 31 | Item: { id: uuidv4() } 32 | } 33 | }; 34 | }); 35 | 36 | console.log(`saving a total of [${items.length}] items`); 37 | 38 | let chunks = _.chunk(items, 25); 39 | let tasks = chunks.map(batchWrite); 40 | 41 | yield tasks; 42 | 43 | console.log(`finished saving [${items.length}] items`); 44 | 45 | yield cloudwatch.putMetric('theburningmonk.com', 'dynamodb_scaling_reqs_count', tableName, count); 46 | 47 | console.log(`tracked request count in cloudwatch`); 48 | 49 | let end = new Date().getTime(); 50 | let duration = end - start; 51 | if (duration < 1000) { 52 | let delay = 1000 - duration; 53 | console.log(`waiting a further ${delay}ms`); 54 | yield Promise.delay(delay); 55 | } 56 | }); 57 | 58 | let recurse = co.wrap(function* (funcName, payload) { 59 | console.log("recursing..."); 60 | 61 | let req = { 62 | FunctionName: funcName, 63 | InvocationType: "Event", 64 | Payload: payload 65 | }; 66 | yield lambda.invoke(req).promise(); 67 | }); 68 | 69 | // input should be of shape: 70 | // { 71 | // tableName: ..., 72 | // tick: 0, 73 | // recursionLeft: 0 74 | // } 75 | module.exports = co.wrap(function* (input, context, callback, tickToItemCount) { 76 | context.callbackWaitsForEmptyEventLoop = false; 77 | console.log(JSON.stringify(input)); 78 | 79 | let funcName = context.functionName; 80 | let tableName = input.tableName; 81 | let tick = input.tick || 0; 82 | let recursionLeft = input.recursionLeft || 0; 83 | 84 | if (recursionLeft <= 0) { 85 | callback(null, "All done"); 86 | return; 87 | } 88 | 89 | while (context.getRemainingTimeInMillis() > 2000) { 90 | try { 91 | let count = tickToItemCount(tick); 92 | yield putItems(tableName, count); 93 | } catch (err) { 94 | } 95 | 96 | tick += 1; 97 | console.log(`there are [${context.getRemainingTimeInMillis()}]ms left`); 98 | } 99 | 100 | let output = _.clone(input); 101 | output.recursionLeft = recursionLeft - 1; 102 | output.tick = tick; 103 | 104 | yield recurse(funcName, JSON.stringify(output)); 105 | 106 | callback(null, output); 107 | }); -------------------------------------------------------------------------------- /functions/scale_up_dynamodb.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const co = require('co'); 5 | const cloudwatch = require('./cloudwatch'); 6 | const dynamodb = require('./dynamodb'); 7 | 8 | let getMaxReqsCount = co.wrap(function* (namespace, metricName, tableName) { 9 | let datum = yield cloudwatch.getLast5MinMetrics(namespace, metricName, tableName); 10 | return _.max(datum); 11 | }); 12 | 13 | module.exports.handler = co.wrap(function* (event, context, callback) { 14 | console.log(JSON.stringify(event)); 15 | 16 | let message = JSON.parse(event.Records[0].Sns.Message); 17 | let alarmName = message.AlarmName; 18 | let namespace = message.Trigger.Namespace; 19 | let metricName = message.Trigger.MetricName; 20 | let tableName = message.Trigger.Dimensions[0].value; 21 | let utilizationLevel = parseInt(tableName.substring(tableName.lastIndexOf("_") + 1)) / 100; 22 | let threshold = message.Trigger.Threshold; 23 | 24 | let maxReqsCount = yield getMaxReqsCount(namespace, metricName, tableName); 25 | let newThroughput = Math.min(1000, maxReqsCount / utilizationLevel / 60); 26 | let newThreshold = newThroughput * 60 * utilizationLevel; 27 | 28 | console.log(` 29 | Alarm Bame: ${alarmName} 30 | Metric 31 | Namespace: ${namespace} 32 | Metric Name: ${metricName} 33 | Table Name: ${tableName} 34 | Old Threshold: ${threshold} 35 | Max reqs count in last 5 min: ${maxReqsCount} 36 | New Throughput: ${newThroughput} 37 | New Threshold: ${newThreshold} 38 | `); 39 | 40 | if (newThroughput) { 41 | yield dynamodb.updateThroughput(tableName, newThroughput); 42 | yield cloudwatch.cloneAndPutMetricAlarm( 43 | alarmName, 44 | x => x.Threshold = newThreshold); 45 | } 46 | 47 | yield cloudwatch.setAlarmToOK(alarmName); 48 | 49 | callback(null, "ok"); 50 | }); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "better-dynamodb-scaling", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "aws-sdk": "^2.80.0", 13 | "bluebird": "^3.5.0", 14 | "co": "^4.6.0", 15 | "lodash": "^4.17.4", 16 | "uuid": "^3.1.0" 17 | }, 18 | "devDependencies": { 19 | "serverless": "^1.16.1" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: better-dynamodb-scaling 2 | 3 | provider: 4 | name: aws 5 | runtime: nodejs6.10 6 | stage: dev 7 | region: us-east-1 8 | iamRoleStatements: 9 | - Effect: Allow 10 | Action: 11 | - cloudwatch:PutMetricAlarm 12 | - cloudwatch:PutMetricData 13 | - cloudwatch:DescribeAlarms 14 | - cloudwatch:GetMetricStatistics 15 | - cloudwatch:SetAlarmState 16 | Resource: "*" 17 | - Effect: Allow 18 | Action: 19 | - dynamodb:BatchWriteItem 20 | - dynamodb:PutItem 21 | - dynamodb:UpdateTable 22 | Resource: "*" 23 | - Effect: Allow 24 | Action: 25 | - lambda:InvokeFunction 26 | Resource: "arn:aws:lambda:us-east-1:*:function:${self:service}-${opt:stage, self:provider.stage}-*" 27 | 28 | functions: 29 | scale_up_dynamodb: 30 | handler: functions/scale_up_dynamodb.handler 31 | events: 32 | - sns: 33 | topicName: scale_up_dynamodb 34 | displayName: Scale Up DynamoDB table 35 | 36 | generate_linear_load: 37 | handler: functions/generate_linear_load.handler 38 | timeout: 300 39 | 40 | generate_head_heavy_load: 41 | handler: functions/generate_head_heavy_load.handler 42 | timeout: 300 43 | 44 | change_cw_alarm: 45 | handler: functions/change_cw_alarm.handler 46 | environment: 47 | accountId: 48 | Ref: AWS::AccountId 49 | region: 50 | ${opt:region, self:provider.region} 51 | events: 52 | - cloudwatchEvent: 53 | event: 54 | source: 55 | - aws.monitoring 56 | detail-type: 57 | - AWS API Call via CloudTrail 58 | detail: 59 | eventSource: 60 | - monitoring.amazonaws.com 61 | eventName: 62 | - PutMetricAlarm --------------------------------------------------------------------------------