├── .gitignore ├── README.md ├── dynamodb └── 2012-08-10 │ ├── examples-1.json │ ├── paginators-1.json │ ├── service-2.json │ └── waiters-2.json ├── handler.py ├── serverless.yml └── tables.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Distribution / packaging 2 | .Python 3 | env/ 4 | build/ 5 | develop-eggs/ 6 | dist/ 7 | downloads/ 8 | eggs/ 9 | .eggs/ 10 | lib/ 11 | lib64/ 12 | parts/ 13 | sdist/ 14 | var/ 15 | *.egg-info/ 16 | .installed.cfg 17 | *.egg 18 | 19 | # Serverless directories 20 | .serverless -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Automated DynamoDB Backups ⚡️ 2 | 3 | # Usage 4 | 5 | 1. Install the [Serverless Framework](https://serverless.com/framework/): 6 | 7 | ```bash 8 | $ npm install -g serverless 9 | ``` 10 | 11 | 2. Install this repository as a Serverless project: 12 | 13 | ```bash 14 | $ sls install --url https://github.com/alexdebrie/serverless-dynamodb-backups && cd serverless-dynamodb-backups 15 | ``` 16 | 17 | 3. Update the required [configuration](#configuration) in the `custom` block of `serverless.yml`. 18 | 19 | 4. Deploy! 20 | 21 | ```bash 22 | $ sls deploy 23 | ``` 24 | 25 | # Configuration 26 | 27 | There are three ways you can specify which tables to backup: 28 | 29 | 1. **Regex on all tables in region.** The function can call the `ListTables` API and include all tables whose name matches a given regular expression. This is the most dynamic configuration, as you won't need to redeploy this function every time you add a new DynamoDB table. This works best if you have a specified naming scheme for your tables, such as prefixing all tables with the stage. Then you can add a pattern of `"^(prod-)."` to match all tables that start with `prod-`. To use this method, put your regular expression in the `tableRegex` property. 30 | 31 | 2. **Specify multiple table names in an included file.** If you have a list of tables you want to back up, you can place their names in a local file and specify the name of the file in the `tableFile` property. The format must be valid JSON that is a list of strings. A `tables.json` file is included in this repo as an example. 32 | 33 | 3. **Specify a single table via environment variable.** If you only have one table to backup, you can specify its name via the `tableName` property. 34 | 35 | In addition to the table configuration, there is also the following configuration: 36 | 37 | - `backupRate` - **required** - The schedule on which you want to backup your table. You can use either `rate` syntax (`rate(1 hour)`) or `cron` syntax (`cron(0 12 * * ? *)`). See [here](https://serverless.com/framework/docs/providers/aws/events/schedule/) for more details on configuration. 38 | - `slackWebhook` - **optional** - An HTTPS endpoint for an [incoming webhook](https://api.slack.com/incoming-webhooks) to Slack. If provided, it will send success + error messages to a Slack channel when it runs. 39 | 40 | - `backupRemovalEnabled` - **optional** - Setting this value to **true** will enable cleanup of old backups. See the below option, `backupRetentionDays`, to specify the retention period. By default, backup removal is disabled. 41 | 42 | - `backupRetentionDays` - **optional** - Specify the number of days to retain old snapshots. For example, setting the value to **2** will remove all snapshots that are older then 2 days from today. 43 | 44 | # Notes 45 | 46 | - As of 12/11/2018, DynamoDB backups aren't working for all tables. It appears to be tables created after a certain time, though I don't know what that cutoff is. If your table is ineligible, you'll get a `ContinuousBackupsUnavailableException`. 47 | - The `botocore` package bundled with Lambda doesn't include the new features announced at reInvent. As a result, I added the `dynamodb` data directory from a more recent version of `botocore` and set the `AWS_DATA_PATH` environment variable to recognize it. 48 | 49 | # Potential improvements 50 | 51 | - **Better control on notifications?** We could implement email or SMS messages, as well as the ability to only notify on failures. 52 | 53 | -------------------------------------------------------------------------------- /dynamodb/2012-08-10/examples-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "examples": { 4 | "BatchGetItem": [ 5 | { 6 | "input": { 7 | "RequestItems": { 8 | "Music": { 9 | "Keys": [ 10 | { 11 | "Artist": { 12 | "S": "No One You Know" 13 | }, 14 | "SongTitle": { 15 | "S": "Call Me Today" 16 | } 17 | }, 18 | { 19 | "Artist": { 20 | "S": "Acme Band" 21 | }, 22 | "SongTitle": { 23 | "S": "Happy Day" 24 | } 25 | }, 26 | { 27 | "Artist": { 28 | "S": "No One You Know" 29 | }, 30 | "SongTitle": { 31 | "S": "Scared of My Shadow" 32 | } 33 | } 34 | ], 35 | "ProjectionExpression": "AlbumTitle" 36 | } 37 | } 38 | }, 39 | "output": { 40 | "Responses": { 41 | "Music": [ 42 | { 43 | "AlbumTitle": { 44 | "S": "Somewhat Famous" 45 | } 46 | }, 47 | { 48 | "AlbumTitle": { 49 | "S": "Blue Sky Blues" 50 | } 51 | }, 52 | { 53 | "AlbumTitle": { 54 | "S": "Louder Than Ever" 55 | } 56 | } 57 | ] 58 | } 59 | }, 60 | "comments": { 61 | "input": { 62 | }, 63 | "output": { 64 | } 65 | }, 66 | "description": "This example reads multiple items from the Music table using a batch of three GetItem requests. Only the AlbumTitle attribute is returned.", 67 | "id": "to-retrieve-multiple-items-from-a-table-1476118438992", 68 | "title": "To retrieve multiple items from a table" 69 | } 70 | ], 71 | "BatchWriteItem": [ 72 | { 73 | "input": { 74 | "RequestItems": { 75 | "Music": [ 76 | { 77 | "PutRequest": { 78 | "Item": { 79 | "AlbumTitle": { 80 | "S": "Somewhat Famous" 81 | }, 82 | "Artist": { 83 | "S": "No One You Know" 84 | }, 85 | "SongTitle": { 86 | "S": "Call Me Today" 87 | } 88 | } 89 | } 90 | }, 91 | { 92 | "PutRequest": { 93 | "Item": { 94 | "AlbumTitle": { 95 | "S": "Songs About Life" 96 | }, 97 | "Artist": { 98 | "S": "Acme Band" 99 | }, 100 | "SongTitle": { 101 | "S": "Happy Day" 102 | } 103 | } 104 | } 105 | }, 106 | { 107 | "PutRequest": { 108 | "Item": { 109 | "AlbumTitle": { 110 | "S": "Blue Sky Blues" 111 | }, 112 | "Artist": { 113 | "S": "No One You Know" 114 | }, 115 | "SongTitle": { 116 | "S": "Scared of My Shadow" 117 | } 118 | } 119 | } 120 | } 121 | ] 122 | } 123 | }, 124 | "output": { 125 | }, 126 | "comments": { 127 | "input": { 128 | }, 129 | "output": { 130 | } 131 | }, 132 | "description": "This example adds three new items to the Music table using a batch of three PutItem requests.", 133 | "id": "to-add-multiple-items-to-a-table-1476118519747", 134 | "title": "To add multiple items to a table" 135 | } 136 | ], 137 | "CreateTable": [ 138 | { 139 | "input": { 140 | "AttributeDefinitions": [ 141 | { 142 | "AttributeName": "Artist", 143 | "AttributeType": "S" 144 | }, 145 | { 146 | "AttributeName": "SongTitle", 147 | "AttributeType": "S" 148 | } 149 | ], 150 | "KeySchema": [ 151 | { 152 | "AttributeName": "Artist", 153 | "KeyType": "HASH" 154 | }, 155 | { 156 | "AttributeName": "SongTitle", 157 | "KeyType": "RANGE" 158 | } 159 | ], 160 | "ProvisionedThroughput": { 161 | "ReadCapacityUnits": 5, 162 | "WriteCapacityUnits": 5 163 | }, 164 | "TableName": "Music" 165 | }, 166 | "output": { 167 | "TableDescription": { 168 | "AttributeDefinitions": [ 169 | { 170 | "AttributeName": "Artist", 171 | "AttributeType": "S" 172 | }, 173 | { 174 | "AttributeName": "SongTitle", 175 | "AttributeType": "S" 176 | } 177 | ], 178 | "CreationDateTime": "1421866952.062", 179 | "ItemCount": 0, 180 | "KeySchema": [ 181 | { 182 | "AttributeName": "Artist", 183 | "KeyType": "HASH" 184 | }, 185 | { 186 | "AttributeName": "SongTitle", 187 | "KeyType": "RANGE" 188 | } 189 | ], 190 | "ProvisionedThroughput": { 191 | "ReadCapacityUnits": 5, 192 | "WriteCapacityUnits": 5 193 | }, 194 | "TableName": "Music", 195 | "TableSizeBytes": 0, 196 | "TableStatus": "CREATING" 197 | } 198 | }, 199 | "comments": { 200 | "input": { 201 | }, 202 | "output": { 203 | } 204 | }, 205 | "description": "This example creates a table named Music.", 206 | "id": "to-create-a-table-1476116291743", 207 | "title": "To create a table" 208 | } 209 | ], 210 | "DeleteItem": [ 211 | { 212 | "input": { 213 | "Key": { 214 | "Artist": { 215 | "S": "No One You Know" 216 | }, 217 | "SongTitle": { 218 | "S": "Scared of My Shadow" 219 | } 220 | }, 221 | "TableName": "Music" 222 | }, 223 | "output": { 224 | "ConsumedCapacity": { 225 | "CapacityUnits": 1, 226 | "TableName": "Music" 227 | } 228 | }, 229 | "comments": { 230 | "input": { 231 | }, 232 | "output": { 233 | } 234 | }, 235 | "description": "This example deletes an item from the Music table.", 236 | "id": "to-delete-an-item-1475884573758", 237 | "title": "To delete an item" 238 | } 239 | ], 240 | "DeleteTable": [ 241 | { 242 | "input": { 243 | "TableName": "Music" 244 | }, 245 | "output": { 246 | "TableDescription": { 247 | "ItemCount": 0, 248 | "ProvisionedThroughput": { 249 | "NumberOfDecreasesToday": 1, 250 | "ReadCapacityUnits": 5, 251 | "WriteCapacityUnits": 5 252 | }, 253 | "TableName": "Music", 254 | "TableSizeBytes": 0, 255 | "TableStatus": "DELETING" 256 | } 257 | }, 258 | "comments": { 259 | "input": { 260 | }, 261 | "output": { 262 | } 263 | }, 264 | "description": "This example deletes the Music table.", 265 | "id": "to-delete-a-table-1475884368755", 266 | "title": "To delete a table" 267 | } 268 | ], 269 | "DescribeLimits": [ 270 | { 271 | "input": { 272 | }, 273 | "output": { 274 | "AccountMaxReadCapacityUnits": 20000, 275 | "AccountMaxWriteCapacityUnits": 20000, 276 | "TableMaxReadCapacityUnits": 10000, 277 | "TableMaxWriteCapacityUnits": 10000 278 | }, 279 | "comments": { 280 | "input": { 281 | }, 282 | "output": { 283 | } 284 | }, 285 | "description": "The following example returns the maximum read and write capacity units per table, and for the AWS account, in the current AWS region.", 286 | "id": "to-determine-capacity-limits-per-table-and-account-in-the-current-aws-region-1475884162064", 287 | "title": "To determine capacity limits per table and account, in the current AWS region" 288 | } 289 | ], 290 | "DescribeTable": [ 291 | { 292 | "input": { 293 | "TableName": "Music" 294 | }, 295 | "output": { 296 | "Table": { 297 | "AttributeDefinitions": [ 298 | { 299 | "AttributeName": "Artist", 300 | "AttributeType": "S" 301 | }, 302 | { 303 | "AttributeName": "SongTitle", 304 | "AttributeType": "S" 305 | } 306 | ], 307 | "CreationDateTime": "1421866952.062", 308 | "ItemCount": 0, 309 | "KeySchema": [ 310 | { 311 | "AttributeName": "Artist", 312 | "KeyType": "HASH" 313 | }, 314 | { 315 | "AttributeName": "SongTitle", 316 | "KeyType": "RANGE" 317 | } 318 | ], 319 | "ProvisionedThroughput": { 320 | "NumberOfDecreasesToday": 1, 321 | "ReadCapacityUnits": 5, 322 | "WriteCapacityUnits": 5 323 | }, 324 | "TableName": "Music", 325 | "TableSizeBytes": 0, 326 | "TableStatus": "ACTIVE" 327 | } 328 | }, 329 | "comments": { 330 | "input": { 331 | }, 332 | "output": { 333 | } 334 | }, 335 | "description": "This example describes the Music table.", 336 | "id": "to-describe-a-table-1475884440502", 337 | "title": "To describe a table" 338 | } 339 | ], 340 | "GetItem": [ 341 | { 342 | "input": { 343 | "Key": { 344 | "Artist": { 345 | "S": "Acme Band" 346 | }, 347 | "SongTitle": { 348 | "S": "Happy Day" 349 | } 350 | }, 351 | "TableName": "Music" 352 | }, 353 | "output": { 354 | "Item": { 355 | "AlbumTitle": { 356 | "S": "Songs About Life" 357 | }, 358 | "Artist": { 359 | "S": "Acme Band" 360 | }, 361 | "SongTitle": { 362 | "S": "Happy Day" 363 | } 364 | } 365 | }, 366 | "comments": { 367 | "input": { 368 | }, 369 | "output": { 370 | } 371 | }, 372 | "description": "This example retrieves an item from the Music table. The table has a partition key and a sort key (Artist and SongTitle), so you must specify both of these attributes.", 373 | "id": "to-read-an-item-from-a-table-1475884258350", 374 | "title": "To read an item from a table" 375 | } 376 | ], 377 | "ListTables": [ 378 | { 379 | "input": { 380 | }, 381 | "output": { 382 | "TableNames": [ 383 | "Forum", 384 | "ProductCatalog", 385 | "Reply", 386 | "Thread" 387 | ] 388 | }, 389 | "comments": { 390 | "input": { 391 | }, 392 | "output": { 393 | } 394 | }, 395 | "description": "This example lists all of the tables associated with the current AWS account and endpoint.", 396 | "id": "to-list-tables-1475884741238", 397 | "title": "To list tables" 398 | } 399 | ], 400 | "PutItem": [ 401 | { 402 | "input": { 403 | "Item": { 404 | "AlbumTitle": { 405 | "S": "Somewhat Famous" 406 | }, 407 | "Artist": { 408 | "S": "No One You Know" 409 | }, 410 | "SongTitle": { 411 | "S": "Call Me Today" 412 | } 413 | }, 414 | "ReturnConsumedCapacity": "TOTAL", 415 | "TableName": "Music" 416 | }, 417 | "output": { 418 | "ConsumedCapacity": { 419 | "CapacityUnits": 1, 420 | "TableName": "Music" 421 | } 422 | }, 423 | "comments": { 424 | "input": { 425 | }, 426 | "output": { 427 | } 428 | }, 429 | "description": "This example adds a new item to the Music table.", 430 | "id": "to-add-an-item-to-a-table-1476116191110", 431 | "title": "To add an item to a table" 432 | } 433 | ], 434 | "Query": [ 435 | { 436 | "input": { 437 | "ExpressionAttributeValues": { 438 | ":v1": { 439 | "S": "No One You Know" 440 | } 441 | }, 442 | "KeyConditionExpression": "Artist = :v1", 443 | "ProjectionExpression": "SongTitle", 444 | "TableName": "Music" 445 | }, 446 | "output": { 447 | "ConsumedCapacity": { 448 | }, 449 | "Count": 2, 450 | "Items": [ 451 | { 452 | "SongTitle": { 453 | "S": "Call Me Today" 454 | } 455 | } 456 | ], 457 | "ScannedCount": 2 458 | }, 459 | "comments": { 460 | "input": { 461 | }, 462 | "output": { 463 | } 464 | }, 465 | "description": "This example queries items in the Music table. The table has a partition key and sort key (Artist and SongTitle), but this query only specifies the partition key value. It returns song titles by the artist named \"No One You Know\".", 466 | "id": "to-query-an-item-1475883874631", 467 | "title": "To query an item" 468 | } 469 | ], 470 | "Scan": [ 471 | { 472 | "input": { 473 | "ExpressionAttributeNames": { 474 | "AT": "AlbumTitle", 475 | "ST": "SongTitle" 476 | }, 477 | "ExpressionAttributeValues": { 478 | ":a": { 479 | "S": "No One You Know" 480 | } 481 | }, 482 | "FilterExpression": "Artist = :a", 483 | "ProjectionExpression": "#ST, #AT", 484 | "TableName": "Music" 485 | }, 486 | "output": { 487 | "ConsumedCapacity": { 488 | }, 489 | "Count": 2, 490 | "Items": [ 491 | { 492 | "AlbumTitle": { 493 | "S": "Somewhat Famous" 494 | }, 495 | "SongTitle": { 496 | "S": "Call Me Today" 497 | } 498 | }, 499 | { 500 | "AlbumTitle": { 501 | "S": "Blue Sky Blues" 502 | }, 503 | "SongTitle": { 504 | "S": "Scared of My Shadow" 505 | } 506 | } 507 | ], 508 | "ScannedCount": 3 509 | }, 510 | "comments": { 511 | "input": { 512 | }, 513 | "output": { 514 | } 515 | }, 516 | "description": "This example scans the entire Music table, and then narrows the results to songs by the artist \"No One You Know\". For each item, only the album title and song title are returned.", 517 | "id": "to-scan-a-table-1475883652470", 518 | "title": "To scan a table" 519 | } 520 | ], 521 | "UpdateItem": [ 522 | { 523 | "input": { 524 | "ExpressionAttributeNames": { 525 | "#AT": "AlbumTitle", 526 | "#Y": "Year" 527 | }, 528 | "ExpressionAttributeValues": { 529 | ":t": { 530 | "S": "Louder Than Ever" 531 | }, 532 | ":y": { 533 | "N": "2015" 534 | } 535 | }, 536 | "Key": { 537 | "Artist": { 538 | "S": "Acme Band" 539 | }, 540 | "SongTitle": { 541 | "S": "Happy Day" 542 | } 543 | }, 544 | "ReturnValues": "ALL_NEW", 545 | "TableName": "Music", 546 | "UpdateExpression": "SET #Y = :y, #AT = :t" 547 | }, 548 | "output": { 549 | "Attributes": { 550 | "AlbumTitle": { 551 | "S": "Songs About Life" 552 | }, 553 | "Artist": { 554 | "S": "Acme Band" 555 | }, 556 | "SongTitle": { 557 | "S": "Happy Day" 558 | } 559 | } 560 | }, 561 | "comments": { 562 | "input": { 563 | }, 564 | "output": { 565 | } 566 | }, 567 | "description": "This example updates an item in the Music table. It adds a new attribute (Year) and modifies the AlbumTitle attribute. All of the attributes in the item, as they appear after the update, are returned in the response.", 568 | "id": "to-update-an-item-in-a-table-1476118250055", 569 | "title": "To update an item in a table" 570 | } 571 | ], 572 | "UpdateTable": [ 573 | { 574 | "input": { 575 | "ProvisionedThroughput": { 576 | "ReadCapacityUnits": 10, 577 | "WriteCapacityUnits": 10 578 | }, 579 | "TableName": "MusicCollection" 580 | }, 581 | "output": { 582 | "TableDescription": { 583 | "AttributeDefinitions": [ 584 | { 585 | "AttributeName": "Artist", 586 | "AttributeType": "S" 587 | }, 588 | { 589 | "AttributeName": "SongTitle", 590 | "AttributeType": "S" 591 | } 592 | ], 593 | "CreationDateTime": "1421866952.062", 594 | "ItemCount": 0, 595 | "KeySchema": [ 596 | { 597 | "AttributeName": "Artist", 598 | "KeyType": "HASH" 599 | }, 600 | { 601 | "AttributeName": "SongTitle", 602 | "KeyType": "RANGE" 603 | } 604 | ], 605 | "ProvisionedThroughput": { 606 | "LastIncreaseDateTime": "1421874759.194", 607 | "NumberOfDecreasesToday": 1, 608 | "ReadCapacityUnits": 1, 609 | "WriteCapacityUnits": 1 610 | }, 611 | "TableName": "MusicCollection", 612 | "TableSizeBytes": 0, 613 | "TableStatus": "UPDATING" 614 | } 615 | }, 616 | "comments": { 617 | "input": { 618 | }, 619 | "output": { 620 | } 621 | }, 622 | "description": "This example increases the provisioned read and write capacity on the Music table.", 623 | "id": "to-modify-a-tables-provisioned-throughput-1476118076147", 624 | "title": "To modify a table's provisioned throughput" 625 | } 626 | ] 627 | } 628 | } 629 | -------------------------------------------------------------------------------- /dynamodb/2012-08-10/paginators-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "pagination": { 3 | "ListTables": { 4 | "input_token": "ExclusiveStartTableName", 5 | "output_token": "LastEvaluatedTableName", 6 | "limit_key": "Limit", 7 | "result_key": "TableNames" 8 | }, 9 | "Query": { 10 | "input_token": "ExclusiveStartKey", 11 | "output_token": "LastEvaluatedKey", 12 | "limit_key": "Limit", 13 | "result_key": [ 14 | "Items", 15 | "Count", 16 | "ScannedCount" 17 | ], 18 | "non_aggregate_keys": [ 19 | "ConsumedCapacity" 20 | ] 21 | }, 22 | "Scan": { 23 | "input_token": "ExclusiveStartKey", 24 | "output_token": "LastEvaluatedKey", 25 | "limit_key": "Limit", 26 | "result_key": [ 27 | "Items", 28 | "Count", 29 | "ScannedCount" 30 | ], 31 | "non_aggregate_keys": [ 32 | "ConsumedCapacity" 33 | ] 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /dynamodb/2012-08-10/waiters-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 2, 3 | "waiters": { 4 | "TableExists": { 5 | "delay": 20, 6 | "operation": "DescribeTable", 7 | "maxAttempts": 25, 8 | "acceptors": [ 9 | { 10 | "expected": "ACTIVE", 11 | "matcher": "path", 12 | "state": "success", 13 | "argument": "Table.TableStatus" 14 | }, 15 | { 16 | "expected": "ResourceNotFoundException", 17 | "matcher": "error", 18 | "state": "retry" 19 | } 20 | ] 21 | }, 22 | "TableNotExists": { 23 | "delay": 20, 24 | "operation": "DescribeTable", 25 | "maxAttempts": 25, 26 | "acceptors": [ 27 | { 28 | "expected": "ResourceNotFoundException", 29 | "matcher": "error", 30 | "state": "success" 31 | } 32 | ] 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /handler.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | import json 3 | import os 4 | import re 5 | 6 | # To get updated botocore data files 7 | os.environ['AWS_DATA_PATH'] = '.' 8 | 9 | import boto3 10 | import botocore 11 | from botocore.exceptions import ClientError 12 | from botocore.vendored import requests 13 | 14 | CLIENT = boto3.client('dynamodb') 15 | SLACK_WEBHOOK = os.environ.get('SLACK_WEBHOOK') 16 | REGION = os.environ.get('AWS_DEFAULT_REGION') 17 | CONSOLE_ENDPOINT = 'https://console.aws.amazon.com/dynamodb/home?region={region}#backups:'.format(region=REGION) 18 | 19 | 20 | def main(event, context): 21 | tables = get_tables_to_backup() 22 | results = { 23 | "success": [], 24 | "failure": [] 25 | } 26 | 27 | for table in tables: 28 | try: 29 | create_backup(table) 30 | results['success'].append(table) 31 | except Exception as e: 32 | print("Error creating backup for table {table}.\n. Error: {err}".format(table=table, err=str(e))) 33 | results['failure'].append(table) 34 | 35 | if os.environ.get('BACKUP_REMOVAL_ENABLED') == 'true': 36 | try: 37 | remove_stale_backups(tables) 38 | except Exception as e: 39 | print("Error removing stale backups. Error: {err}".format(err=str(e))) 40 | 41 | message = format_message(results) 42 | send_to_slack(message) 43 | 44 | 45 | def create_backup(table): 46 | timestamp = datetime.now().strftime('%Y%m%d%H%M%S') 47 | backup_name = table + "_" + timestamp 48 | CLIENT.create_backup( 49 | TableName=table, 50 | BackupName=backup_name 51 | ) 52 | 53 | def remove_stale_backups(tables): 54 | 55 | upper_bound = datetime.now() - timedelta(days=int(os.environ.get('BACKUP_RETENTION_DAYS'))) 56 | 57 | print("Removing backups before the following date: {date}".format(date=upper_bound)) 58 | 59 | if CLIENT.can_paginate('list_backups'): 60 | paginator = CLIENT.get_paginator('list_backups') 61 | for page in paginator.paginate(TimeRangeUpperBound=upper_bound): 62 | for table in page['BackupSummaries']: 63 | if table['TableName'] in tables: 64 | CLIENT.delete_backup(BackupArn=table['BackupArn']) 65 | else: 66 | backup = CLIENT.list_backups(TimeRangeUpperBound=upper_bound) 67 | for table in backup['BackupSummaries']: 68 | if table['TableName'] in tables: 69 | CLIENT.delete_backup(BackupArn=table['BackupArn']) 70 | 71 | 72 | def format_message(results): 73 | msg = "" 74 | 75 | if not results['success'] and not results['failure']: 76 | return "Tried running DynamoDB backup, but no tables were specified.\nPlease check your configuration." 77 | 78 | msg += "Tried to backup {total} DynamoDB tables. {successes} succeeded, and {failures} failed. See all backups <{url}|here>.".format( 79 | total=(len(results['success']) + len(results['failure'])), 80 | successes=len(results['success']), 81 | failures=len(results['failure']), 82 | url=CONSOLE_ENDPOINT.format(region=REGION) 83 | ) 84 | 85 | if results['success']: 86 | msg += "\nThe following tables were successful:\n - " 87 | msg += "\n - ".join(results['success']) 88 | 89 | if results['failure']: 90 | msg += "\nThe following tables failed:\n - " 91 | msg += "\n - ".join(results['failure']) 92 | 93 | return msg 94 | 95 | 96 | def send_to_slack(message): 97 | if not SLACK_WEBHOOK: 98 | print('No SLACK_WEBHOOK provided. Not sending a message...') 99 | return 100 | data = {"text": message} 101 | resp = requests.post(SLACK_WEBHOOK, json=data) 102 | 103 | resp.raise_for_status() 104 | 105 | 106 | def get_tables_to_backup(): 107 | """Determines which tables to backup. The determination is made based on 108 | the config options. Return value is a list. 109 | 110 | Order is as follows: 111 | 112 | 1. If the TABLE_REGEX environment variable is set, call the `ListTables` API 113 | for DynamoDB and return tables that match the given TABLE_REGEX; 114 | 115 | 2. If the TABLE_FILE environment variable is set, load the TABLE_FILE and 116 | return the tables list. 117 | 118 | 3. If the TABLE_NAME environment variable is set, return the TABLE_NAME. 119 | 120 | It will not combine multiple options. It will return the value(s) from the first 121 | option with the environment variable present. 122 | """ 123 | if os.environ.get('TABLE_REGEX'): 124 | return get_tables_regex(os.environ.get('TABLE_REGEX')) 125 | elif os.environ.get('TABLE_FILE'): 126 | return get_tables_from_file(os.environ.get('TABLE_FILE')) 127 | elif os.environ.get('TABLE_NAME'): 128 | return [os.environ.get('TABLE_NAME')] 129 | 130 | print("No tables configured. Please use TABLE_REGEX, TABLE_FILE, OR TABLE_NAME environment variables.") 131 | 132 | return [] 133 | 134 | 135 | def get_tables_regex(pattern): 136 | print("Using regex pattern {} to find tables.".format(pattern)) 137 | tables = [] 138 | paginator = CLIENT.get_paginator('list_tables') 139 | for page in paginator.paginate(): 140 | for table in page['TableNames']: 141 | if re.match(pattern, table): 142 | tables.append(table) 143 | 144 | return tables 145 | 146 | 147 | def get_tables_from_file(filename): 148 | print("Using local file {} to find tables.".format(filename)) 149 | with open(filename, 'r') as f: 150 | tables = json.load(f) 151 | 152 | return tables 153 | 154 | 155 | if __name__ == "__main__": 156 | main('', '') 157 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: serverless-dynamodb-backups 2 | 3 | custom: 4 | tableRegex: "" # Set if you want to use a Regex pattern against all tables in region. 5 | tableFile: "" # Set if you want to read table names from a file. 6 | tableName: "" # Set if you want to backup a single table 7 | backupRate: rate(10 minutes) 8 | backupRetentionDays: 1 9 | backupRemovalEnabled: true 10 | slackWebhook: "" 11 | 12 | provider: 13 | name: aws 14 | runtime: python3.13 15 | timeout: 30 16 | memorySize: 128 17 | iamRoleStatements: 18 | - Effect: "Allow" 19 | Action: 20 | - "dynamodb:CreateBackup" 21 | Resource: 22 | Fn::Join: 23 | - ":" 24 | - - "arn:aws:dynamodb" 25 | - Ref: 'AWS::Region' 26 | - Ref: 'AWS::AccountId' 27 | - "table/*" 28 | - Effect: "Allow" 29 | Action: 30 | - "dynamodb:ListTables" 31 | Resource: "*" 32 | environment: 33 | BACKUP_REMOVAL_ENABLED: ${self:custom.backupRemovalEnabled} 34 | BACKUP_RETENTION_DAYS: ${self:custom.backupRetentionDays} 35 | TABLE_REGEX: ${self:custom.tableRegex} 36 | TABLE_FILE: ${self:custom.tableFile} 37 | TABLE_NAME: ${self:custom.tableName} 38 | SLACK_WEBHOOK: ${self:custom.slackWebhook} 39 | 40 | functions: 41 | createBackup: 42 | handler: handler.main 43 | events: 44 | - schedule: ${self:custom.backupRate} 45 | -------------------------------------------------------------------------------- /tables.json: -------------------------------------------------------------------------------- 1 | ["test-table-1", "test-table-2"] 2 | --------------------------------------------------------------------------------