├── .dockerignore ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── LICENSE.txt ├── Makefile ├── README.md ├── pyproject.toml ├── requirements.txt ├── s3tk ├── __init__.py └── checks.py └── tests ├── __init__.py └── test_s3tk.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | *.egg-info/ 4 | venv/ 5 | .cache/ 6 | *.pyc 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.5.0 (unreleased) 2 | 3 | - Dropped support for Python < 3.10 4 | 5 | ## 0.4.0 (2023-07-23) 6 | 7 | - Fixed warning with clint 8 | - Dropped support for Python < 3.8 9 | 10 | ## 0.3.1 (2020-08-17) 11 | 12 | - Added experimental `--object-level-logging` option to `scan` command 13 | 14 | ## 0.3.0 (2020-01-05) 15 | 16 | - Added check for public access allowed 17 | - Added `block-public-access` command 18 | 19 | ## 0.2.1 (2018-10-16) 20 | 21 | - Fixed error with joblib 0.12 22 | 23 | ## 0.2.0 (2018-04-15) 24 | 25 | - Scan default encryption by default 26 | - More greppable output 27 | - Performance optimization for single objects 28 | 29 | ## 0.1.8 (2018-02-16) 30 | 31 | - Added `delete-unencrypted-versions` command 32 | - Added `--acl` option to `reset-object-acl` command 33 | - Added check for existing object ACL before reset 34 | - Fixed issue with unicode keys in Python 2 35 | 36 | ## 0.1.7 (2017-11-13) 37 | 38 | - Added `enable-default-encryption` command 39 | - Added `--dry-run` and `public-uploads` options to `set-policy` command 40 | - Added summary to `scan-object-acl` and `encrypt` commands 41 | - Added `--default-encryption` and `--sns-topic` to `scan` command 42 | 43 | ## 0.1.6 (2017-10-01) 44 | 45 | - Added `scan-dns` command 46 | - Added `set-policy` command 47 | - Added `delete-policy` command 48 | - Added `--named` option to `list-policy` command 49 | - 2x performance for object commands 50 | 51 | ## 0.1.5 (2017-09-18) 52 | 53 | - Fixed error with `enable-logging` 54 | 55 | ## 0.1.4 (2017-09-17) 56 | 57 | - Added `scan-object-acl` command 58 | - Added `--only` and `--except` options 59 | - Added `--log-bucket` and `--log-prefix` options to `scan` command 60 | - Added `--log-prefix` option to `enable-logging` command 61 | 62 | ## 0.1.3 (2017-09-14) 63 | 64 | - Fixed policy check 65 | - Added `list-policy` command 66 | - Added `reset-object-acl` command 67 | - Added support for wildcards 68 | - Added support for customer-provided encryption key 69 | - Added `--version` option 70 | - Parallelize encryption 71 | 72 | ## 0.1.2 (2017-09-13) 73 | 74 | - Fixed issue with packaging 75 | 76 | ## 0.1.1 (2017-09-12) 77 | 78 | - Fixed json error 79 | - Better message for missing credentials 80 | 81 | ## 0.1.0 (2017-09-12) 82 | 83 | - First release 84 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-alpine 2 | 3 | LABEL org.opencontainers.image.authors="Andrew Kane " 4 | 5 | ENV INSTALL_PATH=/app 6 | 7 | RUN mkdir -p $INSTALL_PATH 8 | 9 | WORKDIR $INSTALL_PATH 10 | 11 | COPY . ./ 12 | 13 | RUN pip install --no-cache-dir -r requirements.txt 14 | 15 | RUN pip install . 16 | 17 | RUN pip install awscli 18 | 19 | CMD ["s3tk"] 20 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-2025 Andrew Kane 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: lint build publish clean docker 2 | 3 | lint: 4 | pycodestyle . --ignore=E501 5 | 6 | build: 7 | python3 -m build 8 | 9 | publish: clean build 10 | twine upload dist/* 11 | 12 | clean: 13 | rm -rf .pytest_cache dist s3tk.egg-info 14 | 15 | docker: clean 16 | docker build --pull --no-cache --platform linux/amd64 -t ankane/s3tk:latest . 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # s3tk 2 | 3 | A security toolkit for Amazon S3 4 | 5 | ![Screenshot](https://gist.githubusercontent.com/ankane/13a9230353c78c0d5c35fd9319a23d98/raw/434b9c54bff9d41c398aa3b57f0d0494217ef7fa/console2.gif) 6 | 7 | :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource) 8 | 9 | ## Installation 10 | 11 | Run: 12 | 13 | ```sh 14 | pip install s3tk 15 | ``` 16 | 17 | You can use the [AWS CLI](https://github.com/aws/aws-cli) or [AWS Vault](https://github.com/99designs/aws-vault) to set up your AWS credentials: 18 | 19 | ```sh 20 | pip install awscli 21 | aws configure 22 | ``` 23 | 24 | See [IAM policies](#iam-policies) needed for each command. 25 | 26 | ## Commands 27 | 28 | ### Scan 29 | 30 | Scan your buckets for: 31 | 32 | - ACL open to public 33 | - policy open to public 34 | - public access blocked 35 | - logging enabled 36 | - versioning enabled 37 | - default encryption enabled 38 | 39 | ```sh 40 | s3tk scan 41 | ``` 42 | 43 | Only run on specific buckets 44 | 45 | ```sh 46 | s3tk scan my-bucket my-bucket-2 47 | ``` 48 | 49 | Also works with wildcards 50 | 51 | ```sh 52 | s3tk scan "my-bucket*" 53 | ``` 54 | 55 | Confirm correct log bucket(s) and prefix 56 | 57 | ``` 58 | s3tk scan --log-bucket my-s3-logs --log-bucket other-region-logs --log-prefix "{bucket}/" 59 | ``` 60 | 61 | Check CloudTrail object-level logging [experimental] 62 | 63 | ```sh 64 | s3tk scan --object-level-logging 65 | ``` 66 | 67 | Skip logging, versioning, or default encryption 68 | 69 | ```sh 70 | s3tk scan --skip-logging --skip-versioning --skip-default-encryption 71 | ``` 72 | 73 | Get email notifications of failures (via SNS) 74 | 75 | ```sh 76 | s3tk scan --sns-topic arn:aws:sns:... 77 | ``` 78 | 79 | ### List Policy 80 | 81 | List bucket policies 82 | 83 | ```sh 84 | s3tk list-policy 85 | ``` 86 | 87 | Only run on specific buckets 88 | 89 | ```sh 90 | s3tk list-policy my-bucket my-bucket-2 91 | ``` 92 | 93 | Show named statements 94 | 95 | ```sh 96 | s3tk list-policy --named 97 | ``` 98 | 99 | ### Set Policy 100 | 101 | **Note:** This replaces the previous policy 102 | 103 | Only private uploads 104 | 105 | ```sh 106 | s3tk set-policy my-bucket --no-object-acl 107 | ``` 108 | 109 | ### Delete Policy 110 | 111 | Delete policy 112 | 113 | ```sh 114 | s3tk delete-policy my-bucket 115 | ``` 116 | 117 | ### Block Public Access 118 | 119 | Block public access on specific buckets 120 | 121 | ```sh 122 | s3tk block-public-access my-bucket my-bucket-2 123 | ``` 124 | 125 | Use the `--dry-run` flag to test 126 | 127 | ### Enable Logging 128 | 129 | Enable logging on all buckets 130 | 131 | ```sh 132 | s3tk enable-logging --log-bucket my-s3-logs 133 | ``` 134 | 135 | Only on specific buckets 136 | 137 | ```sh 138 | s3tk enable-logging my-bucket my-bucket-2 --log-bucket my-s3-logs 139 | ``` 140 | 141 | Set log prefix (`{bucket}/` by default) 142 | 143 | ```sh 144 | s3tk enable-logging --log-bucket my-s3-logs --log-prefix "logs/{bucket}/" 145 | ``` 146 | 147 | Use the `--dry-run` flag to test 148 | 149 | A few notes about logging: 150 | 151 | - buckets with logging already enabled are not updated at all 152 | - the log bucket must in the same region as the source bucket - run this command multiple times for different regions 153 | - it can take over an hour for logs to show up 154 | 155 | ### Enable Versioning 156 | 157 | Enable versioning on all buckets 158 | 159 | ```sh 160 | s3tk enable-versioning 161 | ``` 162 | 163 | Only on specific buckets 164 | 165 | ```sh 166 | s3tk enable-versioning my-bucket my-bucket-2 167 | ``` 168 | 169 | Use the `--dry-run` flag to test 170 | 171 | ### Enable Default Encryption 172 | 173 | Enable [default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html) on all buckets 174 | 175 | ```sh 176 | s3tk enable-default-encryption 177 | ``` 178 | 179 | Only on specific buckets 180 | 181 | ```sh 182 | s3tk enable-default-encryption my-bucket my-bucket-2 183 | ``` 184 | 185 | This does not encrypt existing objects - use the `encrypt` command for this 186 | 187 | Use the `--dry-run` flag to test 188 | 189 | ### Scan Object ACL 190 | 191 | Scan ACL on all objects in a bucket 192 | 193 | ```sh 194 | s3tk scan-object-acl my-bucket 195 | ``` 196 | 197 | Only certain objects 198 | 199 | ```sh 200 | s3tk scan-object-acl my-bucket --only "*.pdf" 201 | ``` 202 | 203 | Except certain objects 204 | 205 | ```sh 206 | s3tk scan-object-acl my-bucket --except "*.jpg" 207 | ``` 208 | 209 | ### Reset Object ACL 210 | 211 | Reset ACL on all objects in a bucket 212 | 213 | ```sh 214 | s3tk reset-object-acl my-bucket 215 | ``` 216 | 217 | This makes all objects private. See [bucket policies](#bucket-policies) for how to enforce going forward. 218 | 219 | Use the `--dry-run` flag to test 220 | 221 | Specify certain objects the same way as [scan-object-acl](#scan-object-acl) 222 | 223 | ### Encrypt 224 | 225 | Encrypt all objects in a bucket with [server-side encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/serv-side-encryption.html) 226 | 227 | ```sh 228 | s3tk encrypt my-bucket 229 | ``` 230 | 231 | Use [S3-managed keys](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) by default. For [KMS-managed keys](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html), use: 232 | 233 | ```sh 234 | s3tk encrypt my-bucket --kms-key-id arn:aws:kms:... 235 | ``` 236 | 237 | For [customer-provided keys](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html), use: 238 | 239 | ```sh 240 | s3tk encrypt my-bucket --customer-key secret-key 241 | ``` 242 | 243 | Use the `--dry-run` flag to test 244 | 245 | Specify certain objects the same way as [scan-object-acl](#scan-object-acl) 246 | 247 | **Note:** Objects will lose any custom ACL 248 | 249 | ### Delete Unencrypted Versions 250 | 251 | Delete all unencrypted versions of objects in a bucket 252 | 253 | ```sh 254 | s3tk delete-unencrypted-versions my-bucket 255 | ``` 256 | 257 | For safety, this will not delete any current versions of objects 258 | 259 | Use the `--dry-run` flag to test 260 | 261 | Specify certain objects the same way as [scan-object-acl](#scan-object-acl) 262 | 263 | ### Scan DNS 264 | 265 | Scan Route 53 for buckets to make sure you own them 266 | 267 | ```sh 268 | s3tk scan-dns 269 | ``` 270 | 271 | Otherwise, you may be susceptible to [subdomain takeover](https://hackerone.com/reports/207576) 272 | 273 | ## Credentials 274 | 275 | Credentials can be specified in `~/.aws/credentials` or with environment variables. See [this guide](https://boto3.readthedocs.io/en/latest/guide/configuration.html) for an explanation of environment variables. 276 | 277 | You can specify a profile to use with: 278 | 279 | ```sh 280 | AWS_PROFILE=your-profile s3tk 281 | ``` 282 | 283 | ## IAM Policies 284 | 285 | Here are the permissions needed for each command. Only include statements you need. 286 | 287 | ```json 288 | { 289 | "Version": "2012-10-17", 290 | "Statement": [ 291 | { 292 | "Sid": "Scan", 293 | "Effect": "Allow", 294 | "Action": [ 295 | "s3:ListAllMyBuckets", 296 | "s3:GetBucketAcl", 297 | "s3:GetBucketPolicy", 298 | "s3:GetBucketPublicAccessBlock", 299 | "s3:GetBucketLogging", 300 | "s3:GetBucketVersioning", 301 | "s3:GetEncryptionConfiguration" 302 | ], 303 | "Resource": "*" 304 | }, 305 | { 306 | "Sid": "ScanObjectLevelLogging", 307 | "Effect": "Allow", 308 | "Action": [ 309 | "cloudtrail:ListTrails", 310 | "cloudtrail:GetTrail", 311 | "cloudtrail:GetEventSelectors", 312 | "s3:GetBucketLocation" 313 | ], 314 | "Resource": "*" 315 | }, 316 | { 317 | "Sid": "ScanDNS", 318 | "Effect": "Allow", 319 | "Action": [ 320 | "s3:ListAllMyBuckets", 321 | "route53:ListHostedZones", 322 | "route53:ListResourceRecordSets" 323 | ], 324 | "Resource": "*" 325 | }, 326 | { 327 | "Sid": "ListPolicy", 328 | "Effect": "Allow", 329 | "Action": [ 330 | "s3:ListAllMyBuckets", 331 | "s3:GetBucketPolicy" 332 | ], 333 | "Resource": "*" 334 | }, 335 | { 336 | "Sid": "SetPolicy", 337 | "Effect": "Allow", 338 | "Action": [ 339 | "s3:PutBucketPolicy" 340 | ], 341 | "Resource": "*" 342 | }, 343 | { 344 | "Sid": "DeletePolicy", 345 | "Effect": "Allow", 346 | "Action": [ 347 | "s3:DeleteBucketPolicy" 348 | ], 349 | "Resource": "*" 350 | }, 351 | { 352 | "Sid": "BlockPublicAccess", 353 | "Effect": "Allow", 354 | "Action": [ 355 | "s3:ListAllMyBuckets", 356 | "s3:PutBucketPublicAccessBlock" 357 | ], 358 | "Resource": "*" 359 | }, 360 | { 361 | "Sid": "EnableLogging", 362 | "Effect": "Allow", 363 | "Action": [ 364 | "s3:ListAllMyBuckets", 365 | "s3:PutBucketLogging" 366 | ], 367 | "Resource": "*" 368 | }, 369 | { 370 | "Sid": "EnableVersioning", 371 | "Effect": "Allow", 372 | "Action": [ 373 | "s3:ListAllMyBuckets", 374 | "s3:PutBucketVersioning" 375 | ], 376 | "Resource": "*" 377 | }, 378 | { 379 | "Sid": "EnableDefaultEncryption", 380 | "Effect": "Allow", 381 | "Action": [ 382 | "s3:ListAllMyBuckets", 383 | "s3:PutEncryptionConfiguration" 384 | ], 385 | "Resource": "*" 386 | }, 387 | { 388 | "Sid": "ResetObjectAcl", 389 | "Effect": "Allow", 390 | "Action": [ 391 | "s3:ListBucket", 392 | "s3:GetObjectAcl", 393 | "s3:PutObjectAcl" 394 | ], 395 | "Resource": [ 396 | "arn:aws:s3:::my-bucket", 397 | "arn:aws:s3:::my-bucket/*" 398 | ] 399 | }, 400 | { 401 | "Sid": "Encrypt", 402 | "Effect": "Allow", 403 | "Action": [ 404 | "s3:ListBucket", 405 | "s3:GetObject", 406 | "s3:PutObject" 407 | ], 408 | "Resource": [ 409 | "arn:aws:s3:::my-bucket", 410 | "arn:aws:s3:::my-bucket/*" 411 | ] 412 | }, 413 | { 414 | "Sid": "DeleteUnencryptedVersions", 415 | "Effect": "Allow", 416 | "Action": [ 417 | "s3:ListBucketVersions", 418 | "s3:GetObjectVersion", 419 | "s3:DeleteObjectVersion" 420 | ], 421 | "Resource": [ 422 | "arn:aws:s3:::my-bucket", 423 | "arn:aws:s3:::my-bucket/*" 424 | ] 425 | } 426 | ] 427 | } 428 | ``` 429 | 430 | ## Access Logs 431 | 432 | [Amazon Athena](https://aws.amazon.com/athena/) is great for querying S3 logs. Create a table (thanks to [this post](http://aws.mannem.me/?p=1462) for the table structure) with: 433 | 434 | ```sql 435 | CREATE EXTERNAL TABLE my_bucket ( 436 | bucket_owner string, 437 | bucket string, 438 | time string, 439 | remote_ip string, 440 | requester string, 441 | request_id string, 442 | operation string, 443 | key string, 444 | request_verb string, 445 | request_url string, 446 | request_proto string, 447 | status_code string, 448 | error_code string, 449 | bytes_sent string, 450 | object_size string, 451 | total_time string, 452 | turn_around_time string, 453 | referrer string, 454 | user_agent string, 455 | version_id string 456 | ) 457 | ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' 458 | WITH SERDEPROPERTIES ( 459 | 'serialization.format' = '1', 460 | 'input.regex' = '([^ ]*) ([^ ]*) \\[(.*?)\\] ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) \\\"([^ ]*) ([^ ]*) (- |[^ ]*)\\\" (-|[0-9]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) (\"[^\"]*\\") ([^ ]*)$' 461 | ) LOCATION 's3://my-s3-logs/my-bucket/'; 462 | ``` 463 | 464 | Change the last line to point to your log bucket (and prefix) and query away 465 | 466 | ```sql 467 | SELECT 468 | date_parse(time, '%d/%b/%Y:%H:%i:%S +0000') AS time, 469 | request_url, 470 | remote_ip, 471 | user_agent 472 | FROM 473 | my_bucket 474 | WHERE 475 | requester = '-' 476 | AND status_code LIKE '2%' 477 | AND request_url LIKE '/some-keys%' 478 | ORDER BY 1 479 | ``` 480 | 481 | ## CloudTrail Logs 482 | 483 | Amazon Athena is also great for querying CloudTrail logs. Create a table (thanks to [this post](https://www.1strategy.com/blog/2017/07/25/auditing-aws-activity-with-cloudtrail-and-athena/) for the table structure) with: 484 | 485 | ```sql 486 | CREATE EXTERNAL TABLE cloudtrail_logs ( 487 | eventversion STRING, 488 | userIdentity STRUCT< 489 | type:STRING, 490 | principalid:STRING, 491 | arn:STRING, 492 | accountid:STRING, 493 | invokedby:STRING, 494 | accesskeyid:STRING, 495 | userName:String, 496 | sessioncontext:STRUCT< 497 | attributes:STRUCT< 498 | mfaauthenticated:STRING, 499 | creationdate:STRING>, 500 | sessionIssuer:STRUCT< 501 | type:STRING, 502 | principalId:STRING, 503 | arn:STRING, 504 | accountId:STRING, 505 | userName:STRING>>>, 506 | eventTime STRING, 507 | eventSource STRING, 508 | eventName STRING, 509 | awsRegion STRING, 510 | sourceIpAddress STRING, 511 | userAgent STRING, 512 | errorCode STRING, 513 | errorMessage STRING, 514 | requestId STRING, 515 | eventId STRING, 516 | resources ARRAY>, 520 | eventType STRING, 521 | apiVersion STRING, 522 | readOnly BOOLEAN, 523 | recipientAccountId STRING, 524 | sharedEventID STRING, 525 | vpcEndpointId STRING, 526 | requestParameters STRING, 527 | responseElements STRING, 528 | additionalEventData STRING, 529 | serviceEventDetails STRING 530 | ) 531 | ROW FORMAT SERDE 'com.amazon.emr.hive.serde.CloudTrailSerde' 532 | STORED AS INPUTFORMAT 'com.amazon.emr.cloudtrail.CloudTrailInputFormat' 533 | OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' 534 | LOCATION 's3://my-cloudtrail-logs/' 535 | ``` 536 | 537 | Change the last line to point to your CloudTrail log bucket and query away 538 | 539 | ```sql 540 | SELECT 541 | eventTime, 542 | eventName, 543 | userIdentity.userName, 544 | requestParameters 545 | FROM 546 | cloudtrail_logs 547 | WHERE 548 | eventName LIKE '%Bucket%' 549 | ORDER BY 1 550 | ``` 551 | 552 | ## Best Practices 553 | 554 | Keep things simple and follow the principle of least privilege to reduce the chance of mistakes. 555 | 556 | - Strictly limit who can perform bucket-related operations 557 | - Avoid mixing objects with different permissions in the same bucket (use a bucket policy to enforce this) 558 | - Don’t specify public read permissions on a bucket level (no `GetObject` in bucket policy) 559 | - Monitor configuration frequently for changes 560 | 561 | ## Bucket Policies 562 | 563 | Only private uploads 564 | 565 | ```json 566 | { 567 | "Version": "2012-10-17", 568 | "Statement": [ 569 | { 570 | "Effect": "Deny", 571 | "Principal": "*", 572 | "Action": "s3:PutObjectAcl", 573 | "Resource": "arn:aws:s3:::my-bucket/*" 574 | } 575 | ] 576 | } 577 | ``` 578 | 579 | ## Performance 580 | 581 | For commands that iterate over bucket objects (`scan-object-acl`, `reset-object-acl`, `encrypt`, and `delete-unencrypted-versions`), run s3tk on an EC2 server for minimum latency. 582 | 583 | ## Notes 584 | 585 | The `set-policy`, `block-public-access`, `enable-logging`, `enable-versioning`, and `enable-default-encryption` commands are provided for convenience. We recommend [Terraform](https://www.terraform.io/) for managing your buckets. 586 | 587 | ```tf 588 | resource "aws_s3_bucket" "my_bucket" { 589 | bucket = "my-bucket" 590 | acl = "private" 591 | 592 | logging { 593 | target_bucket = "my-s3-logs" 594 | target_prefix = "my-bucket/" 595 | } 596 | 597 | versioning { 598 | enabled = true 599 | } 600 | } 601 | 602 | resource "aws_s3_bucket_public_access_block" "my_bucket" { 603 | bucket = "${aws_s3_bucket.my_bucket.id}" 604 | 605 | block_public_acls = true 606 | block_public_policy = true 607 | ignore_public_acls = true 608 | restrict_public_buckets = true 609 | } 610 | ``` 611 | 612 | ## Upgrading 613 | 614 | Run: 615 | 616 | ```sh 617 | pip install s3tk --upgrade 618 | ``` 619 | 620 | To use master, run: 621 | 622 | ```sh 623 | pip install git+https://github.com/ankane/s3tk.git --upgrade 624 | ``` 625 | 626 | ## Docker 627 | 628 | Run: 629 | 630 | ```sh 631 | docker run -it ankane/s3tk aws configure 632 | ``` 633 | 634 | Commit your credentials: 635 | 636 | ```sh 637 | docker commit $(docker ps -l -q) my-s3tk 638 | ``` 639 | 640 | And run: 641 | 642 | ```sh 643 | docker run -it my-s3tk s3tk scan 644 | ``` 645 | 646 | ## History 647 | 648 | View the [changelog](https://github.com/ankane/s3tk/blob/master/CHANGELOG.md) 649 | 650 | ## Contributing 651 | 652 | Everyone is encouraged to help improve this project. Here are a few ways you can help: 653 | 654 | - [Report bugs](https://github.com/ankane/s3tk/issues) 655 | - Fix bugs and [submit pull requests](https://github.com/ankane/s3tk/pulls) 656 | - Write, clarify, or fix documentation 657 | - Suggest or add new features 658 | 659 | To get started with development: 660 | 661 | ```sh 662 | git clone https://github.com/ankane/s3tk.git 663 | cd s3tk 664 | pip install -r requirements.txt 665 | ``` 666 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "s3tk" 7 | version = "0.4.0" 8 | description = "A security toolkit for Amazon S3" 9 | readme = "README.md" 10 | authors = [ 11 | {name = "Andrew Kane", email = "andrew@ankane.org"} 12 | ] 13 | license = "BSD-3-Clause" 14 | requires-python = ">= 3.10" 15 | dependencies = [ 16 | "boto3>=1.9.46", 17 | "botocore>=1.12.46", 18 | "clint", 19 | "click", 20 | "joblib" 21 | ] 22 | 23 | [project.urls] 24 | Homepage = "https://github.com/ankane/s3tk" 25 | 26 | [project.scripts] 27 | s3tk = "s3tk:main" 28 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | -------------------------------------------------------------------------------- /s3tk/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import fnmatch 4 | from collections import Counter, OrderedDict 5 | import warnings 6 | import boto3 7 | import botocore 8 | import click 9 | from joblib import Parallel, delayed 10 | from .checks import AclCheck, PolicyCheck, PublicAccessCheck, LoggingCheck, VersioningCheck, EncryptionCheck, ObjectLoggingCheck 11 | 12 | # fix for https://github.com/kennethreitz-archive/clint/issues/185 13 | with warnings.catch_warnings(): 14 | warnings.simplefilter("ignore") 15 | from clint.textui import colored, puts, indent 16 | 17 | __version__ = '0.4.0' 18 | 19 | canned_acls = [ 20 | { 21 | 'acl': 'private', 22 | 'grants': [] 23 | }, 24 | { 25 | 'acl': 'public-read', 26 | 'grants': [ 27 | {'Grantee': {'Type': 'Group', 'URI': 'http://acs.amazonaws.com/groups/global/AllUsers'}, 'Permission': 'READ'} 28 | ] 29 | }, 30 | { 31 | 'acl': 'public-read-write', 32 | 'grants': [ 33 | {'Grantee': {'Type': 'Group', 'URI': 'http://acs.amazonaws.com/groups/global/AllUsers'}, 'Permission': 'READ'}, 34 | {'Grantee': {u'Type': 'Group', u'URI': 'http://acs.amazonaws.com/groups/global/AllUsers'}, 'Permission': 'WRITE'} 35 | ] 36 | }, 37 | { 38 | 'acl': 'authenticated-read', 39 | 'grants': [ 40 | {'Grantee': {'Type': 'Group', 'URI': 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'}, 'Permission': 'READ'} 41 | ] 42 | }, 43 | { 44 | 'acl': 'aws-exec-read', 45 | 'grants': [ 46 | {'Grantee': {'Type': 'CanonicalUser', 'DisplayName': 'za-team', 'ID': '6aa5a366c34c1cbe25dc49211496e913e0351eb0e8c37aa3477e40942ec6b97c'}, 'Permission': 'READ'} 47 | ] 48 | } 49 | ] 50 | 51 | cached_s3 = None 52 | 53 | 54 | def s3(): 55 | # memoize 56 | global cached_s3 57 | if cached_s3 is None: 58 | cached_s3 = boto3.resource('s3') 59 | return cached_s3 60 | 61 | 62 | def notice(message): 63 | puts(colored.yellow(message)) 64 | 65 | 66 | def abort(message): 67 | puts(colored.red(message)) 68 | sys.exit(1) 69 | 70 | 71 | def unicode_key(key): 72 | return key 73 | 74 | 75 | def perform(check): 76 | check.perform() 77 | 78 | with indent(2): 79 | if check.status == 'passed': 80 | puts(colored.green('✔ ' + check.name + ' ' + unicode_key(check.pass_message))) 81 | elif check.status == 'failed': 82 | puts(colored.red('✘ ' + check.name + ' ' + check.fail_message)) 83 | else: 84 | puts(colored.red('✘ ' + check.name + ' access denied')) 85 | 86 | return check 87 | 88 | 89 | def fetch_buckets(buckets): 90 | if buckets: 91 | if any('*' in b for b in buckets): 92 | return [b for b in s3().buckets.all() if any(fnmatch.fnmatch(b.name, bn) for bn in buckets)] 93 | else: 94 | return [s3().Bucket(bn) for bn in buckets] 95 | else: 96 | return s3().buckets.all() 97 | 98 | 99 | def fix_check(klass, buckets, dry_run, fix_args={}): 100 | for bucket in fetch_buckets(buckets): 101 | check = klass(bucket) 102 | check.perform() 103 | 104 | if check.status == 'passed': 105 | message = colored.green('already ' + check.pass_message) 106 | elif check.status == 'denied': 107 | message = colored.red('access denied') 108 | else: 109 | if dry_run: 110 | message = colored.yellow('to be ' + check.pass_message) 111 | else: 112 | try: 113 | check.fix(fix_args) 114 | message = colored.blue('just ' + check.pass_message) 115 | except botocore.exceptions.ClientError as e: 116 | message = colored.red(str(e)) 117 | 118 | puts(bucket.name + ' ' + message) 119 | 120 | 121 | def encrypt_object(bucket_name, key, dry_run, kms_key_id, customer_key): 122 | obj = s3().Object(bucket_name, key) 123 | str_key = unicode_key(key) 124 | 125 | try: 126 | if customer_key: 127 | obj.load(SSECustomerAlgorithm='AES256', SSECustomerKey=customer_key) 128 | 129 | encrypted = None 130 | if customer_key: 131 | encrypted = obj.sse_customer_algorithm is not None 132 | elif kms_key_id: 133 | encrypted = obj.server_side_encryption == 'aws:kms' 134 | else: 135 | encrypted = obj.server_side_encryption == 'AES256' 136 | 137 | if encrypted: 138 | puts(str_key + ' ' + colored.green('already encrypted')) 139 | return 'already encrypted' 140 | else: 141 | if dry_run: 142 | puts(str_key + ' ' + colored.yellow('to be encrypted')) 143 | return 'to be encrypted' 144 | else: 145 | copy_source = {'Bucket': bucket_name, 'Key': obj.key} 146 | 147 | # TODO support going from customer encryption to other forms 148 | if kms_key_id: 149 | obj.copy_from( 150 | CopySource=copy_source, 151 | ServerSideEncryption='aws:kms', 152 | SSEKMSKeyId=kms_key_id 153 | ) 154 | elif customer_key: 155 | obj.copy_from( 156 | CopySource=copy_source, 157 | SSECustomerAlgorithm='AES256', 158 | SSECustomerKey=customer_key 159 | ) 160 | else: 161 | obj.copy_from( 162 | CopySource=copy_source, 163 | ServerSideEncryption='AES256' 164 | ) 165 | 166 | puts(str_key + ' ' + colored.blue('just encrypted')) 167 | return 'just encrypted' 168 | 169 | except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError) as e: 170 | puts(str_key + ' ' + colored.red(str(e))) 171 | return 'error' 172 | 173 | 174 | def determine_mode(acl): 175 | owner = acl.owner 176 | grants = acl.grants 177 | non_owner_grants = [grant for grant in grants if not (grant['Grantee'].get('ID') == owner['ID'] and grant['Permission'] == 'FULL_CONTROL')] 178 | 179 | # TODO bucket-owner-read and bucket-owner-full-control 180 | return next((ca['acl'] for ca in canned_acls if ca['grants'] == non_owner_grants), 'custom') 181 | 182 | 183 | def scan_object(bucket_name, key): 184 | obj = s3().Object(bucket_name, key) 185 | str_key = unicode_key(key) 186 | 187 | try: 188 | mode = determine_mode(obj.Acl()) 189 | 190 | if mode == 'private': 191 | puts(str_key + ' ' + colored.green(mode)) 192 | else: 193 | puts(str_key + ' ' + colored.yellow(mode)) 194 | 195 | return mode 196 | except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError) as e: 197 | puts(str_key + ' ' + colored.red(str(e))) 198 | return 'error' 199 | 200 | 201 | def reset_object(bucket_name, key, dry_run, acl): 202 | obj = s3().Object(bucket_name, key) 203 | str_key = unicode_key(key) 204 | 205 | try: 206 | obj_acl = obj.Acl() 207 | mode = determine_mode(obj_acl) 208 | 209 | if mode == acl: 210 | puts(str_key + ' ' + colored.green('ACL already ' + acl)) 211 | return 'ACL already ' + acl 212 | elif dry_run: 213 | puts(str_key + ' ' + colored.yellow('ACL to be updated to ' + acl)) 214 | return 'ACL to be updated to ' + acl 215 | else: 216 | obj_acl.put(ACL=acl) 217 | puts(str_key + ' ' + colored.blue('ACL updated to ' + acl)) 218 | return 'ACL updated to ' + acl 219 | 220 | except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError) as e: 221 | puts(str_key + ' ' + colored.red(str(e))) 222 | return 'error' 223 | 224 | 225 | def delete_unencrypted_version(bucket_name, key, id, dry_run): 226 | object_version = s3().ObjectVersion(bucket_name, key, id) 227 | 228 | try: 229 | obj = object_version.get() 230 | if obj.get('ServerSideEncryption') or obj.get('SSECustomerAlgorithm'): 231 | puts(key + ' ' + id + ' ' + colored.green('encrypted')) 232 | return 'encrypted' 233 | else: 234 | if dry_run: 235 | puts(key + ' ' + id + ' ' + colored.blue('to be deleted')) 236 | return 'to be deleted' 237 | else: 238 | puts(key + ' ' + id + ' ' + colored.blue('deleted')) 239 | object_version.delete() 240 | return 'deleted' 241 | except (botocore.exceptions.ClientError, botocore.exceptions.NoCredentialsError) as e: 242 | puts(key + ' ' + id + ' ' + colored.red(str(e))) 243 | return 'error' 244 | 245 | 246 | def object_matches(key, only, _except): 247 | match = True 248 | 249 | if only: 250 | match = fnmatch.fnmatch(key, only) 251 | 252 | if _except and match: 253 | match = not fnmatch.fnmatch(key, _except) 254 | 255 | return match 256 | 257 | 258 | def parallelize(bucket, only, _except, fn, args=(), versions=False): 259 | bucket = s3().Bucket(bucket) 260 | 261 | # use prefix for performance 262 | prefix = None 263 | if only: 264 | # get the first prefix before wildcard 265 | prefix = '/'.join(only.split('*')[0].split('/')[:-1]) 266 | if prefix: 267 | prefix = prefix + '/' 268 | 269 | if versions: 270 | object_versions = bucket.object_versions.filter(Prefix=prefix) if prefix else bucket.object_versions.all() 271 | # delete markers have no size 272 | return Parallel(n_jobs=24)(delayed(fn)(bucket.name, ov.object_key, ov.id, *args) for ov in object_versions if object_matches(ov.object_key, only, _except) and not ov.is_latest and ov.size is not None) 273 | else: 274 | objects = bucket.objects.filter(Prefix=prefix) if prefix else bucket.objects.all() 275 | 276 | if only and '*' not in only: 277 | objects = [s3().Object(bucket, only)] 278 | 279 | return Parallel(n_jobs=24)(delayed(fn)(bucket.name, os.key, *args) for os in objects if object_matches(os.key, only, _except)) 280 | 281 | 282 | def public_statement(bucket): 283 | return OrderedDict([ 284 | ('Sid', 'Public'), 285 | ('Effect', 'Allow'), 286 | ('Principal', '*'), 287 | ('Action', 's3:GetObject'), 288 | ('Resource', 'arn:aws:s3:::%s/*' % bucket.name) 289 | ]) 290 | 291 | 292 | def no_object_acl_statement(bucket): 293 | return OrderedDict([ 294 | ('Sid', 'NoObjectAcl'), 295 | ('Effect', 'Deny'), 296 | ('Principal', '*'), 297 | ('Action', 's3:PutObjectAcl'), 298 | ('Resource', 'arn:aws:s3:::%s/*' % bucket.name) 299 | ]) 300 | 301 | 302 | def public_uploads_statement(bucket): 303 | return OrderedDict([ 304 | ('Sid', 'PublicUploads'), 305 | ('Effect', 'Deny'), 306 | ('Principal', '*'), 307 | ('Action', ['s3:PutObject', 's3:PutObjectAcl']), 308 | ('Resource', 'arn:aws:s3:::%s/*' % bucket.name), 309 | ('Condition', {'StringNotEquals': {'s3:x-amz-acl': 'public-read'}}) 310 | ]) 311 | 312 | 313 | def no_uploads_statement(bucket): 314 | return OrderedDict([ 315 | ('Sid', 'NoUploads'), 316 | ('Effect', 'Deny'), 317 | ('Principal', '*'), 318 | ('Action', 's3:PutObject'), 319 | ('Resource', 'arn:aws:s3:::%s/*' % bucket.name) 320 | ]) 321 | 322 | 323 | def encryption_statement(bucket): 324 | return OrderedDict([ 325 | ('Sid', 'Encryption'), 326 | ('Effect', 'Deny'), 327 | ('Principal', '*'), 328 | ('Action', 's3:PutObject'), 329 | ('Resource', 'arn:aws:s3:::%s/*' % bucket.name), 330 | ('Condition', {'StringNotEquals': {'s3:x-amz-server-side-encryption': 'AES256'}}) 331 | ]) 332 | 333 | 334 | def statement_matches(s1, s2): 335 | s1 = dict(s1) 336 | s2 = dict(s2) 337 | s1.pop('Sid', None) 338 | s2.pop('Sid', None) 339 | return s1 == s2 340 | 341 | 342 | def fetch_policy(bucket): 343 | policy = None 344 | try: 345 | policy = bucket.Policy().policy 346 | except botocore.exceptions.ClientError as e: 347 | if 'NoSuchBucket' not in str(e): 348 | raise 349 | 350 | if policy: 351 | policy = json.loads(policy, object_pairs_hook=OrderedDict) 352 | 353 | return policy 354 | 355 | 356 | def print_dns_bucket(name, buckets, found_buckets): 357 | if name not in found_buckets: 358 | puts(name) 359 | with indent(2): 360 | if name in buckets: 361 | puts(colored.green('owned')) 362 | else: 363 | puts(colored.red('not owned')) 364 | 365 | puts() 366 | 367 | found_buckets.add(name) 368 | 369 | 370 | def print_policy(policy): 371 | with indent(2): 372 | if any(policy['Statement']): 373 | puts(colored.yellow(json.dumps(policy, indent=4))) 374 | else: 375 | puts(colored.yellow("None")) 376 | 377 | 378 | def summarize(values): 379 | summary = Counter(values) 380 | 381 | puts() 382 | puts("Summary") 383 | for k, v in summary.most_common(): 384 | puts(k + ': ' + str(v)) 385 | 386 | 387 | def fetch_event_selectors(): 388 | # TODO get trails across all regions 389 | # even regions without buckets may have multi-region trails 390 | client = boto3.client('cloudtrail') 391 | paginator = client.get_paginator('list_trails') 392 | 393 | event_selectors = {} 394 | for page in paginator.paginate(): 395 | for trail in page['Trails']: 396 | name = trail['Name'] 397 | region_client = boto3.client('cloudtrail', region_name=trail['HomeRegion']) 398 | response = region_client.get_event_selectors(TrailName=name) 399 | for event_selector in response['EventSelectors']: 400 | read_write_type = event_selector['ReadWriteType'] 401 | for data_resource in event_selector['DataResources']: 402 | if data_resource['Type'] == 'AWS::S3::Object': 403 | for value in data_resource['Values']: 404 | if value == 'arn:aws:s3': 405 | trail_response = region_client.get_trail(Name=name)['Trail'] 406 | if trail_response['IsMultiRegionTrail']: 407 | bucket = ('global') 408 | else: 409 | bucket = ('region', trail['HomeRegion']) 410 | path = '' 411 | else: 412 | parts = value.split("/", 2) 413 | bucket = ('bucket', parts[0].replace('arn:aws:s3:::', '')) 414 | path = parts[1] 415 | if bucket not in event_selectors: 416 | event_selectors[bucket] = [] 417 | event_selectors[bucket].append({'trail': name, 'path': path, 'read_write_type': read_write_type}) 418 | return event_selectors 419 | 420 | 421 | @click.group() 422 | @click.version_option(version=__version__) 423 | def cli(): 424 | pass 425 | 426 | 427 | @cli.command() 428 | @click.argument('buckets', nargs=-1) 429 | @click.option('--log-bucket', multiple=True, help='Check log bucket(s)') 430 | @click.option('--log-prefix', help='Check log prefix') 431 | @click.option('--skip-logging', is_flag=True, help='Skip logging check') 432 | @click.option('--skip-versioning', is_flag=True, help='Skip versioning check') 433 | @click.option('--skip-default-encryption', is_flag=True, help='Skip default encryption check') 434 | @click.option('--default-encryption', is_flag=True) # no op, can't hide from help until click 7 released 435 | @click.option('--object-level-logging', is_flag=True) 436 | @click.option('--sns-topic', help='Send SNS notification for failures') 437 | def scan(buckets, log_bucket=None, log_prefix=None, skip_logging=False, skip_versioning=False, skip_default_encryption=False, default_encryption=True, object_level_logging=False, sns_topic=None): 438 | event_selectors = fetch_event_selectors() if object_level_logging else {} 439 | 440 | checks = [] 441 | for bucket in fetch_buckets(buckets): 442 | puts(bucket.name) 443 | 444 | checks.append(perform(AclCheck(bucket))) 445 | 446 | checks.append(perform(PolicyCheck(bucket))) 447 | 448 | checks.append(perform(PublicAccessCheck(bucket))) 449 | 450 | if not skip_logging: 451 | checks.append(perform(LoggingCheck(bucket, log_bucket=log_bucket, log_prefix=log_prefix))) 452 | 453 | if not skip_versioning: 454 | checks.append(perform(VersioningCheck(bucket))) 455 | 456 | if not skip_default_encryption: 457 | checks.append(perform(EncryptionCheck(bucket))) 458 | 459 | if object_level_logging: 460 | checks.append(perform(ObjectLoggingCheck(bucket, event_selectors=event_selectors))) 461 | 462 | puts() 463 | 464 | failed_checks = [c for c in checks if c.status != 'passed'] 465 | if any(failed_checks): 466 | if sns_topic: 467 | topic = boto3.resource('sns').Topic(sns_topic) 468 | message = '' 469 | for check in failed_checks: 470 | msg = check.fail_message if check.status == 'failed' else 'access denied' 471 | message += check.bucket.name + ': ' + check.name + ' ' + msg + '\n' 472 | topic.publish(Message=message, Subject='[s3tk] Scan Failures') 473 | sys.exit(1) 474 | 475 | 476 | @cli.command(name='scan-dns') 477 | def scan_dns(): 478 | buckets = set([b.name for b in s3().buckets.all()]) 479 | found_buckets = set() 480 | 481 | client = boto3.client('route53') 482 | paginator = client.get_paginator('list_hosted_zones') 483 | 484 | for page in paginator.paginate(): 485 | for hosted_zone in page['HostedZones']: 486 | paginator2 = client.get_paginator('list_resource_record_sets') 487 | for page2 in paginator2.paginate(HostedZoneId=hosted_zone['Id']): 488 | for resource_set in page2['ResourceRecordSets']: 489 | if resource_set.get('AliasTarget'): 490 | value = resource_set['AliasTarget']['DNSName'] 491 | if value.startswith('s3-website-') and value.endswith('.amazonaws.com.'): 492 | print_dns_bucket(resource_set['Name'][:-1], buckets, found_buckets) 493 | elif resource_set.get('ResourceRecords'): 494 | for record in resource_set['ResourceRecords']: 495 | value = record['Value'] 496 | if value.endswith('.s3.amazonaws.com'): 497 | print_dns_bucket('.'.join(value.split('.')[:-3]), buckets, found_buckets) 498 | if 's3-website-' in value and value.endswith('.amazonaws.com'): 499 | print_dns_bucket(resource_set['Name'][:-1], buckets, found_buckets) 500 | 501 | 502 | @cli.command(name='block-public-access') 503 | @click.argument('buckets', nargs=-1) 504 | @click.option('--dry-run', is_flag=True, help='Dry run') 505 | def block_public_access(buckets, dry_run=False): 506 | if not buckets: 507 | abort('Must specify at least one bucket or wildcard') 508 | fix_check(PublicAccessCheck, buckets, dry_run) 509 | 510 | 511 | @cli.command(name='enable-logging') 512 | @click.argument('buckets', nargs=-1) 513 | @click.option('--dry-run', is_flag=True, help='Dry run') 514 | @click.option('--log-bucket', required=True, help='Bucket to store logs') 515 | @click.option('--log-prefix', help='Log prefix') 516 | def enable_logging(buckets, log_bucket=None, log_prefix=None, dry_run=False): 517 | fix_check(LoggingCheck, buckets, dry_run, {'log_bucket': log_bucket, 'log_prefix': log_prefix}) 518 | 519 | 520 | @cli.command(name='enable-versioning') 521 | @click.argument('buckets', nargs=-1) 522 | @click.option('--dry-run', is_flag=True, help='Dry run') 523 | def enable_versioning(buckets, dry_run=False): 524 | fix_check(VersioningCheck, buckets, dry_run) 525 | 526 | 527 | @cli.command(name='enable-default-encryption') 528 | @click.argument('buckets', nargs=-1) 529 | @click.option('--dry-run', is_flag=True, help='Dry run') 530 | def enable_default_encryption(buckets, dry_run=False): 531 | fix_check(EncryptionCheck, buckets, dry_run) 532 | 533 | 534 | @cli.command() 535 | @click.argument('bucket') 536 | @click.option('--only', help='Only certain objects') 537 | @click.option('--except', '_except', help='Except certain objects') 538 | @click.option('--dry-run', is_flag=True, help='Dry run') 539 | @click.option('--kms-key-id', help='KMS key id') 540 | @click.option('--customer-key', help='Customer key') 541 | def encrypt(bucket, only=None, _except=None, dry_run=False, kms_key_id=None, customer_key=None): 542 | summarize(parallelize(bucket, only, _except, encrypt_object, (dry_run, kms_key_id, customer_key,))) 543 | 544 | 545 | @cli.command(name='scan-object-acl') 546 | @click.argument('bucket') 547 | @click.option('--only', help='Only certain objects') 548 | @click.option('--except', '_except', help='Except certain objects') 549 | def scan_object_acl(bucket, only=None, _except=None): 550 | summarize(parallelize(bucket, only, _except, scan_object)) 551 | 552 | 553 | @cli.command(name='reset-object-acl') 554 | @click.argument('bucket') 555 | @click.option('--only', help='Only certain objects') 556 | @click.option('--except', '_except', help='Except certain objects') 557 | @click.option('--acl', default='private', help='ACL to use') 558 | @click.option('--dry-run', is_flag=True, help='Dry run') 559 | def reset_object_acl(bucket, only=None, _except=None, acl=None, dry_run=False): 560 | summarize(parallelize(bucket, only, _except, reset_object, (dry_run, acl,))) 561 | 562 | 563 | @cli.command(name='delete-unencrypted-versions') 564 | @click.argument('bucket') 565 | @click.option('--only', help='Only certain objects') 566 | @click.option('--except', '_except', help='Except certain objects') 567 | @click.option('--dry-run', is_flag=True, help='Dry run') 568 | def delete_unencrypted_versions(bucket, only=None, _except=None, dry_run=False): 569 | summarize(parallelize(bucket, only, _except, delete_unencrypted_version, (dry_run,), True)) 570 | 571 | 572 | @cli.command(name='list-policy') 573 | @click.argument('buckets', nargs=-1) 574 | @click.option('--named', is_flag=True, help='Print named statements') 575 | def list_policy(buckets, named=False): 576 | for bucket in fetch_buckets(buckets): 577 | puts(bucket.name) 578 | 579 | policy = fetch_policy(bucket) 580 | 581 | with indent(2): 582 | if policy is None: 583 | puts(colored.yellow('None')) 584 | else: 585 | if named: 586 | public = public_statement(bucket) 587 | no_object_acl = no_object_acl_statement(bucket) 588 | public_uploads = public_uploads_statement(bucket) 589 | no_uploads = no_uploads_statement(bucket) 590 | encryption = encryption_statement(bucket) 591 | 592 | for statement in policy['Statement']: 593 | if statement_matches(statement, public): 594 | named_statement = 'Public' 595 | elif statement_matches(statement, no_object_acl): 596 | named_statement = 'No object ACL' 597 | elif statement_matches(statement, public_uploads): 598 | named_statement = 'Public uploads' 599 | elif statement_matches(statement, no_uploads): 600 | named_statement = 'No uploads' 601 | elif statement_matches(statement, encryption): 602 | named_statement = 'Encryption' 603 | else: 604 | named_statement = 'Custom' 605 | 606 | puts(colored.yellow(named_statement)) 607 | 608 | else: 609 | puts(colored.yellow(json.dumps(policy, indent=4))) 610 | 611 | puts() 612 | 613 | 614 | @cli.command(name='set-policy') 615 | @click.argument('bucket') 616 | @click.option('--public', is_flag=True, help='Make all objects public') 617 | @click.option('--no-object-acl', is_flag=True, help='Prevent object ACL') 618 | @click.option('--public-uploads', is_flag=True, help='Only public uploads') 619 | @click.option('--no-uploads', is_flag=True, help='Prevent new uploads') 620 | @click.option('--encryption', is_flag=True, help='Require encryption') 621 | @click.option('--dry-run', is_flag=True, help='Dry run') 622 | def set_policy(bucket, public=False, no_object_acl=False, public_uploads=False, no_uploads=False, encryption=False, dry_run=False): 623 | bucket = s3().Bucket(bucket) 624 | bucket_policy = bucket.Policy() 625 | 626 | statements = [] 627 | 628 | if public: 629 | statements.append(public_statement(bucket)) 630 | 631 | if no_object_acl: 632 | statements.append(no_object_acl_statement(bucket)) 633 | 634 | if public_uploads: 635 | statements.append(public_uploads_statement(bucket)) 636 | 637 | if no_uploads: 638 | statements.append(no_uploads_statement(bucket)) 639 | 640 | if encryption: 641 | statements.append(encryption_statement(bucket)) 642 | 643 | if any(statements): 644 | puts('New policy') 645 | policy = OrderedDict([ 646 | ('Version', '2012-10-17'), 647 | ('Statement', statements) 648 | ]) 649 | print_policy(policy) 650 | 651 | if not dry_run: 652 | bucket_policy.put(Policy=json.dumps(policy)) 653 | else: 654 | abort('No policies specified') 655 | 656 | 657 | # experimental 658 | @cli.command(name='update-policy') 659 | @click.argument('bucket') 660 | @click.option('--encryption/--no-encryption', default=None, help='Require encryption') 661 | @click.option('--dry-run', is_flag=True, help='Dry run') 662 | def update_policy(bucket, encryption=None, dry_run=False): 663 | bucket = s3().Bucket(bucket) 664 | 665 | policy = fetch_policy(bucket) 666 | if not policy: 667 | policy = OrderedDict([ 668 | ('Version', '2012-10-17'), 669 | ('Statement', []) 670 | ]) 671 | 672 | es = encryption_statement(bucket) 673 | es_index = next((i for i, s in enumerate(policy['Statement']) if statement_matches(s, es)), -1) 674 | 675 | if es_index != -1: 676 | if encryption: 677 | puts("No encryption change") 678 | print_policy(policy) 679 | elif encryption is False: 680 | puts("Removing encryption") 681 | policy['Statement'].pop(es_index) 682 | print_policy(policy) 683 | 684 | if not dry_run: 685 | if any(policy['Statement']): 686 | bucket.Policy().put(Policy=json.dumps(policy)) 687 | else: 688 | bucket.Policy().delete() 689 | else: 690 | if encryption: 691 | puts("Adding encryption") 692 | policy['Statement'].append(es) 693 | print_policy(policy) 694 | 695 | if not dry_run: 696 | bucket.Policy().put(Policy=json.dumps(policy)) 697 | elif encryption is False: 698 | puts(colored.yellow("No encryption change")) 699 | print_policy(policy) 700 | 701 | 702 | @cli.command(name='delete-policy') 703 | @click.argument('bucket') 704 | def delete_policy(bucket): 705 | s3().Bucket(bucket).Policy().delete() 706 | puts('Policy deleted') 707 | 708 | 709 | def main(): 710 | try: 711 | cli() 712 | except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: 713 | abort(str(e)) 714 | except KeyboardInterrupt: 715 | sys.exit(1) 716 | -------------------------------------------------------------------------------- /s3tk/checks.py: -------------------------------------------------------------------------------- 1 | import json 2 | import botocore 3 | 4 | 5 | class Check: 6 | def __init__(self, bucket, **kwargs): 7 | self.bucket = bucket 8 | self.options = kwargs 9 | 10 | def perform(self): 11 | try: 12 | self.status = 'passed' if self._passed() else 'failed' 13 | except botocore.exceptions.ClientError: 14 | self.status = 'denied' 15 | 16 | def fix(self, options): 17 | self._fix(options) 18 | self.status = 'passed' 19 | 20 | 21 | class AclCheck(Check): 22 | name = 'ACL' 23 | pass_message = 'not open to public' 24 | fail_message = 'open to public' 25 | bad_grantees = [ 26 | 'http://acs.amazonaws.com/groups/global/AllUsers', 27 | 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers' 28 | ] 29 | 30 | def _passed(self): 31 | for grant in self.bucket.Acl().grants: 32 | if grant['Grantee'].get('URI', None) in self.bad_grantees: 33 | return False 34 | return True 35 | 36 | 37 | class PolicyCheck(Check): 38 | name = 'Policy' 39 | pass_message = 'not open to public' 40 | fail_message = 'open to public' 41 | 42 | def _passed(self): 43 | policy = None 44 | try: 45 | policy = self.bucket.Policy().policy 46 | except botocore.exceptions.ClientError as e: 47 | if 'NoSuchBucket' not in str(e): 48 | raise 49 | 50 | if policy is not None: 51 | policy = json.loads(policy) 52 | for s in policy['Statement']: 53 | if s['Effect'] == 'Allow' and (s['Principal'] == '*' or s['Principal'] == {'AWS': '*'}): 54 | return False 55 | 56 | return True 57 | 58 | 59 | class PublicAccessCheck(Check): 60 | name = 'Public access' 61 | pass_message = 'blocked' 62 | fail_message = 'not explicitly blocked' 63 | 64 | def _passed(self): 65 | response = None 66 | 67 | try: 68 | response = self.bucket.meta.client.get_public_access_block( 69 | Bucket=self.bucket.name 70 | ) 71 | except botocore.exceptions.ClientError as e: 72 | if 'NoSuchPublicAccessBlockConfiguration' not in str(e): 73 | raise 74 | 75 | return False 76 | 77 | config = response['PublicAccessBlockConfiguration'] 78 | return (config['BlockPublicAcls'] and config['IgnorePublicAcls'] and config['BlockPublicPolicy'] and config['RestrictPublicBuckets']) 79 | 80 | def _fix(self, options): 81 | self.bucket.meta.client.put_public_access_block( 82 | Bucket=self.bucket.name, 83 | PublicAccessBlockConfiguration={ 84 | 'BlockPublicAcls': True, 85 | 'IgnorePublicAcls': True, 86 | 'BlockPublicPolicy': True, 87 | 'RestrictPublicBuckets': True 88 | } 89 | ) 90 | 91 | 92 | class LoggingCheck(Check): 93 | name = 'Logging' 94 | pass_message = 'enabled' 95 | fail_message = 'disabled' 96 | 97 | def _passed(self): 98 | enabled = self.bucket.Logging().logging_enabled 99 | log_bucket = self.options.get('log_bucket', None) 100 | log_prefix = self.options.get('log_prefix', None) 101 | if log_prefix: 102 | log_prefix = log_prefix.replace("{bucket}", self.bucket.name) 103 | 104 | if not enabled: 105 | return False 106 | elif log_bucket and enabled['TargetBucket'] not in log_bucket: 107 | self.fail_message = 'to wrong bucket: ' + enabled['TargetBucket'] 108 | return False 109 | elif log_prefix and enabled['TargetPrefix'] != log_prefix: 110 | self.fail_message = 'to wrong prefix: ' + enabled['TargetPrefix'] 111 | return False 112 | 113 | self.pass_message = 'to ' + enabled['TargetBucket'] 114 | if enabled['TargetPrefix']: 115 | self.pass_message = self.pass_message + '/' + enabled['TargetPrefix'] 116 | 117 | return True 118 | 119 | def _fix(self, options): 120 | log_prefix = (options['log_prefix'] or '{bucket}/').replace("{bucket}", self.bucket.name) 121 | 122 | self.bucket.Logging().put( 123 | BucketLoggingStatus={ 124 | 'LoggingEnabled': { 125 | 'TargetBucket': options['log_bucket'], 126 | 'TargetGrants': [ 127 | { 128 | 'Grantee': { 129 | 'Type': 'Group', 130 | 'URI': 'http://acs.amazonaws.com/groups/s3/LogDelivery' 131 | }, 132 | 'Permission': 'WRITE' 133 | }, 134 | { 135 | 'Grantee': { 136 | 'Type': 'Group', 137 | 'URI': 'http://acs.amazonaws.com/groups/s3/LogDelivery' 138 | }, 139 | 'Permission': 'READ_ACP' 140 | }, 141 | ], 142 | 'TargetPrefix': log_prefix 143 | } 144 | } 145 | ) 146 | 147 | 148 | class VersioningCheck(Check): 149 | name = 'Versioning' 150 | pass_message = 'enabled' 151 | fail_message = 'disabled' 152 | 153 | def _passed(self): 154 | return self.bucket.Versioning().status == 'Enabled' 155 | 156 | def _fix(self, options): 157 | self.bucket.Versioning().enable() 158 | 159 | 160 | class EncryptionCheck(Check): 161 | name = 'Default encryption' 162 | pass_message = 'enabled' 163 | fail_message = 'disabled' 164 | 165 | def _passed(self): 166 | response = None 167 | try: 168 | response = self.bucket.meta.client.get_bucket_encryption( 169 | Bucket=self.bucket.name 170 | ) 171 | except botocore.exceptions.ClientError as e: 172 | if 'ServerSideEncryptionConfigurationNotFoundError' not in str(e): 173 | raise 174 | 175 | return response is not None 176 | 177 | def _fix(self, options): 178 | self.bucket.meta.client.put_bucket_encryption( 179 | Bucket=self.bucket.name, 180 | ServerSideEncryptionConfiguration={ 181 | 'Rules': [ 182 | { 183 | 'ApplyServerSideEncryptionByDefault': { 184 | 'SSEAlgorithm': 'AES256' 185 | } 186 | } 187 | ] 188 | } 189 | ) 190 | 191 | 192 | class ObjectLoggingCheck(Check): 193 | name = 'CloudTrail object-level logging' 194 | pass_message = 'enabled' 195 | fail_message = 'disabled' 196 | 197 | def _passed(self): 198 | event_selectors = self.options['event_selectors'] 199 | 200 | selectors = [] 201 | selectors += event_selectors.get(('global'), []) 202 | 203 | # handle single-region trails 204 | if any(k for k in event_selectors.keys() if k[0] == 'region'): 205 | region = self.bucket.meta.client.get_bucket_location(Bucket=self.bucket.name)['LocationConstraint'] 206 | # https://github.com/aws/aws-sdk-net/issues/323 207 | if region is None: 208 | region = 'us-east-1' 209 | selectors += event_selectors.get(('region', region), []) 210 | 211 | selectors += event_selectors.get(('bucket', self.bucket.name), []) 212 | 213 | passed = any(selectors) 214 | if passed: 215 | messages = [] 216 | for event_selector in selectors: 217 | message = event_selector['trail'] + ' (' 218 | 219 | if event_selector['read_write_type'] == 'All': 220 | message += 'read & write' 221 | elif event_selector['read_write_type'] == 'ReadOnly': 222 | message += 'read' 223 | elif event_selector['read_write_type'] == 'WriteOnly': 224 | message += 'write' 225 | else: 226 | message += 'unknown' 227 | 228 | if event_selector['path'] != '': 229 | message += ' for /' + event_selector['path'] 230 | 231 | messages.append(message + ')') 232 | 233 | self.pass_message = 'to ' + ' and '.join(messages) 234 | 235 | return passed 236 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankane/s3tk/12472b752c255f5b1505da5a1998402f45f7c6d3/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_s3tk.py: -------------------------------------------------------------------------------- 1 | class TestS3tk(object): 2 | def test_works(self): 3 | assert True 4 | --------------------------------------------------------------------------------