├── .changes ├── 0.0.1.json ├── 0.1.0.json ├── 0.1.1.json ├── 0.1.10.json ├── 0.1.11.json ├── 0.1.12.json ├── 0.1.13.json ├── 0.1.2.json ├── 0.1.3.json ├── 0.1.4.json ├── 0.1.5.json ├── 0.1.6.json ├── 0.1.7.json ├── 0.1.8.json ├── 0.1.9.json ├── 0.10.0.json ├── 0.10.1.json ├── 0.10.2.json ├── 0.10.3.json ├── 0.10.4.json ├── 0.11.0.json ├── 0.11.1.json ├── 0.11.2.json ├── 0.11.3.json ├── 0.11.4.json ├── 0.11.5.json ├── 0.12.0.json ├── 0.13.0.json ├── 0.2.0.json ├── 0.2.1.json ├── 0.3.0.json ├── 0.3.1.json ├── 0.3.2.json ├── 0.3.3.json ├── 0.3.4.json ├── 0.3.5.json ├── 0.3.6.json ├── 0.3.7.json ├── 0.4.0.json ├── 0.4.1.json ├── 0.4.2.json ├── 0.5.0.json ├── 0.5.1.json ├── 0.5.2.json ├── 0.6.0.json ├── 0.6.1.json ├── 0.6.2.json ├── 0.7.0.json ├── 0.8.0.json ├── 0.8.1.json ├── 0.8.2.json └── 0.9.0.json ├── .coveragerc ├── .github ├── SECURITY.md ├── codeql.yml └── workflows │ ├── codeql.yml │ ├── fail-master-prs.yml │ ├── lint.yml │ ├── run-crt-test.yml │ └── run-tests.yml ├── .gitignore ├── .pre-commit-config.yaml ├── ACCEPTANCE_TESTS.rst ├── CHANGELOG.rst ├── CONTRIBUTING.rst ├── LICENSE.txt ├── MANIFEST.in ├── NOTICE.txt ├── README.rst ├── pyproject.toml ├── requirements-dev-lock.txt ├── requirements-dev.txt ├── requirements.txt ├── s3transfer ├── __init__.py ├── bandwidth.py ├── compat.py ├── constants.py ├── copies.py ├── crt.py ├── delete.py ├── download.py ├── exceptions.py ├── futures.py ├── manager.py ├── processpool.py ├── subscribers.py ├── tasks.py ├── upload.py └── utils.py ├── scripts ├── ci │ ├── install │ ├── install-dev-deps │ ├── run-crt-tests │ ├── run-integ-tests │ └── run-tests ├── new-change ├── performance │ ├── benchmark │ ├── benchmark-download │ ├── benchmark-upload │ ├── download-file │ ├── processpool-download │ ├── summarize │ └── upload-file └── stress │ └── timeout ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── functional │ ├── __init__.py │ ├── test_copy.py │ ├── test_crt.py │ ├── test_delete.py │ ├── test_download.py │ ├── test_manager.py │ ├── test_processpool.py │ ├── test_upload.py │ └── test_utils.py ├── integration │ ├── __init__.py │ ├── test_copy.py │ ├── test_crt.py │ ├── test_delete.py │ ├── test_download.py │ ├── test_processpool.py │ ├── test_s3transfer.py │ └── test_upload.py └── unit │ ├── __init__.py │ ├── test_bandwidth.py │ ├── test_compat.py │ ├── test_copies.py │ ├── test_crt.py │ ├── test_delete.py │ ├── test_download.py │ ├── test_futures.py │ ├── test_manager.py │ ├── test_processpool.py │ ├── test_s3transfer.py │ ├── test_subscribers.py │ ├── test_tasks.py │ ├── test_upload.py │ └── test_utils.py └── tox.ini /.changes/0.0.1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "manager", 4 | "description": "Add boto3 s3 transfer logic to package. (`issue 2 `__)", 5 | "type": "feature" 6 | } 7 | ] 8 | -------------------------------------------------------------------------------- /.changes/0.1.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "copy", 4 | "description": "Add support for managed copies.", 5 | "type": "feature" 6 | }, 7 | { 8 | "category": "download", 9 | "description": "Add support for downloading to a filename, seekable file-like object, and nonseekable file-like object.", 10 | "type": "feature" 11 | }, 12 | { 13 | "category": "general", 14 | "description": "Add ``TransferManager`` class. All public functionality for ``s3transfer`` is exposed through this class.", 15 | "type": "feature" 16 | }, 17 | { 18 | "category": "subscribers", 19 | "description": "Add subscriber interface. Currently supports on_queued, on_progress, and on_done status changes.", 20 | "type": "feature" 21 | }, 22 | { 23 | "category": "upload", 24 | "description": "Add support for uploading a filename, seekable file-like object, and nonseekable file-like object.", 25 | "type": "feature" 26 | } 27 | ] -------------------------------------------------------------------------------- /.changes/0.1.1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "deadlock", 4 | "description": "Fix deadlock issue described here: https://bugs.python.org/issue20319 with using concurrent.futures.wait", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.1.10.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``TransferManager``", 4 | "description": "Expose ability to use own executor class for ``TransferManager``", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.1.11.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "TransferManager", 4 | "description": "Properly handle unicode exceptions in the context manager. Fixes `#85 `__", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.1.12.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``max_bandwidth``", 4 | "description": "Add ability to set maximum bandwidth consumption for streaming of S3 uploads and downloads", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.1.13.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``RequestPayer``", 4 | "description": "Plumb ``RequestPayer` argument to the ``CompleteMultipartUpload` operation (`#103 `__).", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.1.2.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "download", 4 | "description": "Patch memory leak related to unnecessarily holding onto futures for downloads.", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.1.3.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "delete", 4 | "description": "Add a ``.delete()`` method to the transfer manager.", 5 | "type": "feature" 6 | }, 7 | { 8 | "category": "seekable upload", 9 | "description": "Fix issue where seeked position of seekable file for a nonmultipart upload was not being taken into account.", 10 | "type": "bugfix" 11 | } 12 | ] -------------------------------------------------------------------------------- /.changes/0.1.4.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "chunksize", 4 | "description": "Automatically adjust the chunksize if it doesn't meet S3s requirements.", 5 | "type": "feature" 6 | }, 7 | { 8 | "category": "Download", 9 | "description": "Add support for downloading to special UNIX file by name", 10 | "type": "bugfix" 11 | } 12 | ] -------------------------------------------------------------------------------- /.changes/0.1.5.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Cntrl-C", 4 | "description": "Fix issue of hangs when Cntrl-C happens for many queued transfers", 5 | "type": "bugfix" 6 | }, 7 | { 8 | "category": "cancel", 9 | "description": "Expose messages for cancels", 10 | "type": "feature" 11 | } 12 | ] -------------------------------------------------------------------------------- /.changes/0.1.6.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "download", 4 | "description": "Fix issue where S3 Object was not downloaded to disk when empty", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.1.7.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "TransferManager", 4 | "description": "Fix memory leak when using same client to create multiple TransferManagers", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.1.8.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "download", 4 | "description": "Support downloading to FIFOs.", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.1.9.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``TransferFuture``", 4 | "description": "Add support for setting exceptions on transfer future", 5 | "type": "feature" 6 | } 7 | ] 8 | -------------------------------------------------------------------------------- /.changes/0.10.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``s3``", 4 | "description": "Added CRT support for S3 Express One Zone", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.10.1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``urllib3``", 4 | "description": "Fixed retry handling for IncompleteRead exception raised by urllib3 2.x during data transfer", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.10.2.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``awscrt``", 4 | "description": "Pass operation name to awscrt.s3 to improve error handling.", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.10.3.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Python", 4 | "description": "Added provisional Python 3.13 support to s3transfer", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.10.4.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``s3``", 4 | "description": "Added Multi-Region Access Points support to CRT transfers.", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.11.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "manager", 4 | "description": "Use CRC32 by default and support user provided full-object checksums.", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.11.1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Dependencies", 4 | "description": "Update the floor version of botocore to 1.36.0 to match imports.", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.11.2.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "upload", 4 | "description": "Only set a default checksum if the ``request_checksum_calculation`` config is set to ``when_supported``. Fixes `boto/s3transfer#327 `__.", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.11.3.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``awscrt``", 4 | "description": "Fix urlencoding issues for request signing with the awscrt.", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.11.4.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Dependencies", 4 | "description": "Update the floor version of botocore to 1.37.4 to match imports.", 5 | "type": "enhancement" 6 | }, 7 | { 8 | "category": "Tasks", 9 | "description": "Pass Botocore context from parent to child threads.", 10 | "type": "enhancement" 11 | } 12 | ] -------------------------------------------------------------------------------- /.changes/0.11.5.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``s3``", 4 | "description": "Implement memory performance fixes for downloads to non-seekable streams", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.12.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Python", 4 | "description": "End of support for Python 3.8", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.13.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``GetObjectTask``", 4 | "description": "Validate ETag of stored object during multipart downloads", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.2.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``ProcessPoolDownloader``", 4 | "description": "Add ``ProcessPoolDownloader`` class to speed up download throughput by using processes instead of threads.", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.2.1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "ProcessPool", 4 | "description": "Adds user agent suffix.", 5 | "type": "enhancment" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.3.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Python", 4 | "description": "Dropped support for Python 2.6 and 3.3.", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.3.1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``TransferManager``", 4 | "description": "Expose ``client`` and ``config`` properties", 5 | "type": "enhancement" 6 | }, 7 | { 8 | "category": "Tags", 9 | "description": "Add support for ``Tagging`` and ``TaggingDirective``", 10 | "type": "enhancement" 11 | } 12 | ] -------------------------------------------------------------------------------- /.changes/0.3.2.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "s3", 4 | "description": "Fixes boto/botocore`#1916 `__", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.3.3.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "dependency", 4 | "description": "Updated botocore version range to allow for developmental installs.", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.3.4.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "s3", 4 | "description": "Add server side encryption context into allowed list", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.3.5.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``s3``", 4 | "description": "Block TransferManager methods for S3 Object Lambda resources", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.3.6.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "packaging", 4 | "description": "Fix setup.py metadata for `futures` on Python 2.7", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.3.7.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "ReadFileChunk", 4 | "description": "Fix seek behavior in ReadFileChunk class", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.4.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``crt``", 4 | "description": "Add optional AWS Common Runtime (CRT) support. The AWS CRT provides a C-based S3 transfer client that can improve transfer throughput.", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.4.1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``crt``", 4 | "description": "Add ``set_exception`` to ``CRTTransferFuture`` to allow setting exceptions in subscribers.", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.4.2.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "s3", 4 | "description": "Add support for ``ExpectedBucketOwner``. Fixes `#181 `__.", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.5.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Python", 4 | "description": "Dropped support for Python 2.7", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.5.1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Python", 4 | "description": "Officially add Python 3.10 support", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.5.2.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``s3``", 4 | "description": "Added support for flexible checksums when uploading or downloading objects.", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.6.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Python", 4 | "description": "Dropped support for Python 3.6", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.6.1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "copy", 4 | "description": "Added support for ``ChecksumAlgorithm`` when uploading copy data in parts.", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.6.2.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Python", 4 | "description": "Added provisional Python 3.12 support to s3transfer", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.7.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``SSE-C``", 4 | "description": "Pass SSECustomer* arguments to CompleteMultipartUpload for upload operations", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.8.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``crt``", 4 | "description": "Automatically configure CRC32 checksums for uploads and checksum validation for downloads through the CRT transfer manager.", 5 | "type": "enhancement" 6 | }, 7 | { 8 | "category": "``crt``", 9 | "description": "S3transfer now supports a wider range of CRT functionality for uploads to improve throughput in the CLI/Boto3.", 10 | "type": "feature" 11 | }, 12 | { 13 | "category": "``Botocore``", 14 | "description": "S3Transfer now requires Botocore >=1.32.7", 15 | "type": "enhancement" 16 | }, 17 | { 18 | "category": "``crt``", 19 | "description": "Update ``target_throughput`` defaults. If not configured, s3transfer will use the AWS CRT to attempt to determine a recommended target throughput to use based on the system. If there is no recommended throughput, s3transfer now falls back to ten gigabits per second.", 20 | "type": "enhancement" 21 | }, 22 | { 23 | "category": "``crt``", 24 | "description": "Add support for uploading and downloading file-like objects using CRT transfer manager. It supports both seekable and non-seekable file-like objects.", 25 | "type": "enhancement" 26 | } 27 | ] -------------------------------------------------------------------------------- /.changes/0.8.1.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "``s3``", 4 | "description": "Added support for defaulting checksums to CRC32 for s3express.", 5 | "type": "enhancement" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.8.2.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Subscribers", 4 | "description": "Added caching for Subscribers to improve throughput by up to 24% in high volume transfer", 5 | "type": "bugfix" 6 | } 7 | ] -------------------------------------------------------------------------------- /.changes/0.9.0.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "category": "Python", 4 | "description": "End of support for Python 3.7", 5 | "type": "feature" 6 | } 7 | ] -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | include = 4 | s3transfer/* 5 | -------------------------------------------------------------------------------- /.github/SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Reporting a Vulnerability 2 | 3 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security 4 | via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. 5 | Please do **not** create a public GitHub issue. 6 | -------------------------------------------------------------------------------- /.github/codeql.yml: -------------------------------------------------------------------------------- 1 | paths: 2 | - "s3transfer/" 3 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: ["develop"] 6 | pull_request: 7 | branches: ["develop"] 8 | schedule: 9 | - cron: "0 0 * * 5" 10 | 11 | permissions: "read-all" 12 | 13 | jobs: 14 | analyze: 15 | name: "Analyze" 16 | runs-on: "ubuntu-latest" 17 | permissions: 18 | actions: read 19 | contents: read 20 | security-events: write 21 | steps: 22 | - name: "Checkout repository" 23 | uses: "actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3" 24 | 25 | - name: "Run CodeQL init" 26 | uses: "github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a" 27 | with: 28 | config-file: "./.github/codeql.yml" 29 | languages: "python" 30 | 31 | - name: "Run CodeQL autobuild" 32 | uses: "github/codeql-action/autobuild@cdcdbb579706841c47f7063dda365e292e5cad7a" 33 | 34 | - name: "Run CodeQL analyze" 35 | uses: "github/codeql-action/analyze@cdcdbb579706841c47f7063dda365e292e5cad7a" 36 | -------------------------------------------------------------------------------- /.github/workflows/fail-master-prs.yml: -------------------------------------------------------------------------------- 1 | name: PRs against master are not accepted, please target develop branch 2 | 3 | on: 4 | pull_request: 5 | branches: [ master ] 6 | 7 | permissions: 8 | contents: read 9 | 10 | jobs: 11 | fail: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Fail PRs against master 15 | run: | 16 | echo "PRs must be made against the develop branch." 17 | exit 1 18 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint code 2 | 3 | on: 4 | push: 5 | pull_request: 6 | branches-ignore: [ master ] 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | lint: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 17 | - name: Set up Python 3.9 18 | uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 19 | with: 20 | python-version: 3.9 21 | - name: Run pre-commit 22 | uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 23 | -------------------------------------------------------------------------------- /.github/workflows/run-crt-test.yml: -------------------------------------------------------------------------------- 1 | 2 | name: Run CRT tests 3 | 4 | on: 5 | push: 6 | pull_request: 7 | branches-ignore: [ master ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14-dev"] 16 | os: [ubuntu-latest, macOS-latest, windows-latest] 17 | 18 | steps: 19 | - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | cache: 'pip' 25 | allow-prereleases: true 26 | - name: Install dependencies and CRT 27 | run: | 28 | python scripts/ci/install --extras crt 29 | - name: Run tests 30 | run: | 31 | python scripts/ci/run-crt-tests 32 | -------------------------------------------------------------------------------- /.github/workflows/run-tests.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | push: 5 | pull_request: 6 | branches-ignore: [ master ] 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | build: 13 | 14 | runs-on: ubuntu-latest 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14-dev"] 19 | os: [ubuntu-latest, macOS-latest, windows-latest] 20 | 21 | steps: 22 | - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | cache: 'pip' 28 | allow-prereleases: true 29 | - name: Install dependencies 30 | run: | 31 | python scripts/ci/install 32 | - name: Run tests 33 | run: | 34 | python scripts/ci/run-tests --with-cov 35 | - name: codecov 36 | uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d 37 | with: 38 | directory: tests 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | *.DS_Store 3 | 4 | # Packages 5 | *.egg 6 | *.egg-info 7 | dist 8 | build 9 | eggs 10 | parts 11 | var 12 | sdist 13 | develop-eggs 14 | .installed.cfg 15 | 16 | # Installer logs 17 | pip-log.txt 18 | 19 | # Unit test / coverage reports 20 | .coverage 21 | .tox 22 | .cache 23 | 24 | #Translations 25 | *.mo 26 | 27 | #Mr Developer 28 | .mr.developer.cfg 29 | 30 | # Emacs backup files 31 | *~ 32 | 33 | # Eclipse IDE 34 | /.project 35 | /.pydevproject 36 | 37 | # IDEA IDE 38 | .idea* 39 | src/ 40 | 41 | 42 | # Virtualenvs 43 | .venv 44 | venv 45 | env 46 | env2 47 | env3 48 | 49 | # Completions Index 50 | completions.idx 51 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: ^(.changes/|CHANGELOG.rst|s3transfer/compat.py) 2 | repos: 3 | - repo: 'https://github.com/pre-commit/pre-commit-hooks' 4 | rev: v4.5.0 5 | hooks: 6 | - id: check-yaml 7 | - id: end-of-file-fixer 8 | - id: trailing-whitespace 9 | - repo: https://github.com/astral-sh/ruff-pre-commit 10 | rev: v0.4.8 11 | hooks: 12 | - id: ruff 13 | args: [ --fix ] 14 | - id: ruff-format 15 | -------------------------------------------------------------------------------- /ACCEPTANCE_TESTS.rst: -------------------------------------------------------------------------------- 1 | S3 Acceptance Tests 2 | =================== 3 | 4 | List of all of the various scenarios that need to be handled in implementing 5 | a S3 transfer manager. 6 | 7 | Upload Tests 8 | ------------ 9 | 10 | General 11 | ~~~~~~~ 12 | * [x] Upload single nonmultipart file 13 | * [x] Upload single multipart file 14 | * [x] Upload multiple nonmultipart files 15 | * [x] Upload multiple multipart files 16 | * [x] Failed/cancelled multipart upload is aborted and leaves no orphaned parts especially for: 17 | 18 | * [x] Failure of ``UploadPart`` 19 | * [x] Failure of ``CompleteMultipartUpload`` 20 | * [x] Failure unrelated to making an API call during upload such as read failure 21 | 22 | * [ ] Ctrl-C of any upload does not hang and the wait time is ``avg(transfer_time_iter_chunk) * some_margin`` 23 | * [ ] Upload empty file 24 | * [ ] Upload nonseekable nonmultipart binary stream 25 | * [ ] Upload nonseekable multipart binary stream 26 | 27 | 28 | Region 29 | ~~~~~~ 30 | * [ ] Provide no or incorrect region for sig4 and be able to redirect request in fewest amount of calls as possible for multipart upload. 31 | 32 | 33 | Validation 34 | ~~~~~~~~~~ 35 | * [ ] Before upload, validate upload size of file is less than 5 TB. 36 | * [ ] Before upload, modify chunksize to an acceptable size when needed: 37 | 38 | * [ ] Make chunksize 5 MB when the provided chunksize is less 39 | * [ ] Make chunksize 5 GB when the provided chunksize is more 40 | * [ ] Increase chunksize till the maximum number of parts for multipart upload is less than or equal to 10,000 parts 41 | 42 | * [ ] Before upload, ensure upload is nonmultipart if the file size is less than 5 MB no matter the provided multipart threshold. 43 | 44 | 45 | Extra Parameters 46 | ~~~~~~~~~~~~~~~~ 47 | * [ ] Upload multipart and nonmultipart file with any of the following properties: 48 | 49 | * [x] ACL's 50 | * [x] CacheControl 51 | * [x] ContentDisposition 52 | * [x] ContentEncoding 53 | * [x] ContentLanguage 54 | * [x] ContentType 55 | * [x] Expires 56 | * [x] Metadata 57 | * [x] Grants 58 | * [x] StorageClass 59 | * [x] SSE (including KMS) 60 | * [ ] Website Redirect 61 | 62 | * [x] Upload multipart and nonmultipart file with a sse-c key 63 | * [x] Upload multipart and nonmultipart file with requester pays 64 | 65 | 66 | Performance 67 | ~~~~~~~~~~~ 68 | * [ ] Maximum memory usage does not grow linearly with linearly increasing file size for any upload. 69 | * [ ] Maximum memory usage does not grow linearly with linearly increasing number of uploads. 70 | 71 | 72 | Download Tests 73 | -------------- 74 | 75 | General 76 | ~~~~~~~ 77 | * [x] Download single nonmultipart object 78 | * [x] Download single multipart object 79 | * [x] Download multiple nonmultipart objects 80 | * [x] Download multiple multipart objects 81 | * [x] Download of any object is written to temporary file and renamed to final filename once the object is completely downloaded 82 | * [x] Failed downloads of any object cleans up temporary file 83 | * [x] Provide a transfer size for any download in lieu of using HeadObject 84 | * [ ] Ctrl-C of any download does not hang and the wait time is ``avg(transfer_time_iter_chunk) * some_margin`` 85 | * [ ] Download nonmultipart object as nonseekable binary stream 86 | * [ ] Download multipart object as nonseekable binary stream 87 | 88 | 89 | Region 90 | ~~~~~~ 91 | * [ ] Provide no or incorrect region for sig4 and be able to redirect request in fewest amount of calls as possible for multipart download. 92 | 93 | 94 | Retry Logic 95 | ~~~~~~~~~~~ 96 | * [x] Retry on connection related errors when downloading data 97 | * [ ] Compare MD5 to ``ETag`` and retry for mismatches if all following scenarios are met: 98 | 99 | * If MD5 is available 100 | * Response does not have a ``ServerSideEncryption`` header equal to ``aws:kms`` 101 | * Response does not have ``SSECustomerAlgorithm`` 102 | * ``ETag`` does not have ``-`` in its value indicating a multipart transfer 103 | 104 | 105 | Extra Parameters 106 | ~~~~~~~~~~~~~~~~ 107 | * [x] Download an object of a specific version 108 | * [x] Download an object encrypted with sse-c 109 | * [x] Download an object using requester pays 110 | 111 | 112 | Performance 113 | ~~~~~~~~~~~ 114 | * [ ] Maximum memory usage does not grow linearly with linearly increasing file size for any download. 115 | * [ ] Maximum memory usage does not grow linearly with linearly increasing number of downloads. 116 | 117 | 118 | Copy Tests 119 | ---------- 120 | 121 | General 122 | ~~~~~~~ 123 | * [x] Copy single nonmultipart object 124 | * [x] Copy single multipart object 125 | * [x] Copy multiple nonmultipart objects 126 | * [x] Copy multiple multipart objects 127 | * [x] Provide a transfer size for any copy in lieu of using HeadObject. 128 | * [x] Failed/cancelled multipart copy is aborted and leaves no orphaned parts 129 | * [ ] Ctrl-C of any copy does not hang and the wait time is ``avg(transfer_time_iter_chunk) * some_margin`` 130 | 131 | 132 | Region 133 | ~~~~~~ 134 | * [ ] Provide no or incorrect region for sig4 and be able to redirect request in fewest amount of calls as possible for multipart copy. 135 | 136 | 137 | Validation 138 | ~~~~~~~~~~ 139 | * [ ] Before copy, modify chunksize to an acceptable size when needed: 140 | 141 | * [ ] Make chunksize 5 MB when the provided chunksize is less 142 | * [ ] Make chunksize 5 GB when the provided chunksize is more 143 | * [ ] Increase chunksize till the maximum number of parts for multipart copy is less than or equal to 10,000 parts 144 | 145 | * [ ] Before copy, ensure copy is nonmultipart if the file size is less than 5 MB no matter the provided multipart threshold. 146 | 147 | 148 | Extra Parameters 149 | ~~~~~~~~~~~~~~~~ 150 | * [ ] Copy multipart and nonmultipart file with any of the following properties: 151 | 152 | * [x] ACL's 153 | * [x] CacheControl 154 | * [x] ContentDisposition 155 | * [x] ContentEncoding 156 | * [x] ContentLanguage 157 | * [x] ContentType 158 | * [x] Expires 159 | * [x] Metadata 160 | * [x] Grants 161 | * [x] StorageClass 162 | * [x] SSE (including KMS) 163 | * [ ] Website Redirect 164 | 165 | * [x] Copy multipart and nonmultipart copies with copy source parameters: 166 | 167 | * [x] CopySourceIfMatch 168 | * [x] CopySourceIfModifiedSince 169 | * [x] CopySourceIfNoneMatch 170 | * [x] CopySourceIfUnmodifiedSince 171 | 172 | * [x] Copy nonmultipart object with metadata directive and do not use metadata directive for multipart object 173 | * [x] Copy multipart and nonmultipart objects of a specific version 174 | * [x] Copy multipart and nonmultipart objects using requester pays 175 | * [x] Copy multipart and nonmultipart objects using a sse-c key 176 | * [x] Copy multipart and nonmultipart objects using a copy source sse-c key 177 | * [x] Copy multipart and nonmultipart objects using a copy source sse-c key and sse-c key 178 | 179 | 180 | Cross-Bucket 181 | ~~~~~~~~~~~~ 182 | * [ ] Copy single nonmultipart object across sigv4 regions 183 | * [ ] Copy single multipart object across sigv4 regions 184 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | CHANGELOG 3 | ========= 4 | 5 | 0.13.0 6 | ====== 7 | 8 | * feature:``GetObjectTask``: Validate ETag of stored object during multipart downloads 9 | 10 | 11 | 0.12.0 12 | ====== 13 | 14 | * feature:Python: End of support for Python 3.8 15 | 16 | 17 | 0.11.5 18 | ====== 19 | 20 | * enhancement:``s3``: Implement memory performance fixes for downloads to non-seekable streams 21 | 22 | 23 | 0.11.4 24 | ====== 25 | 26 | * enhancement:Dependencies: Update the floor version of botocore to 1.37.4 to match imports. 27 | * enhancement:Tasks: Pass Botocore context from parent to child threads. 28 | 29 | 30 | 0.11.3 31 | ====== 32 | 33 | * bugfix:``awscrt``: Fix urlencoding issues for request signing with the awscrt. 34 | 35 | 36 | 0.11.2 37 | ====== 38 | 39 | * bugfix:upload: Only set a default checksum if the ``request_checksum_calculation`` config is set to ``when_supported``. Fixes `boto/s3transfer#327 `__. 40 | 41 | 42 | 0.11.1 43 | ====== 44 | 45 | * bugfix:Dependencies: Update the floor version of botocore to 1.36.0 to match imports. 46 | 47 | 48 | 0.11.0 49 | ====== 50 | 51 | * feature:manager: Use CRC32 by default and support user provided full-object checksums. 52 | 53 | 54 | 0.10.4 55 | ====== 56 | 57 | * enhancement:``s3``: Added Multi-Region Access Points support to CRT transfers. 58 | 59 | 60 | 0.10.3 61 | ====== 62 | 63 | * enhancement:Python: Added provisional Python 3.13 support to s3transfer 64 | 65 | 66 | 0.10.2 67 | ====== 68 | 69 | * bugfix:``awscrt``: Pass operation name to awscrt.s3 to improve error handling. 70 | 71 | 72 | 0.10.1 73 | ====== 74 | 75 | * bugfix:``urllib3``: Fixed retry handling for IncompleteRead exception raised by urllib3 2.x during data transfer 76 | 77 | 78 | 0.10.0 79 | ====== 80 | 81 | * feature:``s3``: Added CRT support for S3 Express One Zone 82 | 83 | 84 | 0.9.0 85 | ===== 86 | 87 | * feature:Python: End of support for Python 3.7 88 | 89 | 90 | 0.8.2 91 | ===== 92 | 93 | * bugfix:Subscribers: Added caching for Subscribers to improve throughput by up to 24% in high volume transfer 94 | 95 | 96 | 0.8.1 97 | ===== 98 | 99 | * enhancement:``s3``: Added support for defaulting checksums to CRC32 for s3express. 100 | 101 | 102 | 0.8.0 103 | ===== 104 | 105 | * enhancement:``crt``: Automatically configure CRC32 checksums for uploads and checksum validation for downloads through the CRT transfer manager. 106 | * feature:``crt``: S3transfer now supports a wider range of CRT functionality for uploads to improve throughput in the CLI/Boto3. 107 | * enhancement:``Botocore``: S3Transfer now requires Botocore >=1.32.7 108 | * enhancement:``crt``: Update ``target_throughput`` defaults. If not configured, s3transfer will use the AWS CRT to attempt to determine a recommended target throughput to use based on the system. If there is no recommended throughput, s3transfer now falls back to ten gigabits per second. 109 | * enhancement:``crt``: Add support for uploading and downloading file-like objects using CRT transfer manager. It supports both seekable and non-seekable file-like objects. 110 | 111 | 112 | 0.7.0 113 | ===== 114 | 115 | * feature:``SSE-C``: Pass SSECustomer* arguments to CompleteMultipartUpload for upload operations 116 | 117 | 118 | 0.6.2 119 | ===== 120 | 121 | * enhancement:Python: Added provisional Python 3.12 support to s3transfer 122 | 123 | 124 | 0.6.1 125 | ===== 126 | 127 | * bugfix:copy: Added support for ``ChecksumAlgorithm`` when uploading copy data in parts. 128 | 129 | 130 | 0.6.0 131 | ===== 132 | 133 | * feature:Python: Dropped support for Python 3.6 134 | 135 | 136 | 0.5.2 137 | ===== 138 | 139 | * enhancement:``s3``: Added support for flexible checksums when uploading or downloading objects. 140 | 141 | 142 | 0.5.1 143 | ===== 144 | 145 | * enhancement:Python: Officially add Python 3.10 support 146 | 147 | 148 | 0.5.0 149 | ===== 150 | 151 | * feature:Python: Dropped support for Python 2.7 152 | 153 | 154 | 0.4.2 155 | ===== 156 | 157 | * enhancement:s3: Add support for ``ExpectedBucketOwner``. Fixes `#181 `__. 158 | 159 | 160 | 0.4.1 161 | ===== 162 | 163 | * enhancement:``crt``: Add ``set_exception`` to ``CRTTransferFuture`` to allow setting exceptions in subscribers. 164 | 165 | 166 | 0.4.0 167 | ===== 168 | 169 | * feature:``crt``: Add optional AWS Common Runtime (CRT) support. The AWS CRT provides a C-based S3 transfer client that can improve transfer throughput. 170 | 171 | 172 | 0.3.7 173 | ===== 174 | 175 | * bugfix:ReadFileChunk: Fix seek behavior in ReadFileChunk class 176 | 177 | 178 | 0.3.6 179 | ===== 180 | 181 | * bugfix:packaging: Fix setup.py metadata for `futures` on Python 2.7 182 | 183 | 184 | 0.3.5 185 | ===== 186 | 187 | * enhancement:``s3``: Block TransferManager methods for S3 Object Lambda resources 188 | 189 | 190 | 0.3.4 191 | ===== 192 | 193 | * enhancement:s3: Add server side encryption context into allowed list 194 | 195 | 196 | 0.3.3 197 | ===== 198 | 199 | * bugfix:dependency: Updated botocore version range to allow for developmental installs. 200 | 201 | 202 | 0.3.2 203 | ===== 204 | 205 | * bugfix:s3: Fixes boto/botocore`#1916 `__ 206 | 207 | 208 | 0.3.1 209 | ===== 210 | 211 | * enhancement:``TransferManager``: Expose ``client`` and ``config`` properties 212 | * enhancement:Tags: Add support for ``Tagging`` and ``TaggingDirective`` 213 | 214 | 215 | 0.3.0 216 | ===== 217 | 218 | * feature:Python: Dropped support for Python 2.6 and 3.3. 219 | 220 | 221 | 0.2.1 222 | ===== 223 | 224 | * enhancment:ProcessPool: Adds user agent suffix. 225 | 226 | 227 | 0.2.0 228 | ===== 229 | 230 | * feature:``ProcessPoolDownloader``: Add ``ProcessPoolDownloader`` class to speed up download throughput by using processes instead of threads. 231 | 232 | 233 | 0.1.13 234 | ====== 235 | 236 | * bugfix:``RequestPayer``: Plumb ``RequestPayer` argument to the ``CompleteMultipartUpload` operation (`#103 `__). 237 | 238 | 239 | 0.1.12 240 | ====== 241 | 242 | * enhancement:``max_bandwidth``: Add ability to set maximum bandwidth consumption for streaming of S3 uploads and downloads 243 | 244 | 245 | 0.1.11 246 | ====== 247 | 248 | * bugfix:TransferManager: Properly handle unicode exceptions in the context manager. Fixes `#85 `__ 249 | 250 | 251 | 0.1.10 252 | ====== 253 | 254 | * feature:``TransferManager``: Expose ability to use own executor class for ``TransferManager`` 255 | 256 | 257 | 0.1.9 258 | ===== 259 | 260 | * feature:``TransferFuture``: Add support for setting exceptions on transfer future 261 | 262 | 263 | 0.1.8 264 | ===== 265 | 266 | * feature:download: Support downloading to FIFOs. 267 | 268 | 269 | 0.1.7 270 | ===== 271 | 272 | * bugfix:TransferManager: Fix memory leak when using same client to create multiple TransferManagers 273 | 274 | 275 | 0.1.6 276 | ===== 277 | 278 | * bugfix:download: Fix issue where S3 Object was not downloaded to disk when empty 279 | 280 | 281 | 0.1.5 282 | ===== 283 | 284 | * bugfix:Cntrl-C: Fix issue of hangs when Cntrl-C happens for many queued transfers 285 | * feature:cancel: Expose messages for cancels 286 | 287 | 288 | 0.1.4 289 | ===== 290 | 291 | * feature:chunksize: Automatically adjust the chunksize if it doesn't meet S3s requirements. 292 | * bugfix:Download: Add support for downloading to special UNIX file by name 293 | 294 | 295 | 0.1.3 296 | ===== 297 | 298 | * feature:delete: Add a ``.delete()`` method to the transfer manager. 299 | * bugfix:seekable upload: Fix issue where seeked position of seekable file for a nonmultipart upload was not being taken into account. 300 | 301 | 302 | 0.1.2 303 | ===== 304 | 305 | * bugfix:download: Patch memory leak related to unnecessarily holding onto futures for downloads. 306 | 307 | 308 | 0.1.1 309 | ===== 310 | 311 | * bugfix:deadlock: Fix deadlock issue described here: https://bugs.python.org/issue20319 with using concurrent.futures.wait 312 | 313 | 314 | 0.1.0 315 | ===== 316 | 317 | * feature:copy: Add support for managed copies. 318 | * feature:download: Add support for downloading to a filename, seekable file-like object, and nonseekable file-like object. 319 | * feature:general: Add ``TransferManager`` class. All public functionality for ``s3transfer`` is exposed through this class. 320 | * feature:subscribers: Add subscriber interface. Currently supports on_queued, on_progress, and on_done status changes. 321 | * feature:upload: Add support for uploading a filename, seekable file-like object, and nonseekable file-like object. 322 | 323 | 324 | 0.0.1 325 | ===== 326 | 327 | * feature:manager: Add boto3 s3 transfer logic to package. (`issue 2 `__) 328 | 329 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Contributing Code 2 | ----------------- 3 | A good pull request: 4 | 5 | - Is clear. 6 | - Works across all supported versions of Python. 7 | - Follows the existing style of the code base (see Codestyle section). 8 | - Has comments included as needed. 9 | 10 | - A test case that demonstrates the previous flaw that now passes with 11 | the included patch, or demonstrates the newly added feature. 12 | - If it adds/changes a public API, it must also include documentation 13 | for those changes. 14 | - Must be appropriately licensed (Apache 2.0). 15 | 16 | Reporting An Issue/Feature 17 | -------------------------- 18 | First, check to see if there's an existing 19 | `issue `__/`pull requests `__ for the bug/feature. 20 | 21 | If there isn't an existing issue there, please file an issue *first*. The 22 | ideal report includes: 23 | 24 | - A description of the problem/suggestion. 25 | - How to recreate the bug. 26 | - If relevant, including the versions of your: 27 | 28 | - Python interpreter 29 | - s3transfer 30 | - Optionally of the other dependencies involved (e.g. Botocore) 31 | 32 | - If possible, create a pull request with a (failing) test case 33 | demonstrating what's wrong. This makes the process for fixing bugs 34 | quicker & gets issues resolved sooner. 35 | 36 | Codestyle 37 | --------- 38 | This project uses `ruff `__ to enforce 39 | codstyle requirements. We've codified this process using a tool called 40 | `pre-commit `__. pre-commit allows us to specify a 41 | config file with all tools required for code linting, and surfaces either a 42 | git commit hook, or single command, for enforcing these. 43 | 44 | To validate your PR prior to publishing, you can use the following 45 | `installation guide `__ to setup pre-commit. 46 | 47 | If you don't want to use the git commit hook, you can run the below command 48 | to automatically perform the codestyle validation: 49 | 50 | .. code-block:: bash 51 | 52 | $ pre-commit run 53 | 54 | This will automatically perform simple updates (such as white space clean up) 55 | and provide a list of any failing checks. After these are addressed, 56 | you can commit the changes prior to publishing the PR. 57 | These checks are also included in our CI setup under the "Lint" workflow which 58 | will provide output on Github for anything missed locally. 59 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include LICENSE.txt 3 | include requirements-test.txt 4 | graft tests 5 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | s3transfer 2 | Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | s3transfer - An Amazon S3 Transfer Manager for Python 3 | ===================================================== 4 | 5 | S3transfer is a Python library for managing Amazon S3 transfers. 6 | This project is maintained and published by Amazon Web Services. 7 | 8 | .. note:: 9 | 10 | This project is not currently GA. If you are planning to use this code in 11 | production, make sure to lock to a minor version as interfaces may break 12 | from minor version to minor version. For a basic, stable interface of 13 | s3transfer, try the interfaces exposed in `boto3 `__ 14 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.pytest.ini_options] 2 | markers = [ 3 | "slow: marks tests as slow", 4 | ] 5 | 6 | [tool.ruff] 7 | exclude = [ 8 | ".bzr", 9 | ".direnv", 10 | ".eggs", 11 | ".git", 12 | ".git-rewrite", 13 | ".hg", 14 | ".ipynb_checkpoints", 15 | ".mypy_cache", 16 | ".nox", 17 | ".pants.d", 18 | ".pyenv", 19 | ".pytest_cache", 20 | ".pytype", 21 | ".ruff_cache", 22 | ".svn", 23 | ".tox", 24 | ".venv", 25 | ".vscode", 26 | "__pypackages__", 27 | "_build", 28 | "buck-out", 29 | "build", 30 | "dist", 31 | "node_modules", 32 | "site-packages", 33 | "venv", 34 | ] 35 | 36 | # Format same as Black. 37 | line-length = 79 38 | indent-width = 4 39 | 40 | target-version = "py39" 41 | 42 | [tool.ruff.lint] 43 | # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. 44 | # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or 45 | # McCabe complexity (`C901`) by default. 46 | select = ["E4", "E7", "E9", "F", "I", "UP"] 47 | ignore = [] 48 | 49 | # Allow fix for all enabled rules (when `--fix`) is provided. 50 | fixable = ["ALL"] 51 | unfixable = [] 52 | 53 | # Allow unused variables when underscore-prefixed. 54 | dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" 55 | 56 | [tool.ruff.format] 57 | # Like Black, use double quotes for strings, spaces for indents 58 | # and trailing commas. 59 | quote-style = "preserve" 60 | indent-style = "space" 61 | skip-magic-trailing-comma = false 62 | line-ending = "auto" 63 | 64 | docstring-code-format = false 65 | docstring-code-line-length = "dynamic" 66 | -------------------------------------------------------------------------------- /requirements-dev-lock.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.9 3 | # by the following command: 4 | # 5 | # pip-compile --allow-unsafe --generate-hashes --output-file=requirements-dev-lock.txt requirements-dev.txt 6 | # 7 | atomicwrites==1.4.1 \ 8 | --hash=sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11 9 | # via -r requirements-dev.txt 10 | colorama==0.4.6 \ 11 | --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ 12 | --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 13 | # via -r requirements-dev.txt 14 | coverage[toml]==7.2.7 \ 15 | --hash=sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f \ 16 | --hash=sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2 \ 17 | --hash=sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a \ 18 | --hash=sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a \ 19 | --hash=sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01 \ 20 | --hash=sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6 \ 21 | --hash=sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7 \ 22 | --hash=sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f \ 23 | --hash=sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02 \ 24 | --hash=sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c \ 25 | --hash=sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063 \ 26 | --hash=sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a \ 27 | --hash=sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5 \ 28 | --hash=sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959 \ 29 | --hash=sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97 \ 30 | --hash=sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6 \ 31 | --hash=sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f \ 32 | --hash=sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9 \ 33 | --hash=sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5 \ 34 | --hash=sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f \ 35 | --hash=sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562 \ 36 | --hash=sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe \ 37 | --hash=sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9 \ 38 | --hash=sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f \ 39 | --hash=sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb \ 40 | --hash=sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb \ 41 | --hash=sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1 \ 42 | --hash=sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb \ 43 | --hash=sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250 \ 44 | --hash=sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e \ 45 | --hash=sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511 \ 46 | --hash=sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5 \ 47 | --hash=sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59 \ 48 | --hash=sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2 \ 49 | --hash=sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d \ 50 | --hash=sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3 \ 51 | --hash=sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4 \ 52 | --hash=sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de \ 53 | --hash=sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9 \ 54 | --hash=sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833 \ 55 | --hash=sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0 \ 56 | --hash=sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9 \ 57 | --hash=sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d \ 58 | --hash=sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050 \ 59 | --hash=sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d \ 60 | --hash=sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6 \ 61 | --hash=sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353 \ 62 | --hash=sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb \ 63 | --hash=sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e \ 64 | --hash=sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8 \ 65 | --hash=sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495 \ 66 | --hash=sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2 \ 67 | --hash=sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd \ 68 | --hash=sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27 \ 69 | --hash=sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1 \ 70 | --hash=sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818 \ 71 | --hash=sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4 \ 72 | --hash=sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e \ 73 | --hash=sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850 \ 74 | --hash=sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3 75 | # via 76 | # -r requirements-dev.txt 77 | # pytest-cov 78 | exceptiongroup==1.2.2 \ 79 | --hash=sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b \ 80 | --hash=sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc 81 | # via pytest 82 | iniconfig==2.0.0 \ 83 | --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ 84 | --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 85 | # via pytest 86 | packaging==24.1 \ 87 | --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ 88 | --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 89 | # via pytest 90 | pluggy==1.5.0 \ 91 | --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \ 92 | --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669 93 | # via pytest 94 | psutil==4.4.2 \ 95 | --hash=sha256:10fbb631142a3200623f4ab49f8bf82c32b79b8fe179f6056d01da3dfc589da1 \ 96 | --hash=sha256:15aba78f0262d7839702913f5d2ce1e97c89e31456bb26da1a5f9f7d7fe6d336 \ 97 | --hash=sha256:1c37e6428f7fe3aeea607f9249986d9bb933bb98133c7919837fd9aac4996b07 \ 98 | --hash=sha256:69e30d789c495b781f7cd47c13ee64452c58abfc7132d6dd1b389af312a78239 \ 99 | --hash=sha256:7481f299ae0e966a10cb8dd93a327efd8f51995d9bdc8810dcc65d3b12d856ee \ 100 | --hash=sha256:c2b0d8d1d8b5669b9884d0dd49ccb4094d163858d672d3d13a3fa817bc8a3197 \ 101 | --hash=sha256:d96d31d83781c7f3d0df8ccb1cc50650ca84d4722c5070b71ce8f1cc112e02e0 \ 102 | --hash=sha256:e423dd9cb12256c742d1d56ec38bc7d2a7fa09287c82c41e475e68b9f932c2af \ 103 | --hash=sha256:e44d6b758a96539e3e02336430d3f85263d43c470c5bad93572e9b6a86c67f76 104 | # via -r requirements-dev.txt 105 | pytest==8.1.1 \ 106 | --hash=sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7 \ 107 | --hash=sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044 108 | # via 109 | # -r requirements-dev.txt 110 | # pytest-cov 111 | pytest-cov==5.0.0 \ 112 | --hash=sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652 \ 113 | --hash=sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857 114 | # via -r requirements-dev.txt 115 | tabulate==0.7.5 \ 116 | --hash=sha256:9071aacbd97a9a915096c1aaf0dc684ac2672904cd876db5904085d6dac9810e 117 | # via -r requirements-dev.txt 118 | tomli==2.0.1 \ 119 | --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ 120 | --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f 121 | # via 122 | # coverage 123 | # pytest 124 | wheel==0.43.0 \ 125 | --hash=sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85 \ 126 | --hash=sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81 127 | # via -r requirements-dev.txt 128 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | psutil>=4.1.0,<5.0.0 2 | tabulate==0.7.5 3 | coverage==7.2.7 4 | wheel==0.43.0 5 | setuptools==71.1.0;python_version>="3.12" 6 | packaging==24.1;python_version>="3.12" # Requirement for setuptools>=71 7 | 8 | # Pytest specific deps 9 | pytest==8.1.1 10 | pytest-cov==5.0.0 11 | atomicwrites>=1.0 # Windows requirement 12 | colorama>0.3.0 # Windows requirement 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e git+https://github.com/boto/botocore.git@develop#egg=botocore 2 | -------------------------------------------------------------------------------- /s3transfer/compat.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import errno 14 | import inspect 15 | import os 16 | import socket 17 | import sys 18 | 19 | from botocore.compat import six 20 | 21 | if sys.platform.startswith('win'): 22 | def rename_file(current_filename, new_filename): 23 | try: 24 | os.remove(new_filename) 25 | except OSError as e: 26 | if not e.errno == errno.ENOENT: 27 | # We only want to a ignore trying to remove 28 | # a file that does not exist. If it fails 29 | # for any other reason we should be propagating 30 | # that exception. 31 | raise 32 | os.rename(current_filename, new_filename) 33 | else: 34 | rename_file = os.rename 35 | 36 | 37 | def accepts_kwargs(func): 38 | return inspect.getfullargspec(func)[2] 39 | 40 | 41 | # In python 3, socket.error is OSError, which is too general 42 | # for what we want (i.e FileNotFoundError is a subclass of OSError). 43 | # In python 3, all the socket related errors are in a newly created 44 | # ConnectionError. 45 | SOCKET_ERROR = ConnectionError 46 | MAXINT = None 47 | 48 | 49 | def seekable(fileobj): 50 | """Backwards compat function to determine if a fileobj is seekable 51 | 52 | :param fileobj: The file-like object to determine if seekable 53 | 54 | :returns: True, if seekable. False, otherwise. 55 | """ 56 | # If the fileobj has a seekable attr, try calling the seekable() 57 | # method on it. 58 | if hasattr(fileobj, 'seekable'): 59 | return fileobj.seekable() 60 | # If there is no seekable attr, check if the object can be seeked 61 | # or telled. If it can, try to seek to the current position. 62 | elif hasattr(fileobj, 'seek') and hasattr(fileobj, 'tell'): 63 | try: 64 | fileobj.seek(0, 1) 65 | return True 66 | except OSError: 67 | # If an io related error was thrown then it is not seekable. 68 | return False 69 | # Else, the fileobj is not seekable 70 | return False 71 | 72 | 73 | def readable(fileobj): 74 | """Determines whether or not a file-like object is readable. 75 | 76 | :param fileobj: The file-like object to determine if readable 77 | 78 | :returns: True, if readable. False otherwise. 79 | """ 80 | if hasattr(fileobj, 'readable'): 81 | return fileobj.readable() 82 | 83 | return hasattr(fileobj, 'read') 84 | 85 | 86 | def fallocate(fileobj, size): 87 | if hasattr(os, 'posix_fallocate'): 88 | os.posix_fallocate(fileobj.fileno(), 0, size) 89 | else: 90 | fileobj.truncate(size) 91 | 92 | 93 | # Import at end of file to avoid circular dependencies 94 | from multiprocessing.managers import BaseManager # noqa: F401,E402 95 | -------------------------------------------------------------------------------- /s3transfer/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import s3transfer 14 | 15 | KB = 1024 16 | MB = KB * KB 17 | GB = MB * KB 18 | 19 | ALLOWED_DOWNLOAD_ARGS = [ 20 | 'ChecksumMode', 21 | 'VersionId', 22 | 'SSECustomerAlgorithm', 23 | 'SSECustomerKey', 24 | 'SSECustomerKeyMD5', 25 | 'RequestPayer', 26 | 'ExpectedBucketOwner', 27 | ] 28 | 29 | FULL_OBJECT_CHECKSUM_ARGS = [ 30 | 'ChecksumCRC32', 31 | 'ChecksumCRC32C', 32 | 'ChecksumCRC64NVME', 33 | 'ChecksumSHA1', 34 | 'ChecksumSHA256', 35 | ] 36 | 37 | USER_AGENT = f's3transfer/{s3transfer.__version__}' 38 | PROCESS_USER_AGENT = f'{USER_AGENT} processpool' 39 | -------------------------------------------------------------------------------- /s3transfer/delete.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from s3transfer.tasks import SubmissionTask, Task 14 | 15 | 16 | class DeleteSubmissionTask(SubmissionTask): 17 | """Task for submitting tasks to execute an object deletion.""" 18 | 19 | def _submit(self, client, request_executor, transfer_future, **kwargs): 20 | """ 21 | :param client: The client associated with the transfer manager 22 | 23 | :type config: s3transfer.manager.TransferConfig 24 | :param config: The transfer config associated with the transfer 25 | manager 26 | 27 | :type osutil: s3transfer.utils.OSUtil 28 | :param osutil: The os utility associated to the transfer manager 29 | 30 | :type request_executor: s3transfer.futures.BoundedExecutor 31 | :param request_executor: The request executor associated with the 32 | transfer manager 33 | 34 | :type transfer_future: s3transfer.futures.TransferFuture 35 | :param transfer_future: The transfer future associated with the 36 | transfer request that tasks are being submitted for 37 | """ 38 | call_args = transfer_future.meta.call_args 39 | 40 | self._transfer_coordinator.submit( 41 | request_executor, 42 | DeleteObjectTask( 43 | transfer_coordinator=self._transfer_coordinator, 44 | main_kwargs={ 45 | 'client': client, 46 | 'bucket': call_args.bucket, 47 | 'key': call_args.key, 48 | 'extra_args': call_args.extra_args, 49 | }, 50 | is_final=True, 51 | ), 52 | ) 53 | 54 | 55 | class DeleteObjectTask(Task): 56 | def _main(self, client, bucket, key, extra_args): 57 | """ 58 | 59 | :param client: The S3 client to use when calling DeleteObject 60 | 61 | :type bucket: str 62 | :param bucket: The name of the bucket. 63 | 64 | :type key: str 65 | :param key: The name of the object to delete. 66 | 67 | :type extra_args: dict 68 | :param extra_args: Extra arguments to pass to the DeleteObject call. 69 | 70 | """ 71 | client.delete_object(Bucket=bucket, Key=key, **extra_args) 72 | -------------------------------------------------------------------------------- /s3transfer/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from concurrent.futures import CancelledError 14 | 15 | 16 | class RetriesExceededError(Exception): 17 | def __init__(self, last_exception, msg='Max Retries Exceeded'): 18 | super().__init__(msg) 19 | self.last_exception = last_exception 20 | 21 | 22 | class S3UploadFailedError(Exception): 23 | pass 24 | 25 | 26 | class S3DownloadFailedError(Exception): 27 | pass 28 | 29 | 30 | class InvalidSubscriberMethodError(Exception): 31 | pass 32 | 33 | 34 | class TransferNotDoneError(Exception): 35 | pass 36 | 37 | 38 | class FatalError(CancelledError): 39 | """A CancelledError raised from an error in the TransferManager""" 40 | 41 | pass 42 | -------------------------------------------------------------------------------- /s3transfer/subscribers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from functools import lru_cache 14 | 15 | from s3transfer.compat import accepts_kwargs 16 | from s3transfer.exceptions import InvalidSubscriberMethodError 17 | 18 | 19 | class BaseSubscriber: 20 | """The base subscriber class 21 | 22 | It is recommended that all subscriber implementations subclass and then 23 | override the subscription methods (i.e. on_{subsribe_type}() methods). 24 | """ 25 | 26 | VALID_SUBSCRIBER_TYPES = ['queued', 'progress', 'done'] 27 | 28 | def __new__(cls, *args, **kwargs): 29 | cls._validate_subscriber_methods() 30 | return super().__new__(cls) 31 | 32 | @classmethod 33 | @lru_cache 34 | def _validate_subscriber_methods(cls): 35 | for subscriber_type in cls.VALID_SUBSCRIBER_TYPES: 36 | subscriber_method = getattr(cls, 'on_' + subscriber_type) 37 | if not callable(subscriber_method): 38 | raise InvalidSubscriberMethodError( 39 | f'Subscriber method {subscriber_method} must be callable.' 40 | ) 41 | 42 | if not accepts_kwargs(subscriber_method): 43 | raise InvalidSubscriberMethodError( 44 | f'Subscriber method {subscriber_method} must accept keyword ' 45 | 'arguments (**kwargs)' 46 | ) 47 | 48 | def on_queued(self, future, **kwargs): 49 | """Callback to be invoked when transfer request gets queued 50 | 51 | This callback can be useful for: 52 | 53 | * Keeping track of how many transfers have been requested 54 | * Providing the expected transfer size through 55 | future.meta.provide_transfer_size() so a HeadObject would not 56 | need to be made for copies and downloads. 57 | 58 | :type future: s3transfer.futures.TransferFuture 59 | :param future: The TransferFuture representing the requested transfer. 60 | """ 61 | pass 62 | 63 | def on_progress(self, future, bytes_transferred, **kwargs): 64 | """Callback to be invoked when progress is made on transfer 65 | 66 | This callback can be useful for: 67 | 68 | * Recording and displaying progress 69 | 70 | :type future: s3transfer.futures.TransferFuture 71 | :param future: The TransferFuture representing the requested transfer. 72 | 73 | :type bytes_transferred: int 74 | :param bytes_transferred: The number of bytes transferred for that 75 | invocation of the callback. Note that a negative amount can be 76 | provided, which usually indicates that an in-progress request 77 | needed to be retried and thus progress was rewound. 78 | """ 79 | pass 80 | 81 | def on_done(self, future, **kwargs): 82 | """Callback to be invoked once a transfer is done 83 | 84 | This callback can be useful for: 85 | 86 | * Recording and displaying whether the transfer succeeded or 87 | failed using future.result() 88 | * Running some task after the transfer completed like changing 89 | the last modified time of a downloaded file. 90 | 91 | :type future: s3transfer.futures.TransferFuture 92 | :param future: The TransferFuture representing the requested transfer. 93 | """ 94 | pass 95 | -------------------------------------------------------------------------------- /scripts/ci/install: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import os 4 | import shutil 5 | from contextlib import contextmanager 6 | from subprocess import check_call 7 | 8 | _dname = os.path.dirname 9 | 10 | REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__)))) 11 | 12 | 13 | @contextmanager 14 | def cd(path): 15 | """Change directory while inside context manager.""" 16 | cwd = os.getcwd() 17 | try: 18 | os.chdir(path) 19 | yield 20 | finally: 21 | os.chdir(cwd) 22 | 23 | 24 | def run(command): 25 | return check_call(command, shell=True) 26 | 27 | 28 | if __name__ == "__main__": 29 | parser = argparse.ArgumentParser() 30 | group = parser.add_mutually_exclusive_group() 31 | group.add_argument( 32 | '-e', 33 | '--extras', 34 | help='Install extras_require along with normal install', 35 | ) 36 | args = parser.parse_args() 37 | with cd(REPO_ROOT): 38 | run("pip install -r requirements.txt") 39 | run('python scripts/ci/install-dev-deps') 40 | if os.path.isdir('dist') and os.listdir('dist'): 41 | shutil.rmtree('dist') 42 | run('python setup.py bdist_wheel') 43 | wheel_dist = os.listdir('dist')[0] 44 | package = os.path.join('dist', wheel_dist) 45 | if args.extras: 46 | package = f"'{package}[{args.extras}]'" 47 | run(f'pip install {package}') 48 | -------------------------------------------------------------------------------- /scripts/ci/install-dev-deps: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | from contextlib import contextmanager 5 | from subprocess import check_call 6 | 7 | _dname = os.path.dirname 8 | 9 | REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__)))) 10 | 11 | 12 | @contextmanager 13 | def cd(path): 14 | """Change directory while inside context manager.""" 15 | cwd = os.getcwd() 16 | try: 17 | os.chdir(path) 18 | yield 19 | finally: 20 | os.chdir(cwd) 21 | 22 | 23 | def run(command): 24 | return check_call(command, shell=True) 25 | 26 | 27 | if __name__ == "__main__": 28 | with cd(REPO_ROOT): 29 | if sys.version_info[:2] >= (3, 12): 30 | # Python 3.12+ no longer includes setuptools by default. 31 | 32 | # Setuptools 71+ now prefers already installed versions 33 | # of packaging _and_ broke the API for packaging<22.0. 34 | # We'll pin to match what's in requirements-dev.txt. 35 | run("pip install setuptools==71.1.0 packaging==24.1") 36 | 37 | run("pip install -r requirements-dev-lock.txt") 38 | -------------------------------------------------------------------------------- /scripts/ci/run-crt-tests: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Don't run tests from the root repo dir. 3 | # We want to ensure we're importing from the installed 4 | # binary package not from the CWD. 5 | 6 | import os 7 | import sys 8 | from contextlib import contextmanager 9 | from subprocess import check_call 10 | 11 | _dname = os.path.dirname 12 | 13 | REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__)))) 14 | 15 | 16 | @contextmanager 17 | def cd(path): 18 | """Change directory while inside context manager.""" 19 | cwd = os.getcwd() 20 | try: 21 | os.chdir(path) 22 | yield 23 | finally: 24 | os.chdir(cwd) 25 | 26 | 27 | def run(command, env=None): 28 | return check_call(command, shell=True, env=env) 29 | 30 | 31 | try: 32 | import awscrt # noqa: F401 33 | except ImportError: 34 | print("MISSING DEPENDENCY: awscrt must be installed to run the crt tests.") 35 | sys.exit(1) 36 | 37 | 38 | if __name__ == "__main__": 39 | with cd(os.path.join(REPO_ROOT, "tests")): 40 | run(f"{REPO_ROOT}/scripts/ci/run-tests unit/ functional/") 41 | -------------------------------------------------------------------------------- /scripts/ci/run-integ-tests: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Don't run tests from the root repo dir. 3 | # We want to ensure we're importing from the installed 4 | # binary package not from the CWD. 5 | 6 | import os 7 | from contextlib import contextmanager 8 | from subprocess import check_call 9 | 10 | _dname = os.path.dirname 11 | 12 | REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__)))) 13 | 14 | 15 | @contextmanager 16 | def cd(path): 17 | """Change directory while inside context manager.""" 18 | cwd = os.getcwd() 19 | try: 20 | os.chdir(path) 21 | yield 22 | finally: 23 | os.chdir(cwd) 24 | 25 | 26 | def run(command, env=None): 27 | return check_call(command, shell=True, env=env) 28 | 29 | 30 | if __name__ == "__main__": 31 | with cd(os.path.join(REPO_ROOT, "tests")): 32 | run(f"{REPO_ROOT}/scripts/ci/run-tests --with-cov integration") 33 | -------------------------------------------------------------------------------- /scripts/ci/run-tests: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Don't run tests from the root repo dir. 3 | # We want to ensure we're importing from the installed 4 | # binary package not from the CWD. 5 | 6 | import argparse 7 | import os 8 | from contextlib import contextmanager 9 | from subprocess import check_call 10 | 11 | _dname = os.path.dirname 12 | 13 | REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__)))) 14 | PACKAGE = "s3transfer" 15 | 16 | 17 | @contextmanager 18 | def cd(path): 19 | """Change directory while inside context manager.""" 20 | cwd = os.getcwd() 21 | try: 22 | os.chdir(path) 23 | yield 24 | finally: 25 | os.chdir(cwd) 26 | 27 | 28 | def run(command, env=None): 29 | return check_call(command, shell=True, env=env) 30 | 31 | 32 | def process_args(args): 33 | runner = args.test_runner 34 | test_args = "" 35 | if args.with_cov: 36 | test_args += f"--cov={PACKAGE} --cov-report xml " 37 | dirs = " ".join(args.test_dirs) 38 | 39 | return runner, test_args, dirs 40 | 41 | 42 | if __name__ == "__main__": 43 | parser = argparse.ArgumentParser() 44 | parser.add_argument( 45 | "test_dirs", 46 | default=["unit/", "functional/"], 47 | nargs="*", 48 | help="One or more directories containing tests.", 49 | ) 50 | parser.add_argument( 51 | "-r", 52 | "--test-runner", 53 | default="pytest", 54 | help="Test runner to execute tests. Defaults to pytest.", 55 | ) 56 | parser.add_argument( 57 | "-c", 58 | "--with-cov", 59 | default=False, 60 | action="store_true", 61 | help="Run default test-runner with code coverage enabled.", 62 | ) 63 | raw_args = parser.parse_args() 64 | test_runner, test_args, test_dirs = process_args(raw_args) 65 | 66 | with cd(os.path.join(REPO_ROOT, "tests")): 67 | cmd = f"{test_runner} {test_args}{test_dirs}" 68 | print(f"Running {cmd}...") 69 | run(cmd) 70 | 71 | # Run the serial implementation of s3transfer 72 | os.environ["USE_SERIAL_EXECUTOR"] = "True" 73 | print(f"Running serial execution for {cmd}...") 74 | run(cmd, env=os.environ) 75 | -------------------------------------------------------------------------------- /scripts/new-change: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Generate a new changelog entry. 3 | 4 | Usage 5 | ===== 6 | 7 | To generate a new changelog entry:: 8 | 9 | scripts/new-change 10 | 11 | This will open up a file in your editor (via the ``EDITOR`` env var). 12 | You'll see this template:: 13 | 14 | # Type should be one of: feature, bugfix 15 | type: 16 | 17 | # Category is the high level feature area. 18 | # This can be a service identifier (e.g ``s3``), 19 | # or something like: Paginator. 20 | category: 21 | 22 | # A brief description of the change. You can 23 | # use github style references to issues such as 24 | # "fixes #489", "boto/boto3#100", etc. These 25 | # will get automatically replaced with the correct 26 | # link. 27 | description: 28 | 29 | Fill in the appropriate values, save and exit the editor. 30 | Make sure to commit these changes as part of your pull request. 31 | 32 | If, when your editor is open, you decide don't don't want to add a changelog 33 | entry, save an empty file and no entry will be generated. 34 | 35 | You can then use the ``scripts/gen-changelog`` to generate the 36 | CHANGELOG.rst file. 37 | 38 | """ 39 | 40 | import argparse 41 | import json 42 | import os 43 | import random 44 | import re 45 | import string 46 | import subprocess 47 | import sys 48 | import tempfile 49 | 50 | VALID_CHARS = set(string.ascii_letters + string.digits) 51 | CHANGES_DIR = os.path.join( 52 | os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.changes' 53 | ) 54 | TEMPLATE = """\ 55 | # Type should be one of: feature, bugfix, enhancement, api-change 56 | # feature: A larger feature or change in behavior, usually resulting in a 57 | # minor version bump. 58 | # bugfix: Fixing a bug in an existing code path. 59 | # enhancement: Small change to an underlying implementation detail. 60 | # api-change: Changes to a modeled API. 61 | type: {change_type} 62 | 63 | # Category is the high level feature area. 64 | # This can be a service identifier (e.g ``s3``), 65 | # or something like: Paginator. 66 | category: {category} 67 | 68 | # A brief description of the change. You can 69 | # use github style references to issues such as 70 | # "fixes #489", "boto/boto3#100", etc. These 71 | # will get automatically replaced with the correct 72 | # link. 73 | description: {description} 74 | """ 75 | 76 | 77 | def new_changelog_entry(args): 78 | # Changelog values come from one of two places. 79 | # Either all values are provided on the command line, 80 | # or we open a text editor and let the user provide 81 | # enter their values. 82 | if all_values_provided(args): 83 | parsed_values = { 84 | 'type': args.change_type, 85 | 'category': args.category, 86 | 'description': args.description, 87 | } 88 | else: 89 | parsed_values = get_values_from_editor(args) 90 | if has_empty_values(parsed_values): 91 | sys.stderr.write( 92 | "Empty changelog values received, skipping entry creation.\n" 93 | ) 94 | return 1 95 | replace_issue_references(parsed_values, args.repo) 96 | write_new_change(parsed_values) 97 | return 0 98 | 99 | 100 | def has_empty_values(parsed_values): 101 | return not ( 102 | parsed_values.get('type') 103 | and parsed_values.get('category') 104 | and parsed_values.get('description') 105 | ) 106 | 107 | 108 | def all_values_provided(args): 109 | return args.change_type and args.category and args.description 110 | 111 | 112 | def get_values_from_editor(args): 113 | with tempfile.NamedTemporaryFile('w') as f: 114 | contents = TEMPLATE.format( 115 | change_type=args.change_type, 116 | category=args.category, 117 | description=args.description, 118 | ) 119 | f.write(contents) 120 | f.flush() 121 | env = os.environ 122 | editor = env.get('VISUAL', env.get('EDITOR', 'vim')) 123 | p = subprocess.Popen(f'{editor} {f.name}', shell=True) 124 | p.communicate() 125 | with open(f.name) as f: 126 | filled_in_contents = f.read() 127 | parsed_values = parse_filled_in_contents(filled_in_contents) 128 | return parsed_values 129 | 130 | 131 | def replace_issue_references(parsed, repo_name): 132 | description = parsed['description'] 133 | 134 | def linkify(match): 135 | number = match.group()[1:] 136 | return f'`{match.group()} `__' 137 | 138 | new_description = re.sub(r'#\d+', linkify, description) 139 | parsed['description'] = new_description 140 | 141 | 142 | def write_new_change(parsed_values): 143 | if not os.path.isdir(CHANGES_DIR): 144 | os.makedirs(CHANGES_DIR) 145 | # Assume that new changes go into the next release. 146 | dirname = os.path.join(CHANGES_DIR, 'next-release') 147 | if not os.path.isdir(dirname): 148 | os.makedirs(dirname) 149 | # Need to generate a unique filename for this change. 150 | # We'll try a couple things until we get a unique match. 151 | category = parsed_values['category'] 152 | short_summary = ''.join(filter(lambda x: x in VALID_CHARS, category)) 153 | filename = '{type_name}-{summary}'.format( 154 | type_name=parsed_values['type'], summary=short_summary 155 | ) 156 | possible_filename = os.path.join( 157 | dirname, f'{filename}-{str(random.randint(1, 100000))}.json' 158 | ) 159 | while os.path.isfile(possible_filename): 160 | possible_filename = os.path.join( 161 | dirname, f'{filename}-{str(random.randint(1, 100000))}.json' 162 | ) 163 | with open(possible_filename, 'w') as f: 164 | f.write(json.dumps(parsed_values, indent=2) + "\n") 165 | 166 | 167 | def parse_filled_in_contents(contents): 168 | """Parse filled in file contents and returns parsed dict. 169 | 170 | Return value will be:: 171 | { 172 | "type": "bugfix", 173 | "category": "category", 174 | "description": "This is a description" 175 | } 176 | 177 | """ 178 | if not contents.strip(): 179 | return {} 180 | parsed = {} 181 | lines = iter(contents.splitlines()) 182 | for line in lines: 183 | line = line.strip() 184 | if line.startswith('#'): 185 | continue 186 | if 'type' not in parsed and line.startswith('type:'): 187 | parsed['type'] = line.split(':')[1].strip() 188 | elif 'category' not in parsed and line.startswith('category:'): 189 | parsed['category'] = line.split(':')[1].strip() 190 | elif 'description' not in parsed and line.startswith('description:'): 191 | # Assume that everything until the end of the file is part 192 | # of the description, so we can break once we pull in the 193 | # remaining lines. 194 | first_line = line.split(':')[1].strip() 195 | full_description = '\n'.join([first_line] + list(lines)) 196 | parsed['description'] = full_description.strip() 197 | break 198 | return parsed 199 | 200 | 201 | def main(): 202 | parser = argparse.ArgumentParser() 203 | parser.add_argument( 204 | '-t', 205 | '--type', 206 | dest='change_type', 207 | default='', 208 | choices=('bugfix', 'feature', 'enhancement', 'api-change'), 209 | ) 210 | parser.add_argument('-c', '--category', dest='category', default='') 211 | parser.add_argument('-d', '--description', dest='description', default='') 212 | parser.add_argument( 213 | '-r', 214 | '--repo', 215 | default='boto/boto3', 216 | help='Optional repo name, e.g: boto/boto3', 217 | ) 218 | args = parser.parse_args() 219 | sys.exit(new_changelog_entry(args)) 220 | 221 | 222 | if __name__ == '__main__': 223 | main() 224 | -------------------------------------------------------------------------------- /scripts/performance/benchmark: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Use for benchmarking performance of other scripts. Provides data about 4 | time, memory use, cpu usage, network in, network out about the script ran in 5 | the form of a csv. 6 | 7 | 8 | Usage 9 | ===== 10 | 11 | NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running. 12 | 13 | To use the script, run:: 14 | 15 | ./benchmark "./my-script-to-run" 16 | 17 | 18 | If no ``--output-file`` was provided, the data will be saved to 19 | ``performance.csv`` 20 | """ 21 | 22 | import argparse 23 | import os 24 | import subprocess 25 | import sys 26 | import time 27 | 28 | import psutil 29 | 30 | # Determine the interface to track network IO depending on the platform. 31 | if sys.platform.startswith('linux'): 32 | INTERFACE = 'eth0' 33 | elif sys.platform == 'darwin': 34 | INTERFACE = 'en0' 35 | else: 36 | # TODO: Add support for windows. This would require figuring out what 37 | # interface to use on windows. 38 | raise RuntimeError(f'Script cannot be run on {sys.platform}') 39 | 40 | 41 | def benchmark(args): 42 | parent_pid = os.getpid() 43 | child_p = run_script(args) 44 | try: 45 | # Benchmark the process where the script is being ran. 46 | return run_benchmark(child_p.pid, args.output_file, args.data_interval) 47 | except KeyboardInterrupt: 48 | # If there is an interrupt, then try to clean everything up. 49 | proc = psutil.Process(parent_pid) 50 | procs = proc.children(recursive=True) 51 | 52 | for child in procs: 53 | child.terminate() 54 | 55 | gone, alive = psutil.wait_procs(procs, timeout=1) 56 | for child in alive: 57 | child.kill() 58 | return 1 59 | 60 | 61 | def run_script(args): 62 | return subprocess.Popen(args.script, shell=True) 63 | 64 | 65 | def run_benchmark(pid, output_file, data_interval): 66 | p = psutil.Process(pid) 67 | previous_net = psutil.net_io_counters(pernic=True)[INTERFACE] 68 | previous_time = time.time() 69 | 70 | with open(output_file, 'w') as f: 71 | while p.is_running(): 72 | if p.status() == psutil.STATUS_ZOMBIE: 73 | p.kill() 74 | break 75 | time.sleep(data_interval) 76 | process_to_measure = _get_underlying_python_process(p) 77 | try: 78 | # Collect the memory and cpu usage. 79 | memory_used = process_to_measure.memory_info().rss 80 | cpu_percent = process_to_measure.cpu_percent() 81 | current_net = psutil.net_io_counters(pernic=True)[INTERFACE] 82 | except (psutil.AccessDenied, psutil.ZombieProcess): 83 | # Trying to get process information from a closed or zombie process will 84 | # result in corresponding exceptions. 85 | break 86 | 87 | # Collect data on the in/out network io. 88 | sent_delta = current_net.bytes_sent - previous_net.bytes_sent 89 | recv_delta = current_net.bytes_recv - previous_net.bytes_recv 90 | 91 | # Determine the lapsed time to determine the network io rate. 92 | current_time = time.time() 93 | previous_net = current_net 94 | dt = current_time - previous_time 95 | previous_time = current_time 96 | sent_rate = sent_delta / dt 97 | recv_rate = recv_delta / dt 98 | 99 | # Save all of the data into a CSV file. 100 | f.write( 101 | f"{current_time},{memory_used},{cpu_percent}," 102 | f"{sent_rate},{recv_rate}\n" 103 | ) 104 | f.flush() 105 | return 0 106 | 107 | 108 | def _get_underlying_python_process(process): 109 | # For some scripts such as the streaming CLI commands, the process is 110 | # nested under a shell script that does not account for the python process. 111 | # We want to always be measuring the python process. 112 | children = process.children(recursive=True) 113 | for child_process in children: 114 | if 'python' in child_process.name().lower(): 115 | return child_process 116 | return process 117 | 118 | 119 | def main(): 120 | parser = argparse.ArgumentParser(usage=__doc__) 121 | parser.add_argument('script', help='The script to run for benchmarking') 122 | parser.add_argument( 123 | '--data-interval', 124 | default=1, 125 | type=float, 126 | help='The interval in seconds to poll for data points', 127 | ) 128 | parser.add_argument( 129 | '--output-file', 130 | default='performance.csv', 131 | help='The file to output the data collected to', 132 | ) 133 | args = parser.parse_args() 134 | return benchmark(args) 135 | 136 | 137 | if __name__ == '__main__': 138 | sys.exit(main()) 139 | -------------------------------------------------------------------------------- /scripts/performance/benchmark-download: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Benchmark the downloading of a file using s3transfer. You can also chose how 4 | type of file that is downloaded (i.e. filename, seekable, nonseekable). 5 | 6 | Usage 7 | ===== 8 | 9 | NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running. 10 | 11 | To benchmark with using a temporary file and key that is generated for you:: 12 | 13 | ./benchmark-download --file-size 10MB --file-type filename \\ 14 | --s3-bucket mybucket 15 | 16 | To benchmark with your own s3 key: 17 | 18 | ./benchmark-upload --existing-s3-key mykey --file-type filename \\ 19 | --s3-bucket mybucket 20 | 21 | """ 22 | 23 | import argparse 24 | import os 25 | import shutil 26 | import subprocess 27 | import tempfile 28 | 29 | from botocore.session import get_session 30 | 31 | from s3transfer.manager import TransferManager 32 | 33 | TEMP_FILE = 'temp' 34 | TEMP_KEY = 'temp' 35 | KB = 1024 36 | SIZE_SUFFIX = { 37 | 'kb': 1024, 38 | 'mb': 1024**2, 39 | 'gb': 1024**3, 40 | 'tb': 1024**4, 41 | 'kib': 1024, 42 | 'mib': 1024**2, 43 | 'gib': 1024**3, 44 | 'tib': 1024**4, 45 | } 46 | 47 | 48 | def human_readable_to_bytes(value): 49 | """Converts a human readable size to bytes. 50 | :param value: A string such as "10MB". If a suffix is not included, 51 | then the value is assumed to be an integer representing the size 52 | in bytes. 53 | :returns: The converted value in bytes as an integer 54 | """ 55 | value = value.lower() 56 | if value[-2:] == 'ib': 57 | # Assume IEC suffix. 58 | suffix = value[-3:].lower() 59 | else: 60 | suffix = value[-2:].lower() 61 | has_size_identifier = len(value) >= 2 and suffix in SIZE_SUFFIX 62 | if not has_size_identifier: 63 | try: 64 | return int(value) 65 | except ValueError: 66 | raise ValueError(f"Invalid size value: {value}") 67 | else: 68 | multiplier = SIZE_SUFFIX[suffix] 69 | return int(value[: -len(suffix)]) * multiplier 70 | 71 | 72 | def create_file(filename, file_size): 73 | with open(filename, 'wb') as f: 74 | for i in range(0, file_size, KB): 75 | f.write(b'a' * i) 76 | 77 | 78 | def benchmark_download(args): 79 | # Create a temporary directory to use for scratch work. 80 | tempdir = tempfile.mkdtemp() 81 | temp_file = os.path.join(tempdir, TEMP_FILE) 82 | 83 | if args.target_download: 84 | temp_file = os.path.abspath(os.path.expanduser(args.target_download)) 85 | session = get_session() 86 | client = session.create_client('s3') 87 | s3_key = args.existing_s3_key 88 | try: 89 | # If an existing s3 key was not specified, then create a temporary 90 | # file of that size for the user and upload it. 91 | if not args.existing_s3_key: 92 | # Create the temporary file. 93 | create_file(temp_file, args.file_size) 94 | 95 | # Create the temporary s3 key 96 | s3_key = TEMP_KEY 97 | upload_file(client, temp_file, args.s3_bucket) 98 | 99 | download_file_script = ( 100 | f'./download-file --file-name {temp_file} --file-type {args.file_type} --s3-bucket {args.s3_bucket} ' 101 | f'--s3-key {s3_key}' 102 | ) 103 | benchmark_args = ['./benchmark', download_file_script] 104 | if args.output_file: 105 | benchmark_args.extend(['--output-file', args.output_file]) 106 | subprocess.check_call(benchmark_args) 107 | finally: 108 | shutil.rmtree(tempdir) 109 | if not args.existing_s3_key: 110 | client.delete_object(Bucket=args.s3_bucket, Key=s3_key) 111 | 112 | 113 | def upload_file(client, filename, bucket): 114 | with TransferManager(client) as manager: 115 | manager.upload(filename, bucket, TEMP_KEY) 116 | 117 | 118 | def main(): 119 | parser = argparse.ArgumentParser() 120 | file_group = parser.add_mutually_exclusive_group(required=True) 121 | file_group.add_argument( 122 | '--file-size', 123 | type=human_readable_to_bytes, 124 | help=( 125 | 'The size of the temporary file to create and then upload to s3. ' 126 | 'You can also specify your own key with --existing-s3-key to ' 127 | 'avoid going through this setup step.' 128 | ), 129 | ) 130 | parser.add_argument( 131 | '--file-type', 132 | choices=['filename', 'seekable', 'nonseekable'], 133 | required=True, 134 | help='The way to represent the file when downloading', 135 | ) 136 | parser.add_argument( 137 | '--s3-bucket', 138 | required=True, 139 | help='The S3 bucket to download the file to', 140 | ) 141 | file_group.add_argument( 142 | '--existing-s3-key', 143 | help=( 144 | 'The existing s3 key to download from. You can also use ' 145 | '--file-size to create a temporary file and key to download from.' 146 | ), 147 | ) 148 | parser.add_argument( 149 | '--target-download', 150 | help=( 151 | 'The filename to download to. Note that this file will ' 152 | 'always be cleaned up for you.' 153 | ), 154 | ) 155 | parser.add_argument( 156 | '--output-file', 157 | help=( 158 | 'The file to output the data collected to. The default ' 159 | 'location performance.csv' 160 | ), 161 | ) 162 | args = parser.parse_args() 163 | benchmark_download(args) 164 | 165 | 166 | if __name__ == '__main__': 167 | main() 168 | -------------------------------------------------------------------------------- /scripts/performance/benchmark-upload: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Benchmark the uploading of a file using s3transfer. You can also chose how 4 | type of file that is uploaded (i.e. filename, seekable, nonseekable). 5 | 6 | Usage 7 | ===== 8 | 9 | NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running. 10 | 11 | To benchmark with using a temporary file that is generated for you:: 12 | 13 | ./benchmark-upload --file-size 10MB --file-type filename \\ 14 | --s3-bucket mybucket 15 | 16 | To benchmark with your own local file:: 17 | 18 | ./benchmark-upload --source-file myfile --file-type filename \\ 19 | --s3-bucket mybucket 20 | 21 | """ 22 | 23 | import argparse 24 | import os 25 | import shutil 26 | import subprocess 27 | import tempfile 28 | 29 | from botocore.session import get_session 30 | 31 | TEMP_FILE = 'temp' 32 | TEMP_KEY = 'temp' 33 | KB = 1024 34 | SIZE_SUFFIX = { 35 | 'kb': 1024, 36 | 'mb': 1024**2, 37 | 'gb': 1024**3, 38 | 'tb': 1024**4, 39 | 'kib': 1024, 40 | 'mib': 1024**2, 41 | 'gib': 1024**3, 42 | 'tib': 1024**4, 43 | } 44 | 45 | 46 | def human_readable_to_bytes(value): 47 | """Converts a human readable size to bytes. 48 | :param value: A string such as "10MB". If a suffix is not included, 49 | then the value is assumed to be an integer representing the size 50 | in bytes. 51 | :returns: The converted value in bytes as an integer 52 | """ 53 | value = value.lower() 54 | if value[-2:] == 'ib': 55 | # Assume IEC suffix. 56 | suffix = value[-3:].lower() 57 | else: 58 | suffix = value[-2:].lower() 59 | has_size_identifier = len(value) >= 2 and suffix in SIZE_SUFFIX 60 | if not has_size_identifier: 61 | try: 62 | return int(value) 63 | except ValueError: 64 | raise ValueError(f"Invalid size value: {value}") 65 | else: 66 | multiplier = SIZE_SUFFIX[suffix] 67 | return int(value[: -len(suffix)]) * multiplier 68 | 69 | 70 | def create_file(filename, file_size): 71 | with open(filename, 'wb') as f: 72 | for i in range(0, file_size, KB): 73 | f.write(b'a' * i) 74 | 75 | 76 | def benchmark_upload(args): 77 | source_file = args.source_file 78 | session = get_session() 79 | client = session.create_client('s3') 80 | tempdir = None 81 | try: 82 | # If a source file was not specified, then create a temporary file of 83 | # that size for the user. 84 | if not source_file: 85 | tempdir = tempfile.mkdtemp() 86 | source_file = os.path.join(tempdir, TEMP_FILE) 87 | create_file(source_file, args.file_size) 88 | 89 | upload_file_script = ( 90 | f'./upload-file --file-name {source_file} --file-type {args.file_type} --s3-bucket {args.s3_bucket} ' 91 | f'--s3-key {TEMP_KEY}' 92 | ) 93 | benchmark_args = ['./benchmark', upload_file_script] 94 | if args.output_file: 95 | benchmark_args.extend(['--output-file', args.output_file]) 96 | subprocess.check_call(benchmark_args) 97 | finally: 98 | if tempdir: 99 | shutil.rmtree(tempdir) 100 | client.delete_object(Bucket=args.s3_bucket, Key=TEMP_KEY) 101 | 102 | 103 | def main(): 104 | parser = argparse.ArgumentParser(usage=__doc__) 105 | source_file_group = parser.add_mutually_exclusive_group(required=True) 106 | source_file_group.add_argument( 107 | '--source-file', 108 | help=( 109 | 'The local file to upload. Note this is optional. You can also ' 110 | 'use --file-size which will create a temporary file for you.' 111 | ), 112 | ) 113 | source_file_group.add_argument( 114 | '--file-size', 115 | type=human_readable_to_bytes, 116 | help=( 117 | 'The size of the temporary file to create. You can also specify ' 118 | 'your own file with --source-file' 119 | ), 120 | ) 121 | parser.add_argument( 122 | '--file-type', 123 | choices=['filename', 'seekable', 'nonseekable'], 124 | required=True, 125 | help='The way to represent the file when uploading', 126 | ) 127 | parser.add_argument( 128 | '--s3-bucket', 129 | required=True, 130 | help='The S3 bucket to upload the file to', 131 | ) 132 | parser.add_argument( 133 | '--output-file', 134 | help=( 135 | 'The file to output the data collected to. The default ' 136 | 'location performance.csv' 137 | ), 138 | ) 139 | args = parser.parse_args() 140 | benchmark_upload(args) 141 | 142 | 143 | if __name__ == '__main__': 144 | main() 145 | -------------------------------------------------------------------------------- /scripts/performance/download-file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Downloads a file using s3transfer. You can also chose how type of file that 4 | is downloaded (i.e. filename, seekable, nonseekable). 5 | 6 | Usage 7 | ===== 8 | 9 | NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running. 10 | 11 | To download a file:: 12 | 13 | ./download-file --file-name myfilename --file-type filename \\ 14 | --s3-bucket mybucket --s3-key mykey 15 | 16 | """ 17 | 18 | import argparse 19 | 20 | from botocore.session import get_session 21 | 22 | from s3transfer.manager import TransferManager 23 | 24 | 25 | class NonSeekableWriter: 26 | """A wrapper to hide the ability to seek for a fileobj""" 27 | 28 | def __init__(self, fileobj): 29 | self._fileobj = fileobj 30 | 31 | def write(self, b): 32 | return self._fileobj.write(b) 33 | 34 | 35 | class Downloader: 36 | def download(self, args): 37 | session = get_session() 38 | client = session.create_client('s3') 39 | file_type = args.file_type 40 | if args.debug: 41 | session.set_debug_logger('') 42 | with TransferManager(client) as manager: 43 | getattr(self, 'download_' + file_type)( 44 | manager, args.file_name, args.s3_bucket, args.s3_key 45 | ) 46 | 47 | def download_filename(self, manager, filename, bucket, s3_key): 48 | manager.download(bucket, s3_key, filename) 49 | 50 | def download_seekable(self, manager, filename, bucket, s3_key): 51 | with open(filename, 'wb') as f: 52 | future = manager.download(bucket, s3_key, f) 53 | future.result() 54 | 55 | def download_nonseekable(self, manager, filename, bucket, s3_key): 56 | with open(filename, 'wb') as f: 57 | future = manager.download(bucket, s3_key, NonSeekableWriter(f)) 58 | future.result() 59 | 60 | 61 | def main(): 62 | parser = argparse.ArgumentParser(usage=__doc__) 63 | parser.add_argument('--file-name', required=True, help='The name of file') 64 | parser.add_argument( 65 | '--file-type', 66 | choices=['filename', 'seekable', 'nonseekable'], 67 | required=True, 68 | help='The way to represent the file when downloading', 69 | ) 70 | parser.add_argument( 71 | '--s3-bucket', 72 | required=True, 73 | help='The S3 bucket to download the file to', 74 | ) 75 | parser.add_argument( 76 | '--s3-key', required=True, help='The key to download to' 77 | ) 78 | parser.add_argument( 79 | '--debug', 80 | action='store_true', 81 | help='Whether to turn debugging on. This will get printed to stderr', 82 | ) 83 | args = parser.parse_args() 84 | Downloader().download(args) 85 | 86 | 87 | if __name__ == '__main__': 88 | main() 89 | -------------------------------------------------------------------------------- /scripts/performance/processpool-download: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Downloads using s3transfer.processpool.ProcessPoolDownloader 4 | 5 | Usage 6 | ===== 7 | 8 | NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running. 9 | 10 | To download a file:: 11 | 12 | ./proccesspool-download -f myfilename -b mybucket -k mykey 13 | 14 | To download a prefix recursively to a directory:: 15 | 16 | ./proccesspool-download -d mydirname -b mybucket -p myprefix/ 17 | 18 | """ 19 | 20 | import argparse 21 | import os 22 | 23 | import botocore.session 24 | 25 | from s3transfer.processpool import ProcessPoolDownloader, ProcessTransferConfig 26 | 27 | MB = 1024 * 1024 28 | 29 | 30 | def download(bucket, key, filename, num_processes, mb_chunksize): 31 | config = ProcessTransferConfig( 32 | multipart_chunksize=mb_chunksize * MB, 33 | max_request_processes=num_processes, 34 | ) 35 | with ProcessPoolDownloader(config=config) as downloader: 36 | future = downloader.download_file( 37 | bucket=bucket, key=key, filename=filename 38 | ) 39 | future.result() 40 | 41 | 42 | def recursive_download(bucket, prefix, dirname, num_processes, mb_chunksize): 43 | config = ProcessTransferConfig( 44 | multipart_chunksize=mb_chunksize * MB, 45 | max_request_processes=num_processes, 46 | ) 47 | s3 = botocore.session.get_session().create_client('s3') 48 | with ProcessPoolDownloader(config=config) as downloader: 49 | paginator = s3.get_paginator('list_objects') 50 | for response in paginator.paginate(Bucket=bucket, Prefix=prefix): 51 | contents = response.get('Contents', []) 52 | for content in contents: 53 | key = content['Key'] 54 | filename = os.path.join(dirname, key[len(prefix) :]) 55 | parent_dirname = os.path.dirname(filename) 56 | if not os.path.exists(parent_dirname): 57 | os.makedirs(parent_dirname) 58 | # An expected size is provided so an additional HeadObject 59 | # does not need to be made for each of these objects that 60 | # get downloaded. 61 | downloader.download_file( 62 | bucket, 63 | key, 64 | filename=filename, 65 | expected_size=content['Size'], 66 | ) 67 | 68 | 69 | def main(): 70 | parser = argparse.ArgumentParser(usage=__doc__) 71 | parser.add_argument( 72 | '-b', '--bucket', required=True, help='The S3 bucket to download from' 73 | ) 74 | single_file_group = parser.add_argument_group('Single file downloads') 75 | single_file_group.add_argument( 76 | '-k', '--key', help='The key to download from' 77 | ) 78 | single_file_group.add_argument( 79 | '-f', '--filename', help='The name of file to download to' 80 | ) 81 | recursive_file_group = parser.add_argument_group( 82 | 'Recursive file downloads' 83 | ) 84 | recursive_file_group.add_argument( 85 | '-p', '--prefix', help='The prefix to download from' 86 | ) 87 | recursive_file_group.add_argument( 88 | '-d', '--dirname', help='The directory to download to' 89 | ) 90 | parser.add_argument( 91 | '-n', 92 | '--num-processes', 93 | type=int, 94 | default=10, 95 | help='The number of processes to run the download. 10 by default.', 96 | ) 97 | parser.add_argument( 98 | '-c', 99 | '--mb-chunksize', 100 | type=int, 101 | default=8, 102 | help='The part size in MB to use for the download. 8 MB by default.', 103 | ) 104 | args = parser.parse_args() 105 | if args.filename and args.key: 106 | download( 107 | args.bucket, 108 | args.key, 109 | args.filename, 110 | args.num_processes, 111 | args.mb_chunksize, 112 | ) 113 | elif args.prefix and args.dirname: 114 | recursive_download( 115 | args.bucket, 116 | args.prefix, 117 | args.dirname, 118 | args.num_processes, 119 | args.mb_chunksize, 120 | ) 121 | else: 122 | raise ValueError( 123 | 'Either --key and --filename must be provided or ' 124 | '--prefix and --dirname must be provided.' 125 | ) 126 | 127 | 128 | if __name__ == '__main__': 129 | main() 130 | -------------------------------------------------------------------------------- /scripts/performance/summarize: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Summarizes results of benchmarking. 4 | 5 | Usage 6 | ===== 7 | 8 | Run this script with:: 9 | 10 | ./summarize performance.csv 11 | 12 | 13 | And that should output:: 14 | 15 | +------------------------+----------+----------------------+ 16 | | Metric over 1 run(s) | Mean | Standard Deviation | 17 | +========================+==========+======================+ 18 | | Total Time (seconds) | 1.200 | 0.0 | 19 | +------------------------+----------+----------------------+ 20 | | Maximum Memory | 42.3 MiB | 0 Bytes | 21 | +------------------------+----------+----------------------+ 22 | | Maximum CPU (percent) | 88.1 | 0.0 | 23 | +------------------------+----------+----------------------+ 24 | | Average Memory | 33.9 MiB | 0 Bytes | 25 | +------------------------+----------+----------------------+ 26 | | Average CPU (percent) | 30.5 | 0.0 | 27 | +------------------------+----------+----------------------+ 28 | 29 | 30 | The script can also be ran with multiple files: 31 | 32 | ./summarize performance.csv performance-2.csv 33 | 34 | And will have a similar output: 35 | 36 | +------------------------+----------+----------------------+ 37 | | Metric over 2 run(s) | Mean | Standard Deviation | 38 | +========================+==========+======================+ 39 | | Total Time (seconds) | 1.155 | 0.0449999570847 | 40 | +------------------------+----------+----------------------+ 41 | | Maximum Memory | 42.5 MiB | 110.0 KiB | 42 | +------------------------+----------+----------------------+ 43 | | Maximum CPU (percent) | 94.5 | 6.45 | 44 | +------------------------+----------+----------------------+ 45 | | Average Memory | 35.6 MiB | 1.7 MiB | 46 | +------------------------+----------+----------------------+ 47 | | Average CPU (percent) | 27.5 | 3.03068181818 | 48 | +------------------------+----------+----------------------+ 49 | 50 | 51 | You can also specify the ``--output-format json`` option to print the 52 | summary as JSON instead of a pretty printed table:: 53 | 54 | { 55 | "total_time": 72.76999998092651, 56 | "std_dev_average_memory": 0.0, 57 | "std_dev_total_time": 0.0, 58 | "average_memory": 56884518.57534247, 59 | "std_dev_average_cpu": 0.0, 60 | "std_dev_max_memory": 0.0, 61 | "average_cpu": 61.19315068493151, 62 | "max_memory": 58331136.0 63 | } 64 | 65 | """ 66 | 67 | import argparse 68 | import csv 69 | import json 70 | from math import sqrt 71 | 72 | from tabulate import tabulate 73 | 74 | 75 | def human_readable_size(value): 76 | """Converts integer values in bytes to human readable values""" 77 | hummanize_suffixes = ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB') 78 | base = 1024 79 | bytes_int = float(value) 80 | 81 | if bytes_int == 1: 82 | return '1 Byte' 83 | elif bytes_int < base: 84 | return '%d Bytes' % bytes_int 85 | 86 | for i, suffix in enumerate(hummanize_suffixes): 87 | unit = base ** (i + 2) 88 | if round((bytes_int / unit) * base) < base: 89 | return f'{(base * bytes_int / unit):.1f} {suffix}' 90 | 91 | 92 | class Summarizer: 93 | DATA_INDEX_IN_ROW = {'time': 0, 'memory': 1, 'cpu': 2} 94 | 95 | def __init__(self): 96 | self.total_files = 0 97 | self._num_rows = 0 98 | self._start_time = None 99 | self._end_time = None 100 | self._totals = { 101 | 'time': [], 102 | 'average_memory': [], 103 | 'average_cpu': [], 104 | 'max_memory': [], 105 | 'max_cpu': [], 106 | } 107 | self._averages = { 108 | 'memory': 0.0, 109 | 'cpu': 0.0, 110 | } 111 | self._maximums = {'memory': 0.0, 'cpu': 0.0} 112 | 113 | @property 114 | def total_time(self): 115 | return self._average_across_all_files('time') 116 | 117 | @property 118 | def max_cpu(self): 119 | return self._average_across_all_files('max_cpu') 120 | 121 | @property 122 | def max_memory(self): 123 | return self._average_across_all_files('max_memory') 124 | 125 | @property 126 | def average_cpu(self): 127 | return self._average_across_all_files('average_cpu') 128 | 129 | @property 130 | def average_memory(self): 131 | return self._average_across_all_files('average_memory') 132 | 133 | @property 134 | def std_dev_total_time(self): 135 | return self._standard_deviation_across_all_files('time') 136 | 137 | @property 138 | def std_dev_max_cpu(self): 139 | return self._standard_deviation_across_all_files('max_cpu') 140 | 141 | @property 142 | def std_dev_max_memory(self): 143 | return self._standard_deviation_across_all_files('max_memory') 144 | 145 | @property 146 | def std_dev_average_cpu(self): 147 | return self._standard_deviation_across_all_files('average_cpu') 148 | 149 | @property 150 | def std_dev_average_memory(self): 151 | return self._standard_deviation_across_all_files('average_memory') 152 | 153 | def _average_across_all_files(self, name): 154 | return sum(self._totals[name]) / len(self._totals[name]) 155 | 156 | def _standard_deviation_across_all_files(self, name): 157 | mean = self._average_across_all_files(name) 158 | differences = [total - mean for total in self._totals[name]] 159 | sq_differences = [difference**2 for difference in differences] 160 | return sqrt(sum(sq_differences) / len(self._totals[name])) 161 | 162 | def summarize_as_table(self): 163 | """Formats the processed data as pretty printed table. 164 | 165 | :return: str of formatted table 166 | """ 167 | h = human_readable_size 168 | table = [ 169 | [ 170 | 'Total Time (seconds)', 171 | f'{self.total_time:.3f}', 172 | self.std_dev_total_time, 173 | ], 174 | ['Maximum Memory', h(self.max_memory), h(self.std_dev_max_memory)], 175 | [ 176 | 'Maximum CPU (percent)', 177 | f'{self.max_cpu:.1f}', 178 | self.std_dev_max_cpu, 179 | ], 180 | [ 181 | 'Average Memory', 182 | h(self.average_memory), 183 | h(self.std_dev_average_memory), 184 | ], 185 | [ 186 | 'Average CPU (percent)', 187 | f'{self.average_cpu:.1f}', 188 | self.std_dev_average_cpu, 189 | ], 190 | ] 191 | return tabulate( 192 | table, 193 | headers=[ 194 | f'Metric over {self.total_files} run(s)', 195 | 'Mean', 196 | 'Standard Deviation', 197 | ], 198 | tablefmt="grid", 199 | ) 200 | 201 | def summarize_as_json(self): 202 | """Return JSON summary of processed data. 203 | 204 | :return: str of formatted JSON 205 | """ 206 | return json.dumps( 207 | { 208 | 'total_time': self.total_time, 209 | 'std_dev_total_time': self.std_dev_total_time, 210 | 'max_memory': self.max_memory, 211 | 'std_dev_max_memory': self.std_dev_max_memory, 212 | 'average_memory': self.average_memory, 213 | 'std_dev_average_memory': self.std_dev_average_memory, 214 | 'std_dev_max_cpu': self.std_dev_max_cpu, 215 | 'max_cpu': self.max_cpu, 216 | 'average_cpu': self.average_cpu, 217 | 'std_dev_average_cpu': self.std_dev_average_cpu, 218 | }, 219 | indent=2, 220 | ) 221 | 222 | def process(self, args): 223 | """Processes the data from the CSV file""" 224 | for benchmark_file in args.benchmark_files: 225 | self.process_individual_file(benchmark_file) 226 | self.total_files += 1 227 | 228 | def process_individual_file(self, benchmark_file): 229 | with open(benchmark_file) as f: 230 | reader = csv.reader(f) 231 | # Process each row from the CSV file 232 | row = None 233 | for row in reader: 234 | self._validate_row(row, benchmark_file) 235 | self.process_data_row(row) 236 | self._validate_row(row, benchmark_file) 237 | self._end_time = self._get_time(row) 238 | self._finalize_processed_data_for_file() 239 | 240 | def _validate_row(self, row, filename): 241 | if not row: 242 | raise RuntimeError( 243 | f'Row: {row} could not be processed. The CSV file ({filename}) may be ' 244 | 'empty.' 245 | ) 246 | 247 | def process_data_row(self, row): 248 | # If the row is the first row collect the start time. 249 | if self._num_rows == 0: 250 | self._start_time = self._get_time(row) 251 | self._num_rows += 1 252 | self.process_data_point(row, 'memory') 253 | self.process_data_point(row, 'cpu') 254 | 255 | def process_data_point(self, row, name): 256 | # Determine where in the CSV row the requested data is located. 257 | index = self.DATA_INDEX_IN_ROW[name] 258 | # Get the data point. 259 | data_point = float(row[index]) 260 | self._add_to_average(name, data_point) 261 | self._account_for_maximum(name, data_point) 262 | 263 | def _finalize_processed_data_for_file(self): 264 | # Add numbers to the total, which keeps track of data over 265 | # all files provided. 266 | self._totals['time'].append(self._end_time - self._start_time) 267 | self._totals['max_cpu'].append(self._maximums['cpu']) 268 | self._totals['max_memory'].append(self._maximums['memory']) 269 | self._totals['average_cpu'].append( 270 | self._averages['cpu'] / self._num_rows 271 | ) 272 | self._totals['average_memory'].append( 273 | self._averages['memory'] / self._num_rows 274 | ) 275 | 276 | # Reset some of the data needed to be tracked for each specific 277 | # file. 278 | self._num_rows = 0 279 | self._maximums = self._maximums.fromkeys(self._maximums, 0.0) 280 | self._averages = self._averages.fromkeys(self._averages, 0.0) 281 | 282 | def _get_time(self, row): 283 | return float(row[self.DATA_INDEX_IN_ROW['time']]) 284 | 285 | def _add_to_average(self, name, data_point): 286 | self._averages[name] += data_point 287 | 288 | def _account_for_maximum(self, name, data_point): 289 | if data_point > self._maximums[name]: 290 | self._maximums[name] = data_point 291 | 292 | 293 | def main(): 294 | parser = argparse.ArgumentParser(usage=__doc__) 295 | parser.add_argument( 296 | 'benchmark_files', 297 | nargs='+', 298 | help=( 299 | 'The CSV output file from the benchmark script. If you provide' 300 | 'more than one of these files, it will give you the average ' 301 | 'across all of the files for each metric.' 302 | ), 303 | ) 304 | parser.add_argument( 305 | '-f', 306 | '--output-format', 307 | default='table', 308 | choices=['table', 'json'], 309 | help=( 310 | 'Specify what output format to use for displaying results. ' 311 | 'By default, a pretty printed table is used, but you can also ' 312 | 'specify "json" to display pretty printed JSON.' 313 | ), 314 | ) 315 | args = parser.parse_args() 316 | summarizer = Summarizer() 317 | summarizer.process(args) 318 | if args.output_format == 'table': 319 | result = summarizer.summarize_as_table() 320 | else: 321 | result = summarizer.summarize_as_json() 322 | print(result) 323 | 324 | 325 | if __name__ == '__main__': 326 | main() 327 | -------------------------------------------------------------------------------- /scripts/performance/upload-file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Uploads a file using s3transfer. You can also chose how type of file that 4 | is uploaded (i.e. filename, seekable, nonseekable). 5 | 6 | Usage 7 | ===== 8 | 9 | NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running. 10 | 11 | To upload a file:: 12 | 13 | ./upload-file --file-name myfilename --file-type filename \\ 14 | --s3-bucket mybucket --s3-key mykey 15 | 16 | """ 17 | 18 | import argparse 19 | 20 | from botocore.session import get_session 21 | 22 | from s3transfer.manager import TransferManager 23 | 24 | 25 | class NonSeekableReader: 26 | """A wrapper to hide the ability to seek for a fileobj""" 27 | 28 | def __init__(self, fileobj): 29 | self._fileobj = fileobj 30 | 31 | def read(self, amt=-1): 32 | return self._fileobj.read(amt) 33 | 34 | 35 | class Uploader: 36 | def upload(self, args): 37 | session = get_session() 38 | client = session.create_client('s3') 39 | file_type = args.file_type 40 | if args.debug: 41 | session.set_debug_logger('') 42 | with TransferManager(client) as manager: 43 | getattr(self, 'upload_' + file_type)( 44 | manager, args.file_name, args.s3_bucket, args.s3_key 45 | ) 46 | 47 | def upload_filename(self, manager, filename, bucket, s3_key): 48 | manager.upload(filename, bucket, s3_key) 49 | 50 | def upload_seekable(self, manager, filename, bucket, s3_key): 51 | with open(filename, 'rb') as f: 52 | future = manager.upload(f, bucket, s3_key) 53 | future.result() 54 | 55 | def upload_nonseekable(self, manager, filename, bucket, s3_key): 56 | with open(filename, 'rb') as f: 57 | future = manager.upload(NonSeekableReader(f), bucket, s3_key) 58 | future.result() 59 | 60 | 61 | def main(): 62 | parser = argparse.ArgumentParser(usage=__doc__) 63 | parser.add_argument('--file-name', required=True, help='The name of file') 64 | parser.add_argument( 65 | '--file-type', 66 | choices=['filename', 'seekable', 'nonseekable'], 67 | required=True, 68 | help='The way to represent the file when uploading', 69 | ) 70 | parser.add_argument( 71 | '--s3-bucket', 72 | required=True, 73 | help='The S3 bucket to upload the file to', 74 | ) 75 | parser.add_argument('--s3-key', required=True, help='The key to upload to') 76 | parser.add_argument( 77 | '--debug', 78 | action='store_true', 79 | help='Whether to turn debugging on. This will get printed to stderr', 80 | ) 81 | args = parser.parse_args() 82 | Uploader().upload(args) 83 | 84 | 85 | if __name__ == '__main__': 86 | main() 87 | -------------------------------------------------------------------------------- /scripts/stress/timeout: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Use to put a timeout on the length of time a script can run. This is 4 | especially useful for checking for scripts that hang. 5 | 6 | Usage 7 | ===== 8 | 9 | NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running. 10 | 11 | To use the script, run:: 12 | 13 | ./timeout "./my-script-to-run" --timeout-after 5 14 | 15 | """ 16 | 17 | import argparse 18 | import os 19 | import subprocess 20 | import sys 21 | import time 22 | 23 | import psutil 24 | 25 | 26 | class TimeoutException(Exception): 27 | def __init__(self, timeout_len): 28 | msg = f'Script failed to complete within {timeout_len} seconds' 29 | Exception.__init__(self, msg) 30 | 31 | 32 | def timeout(args): 33 | parent_pid = os.getpid() 34 | child_p = run_script(args) 35 | try: 36 | run_timeout(child_p.pid, args.timeout_after) 37 | except (TimeoutException, KeyboardInterrupt) as e: 38 | proc = psutil.Process(parent_pid) 39 | procs = proc.children(recursive=True) 40 | 41 | for child in procs: 42 | child.terminate() 43 | 44 | gone, alive = psutil.wait_procs(procs, timeout=1) 45 | for child in alive: 46 | child.kill() 47 | raise e 48 | 49 | 50 | def run_timeout(pid, timeout_len): 51 | p = psutil.Process(pid) 52 | start_time = time.time() 53 | while p.is_running(): 54 | if p.status() == psutil.STATUS_ZOMBIE: 55 | p.kill() 56 | break 57 | current_time = time.time() 58 | # Raise a timeout if the duration of the process is longer than 59 | # the desired timeout. 60 | if current_time - start_time > timeout_len: 61 | raise TimeoutException(timeout_len) 62 | time.sleep(1) 63 | 64 | 65 | def run_script(args): 66 | return subprocess.Popen(args.script, shell=True) 67 | 68 | 69 | def main(): 70 | parser = argparse.ArgumentParser(usage=__doc__) 71 | parser.add_argument('script', help='The script to run for benchmarking') 72 | parser.add_argument( 73 | '--timeout-after', 74 | required=True, 75 | type=float, 76 | help=( 77 | 'The length of time in seconds allowed for the script to run ' 78 | 'before it time\'s out.' 79 | ), 80 | ) 81 | args = parser.parse_args() 82 | return timeout(args) 83 | 84 | 85 | if __name__ == '__main__': 86 | sys.exit(main()) 87 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 0 3 | 4 | [metadata] 5 | requires_dist = 6 | botocore>=1.37.4,<2.0a.0 7 | 8 | [options.extras_require] 9 | crt = botocore[crt]>=1.37.4,<2.0a0 10 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import re 4 | 5 | from setuptools import find_packages, setup 6 | 7 | ROOT = os.path.dirname(__file__) 8 | VERSION_RE = re.compile(r'''__version__ = ['"]([0-9.]+)['"]''') 9 | 10 | 11 | requires = [ 12 | 'botocore>=1.37.4,<2.0a.0', 13 | ] 14 | 15 | 16 | def get_version(): 17 | init = open(os.path.join(ROOT, 's3transfer', '__init__.py')).read() 18 | return VERSION_RE.search(init).group(1) 19 | 20 | 21 | setup( 22 | name='s3transfer', 23 | version=get_version(), 24 | description='An Amazon S3 Transfer Manager', 25 | long_description=open('README.rst').read(), 26 | author='Amazon Web Services', 27 | author_email='kyknapp1@gmail.com', 28 | url='https://github.com/boto/s3transfer', 29 | packages=find_packages(exclude=['tests*']), 30 | include_package_data=True, 31 | install_requires=requires, 32 | extras_require={ 33 | 'crt': 'botocore[crt]>=1.37.4,<2.0a.0', 34 | }, 35 | license="Apache License 2.0", 36 | python_requires=">= 3.9", 37 | classifiers=[ 38 | 'Development Status :: 3 - Alpha', 39 | 'Intended Audience :: Developers', 40 | 'Natural Language :: English', 41 | 'License :: OSI Approved :: Apache Software License', 42 | 'Programming Language :: Python', 43 | 'Programming Language :: Python :: 3', 44 | 'Programming Language :: Python :: 3 :: Only', 45 | 'Programming Language :: Python :: 3.9', 46 | 'Programming Language :: Python :: 3.10', 47 | 'Programming Language :: Python :: 3.11', 48 | 'Programming Language :: Python :: 3.12', 49 | 'Programming Language :: Python :: 3.13', 50 | ], 51 | ) 52 | -------------------------------------------------------------------------------- /tests/functional/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | -------------------------------------------------------------------------------- /tests/functional/test_delete.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from s3transfer.manager import TransferManager 14 | from tests import BaseGeneralInterfaceTest 15 | 16 | 17 | class TestDeleteObject(BaseGeneralInterfaceTest): 18 | __test__ = True 19 | 20 | def setUp(self): 21 | super().setUp() 22 | self.bucket = 'mybucket' 23 | self.key = 'mykey' 24 | self.manager = TransferManager(self.client) 25 | 26 | @property 27 | def method(self): 28 | """The transfer manager method to invoke i.e. upload()""" 29 | return self.manager.delete 30 | 31 | def create_call_kwargs(self): 32 | """The kwargs to be passed to the transfer manager method""" 33 | return { 34 | 'bucket': self.bucket, 35 | 'key': self.key, 36 | } 37 | 38 | def create_invalid_extra_args(self): 39 | return { 40 | 'BadKwargs': True, 41 | } 42 | 43 | def create_stubbed_responses(self): 44 | """A list of stubbed responses that will cause the request to succeed 45 | 46 | The elements of this list is a dictionary that will be used as key 47 | word arguments to botocore.Stubber.add_response(). For example:: 48 | 49 | [{'method': 'put_object', 'service_response': {}}] 50 | """ 51 | return [ 52 | { 53 | 'method': 'delete_object', 54 | 'service_response': {}, 55 | 'expected_params': {'Bucket': self.bucket, 'Key': self.key}, 56 | } 57 | ] 58 | 59 | def create_expected_progress_callback_info(self): 60 | return [] 61 | 62 | def test_known_allowed_args_in_input_shape(self): 63 | op_model = self.client.meta.service_model.operation_model( 64 | 'DeleteObject' 65 | ) 66 | for allowed_arg in self.manager.ALLOWED_DELETE_ARGS: 67 | self.assertIn(allowed_arg, op_model.input_shape.members) 68 | 69 | def test_raise_exception_on_s3_object_lambda_resource(self): 70 | s3_object_lambda_arn = ( 71 | 'arn:aws:s3-object-lambda:us-west-2:123456789012:' 72 | 'accesspoint:my-accesspoint' 73 | ) 74 | with self.assertRaisesRegex(ValueError, 'methods do not support'): 75 | self.manager.delete(s3_object_lambda_arn, self.key) 76 | -------------------------------------------------------------------------------- /tests/functional/test_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the 'License'). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the 'license' file accompanying this file. This file is 10 | # distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from io import BytesIO 14 | 15 | from botocore.awsrequest import create_request_object 16 | 17 | from s3transfer.exceptions import CancelledError, FatalError 18 | from s3transfer.futures import BaseExecutor 19 | from s3transfer.manager import TransferConfig, TransferManager 20 | from tests import StubbedClientTest, mock, skip_if_using_serial_implementation 21 | 22 | 23 | class ArbitraryException(Exception): 24 | pass 25 | 26 | 27 | class SignalTransferringBody(BytesIO): 28 | """A mocked body with the ability to signal when transfers occur""" 29 | 30 | def __init__(self): 31 | super().__init__() 32 | self.signal_transferring_call_count = 0 33 | self.signal_not_transferring_call_count = 0 34 | 35 | def signal_transferring(self): 36 | self.signal_transferring_call_count += 1 37 | 38 | def signal_not_transferring(self): 39 | self.signal_not_transferring_call_count += 1 40 | 41 | def seek(self, where, whence=0): 42 | pass 43 | 44 | def tell(self): 45 | return 0 46 | 47 | def read(self, amount=0): 48 | return b'' 49 | 50 | 51 | class TestTransferManager(StubbedClientTest): 52 | @skip_if_using_serial_implementation( 53 | 'Exception is thrown once all transfers are submitted. ' 54 | 'However for the serial implementation, transfers are performed ' 55 | 'in main thread meaning all transfers will complete before the ' 56 | 'exception being thrown.' 57 | ) 58 | def test_error_in_context_manager_cancels_incomplete_transfers(self): 59 | # The purpose of this test is to make sure if an error is raised 60 | # in the body of the context manager, incomplete transfers will 61 | # be cancelled with value of the exception wrapped by a CancelledError 62 | 63 | # NOTE: The fact that delete() was chosen to test this is arbitrary 64 | # other than it is the easiet to set up for the stubber. 65 | # The specific operation is not important to the purpose of this test. 66 | num_transfers = 100 67 | futures = [] 68 | ref_exception_msg = 'arbitrary exception' 69 | 70 | for _ in range(num_transfers): 71 | self.stubber.add_response('delete_object', {}) 72 | 73 | manager = TransferManager( 74 | self.client, 75 | TransferConfig( 76 | max_request_concurrency=1, max_submission_concurrency=1 77 | ), 78 | ) 79 | try: 80 | with manager: 81 | for i in range(num_transfers): 82 | futures.append(manager.delete('mybucket', 'mykey')) 83 | raise ArbitraryException(ref_exception_msg) 84 | except ArbitraryException: 85 | # At least one of the submitted futures should have been 86 | # cancelled. 87 | with self.assertRaisesRegex(FatalError, ref_exception_msg): 88 | for future in futures: 89 | future.result() 90 | 91 | @skip_if_using_serial_implementation( 92 | 'Exception is thrown once all transfers are submitted. ' 93 | 'However for the serial implementation, transfers are performed ' 94 | 'in main thread meaning all transfers will complete before the ' 95 | 'exception being thrown.' 96 | ) 97 | def test_cntrl_c_in_context_manager_cancels_incomplete_transfers(self): 98 | # The purpose of this test is to make sure if an error is raised 99 | # in the body of the context manager, incomplete transfers will 100 | # be cancelled with value of the exception wrapped by a CancelledError 101 | 102 | # NOTE: The fact that delete() was chosen to test this is arbitrary 103 | # other than it is the easiet to set up for the stubber. 104 | # The specific operation is not important to the purpose of this test. 105 | num_transfers = 100 106 | futures = [] 107 | 108 | for _ in range(num_transfers): 109 | self.stubber.add_response('delete_object', {}) 110 | 111 | manager = TransferManager( 112 | self.client, 113 | TransferConfig( 114 | max_request_concurrency=1, max_submission_concurrency=1 115 | ), 116 | ) 117 | try: 118 | with manager: 119 | for i in range(num_transfers): 120 | futures.append(manager.delete('mybucket', 'mykey')) 121 | raise KeyboardInterrupt() 122 | except KeyboardInterrupt: 123 | # At least one of the submitted futures should have been 124 | # cancelled. 125 | with self.assertRaisesRegex(CancelledError, 'KeyboardInterrupt()'): 126 | for future in futures: 127 | future.result() 128 | 129 | def test_enable_disable_callbacks_only_ever_registered_once(self): 130 | body = SignalTransferringBody() 131 | request = create_request_object( 132 | { 133 | 'method': 'PUT', 134 | 'url': 'https://s3.amazonaws.com', 135 | 'body': body, 136 | 'headers': {}, 137 | 'context': {}, 138 | } 139 | ) 140 | # Create two TransferManager's using the same client 141 | TransferManager(self.client) 142 | TransferManager(self.client) 143 | self.client.meta.events.emit( 144 | 'request-created.s3', request=request, operation_name='PutObject' 145 | ) 146 | # The client should have only have the enable/disable callback 147 | # handlers registered once depite being used for two different 148 | # TransferManagers. 149 | self.assertEqual( 150 | body.signal_transferring_call_count, 151 | 1, 152 | 'The enable_callback() should have only ever been registered once', 153 | ) 154 | self.assertEqual( 155 | body.signal_not_transferring_call_count, 156 | 1, 157 | 'The disable_callback() should have only ever been registered ' 158 | 'once', 159 | ) 160 | 161 | def test_use_custom_executor_implementation(self): 162 | mocked_executor_cls = mock.Mock(BaseExecutor) 163 | transfer_manager = TransferManager( 164 | self.client, executor_cls=mocked_executor_cls 165 | ) 166 | transfer_manager.delete('bucket', 'key') 167 | self.assertTrue(mocked_executor_cls.return_value.submit.called) 168 | 169 | def test_unicode_exception_in_context_manager(self): 170 | with self.assertRaises(ArbitraryException): 171 | with TransferManager(self.client): 172 | raise ArbitraryException('\u2713') 173 | 174 | def test_client_property(self): 175 | manager = TransferManager(self.client) 176 | self.assertIs(manager.client, self.client) 177 | 178 | def test_config_property(self): 179 | config = TransferConfig() 180 | manager = TransferManager(self.client, config) 181 | self.assertIs(manager.config, config) 182 | 183 | def test_can_disable_bucket_validation(self): 184 | s3_object_lambda_arn = ( 185 | 'arn:aws:s3-object-lambda:us-west-2:123456789012:' 186 | 'accesspoint:my-accesspoint' 187 | ) 188 | config = TransferConfig() 189 | manager = TransferManager(self.client, config) 190 | manager.VALIDATE_SUPPORTED_BUCKET_VALUES = False 191 | manager.delete(s3_object_lambda_arn, 'my-key') 192 | -------------------------------------------------------------------------------- /tests/functional/test_processpool.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import glob 14 | import os 15 | from io import BytesIO 16 | from multiprocessing.managers import BaseManager 17 | 18 | import botocore.exceptions 19 | import botocore.session 20 | from botocore.stub import Stubber 21 | 22 | from s3transfer.exceptions import CancelledError 23 | from s3transfer.processpool import ProcessPoolDownloader, ProcessTransferConfig 24 | from tests import FileCreator, mock, unittest 25 | 26 | 27 | class StubbedClient: 28 | def __init__(self): 29 | self._client = botocore.session.get_session().create_client( 30 | 's3', 31 | 'us-west-2', 32 | aws_access_key_id='foo', 33 | aws_secret_access_key='bar', 34 | ) 35 | self._stubber = Stubber(self._client) 36 | self._stubber.activate() 37 | self._caught_stubber_errors = [] 38 | 39 | def get_object(self, **kwargs): 40 | return self._client.get_object(**kwargs) 41 | 42 | def head_object(self, **kwargs): 43 | return self._client.head_object(**kwargs) 44 | 45 | def add_response(self, *args, **kwargs): 46 | self._stubber.add_response(*args, **kwargs) 47 | 48 | def add_client_error(self, *args, **kwargs): 49 | self._stubber.add_client_error(*args, **kwargs) 50 | 51 | 52 | class StubbedClientManager(BaseManager): 53 | pass 54 | 55 | 56 | StubbedClientManager.register('StubbedClient', StubbedClient) 57 | 58 | 59 | # Ideally a Mock would be used here. However, they cannot be pickled 60 | # for Windows. So instead we define a factory class at the module level that 61 | # can return a stubbed client we initialized in the setUp. 62 | class StubbedClientFactory: 63 | def __init__(self, stubbed_client): 64 | self._stubbed_client = stubbed_client 65 | 66 | def __call__(self, *args, **kwargs): 67 | # The __call__ is defined so we can provide an instance of the 68 | # StubbedClientFactory to mock.patch() and have the instance be 69 | # returned when the patched class is instantiated. 70 | return self 71 | 72 | def create_client(self): 73 | return self._stubbed_client 74 | 75 | 76 | class TestProcessPoolDownloader(unittest.TestCase): 77 | def setUp(self): 78 | # The stubbed client needs to run in a manager to be shared across 79 | # processes and have it properly consume the stubbed response across 80 | # processes. 81 | self.manager = StubbedClientManager() 82 | self.manager.start() 83 | self.stubbed_client = self.manager.StubbedClient() 84 | self.stubbed_client_factory = StubbedClientFactory(self.stubbed_client) 85 | 86 | self.client_factory_patch = mock.patch( 87 | 's3transfer.processpool.ClientFactory', self.stubbed_client_factory 88 | ) 89 | self.client_factory_patch.start() 90 | self.files = FileCreator() 91 | 92 | self.config = ProcessTransferConfig(max_request_processes=1) 93 | self.downloader = ProcessPoolDownloader(config=self.config) 94 | self.bucket = 'mybucket' 95 | self.key = 'mykey' 96 | self.filename = self.files.full_path('filename') 97 | self.remote_contents = b'my content' 98 | self.stream = BytesIO(self.remote_contents) 99 | 100 | def tearDown(self): 101 | self.manager.shutdown() 102 | self.client_factory_patch.stop() 103 | self.files.remove_all() 104 | 105 | def assert_contents(self, filename, expected_contents): 106 | self.assertTrue(os.path.exists(filename)) 107 | with open(filename, 'rb') as f: 108 | self.assertEqual(f.read(), expected_contents) 109 | 110 | def test_download_file(self): 111 | self.stubbed_client.add_response( 112 | 'head_object', {'ContentLength': len(self.remote_contents)} 113 | ) 114 | self.stubbed_client.add_response('get_object', {'Body': self.stream}) 115 | with self.downloader: 116 | self.downloader.download_file(self.bucket, self.key, self.filename) 117 | self.assert_contents(self.filename, self.remote_contents) 118 | 119 | def test_download_multiple_files(self): 120 | self.stubbed_client.add_response('get_object', {'Body': self.stream}) 121 | self.stubbed_client.add_response( 122 | 'get_object', {'Body': BytesIO(self.remote_contents)} 123 | ) 124 | with self.downloader: 125 | self.downloader.download_file( 126 | self.bucket, 127 | self.key, 128 | self.filename, 129 | expected_size=len(self.remote_contents), 130 | ) 131 | other_file = self.files.full_path('filename2') 132 | self.downloader.download_file( 133 | self.bucket, 134 | self.key, 135 | other_file, 136 | expected_size=len(self.remote_contents), 137 | ) 138 | self.assert_contents(self.filename, self.remote_contents) 139 | self.assert_contents(other_file, self.remote_contents) 140 | 141 | def test_download_file_ranged_download(self): 142 | half_of_content_length = int(len(self.remote_contents) / 2) 143 | self.stubbed_client.add_response( 144 | 'head_object', {'ContentLength': len(self.remote_contents)} 145 | ) 146 | self.stubbed_client.add_response( 147 | 'get_object', 148 | {'Body': BytesIO(self.remote_contents[:half_of_content_length])}, 149 | ) 150 | self.stubbed_client.add_response( 151 | 'get_object', 152 | {'Body': BytesIO(self.remote_contents[half_of_content_length:])}, 153 | ) 154 | downloader = ProcessPoolDownloader( 155 | config=ProcessTransferConfig( 156 | multipart_chunksize=half_of_content_length, 157 | multipart_threshold=half_of_content_length, 158 | max_request_processes=1, 159 | ) 160 | ) 161 | with downloader: 162 | downloader.download_file(self.bucket, self.key, self.filename) 163 | self.assert_contents(self.filename, self.remote_contents) 164 | 165 | def test_download_file_extra_args(self): 166 | self.stubbed_client.add_response( 167 | 'head_object', 168 | {'ContentLength': len(self.remote_contents)}, 169 | expected_params={ 170 | 'Bucket': self.bucket, 171 | 'Key': self.key, 172 | 'VersionId': 'versionid', 173 | }, 174 | ) 175 | self.stubbed_client.add_response( 176 | 'get_object', 177 | {'Body': self.stream}, 178 | expected_params={ 179 | 'Bucket': self.bucket, 180 | 'Key': self.key, 181 | 'VersionId': 'versionid', 182 | }, 183 | ) 184 | with self.downloader: 185 | self.downloader.download_file( 186 | self.bucket, 187 | self.key, 188 | self.filename, 189 | extra_args={'VersionId': 'versionid'}, 190 | ) 191 | self.assert_contents(self.filename, self.remote_contents) 192 | 193 | def test_download_file_expected_size(self): 194 | self.stubbed_client.add_response('get_object', {'Body': self.stream}) 195 | with self.downloader: 196 | self.downloader.download_file( 197 | self.bucket, 198 | self.key, 199 | self.filename, 200 | expected_size=len(self.remote_contents), 201 | ) 202 | self.assert_contents(self.filename, self.remote_contents) 203 | 204 | def test_cleans_up_tempfile_on_failure(self): 205 | self.stubbed_client.add_client_error('get_object', 'NoSuchKey') 206 | with self.downloader: 207 | self.downloader.download_file( 208 | self.bucket, 209 | self.key, 210 | self.filename, 211 | expected_size=len(self.remote_contents), 212 | ) 213 | self.assertFalse(os.path.exists(self.filename)) 214 | # Any tempfile should have been erased as well 215 | possible_matches = glob.glob(f'{self.filename}*' + os.extsep) 216 | self.assertEqual(possible_matches, []) 217 | 218 | def test_validates_extra_args(self): 219 | with self.downloader: 220 | with self.assertRaises(ValueError): 221 | self.downloader.download_file( 222 | self.bucket, 223 | self.key, 224 | self.filename, 225 | extra_args={'NotSupported': 'NotSupported'}, 226 | ) 227 | 228 | def test_result_with_success(self): 229 | self.stubbed_client.add_response('get_object', {'Body': self.stream}) 230 | with self.downloader: 231 | future = self.downloader.download_file( 232 | self.bucket, 233 | self.key, 234 | self.filename, 235 | expected_size=len(self.remote_contents), 236 | ) 237 | self.assertIsNone(future.result()) 238 | 239 | def test_result_with_exception(self): 240 | self.stubbed_client.add_client_error('get_object', 'NoSuchKey') 241 | with self.downloader: 242 | future = self.downloader.download_file( 243 | self.bucket, 244 | self.key, 245 | self.filename, 246 | expected_size=len(self.remote_contents), 247 | ) 248 | with self.assertRaises(botocore.exceptions.ClientError): 249 | future.result() 250 | 251 | def test_result_with_cancel(self): 252 | self.stubbed_client.add_response('get_object', {'Body': self.stream}) 253 | with self.downloader: 254 | future = self.downloader.download_file( 255 | self.bucket, 256 | self.key, 257 | self.filename, 258 | expected_size=len(self.remote_contents), 259 | ) 260 | future.cancel() 261 | with self.assertRaises(CancelledError): 262 | future.result() 263 | 264 | def test_shutdown_with_no_downloads(self): 265 | downloader = ProcessPoolDownloader() 266 | try: 267 | downloader.shutdown() 268 | except AttributeError: 269 | self.fail( 270 | 'The downloader should be able to be shutdown even though ' 271 | 'the downloader was never started.' 272 | ) 273 | 274 | def test_shutdown_with_no_downloads_and_ctrl_c(self): 275 | # Special shutdown logic happens if a KeyboardInterrupt is raised in 276 | # the context manager. However, this logic can not happen if the 277 | # downloader was never started. So a KeyboardInterrupt should be 278 | # the only exception propagated. 279 | with self.assertRaises(KeyboardInterrupt): 280 | with self.downloader: 281 | raise KeyboardInterrupt() 282 | -------------------------------------------------------------------------------- /tests/functional/test_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import os 14 | import shutil 15 | import socket 16 | import tempfile 17 | 18 | from s3transfer.utils import OSUtils 19 | from tests import skip_if_windows, unittest 20 | 21 | 22 | @skip_if_windows('Windows does not support UNIX special files') 23 | class TestOSUtilsSpecialFiles(unittest.TestCase): 24 | def setUp(self): 25 | self.tempdir = tempfile.mkdtemp() 26 | self.filename = os.path.join(self.tempdir, 'myfile') 27 | 28 | def tearDown(self): 29 | shutil.rmtree(self.tempdir) 30 | 31 | def test_character_device(self): 32 | self.assertTrue(OSUtils().is_special_file('/dev/null')) 33 | 34 | def test_fifo(self): 35 | os.mkfifo(self.filename) 36 | self.assertTrue(OSUtils().is_special_file(self.filename)) 37 | 38 | def test_socket(self): 39 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 40 | sock.bind(self.filename) 41 | self.assertTrue(OSUtils().is_special_file(self.filename)) 42 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the 'License'). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the 'license' file accompanying this file. This file is 10 | # distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import botocore 14 | import botocore.session 15 | from botocore.exceptions import WaiterError 16 | 17 | from s3transfer.manager import TransferManager 18 | from s3transfer.subscribers import BaseSubscriber 19 | from tests import FileCreator, random_bucket_name, unittest 20 | 21 | 22 | def recursive_delete(client, bucket_name): 23 | # Ensure the bucket exists before attempting to wipe it out 24 | exists_waiter = client.get_waiter('bucket_exists') 25 | exists_waiter.wait(Bucket=bucket_name) 26 | page = client.get_paginator('list_objects') 27 | # Use pages paired with batch delete_objects(). 28 | for page in page.paginate(Bucket=bucket_name): 29 | keys = [{'Key': obj['Key']} for obj in page.get('Contents', [])] 30 | if keys: 31 | client.delete_objects(Bucket=bucket_name, Delete={'Objects': keys}) 32 | for _ in range(5): 33 | try: 34 | client.delete_bucket(Bucket=bucket_name) 35 | break 36 | except client.exceptions.NoSuchBucket: 37 | exists_waiter.wait(Bucket=bucket_name) 38 | except Exception: 39 | # We can sometimes get exceptions when trying to 40 | # delete a bucket. We'll let the waiter make 41 | # the final call as to whether the bucket was able 42 | # to be deleted. 43 | not_exists_waiter = client.get_waiter('bucket_not_exists') 44 | try: 45 | not_exists_waiter.wait(Bucket=bucket_name) 46 | except botocore.exceptions.WaiterError: 47 | continue 48 | 49 | 50 | class BaseTransferManagerIntegTest(unittest.TestCase): 51 | """Tests for the high level s3transfer module.""" 52 | 53 | @classmethod 54 | def setUpClass(cls): 55 | cls.region = 'us-west-2' 56 | cls.session = botocore.session.get_session() 57 | cls.client = cls.session.create_client('s3', cls.region) 58 | cls.bucket_name = random_bucket_name() 59 | cls.client.create_bucket( 60 | Bucket=cls.bucket_name, 61 | CreateBucketConfiguration={'LocationConstraint': cls.region}, 62 | ObjectOwnership='ObjectWriter', 63 | ) 64 | cls.client.delete_public_access_block(Bucket=cls.bucket_name) 65 | 66 | def setUp(self): 67 | self.files = FileCreator() 68 | 69 | def tearDown(self): 70 | self.files.remove_all() 71 | 72 | @classmethod 73 | def tearDownClass(cls): 74 | recursive_delete(cls.client, cls.bucket_name) 75 | 76 | def delete_object(self, key): 77 | self.client.delete_object(Bucket=self.bucket_name, Key=key) 78 | 79 | def object_exists(self, key, extra_args=None): 80 | try: 81 | self.wait_object_exists(key, extra_args) 82 | return True 83 | except WaiterError: 84 | return False 85 | 86 | def object_not_exists(self, key, extra_args=None): 87 | if extra_args is None: 88 | extra_args = {} 89 | try: 90 | self.client.get_waiter('object_not_exists').wait( 91 | Bucket=self.bucket_name, Key=key, **extra_args 92 | ) 93 | return True 94 | except WaiterError: 95 | return False 96 | 97 | def wait_object_exists(self, key, extra_args=None): 98 | if extra_args is None: 99 | extra_args = {} 100 | for _ in range(5): 101 | self.client.get_waiter('object_exists').wait( 102 | Bucket=self.bucket_name, Key=key, **extra_args 103 | ) 104 | 105 | def create_transfer_manager(self, config=None): 106 | return TransferManager(self.client, config=config) 107 | 108 | def upload_file(self, filename, key, extra_args=None): 109 | transfer = self.create_transfer_manager() 110 | with open(filename, 'rb') as f: 111 | transfer.upload(f, self.bucket_name, key, extra_args) 112 | self.wait_object_exists(key, extra_args) 113 | self.addCleanup(self.delete_object, key) 114 | 115 | 116 | class WaitForTransferStart(BaseSubscriber): 117 | def __init__(self, bytes_transfer_started_event): 118 | self._bytes_transfer_started_event = bytes_transfer_started_event 119 | 120 | def on_progress(self, **kwargs): 121 | if not self._bytes_transfer_started_event.is_set(): 122 | self._bytes_transfer_started_event.set() 123 | -------------------------------------------------------------------------------- /tests/integration/test_copy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from s3transfer.manager import TransferConfig 14 | from tests import RecordingSubscriber 15 | from tests.integration import BaseTransferManagerIntegTest 16 | 17 | 18 | class TestCopy(BaseTransferManagerIntegTest): 19 | def setUp(self): 20 | super().setUp() 21 | self.multipart_threshold = 5 * 1024 * 1024 22 | self.config = TransferConfig( 23 | multipart_threshold=self.multipart_threshold 24 | ) 25 | 26 | def test_copy_below_threshold(self): 27 | transfer_manager = self.create_transfer_manager(self.config) 28 | key = '1mb.txt' 29 | new_key = '1mb-copy.txt' 30 | 31 | filename = self.files.create_file_with_size(key, filesize=1024 * 1024) 32 | self.upload_file(filename, key) 33 | 34 | future = transfer_manager.copy( 35 | copy_source={'Bucket': self.bucket_name, 'Key': key}, 36 | bucket=self.bucket_name, 37 | key=new_key, 38 | ) 39 | 40 | future.result() 41 | self.assertTrue(self.object_exists(new_key)) 42 | 43 | def test_copy_above_threshold(self): 44 | transfer_manager = self.create_transfer_manager(self.config) 45 | key = '20mb.txt' 46 | new_key = '20mb-copy.txt' 47 | 48 | filename = self.files.create_file_with_size( 49 | key, filesize=20 * 1024 * 1024 50 | ) 51 | self.upload_file(filename, key) 52 | 53 | future = transfer_manager.copy( 54 | copy_source={'Bucket': self.bucket_name, 'Key': key}, 55 | bucket=self.bucket_name, 56 | key=new_key, 57 | ) 58 | 59 | future.result() 60 | self.assertTrue(self.object_exists(new_key)) 61 | 62 | def test_progress_subscribers_on_copy(self): 63 | subscriber = RecordingSubscriber() 64 | transfer_manager = self.create_transfer_manager(self.config) 65 | key = '20mb.txt' 66 | new_key = '20mb-copy.txt' 67 | 68 | filename = self.files.create_file_with_size( 69 | key, filesize=20 * 1024 * 1024 70 | ) 71 | self.upload_file(filename, key) 72 | 73 | future = transfer_manager.copy( 74 | copy_source={'Bucket': self.bucket_name, 'Key': key}, 75 | bucket=self.bucket_name, 76 | key=new_key, 77 | subscribers=[subscriber], 78 | ) 79 | 80 | future.result() 81 | # The callback should have been called enough times such that 82 | # the total amount of bytes we've seen (via the "amount" 83 | # arg to the callback function) should be the size 84 | # of the file we uploaded. 85 | self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024) 86 | -------------------------------------------------------------------------------- /tests/integration/test_delete.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from tests.integration import BaseTransferManagerIntegTest 14 | 15 | 16 | class TestDeleteObject(BaseTransferManagerIntegTest): 17 | def test_can_delete_object(self): 18 | key_name = 'mykey' 19 | self.client.put_object( 20 | Bucket=self.bucket_name, Key=key_name, Body=b'hello world' 21 | ) 22 | self.assertTrue(self.object_exists(key_name)) 23 | 24 | transfer_manager = self.create_transfer_manager() 25 | future = transfer_manager.delete(bucket=self.bucket_name, key=key_name) 26 | future.result() 27 | 28 | self.assertTrue(self.object_not_exists(key_name)) 29 | -------------------------------------------------------------------------------- /tests/integration/test_download.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import glob 14 | import os 15 | import threading 16 | import time 17 | from concurrent.futures import CancelledError 18 | 19 | from s3transfer.manager import TransferConfig 20 | from tests import ( 21 | NonSeekableWriter, 22 | RecordingSubscriber, 23 | assert_files_equal, 24 | skip_if_using_serial_implementation, 25 | skip_if_windows, 26 | ) 27 | from tests.integration import ( 28 | BaseTransferManagerIntegTest, 29 | WaitForTransferStart, 30 | ) 31 | 32 | 33 | class TestDownload(BaseTransferManagerIntegTest): 34 | def setUp(self): 35 | super().setUp() 36 | self.multipart_threshold = 5 * 1024 * 1024 37 | self.config = TransferConfig( 38 | multipart_threshold=self.multipart_threshold 39 | ) 40 | 41 | def test_below_threshold(self): 42 | transfer_manager = self.create_transfer_manager(self.config) 43 | 44 | filename = self.files.create_file_with_size( 45 | 'foo.txt', filesize=1024 * 1024 46 | ) 47 | self.upload_file(filename, '1mb.txt') 48 | 49 | download_path = os.path.join(self.files.rootdir, '1mb.txt') 50 | future = transfer_manager.download( 51 | self.bucket_name, '1mb.txt', download_path 52 | ) 53 | future.result() 54 | assert_files_equal(filename, download_path) 55 | 56 | def test_above_threshold(self): 57 | transfer_manager = self.create_transfer_manager(self.config) 58 | 59 | filename = self.files.create_file_with_size( 60 | 'foo.txt', filesize=20 * 1024 * 1024 61 | ) 62 | self.upload_file(filename, '20mb.txt') 63 | 64 | download_path = os.path.join(self.files.rootdir, '20mb.txt') 65 | future = transfer_manager.download( 66 | self.bucket_name, '20mb.txt', download_path 67 | ) 68 | future.result() 69 | assert_files_equal(filename, download_path) 70 | 71 | @skip_if_using_serial_implementation( 72 | 'Exception is thrown once the transfer is submitted. ' 73 | 'However for the serial implementation, transfers are performed ' 74 | 'in main thread meaning the transfer will complete before the ' 75 | 'KeyboardInterrupt being thrown.' 76 | ) 77 | def test_large_download_exits_quicky_on_exception(self): 78 | transfer_manager = self.create_transfer_manager(self.config) 79 | 80 | filename = self.files.create_file_with_size( 81 | 'foo.txt', filesize=60 * 1024 * 1024 82 | ) 83 | self.upload_file(filename, '60mb.txt') 84 | 85 | download_path = os.path.join(self.files.rootdir, '60mb.txt') 86 | timeout = 10 87 | bytes_transferring = threading.Event() 88 | subscriber = WaitForTransferStart(bytes_transferring) 89 | try: 90 | with transfer_manager: 91 | future = transfer_manager.download( 92 | self.bucket_name, 93 | '60mb.txt', 94 | download_path, 95 | subscribers=[subscriber], 96 | ) 97 | if not bytes_transferring.wait(timeout): 98 | future.cancel() 99 | raise RuntimeError( 100 | "Download transfer did not start after waiting for " 101 | f"{timeout} seconds." 102 | ) 103 | # Raise an exception which should cause the preceding 104 | # download to cancel and exit quickly 105 | start_time = time.time() 106 | raise KeyboardInterrupt() 107 | except KeyboardInterrupt: 108 | pass 109 | end_time = time.time() 110 | # The maximum time allowed for the transfer manager to exit. 111 | # This means that it should take less than a couple second after 112 | # sleeping to exit. 113 | max_allowed_exit_time = 5 114 | actual_time_to_exit = end_time - start_time 115 | self.assertLess( 116 | actual_time_to_exit, 117 | max_allowed_exit_time, 118 | f"Failed to exit under {max_allowed_exit_time}. Instead exited in {actual_time_to_exit}.", 119 | ) 120 | 121 | # Make sure the future was cancelled because of the KeyboardInterrupt 122 | with self.assertRaisesRegex(CancelledError, 'KeyboardInterrupt()'): 123 | future.result() 124 | 125 | # Make sure the actual file and the temporary do not exist 126 | # by globbing for the file and any of its extensions 127 | possible_matches = glob.glob(f'{download_path}*') 128 | self.assertEqual(possible_matches, []) 129 | 130 | @skip_if_using_serial_implementation( 131 | 'Exception is thrown once the transfer is submitted. ' 132 | 'However for the serial implementation, transfers are performed ' 133 | 'in main thread meaning the transfer will complete before the ' 134 | 'KeyboardInterrupt being thrown.' 135 | ) 136 | def test_many_files_exits_quicky_on_exception(self): 137 | # Set the max request queue size and number of submission threads 138 | # to something small to simulate having a large queue 139 | # of transfer requests to complete and it is backed up. 140 | self.config.max_request_queue_size = 1 141 | self.config.max_submission_concurrency = 1 142 | transfer_manager = self.create_transfer_manager(self.config) 143 | 144 | filename = self.files.create_file_with_size( 145 | 'foo.txt', filesize=1024 * 1024 146 | ) 147 | self.upload_file(filename, '1mb.txt') 148 | 149 | filenames = [] 150 | futures = [] 151 | for i in range(10): 152 | filenames.append(os.path.join(self.files.rootdir, 'file' + str(i))) 153 | 154 | try: 155 | with transfer_manager: 156 | start_time = time.time() 157 | for filename in filenames: 158 | futures.append( 159 | transfer_manager.download( 160 | self.bucket_name, '1mb.txt', filename 161 | ) 162 | ) 163 | # Raise an exception which should cause the preceding 164 | # transfer to cancel and exit quickly 165 | raise KeyboardInterrupt() 166 | except KeyboardInterrupt: 167 | pass 168 | end_time = time.time() 169 | # The maximum time allowed for the transfer manager to exit. 170 | # This means that it should take less than a couple seconds to exit. 171 | max_allowed_exit_time = 5 172 | self.assertLess( 173 | end_time - start_time, 174 | max_allowed_exit_time, 175 | f"Failed to exit under {max_allowed_exit_time}. Instead exited in {end_time - start_time}.", 176 | ) 177 | 178 | # Make sure at least one of the futures got cancelled 179 | with self.assertRaisesRegex(CancelledError, 'KeyboardInterrupt()'): 180 | for future in futures: 181 | future.result() 182 | 183 | # For the transfer that did get cancelled, make sure the temporary 184 | # file got removed. 185 | possible_matches = glob.glob(f'{future.meta.call_args.fileobj}*') 186 | self.assertEqual(possible_matches, []) 187 | 188 | def test_progress_subscribers_on_download(self): 189 | subscriber = RecordingSubscriber() 190 | transfer_manager = self.create_transfer_manager(self.config) 191 | 192 | filename = self.files.create_file_with_size( 193 | 'foo.txt', filesize=20 * 1024 * 1024 194 | ) 195 | self.upload_file(filename, '20mb.txt') 196 | 197 | download_path = os.path.join(self.files.rootdir, '20mb.txt') 198 | 199 | future = transfer_manager.download( 200 | self.bucket_name, 201 | '20mb.txt', 202 | download_path, 203 | subscribers=[subscriber], 204 | ) 205 | future.result() 206 | self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024) 207 | 208 | def test_below_threshold_for_fileobj(self): 209 | transfer_manager = self.create_transfer_manager(self.config) 210 | 211 | filename = self.files.create_file_with_size( 212 | 'foo.txt', filesize=1024 * 1024 213 | ) 214 | self.upload_file(filename, '1mb.txt') 215 | 216 | download_path = os.path.join(self.files.rootdir, '1mb.txt') 217 | with open(download_path, 'wb') as f: 218 | future = transfer_manager.download(self.bucket_name, '1mb.txt', f) 219 | future.result() 220 | assert_files_equal(filename, download_path) 221 | 222 | def test_above_threshold_for_fileobj(self): 223 | transfer_manager = self.create_transfer_manager(self.config) 224 | 225 | filename = self.files.create_file_with_size( 226 | 'foo.txt', filesize=20 * 1024 * 1024 227 | ) 228 | self.upload_file(filename, '20mb.txt') 229 | 230 | download_path = os.path.join(self.files.rootdir, '20mb.txt') 231 | with open(download_path, 'wb') as f: 232 | future = transfer_manager.download(self.bucket_name, '20mb.txt', f) 233 | future.result() 234 | assert_files_equal(filename, download_path) 235 | 236 | def test_below_threshold_for_nonseekable_fileobj(self): 237 | transfer_manager = self.create_transfer_manager(self.config) 238 | 239 | filename = self.files.create_file_with_size( 240 | 'foo.txt', filesize=1024 * 1024 241 | ) 242 | self.upload_file(filename, '1mb.txt') 243 | 244 | download_path = os.path.join(self.files.rootdir, '1mb.txt') 245 | with open(download_path, 'wb') as f: 246 | future = transfer_manager.download( 247 | self.bucket_name, '1mb.txt', NonSeekableWriter(f) 248 | ) 249 | future.result() 250 | assert_files_equal(filename, download_path) 251 | 252 | def test_above_threshold_for_nonseekable_fileobj(self): 253 | transfer_manager = self.create_transfer_manager(self.config) 254 | 255 | filename = self.files.create_file_with_size( 256 | 'foo.txt', filesize=20 * 1024 * 1024 257 | ) 258 | self.upload_file(filename, '20mb.txt') 259 | 260 | download_path = os.path.join(self.files.rootdir, '20mb.txt') 261 | with open(download_path, 'wb') as f: 262 | future = transfer_manager.download( 263 | self.bucket_name, '20mb.txt', NonSeekableWriter(f) 264 | ) 265 | future.result() 266 | assert_files_equal(filename, download_path) 267 | 268 | @skip_if_windows('Windows does not support UNIX special files') 269 | def test_download_to_special_file(self): 270 | transfer_manager = self.create_transfer_manager(self.config) 271 | filename = self.files.create_file_with_size( 272 | 'foo.txt', filesize=1024 * 1024 273 | ) 274 | self.upload_file(filename, '1mb.txt') 275 | future = transfer_manager.download( 276 | self.bucket_name, '1mb.txt', '/dev/null' 277 | ) 278 | try: 279 | future.result() 280 | except Exception as e: 281 | self.fail( 282 | 'Should have been able to download to /dev/null but received ' 283 | f'following exception {e}' 284 | ) 285 | -------------------------------------------------------------------------------- /tests/integration/test_processpool.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import glob 14 | import os 15 | import time 16 | 17 | from s3transfer.processpool import ProcessPoolDownloader, ProcessTransferConfig 18 | from tests import assert_files_equal 19 | from tests.integration import BaseTransferManagerIntegTest 20 | 21 | 22 | class TestProcessPoolDownloader(BaseTransferManagerIntegTest): 23 | def setUp(self): 24 | super().setUp() 25 | self.multipart_threshold = 5 * 1024 * 1024 26 | self.config = ProcessTransferConfig( 27 | multipart_threshold=self.multipart_threshold 28 | ) 29 | self.client_kwargs = {'region_name': self.region} 30 | 31 | def create_process_pool_downloader(self, client_kwargs=None, config=None): 32 | if client_kwargs is None: 33 | client_kwargs = self.client_kwargs 34 | if config is None: 35 | config = self.config 36 | return ProcessPoolDownloader( 37 | client_kwargs=client_kwargs, config=config 38 | ) 39 | 40 | def test_below_threshold(self): 41 | downloader = self.create_process_pool_downloader() 42 | filename = self.files.create_file_with_size( 43 | 'foo.txt', filesize=1024 * 1024 44 | ) 45 | self.upload_file(filename, '1mb.txt') 46 | 47 | download_path = os.path.join(self.files.rootdir, '1mb.txt') 48 | with downloader: 49 | downloader.download_file( 50 | self.bucket_name, '1mb.txt', download_path 51 | ) 52 | assert_files_equal(filename, download_path) 53 | 54 | def test_above_threshold(self): 55 | downloader = self.create_process_pool_downloader() 56 | filename = self.files.create_file_with_size( 57 | 'foo.txt', filesize=20 * 1024 * 1024 58 | ) 59 | self.upload_file(filename, '20mb.txt') 60 | 61 | download_path = os.path.join(self.files.rootdir, '20mb.txt') 62 | with downloader: 63 | downloader.download_file( 64 | self.bucket_name, '20mb.txt', download_path 65 | ) 66 | assert_files_equal(filename, download_path) 67 | 68 | def test_large_download_exits_quickly_on_exception(self): 69 | downloader = self.create_process_pool_downloader() 70 | 71 | filename = self.files.create_file_with_size( 72 | 'foo.txt', filesize=60 * 1024 * 1024 73 | ) 74 | self.upload_file(filename, '60mb.txt') 75 | 76 | download_path = os.path.join(self.files.rootdir, '60mb.txt') 77 | sleep_time = 0.2 78 | try: 79 | with downloader: 80 | downloader.download_file( 81 | self.bucket_name, '60mb.txt', download_path 82 | ) 83 | # Sleep for a little to get the transfer process going 84 | time.sleep(sleep_time) 85 | # Raise an exception which should cause the preceding 86 | # download to cancel and exit quickly 87 | start_time = time.time() 88 | raise KeyboardInterrupt() 89 | except KeyboardInterrupt: 90 | pass 91 | end_time = time.time() 92 | # The maximum time allowed for the transfer manager to exit. 93 | # This means that it should take less than a couple second after 94 | # sleeping to exit. 95 | max_allowed_exit_time = 5 96 | self.assertLess( 97 | end_time - start_time, 98 | max_allowed_exit_time, 99 | f"Failed to exit under {max_allowed_exit_time}. Instead exited in {end_time - start_time}.", 100 | ) 101 | 102 | # Make sure the actual file and the temporary do not exist 103 | # by globbing for the file and any of its extensions 104 | possible_matches = glob.glob(f'{download_path}*') 105 | self.assertEqual(possible_matches, []) 106 | 107 | def test_many_files_exits_quickly_on_exception(self): 108 | downloader = self.create_process_pool_downloader() 109 | 110 | filename = self.files.create_file_with_size( 111 | '1mb.txt', filesize=1024 * 1024 112 | ) 113 | self.upload_file(filename, '1mb.txt') 114 | 115 | filenames = [] 116 | base_filename = os.path.join(self.files.rootdir, 'file') 117 | for i in range(10): 118 | filenames.append(base_filename + str(i)) 119 | 120 | try: 121 | with downloader: 122 | start_time = time.time() 123 | for filename in filenames: 124 | downloader.download_file( 125 | self.bucket_name, '1mb.txt', filename 126 | ) 127 | # Raise an exception which should cause the preceding 128 | # transfer to cancel and exit quickly 129 | raise KeyboardInterrupt() 130 | except KeyboardInterrupt: 131 | pass 132 | end_time = time.time() 133 | # The maximum time allowed for the transfer manager to exit. 134 | # This means that it should take less than a couple seconds to exit. 135 | max_allowed_exit_time = 5 136 | self.assertLess( 137 | end_time - start_time, 138 | max_allowed_exit_time, 139 | f"Failed to exit under {max_allowed_exit_time}. Instead exited in {end_time - start_time}.", 140 | ) 141 | 142 | # For the transfer that did get cancelled, make sure the temporary 143 | # file got removed. 144 | possible_matches = glob.glob(f'{base_filename}*') 145 | self.assertEqual(possible_matches, []) 146 | -------------------------------------------------------------------------------- /tests/integration/test_upload.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import threading 14 | import time 15 | from concurrent.futures import CancelledError 16 | from io import BytesIO 17 | 18 | from s3transfer.manager import TransferConfig 19 | from tests import ( 20 | NonSeekableReader, 21 | RecordingSubscriber, 22 | skip_if_using_serial_implementation, 23 | ) 24 | from tests.integration import ( 25 | BaseTransferManagerIntegTest, 26 | WaitForTransferStart, 27 | ) 28 | 29 | 30 | class TestUpload(BaseTransferManagerIntegTest): 31 | def setUp(self): 32 | super().setUp() 33 | self.multipart_threshold = 5 * 1024 * 1024 34 | self.config = TransferConfig( 35 | multipart_threshold=self.multipart_threshold 36 | ) 37 | 38 | def get_input_fileobj(self, size, name=''): 39 | return self.files.create_file_with_size(name, size) 40 | 41 | def test_upload_below_threshold(self): 42 | transfer_manager = self.create_transfer_manager(self.config) 43 | file = self.get_input_fileobj(size=1024 * 1024, name='1mb.txt') 44 | future = transfer_manager.upload(file, self.bucket_name, '1mb.txt') 45 | self.addCleanup(self.delete_object, '1mb.txt') 46 | 47 | future.result() 48 | self.assertTrue(self.object_exists('1mb.txt')) 49 | 50 | def test_upload_above_threshold(self): 51 | transfer_manager = self.create_transfer_manager(self.config) 52 | file = self.get_input_fileobj(size=20 * 1024 * 1024, name='20mb.txt') 53 | future = transfer_manager.upload(file, self.bucket_name, '20mb.txt') 54 | self.addCleanup(self.delete_object, '20mb.txt') 55 | 56 | future.result() 57 | self.assertTrue(self.object_exists('20mb.txt')) 58 | 59 | @skip_if_using_serial_implementation( 60 | 'Exception is thrown once the transfer is submitted. ' 61 | 'However for the serial implementation, transfers are performed ' 62 | 'in main thread meaning the transfer will complete before the ' 63 | 'KeyboardInterrupt being thrown.' 64 | ) 65 | def test_large_upload_exits_quicky_on_exception(self): 66 | transfer_manager = self.create_transfer_manager(self.config) 67 | 68 | filename = self.get_input_fileobj( 69 | name='foo.txt', size=20 * 1024 * 1024 70 | ) 71 | 72 | timeout = 10 73 | bytes_transferring = threading.Event() 74 | subscriber = WaitForTransferStart(bytes_transferring) 75 | try: 76 | with transfer_manager: 77 | future = transfer_manager.upload( 78 | filename, 79 | self.bucket_name, 80 | '20mb.txt', 81 | subscribers=[subscriber], 82 | ) 83 | if not bytes_transferring.wait(timeout): 84 | future.cancel() 85 | raise RuntimeError( 86 | "Download transfer did not start after waiting for " 87 | f"{timeout} seconds." 88 | ) 89 | # Raise an exception which should cause the preceding 90 | # download to cancel and exit quickly 91 | start_time = time.time() 92 | raise KeyboardInterrupt() 93 | except KeyboardInterrupt: 94 | pass 95 | end_time = time.time() 96 | # The maximum time allowed for the transfer manager to exit. 97 | # This means that it should take less than a couple second after 98 | # sleeping to exit. 99 | max_allowed_exit_time = 5 100 | actual_time_to_exit = end_time - start_time 101 | self.assertLess( 102 | actual_time_to_exit, 103 | max_allowed_exit_time, 104 | f"Failed to exit under {max_allowed_exit_time}. Instead exited in {actual_time_to_exit}.", 105 | ) 106 | 107 | try: 108 | future.result() 109 | self.skipTest( 110 | 'Upload completed before interrupted and therefore ' 111 | 'could not cancel the upload' 112 | ) 113 | except CancelledError as e: 114 | self.assertEqual(str(e), 'KeyboardInterrupt()') 115 | # If the transfer did get cancelled, 116 | # make sure the object does not exist. 117 | self.assertTrue(self.object_not_exists('20mb.txt')) 118 | 119 | @skip_if_using_serial_implementation( 120 | 'Exception is thrown once the transfers are submitted. ' 121 | 'However for the serial implementation, transfers are performed ' 122 | 'in main thread meaning the transfers will complete before the ' 123 | 'KeyboardInterrupt being thrown.' 124 | ) 125 | def test_many_files_exits_quicky_on_exception(self): 126 | # Set the max request queue size and number of submission threads 127 | # to something small to simulate having a large queue 128 | # of transfer requests to complete and it is backed up. 129 | self.config.max_request_queue_size = 1 130 | self.config.max_submission_concurrency = 1 131 | transfer_manager = self.create_transfer_manager(self.config) 132 | 133 | fileobjs = [] 134 | keynames = [] 135 | futures = [] 136 | for i in range(10): 137 | filename = 'file' + str(i) 138 | keynames.append(filename) 139 | fileobjs.append( 140 | self.get_input_fileobj(name=filename, size=1024 * 1024) 141 | ) 142 | 143 | try: 144 | with transfer_manager: 145 | for i, fileobj in enumerate(fileobjs): 146 | futures.append( 147 | transfer_manager.upload( 148 | fileobj, self.bucket_name, keynames[i] 149 | ) 150 | ) 151 | # Raise an exception which should cause the preceding 152 | # transfer to cancel and exit quickly 153 | start_time = time.time() 154 | raise KeyboardInterrupt() 155 | except KeyboardInterrupt: 156 | pass 157 | end_time = time.time() 158 | # The maximum time allowed for the transfer manager to exit. 159 | # This means that it should take less than a couple seconds to exit. 160 | max_allowed_exit_time = 5 161 | self.assertLess( 162 | end_time - start_time, 163 | max_allowed_exit_time, 164 | f"Failed to exit under {max_allowed_exit_time}. Instead exited in {end_time - start_time}.", 165 | ) 166 | 167 | # Make sure at least one of the futures got cancelled 168 | with self.assertRaisesRegex(CancelledError, 'KeyboardInterrupt()'): 169 | for future in futures: 170 | future.result() 171 | # For the transfer that did get cancelled, make sure the object 172 | # does not exist. 173 | self.assertTrue(self.object_not_exists(future.meta.call_args.key)) 174 | 175 | def test_progress_subscribers_on_upload(self): 176 | subscriber = RecordingSubscriber() 177 | transfer_manager = self.create_transfer_manager(self.config) 178 | file = self.get_input_fileobj(size=20 * 1024 * 1024, name='20mb.txt') 179 | future = transfer_manager.upload( 180 | file, self.bucket_name, '20mb.txt', subscribers=[subscriber] 181 | ) 182 | self.addCleanup(self.delete_object, '20mb.txt') 183 | 184 | future.result() 185 | # The callback should have been called enough times such that 186 | # the total amount of bytes we've seen (via the "amount" 187 | # arg to the callback function) should be the size 188 | # of the file we uploaded. 189 | self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024) 190 | 191 | 192 | class TestUploadSeekableStream(TestUpload): 193 | def get_input_fileobj(self, size, name=''): 194 | return BytesIO(b'0' * size) 195 | 196 | 197 | class TestUploadNonSeekableStream(TestUpload): 198 | def get_input_fileobj(self, size, name=''): 199 | return NonSeekableReader(b'0' * size) 200 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the 'License'). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the 'license' file accompanying this file. This file is 10 | # distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | -------------------------------------------------------------------------------- /tests/unit/test_compat.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import multiprocessing 14 | import os 15 | import shutil 16 | import signal 17 | import tempfile 18 | from io import BytesIO 19 | 20 | from s3transfer.compat import BaseManager, readable, seekable 21 | from tests import skip_if_windows, unittest 22 | 23 | 24 | class ErrorRaisingSeekWrapper: 25 | """An object wrapper that throws an error when seeked on 26 | 27 | :param fileobj: The fileobj that it wraps 28 | :param exception: The exception to raise when seeked on. 29 | """ 30 | 31 | def __init__(self, fileobj, exception): 32 | self._fileobj = fileobj 33 | self._exception = exception 34 | 35 | def seek(self, offset, whence=0): 36 | raise self._exception 37 | 38 | def tell(self): 39 | return self._fileobj.tell() 40 | 41 | 42 | class TestSeekable(unittest.TestCase): 43 | def setUp(self): 44 | self.tempdir = tempfile.mkdtemp() 45 | self.filename = os.path.join(self.tempdir, 'foo') 46 | 47 | def tearDown(self): 48 | shutil.rmtree(self.tempdir) 49 | 50 | def test_seekable_fileobj(self): 51 | with open(self.filename, 'w') as f: 52 | self.assertTrue(seekable(f)) 53 | 54 | def test_non_file_like_obj(self): 55 | # Fails because there is no seekable(), seek(), nor tell() 56 | self.assertFalse(seekable(object())) 57 | 58 | def test_non_seekable_ioerror(self): 59 | # Should return False if IOError is thrown. 60 | with open(self.filename, 'w') as f: 61 | self.assertFalse(seekable(ErrorRaisingSeekWrapper(f, OSError()))) 62 | 63 | def test_non_seekable_oserror(self): 64 | # Should return False if OSError is thrown. 65 | with open(self.filename, 'w') as f: 66 | self.assertFalse(seekable(ErrorRaisingSeekWrapper(f, OSError()))) 67 | 68 | 69 | class TestReadable(unittest.TestCase): 70 | def test_readable_fileobj(self): 71 | with tempfile.TemporaryFile() as f: 72 | self.assertTrue(readable(f)) 73 | 74 | def test_readable_file_like_obj(self): 75 | self.assertTrue(readable(BytesIO())) 76 | 77 | def test_non_file_like_obj(self): 78 | self.assertFalse(readable(object())) 79 | 80 | 81 | class TestBaseManager(unittest.TestCase): 82 | def create_pid_manager(self): 83 | class PIDManager(BaseManager): 84 | def __init__(self): 85 | # Python 3.14 changed the non-macOS POSIX default to forkserver 86 | # but the code in this module does not work with it 87 | # See https://github.com/python/cpython/issues/125714 88 | if multiprocessing.get_start_method() == 'forkserver': 89 | ctx = multiprocessing.get_context(method='fork') 90 | else: 91 | ctx = multiprocessing.get_context() 92 | super().__init__(ctx=ctx) 93 | 94 | PIDManager.register('getpid', os.getpid) 95 | return PIDManager() 96 | 97 | def get_pid(self, pid_manager): 98 | pid = pid_manager.getpid() 99 | # A proxy object is returned back. The needed value can be acquired 100 | # from the repr and converting that to an integer 101 | return int(str(pid)) 102 | 103 | @skip_if_windows('os.kill() with SIGINT not supported on Windows') 104 | def test_can_provide_signal_handler_initializers_to_start(self): 105 | manager = self.create_pid_manager() 106 | manager.start(signal.signal, (signal.SIGINT, signal.SIG_IGN)) 107 | pid = self.get_pid(manager) 108 | try: 109 | os.kill(pid, signal.SIGINT) 110 | except KeyboardInterrupt: 111 | pass 112 | # Try using the manager after the os.kill on the parent process. The 113 | # manager should not have died and should still be usable. 114 | self.assertEqual(pid, self.get_pid(manager)) 115 | -------------------------------------------------------------------------------- /tests/unit/test_copies.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from s3transfer.copies import CopyObjectTask, CopyPartTask 14 | from tests import BaseTaskTest, RecordingSubscriber 15 | 16 | 17 | class BaseCopyTaskTest(BaseTaskTest): 18 | def setUp(self): 19 | super().setUp() 20 | self.bucket = 'mybucket' 21 | self.key = 'mykey' 22 | self.copy_source = {'Bucket': 'mysourcebucket', 'Key': 'mysourcekey'} 23 | self.extra_args = {} 24 | self.callbacks = [] 25 | self.size = 5 26 | 27 | 28 | class TestCopyObjectTask(BaseCopyTaskTest): 29 | def get_copy_task(self, **kwargs): 30 | default_kwargs = { 31 | 'client': self.client, 32 | 'copy_source': self.copy_source, 33 | 'bucket': self.bucket, 34 | 'key': self.key, 35 | 'extra_args': self.extra_args, 36 | 'callbacks': self.callbacks, 37 | 'size': self.size, 38 | } 39 | default_kwargs.update(kwargs) 40 | return self.get_task(CopyObjectTask, main_kwargs=default_kwargs) 41 | 42 | def test_main(self): 43 | self.stubber.add_response( 44 | 'copy_object', 45 | service_response={}, 46 | expected_params={ 47 | 'Bucket': self.bucket, 48 | 'Key': self.key, 49 | 'CopySource': self.copy_source, 50 | }, 51 | ) 52 | task = self.get_copy_task() 53 | task() 54 | 55 | self.stubber.assert_no_pending_responses() 56 | 57 | def test_extra_args(self): 58 | self.extra_args['ACL'] = 'private' 59 | self.stubber.add_response( 60 | 'copy_object', 61 | service_response={}, 62 | expected_params={ 63 | 'Bucket': self.bucket, 64 | 'Key': self.key, 65 | 'CopySource': self.copy_source, 66 | 'ACL': 'private', 67 | }, 68 | ) 69 | task = self.get_copy_task() 70 | task() 71 | 72 | self.stubber.assert_no_pending_responses() 73 | 74 | def test_callbacks_invoked(self): 75 | subscriber = RecordingSubscriber() 76 | self.callbacks.append(subscriber.on_progress) 77 | self.stubber.add_response( 78 | 'copy_object', 79 | service_response={}, 80 | expected_params={ 81 | 'Bucket': self.bucket, 82 | 'Key': self.key, 83 | 'CopySource': self.copy_source, 84 | }, 85 | ) 86 | task = self.get_copy_task() 87 | task() 88 | 89 | self.stubber.assert_no_pending_responses() 90 | self.assertEqual(subscriber.calculate_bytes_seen(), self.size) 91 | 92 | 93 | class TestCopyPartTask(BaseCopyTaskTest): 94 | def setUp(self): 95 | super().setUp() 96 | self.copy_source_range = 'bytes=5-9' 97 | self.extra_args['CopySourceRange'] = self.copy_source_range 98 | self.upload_id = 'myuploadid' 99 | self.part_number = 1 100 | self.result_etag = 'my-etag' 101 | self.checksum_sha1 = 'my-checksum_sha1' 102 | 103 | def get_copy_task(self, **kwargs): 104 | default_kwargs = { 105 | 'client': self.client, 106 | 'copy_source': self.copy_source, 107 | 'bucket': self.bucket, 108 | 'key': self.key, 109 | 'upload_id': self.upload_id, 110 | 'part_number': self.part_number, 111 | 'extra_args': self.extra_args, 112 | 'callbacks': self.callbacks, 113 | 'size': self.size, 114 | } 115 | default_kwargs.update(kwargs) 116 | return self.get_task(CopyPartTask, main_kwargs=default_kwargs) 117 | 118 | def test_main(self): 119 | self.stubber.add_response( 120 | 'upload_part_copy', 121 | service_response={'CopyPartResult': {'ETag': self.result_etag}}, 122 | expected_params={ 123 | 'Bucket': self.bucket, 124 | 'Key': self.key, 125 | 'CopySource': self.copy_source, 126 | 'UploadId': self.upload_id, 127 | 'PartNumber': self.part_number, 128 | 'CopySourceRange': self.copy_source_range, 129 | }, 130 | ) 131 | task = self.get_copy_task() 132 | self.assertEqual( 133 | task(), {'PartNumber': self.part_number, 'ETag': self.result_etag} 134 | ) 135 | self.stubber.assert_no_pending_responses() 136 | 137 | def test_main_with_checksum(self): 138 | self.stubber.add_response( 139 | 'upload_part_copy', 140 | service_response={ 141 | 'CopyPartResult': { 142 | 'ETag': self.result_etag, 143 | 'ChecksumSHA1': self.checksum_sha1, 144 | } 145 | }, 146 | expected_params={ 147 | 'Bucket': self.bucket, 148 | 'Key': self.key, 149 | 'CopySource': self.copy_source, 150 | 'UploadId': self.upload_id, 151 | 'PartNumber': self.part_number, 152 | 'CopySourceRange': self.copy_source_range, 153 | }, 154 | ) 155 | task = self.get_copy_task(checksum_algorithm="sha1") 156 | self.assertEqual( 157 | task(), 158 | { 159 | 'PartNumber': self.part_number, 160 | 'ETag': self.result_etag, 161 | 'ChecksumSHA1': self.checksum_sha1, 162 | }, 163 | ) 164 | self.stubber.assert_no_pending_responses() 165 | 166 | def test_extra_args(self): 167 | self.extra_args['RequestPayer'] = 'requester' 168 | self.stubber.add_response( 169 | 'upload_part_copy', 170 | service_response={'CopyPartResult': {'ETag': self.result_etag}}, 171 | expected_params={ 172 | 'Bucket': self.bucket, 173 | 'Key': self.key, 174 | 'CopySource': self.copy_source, 175 | 'UploadId': self.upload_id, 176 | 'PartNumber': self.part_number, 177 | 'CopySourceRange': self.copy_source_range, 178 | 'RequestPayer': 'requester', 179 | }, 180 | ) 181 | task = self.get_copy_task() 182 | self.assertEqual( 183 | task(), {'PartNumber': self.part_number, 'ETag': self.result_etag} 184 | ) 185 | self.stubber.assert_no_pending_responses() 186 | 187 | def test_callbacks_invoked(self): 188 | subscriber = RecordingSubscriber() 189 | self.callbacks.append(subscriber.on_progress) 190 | self.stubber.add_response( 191 | 'upload_part_copy', 192 | service_response={'CopyPartResult': {'ETag': self.result_etag}}, 193 | expected_params={ 194 | 'Bucket': self.bucket, 195 | 'Key': self.key, 196 | 'CopySource': self.copy_source, 197 | 'UploadId': self.upload_id, 198 | 'PartNumber': self.part_number, 199 | 'CopySourceRange': self.copy_source_range, 200 | }, 201 | ) 202 | task = self.get_copy_task() 203 | self.assertEqual( 204 | task(), {'PartNumber': self.part_number, 'ETag': self.result_etag} 205 | ) 206 | self.stubber.assert_no_pending_responses() 207 | self.assertEqual(subscriber.calculate_bytes_seen(), self.size) 208 | -------------------------------------------------------------------------------- /tests/unit/test_delete.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from s3transfer.delete import DeleteObjectTask 14 | from tests import BaseTaskTest 15 | 16 | 17 | class TestDeleteObjectTask(BaseTaskTest): 18 | def setUp(self): 19 | super().setUp() 20 | self.bucket = 'mybucket' 21 | self.key = 'mykey' 22 | self.extra_args = {} 23 | self.callbacks = [] 24 | 25 | def get_delete_task(self, **kwargs): 26 | default_kwargs = { 27 | 'client': self.client, 28 | 'bucket': self.bucket, 29 | 'key': self.key, 30 | 'extra_args': self.extra_args, 31 | } 32 | default_kwargs.update(kwargs) 33 | return self.get_task(DeleteObjectTask, main_kwargs=default_kwargs) 34 | 35 | def test_main(self): 36 | self.stubber.add_response( 37 | 'delete_object', 38 | service_response={}, 39 | expected_params={ 40 | 'Bucket': self.bucket, 41 | 'Key': self.key, 42 | }, 43 | ) 44 | task = self.get_delete_task() 45 | task() 46 | 47 | self.stubber.assert_no_pending_responses() 48 | 49 | def test_extra_args(self): 50 | self.extra_args['MFA'] = 'mfa-code' 51 | self.extra_args['VersionId'] = '12345' 52 | self.stubber.add_response( 53 | 'delete_object', 54 | service_response={}, 55 | expected_params={ 56 | 'Bucket': self.bucket, 57 | 'Key': self.key, 58 | # These extra_args should be injected into the 59 | # expected params for the delete_object call. 60 | 'MFA': 'mfa-code', 61 | 'VersionId': '12345', 62 | }, 63 | ) 64 | task = self.get_delete_task() 65 | task() 66 | 67 | self.stubber.assert_no_pending_responses() 68 | -------------------------------------------------------------------------------- /tests/unit/test_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the "license" file accompanying this file. This file is 10 | # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | import time 14 | from concurrent.futures import ThreadPoolExecutor 15 | 16 | from s3transfer.exceptions import CancelledError, FatalError 17 | from s3transfer.futures import TransferCoordinator 18 | from s3transfer.manager import TransferConfig, TransferCoordinatorController 19 | from tests import TransferCoordinatorWithInterrupt, unittest 20 | 21 | 22 | class FutureResultException(Exception): 23 | pass 24 | 25 | 26 | class TestTransferConfig(unittest.TestCase): 27 | def test_exception_on_zero_attr_value(self): 28 | with self.assertRaises(ValueError): 29 | TransferConfig(max_request_queue_size=0) 30 | 31 | 32 | class TestTransferCoordinatorController(unittest.TestCase): 33 | def setUp(self): 34 | self.coordinator_controller = TransferCoordinatorController() 35 | 36 | def sleep_then_announce_done(self, transfer_coordinator, sleep_time): 37 | time.sleep(sleep_time) 38 | transfer_coordinator.set_result('done') 39 | transfer_coordinator.announce_done() 40 | 41 | def assert_coordinator_is_cancelled(self, transfer_coordinator): 42 | self.assertEqual(transfer_coordinator.status, 'cancelled') 43 | 44 | def test_add_transfer_coordinator(self): 45 | transfer_coordinator = TransferCoordinator() 46 | # Add the transfer coordinator 47 | self.coordinator_controller.add_transfer_coordinator( 48 | transfer_coordinator 49 | ) 50 | # Ensure that is tracked. 51 | self.assertEqual( 52 | self.coordinator_controller.tracked_transfer_coordinators, 53 | {transfer_coordinator}, 54 | ) 55 | 56 | def test_remove_transfer_coordinator(self): 57 | transfer_coordinator = TransferCoordinator() 58 | # Add the coordinator 59 | self.coordinator_controller.add_transfer_coordinator( 60 | transfer_coordinator 61 | ) 62 | # Now remove the coordinator 63 | self.coordinator_controller.remove_transfer_coordinator( 64 | transfer_coordinator 65 | ) 66 | # Make sure that it is no longer getting tracked. 67 | self.assertEqual( 68 | self.coordinator_controller.tracked_transfer_coordinators, set() 69 | ) 70 | 71 | def test_cancel(self): 72 | transfer_coordinator = TransferCoordinator() 73 | # Add the transfer coordinator 74 | self.coordinator_controller.add_transfer_coordinator( 75 | transfer_coordinator 76 | ) 77 | # Cancel with the canceler 78 | self.coordinator_controller.cancel() 79 | # Check that coordinator got canceled 80 | self.assert_coordinator_is_cancelled(transfer_coordinator) 81 | 82 | def test_cancel_with_message(self): 83 | message = 'my cancel message' 84 | transfer_coordinator = TransferCoordinator() 85 | self.coordinator_controller.add_transfer_coordinator( 86 | transfer_coordinator 87 | ) 88 | self.coordinator_controller.cancel(message) 89 | transfer_coordinator.announce_done() 90 | with self.assertRaisesRegex(CancelledError, message): 91 | transfer_coordinator.result() 92 | 93 | def test_cancel_with_provided_exception(self): 94 | message = 'my cancel message' 95 | transfer_coordinator = TransferCoordinator() 96 | self.coordinator_controller.add_transfer_coordinator( 97 | transfer_coordinator 98 | ) 99 | self.coordinator_controller.cancel(message, exc_type=FatalError) 100 | transfer_coordinator.announce_done() 101 | with self.assertRaisesRegex(FatalError, message): 102 | transfer_coordinator.result() 103 | 104 | def test_wait_for_done_transfer_coordinators(self): 105 | # Create a coordinator and add it to the canceler 106 | transfer_coordinator = TransferCoordinator() 107 | self.coordinator_controller.add_transfer_coordinator( 108 | transfer_coordinator 109 | ) 110 | 111 | sleep_time = 0.02 112 | with ThreadPoolExecutor(max_workers=1) as executor: 113 | # In a separate thread sleep and then set the transfer coordinator 114 | # to done after sleeping. 115 | start_time = time.time() 116 | executor.submit( 117 | self.sleep_then_announce_done, transfer_coordinator, sleep_time 118 | ) 119 | # Now call wait to wait for the transfer coordinator to be done. 120 | self.coordinator_controller.wait() 121 | end_time = time.time() 122 | wait_time = end_time - start_time 123 | # The time waited should not be less than the time it took to sleep in 124 | # the separate thread because the wait ending should be dependent on 125 | # the sleeping thread announcing that the transfer coordinator is done. 126 | self.assertTrue(sleep_time <= wait_time) 127 | 128 | def test_wait_does_not_propogate_exceptions_from_result(self): 129 | transfer_coordinator = TransferCoordinator() 130 | transfer_coordinator.set_exception(FutureResultException()) 131 | transfer_coordinator.announce_done() 132 | try: 133 | self.coordinator_controller.wait() 134 | except FutureResultException as e: 135 | self.fail(f'{e} should not have been raised.') 136 | 137 | def test_wait_can_be_interrupted(self): 138 | inject_interrupt_coordinator = TransferCoordinatorWithInterrupt() 139 | self.coordinator_controller.add_transfer_coordinator( 140 | inject_interrupt_coordinator 141 | ) 142 | with self.assertRaises(KeyboardInterrupt): 143 | self.coordinator_controller.wait() 144 | -------------------------------------------------------------------------------- /tests/unit/test_subscribers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the 'License'). You 4 | # may not use this file except in compliance with the License. A copy of 5 | # the License is located at 6 | # 7 | # http://aws.amazon.com/apache2.0/ 8 | # 9 | # or in the 'license' file accompanying this file. This file is 10 | # distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11 | # ANY KIND, either express or implied. See the License for the specific 12 | # language governing permissions and limitations under the License. 13 | from s3transfer.exceptions import InvalidSubscriberMethodError 14 | from s3transfer.subscribers import BaseSubscriber 15 | from tests import unittest 16 | 17 | 18 | class ExtraMethodsSubscriber(BaseSubscriber): 19 | def extra_method(self): 20 | return 'called extra method' 21 | 22 | 23 | class NotCallableSubscriber(BaseSubscriber): 24 | on_done = 'foo' 25 | 26 | 27 | class NoKwargsSubscriber(BaseSubscriber): 28 | def on_done(self): 29 | pass 30 | 31 | 32 | class OverrideMethodSubscriber(BaseSubscriber): 33 | def on_queued(self, **kwargs): 34 | return kwargs 35 | 36 | 37 | class OverrideConstructorSubscriber(BaseSubscriber): 38 | def __init__(self, arg1, arg2): 39 | self.arg1 = arg1 40 | self.arg2 = arg2 41 | 42 | 43 | class TestSubscribers(unittest.TestCase): 44 | def test_can_instantiate_base_subscriber(self): 45 | try: 46 | BaseSubscriber() 47 | except InvalidSubscriberMethodError: 48 | self.fail('BaseSubscriber should be instantiable') 49 | 50 | def test_can_call_base_subscriber_method(self): 51 | subscriber = BaseSubscriber() 52 | try: 53 | subscriber.on_done(future=None) 54 | except Exception as e: 55 | self.fail( 56 | 'Should be able to call base class subscriber method. ' 57 | f'instead got: {e}' 58 | ) 59 | 60 | def test_subclass_can_have_and_call_additional_methods(self): 61 | subscriber = ExtraMethodsSubscriber() 62 | self.assertEqual(subscriber.extra_method(), 'called extra method') 63 | 64 | def test_can_subclass_and_override_method_from_base_subscriber(self): 65 | subscriber = OverrideMethodSubscriber() 66 | # Make sure that the overridden method is called 67 | self.assertEqual(subscriber.on_queued(foo='bar'), {'foo': 'bar'}) 68 | 69 | def test_can_subclass_and_override_constructor_from_base_class(self): 70 | subscriber = OverrideConstructorSubscriber('foo', arg2='bar') 71 | # Make sure you can create a custom constructor. 72 | self.assertEqual(subscriber.arg1, 'foo') 73 | self.assertEqual(subscriber.arg2, 'bar') 74 | 75 | def test_invalid_arguments_in_constructor_of_subclass_subscriber(self): 76 | # The override constructor should still have validation of 77 | # constructor args. 78 | with self.assertRaises(TypeError): 79 | OverrideConstructorSubscriber() 80 | 81 | def test_not_callable_in_subclass_subscriber_method(self): 82 | with self.assertRaisesRegex( 83 | InvalidSubscriberMethodError, 'must be callable' 84 | ): 85 | NotCallableSubscriber() 86 | 87 | def test_no_kwargs_in_subclass_subscriber_method(self): 88 | with self.assertRaisesRegex( 89 | InvalidSubscriberMethodError, 'must accept keyword' 90 | ): 91 | NoKwargsSubscriber() 92 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py39,py310,py311,py312,py313 3 | 4 | # Comment to build sdist and install into virtualenv 5 | # This is helpful to test installation but takes extra time 6 | skipsdist = True 7 | 8 | [testenv] 9 | commands = 10 | {toxinidir}/scripts/ci/install 11 | {toxinidir}/scripts/ci/run-tests 12 | --------------------------------------------------------------------------------