├── .coveragerc ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.md │ ├── feature_request.md │ └── i-need-help.md ├── pull_request_template.md └── workflows │ ├── iam-policy.json │ └── main.yml ├── .gitignore ├── .readthedocs.yaml ├── AUTHORS.rst ├── CODE_OF_CONDUCT.rst ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── NOTICE ├── README.rst ├── chore.txt ├── codecov.yml ├── count_code.py ├── docs ├── Makefile ├── make.bat └── source │ ├── 01-Pure-S3-Path-Manipulation.ipynb │ ├── 02-S3-Read-API.ipynb │ ├── 03-S3-Write-API.ipynb │ ├── 04-Example-App-Sync-Two-S3-Folders.ipynb │ ├── 05-Example-App-Reinvent-Cloud-Drive.ipynb │ ├── _static │ ├── .custom-style.rst │ ├── css │ │ └── custom-style.css │ ├── images │ │ └── calculate-total-size.png │ ├── js │ │ └── sorttable.js │ ├── s3pathlib-favicon.ico │ └── s3pathlib-logo.png │ ├── conf.py │ ├── index.rst │ ├── release-history.rst │ └── s3pathlib │ ├── __init__.rst │ ├── aws.rst │ ├── better_client │ ├── __init__.rst │ ├── api.rst │ ├── delete_object.rst │ ├── head_bucket.rst │ ├── head_object.rst │ ├── list_object_versions.rst │ ├── list_objects.rst │ ├── tagging.rst │ └── upload.rst │ ├── compat.rst │ ├── constants.rst │ ├── core │ ├── __init__.rst │ ├── attribute.rst │ ├── base.rst │ ├── bucket.rst │ ├── comparison.rst │ ├── copy.rst │ ├── delete.rst │ ├── exists.rst │ ├── filterable_property.rst │ ├── is_test.rst │ ├── iter_object_versions.rst │ ├── iter_objects.rst │ ├── joinpath.rst │ ├── metadata.rst │ ├── mutate.rst │ ├── opener.rst │ ├── relative.rst │ ├── resolve_s3_client.rst │ ├── rw.rst │ ├── s3path.rst │ ├── serde.rst │ ├── sync.rst │ ├── tagging.rst │ ├── upload.rst │ └── uri.rst │ ├── exc.rst │ ├── marker.rst │ ├── metadata.rst │ ├── tag.rst │ ├── type.rst │ ├── utils.rst │ └── validate.rst ├── pygitrepo-config.json ├── release-history.rst ├── requirements-dev.txt ├── requirements-doc.txt ├── requirements-test.txt ├── requirements.txt ├── s3pathlib ├── __init__.py ├── _version.py ├── aws.py ├── better_client │ ├── __init__.py │ ├── api.py │ ├── delete_object.py │ ├── head_bucket.py │ ├── head_object.py │ ├── list_object_versions.py │ ├── list_objects.py │ ├── tagging.py │ └── upload.py ├── compat.py ├── constants.py ├── core │ ├── __init__.py │ ├── attribute.py │ ├── base.py │ ├── bucket.py │ ├── comparison.py │ ├── copy.py │ ├── delete.py │ ├── exists.py │ ├── filterable_property.py │ ├── is_test.py │ ├── iter_object_versions.py │ ├── iter_objects.py │ ├── joinpath.py │ ├── metadata.py │ ├── mutate.py │ ├── opener.py │ ├── relative.py │ ├── resolve_s3_client.py │ ├── rw.py │ ├── s3path.py │ ├── serde.py │ ├── sync.py │ ├── tagging.py │ ├── upload.py │ └── uri.py ├── docs │ └── __init__.py ├── exc.py ├── marker.py ├── metadata.py ├── tag.py ├── tests │ ├── __init__.py │ ├── helpers.py │ ├── mock.py │ └── paths.py ├── type.py ├── utils.py └── validate.py ├── setup.py ├── tests ├── all.py ├── api │ ├── README.rst │ ├── all.py │ └── test_import.py ├── better_client │ ├── README.rst │ ├── all.py │ ├── dummy_data.py │ ├── test_client_delete_object.py │ ├── test_client_head_bucket.py │ ├── test_client_head_object.py │ ├── test_client_list_object_versions.py │ ├── test_client_list_objects.py │ ├── test_client_tagging.py │ └── test_client_upload.py ├── conftest.py ├── core │ ├── .gitignore │ ├── all.py │ ├── test_core_attribute.py │ ├── test_core_base.py │ ├── test_core_bucket.py │ ├── test_core_comparison.py │ ├── test_core_copy.py │ ├── test_core_delete.py │ ├── test_core_exists.py │ ├── test_core_filterable_property.py │ ├── test_core_is_test.py │ ├── test_core_iter_object_versions.py │ ├── test_core_iter_objects.py │ ├── test_core_joinpath.py │ ├── test_core_metadata.py │ ├── test_core_mutate.py │ ├── test_core_opener.py │ ├── test_core_relative.py │ ├── test_core_resolve_s3_client.py │ ├── test_core_rw.py │ ├── test_core_serde.py │ ├── test_core_sync.py │ ├── test_core_tagging.py │ ├── test_core_upload.py │ ├── test_core_uri.py │ ├── test_iter_objects │ │ ├── README.txt │ │ ├── folder-description.txt │ │ ├── folder1 │ │ │ ├── 1.txt │ │ │ ├── 2.txt │ │ │ └── 3.txt │ │ ├── folder2 │ │ │ ├── 4.txt │ │ │ ├── 5.txt │ │ │ └── 6.txt │ │ └── folder3 │ │ │ ├── 7.txt │ │ │ ├── 8.txt │ │ │ └── 9.txt │ └── test_upload_dir │ │ ├── 1.txt │ │ └── subfolder │ │ └── 2.txt ├── data │ ├── list_objects_folder │ │ ├── README.txt │ │ ├── folder-description.txt │ │ ├── folder1 │ │ │ ├── 1.txt │ │ │ ├── 2.txt │ │ │ └── 3.txt │ │ ├── folder2 │ │ │ ├── 4.txt │ │ │ ├── 5.txt │ │ │ └── 6.txt │ │ └── folder3 │ │ │ ├── 7.txt │ │ │ ├── 8.txt │ │ │ └── 9.txt │ └── upload_dir_folder │ │ ├── 1.txt │ │ ├── data1.json │ │ └── subfolder │ │ ├── 2.txt │ │ └── data2.json ├── test_exc.py ├── test_marker.py ├── test_metadata.py ├── test_tag.py ├── test_utils.py └── test_validate.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | # Coverage.py is a tool for measuring code coverage of Python programs. 2 | # for more info: https://coverage.readthedocs.io/en/latest/config.html 3 | [run] 4 | omit = 5 | s3pathlib/docs/* 6 | s3pathlib/tests/* 7 | s3pathlib/_version.py 8 | s3pathlib/compat.py 9 | 10 | [report] 11 | # Regexes for lines to exclude from consideration 12 | exclude_lines = 13 | # Have to re-enable the standard pragma 14 | pragma: no cover 15 | 16 | # Don't complain about missing debug-only code: 17 | def __repr__ 18 | if self\.debug 19 | 20 | # Don't complain if tests don't hit defensive assertion code: 21 | raise AssertionError 22 | raise NotImplementedError 23 | 24 | # Don't complain if non-runnable code isn't run: 25 | if 0: 26 | if __name__ == .__main__.: 27 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Report bugs to the author or the community, we appreciate your contribution! 4 | title: "[Bug]: describe the bug here" 5 | labels: bug 6 | assignees: MacHu-GWU 7 | 8 | --- 9 | 10 | ## Describe the Bug 11 | 12 | **What is the expected behavior** 13 | 14 | ... 15 | 16 | **What is the actual behavior** 17 | 18 | ... 19 | 20 | ## How to Reproduce this Bug 21 | 22 | **Code to reproduce the bug**: 23 | 24 | ```python 25 | import something 26 | ``` 27 | 28 | **Describe your Environment**: 29 | 30 | - Operation System: Windows / MacOS / Linux 31 | - What Python: CPython / Conda / PyPI 32 | - Python version: 2.7 / 3.6 / 3.7 / 3.8 / 3.9 / 3.10 33 | - Network environment: home / work with VPN / cloud 34 | 35 | ## Any Additional Information 36 | 37 | [add any additional information here] 38 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[Feature] describe the feature you want here" 5 | labels: feature 6 | assignees: MacHu-GWU 7 | 8 | --- 9 | 10 | ## Describe the feature you want 11 | 12 | I want to use this API 13 | 14 | ```python 15 | from s3pathlib import S3Path 16 | 17 | for s3path in S3Path("my-bucket", "my-folder").iter_objects(): 18 | ... 19 | ``` 20 | 21 | Gives me: 22 | 23 | ```python 24 | ValueError 25 | ``` 26 | 27 | ## Why you believe it benefits the users 28 | 29 | [describe your solid reason here] 30 | 31 | ## Any Additional Information 32 | 33 | [add any additional information here] 34 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/i-need-help.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: I need help 3 | about: Ask the author or the community for help! 4 | title: "[Help]: describe your question here" 5 | labels: question 6 | assignees: MacHu-GWU 7 | 8 | --- 9 | 10 | ## My Objective 11 | 12 | [describe your objective here] I am trying to do XYZ 13 | 14 | ## My Existing Work 15 | 16 | [you may have tried some codes, describe it here, tell me what is your expected output / behavior] 17 | 18 | I am expecting this code: 19 | 20 | ```python 21 | name = "Alice" 22 | print(f"Hello {name}") 23 | ``` 24 | 25 | gives me this output: 26 | 27 | ```python 28 | Hello Alice 29 | ``` 30 | 31 | However it raise an error: 32 | 33 | ```python 34 | ValueError 35 | ``` 36 | 37 | ## Any Additional Information 38 | 39 | [add any additional information here] 40 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Title 2 | 3 | Please **pick a concise, informative and complete title** for your PR. 4 | 5 | ## Motivation 6 | 7 | Please explain the motivation behind this PR in the description. 8 | 9 | ## Issue #, if available 10 | 11 | If you're adding a new feature, then consider opening an issue and discussing it with the maintainers (GitHub Username ``MacHu-GWU``) before you actually do the hard work. 12 | 13 | ## Description of changes: 14 | 15 | Please explain the changes you have made here. 16 | 17 | By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. 18 | 19 | ## Tests 20 | 21 | If you're fixing a bug, consider [test-driven development](https://en.wikipedia.org/wiki/Test-driven_development): 22 | 23 | 1. Create a unit test that demonstrates the bug. The test should **fail**. 24 | 2. Implement your bug fix. 25 | 3. The test you created should now **pass**. 26 | 27 | If you're implementing a new feature, include unit tests for it. 28 | 29 | Make sure all existing unit tests pass. 30 | 31 | ## Work in progress 32 | 33 | If you're still working on your PR, include "WIP" in the title. 34 | We'll skip reviewing it for the time being. 35 | Once you're ready to review, remove the "WIP" from the title, and ping one of the maintainers (e.g. mpenkov). 36 | 37 | ## Checklist 38 | 39 | Before you create the PR, please make sure you have: 40 | 41 | - [ ] Picked a concise, informative and complete title 42 | - [ ] Clearly explained the motivation behind the PR 43 | - [ ] Linked to any existing issues that your PR will be solving 44 | - [ ] Included tests for any new functionality 45 | - [ ] Checked that all unit tests pass 46 | 47 | ## Workflow 48 | 49 | Please avoid rebasing and force-pushing to the branch of the PR once a review is in progress. 50 | Rebasing can make your commits look a bit cleaner, but it also makes life more difficult from the reviewer, because they are no longer able to distinguish between code that has already been reviewed, and unreviewed code. 51 | -------------------------------------------------------------------------------- /.github/workflows/iam-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "VisualEditor0", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "s3:AbortMultipartUpload", 9 | "s3:CreateBucket", 10 | "s3:DeleteBucket", 11 | "s3:DeleteBucketWebsite", 12 | "s3:DeleteObject", 13 | "s3:DeleteObjectTagging", 14 | "s3:DeleteObjectVersion", 15 | "s3:DeleteObjectVersionTagging", 16 | "s3:GetAccelerateConfiguration", 17 | "s3:GetAnalyticsConfiguration", 18 | "s3:GetBucketAcl", 19 | "s3:GetBucketCORS", 20 | "s3:GetBucketLocation", 21 | "s3:GetBucketLogging", 22 | "s3:GetBucketNotification", 23 | "s3:GetBucketObjectLockConfiguration", 24 | "s3:GetBucketOwnershipControls", 25 | "s3:GetBucketPolicy", 26 | "s3:GetBucketPolicyStatus", 27 | "s3:GetBucketPublicAccessBlock", 28 | "s3:GetBucketRequestPayment", 29 | "s3:GetBucketTagging", 30 | "s3:GetBucketVersioning", 31 | "s3:GetBucketWebsite", 32 | "s3:GetEncryptionConfiguration", 33 | "s3:GetIntelligentTieringConfiguration", 34 | "s3:GetInventoryConfiguration", 35 | "s3:GetLifecycleConfiguration", 36 | "s3:GetMetricsConfiguration", 37 | "s3:GetObject", 38 | "s3:GetObjectAcl", 39 | "s3:GetObjectLegalHold", 40 | "s3:GetObjectRetention", 41 | "s3:GetObjectTagging", 42 | "s3:GetObjectTorrent", 43 | "s3:GetObjectVersion", 44 | "s3:GetObjectVersionAcl", 45 | "s3:GetObjectVersionForReplication", 46 | "s3:GetObjectVersionTagging", 47 | "s3:GetObjectVersionTorrent", 48 | "s3:GetReplicationConfiguration", 49 | "s3:ListBucket", 50 | "s3:ListBucketMultipartUploads", 51 | "s3:ListBucketVersions", 52 | "s3:ListMultipartUploadParts", 53 | "s3:PutAccelerateConfiguration", 54 | "s3:PutBucketCORS", 55 | "s3:PutBucketLogging", 56 | "s3:PutBucketNotification", 57 | "s3:PutBucketObjectLockConfiguration", 58 | "s3:PutBucketOwnershipControls", 59 | "s3:PutBucketRequestPayment", 60 | "s3:PutBucketTagging", 61 | "s3:PutBucketVersioning", 62 | "s3:PutBucketWebsite", 63 | "s3:PutEncryptionConfiguration", 64 | "s3:PutIntelligentTieringConfiguration", 65 | "s3:PutInventoryConfiguration", 66 | "s3:PutLifecycleConfiguration", 67 | "s3:PutMetricsConfiguration", 68 | "s3:PutObject", 69 | "s3:PutObjectLegalHold", 70 | "s3:PutObjectRetention", 71 | "s3:PutObjectTagging", 72 | "s3:PutObjectVersionTagging", 73 | "s3:PutReplicationConfiguration", 74 | "s3:ReplicateDelete", 75 | "s3:ReplicateObject", 76 | "s3:ReplicateTags", 77 | "s3:RestoreObject" 78 | ], 79 | "Resource": [ 80 | "arn:aws:s3:::${aws_account_id}-us-east-1-s3pathlib-test", 81 | "arn:aws:s3:::${aws_account_id}-us-east-1-s3pathlib-test-versioning-on", 82 | "arn:aws:s3:::${aws_account_id}-us-east-1-s3pathlib-test/*", 83 | "arn:aws:s3:::${aws_account_id}-us-east-1-s3pathlib-test-versioning-on/*" 84 | ] 85 | }, 86 | { 87 | "Sid": "VisualEditor1", 88 | "Effect": "Allow", 89 | "Action": "sts:GetCallerIdentity", 90 | "Resource": "*" 91 | } 92 | ] 93 | } -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | # comprehensive github action yml reference: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions 2 | 3 | --- 4 | name: CI 5 | 6 | on: 7 | push: # any push event to main will trigger this 8 | branches: ["main"] 9 | pull_request: # any pull request to main will trigger this 10 | branches: ["main"] 11 | workflow_dispatch: # allows you to manually trigger run 12 | 13 | jobs: 14 | tests: 15 | name: "${{ matrix.os }} Python ${{ matrix.python-version }}" 16 | runs-on: "${{ matrix.os }}" # for all available VM runtime, see this: https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners 17 | env: # define environment variables 18 | USING_COVERAGE: "3.7,3.8,3.9,3.10,3.11" 19 | strategy: 20 | matrix: 21 | os: ["ubuntu-latest", "windows-latest"] 22 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] 23 | # os: ["ubuntu-latest", ] # for debug only 24 | # os: ["windows-latest", ] # for debug only 25 | # python-version: ["3.7", ] # for debug only 26 | exclude: 27 | - os: windows-latest # this is a useless exclude rules for demonstration use only 28 | python-version: 2.7 29 | steps: 30 | - uses: "actions/checkout@v3" # https://github.com/marketplace/actions/checkout 31 | - uses: "actions/setup-python@v4" # https://github.com/marketplace/actions/setup-python 32 | with: 33 | python-version: "${{ matrix.python-version }}" 34 | 35 | - name: "Install dependencies on MacOS or Linux" 36 | if: matrix.os == 'ubuntu-latest' # for condition steps, you should put if at begin, and use single quote for logical expression 37 | env: 38 | AWS_ACCESS_KEY_ID_FOR_GITHUB_CI: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_GITHUB_CI }} 39 | AWS_SECRET_ACCESS_KEY_FOR_GITHUB_CI: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_GITHUB_CI }} 40 | run: | 41 | set -xe 42 | python -VV 43 | python -m site 44 | python -m pip install --upgrade pip setuptools wheel virtualenv 45 | pip install -r requirements.txt 46 | pip install -r requirements-dev.txt 47 | pip install -r requirements-test.txt 48 | pip install . 49 | - name: "Install dependencies on Windows" 50 | if: matrix.os == 'windows-latest' 51 | env: 52 | AWS_ACCESS_KEY_ID_FOR_GITHUB_CI: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_GITHUB_CI }} 53 | AWS_SECRET_ACCESS_KEY_FOR_GITHUB_CI: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_GITHUB_CI }} 54 | run: | 55 | python -m site 56 | python -m pip install --upgrade pip setuptools wheel virtualenv 57 | pip install -r requirements.txt 58 | pip install -r requirements-dev.txt 59 | pip install -r requirements-test.txt 60 | pip install . 61 | - name: "Run pytest" 62 | env: 63 | AWS_ACCESS_KEY_ID_FOR_GITHUB_CI: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_GITHUB_CI }} 64 | AWS_SECRET_ACCESS_KEY_FOR_GITHUB_CI: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_GITHUB_CI }} 65 | run: "python -m pytest tests --cov=s3pathlib" 66 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # User Settings 2 | s3pathlib_venv/ 3 | s3pathlib.egg-info/ 4 | s3pathlib-*/ 5 | tmp/ 6 | 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | env/ 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | 34 | # PyCharm 35 | .idea/ 36 | 37 | # PyInstaller 38 | # Usually these files are written by a python script from a template 39 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 40 | *.manifest 41 | *.spec 42 | 43 | # Installer logs 44 | pip-log.txt 45 | pip-delete-this-directory.txt 46 | 47 | # Unit test / coverage reports 48 | htmlcov/ 49 | .tox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | .pytest_cache/ 54 | nosetests.xml 55 | coverage.xml 56 | *,cover 57 | .hypothesis/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | 67 | # Flask instance folder 68 | instance/ 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # IPython Notebook 80 | .ipynb_checkpoints 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # dotenv 89 | .env 90 | 91 | # virtualenv 92 | venv/ 93 | ENV/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # ========================= 102 | # Operating System Files 103 | # ========================= 104 | 105 | # OSX 106 | # ========================= 107 | 108 | .DS_Store 109 | .AppleDouble 110 | .LSOverride 111 | 112 | # Thumbnails 113 | ._* 114 | 115 | # Files that might appear in the root of a volume 116 | .DocumentRevisions-V100 117 | .fseventsd 118 | .Spotlight-V100 119 | .TemporaryItems 120 | .Trashes 121 | .VolumeIcon.icns 122 | 123 | # Directories potentially created on remote AFP share 124 | .AppleDB 125 | .AppleDesktop 126 | Network Trash Folder 127 | Temporary Items 128 | .apdisk 129 | 130 | # Windows 131 | # ========================= 132 | 133 | # Windows image file caches 134 | Thumbs.db 135 | ehthumbs.db 136 | 137 | # Folder config file 138 | Desktop.ini 139 | 140 | # Recycle Bin used on file shares 141 | $RECYCLE.BIN/ 142 | 143 | # Windows Installer files 144 | *.cab 145 | *.msi 146 | *.msm 147 | *.msp 148 | 149 | # Windows shortcuts 150 | *.lnk 151 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-20.04 5 | tools: 6 | python: "3.8" 7 | 8 | python: 9 | install: 10 | - method: pip 11 | path: . 12 | extra_requirements: 13 | - docs 14 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | Authors 2 | ============================================================================== 3 | 4 | 5 | The Creator 6 | ------------------------------------------------------------------------------ 7 | - Sanhe Hu `@MacHu-GWU `_ 8 | 9 | 10 | Maintainer Team 11 | ------------------------------------------------------------------------------ 12 | - Sanhe Hu `@MacHu-GWU `_ 13 | 14 | 15 | Contributors 16 | ------------------------------------------------------------------------------ 17 | - Sanhe Hu `@MacHu-GWU `_ 18 | - Ivan Chen `@chen115y `_ 19 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.rst: -------------------------------------------------------------------------------- 1 | Code of Conduct 2 | ============================================================================== 3 | This project has adopted the `Amazon Open Source Code of Conduct `_. 4 | For more information see the `Code of Conduct FAQ `_ or contact opensource-codeofconduct@amazon.com with any additional questions or comments. -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. _contributing-guidelines: 2 | 3 | Contributing Guidelines 4 | ============================================================================== 5 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 6 | documentation, we greatly value feedback and contributions from our community. 7 | 8 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 9 | information to effectively respond to your bug report or contribution. 10 | 11 | 12 | Reporting Bugs/Feature Requests 13 | ------------------------------------------------------------------------------ 14 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 15 | 16 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 17 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 18 | 19 | - A reproducible test case or series of steps 20 | - The version of our code being used 21 | - Any modifications you've made relevant to the bug 22 | - Anything unusual about your environment or deployment 23 | 24 | 25 | Contributing via Pull Requests 26 | ------------------------------------------------------------------------------ 27 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 28 | 29 | 1. You are working against the latest source on the *main* branch. 30 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 31 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 32 | 33 | To send us a pull request, please: 34 | 35 | 1. Fork the repository. 36 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 37 | 3. Ensure local tests pass. 38 | 4. Commit to your fork using clear commit messages. 39 | 5. Send us a pull request, answering any default questions in the pull request interface. 40 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 41 | 42 | GitHub provides additional document on `forking a repository `_ and 43 | `creating a pull request `_. 44 | 45 | 46 | Finding contributions to work on 47 | ------------------------------------------------------------------------------ 48 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 49 | 50 | 51 | Code of Conduct 52 | ------------------------------------------------------------------------------ 53 | This project has adopted the `Amazon Open Source Code of Conduct `_. 54 | 55 | For more information see the `Code of Conduct FAQ `_ or contact opensource-codeofconduct@amazon.com with any additional questions or comments. 56 | 57 | 58 | Security issue notifications 59 | ------------------------------------------------------------------------------ 60 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our `vulnerability reporting page `_. Please do **not** create a public github issue. 61 | 62 | 63 | Licensing 64 | ------------------------------------------------------------------------------ 65 | See the `LICENSE <./LICENSE>`_ file for our project's licensing. We will ask you to confirm the licensing of your contribution. 66 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # A MANIFEST.in file can be added in a project to define the list of files 2 | # to include in the distribution built by the sdist command. 3 | # 4 | # For more info: https://docs.python.org/2/distutils/sourcedist.html#manifest-related-options 5 | 6 | include s3pathlib 7 | include *.txt 8 | include *.rst 9 | exclude *.pyc 10 | exclude *.pyo -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | S3PathLib 2 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -------------------------------------------------------------------------------- /chore.txt: -------------------------------------------------------------------------------- 1 | 0eec75f8addbfc88909a376b113be79d -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: yes 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | range: "0...100" 8 | 9 | parsers: 10 | gcov: 11 | branch_detection: 12 | conditional: yes 13 | loop: yes 14 | method: no 15 | macro: no 16 | 17 | comment: 18 | layout: "reach,diff,flags,files,footer" 19 | behavior: default 20 | require_changes: no 21 | -------------------------------------------------------------------------------- /count_code.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import typing as T 4 | import json 5 | import subprocess 6 | from pathlib_mate import Path 7 | 8 | dir_project_root = Path.dir_here(__file__).absolute() 9 | 10 | 11 | def cloc(path: T.Union[Path, T.List[Path]]) -> dict: 12 | if isinstance(path, list): 13 | cloc_list_file.write_text("\n".join([str(p) for p in path])) 14 | args = [ 15 | "cloc", 16 | f"--list-file={cloc_list_file}", 17 | "--json", 18 | ] 19 | else: 20 | args = [ 21 | "cloc", 22 | f"{path}", 23 | "--json", 24 | ] 25 | result = subprocess.run(args, capture_output=True) 26 | data = json.loads(result.stdout.decode("utf-8")) 27 | del data["header"] 28 | return data 29 | 30 | 31 | def count_code(title, path: T.Union[Path, T.List[Path]]): 32 | data = cloc(path) 33 | print(f"-------------------- {title} --------------------") 34 | print(json.dumps(data, indent=4)) 35 | 36 | 37 | if __name__ == "__main__": 38 | cloc_list_file = dir_project_root.joinpath(".cloc-list-file") 39 | 40 | count_code( 41 | "source code", 42 | [ 43 | dir_project_root.joinpath("s3pathlib"), 44 | ], 45 | ) 46 | count_code( 47 | "test code", 48 | [ 49 | dir_project_root.joinpath("tests"), 50 | ], 51 | ) 52 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = s3pathlib 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=python -msphinx 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=s3pathlib 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The Sphinx module was not found. Make sure you have Sphinx installed, 20 | echo.then set the SPHINXBUILD environment variable to point to the full 21 | echo.path of the 'sphinx-build' executable. Alternatively you may add the 22 | echo.Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/source/04-Example-App-Sync-Two-S3-Folders.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "source": [ 6 | "# Example App - Sync Two S3 Folder\n", 7 | "\n", 8 | "In this tutorial, you will learn how to create a simple app that can sync files from one folder to another folder in S3 while preserving the same folder structure in real-time.\n", 9 | "\n", 10 | "To get started, you should first [configure an S3 put object event trigger](https://docs.aws.amazon.com/lambda/latest/dg/with-s3.html) that can monitor any changes made to the source S3 folder. After that, you can proceed to [create a Lambda function](https://docs.aws.amazon.com/lambda/latest/dg/lambda-python.html) that can handle the event and sync the file to the target S3 folder." 11 | ], 12 | "metadata": { 13 | "collapsed": false, 14 | "pycharm": { 15 | "name": "#%% md\n" 16 | } 17 | } 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 4, 22 | "outputs": [], 23 | "source": [ 24 | "# this is your lambda function code\n", 25 | "\n", 26 | "from s3pathlib import S3Path\n", 27 | "\n", 28 | "s3dir_source = S3Path(\"s3pathlib/example-app/sync-two-s3-folders/source/\")\n", 29 | "s3dir_target = S3Path(\"s3pathlib/example-app/sync-two-s3-folders/target/\")\n", 30 | "\n", 31 | "def lambda_handler(event, context):\n", 32 | " # parse s3 put object event\n", 33 | " s3path_source = S3Path(\n", 34 | " event[\"Records\"][0][\"s3\"][\"bucket\"][\"name\"],\n", 35 | " event[\"Records\"][0][\"s3\"][\"object\"][\"key\"],\n", 36 | " )\n", 37 | " # find out the target s3 location\n", 38 | " s3path_target = s3dir_target.joinpath(\n", 39 | " s3path_source.relative_to(s3dir_source)\n", 40 | " )\n", 41 | " # copy data\n", 42 | " s3path_source.copy_to(s3path_target, overwrite=True)" 43 | ], 44 | "metadata": { 45 | "collapsed": false, 46 | "pycharm": { 47 | "name": "#%%\n" 48 | } 49 | } 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 13, 54 | "outputs": [ 55 | { 56 | "data": { 57 | "text/plain": "S3Path('s3://s3pathlib/example-app/sync-two-s3-folders/target/')" 58 | }, 59 | "execution_count": 13, 60 | "metadata": {}, 61 | "output_type": "execute_result" 62 | } 63 | ], 64 | "source": [ 65 | "# firstly, we clean up the source and target location\n", 66 | "s3dir_source.delete()\n", 67 | "s3dir_target.delete()" 68 | ], 69 | "metadata": { 70 | "collapsed": false, 71 | "pycharm": { 72 | "name": "#%%\n" 73 | } 74 | } 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 14, 79 | "outputs": [ 80 | { 81 | "data": { 82 | "text/plain": "S3Path('s3://s3pathlib/example-app/sync-two-s3-folders/source/folder/file.txt')" 83 | }, 84 | "execution_count": 14, 85 | "metadata": {}, 86 | "output_type": "execute_result" 87 | } 88 | ], 89 | "source": [ 90 | "# then we create a file in the source location\n", 91 | "s3path_source = s3dir_source.joinpath(\"folder/file.txt\")\n", 92 | "s3path_source.write_text(\"hello\")" 93 | ], 94 | "metadata": { 95 | "collapsed": false, 96 | "pycharm": { 97 | "name": "#%%\n" 98 | } 99 | } 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 15, 104 | "outputs": [ 105 | { 106 | "data": { 107 | "text/plain": "0" 108 | }, 109 | "execution_count": 15, 110 | "metadata": {}, 111 | "output_type": "execute_result" 112 | } 113 | ], 114 | "source": [ 115 | "# at begin, the s3 target folder doesn't have any file\n", 116 | "s3dir_target.count_objects()" 117 | ], 118 | "metadata": { 119 | "collapsed": false, 120 | "pycharm": { 121 | "name": "#%%\n" 122 | } 123 | } 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 16, 128 | "outputs": [], 129 | "source": [ 130 | "# we use this code to simulate the s3 put object event\n", 131 | "event = {\n", 132 | " \"Records\": [\n", 133 | " {\n", 134 | " \"s3\": {\n", 135 | " \"bucket\": {\"name\": s3path_source.bucket},\n", 136 | " \"object\": {\"key\": s3path_source.key}\n", 137 | " }\n", 138 | " }\n", 139 | " ]\n", 140 | "}\n", 141 | "lambda_handler(event, None)" 142 | ], 143 | "metadata": { 144 | "collapsed": false, 145 | "pycharm": { 146 | "name": "#%%\n" 147 | } 148 | } 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 11, 153 | "outputs": [ 154 | { 155 | "data": { 156 | "text/plain": "[S3Path('s3://s3pathlib/example-app/sync-two-s3-folders/target/folder/file.txt')]" 157 | }, 158 | "execution_count": 11, 159 | "metadata": {}, 160 | "output_type": "execute_result" 161 | } 162 | ], 163 | "source": [ 164 | "# it should have one file now\n", 165 | "s3dir_target.iter_objects().all()" 166 | ], 167 | "metadata": { 168 | "collapsed": false, 169 | "pycharm": { 170 | "name": "#%%\n" 171 | } 172 | } 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": null, 177 | "outputs": [], 178 | "source": [], 179 | "metadata": { 180 | "collapsed": false, 181 | "pycharm": { 182 | "name": "#%%\n" 183 | } 184 | } 185 | } 186 | ], 187 | "metadata": { 188 | "kernelspec": { 189 | "display_name": "Python 3", 190 | "language": "python", 191 | "name": "python3" 192 | }, 193 | "language_info": { 194 | "codemirror_mode": { 195 | "name": "ipython", 196 | "version": 2 197 | }, 198 | "file_extension": ".py", 199 | "mimetype": "text/x-python", 200 | "name": "python", 201 | "nbconvert_exporter": "python", 202 | "pygments_lexer": "ipython2", 203 | "version": "2.7.6" 204 | } 205 | }, 206 | "nbformat": 4, 207 | "nbformat_minor": 0 208 | } -------------------------------------------------------------------------------- /docs/source/_static/.custom-style.rst: -------------------------------------------------------------------------------- 1 | .. role:: black 2 | .. role:: gray 3 | .. role:: grey 4 | .. role:: silver 5 | .. role:: white 6 | .. role:: maroon 7 | .. role:: red 8 | .. role:: magenta 9 | .. role:: fuchsia 10 | .. role:: pink 11 | .. role:: orange 12 | .. role:: yellow 13 | .. role:: lime 14 | .. role:: green 15 | .. role:: olive 16 | .. role:: teal 17 | .. role:: cyan 18 | .. role:: aqua 19 | .. role:: blue 20 | .. role:: navy 21 | .. role:: purple 22 | 23 | .. role:: under 24 | .. role:: over 25 | .. role:: blink 26 | .. role:: line 27 | .. role:: strike 28 | 29 | .. role:: it 30 | .. role:: ob 31 | 32 | .. role:: small 33 | .. role:: large 34 | -------------------------------------------------------------------------------- /docs/source/_static/css/custom-style.css: -------------------------------------------------------------------------------- 1 | /* additional style */ 2 | .black { 3 | color: black; 4 | } 5 | 6 | .gray { 7 | color: gray; 8 | } 9 | 10 | .grey { 11 | color: gray; 12 | } 13 | 14 | .silver { 15 | color: silver; 16 | } 17 | 18 | .white { 19 | color: white; 20 | } 21 | 22 | .maroon { 23 | color: maroon; 24 | } 25 | 26 | .red { 27 | color: red; 28 | } 29 | 30 | .magenta { 31 | color: magenta; 32 | } 33 | 34 | .fuchsia { 35 | color: fuchsia; 36 | } 37 | 38 | .pink { 39 | color: pink; 40 | } 41 | 42 | .orange { 43 | color: orange; 44 | } 45 | 46 | .yellow { 47 | color: yellow; 48 | } 49 | 50 | .lime { 51 | color: lime; 52 | } 53 | 54 | .green { 55 | color: green; 56 | } 57 | 58 | .olive { 59 | color: olive; 60 | } 61 | 62 | .teal { 63 | color: teal; 64 | } 65 | 66 | .cyan { 67 | color: cyan; 68 | } 69 | 70 | .aqua { 71 | color: aqua; 72 | } 73 | 74 | .blue { 75 | color: blue; 76 | } 77 | 78 | .navy { 79 | color: navy; 80 | } 81 | 82 | .purple { 83 | color: purple; 84 | } 85 | 86 | .under { 87 | text-decoration: underline; 88 | } 89 | 90 | .over { 91 | text-decoration: overline; 92 | } 93 | 94 | .blink { 95 | text-decoration: blink; 96 | } 97 | 98 | .line { 99 | text-decoration: line-through; 100 | } 101 | 102 | .strike { 103 | text-decoration: line-through; 104 | } 105 | 106 | .it { 107 | font-style: italic; 108 | } 109 | 110 | .ob { 111 | font-style: oblique; 112 | } 113 | 114 | .small { 115 | font-size: small; 116 | } 117 | 118 | .large { 119 | font-size: large; 120 | } 121 | 122 | /* sort table library */ 123 | table.sortable th:not(.sorttable_sorted):not(.sorttable_sorted_reverse):not(.sorttable_nosort):after { 124 | content: " \25B4\25BE" 125 | } 126 | table.sortable tbody tr:nth-child(2n) td { 127 | background: #ffffff; 128 | } 129 | table.sortable tbody tr:nth-child(2n+1) td { 130 | background: #f2f2f2; 131 | } 132 | 133 | .search__outer__input { 134 | background-color: #0a0a0a; 135 | } -------------------------------------------------------------------------------- /docs/source/_static/images/calculate-total-size.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/s3pathlib-project/ab535a916a469928b25ccd3fa0ae2e0dbbc1ae36/docs/source/_static/images/calculate-total-size.png -------------------------------------------------------------------------------- /docs/source/_static/s3pathlib-favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/s3pathlib-project/ab535a916a469928b25ccd3fa0ae2e0dbbc1ae36/docs/source/_static/s3pathlib-favicon.ico -------------------------------------------------------------------------------- /docs/source/_static/s3pathlib-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/s3pathlib-project/ab535a916a469928b25ccd3fa0ae2e0dbbc1ae36/docs/source/_static/s3pathlib-logo.png -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Release v\ |release| (:ref:`What's new? `). 2 | 3 | .. include:: ../../README.rst 4 | 5 | 6 | Install 7 | ------------------------------------------------------------------------------ 8 | 9 | ``s3pathlib`` is released on PyPI, so all you need is: 10 | 11 | .. code-block:: console 12 | 13 | $ pip install s3pathlib 14 | 15 | To upgrade to latest version: 16 | 17 | .. code-block:: console 18 | 19 | $ pip install --upgrade s3pathlib 20 | 21 | 22 | Comprehensive Guide 23 | ------------------------------------------------------------------------------ 24 | 25 | .. toctree:: 26 | :maxdepth: 1 27 | :caption: Table of Content 28 | 29 | 01-Pure-S3-Path-Manipulation.ipynb 30 | 02-S3-Read-API.ipynb 31 | 03-S3-Write-API.ipynb 32 | 04-Example-App-Sync-Two-S3-Folders.ipynb 33 | 05-Example-App-Reinvent-Cloud-Drive.ipynb 34 | 35 | 36 | API Document 37 | ------------------------------------------------------------------------------ 38 | 39 | * :ref:`by Name ` 40 | * :ref:`by Structure ` 41 | -------------------------------------------------------------------------------- /docs/source/release-history.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. include:: ../../release-history.rst 4 | -------------------------------------------------------------------------------- /docs/source/s3pathlib/__init__.rst: -------------------------------------------------------------------------------- 1 | s3pathlib 2 | ========= 3 | 4 | .. automodule:: s3pathlib 5 | :members: 6 | 7 | sub packages and modules 8 | ------------------------ 9 | 10 | .. toctree:: 11 | :maxdepth: 1 12 | 13 | better_client 14 | core 15 | aws 16 | compat 17 | constants 18 | exc 19 | marker 20 | metadata 21 | tag 22 | type 23 | utils 24 | validate 25 | -------------------------------------------------------------------------------- /docs/source/s3pathlib/aws.rst: -------------------------------------------------------------------------------- 1 | aws 2 | === 3 | 4 | .. automodule:: s3pathlib.aws 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/better_client/__init__.rst: -------------------------------------------------------------------------------- 1 | better_client 2 | ============= 3 | 4 | .. automodule:: s3pathlib.better_client 5 | :members: 6 | 7 | sub packages and modules 8 | ------------------------ 9 | 10 | .. toctree:: 11 | :maxdepth: 1 12 | 13 | api 14 | delete_object 15 | head_bucket 16 | head_object 17 | list_object_versions 18 | list_objects 19 | tagging 20 | upload 21 | -------------------------------------------------------------------------------- /docs/source/s3pathlib/better_client/api.rst: -------------------------------------------------------------------------------- 1 | api 2 | === 3 | 4 | .. automodule:: s3pathlib.better_client.api 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/better_client/delete_object.rst: -------------------------------------------------------------------------------- 1 | delete_object 2 | ============= 3 | 4 | .. automodule:: s3pathlib.better_client.delete_object 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/better_client/head_bucket.rst: -------------------------------------------------------------------------------- 1 | head_bucket 2 | =========== 3 | 4 | .. automodule:: s3pathlib.better_client.head_bucket 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/better_client/head_object.rst: -------------------------------------------------------------------------------- 1 | head_object 2 | =========== 3 | 4 | .. automodule:: s3pathlib.better_client.head_object 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/better_client/list_object_versions.rst: -------------------------------------------------------------------------------- 1 | list_object_versions 2 | ==================== 3 | 4 | .. automodule:: s3pathlib.better_client.list_object_versions 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/better_client/list_objects.rst: -------------------------------------------------------------------------------- 1 | list_objects 2 | ============ 3 | 4 | .. automodule:: s3pathlib.better_client.list_objects 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/better_client/tagging.rst: -------------------------------------------------------------------------------- 1 | tagging 2 | ======= 3 | 4 | .. automodule:: s3pathlib.better_client.tagging 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/better_client/upload.rst: -------------------------------------------------------------------------------- 1 | upload 2 | ====== 3 | 4 | .. automodule:: s3pathlib.better_client.upload 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/compat.rst: -------------------------------------------------------------------------------- 1 | compat 2 | ====== 3 | 4 | .. automodule:: s3pathlib.compat 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/constants.rst: -------------------------------------------------------------------------------- 1 | constants 2 | ========= 3 | 4 | .. automodule:: s3pathlib.constants 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/__init__.rst: -------------------------------------------------------------------------------- 1 | core 2 | ==== 3 | 4 | .. automodule:: s3pathlib.core 5 | :members: 6 | 7 | sub packages and modules 8 | ------------------------ 9 | 10 | .. toctree:: 11 | :maxdepth: 1 12 | 13 | attribute 14 | base 15 | bucket 16 | comparison 17 | copy 18 | delete 19 | exists 20 | filterable_property 21 | is_test 22 | iter_object_versions 23 | iter_objects 24 | joinpath 25 | metadata 26 | mutate 27 | opener 28 | relative 29 | resolve_s3_client 30 | rw 31 | s3path 32 | serde 33 | sync 34 | tagging 35 | upload 36 | uri 37 | -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/attribute.rst: -------------------------------------------------------------------------------- 1 | attribute 2 | ========= 3 | 4 | .. automodule:: s3pathlib.core.attribute 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/base.rst: -------------------------------------------------------------------------------- 1 | base 2 | ==== 3 | 4 | .. automodule:: s3pathlib.core.base 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/bucket.rst: -------------------------------------------------------------------------------- 1 | bucket 2 | ====== 3 | 4 | .. automodule:: s3pathlib.core.bucket 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/comparison.rst: -------------------------------------------------------------------------------- 1 | comparison 2 | ========== 3 | 4 | .. automodule:: s3pathlib.core.comparison 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/copy.rst: -------------------------------------------------------------------------------- 1 | copy 2 | ==== 3 | 4 | .. automodule:: s3pathlib.core.copy 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/delete.rst: -------------------------------------------------------------------------------- 1 | delete 2 | ====== 3 | 4 | .. automodule:: s3pathlib.core.delete 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/exists.rst: -------------------------------------------------------------------------------- 1 | exists 2 | ====== 3 | 4 | .. automodule:: s3pathlib.core.exists 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/filterable_property.rst: -------------------------------------------------------------------------------- 1 | filterable_property 2 | =================== 3 | 4 | .. automodule:: s3pathlib.core.filterable_property 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/is_test.rst: -------------------------------------------------------------------------------- 1 | is_test 2 | ======= 3 | 4 | .. automodule:: s3pathlib.core.is_test 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/iter_object_versions.rst: -------------------------------------------------------------------------------- 1 | iter_object_versions 2 | ==================== 3 | 4 | .. automodule:: s3pathlib.core.iter_object_versions 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/iter_objects.rst: -------------------------------------------------------------------------------- 1 | iter_objects 2 | ============ 3 | 4 | .. automodule:: s3pathlib.core.iter_objects 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/joinpath.rst: -------------------------------------------------------------------------------- 1 | joinpath 2 | ======== 3 | 4 | .. automodule:: s3pathlib.core.joinpath 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/metadata.rst: -------------------------------------------------------------------------------- 1 | metadata 2 | ======== 3 | 4 | .. automodule:: s3pathlib.core.metadata 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/mutate.rst: -------------------------------------------------------------------------------- 1 | mutate 2 | ====== 3 | 4 | .. automodule:: s3pathlib.core.mutate 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/opener.rst: -------------------------------------------------------------------------------- 1 | opener 2 | ====== 3 | 4 | .. automodule:: s3pathlib.core.opener 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/relative.rst: -------------------------------------------------------------------------------- 1 | relative 2 | ======== 3 | 4 | .. automodule:: s3pathlib.core.relative 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/resolve_s3_client.rst: -------------------------------------------------------------------------------- 1 | resolve_s3_client 2 | ================= 3 | 4 | .. automodule:: s3pathlib.core.resolve_s3_client 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/rw.rst: -------------------------------------------------------------------------------- 1 | rw 2 | == 3 | 4 | .. automodule:: s3pathlib.core.rw 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/s3path.rst: -------------------------------------------------------------------------------- 1 | s3path 2 | ====== 3 | 4 | .. automodule:: s3pathlib.core.s3path 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/serde.rst: -------------------------------------------------------------------------------- 1 | serde 2 | ===== 3 | 4 | .. automodule:: s3pathlib.core.serde 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/sync.rst: -------------------------------------------------------------------------------- 1 | sync 2 | ==== 3 | 4 | .. automodule:: s3pathlib.core.sync 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/tagging.rst: -------------------------------------------------------------------------------- 1 | tagging 2 | ======= 3 | 4 | .. automodule:: s3pathlib.core.tagging 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/upload.rst: -------------------------------------------------------------------------------- 1 | upload 2 | ====== 3 | 4 | .. automodule:: s3pathlib.core.upload 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/core/uri.rst: -------------------------------------------------------------------------------- 1 | uri 2 | === 3 | 4 | .. automodule:: s3pathlib.core.uri 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/exc.rst: -------------------------------------------------------------------------------- 1 | exc 2 | === 3 | 4 | .. automodule:: s3pathlib.exc 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/marker.rst: -------------------------------------------------------------------------------- 1 | marker 2 | ====== 3 | 4 | .. automodule:: s3pathlib.marker 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/metadata.rst: -------------------------------------------------------------------------------- 1 | metadata 2 | ======== 3 | 4 | .. automodule:: s3pathlib.metadata 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/tag.rst: -------------------------------------------------------------------------------- 1 | tag 2 | === 3 | 4 | .. automodule:: s3pathlib.tag 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/type.rst: -------------------------------------------------------------------------------- 1 | type 2 | ==== 3 | 4 | .. automodule:: s3pathlib.type 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/utils.rst: -------------------------------------------------------------------------------- 1 | utils 2 | ===== 3 | 4 | .. automodule:: s3pathlib.utils 5 | :members: -------------------------------------------------------------------------------- /docs/source/s3pathlib/validate.rst: -------------------------------------------------------------------------------- 1 | validate 2 | ======== 3 | 4 | .. automodule:: s3pathlib.validate 5 | :members: -------------------------------------------------------------------------------- /pygitrepo-config.json: -------------------------------------------------------------------------------- 1 | { 2 | // the python package name, use lowercase, digits, and underscore only 3 | // this would be the name for ``pip install ${package_name}`` 4 | "PACKAGE_NAME": "s3pathlib", 5 | // the python major version you use for local development 6 | "DEV_PY_VER_MAJOR": "3", 7 | // the python minor version you use for local development 8 | "DEV_PY_VER_MINOR": "8", 9 | // the python micro version you use for local development 10 | "DEV_PY_VER_MICRO": "11", 11 | 12 | // if you use readthedocs.org to host your document website 13 | // it is the project name in your document website domain 14 | // https://${DOC_HOST_RTD_PROJECT_NAME}.readthedocs.io/ 15 | "DOC_HOST_RTD_PROJECT_NAME": "s3pathlib", 16 | 17 | // if you use AWS S3 to host your document website 18 | // it is the aws profile you use for doc site deployment 19 | // leave empty string "" if you don't use it 20 | "DOC_HOST_AWS_PROFILE": "", 21 | // it is the aws s3 bucket you use to store you document files 22 | "DOC_HOST_S3_BUCKET": "", 23 | 24 | // if it is an AWS Lambda microservice project 25 | // it is the aws profile you use for deployment 26 | "AWS_LAMBDA_DEPLOY_AWS_PROFILE": "", 27 | // it is the aws s3 bucket to store deployment artifacts 28 | "AWS_LAMBDA_DEPLOY_S3_BUCKET": "", 29 | // it is the docker image for AWS Lambda layer build on your local machine 30 | "AWS_LAMBDA_BUILD_DOCKER_IMAGE": "", 31 | // it is the working directory for temp lambda layer build in docker image 32 | "AWS_LAMBDA_BUILD_DOCKER_IMAGE_WORKSPACE_DIR": "" 33 | } 34 | 35 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | # This requirements file should only include dependencies for development 2 | pathlib_mate # autopep8 your code 3 | twine # make distribution archive 4 | wheel # make pre-compiled distribution package 5 | rich 6 | awscli 7 | boto3-stubs[s3,sts] 8 | -------------------------------------------------------------------------------- /requirements-doc.txt: -------------------------------------------------------------------------------- 1 | # This requirements file should only include dependencies for documentations 2 | # sphinx doc builder 3 | sphinx 4 | 5 | # Extensions 6 | sphinx-inline-tabs # allow inline tab 7 | sphinx-jinja # allow jinja2 template 8 | sphinx-copybutton # add copy to clipboard button for code block 9 | sphinx_design # better sphinx design pattern 10 | nbsphinx # embed jupyter notebook 11 | readthedocs-sphinx-search # auto complete in search 12 | rstobj==0.0.7 # generate list table from data 13 | 14 | # docfly # auto API manual, auto Table of Content 15 | docfly==1.0.2 # sphinx-doc automation 16 | 17 | # Theme 18 | furo # modern doc theme 19 | IPython 20 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | # This requirements file should only include dependencies for testing 2 | pytest # test framework 3 | pytest-cov # coverage test 4 | decorator 5 | moto 6 | rich 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | iterproxy 3 | func_args 4 | pathlib_mate>=1.0.1,<2.0.0 5 | boto_session_manager>=1.5.1,<2.0.0 6 | smart_open>=5.1.0,<7.0.0 7 | cached-property>=1.5.2; python_version < '3.8' 8 | -------------------------------------------------------------------------------- /s3pathlib/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Objective Oriented Interface for AWS S3, similar to pathlib. 5 | """ 6 | 7 | from ._version import __version__ 8 | 9 | __short_description__ = "Objective Oriented Interface for AWS S3, similar to pathlib." 10 | __license__ = "Apache License 2.0" 11 | __author__ = "Sanhe Hu" 12 | __author_email__ = "husanhe@gmail.com" 13 | __maintainer__ = "Sanhe Hu" 14 | __maintainer_email__ = "sanhehu@amazon.com" 15 | __github_username__ = "aws-samples" 16 | 17 | try: 18 | from . import utils 19 | from .better_client import api 20 | from .aws import context 21 | from .core import S3Path 22 | 23 | from iterproxy import and_, or_, not_ 24 | except ImportError: # pragma: no cover 25 | pass 26 | except: # pragma: no cover 27 | raise 28 | -------------------------------------------------------------------------------- /s3pathlib/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "2.0.1" 2 | 3 | if __name__ == "__main__": # pragma: no cover 4 | print(__version__) 5 | -------------------------------------------------------------------------------- /s3pathlib/aws.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Manage the AWS environment that ``s3pathlib`` dealing with. 5 | """ 6 | 7 | import typing as T 8 | 9 | try: 10 | import boto3 11 | except ImportError: # pragma: no cover 12 | pass 13 | except: # pragma: no cover 14 | raise 15 | 16 | if T.TYPE_CHECKING: # pragma: no cover 17 | from mypy_boto3_s3 import S3Client 18 | from mypy_boto3_sts import STSClient 19 | 20 | 21 | class Context: 22 | """ 23 | A globally available context object managing AWS SDK credentials. 24 | 25 | TODO: use singleton pattern to create context object 26 | """ 27 | 28 | def __init__(self): 29 | self.boto_ses: T.Optional["boto3.session.Session"] = None 30 | self._aws_region: T.Optional[str] = None 31 | self._aws_account_id: T.Optional[str] = None 32 | self._s3_client: T.Optional["S3Client"] = None 33 | self._sts_client: T.Optional["STSClient"] = None 34 | 35 | # try to create default session 36 | try: 37 | self.boto_ses = boto3.session.Session() 38 | except: # pragma: no cover 39 | pass 40 | 41 | def attach_boto_session(self, boto_ses: "boto3.session.Session"): 42 | """ 43 | Attach a custom boto session, also remove caches. 44 | """ 45 | self.boto_ses = boto_ses 46 | self._s3_client = None 47 | self._sts_client = None 48 | self._aws_account_id = None 49 | self._aws_region = None 50 | 51 | @property 52 | def s3_client(self) -> "S3Client": 53 | """ 54 | Access the s3 client. 55 | 56 | https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client 57 | """ 58 | if self._s3_client is None: 59 | self._s3_client = self.boto_ses.client("s3") 60 | return self._s3_client 61 | 62 | @property 63 | def sts_client(self) -> "STSClient": 64 | """ 65 | Access the s3 client. 66 | 67 | https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#client 68 | """ 69 | if self._sts_client is None: 70 | self._sts_client = self.boto_ses.client("sts") 71 | return self._sts_client 72 | 73 | @property 74 | def aws_account_id(self) -> str: 75 | """ 76 | The AWS Account ID of the current boto session. 77 | """ 78 | if self._aws_account_id is None: 79 | self._aws_account_id = self.sts_client.get_caller_identity()["Account"] 80 | return self._aws_account_id 81 | 82 | @property 83 | def aws_region(self) -> str: 84 | """ 85 | The AWS Region of the current boto session. 86 | """ 87 | if self._aws_region is None: 88 | self._aws_region = self.boto_ses.region_name 89 | return self._aws_region 90 | 91 | 92 | context = Context() 93 | -------------------------------------------------------------------------------- /s3pathlib/better_client/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Improve the native boto3 client. 5 | """ 6 | -------------------------------------------------------------------------------- /s3pathlib/better_client/api.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .head_bucket import is_bucket_exists 4 | from .head_object import ( 5 | head_object, 6 | is_object_exists, 7 | ) 8 | from .tagging import ( 9 | update_bucket_tagging, 10 | update_object_tagging, 11 | ) 12 | from .upload import ( 13 | upload_dir, 14 | ) 15 | from .list_objects import ( 16 | ObjectTypeDefIterproxy, 17 | CommonPrefixTypeDefIterproxy, 18 | ListObjectsV2OutputTypeDefIterproxy, 19 | paginate_list_objects_v2, 20 | is_content_an_object, 21 | calculate_total_size, 22 | count_objects, 23 | ) 24 | from .list_object_versions import ( 25 | ObjectVersionTypeDefIterproxy, 26 | DeleteMarkerEntryTypeDefIterproxy, 27 | ListObjectVersionsOutputTypeDefIterproxy, 28 | paginate_list_object_versions, 29 | ) 30 | from .delete_object import ( 31 | delete_object, 32 | delete_dir, 33 | delete_object_versions, 34 | ) 35 | -------------------------------------------------------------------------------- /s3pathlib/better_client/head_bucket.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Improve the head_bucket_ API. 5 | 6 | .. _head_bucket: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_bucket 7 | """ 8 | 9 | import typing as T 10 | 11 | import botocore.exceptions 12 | 13 | 14 | if T.TYPE_CHECKING: # pragma: no cover 15 | from mypy_boto3_s3 import S3Client 16 | 17 | 18 | def is_bucket_exists( 19 | s3_client: "S3Client", 20 | bucket: str, 21 | ) -> bool: 22 | """ 23 | Check if a bucket exists. 24 | 25 | Example:: 26 | 27 | >>> is_bucket_exists(s3_client, "my-bucket") 28 | True 29 | 30 | :param s3_client: See head_bucket_ 31 | :param bucket: See head_bucket_ 32 | 33 | :return: A Boolean flag to indicate whether the bucket exists. 34 | """ 35 | try: 36 | s3_client.head_bucket(Bucket=bucket) 37 | return True 38 | except botocore.exceptions.ClientError as e: 39 | if "Not Found" in str(e): 40 | return False 41 | else: # pragma: no cover 42 | raise e 43 | -------------------------------------------------------------------------------- /s3pathlib/better_client/head_object.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Improve the head_object_ API. 5 | 6 | .. _head_object: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_object 7 | """ 8 | 9 | import typing as T 10 | from datetime import datetime 11 | 12 | import botocore.exceptions 13 | from func_args import NOTHING, resolve_kwargs 14 | 15 | from .. import exc 16 | 17 | if T.TYPE_CHECKING: # pragma: no cover 18 | from mypy_boto3_s3 import S3Client 19 | from mypy_boto3_s3.type_defs import HeadObjectOutputTypeDef 20 | 21 | 22 | def head_object( 23 | s3_client: "S3Client", 24 | bucket: str, 25 | key: str, 26 | if_match: str = NOTHING, 27 | if_modified_since: datetime = NOTHING, 28 | if_none_match: str = NOTHING, 29 | if_unmodified_since: datetime = NOTHING, 30 | range: str = NOTHING, 31 | version_id: str = NOTHING, 32 | sse_customer_algorithm: str = NOTHING, 33 | sse_customer_key: str = NOTHING, 34 | request_payer: str = NOTHING, 35 | part_number: int = NOTHING, 36 | expected_bucket_owner: str = NOTHING, 37 | checksum_mode: str = NOTHING, 38 | ignore_not_found: bool = False, 39 | ) -> T.Optional[T.Union[dict, "HeadObjectOutputTypeDef"]]: 40 | """ 41 | Wrapper of head_object_. 42 | 43 | Example: 44 | 45 | >>> response = head_object(s3_client, "my-bucket", "file.txt") 46 | >>> if response is None: 47 | ... print("Object not found") 48 | ... else: 49 | ... print(response["LastModified"]) 50 | 51 | :param s3_client: See head_object_ 52 | :param bucket: See head_object_ 53 | :param key: See head_object_ 54 | :param if_match: See head_object_ 55 | :param if_modified_since: See head_object_ 56 | :param if_none_match: See head_object_ 57 | :param if_unmodified_since: See head_object_ 58 | :param range: See head_object_ 59 | :param version_id: See head_object_ 60 | :param sse_customer_algorithm: See head_object_ 61 | :param sse_customer_key: See head_object_ 62 | :param request_payer: See head_object_ 63 | :param part_number: See head_object_ 64 | :param expected_bucket_owner: See head_object_ 65 | :param checksum_mode: See head_object_ 66 | :param ignore_not_found: Default is ``False``; if ``True``, return ``None`` 67 | when object is not found instead of raising an error. 68 | 69 | :return: See head_object_ 70 | """ 71 | try: 72 | dct = s3_client.head_object( 73 | **resolve_kwargs( 74 | Bucket=bucket, 75 | Key=key, 76 | IfMatch=if_match, 77 | IfModifiedSince=if_modified_since, 78 | IfNoneMatch=if_none_match, 79 | IfUnmodifiedSince=if_unmodified_since, 80 | Range=range, 81 | VersionId=version_id, 82 | SSECustomerAlgorithm=sse_customer_algorithm, 83 | SSECustomerKey=sse_customer_key, 84 | RequestPayer=request_payer, 85 | PartNumber=part_number, 86 | ExpectedBucketOwner=expected_bucket_owner, 87 | ChecksumMode=checksum_mode, 88 | ) 89 | ) 90 | return dct 91 | except botocore.exceptions.ClientError as e: 92 | if "Not Found" in str(e): 93 | if ignore_not_found: 94 | return None 95 | else: 96 | raise exc.S3FileNotExist.make(f"s3://{bucket}/{key}") 97 | else: # pragma: no cover 98 | raise e 99 | 100 | 101 | def is_object_exists( 102 | s3_client: "S3Client", 103 | bucket: str, 104 | key: str, 105 | version_id: str = NOTHING, 106 | ) -> bool: 107 | """ 108 | Check if an object exists. If you want to use the head_object_ API response 109 | immediately when the object exists, use :func:`head_object` instead. 110 | 111 | Example:: 112 | 113 | >>> is_object_exists(s3_client, "my-bucket", "file.txt") 114 | True 115 | 116 | :param s3_client: See head_object_ 117 | :param bucket: See head_object_ 118 | :param key: See head_object_ 119 | :param version_id: See head_object_ 120 | 121 | :return: A Boolean flag to indicate whether the object exists. 122 | """ 123 | response = head_object( 124 | s3_client=s3_client, 125 | bucket=bucket, 126 | key=key, 127 | version_id=version_id, 128 | ignore_not_found=True, 129 | ) 130 | if response is None: 131 | return False 132 | else: 133 | return True 134 | -------------------------------------------------------------------------------- /s3pathlib/better_client/tagging.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Update S3 bucket, object tagging. 5 | 6 | .. _get_bucket_tagging: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/get_bucket_tagging.html 7 | .. _put_bucket_tagging: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/put_bucket_tagging.html 8 | 9 | .. _get_object_tagging: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/get_object_tagging.html 10 | .. _put_object_tagging: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/put_object_tagging.html 11 | """ 12 | 13 | import typing as T 14 | import botocore.exceptions 15 | from func_args import NOTHING, resolve_kwargs 16 | 17 | from .. import tag 18 | 19 | 20 | if T.TYPE_CHECKING: # pragma: no cover 21 | from mypy_boto3_s3 import S3Client 22 | 23 | 24 | def update_bucket_tagging( 25 | s3_client: "S3Client", 26 | bucket: str, 27 | tags: tag.TagType, 28 | checksum_algorithm: str = NOTHING, 29 | expected_bucket_owner: str = NOTHING, 30 | ) -> tag.TagType: 31 | """ 32 | Allow you to use ``dict.update`` liked API to update s3 bucket tagging. 33 | It is a combination of get, update and put. 34 | 35 | :return: the updated tags in Python dict. 36 | """ 37 | try: 38 | res = s3_client.get_bucket_tagging( 39 | **resolve_kwargs( 40 | Bucket=bucket, 41 | ExpectedBucketOwner=expected_bucket_owner, 42 | ) 43 | ) 44 | existing_tags = tag.parse_tags(res.get("TagSet", [])) 45 | except botocore.exceptions.ClientError as e: 46 | if "NoSuchTagSet" in str(e): 47 | existing_tags = {} 48 | else: # pragma: no cover 49 | raise e 50 | 51 | existing_tags.update(tags) 52 | s3_client.put_bucket_tagging( 53 | **resolve_kwargs( 54 | Bucket=bucket, 55 | Tagging=dict(TagSet=tag.encode_for_put_bucket_tagging(existing_tags)), 56 | ChecksumAlgorithm=checksum_algorithm, 57 | ExpectedBucketOwner=expected_bucket_owner, 58 | ) 59 | ) 60 | return existing_tags 61 | 62 | 63 | def update_object_tagging( 64 | s3_client: "S3Client", 65 | bucket: str, 66 | key: str, 67 | tags: tag.TagType, 68 | version_id: str = NOTHING, 69 | content_md5: str = NOTHING, 70 | checksum_algorithm: str = NOTHING, 71 | expected_bucket_owner: str = NOTHING, 72 | request_payer: str = NOTHING, 73 | ) -> T.Tuple[T.Optional[str], tag.TagType]: 74 | """ 75 | Allow you to use ``dict.update`` liked API to update s3 object tagging. 76 | It is a combination of get, update and put. 77 | 78 | :return: the tuple of ``(version_id, tags)``, where version_id is optional, 79 | and tags is the updated tags in Python dict. 80 | """ 81 | res = s3_client.get_object_tagging( 82 | **resolve_kwargs( 83 | Bucket=bucket, 84 | Key=key, 85 | VersionId=version_id, 86 | ExpectedBucketOwner=expected_bucket_owner, 87 | RequestPayer=request_payer, 88 | ) 89 | ) 90 | existing_version_id = res.get("VersionId", None) 91 | existing_tags = tag.parse_tags(res.get("TagSet", [])) 92 | existing_tags.update(tags) 93 | s3_client.put_object_tagging( 94 | **resolve_kwargs( 95 | Bucket=bucket, 96 | Key=key, 97 | Tagging=dict(TagSet=tag.encode_for_put_object_tagging(existing_tags)), 98 | VersionId=res.get("VersionId", NOTHING), 99 | ContentMD5=content_md5, 100 | ChecksumAlgorithm=checksum_algorithm, 101 | ExpectedBucketOwner=expected_bucket_owner, 102 | RequestPayer=request_payer, 103 | ) 104 | ) 105 | return existing_version_id, existing_tags 106 | -------------------------------------------------------------------------------- /s3pathlib/better_client/upload.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | .. _upload_file: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/upload_file.html# 5 | """ 6 | 7 | import typing as T 8 | 9 | from pathlib_mate import Path 10 | 11 | from .. import exc 12 | from ..type import PathType 13 | from ..utils import join_s3_uri 14 | from .head_object import is_object_exists 15 | 16 | 17 | if T.TYPE_CHECKING: # pragma: no cover 18 | from mypy_boto3_s3 import S3Client 19 | 20 | 21 | def upload_dir( 22 | s3_client: "S3Client", 23 | bucket: str, 24 | prefix: str, 25 | local_dir: PathType, 26 | pattern: str = "**/*", 27 | overwrite: bool = False, 28 | ) -> int: 29 | """ 30 | Recursively upload a local directory and files in its subdirectory to S3. 31 | 32 | :param s3_client: A ``boto3.session.Session().client("s3")`` object. 33 | :param bucket: S3 bucket name. 34 | :param prefix: The s3 prefix (logic directory) you want to upload to. 35 | :param local_dir: Absolute path of the directory on the local 36 | file system you want to upload. 37 | :param pattern: Linux styled glob pattern match syntax. see this official 38 | guide https://docs.python.org/3/library/pathlib.html#pathlib.Path.glob 39 | for more details. 40 | :param overwrite: If False, none of the file will be uploaded / overwritten 41 | if any of target s3 location already taken. 42 | 43 | :return: number of files uploaded 44 | 45 | .. versionadded:: 1.0.1 46 | """ 47 | # preprocess input arguments 48 | if prefix.endswith("/"): 49 | prefix = prefix[:-1] 50 | 51 | p_local_dir = Path(local_dir) 52 | 53 | if p_local_dir.is_file(): 54 | raise TypeError(f"'{p_local_dir}' is a file, not a directory!") 55 | 56 | if p_local_dir.exists() is False: 57 | raise FileNotFoundError(f"'{p_local_dir}' not found!") 58 | 59 | if len(prefix): 60 | final_prefix = f"{prefix}/" 61 | else: 62 | final_prefix = "" 63 | 64 | # list of (local file path, target s3 key) 65 | todo: T.List[T.Tuple[str, str]] = list() 66 | for p in p_local_dir.glob(pattern): 67 | if p.is_file(): 68 | relative_path = p.relative_to(p_local_dir) 69 | key = "{}{}".format(final_prefix, "/".join(relative_path.parts)) 70 | todo.append((p.abspath, key)) 71 | 72 | # make sure all target s3 location not exists 73 | if overwrite is False: 74 | for abspath, key in todo: 75 | if is_object_exists(s3_client, bucket, key) is True: 76 | s3_uri = join_s3_uri(bucket, key) 77 | raise exc.S3FileAlreadyExist.make(s3_uri) 78 | 79 | # execute upload 80 | for abspath, key in todo: 81 | s3_client.upload_file(abspath, bucket, key) 82 | 83 | return len(todo) 84 | -------------------------------------------------------------------------------- /s3pathlib/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Provide compatibility with older versions of Python and dependent libraries. 5 | """ 6 | 7 | import sys 8 | 9 | try: 10 | import smart_open 11 | 12 | smart_open_version = smart_open.__version__ 13 | ( 14 | smart_open_version_major, 15 | smart_open_version_minor, 16 | _, 17 | ) = smart_open_version.split(".") 18 | smart_open_version_major = int(smart_open_version_major) 19 | smart_open_version_minor = int(smart_open_version_minor) 20 | except ImportError: # pragma: no cover 21 | smart_open = None 22 | smart_open_version_major = None 23 | smart_open_version_minor = None 24 | except: # pragma: no cover 25 | raise 26 | 27 | if sys.version_info.minor < 8: 28 | from cached_property import cached_property 29 | else: 30 | from functools import cached_property 31 | 32 | 33 | class Compat: # pragma: no cover 34 | @property 35 | def smart_open_version_major(self) -> int: 36 | if smart_open_version_major is None: 37 | raise ImportError("You don't have smart_open installed") 38 | return smart_open_version_major 39 | 40 | @property 41 | def smart_open_version_minor(self) -> int: 42 | if smart_open_version_minor is None: 43 | raise ImportError("You don't have smart_open installed") 44 | return smart_open_version_minor 45 | 46 | 47 | compat = Compat() 48 | -------------------------------------------------------------------------------- /s3pathlib/constants.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | IS_DELETE_MARKER = "IsDeleteMarker" 4 | -------------------------------------------------------------------------------- /s3pathlib/core/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | This module implements the core OOP interface :class:`S3Path`. 5 | 6 | Import:: 7 | 8 | >>> from s3pathlib.core import S3Path 9 | # or 10 | >>> from s3pathlib import S3Path 11 | """ 12 | 13 | from .s3path import S3Path 14 | -------------------------------------------------------------------------------- /s3pathlib/core/bucket.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Bucket related API. 5 | """ 6 | 7 | import typing as T 8 | 9 | 10 | if T.TYPE_CHECKING: # pragma: no cover 11 | from .s3path import S3Path 12 | from boto_session_manager import BotoSesManager 13 | 14 | 15 | class BucketAPIMixin: 16 | """ 17 | A mixin class that implements the bucket related methods. 18 | """ 19 | -------------------------------------------------------------------------------- /s3pathlib/core/comparison.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Comparison operator implementation. 5 | """ 6 | 7 | import typing as T 8 | 9 | if T.TYPE_CHECKING: # pragma: no cover 10 | from .s3path import S3Path 11 | 12 | 13 | class ComparisonAPIMixin: 14 | """ 15 | A mixin class that implements the comparison operator magic methods. 16 | """ 17 | @property 18 | def _cparts(self: "S3Path"): 19 | """ 20 | Cached comparison parts, for hashing and comparison 21 | """ 22 | try: 23 | return self._cached_cparts 24 | except AttributeError: 25 | cparts = list() 26 | 27 | if self._bucket: 28 | cparts.append(self._bucket) 29 | else: 30 | cparts.append("") 31 | 32 | cparts.extend(self._parts) 33 | 34 | if self._is_dir: 35 | cparts.append("/") 36 | 37 | self._cached_cparts = cparts 38 | return self._cached_cparts 39 | 40 | def __eq__(self: "S3Path", other: "S3Path") -> bool: 41 | """ 42 | Return ``self == other``. 43 | """ 44 | return self._cparts == other._cparts 45 | 46 | def __lt__(self: "S3Path", other: "S3Path") -> bool: 47 | """ 48 | Return ``self < other``. 49 | """ 50 | return self._cparts < other._cparts 51 | 52 | def __gt__(self: "S3Path", other: "S3Path") -> bool: 53 | """ 54 | Return ``self > other``. 55 | """ 56 | return self._cparts > other._cparts 57 | 58 | def __le__(self: "S3Path", other: "S3Path") -> bool: 59 | """ 60 | Return ``self <= other``. 61 | """ 62 | return self._cparts <= other._cparts 63 | 64 | def __ge__(self: "S3Path", other: "S3Path") -> bool: 65 | """ 66 | Return ``self >= other``. 67 | """ 68 | return self._cparts >= other._cparts 69 | 70 | def __hash__(self: "S3Path") -> int: 71 | """ 72 | Return ``hash(self)`` 73 | """ 74 | try: 75 | return self._hash 76 | except AttributeError: 77 | self._hash = hash(tuple(self._cparts)) 78 | return self._hash 79 | -------------------------------------------------------------------------------- /s3pathlib/core/exists.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Tagging related API. 5 | """ 6 | 7 | import typing as T 8 | from func_args import NOTHING 9 | 10 | from .. import exc 11 | from ..better_client.head_bucket import is_bucket_exists 12 | from ..better_client.head_object import head_object 13 | from ..aws import context 14 | 15 | from .resolve_s3_client import resolve_s3_client 16 | 17 | if T.TYPE_CHECKING: # pragma: no cover 18 | from .s3path import S3Path 19 | from boto_session_manager import BotoSesManager 20 | 21 | 22 | class ExistsAPIMixin: 23 | """ 24 | A mixin class that implements the exists test related methods. 25 | """ 26 | 27 | def exists( 28 | self: "S3Path", 29 | version_id: str = NOTHING, 30 | bsm: T.Optional["BotoSesManager"] = None, 31 | ) -> bool: 32 | """ 33 | Test if an S3Path object actually exists in S3 bucket. 34 | 35 | - For S3 bucket: check if the bucket exists. If you don't have the 36 | access, then it raise exception. 37 | - For S3 object: check if the object exists 38 | - For S3 directory: check if the directory exists, it returns ``True`` 39 | even if the folder doesn't have any object. 40 | - For versioning enabled bucket, you can explicitly check if a specific 41 | version exists, otherwise it will check the latest version. 42 | 43 | Example: 44 | 45 | >>> S3Path("bucket").exists() 46 | >>> S3Path("bucket/file.txt").exists() 47 | >>> S3Path("bucket/folder/").exists() 48 | >>> S3Path("bucket/file.txt").exists(version_id="v123456") 49 | 50 | .. versionadded:: 1.0.1 51 | 52 | .. versionchanged:: 2.0.1 53 | 54 | Add ``version_id`` parameter. 55 | """ 56 | if self.is_bucket(): 57 | s3_client = resolve_s3_client(context, bsm) 58 | return is_bucket_exists(s3_client, self.bucket) 59 | elif self.is_file(): 60 | s3_client = resolve_s3_client(context, bsm) 61 | dct = head_object( 62 | s3_client=s3_client, 63 | bucket=self.bucket, 64 | key=self.key, 65 | version_id=version_id, 66 | ignore_not_found=True, 67 | ) 68 | if dct is None: 69 | return False 70 | else: 71 | if "ResponseMetadata" in dct: 72 | del dct["ResponseMetadata"] 73 | self._meta = dct 74 | return True 75 | elif self.is_dir(): 76 | l = list( 77 | self.iterdir( 78 | batch_size=1, 79 | limit=1, 80 | bsm=bsm, 81 | ) 82 | ) 83 | if len(l): 84 | return True 85 | else: 86 | return False 87 | else: # pragma: no cover 88 | raise TypeError 89 | 90 | def ensure_not_exists( 91 | self: "S3Path", 92 | version_id: str = NOTHING, 93 | bsm: T.Optional["BotoSesManager"] = None, 94 | ) -> None: 95 | """ 96 | A validator method ensure that it doesn't exists. 97 | 98 | .. versionadded:: 1.0.1 99 | 100 | .. versionchanged:: 2.0.1 101 | 102 | Add ``version_id`` parameter. 103 | """ 104 | if self.exists(version_id=version_id, bsm=bsm): 105 | raise exc.S3AlreadyExist( 106 | ( 107 | "cannot write to {}, s3 object ALREADY EXISTS! " 108 | "open console for more details {}." 109 | ).format(self.uri, self.console_url) 110 | ) 111 | -------------------------------------------------------------------------------- /s3pathlib/core/is_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Is the S3Path a XYZ testing. 5 | 6 | - :meth:`~IsTestAPIMixin.is_void` 7 | - :meth:`~IsTestAPIMixin.is_dir` 8 | - :meth:`~IsTestAPIMixin.is_file` 9 | - :meth:`~IsTestAPIMixin.is_bucket` 10 | """ 11 | 12 | import typing as T 13 | 14 | from ..exc import ( 15 | S3PathIsNotFolderError, 16 | S3PathIsNotFileError, 17 | ) 18 | from ..constants import IS_DELETE_MARKER 19 | 20 | if T.TYPE_CHECKING: # pragma: no cover 21 | from .s3path import S3Path 22 | 23 | 24 | class IsTestAPIMixin: 25 | """ 26 | A mixin class that implements the condition test methods. 27 | """ 28 | 29 | def is_void(self: "S3Path") -> bool: 30 | """ 31 | Test if it is a void S3 path. 32 | 33 | **Definition** 34 | 35 | no bucket, no key, no parts, no type, no nothing. 36 | A void path is also a special :meth:`relative path `, 37 | because any path join with void path results to itself. 38 | """ 39 | return (self._bucket is None) and (len(self._parts) == 0) 40 | 41 | def is_dir(self: "S3Path") -> bool: 42 | """ 43 | Test if it is a S3 directory 44 | 45 | **Definition** 46 | 47 | A logical S3 directory that never physically exists. An S3 48 | :meth:`bucket ` is also a special dir, which is the root dir. 49 | 50 | .. versionadded:: 1.0.1 51 | """ 52 | if self._is_dir is None: 53 | return False 54 | else: 55 | return self._is_dir 56 | 57 | def is_file(self: "S3Path") -> bool: 58 | """ 59 | Test if it is a S3 object 60 | 61 | **Definition** 62 | 63 | A S3 object. 64 | 65 | .. versionadded:: 1.0.1 66 | """ 67 | if self._is_dir is None: 68 | return False 69 | else: 70 | return not self._is_dir 71 | 72 | def is_bucket(self: "S3Path") -> bool: 73 | """ 74 | Test if it is a S3 bucket. 75 | 76 | **Definition** 77 | 78 | A S3 bucket, the root folder S3 path is equivalent to a S3 bucket. 79 | A S3 bucket are always :meth:`is_dir() is True ` 80 | 81 | .. versionadded:: 1.0.1 82 | """ 83 | return ( 84 | (self._bucket is not None) 85 | and (len(self._parts) == 0) 86 | and (self._is_dir is True) 87 | ) 88 | 89 | def is_delete_marker(self: "S3Path") -> bool: 90 | """ 91 | Test if it is a delete-marker. A delete-marker is just a version of 92 | an object without content. 93 | 94 | .. versionadded:: 2.0.1 95 | """ 96 | if self.is_file(): 97 | if (self.version_id is not None) and (IS_DELETE_MARKER in self._meta): 98 | return True 99 | else: 100 | return False 101 | else: 102 | return False 103 | 104 | def ensure_object(self: "S3Path") -> None: 105 | """ 106 | A validator method that ensure it represents a S3 object. 107 | 108 | .. versionadded:: 1.0.1 109 | """ 110 | if self.is_file() is not True: 111 | raise S3PathIsNotFileError.make(self.uri) 112 | 113 | def ensure_file(self: "S3Path") -> None: 114 | """ 115 | A validator method that ensure it represents a S3 object. 116 | 117 | .. versionadded:: 1.2.1 118 | """ 119 | return self.ensure_object() 120 | 121 | def ensure_not_object(self: "S3Path") -> None: 122 | """ 123 | A validator method that ensure it IS NOT a S3 object. 124 | 125 | .. versionadded:: 1.2.1 126 | """ 127 | if self.is_file() is True: 128 | raise TypeError(f"S3 URI: {self} IS an s3 object!") 129 | 130 | def ensure_not_file(self: "S3Path") -> None: 131 | """ 132 | A validator method that ensure it IS NOT a S3 object. 133 | 134 | .. versionadded:: 1.2.1 135 | """ 136 | self.ensure_not_object() 137 | 138 | def ensure_dir(self: "S3Path") -> None: 139 | """ 140 | A validator method that ensure it represents a S3 dir. 141 | 142 | .. versionadded:: 1.0.1 143 | """ 144 | if self.is_dir() is not True: 145 | raise S3PathIsNotFolderError.make(self.uri) 146 | 147 | def ensure_not_dir(self: "S3Path") -> None: 148 | """ 149 | A validator method that ensure it IS NOT a S3 dir. 150 | 151 | .. versionadded:: 1.2.1 152 | """ 153 | if self.is_dir() is True: 154 | raise TypeError(f"{self} IS a s3 directory!") 155 | -------------------------------------------------------------------------------- /s3pathlib/core/iter_object_versions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | List object versions related API. 5 | 6 | .. _bsm: https://github.com/aws-samples/boto-session-manager-project 7 | .. _ListObjectVersions: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/paginator/ListObjectVersions.html 8 | """ 9 | 10 | import typing as T 11 | 12 | from func_args import NOTHING 13 | 14 | from ..aws import context 15 | from ..better_client.list_object_versions import paginate_list_object_versions 16 | from .iter_objects import S3PathIterProxy 17 | from .resolve_s3_client import resolve_s3_client 18 | 19 | if T.TYPE_CHECKING: # pragma: no cover 20 | from .s3path import S3Path 21 | from boto_session_manager import BotoSesManager 22 | 23 | 24 | class IterObjectVersionsAPIMixin: 25 | """ 26 | A mixin class that implements the iter object versions methods. 27 | """ 28 | 29 | def list_object_versions( 30 | self: "S3Path", 31 | batch_size: int = 1000, 32 | limit: int = NOTHING, 33 | delimiter: str = NOTHING, 34 | encoding_type: str = NOTHING, 35 | expected_bucket_owner: str = NOTHING, 36 | bsm: T.Optional["BotoSesManager"] = None, 37 | ) -> S3PathIterProxy: 38 | """ 39 | Recursively iterate objects under this prefix, yield :class:`S3Path`. 40 | 41 | :param batch_size: Number of s3 object returned per paginator, 42 | valid value is from 1 ~ 1000. large number can reduce IO. 43 | :param limit: Total number of s3 object to return. 44 | :param delimiter: See ListObjectVersions_. 45 | :param encoding_type: See ListObjectVersions_. 46 | :param expected_bucket_owner: See ListObjectVersions_. 47 | :param bsm: See bsm_. 48 | 49 | .. versionadded:: 2.0.1 50 | """ 51 | s3_client = resolve_s3_client(context, bsm) 52 | bucket = self.bucket 53 | 54 | def _iter_s3path() -> T.Iterable["S3Path"]: 55 | proxy = paginate_list_object_versions( 56 | s3_client=s3_client, 57 | bucket=bucket, 58 | prefix=self.key, 59 | batch_size=batch_size, 60 | limit=limit, 61 | delimiter=delimiter, 62 | encoding_type=encoding_type, 63 | expected_bucket_owner=expected_bucket_owner, 64 | ) 65 | s3path_list = list() 66 | for response in proxy: 67 | ( 68 | versions, 69 | delete_markers, 70 | _, 71 | ) = proxy.extract_versions_and_delete_markers_and_common_prefixes( 72 | response 73 | ) 74 | 75 | s3path_list.extend( 76 | [self._from_version_dict(bucket, dct=dct) for dct in versions] 77 | ) 78 | s3path_list.extend( 79 | [ 80 | self._from_delete_marker(bucket, dct=dct) 81 | for dct in delete_markers 82 | ] 83 | ) 84 | for s3path in sorted( 85 | s3path_list, key=lambda x: x.last_modified_at, reverse=True 86 | ): 87 | yield s3path 88 | 89 | return S3PathIterProxy(_iter_s3path()) 90 | -------------------------------------------------------------------------------- /s3pathlib/core/joinpath.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Join path operator implementation. 5 | """ 6 | 7 | import typing as T 8 | 9 | from .relative import RelativePathAPIMixin 10 | from ..marker import warn_deprecate 11 | 12 | if T.TYPE_CHECKING: # pragma: no cover 13 | from .s3path import S3Path 14 | 15 | 16 | class JoinPathAPIMixin: 17 | """ 18 | A mixin class that implements the join path operator. 19 | """ 20 | def joinpath(self: "S3Path", *other: T.Union[str, "S3Path"]) -> "S3Path": 21 | """ 22 | Join with other relative path or string parts. 23 | 24 | Example:: 25 | 26 | # join with string parts 27 | >>> p = S3Path("bucket") 28 | >>> p.joinpath("folder", "file.txt") 29 | S3Path('s3://bucket/folder/file.txt') 30 | 31 | # join ith relative path or string parts 32 | >>> p = S3Path("bucket") 33 | >>> relpath = S3Path("my-bucket", "data", "folder/").relative_to(S3Path("my-bucket", "data")) 34 | >>> p.joinpath("data", relpath, "file.txt") 35 | S3Path('s3://bucket/data/folder/file.txt') 36 | 37 | :param others: many string or relative path 38 | 39 | .. versionadded:: 1.1.1 40 | """ 41 | args = [ 42 | self, 43 | ] 44 | for part in other: 45 | if isinstance(part, str): 46 | args.append(part) 47 | elif isinstance(part, RelativePathAPIMixin): 48 | if part.is_relpath() is False: 49 | msg = ( 50 | "you can only join with string part or relative path! " 51 | "{} is not a relative path" 52 | ).format(part) 53 | raise TypeError(msg) 54 | else: 55 | args.append(part) 56 | else: 57 | msg = ( 58 | "you can only join with string part or relative path! " 59 | "{} is not a relative path" 60 | ).format(part) 61 | raise TypeError(msg) 62 | return self._from_parts(args) 63 | 64 | def __truediv__( 65 | self: "S3Path", other: T.Union[str, "S3Path", T.List[T.Union[str, "S3Path"]]] 66 | ) -> "S3Path": 67 | """ 68 | A syntax sugar. Basically ``S3Path(s3path, part1, part2, ...)`` 69 | is equal to ``s3path / part1 / part2 / ...`` or 70 | ``s3path / [part1, part2]`` 71 | 72 | Example:: 73 | 74 | >>> S3Path("bucket") / "folder" / "file.txt" 75 | S3Path('s3://bucket/folder/file.txt') 76 | 77 | >>> S3Path("bucket") / ["folder", "file.txt"] 78 | S3Path('s3://bucket/folder/file.txt') 79 | 80 | # relative path also work 81 | >>> S3Path("new-bucket") / (S3Path("bucket/file.txt").relative_to(S3Path("bucket"))) 82 | S3Path('s3://new-bucket/file.txt') 83 | 84 | .. versionadded:: 1.0.11 85 | """ 86 | if self.is_void(): 87 | raise TypeError("You cannot do ``VoidS3Path / other``!") 88 | if isinstance(other, list): 89 | res = self 90 | for part in other: 91 | res = res / part 92 | return res 93 | else: 94 | if ( 95 | self.is_relpath() 96 | and isinstance(other, RelativePathAPIMixin) 97 | and other.is_relpath() is False 98 | ): 99 | raise TypeError("you cannot do ``RelativeS3Path / NonRelativeS3Path``!") 100 | return self.joinpath(other) 101 | -------------------------------------------------------------------------------- /s3pathlib/core/mutate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | S3Path object mutation implementation. 5 | """ 6 | 7 | import typing as T 8 | 9 | from .. import exc 10 | 11 | if T.TYPE_CHECKING: # pragma: no cover 12 | from .s3path import S3Path 13 | 14 | 15 | class MutateAPIMixin: 16 | """ 17 | A mixin class that implements the S3Path object mutation. 18 | """ 19 | 20 | def copy(self: "S3Path") -> "S3Path": 21 | """ 22 | Create a copy of S3Path object that logically equals to this one, 23 | but is actually different identity in memory. Also, the cache data 24 | are cleared. 25 | 26 | Example:: 27 | 28 | >>> p1 = S3Path("bucket", "folder", "file.txt") 29 | >>> p2 = p1.copy() 30 | >>> p1 == p2 31 | True 32 | >>> p1 is p2 33 | False 34 | 35 | .. versionadded:: 1.0.1 36 | """ 37 | return self._from_parsed_parts( 38 | bucket=self._bucket, 39 | parts=list(self._parts), 40 | is_dir=self._is_dir, 41 | ) 42 | 43 | def change( 44 | self: "S3Path", 45 | new_bucket: str = None, 46 | new_abspath: str = None, 47 | new_dirpath: str = None, 48 | new_dirname: str = None, 49 | new_basename: str = None, 50 | new_fname: str = None, 51 | new_ext: str = None, 52 | ) -> "S3Path": 53 | """ 54 | Create a new S3Path by replacing part of the attributes. If no argument 55 | is given, it behaves like :meth:`copy`. 56 | 57 | Example: 58 | 59 | >>> s3path = S3Path("bucket", "folder", "file.txt") 60 | >>> s3path.change(new_bucket="new_bucket") 61 | S3Path('s3://new_bucket/folder/file.txt') 62 | 63 | >>> s3path = S3Path("bucket", "folder", "file.txt") 64 | >>> s3path.change(new_basename="data.json") 65 | S3Path('s3://bucket/folder/data.json') 66 | 67 | >>> s3path = S3Path("bucket", "folder", "file.txt") 68 | >>> s3path.change(new_fname="log") 69 | S3Path('s3://bucket/folder/log.txt') 70 | 71 | :param new_bucket: The new bucket name 72 | :param new_abspath: 73 | :param new_dirpath: 74 | :param new_dirname: 75 | :param new_basename: 76 | :param new_fname: 77 | :param new_ext: 78 | 79 | .. versionadded:: 1.0.2 80 | """ 81 | if new_bucket is None: 82 | new_bucket = self.bucket 83 | 84 | if new_abspath is not None: 85 | exc.ensure_all_none( 86 | new_dirpath=new_dirpath, 87 | new_dirname=new_dirname, 88 | new_basename=new_basename, 89 | new_fname=new_fname, 90 | new_ext=new_ext, 91 | ) 92 | p = self._from_parts([self.bucket, new_abspath]) 93 | return p 94 | 95 | if (new_dirpath is None) and (new_dirname is not None): 96 | dir_parts = self.parent.parent._parts + [new_dirname] 97 | elif (new_dirpath is not None) and (new_dirname is None): 98 | dir_parts = [ 99 | new_dirpath, 100 | ] 101 | elif (new_dirpath is None) and (new_dirname is None): 102 | dir_parts = self.parent._parts 103 | else: 104 | raise ValueError("Cannot having both 'new_dirpath' and 'new_dirname'!") 105 | 106 | if new_basename is None: 107 | if new_fname is None: 108 | new_fname = self.fname 109 | if new_ext is None: 110 | new_ext = self.ext 111 | new_basename = new_fname + new_ext 112 | else: 113 | if (new_fname is not None) or (new_ext is not None): 114 | raise ValueError( 115 | "Cannot having both " 116 | "'new_basename' / 'new_fname', " 117 | "or 'new_basename' / 'new_ext'!" 118 | ) 119 | if new_bucket is None: 120 | p = self._from_parts( 121 | [ 122 | "dummy-bucket", 123 | ] 124 | + dir_parts 125 | + [ 126 | new_basename, 127 | ] 128 | ) 129 | p._bucket = None 130 | else: 131 | p = self._from_parts( 132 | [ 133 | new_bucket, 134 | ] 135 | + dir_parts 136 | + [ 137 | new_basename, 138 | ] 139 | ) 140 | return p 141 | 142 | def to_dir(self: "S3Path") -> "S3Path": 143 | """ 144 | Convert the S3Path to a directory. If the S3Path is a file, then append 145 | a "/" at the end. If the S3Path is already a directory, then do nothing. 146 | 147 | Example: 148 | 149 | >>> S3Path.from_s3_uri("s3://bucket/folder").to_dir() 150 | S3Path('s3://bucket/folder/') 151 | """ 152 | if self.is_dir(): 153 | return self.copy() 154 | elif self.is_file(): 155 | return self.joinpath("/") 156 | else: 157 | raise ValueError("only concrete file or folder S3Path can do .to_dir()") 158 | 159 | def to_file(self: "S3Path") -> "S3Path": 160 | """ 161 | Convert the S3Path to a file. If the S3Path is a directory, then strip 162 | out the last "/". If the S3Path is already a file, then do nothing. 163 | 164 | Example: 165 | 166 | >>> S3Path.from_s3_uri("s3://bucket/file/").to_dir() 167 | S3Path('s3://bucket/file/') 168 | """ 169 | if self.is_file(): 170 | return self.copy() 171 | elif self.is_dir(): 172 | p = self.copy() 173 | p._is_dir = False 174 | return p 175 | else: 176 | raise ValueError("only concrete file or folder S3Path can do .to_file()") 177 | -------------------------------------------------------------------------------- /s3pathlib/core/opener.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Smart open library integration. 5 | 6 | .. _bsm: https://github.com/aws-samples/boto-session-manager-project 7 | .. _smart_open: https://github.com/RaRe-Technologies/smart_open 8 | """ 9 | 10 | import typing as T 11 | 12 | from func_args import NOTHING, resolve_kwargs 13 | 14 | from ..metadata import warn_upper_case_in_metadata_key 15 | from ..aws import context 16 | from ..compat import smart_open, compat 17 | from ..type import MetadataType, TagType 18 | from ..tag import encode_url_query 19 | 20 | from .resolve_s3_client import resolve_s3_client 21 | 22 | if T.TYPE_CHECKING: # pragma: no cover 23 | from .s3path import S3Path 24 | from boto_session_manager import BotoSesManager 25 | 26 | 27 | class OpenerAPIMixin: 28 | """ 29 | A mixin class that implements the file-object protocol. 30 | """ 31 | def open( 32 | self: 'S3Path', 33 | mode: T.Optional[str] = "r", 34 | version_id: T.Optional[str] = NOTHING, 35 | buffering: T.Optional[int] = -1, 36 | encoding: T.Optional[str] = None, 37 | errors: T.Optional[str] = None, 38 | newline: T.Optional[str] = None, 39 | closefd=True, 40 | opener=None, 41 | ignore_ext: bool = False, 42 | compression: T.Optional[str] = None, 43 | multipart_upload: bool = True, 44 | metadata: T.Optional[MetadataType] = NOTHING, 45 | tags: T.Optional[TagType] = NOTHING, 46 | transport_params: T.Optional[dict] = None, 47 | bsm: T.Optional['BotoSesManager'] = None, 48 | ): 49 | """ 50 | Open S3Path as a file-liked object. 51 | 52 | Example:: 53 | 54 | >>> import json 55 | >>> with S3Path("s3://bucket/data.json").open("w") as f: 56 | ... json.dump({"a": 1}, f) 57 | 58 | >>> with S3Path("s3://bucket/data.json").open("r") as f: 59 | ... data = json.load(f) 60 | 61 | :param mode: "r", "w", "rb", "wb". 62 | :param version_id: optional version id you want to read from. 63 | :param buffering: See smart_open_. 64 | :param encoding: See smart_open_. 65 | :param errors: See smart_open_. 66 | :param newline: See smart_open_. 67 | :param closefd: See smart_open_. 68 | :param opener: See smart_open_. 69 | :param ignore_ext: See smart_open_. 70 | :param compression: whether do you want to compress the content. 71 | :param multipart_upload: do you want to use multi-parts upload, 72 | by default it is True. 73 | :param metadata: also put the user defined metadata dictionary. 74 | :param tags: also put the tag dictionary. 75 | :param bsm: See bsm_. 76 | 77 | :return: a file-like object that has ``read()`` and ``write()`` method. 78 | 79 | See smart_open_ for more info. 80 | Also see https://github.com/RaRe-Technologies/smart_open/blob/develop/howto.md#how-to-access-s3-anonymously 81 | for S3 related info. 82 | 83 | .. versionadded:: 1.0.1 84 | 85 | .. versionchanged:: 1.2.1 86 | 87 | add ``metadata`` and ``tags`` parameters 88 | 89 | .. versionchanged:: 2.0.1 90 | 91 | add ``version_id`` parameter 92 | """ 93 | s3_client = resolve_s3_client(context, bsm) 94 | if transport_params is None: 95 | transport_params = dict() 96 | transport_params["client"] = s3_client 97 | transport_params["multipart_upload"] = multipart_upload 98 | # write API doesn't take version_id parameter 99 | # set it to NOTHING in case human made a mistake 100 | if mode.startswith("r") is False: # pragma: no cover 101 | version_id = NOTHING 102 | if metadata is not NOTHING: 103 | warn_upper_case_in_metadata_key(metadata) 104 | if version_id is not NOTHING: 105 | transport_params["version_id"] = version_id 106 | 107 | open_kwargs = dict( 108 | uri=self.uri, 109 | mode=mode, 110 | buffering=buffering, 111 | encoding=encoding, 112 | errors=errors, 113 | newline=newline, 114 | closefd=closefd, 115 | opener=opener, 116 | transport_params=transport_params, 117 | ) 118 | 119 | if compat.smart_open_version_major < 6: # pragma: no cover 120 | open_kwargs["ignore_ext"] = ignore_ext 121 | if compat.smart_open_version_major >= 5 and compat.smart_open_version_major >= 1: # pragma: no cover 122 | if compression is not None: 123 | open_kwargs["compression"] = compression 124 | 125 | # if any of additional parameters exists, we need additional handling 126 | if sum([metadata is not None, tags is not None]) > 0: 127 | s3_client_kwargs = resolve_kwargs( 128 | Metadata=metadata, 129 | Tagging=tags if tags is NOTHING else encode_url_query(tags), 130 | ) 131 | if multipart_upload: 132 | client_kwargs = {"S3.Client.create_multipart_upload": s3_client_kwargs} 133 | else: 134 | client_kwargs = {"S3.Client.put_object": s3_client_kwargs} 135 | if "client_kwargs" in transport_params: # pragma: no cover 136 | transport_params["client_kwargs"].update(client_kwargs) 137 | else: 138 | transport_params["client_kwargs"] = client_kwargs 139 | return smart_open.open(**open_kwargs) 140 | -------------------------------------------------------------------------------- /s3pathlib/core/resolve_s3_client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Resolve what s3 client to use for API call. 5 | """ 6 | 7 | import typing as T 8 | 9 | from ..aws import Context 10 | 11 | if T.TYPE_CHECKING: # pragma: no cover 12 | from boto_session_manager import BotoSesManager 13 | from mypy_boto3_s3 import S3Client 14 | 15 | 16 | def resolve_s3_client( 17 | context: Context, 18 | bsm: T.Optional["BotoSesManager"] = None, 19 | ) -> "S3Client": 20 | """ 21 | Figure out the final boto session to use for API call. 22 | If ``BotoSesManager`` is defined, then prioritize to use it. 23 | """ 24 | if bsm is None: 25 | return context.s3_client 26 | else: 27 | return bsm.s3_client 28 | -------------------------------------------------------------------------------- /s3pathlib/core/s3path.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | The ``S3Path`` public API class. 5 | """ 6 | 7 | # The import order is very important. The later one depends on the earlier one. 8 | 9 | from .base import BaseS3Path 10 | from .is_test import IsTestAPIMixin 11 | from .uri import UriAPIMixin 12 | from .relative import RelativePathAPIMixin 13 | from .comparison import ComparisonAPIMixin 14 | from .attribute import AttributeAPIMixin 15 | from .joinpath import JoinPathAPIMixin 16 | from .mutate import MutateAPIMixin 17 | from .metadata import MetadataAPIMixin 18 | from .bucket import BucketAPIMixin 19 | from .tagging import TaggingAPIMixin 20 | from .iter_objects import IterObjectsAPIMixin 21 | from .iter_object_versions import IterObjectVersionsAPIMixin 22 | from .exists import ExistsAPIMixin 23 | from .rw import ReadAndWriteAPIMixin 24 | from .delete import DeleteAPIMixin 25 | from .upload import UploadAPIMixin 26 | from .copy import CopyAPIMixin 27 | from .sync import SyncAPIMixin 28 | from .serde import SerdeAPIMixin 29 | from .opener import OpenerAPIMixin 30 | 31 | 32 | class S3Path( 33 | BaseS3Path, 34 | IsTestAPIMixin, 35 | UriAPIMixin, 36 | RelativePathAPIMixin, 37 | ComparisonAPIMixin, 38 | AttributeAPIMixin, 39 | JoinPathAPIMixin, 40 | MutateAPIMixin, 41 | MetadataAPIMixin, 42 | BucketAPIMixin, 43 | TaggingAPIMixin, 44 | IterObjectsAPIMixin, 45 | IterObjectVersionsAPIMixin, 46 | ExistsAPIMixin, 47 | ReadAndWriteAPIMixin, 48 | DeleteAPIMixin, 49 | UploadAPIMixin, 50 | CopyAPIMixin, 51 | SyncAPIMixin, 52 | SerdeAPIMixin, 53 | OpenerAPIMixin, 54 | ): 55 | """ 56 | The ``S3Path`` public API class. 57 | """ 58 | -------------------------------------------------------------------------------- /s3pathlib/core/serde.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | serialization and deserialization. 5 | """ 6 | 7 | import typing as T 8 | 9 | if T.TYPE_CHECKING: # pragma: no cover 10 | from .s3path import S3Path 11 | 12 | 13 | class SerdeAPIMixin: 14 | """ 15 | A mixin class that implements the serialization and deserialization. 16 | """ 17 | 18 | def to_dict(self: "S3Path") -> dict: 19 | """ 20 | Serialize to Python dict 21 | 22 | .. versionadded:: 1.0.1 23 | """ 24 | return { 25 | "bucket": self._bucket, 26 | "parts": self._parts, 27 | "is_dir": self._is_dir, 28 | } 29 | 30 | @classmethod 31 | def from_dict(cls: "S3Path", dct: dict) -> "S3Path": 32 | """ 33 | Deserialize from Python dict 34 | 35 | .. versionadded:: 1.0.2 36 | """ 37 | return cls._from_parsed_parts( 38 | bucket=dct["bucket"], 39 | parts=dct["parts"], 40 | is_dir=dct["is_dir"], 41 | ) 42 | -------------------------------------------------------------------------------- /s3pathlib/core/tagging.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Tagging related API. 5 | 6 | .. _get_object_tagging: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/get_object_tagging.html 7 | .. _put_object_tagging: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/put_object_tagging.html 8 | """ 9 | 10 | import typing as T 11 | 12 | from func_args import NOTHING, resolve_kwargs 13 | 14 | from .resolve_s3_client import resolve_s3_client 15 | from ..better_client.tagging import update_object_tagging 16 | from ..aws import context 17 | from ..type import TagType 18 | from ..tag import parse_tags, encode_tag_set 19 | 20 | 21 | if T.TYPE_CHECKING: # pragma: no cover 22 | from .s3path import S3Path 23 | from boto_session_manager import BotoSesManager 24 | 25 | 26 | class TaggingAPIMixin: 27 | """ 28 | A mixin class that implements the tagging related methods. 29 | """ 30 | 31 | def get_tags( 32 | self: "S3Path", 33 | version_id: str = NOTHING, 34 | expected_bucket_owner: str = NOTHING, 35 | request_payer: str = NOTHING, 36 | bsm: T.Optional["BotoSesManager"] = None, 37 | ) -> T.Tuple[T.Optional[str], TagType]: 38 | """ 39 | Get s3 object tags in key value pairs dict. 40 | 41 | :return: ``(version_id, tags)``, tags is in string key value pairs dict. 42 | 43 | .. versionadded:: 1.1.1 44 | 45 | .. versionchanged:: 2.0.1 46 | 47 | Add ``version_id``, ``expected_bucket_owner``, ``request_payer`` parameter. 48 | """ 49 | self.ensure_object() 50 | s3_client = resolve_s3_client(context, bsm) 51 | res = s3_client.get_object_tagging( 52 | **resolve_kwargs( 53 | Bucket=self.bucket, 54 | Key=self.key, 55 | VersionId=version_id, 56 | ExpectedBucketOwner=expected_bucket_owner, 57 | RequestPayer=request_payer, 58 | ) 59 | ) 60 | returned_version_id = res.get("VersionId", None) 61 | tags = parse_tags(res.get("TagSet", [])) 62 | return returned_version_id, tags 63 | 64 | def put_tags( 65 | self: "S3Path", 66 | tags: TagType, 67 | version_id: str = NOTHING, 68 | content_md5: str = NOTHING, 69 | checksum_algorithm: str = NOTHING, 70 | expected_bucket_owner: str = NOTHING, 71 | request_payer: str = NOTHING, 72 | bsm: T.Optional["BotoSesManager"] = None, 73 | ) -> T.Tuple[T.Optional[str], TagType]: 74 | """ 75 | Do full replacement of s3 object tags. 76 | 77 | :param tags: the s3 object tags in string key value pairs dict. 78 | 79 | :return: ``(version_id, tags)``, tags is in string key value pairs dict. 80 | 81 | .. versionadded:: 1.1.1 82 | 83 | .. versionchanged:: 2.0.1 84 | 85 | Add ``version_id``, ``expected_bucket_owner``, ``request_payer`` parameter. 86 | """ 87 | self.ensure_object() 88 | s3_client = resolve_s3_client(context, bsm) 89 | res = s3_client.put_object_tagging( 90 | **resolve_kwargs( 91 | Bucket=self.bucket, 92 | Key=self.key, 93 | Tagging=dict(TagSet=encode_tag_set(tags)), 94 | VersionId=version_id, 95 | ContentMD5=content_md5, 96 | ChecksumAlgorithm=checksum_algorithm, 97 | ExpectedBucketOwner=expected_bucket_owner, 98 | RequestPayer=request_payer, 99 | ) 100 | ) 101 | returned_version_id = res.get("VersionId", None) 102 | return returned_version_id, tags 103 | 104 | def update_tags( 105 | self: "S3Path", 106 | tags: TagType, 107 | version_id: str = NOTHING, 108 | content_md5: str = NOTHING, 109 | checksum_algorithm: str = NOTHING, 110 | expected_bucket_owner: str = NOTHING, 111 | request_payer: str = NOTHING, 112 | bsm: T.Optional["BotoSesManager"] = None, 113 | ) -> T.Tuple[T.Optional[str], TagType]: 114 | """ 115 | Do partial updates of s3 object tags. 116 | 117 | :param tags: the s3 object tags in string key value pairs dict. 118 | 119 | :return: ``(version_id, tags)``, tags is the latest, merged object tags 120 | in string key value pairs dict. 121 | 122 | .. versionadded:: 1.1.1 123 | """ 124 | self.ensure_object() 125 | s3_client = resolve_s3_client(context, bsm) 126 | return update_object_tagging( 127 | s3_client=s3_client, 128 | bucket=self.bucket, 129 | key=self.key, 130 | tags=tags, 131 | version_id=version_id, 132 | content_md5=content_md5, 133 | checksum_algorithm=checksum_algorithm, 134 | expected_bucket_owner=expected_bucket_owner, 135 | request_payer=request_payer, 136 | ) 137 | -------------------------------------------------------------------------------- /s3pathlib/core/upload.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Upload file from local to s3. 5 | """ 6 | 7 | import typing as T 8 | 9 | from pathlib_mate import Path 10 | 11 | from .resolve_s3_client import resolve_s3_client 12 | from ..better_client.upload import upload_dir 13 | from ..type import PathType 14 | from ..aws import context 15 | 16 | if T.TYPE_CHECKING: # pragma: no cover 17 | from .s3path import S3Path 18 | from boto_session_manager import BotoSesManager 19 | from mypy_boto3_s3.type_defs import UploadFile 20 | 21 | 22 | class UploadAPIMixin: 23 | """ 24 | A mixin class that implements upload method. 25 | """ 26 | 27 | def upload_file( 28 | self: "S3Path", 29 | path: PathType, 30 | overwrite: bool = False, 31 | extra_args: dict = None, 32 | callback: callable = None, 33 | config=None, 34 | bsm: T.Optional["BotoSesManager"] = None, 35 | ): 36 | """ 37 | Upload a file from local file system to targeted S3 path 38 | 39 | Example:: 40 | 41 | >>> s3path = S3Path("bucket", "artifacts", "deployment.zip") 42 | >>> s3path.upload_file(path="/tmp/build/deployment.zip", overwrite=True) 43 | 44 | :param path: absolute path of the file on the local file system 45 | you want to upload 46 | :param overwrite: if False, non of the file will be upload / overwritten 47 | if any of target s3 location already taken. 48 | 49 | .. versionadded:: 1.0.1 50 | """ 51 | self.ensure_object() 52 | if overwrite is False: 53 | self.ensure_not_exists(bsm=bsm) 54 | p = Path(path) 55 | s3_client = resolve_s3_client(context, bsm) 56 | return s3_client.upload_file( 57 | Filename=p.abspath, 58 | Bucket=self.bucket, 59 | Key=self.key, 60 | ExtraArgs=extra_args, 61 | Callback=callback, 62 | Config=config, 63 | ) 64 | 65 | def upload_dir( 66 | self: "S3Path", 67 | local_dir: PathType, 68 | pattern: str = "**/*", 69 | overwrite: bool = False, 70 | bsm: T.Optional["BotoSesManager"] = None, 71 | ) -> int: 72 | """ 73 | Upload a directory on local file system and all sub-folders, files to 74 | a S3 prefix (logical directory) 75 | 76 | Example:: 77 | 78 | >>> s3path = S3Path("bucket", "datalake", "orders/") 79 | >>> s3path.upload_dir(path="/data/orders", overwrite=True) 80 | 81 | :param local_dir: absolute path of the directory on the 82 | local file system you want to upload 83 | :param pattern: linux styled glob pattern match syntax. see this 84 | official reference 85 | https://docs.python.org/3/library/pathlib.html#pathlib.Path.glob 86 | for more details 87 | :param overwrite: if False, non of the file will be upload / overwritten 88 | if any of target s3 location already taken. 89 | 90 | :return: number of files uploaded 91 | 92 | .. versionadded:: 1.0.1 93 | """ 94 | self.ensure_dir() 95 | s3_client = resolve_s3_client(context, bsm) 96 | return upload_dir( 97 | s3_client=s3_client, 98 | bucket=self.bucket, 99 | prefix=self.key, 100 | local_dir=local_dir, 101 | pattern=pattern, 102 | overwrite=overwrite, 103 | ) 104 | -------------------------------------------------------------------------------- /s3pathlib/docs/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | doc_data = dict() 4 | -------------------------------------------------------------------------------- /s3pathlib/exc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Exception creator and helpers, argument validators, and more. 5 | """ 6 | 7 | import typing as T 8 | 9 | if T.TYPE_CHECKING: # pragma: no cover 10 | from .core.s3path import S3Path 11 | 12 | 13 | def ensure_one_and_only_one_not_none(**kwargs) -> None: 14 | """ 15 | Ensure only exact one of the keyword argument is not None. 16 | """ 17 | if len(kwargs) == 0: 18 | raise ValueError 19 | if sum([v is not None for _, v in kwargs.items()]) != 1: 20 | raise ValueError( 21 | f"one and only one of arguments from " f"{list(kwargs)} can be not None!" 22 | ) 23 | 24 | 25 | def ensure_all_none(**kwargs) -> None: 26 | """ 27 | Ensure all the keyword arguments are None. 28 | """ 29 | if len(kwargs) == 0: 30 | raise ValueError 31 | if sum([v is not None for _, v in kwargs.items()]) != 0: 32 | raise ValueError(f"arguments from {list(kwargs)} has to be all None!") 33 | 34 | 35 | class _UriRelatedError: 36 | _tpl: str 37 | 38 | @classmethod 39 | def make( 40 | cls: T.Type[T.Union[Exception, "_UriRelatedError"]], 41 | uri: str, 42 | ): 43 | return cls(cls._tpl.format(uri=uri)) 44 | 45 | 46 | class S3NotExist(FileNotFoundError, _UriRelatedError): 47 | _tpl = "{uri!r} does not exist!" 48 | 49 | 50 | class S3BucketNotExist(FileNotFoundError, _UriRelatedError): 51 | _tpl = "S3 bucket {uri!r} does not exist!" 52 | 53 | 54 | class S3FolderNotExist(FileNotFoundError, _UriRelatedError): 55 | _tpl = "S3 folder {uri!r} does not exist!" 56 | 57 | 58 | class S3FileNotExist(FileNotFoundError, _UriRelatedError): 59 | _tpl = "S3 object {uri!r} does not exist!" 60 | 61 | 62 | class S3AlreadyExist(FileExistsError, _UriRelatedError): 63 | _tpl = "{uri!r} already exist!" 64 | 65 | 66 | class S3BucketAlreadyExist(FileExistsError, _UriRelatedError): 67 | _tpl = "S3 bucket {uri!r} already exist!" 68 | 69 | 70 | class S3FolderAlreadyExist(FileExistsError, _UriRelatedError): 71 | _tpl = "S3 folder {uri!r} already exist!" 72 | 73 | 74 | class S3FileAlreadyExist(FileExistsError, _UriRelatedError): 75 | _tpl = "S3 object {uri!r} already exist!" 76 | 77 | 78 | class S3PermissionDenied(PermissionError): 79 | pass 80 | 81 | 82 | class _S3PathTypeError(TypeError): 83 | _expected_type: str 84 | 85 | @classmethod 86 | def make( 87 | cls: T.Type[T.Union[Exception, "_S3PathTypeError"]], 88 | s3path: "S3Path", 89 | ): 90 | return cls(f"{s3path!r} is not a {cls._expected_type}! ") 91 | 92 | 93 | class S3PathIsNotBucketError(_S3PathTypeError): 94 | _expected_type = "bucket" 95 | 96 | 97 | class S3PathIsNotFolderError(_S3PathTypeError): 98 | _expected_type = "dir" 99 | 100 | 101 | class S3PathIsNotFileError(_S3PathTypeError): 102 | _expected_type = "file" 103 | 104 | 105 | class S3PathIsNotRelpathError(_S3PathTypeError): 106 | _expected_type = "relpath" 107 | -------------------------------------------------------------------------------- /s3pathlib/marker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import warnings 4 | import functools 5 | 6 | 7 | def warn_deprecate( 8 | func_name: str, 9 | version: str, 10 | message: str, 11 | ): 12 | warnings.warn(f"{func_name!r} will be deprecated on {version}: {message}") 13 | 14 | 15 | def deprecate_v1(version: str, message: str): 16 | def deco(func): 17 | @functools.wraps(func) 18 | def wrapper(*args, **kwargs): 19 | warn_deprecate(func.__name__, version, message) 20 | return func(*args, **kwargs) 21 | 22 | return wrapper 23 | 24 | return deco 25 | 26 | 27 | def deprecate_v2(version: str, message: str): 28 | from decorator import decorator 29 | 30 | @decorator 31 | def deco(func, *args, **kwargs): 32 | warn_deprecate(func.__name__, version, message) 33 | return func(*args, **kwargs) 34 | 35 | return deco 36 | -------------------------------------------------------------------------------- /s3pathlib/metadata.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | This module provides AWS S3 object metadata manipulation helpers. 5 | """ 6 | 7 | import warnings 8 | 9 | from .type import MetadataType 10 | 11 | 12 | def warn_upper_case_in_metadata_key(metadata: MetadataType): 13 | """ 14 | Warn if there are uppercase letters used in user-defined metadata. 15 | 16 | Ref: 17 | 18 | - https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata 19 | """ 20 | for k, v in metadata.items(): 21 | if k.lower() != k: 22 | msg = ( 23 | f"Based on this document " 24 | f"https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata " 25 | f"Amazon will automatically convert user-defined metadata key to lowercase. " 26 | f"However, you have a key {k!r} in the metadata that uses uppercase letters." 27 | ) 28 | warnings.warn(msg, UserWarning) 29 | -------------------------------------------------------------------------------- /s3pathlib/tag.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | This module provides AWS Tags manipulation helpers. 5 | 6 | .. _get_bucket_tagging: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/get_bucket_tagging.html 7 | .. _get_object_tagging: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/get_object_tagging.html 8 | .. _put_object: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/put_object.html 9 | .. _put_bucket_tagging: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/put_bucket_tagging.html 10 | .. _put_object_tagging: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/put_object_tagging.html 11 | """ 12 | 13 | from urllib.parse import urlencode 14 | 15 | from .type import TagType, TagSetType 16 | 17 | 18 | def parse_tags(data: TagSetType) -> TagType: 19 | """ 20 | Convert the tag set in boto3 API response into pythonic dictionary key value 21 | pairs. 22 | 23 | - get_bucket_tagging_: it's a tag set. 24 | - get_object_tagging_: it's a tag set. 25 | 26 | :param data: the tag set in boto3 API response. 27 | :return: the pythonic dictionary key value pairs. 28 | """ 29 | if isinstance(data, list): 30 | return {dct["Key"]: dct["Value"] for dct in data} 31 | else: # pragma: no cover 32 | raise NotImplementedError 33 | 34 | 35 | def encode_tag_set(tags: TagType) -> TagSetType: 36 | """ 37 | Some API requires: ``[{"Key": "name", "Value": "Alice"}, {...}, ...]`` 38 | for tagging parameter. 39 | 40 | Example:: 41 | 42 | >>> encode_tag_set({"name": "Alice", ...}) 43 | [{"Key": "name", "Value": "Alice"}, ...] 44 | """ 45 | return [{"Key": k, "Value": v} for k, v in tags.items()] 46 | 47 | 48 | def encode_url_query(tags: TagType) -> str: 49 | """ 50 | Some API requires: ``Key1=Value1&Key2=Value2`` for tagging parameter. 51 | 52 | Example:: 53 | 54 | >>> encode_url_query({"name": "Alice", ...}) 55 | "name=Alice&..." 56 | """ 57 | return urlencode(tags) 58 | 59 | 60 | def encode_for_put_object(tags: TagType) -> str: 61 | """ 62 | Encode tags for put_object_. 63 | """ 64 | return encode_url_query(tags) 65 | 66 | 67 | def encode_for_put_bucket_tagging(tags: TagType) -> TagSetType: 68 | """ 69 | Encode tags for put_bucket_tagging_. 70 | """ 71 | return encode_tag_set(tags) 72 | 73 | 74 | def encode_for_put_object_tagging(tags: TagType) -> TagSetType: 75 | """ 76 | Encode tags for put_object_tagging_. 77 | """ 78 | return encode_tag_set(tags) 79 | -------------------------------------------------------------------------------- /s3pathlib/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .helpers import run_cov_test 4 | -------------------------------------------------------------------------------- /s3pathlib/tests/helpers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys 4 | import subprocess 5 | 6 | from .paths import dir_project_root, dir_htmlcov, path_cov_index_html, bin_pytest 7 | 8 | if sys.platform == "win32": 9 | open_cmd = "open" 10 | else: 11 | open_cmd = "open" 12 | 13 | 14 | def normalize_module(module: str) -> str: 15 | if module.endswith(".py"): 16 | return module[:-3] 17 | else: 18 | return module 19 | 20 | 21 | def _run_cov_test( 22 | bin_pytest: str, 23 | script: str, 24 | module: str, 25 | root_dir: str, 26 | htmlcov_dir: str, 27 | ): 28 | """ 29 | A simple wrapper around pytest + coverage cli command. 30 | :param bin_pytest: the path to pytest executable 31 | :param script: the path to test script 32 | :param module: the dot notation to the python module you want to calculate 33 | coverage 34 | :param root_dir: the dir to dump coverage results binary file 35 | :param htmlcov_dir: the dir to dump HTML output 36 | """ 37 | module = normalize_module(module) 38 | args = [ 39 | bin_pytest, 40 | "-s", 41 | "--tb=native", 42 | f"--rootdir={root_dir}", 43 | f"--cov={module}", 44 | "--cov-report", 45 | "term-missing", 46 | "--cov-report", 47 | f"html:{htmlcov_dir}", 48 | script, 49 | ] 50 | subprocess.run(args) 51 | 52 | 53 | def run_cov_test(script: str, module: str, preview: bool = False): 54 | _run_cov_test( 55 | bin_pytest=f"{bin_pytest}", 56 | script=script, 57 | module=module, 58 | root_dir=f"{dir_project_root}", 59 | htmlcov_dir=f"{dir_htmlcov}", 60 | ) 61 | if preview: 62 | subprocess.run(["open", f"{path_cov_index_html}"]) 63 | -------------------------------------------------------------------------------- /s3pathlib/tests/paths.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys 4 | from pathlib import Path 5 | 6 | dir_project_root = Path(__file__).absolute().parent.parent.parent 7 | 8 | # code structure 9 | dir_tests: Path = dir_project_root / "tests" 10 | dir_htmlcov: Path = dir_project_root / "htmlcov" 11 | path_cov_index_html = dir_htmlcov / "index.html" 12 | 13 | # virtual environment 14 | dir_bin: Path = Path(sys.executable).parent 15 | bin_pytest: Path = dir_bin / "pytest" 16 | 17 | # test data 18 | dir_test_data: Path = dir_tests / "data" 19 | dir_test_list_objects_folder: Path = dir_test_data / "list_objects_folder" 20 | dir_test_upload_dir_folder: Path = dir_test_data / "upload_dir_folder" 21 | -------------------------------------------------------------------------------- /s3pathlib/type.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Type hint variables. 5 | """ 6 | 7 | import typing as T 8 | 9 | from pathlib import Path as Path1 10 | from pathlib_mate import Path as Path2 11 | 12 | TagType = T.Dict[str, str] # {"Key": "Value"} 13 | TagSetType = T.List[T.Dict[str, str]] # [{"Key": "Value"}, ...] 14 | 15 | MetadataType = T.Dict[str, str] # {"Key": "Value"} 16 | PathType = T.Union[str, Path1, Path2] # str, pathlib.Path, pathlib_mate.Path 17 | -------------------------------------------------------------------------------- /s3pathlib/validate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import string 4 | 5 | valid_bucket_charset: set = set(string.ascii_lowercase + string.digits + ".-") 6 | letter_and_number: set = set(string.ascii_lowercase + string.digits) 7 | 8 | safe_key_charset: set = set(string.ascii_letters + string.digits + "/!-_.*'()") 9 | req_special_handling_key_charset: set = set("&$@=;:+ ,?") 10 | to_avoid_key_charset: set = set("\\{}^&`[]\"<>#|~%") 11 | valid_key_charset: set = set.union(safe_key_charset, req_special_handling_key_charset) 12 | 13 | 14 | class S3BucketValidationError(Exception): pass 15 | 16 | class S3KeyValidationError(Exception): pass 17 | 18 | 19 | def validate_s3_bucket(bucket: str) -> None: 20 | """ 21 | Raise exception if validation not passed. 22 | 23 | Ref: 24 | 25 | - `Bucket naming rules `_ 26 | """ 27 | if not (3 <= len(bucket) <= 63): 28 | raise ValueError("Bucket names must be between 3 and 63 characters long.") 29 | 30 | invalid_chars = set(bucket).difference(valid_bucket_charset) 31 | if len(invalid_chars) != 0: 32 | raise ValueError( 33 | ( 34 | "Bucket names can consist only of lowercase letters, numbers, " 35 | "dots (.), and hyphens (-). invalid char found {}" 36 | ).format(invalid_chars) 37 | ) 38 | 39 | if (bucket[0] not in letter_and_number) or (bucket[-1] not in letter_and_number): 40 | raise ValueError("Bucket names must begin and end with a letter or number.") 41 | 42 | try: 43 | parts = [int(part) for part in bucket.split(".")] 44 | assert len(parts) == 4 45 | for p in parts: 46 | assert 0 <= p <= 255 47 | raise S3BucketValidationError("Bucket names must not be formatted as an IP address (for example, 192.168.5.4).") 48 | except S3BucketValidationError as e: 49 | raise e 50 | except: 51 | pass 52 | 53 | if bucket.startswith("xn--"): 54 | raise ValueError("Bucket names must not start with the prefix xn--.") 55 | if bucket.endswith("-s3alias"): 56 | raise ValueError("Bucket names must not end with the suffix -s3alias. This suffix is reserved for access point alias names.") 57 | # raise ValueError("Buckets used with Amazon S3 Transfer Acceleration can't have dots (.) in their names.") 58 | 59 | 60 | def validate_s3_key(key: str) -> None: 61 | """ 62 | Raise exception if validation not passed. 63 | 64 | Ref: 65 | 66 | - `Key naming rules `_ 67 | """ 68 | if len(key) > 1024: 69 | raise ValueError( 70 | "S3 key must be less that 1024 chars " 71 | "to construct the S3 console url!" 72 | ) 73 | 74 | invalid_chars = set(key).difference(valid_key_charset) 75 | if len(invalid_chars) != 0: 76 | doc_url = "https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html#object-key-guidelines" 77 | raise ValueError( 78 | ( 79 | "Invalid char found {}, " 80 | "read {} " 81 | "for more info" 82 | ).format(invalid_chars, doc_url) 83 | ) 84 | 85 | 86 | def validate_s3_uri(uri: str) -> None: 87 | """ 88 | Raise exception if validation not passed. 89 | 90 | S3 URI is just ``s3://{bucket}/{key}`` 91 | """ 92 | if not uri.startswith("s3://"): 93 | raise ValueError("S3 URI must starts with 's3://'") 94 | 95 | if uri.count("/") < 3: 96 | raise ValueError( 97 | "S3 URI must have at least three '/', " 98 | "for example: s3://bucket/" 99 | ) 100 | 101 | parts = uri.split("/", 3) 102 | bucket = parts[2] 103 | key = parts[3] 104 | validate_s3_bucket(bucket) 105 | validate_s3_key(key) 106 | 107 | 108 | def validate_s3_arn(arn: str) -> None: 109 | """ 110 | Raise exception if validation not passed. 111 | 112 | S3 ARN is just: 113 | 114 | - for bucket: ``arn:aws:s3:::{bucket}`` 115 | - for object: ``arn:aws:s3:::{bucket}/{key}`` 116 | - for directory: ``arn:aws:s3:::{bucket}/{prefix}/`` 117 | """ 118 | if not arn.startswith("arn:aws:s3:::"): 119 | raise ValueError("S3 ARN must starts with 'arn:aws:s3:::'") 120 | 121 | path = arn.replace("arn:aws:s3:::", "", 1) 122 | 123 | if "/" not in path: # path is the bucket 124 | validate_s3_bucket(path) 125 | else: 126 | bucket, key = path.split("/", 1) 127 | validate_s3_bucket(bucket) 128 | validate_s3_key(key) 129 | -------------------------------------------------------------------------------- /tests/all.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | if __name__ == "__main__": 4 | import pytest 5 | 6 | pytest.main(["-s", "--tb=native"]) 7 | -------------------------------------------------------------------------------- /tests/api/README.rst: -------------------------------------------------------------------------------- 1 | Test code in this folder consider as stable API. 2 | -------------------------------------------------------------------------------- /tests/api/all.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | if __name__ == "__main__": 4 | import pytest 5 | 6 | pytest.main(["-s", "--tb=native"]) 7 | -------------------------------------------------------------------------------- /tests/api/test_import.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | 6 | def test(): 7 | import s3pathlib 8 | 9 | _ = s3pathlib.S3Path 10 | _ = s3pathlib.context 11 | _ = s3pathlib.utils 12 | _ = s3pathlib.api 13 | 14 | 15 | if __name__ == "__main__": 16 | import os 17 | 18 | basename = os.path.basename(__file__) 19 | pytest.main([basename, "-s", "--tb=native"]) 20 | -------------------------------------------------------------------------------- /tests/better_client/README.rst: -------------------------------------------------------------------------------- 1 | We should NOT use S3Path object in test cases of ``better_client``! 2 | -------------------------------------------------------------------------------- /tests/better_client/all.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | if __name__ == "__main__": 4 | import pytest 5 | 6 | pytest.main(["-s", "--tb=native"]) 7 | -------------------------------------------------------------------------------- /tests/better_client/dummy_data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from s3pathlib.utils import smart_join_s3_key 4 | from s3pathlib.tests.mock import BaseTest 5 | from s3pathlib.tests.paths import ( 6 | dir_test_list_objects_folder, 7 | dir_test_upload_dir_folder, 8 | ) 9 | 10 | 11 | class DummyData(BaseTest): 12 | prefix_dummy_data: str 13 | key_hello: str 14 | key_soft_folder: str 15 | prefix_soft_folder: str 16 | key_soft_folder_file: str 17 | key_hard_folder: str 18 | prefix_hard_folder: str 19 | key_hard_folder_file: str 20 | key_empty_hard_folder: str 21 | prefix_empty_hard_folder: str 22 | 23 | key_never_exists: str 24 | prefix_never_exists: str 25 | 26 | @classmethod 27 | def setup_dummy_data(cls): 28 | """ 29 | Following test S3 objects are created. Key endswith "/" are special 30 | empty S3 object representing a folder. 31 | 32 | - ``/{prefix}/hello.txt`` 33 | - ``/{prefix}/soft_folder`` (not exists, just for testing) 34 | - ``/{prefix}/soft_folder/`` (not exists, just for testing) 35 | - ``/{prefix}/soft_folder/file.txt`` 36 | - ``/{prefix}/hard_folder`` (not exists, just for testing) 37 | - ``/{prefix}/hard_folder/`` 38 | - ``/{prefix}/hard_folder/file.txt`` 39 | - ``/{prefix}/empty_hard_folder`` (not exists, just for testing) 40 | - ``/{prefix}/empty_hard_folder/`` 41 | - ``/{prefix}/never_exists`` (not exists, just for testing) 42 | - ``/{prefix}/never_exists/`` (not exists, just for testing) 43 | """ 44 | s3_client = cls.bsm.s3_client 45 | 46 | cls.prefix_dummy_data = smart_join_s3_key( 47 | parts=[cls.get_prefix(), "dummy_data"], 48 | is_dir=True, 49 | ) 50 | cls.key_hello = smart_join_s3_key( 51 | parts=[cls.prefix_dummy_data, "hello.txt"], 52 | is_dir=False, 53 | ) 54 | cls.key_soft_folder = smart_join_s3_key( 55 | [cls.prefix_dummy_data, "soft_folder"], is_dir=False 56 | ) 57 | cls.prefix_soft_folder = cls.key_soft_folder + "/" 58 | cls.key_soft_folder_file = smart_join_s3_key( 59 | parts=[cls.prefix_soft_folder, "file.txt"], 60 | is_dir=False, 61 | ) 62 | cls.key_hard_folder = smart_join_s3_key( 63 | parts=[cls.prefix_dummy_data, "hard_folder"], 64 | is_dir=False, 65 | ) 66 | cls.prefix_hard_folder = cls.key_hard_folder + "/" 67 | cls.key_hard_folder_file = smart_join_s3_key( 68 | parts=[cls.prefix_hard_folder, "file.txt"], 69 | is_dir=False, 70 | ) 71 | cls.key_empty_hard_folder = smart_join_s3_key( 72 | parts=[cls.prefix_dummy_data, "empty_hard_folder"], 73 | is_dir=False, 74 | ) 75 | cls.prefix_empty_hard_folder = cls.key_empty_hard_folder + "/" 76 | cls.key_never_exists = smart_join_s3_key( 77 | parts=[cls.prefix_dummy_data, "never_exists"], 78 | is_dir=False, 79 | ) 80 | cls.prefix_never_exists = cls.key_never_exists + "/" 81 | 82 | bucket = cls.get_bucket() 83 | 84 | s3_client.put_object( 85 | Bucket=bucket, 86 | Key=cls.key_hello, 87 | Body="Hello World!", 88 | ) 89 | 90 | s3_client.put_object( 91 | Bucket=bucket, 92 | Key=cls.key_soft_folder_file, 93 | Body="Hello World!", 94 | ) 95 | 96 | s3_client.put_object( 97 | Bucket=bucket, 98 | Key=cls.prefix_hard_folder, 99 | Body="", 100 | ) 101 | 102 | s3_client.put_object( 103 | Bucket=bucket, 104 | Key=cls.key_hard_folder_file, 105 | Body="Hello World!", 106 | ) 107 | 108 | s3_client.put_object( 109 | Bucket=bucket, 110 | Key=cls.prefix_empty_hard_folder, 111 | Body="", 112 | ) 113 | 114 | prefix_test_list_objects: str 115 | 116 | @classmethod 117 | def setup_list_objects_folder(cls): 118 | from s3pathlib.better_client.upload import upload_dir 119 | 120 | # setup test data for iter_objects 121 | cls.prefix_test_list_objects = smart_join_s3_key( 122 | parts=[cls.get_prefix(), "test_iter_objects"], 123 | is_dir=True, 124 | ) 125 | upload_dir( 126 | s3_client=cls.bsm.s3_client, 127 | bucket=cls.get_bucket(), 128 | prefix=cls.prefix_test_list_objects, 129 | local_dir=f"{dir_test_list_objects_folder}", 130 | pattern="**/*.txt", 131 | overwrite=True, 132 | ) 133 | 134 | prefix_test_upload_dir: str 135 | 136 | @classmethod 137 | def setup_test_upload_dir(cls): 138 | from s3pathlib.better_client.upload import upload_dir 139 | 140 | cls.prefix_test_upload_dir = smart_join_s3_key( 141 | parts=[cls.get_prefix(), "test_upload_dir"], 142 | is_dir=True, 143 | ) 144 | upload_dir( 145 | s3_client=cls.bsm.s3_client, 146 | bucket=cls.get_bucket(), 147 | prefix=cls.prefix_test_upload_dir, 148 | local_dir=f"{dir_test_upload_dir_folder}", 149 | pattern="**/*.txt", 150 | overwrite=True, 151 | ) 152 | -------------------------------------------------------------------------------- /tests/better_client/test_client_head_bucket.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from s3pathlib.better_client.head_bucket import is_bucket_exists 4 | from s3pathlib.tests import run_cov_test 5 | from s3pathlib.tests.mock import BaseTest 6 | 7 | 8 | class BetterHeadBucket(BaseTest): 9 | module = "better_client.head_bucket" 10 | 11 | @classmethod 12 | def custom_setup_class(cls): 13 | cls.bsm.s3_client.create_bucket(Bucket="this-bucket-exists") 14 | 15 | def test(self): 16 | assert is_bucket_exists(self.s3_client, "this-bucket-exists") is True 17 | assert is_bucket_exists(self.s3_client, "this-bucket-not-exists") is False 18 | 19 | 20 | # NOTE: this module should ONLY be tested with MOCK 21 | # DO NOT USE REAL S3 BUCKET 22 | class TestUseMock(BetterHeadBucket): 23 | use_mock = True 24 | 25 | 26 | if __name__ == "__main__": 27 | run_cov_test(__file__, module="s3pathlib.better_client.head_bucket", preview=False) 28 | -------------------------------------------------------------------------------- /tests/better_client/test_client_head_object.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from s3pathlib import exc 5 | from s3pathlib.better_client.head_object import ( 6 | head_object, 7 | is_object_exists, 8 | ) 9 | from s3pathlib.utils import smart_join_s3_key 10 | from s3pathlib.tests import run_cov_test 11 | 12 | from dummy_data import DummyData 13 | 14 | 15 | class BetterHeadObject(DummyData): 16 | module = "better_client.head_object" 17 | 18 | @classmethod 19 | def custom_setup_class(cls): 20 | cls.setup_dummy_data() 21 | 22 | def _test_before_and_after_put_object(self): 23 | # at begin, no object exists 24 | bucket = self.bucket 25 | key = smart_join_s3_key([self.prefix, "file.txt"], is_dir=False) 26 | assert is_object_exists(self.s3_client, bucket=bucket, key=key) is False 27 | 28 | with pytest.raises(exc.S3FileNotExist): 29 | head_object(self.s3_client, bucket=bucket, key=key) 30 | 31 | assert ( 32 | head_object(self.s3_client, bucket=bucket, key=key, ignore_not_found=True) 33 | is None 34 | ) 35 | 36 | # put the object 37 | self.s3_client.put_object(Bucket=bucket, Key=key, Body="hello") 38 | 39 | assert is_object_exists(self.s3_client, bucket=bucket, key=key) is True 40 | 41 | res = head_object(self.s3_client, bucket=bucket, key=key) 42 | assert res["ContentLength"] == len("hello") 43 | 44 | def _test_is_object_exists(self): 45 | s3_client = self.s3_client 46 | bucket = self.bucket 47 | 48 | assert ( 49 | is_object_exists(s3_client=s3_client, bucket=bucket, key=self.key_hello) 50 | is True 51 | ) 52 | assert ( 53 | is_object_exists( 54 | s3_client=s3_client, bucket=bucket, key=self.key_soft_folder 55 | ) 56 | is False 57 | ) 58 | assert ( 59 | is_object_exists( 60 | s3_client=s3_client, bucket=bucket, key=self.prefix_soft_folder 61 | ) 62 | is False 63 | ) 64 | assert ( 65 | is_object_exists( 66 | s3_client=s3_client, bucket=bucket, key=self.key_soft_folder_file 67 | ) 68 | is True 69 | ) 70 | assert ( 71 | is_object_exists( 72 | s3_client=s3_client, bucket=bucket, key=self.key_hard_folder 73 | ) 74 | is False 75 | ) 76 | assert ( 77 | is_object_exists( 78 | s3_client=s3_client, bucket=bucket, key=self.prefix_hard_folder 79 | ) 80 | is True 81 | ) 82 | assert ( 83 | is_object_exists( 84 | s3_client=s3_client, bucket=bucket, key=self.key_hard_folder_file 85 | ) 86 | is True 87 | ) 88 | assert ( 89 | is_object_exists( 90 | s3_client=s3_client, bucket=bucket, key=self.key_empty_hard_folder 91 | ) 92 | is False 93 | ) 94 | assert ( 95 | is_object_exists( 96 | s3_client=s3_client, bucket=bucket, key=self.prefix_empty_hard_folder 97 | ) 98 | is True 99 | ) 100 | 101 | def test(self): 102 | self._test_before_and_after_put_object() 103 | self._test_is_object_exists() 104 | 105 | 106 | # NOTE: this module should ONLY be tested with MOCK 107 | # DO NOT USE REAL S3 BUCKET 108 | class TestUseMock(BetterHeadObject): 109 | use_mock = True 110 | 111 | 112 | if __name__ == "__main__": 113 | run_cov_test(__file__, module="s3pathlib.better_client.head_object", preview=False) 114 | -------------------------------------------------------------------------------- /tests/better_client/test_client_list_object_versions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import typing as T 4 | import pytest 5 | from s3pathlib.better_client.list_object_versions import ( 6 | paginate_list_object_versions, 7 | ) 8 | from s3pathlib.utils import smart_join_s3_key 9 | from s3pathlib.tests import run_cov_test 10 | 11 | from dummy_data import DummyData 12 | 13 | 14 | class BetterListObjectVersions(DummyData): 15 | module = "better_client.list_object_versions" 16 | 17 | def _test_paginate_list_object_versions_for_object(self): 18 | # prepare data 19 | s3_client = self.s3_client 20 | bucket = self.bucket_with_versioning 21 | prefix = smart_join_s3_key( 22 | [self.get_prefix(), "list_object_versions"], is_dir=True 23 | ) 24 | key = smart_join_s3_key([prefix, "for_object/file.txt"], is_dir=False) 25 | 26 | # create 5 version and 2 delete markers 27 | for i in [1, 2]: 28 | s3_client.put_object(Bucket=bucket, Key=key, Body=f"v{i}") 29 | s3_client.delete_object(Bucket=bucket, Key=key) 30 | for i in [3, 4]: 31 | s3_client.put_object(Bucket=bucket, Key=key, Body=f"v{i}") 32 | s3_client.delete_object(Bucket=bucket, Key=key) 33 | for i in [5]: 34 | s3_client.put_object(Bucket=bucket, Key=key, Body=f"v{i}") 35 | 36 | # check the number of versions, delete markers and common prefixes 37 | proxy = paginate_list_object_versions( 38 | s3_client=s3_client, 39 | bucket=bucket, 40 | prefix=key, 41 | ) 42 | ( 43 | versions, 44 | delete_markers, 45 | common_prefixes, 46 | ) = proxy.versions_and_delete_markers_and_common_prefixes() 47 | assert len(versions) == 5 48 | assert len(delete_markers) == 2 49 | assert len(common_prefixes) == 0 50 | 51 | def _test_paginate_list_object_versions_for_folder(self): 52 | # prepare data 53 | s3_client = self.s3_client 54 | bucket = self.bucket_with_versioning 55 | prefix = smart_join_s3_key( 56 | [self.get_prefix(), "list_object_versions_for_folder"], is_dir=True 57 | ) 58 | 59 | def put(suffix: str, content: str): 60 | s3_client.put_object(Bucket=bucket, Key=f"{prefix}{suffix}", Body=content) 61 | 62 | def delete(suffix: str): 63 | s3_client.delete_object(Bucket=bucket, Key=f"{prefix}{suffix}") 64 | 65 | def list_object_versions(delimiter: T.Optional[str] = None): 66 | kwargs = dict( 67 | s3_client=s3_client, 68 | bucket=bucket, 69 | prefix=prefix, 70 | ) 71 | if delimiter: 72 | kwargs["delimiter"] = delimiter 73 | return paginate_list_object_versions(**kwargs) 74 | 75 | put("README.txt", "this is read me v1") 76 | delete("README.txt") 77 | put("README.txt", "this is read me v2") 78 | 79 | put("hard_folder/", "") 80 | 81 | put("hard_folder/hard_copy.txt", "hard copy v1") 82 | delete("hard_folder/hard_copy.txt") 83 | put("hard_folder/hard_copy.txt", "hard copy v2") 84 | 85 | put("soft_folder/soft_copy.txt", "soft copy v1") 86 | delete("soft_folder/soft_copy.txt") 87 | put("soft_folder/soft_copy.txt", "soft copy v2") 88 | put("soft_folder/soft_copy.txt", "soft copy v3") 89 | 90 | put("soft_folder/sub_folder/password.txt", "pwd v1") 91 | 92 | # check the number of versions, delete markers and common prefixes 93 | ( 94 | versions, 95 | delete_markers, 96 | common_prefixes, 97 | ) = list_object_versions().versions_and_delete_markers_and_common_prefixes() 98 | assert len(versions) == 9 99 | assert len(delete_markers) == 3 100 | assert len(common_prefixes) == 0 101 | 102 | # check the number of versions, delete markers and common prefixes 103 | assert len(list_object_versions().versions().all()) == 9 104 | assert len(list_object_versions().delete_markers().all()) == 3 105 | assert len(list_object_versions().common_prefixes().all()) == 0 106 | 107 | # check the number of versions, delete markers and common prefixes 108 | ( 109 | versions, 110 | delete_markers, 111 | common_prefixes, 112 | ) = list_object_versions("/").versions_and_delete_markers_and_common_prefixes() 113 | assert len(versions) == 2 114 | assert len(delete_markers) == 1 115 | assert len(common_prefixes) == 2 116 | 117 | def test(self): 118 | self._test_paginate_list_object_versions_for_object() 119 | self._test_paginate_list_object_versions_for_folder() 120 | 121 | 122 | # class Test(BetterListObjectVersions): 123 | # use_mock = False 124 | 125 | 126 | class TestUseMock(BetterListObjectVersions): 127 | use_mock = True 128 | 129 | 130 | if __name__ == "__main__": 131 | run_cov_test( 132 | __file__, module="s3pathlib.better_client.list_object_versions", preview=False 133 | ) 134 | -------------------------------------------------------------------------------- /tests/better_client/test_client_tagging.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from s3pathlib.better_client.tagging import ( 6 | update_bucket_tagging, 7 | update_object_tagging, 8 | ) 9 | from s3pathlib.tests import run_cov_test 10 | from s3pathlib.tests.mock import BaseTest 11 | 12 | 13 | class BetterUpload(BaseTest): 14 | module = "better_client.tagging" 15 | 16 | def _test_bucket_tagging(self): 17 | s3_client = self.s3_client 18 | bucket = self.bucket 19 | 20 | tags = update_bucket_tagging( 21 | s3_client=s3_client, 22 | bucket=bucket, 23 | tags={"k1": "v1", "k2": "v2"}, 24 | ) 25 | assert tags == {"k1": "v1", "k2": "v2"} 26 | res = s3_client.get_bucket_tagging( 27 | Bucket=bucket, 28 | ) 29 | assert res["TagSet"] == [ 30 | {"Key": "k1", "Value": "v1"}, 31 | {"Key": "k2", "Value": "v2"}, 32 | ] 33 | 34 | tags = update_bucket_tagging( 35 | s3_client=s3_client, 36 | bucket=bucket, 37 | tags={"k2": "v22", "k3": "v3"}, 38 | ) 39 | assert tags == {"k1": "v1", "k2": "v22", "k3": "v3"} 40 | res = s3_client.get_bucket_tagging( 41 | Bucket=bucket, 42 | ) 43 | assert res["TagSet"] == [ 44 | {"Key": "k1", "Value": "v1"}, 45 | {"Key": "k2", "Value": "v22"}, 46 | {"Key": "k3", "Value": "v3"}, 47 | ] 48 | 49 | def _test_object_tagging(self): 50 | s3_client = self.s3_client 51 | bucket = self.bucket 52 | key = f"{self.get_prefix()}/test_object_tagging" 53 | 54 | s3_client.put_object( 55 | Bucket=bucket, 56 | Key=key, 57 | Body="", 58 | ) 59 | tags = update_object_tagging( 60 | s3_client=s3_client, 61 | bucket=bucket, 62 | key=key, 63 | tags={"k1": "v1", "k2": "v2"}, 64 | )[1] 65 | assert tags == {"k1": "v1", "k2": "v2"} 66 | res = s3_client.get_object_tagging(Bucket=bucket, Key=key) 67 | assert res["TagSet"] == [ 68 | {"Key": "k1", "Value": "v1"}, 69 | {"Key": "k2", "Value": "v2"}, 70 | ] 71 | 72 | tags = update_object_tagging( 73 | s3_client=s3_client, 74 | bucket=bucket, 75 | key=key, 76 | tags={"k2": "v22", "k3": "v3"}, 77 | )[1] 78 | assert tags == {"k1": "v1", "k2": "v22", "k3": "v3"} 79 | res = s3_client.get_object_tagging( 80 | Bucket=bucket, 81 | Key=key, 82 | ) 83 | assert res["TagSet"] == [ 84 | {"Key": "k1", "Value": "v1"}, 85 | {"Key": "k2", "Value": "v22"}, 86 | {"Key": "k3", "Value": "v3"}, 87 | ] 88 | 89 | def test(self): 90 | self._test_bucket_tagging() 91 | self._test_object_tagging() 92 | 93 | 94 | # NOTE: this module should ONLY be tested with MOCK 95 | # DO NOT USE REAL S3 BUCKET 96 | class TestUseMock(BetterUpload): 97 | use_mock = True 98 | 99 | 100 | if __name__ == "__main__": 101 | run_cov_test(__file__, module="s3pathlib.better_client.tagging", preview=False) 102 | -------------------------------------------------------------------------------- /tests/better_client/test_client_upload.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from s3pathlib.better_client.upload import ( 6 | upload_dir, 7 | ) 8 | from s3pathlib.utils import smart_join_s3_key 9 | from s3pathlib.tests import run_cov_test 10 | from s3pathlib.tests.mock import BaseTest 11 | from s3pathlib.tests.paths import dir_test_upload_dir_folder 12 | 13 | dir_test_upload_dir_folder.joinpath("emptyfolder").mkdir(exist_ok=True) 14 | 15 | 16 | class BetterUpload(BaseTest): 17 | module = "better_client.upload" 18 | 19 | def _test(self): 20 | s3_client = self.s3_client 21 | bucket = self.bucket 22 | 23 | # regular upload 24 | upload_dir( 25 | s3_client=s3_client, 26 | bucket=bucket, 27 | prefix=smart_join_s3_key( 28 | parts=[self.prefix, "test_upload_dir"], 29 | is_dir=True, 30 | ), 31 | local_dir=f"{dir_test_upload_dir_folder}", 32 | pattern="**/*.txt", 33 | overwrite=True, 34 | ) 35 | 36 | # upload to bucket root directory also works 37 | upload_dir( 38 | s3_client=s3_client, 39 | bucket=bucket, 40 | prefix="", 41 | local_dir=f"{dir_test_upload_dir_folder}", 42 | pattern="**/*.txt", 43 | overwrite=True, 44 | ) 45 | 46 | # raise error when overwrite is False 47 | with pytest.raises(FileExistsError): 48 | upload_dir( 49 | s3_client=s3_client, 50 | bucket=bucket, 51 | prefix=smart_join_s3_key( 52 | parts=[self.prefix, "test_upload_dir"], 53 | is_dir=True, 54 | ), 55 | local_dir=f"{dir_test_upload_dir_folder}", 56 | pattern="**/*.txt", 57 | overwrite=False, 58 | ) 59 | 60 | # input argument error 61 | path = dir_test_upload_dir_folder.joinpath("1.txt") 62 | with pytest.raises(TypeError): 63 | upload_dir( 64 | s3_client=s3_client, 65 | bucket=bucket, 66 | prefix="", 67 | local_dir=f"{path}", 68 | pattern="**/*.txt", 69 | overwrite=False, 70 | ) 71 | 72 | dir_not_exist = dir_test_upload_dir_folder.parent.joinpath("not_exist") 73 | with pytest.raises(FileNotFoundError): 74 | upload_dir( 75 | s3_client=s3_client, 76 | bucket=bucket, 77 | prefix="", 78 | local_dir=f"{dir_not_exist}", 79 | pattern="**/*.txt", 80 | overwrite=False, 81 | ) 82 | 83 | def test(self): 84 | self._test() 85 | 86 | 87 | class Test(BetterUpload): 88 | use_mock = False 89 | 90 | 91 | class TestUseMock(BetterUpload): 92 | use_mock = True 93 | 94 | 95 | if __name__ == "__main__": 96 | run_cov_test(__file__, module="s3pathlib.better_client.upload", preview=False) 97 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from s3pathlib import S3Path 5 | 6 | 7 | @pytest.fixture 8 | def bucket() -> S3Path: 9 | return S3Path("bucket") 10 | 11 | 12 | @pytest.fixture 13 | def directory() -> S3Path: 14 | return S3Path("bucket", "folder/") 15 | 16 | 17 | @pytest.fixture 18 | def file() -> S3Path: 19 | return S3Path("bucket", "folder", "file.txt") 20 | 21 | 22 | @pytest.fixture 23 | def relpath() -> S3Path: 24 | return S3Path("bucket", "folder", "file.txt").relative_to(S3Path("bucket")) 25 | 26 | 27 | @pytest.fixture 28 | def void() -> S3Path: 29 | return S3Path() 30 | -------------------------------------------------------------------------------- /tests/core/.gitignore: -------------------------------------------------------------------------------- 1 | dir1/ 2 | dir2/ 3 | -------------------------------------------------------------------------------- /tests/core/all.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | if __name__ == "__main__": 4 | import pytest 5 | 6 | pytest.main(["-s", "--tb=native"]) 7 | -------------------------------------------------------------------------------- /tests/core/test_core_base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from s3pathlib.core import S3Path 5 | from s3pathlib.tests import run_cov_test 6 | from s3pathlib.tests.mock import BaseTest 7 | 8 | 9 | class BaseS3Path(BaseTest): 10 | """ 11 | Test strategy, try different construction method, inspect internal 12 | implementation variables. 13 | 14 | **中文文档** 15 | 16 | 测试策略, 根据主要的内部属性的值不同, 可以将 S3Path 分为 4 类. 该测试中对于每一类都创建了 17 | 许多逻辑上是相同的 S3Path, 也就是说这些 S3Path 的构建方式虽然不同, 但是内部属性完全一致, 18 | 可以理解为同一个. 在之后的 API 测试中, 每一类我们只需要取一个用例进行测试即可, 因为其他的 19 | 变种在内部实现上都是一致的. 20 | """ 21 | 22 | def _test_classic_aws_s3_object(self): 23 | # these s3path are equivalent 24 | p_list = [ 25 | S3Path("bucket", "a", "b", "c"), 26 | S3Path("bucket/a/b/c"), 27 | S3Path("bucket", "a/b/c"), 28 | S3Path("bucket", "/a/b/c"), 29 | S3Path(S3Path("//bucket//"), "/a/b/c"), 30 | ] 31 | for p in p_list: 32 | assert p._bucket == "bucket" 33 | assert p._parts == ["a", "b", "c"] 34 | assert p._is_dir is False 35 | assert p.parts == ["a", "b", "c"] 36 | 37 | def _test_logical_aws_s3_directory(self): 38 | # these s3path are equivalent 39 | p_list = [ 40 | S3Path("bucket", "a", "b", "c/"), 41 | S3Path("bucket", "/a", "b", "c/"), 42 | S3Path("//bucket", "a//b//c//"), 43 | S3Path("//bucket", "//a//b//c//"), 44 | S3Path("bucket//a//b//c//"), 45 | S3Path("//bucket//a//b//c//"), 46 | S3Path(S3Path("//bucket//"), "//a//b//c//"), 47 | ] 48 | for p in p_list: 49 | assert p._bucket == "bucket" 50 | assert p._parts == ["a", "b", "c"] 51 | assert p._is_dir is True 52 | assert p.parts == ["a", "b", "c"] 53 | 54 | def _test_aws_s3_bucket(self): 55 | # these s3path are equivalent 56 | p_list = [ 57 | S3Path("bucket"), 58 | S3Path("/bucket"), 59 | S3Path("//bucket//"), 60 | S3Path(S3Path("//bucket//")), 61 | ] 62 | for p in p_list: 63 | assert p._bucket == "bucket" 64 | assert p._parts == [] 65 | assert p._is_dir is True 66 | assert p.parts == [] 67 | 68 | def _test_void_aws_s3_path(self): 69 | # these s3path are equivalent 70 | p_list = [S3Path(), S3Path(S3Path(), S3Path(), S3Path())] 71 | for p in p_list: 72 | assert p._bucket is None 73 | assert p._parts == [] 74 | assert p._is_dir is None 75 | assert p.parts == [] 76 | 77 | def _test_uri_and_arn(self): 78 | for p in [ 79 | S3Path("s3://bucket/folder/file.txt"), 80 | S3Path("arn:aws:s3:::bucket/folder/file.txt"), 81 | ]: 82 | assert p._bucket == "bucket" 83 | assert p._parts == ["folder", "file.txt"] 84 | assert p._is_dir is False 85 | 86 | for p in [ 87 | S3Path("s3://bucket/folder/subfolder/"), 88 | S3Path("arn:aws:s3:::bucket/folder/subfolder/"), 89 | ]: 90 | assert p._bucket == "bucket" 91 | assert p._parts == ["folder", "subfolder"] 92 | assert p._is_dir is True 93 | 94 | def _test_type_error(self): 95 | with pytest.raises(TypeError): 96 | S3Path(1, "a", "b", "c") 97 | 98 | with pytest.raises(TypeError): 99 | S3Path("bucket", 1, 2, 3) 100 | 101 | with pytest.raises(TypeError): 102 | S3Path(S3Path("bucket"), S3Path("a", "b", "c")) 103 | 104 | def test(self): 105 | self._test_classic_aws_s3_object() 106 | self._test_logical_aws_s3_directory() 107 | self._test_aws_s3_bucket() 108 | self._test_void_aws_s3_path() 109 | self._test_uri_and_arn() 110 | self._test_type_error() 111 | 112 | 113 | class Test(BaseS3Path): 114 | use_mock = False 115 | 116 | 117 | class TestUseMock(BaseS3Path): 118 | use_mock = True 119 | 120 | 121 | if __name__ == "__main__": 122 | run_cov_test(__file__, module="s3pathlib.core.base", preview=False) 123 | -------------------------------------------------------------------------------- /tests/core/test_core_bucket.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from s3pathlib.tests import run_cov_test 4 | 5 | 6 | class TestBucketAPIMixin: 7 | def test(self): 8 | pass 9 | 10 | 11 | if __name__ == "__main__": 12 | run_cov_test(__file__, module="s3pathlib.core.bucket", preview=False) 13 | -------------------------------------------------------------------------------- /tests/core/test_core_comparison.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from s3pathlib.core import S3Path 4 | from s3pathlib.tests import run_cov_test 5 | 6 | 7 | class TestComparisonAPIMixin: 8 | def test_comparison_and_hash(self): 9 | """ 10 | Test comparison operator 11 | 12 | - ``==`` 13 | - ``!=`` 14 | - ``>`` 15 | - ``<`` 16 | - ``>=`` 17 | - ``<=`` 18 | - ``hash(S3Path())`` 19 | """ 20 | p1 = S3Path("bucket", "file.txt") 21 | p2 = S3Path("bucket", "folder/") 22 | p3 = S3Path("bucket") 23 | p4 = S3Path() 24 | p5 = S3Path("bucket", "file.txt").relative_to(S3Path("bucket")) 25 | p6 = S3Path("bucket", "folder/").relative_to(S3Path("bucket")) 26 | p7 = S3Path("bucket").relative_to(S3Path("bucket")) 27 | p8 = S3Path.make_relpath("file.txt") 28 | p9 = S3Path.make_relpath("folder/") 29 | 30 | p_list = [p1, p2, p3, p4, p5, p6, p7, p8, p9] 31 | for p in p_list: 32 | assert p == p 33 | 34 | assert p1 != p2 35 | assert p5 != p6 36 | assert p4 == p7 37 | assert p5 == p8 38 | assert p6 == p9 39 | 40 | assert p1 > p3 41 | assert p1 >= p3 42 | 43 | assert p3 < p1 44 | assert p3 <= p1 45 | 46 | assert p2 > p3 47 | assert p2 >= p3 48 | 49 | assert p3 < p2 50 | assert p3 <= p2 51 | 52 | p_set = set(p_list + p_list) 53 | assert len(p_set) == 6 54 | for p in p_list: 55 | assert p in p_set 56 | 57 | 58 | if __name__ == "__main__": 59 | run_cov_test(__file__, module="s3pathlib.core.comparison", preview=False) 60 | -------------------------------------------------------------------------------- /tests/core/test_core_copy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from pathlib_mate import Path 5 | from s3pathlib.core import S3Path 6 | from s3pathlib.tests import run_cov_test 7 | from s3pathlib.tests.mock import BaseTest 8 | 9 | dir_here = Path.dir_here(__file__) 10 | 11 | 12 | class CopyAPIMixin(BaseTest): 13 | module = "core.copy" 14 | 15 | def _test_copy_object(self): 16 | # before state 17 | p_src = S3Path(self.s3dir_root, "copy-object", "before.py") 18 | p_src.write_text("a") 19 | 20 | p_dst = S3Path(p_src, "copy-object", "after.py") 21 | p_dst.delete() 22 | 23 | assert p_dst.exists() is False 24 | 25 | # invoke api 26 | count = p_src.copy_to(p_dst, overwrite=False) 27 | 28 | # after state 29 | assert count == 1 30 | assert p_dst.exists() is True 31 | 32 | # raise exception 33 | with pytest.raises(FileExistsError): 34 | p_src.copy_to(p_dst, overwrite=False) 35 | 36 | def _test_copy_dir(self): 37 | # before state 38 | p_src = S3Path(self.s3dir_root, "copy-dir", "before").to_dir() 39 | p_src.delete() 40 | assert p_src.count_objects() == 0 41 | 42 | dir_to_upload = dir_here.joinpath("test_upload_dir").abspath 43 | p_src.upload_dir( 44 | local_dir=dir_to_upload, 45 | pattern="**/*.txt", 46 | overwrite=True, 47 | ) 48 | 49 | p_dst = S3Path(self.s3dir_root, "copy-dir", "after").to_dir() 50 | p_dst.delete() 51 | assert p_dst.count_objects() == 0 52 | 53 | # invoke api 54 | count = p_src.copy_to(dst=p_dst, overwrite=False) 55 | 56 | # validate after state 57 | assert count == 2 58 | 59 | # raise exception 60 | with pytest.raises(FileExistsError): 61 | p_src.copy_to(dst=p_dst, overwrite=False) 62 | 63 | def _test_move_to(self): 64 | # before state 65 | p_src = S3Path(self.s3dir_root, "move-to", "before").to_dir() 66 | p_src.delete() 67 | assert p_src.count_objects() == 0 68 | 69 | dir_to_upload = dir_here.joinpath("test_upload_dir").abspath 70 | p_src.upload_dir( 71 | local_dir=dir_to_upload, 72 | pattern="**/*.txt", 73 | overwrite=True, 74 | ) 75 | 76 | p_dst = S3Path(self.s3dir_root, "move-to", "after/") 77 | p_dst.delete() 78 | 79 | assert p_dst.count_objects() == 0 80 | 81 | # invoke api 82 | count = p_src.move_to(dst=p_dst, overwrite=False) 83 | 84 | # validate after state 85 | assert count == 2 86 | assert p_src.count_objects() == 0 87 | assert p_dst.count_objects() == 2 88 | 89 | def _test_copy_with_metadata_and_tagging(self): 90 | p_src = S3Path(self.s3dir_root, "copy_object", "src.txt") 91 | p_dst = S3Path(self.s3dir_root, "copy_object", "dst.txt") 92 | p_src.delete() 93 | p_dst.delete() 94 | 95 | p_src.write_text( 96 | "hello", 97 | metadata=dict(key_name="a"), 98 | tags=dict(tag_name="a"), 99 | ) 100 | 101 | # copy without metadata and tags argument 102 | p_src.copy_to(p_dst) 103 | 104 | # it will automatically copy metadata and tags from source 105 | p_dst.clear_cache() 106 | assert p_dst.metadata == {"key_name": "a"} 107 | assert p_dst.get_tags()[1] == {"tag_name": "a"} 108 | 109 | # copy with explicit metadata and tags 110 | p_src.copy_to( 111 | p_dst, 112 | metadata=dict(key_name="b"), 113 | tags=dict(tag_name="b"), 114 | overwrite=True, 115 | ) 116 | 117 | # it will overwrite metadata and tags with the explicit value 118 | p_dst.clear_cache() 119 | assert p_dst.metadata == {"key_name": "b"} 120 | assert p_dst.get_tags()[1] == {"tag_name": "b"} 121 | 122 | def test(self): 123 | self._test_copy_object() 124 | self._test_copy_dir() 125 | self._test_move_to() 126 | 127 | self._test_copy_with_metadata_and_tagging() 128 | 129 | 130 | class Test(CopyAPIMixin): 131 | use_mock = False 132 | 133 | 134 | class TestUseMock(CopyAPIMixin): 135 | use_mock = True 136 | 137 | 138 | if __name__ == "__main__": 139 | run_cov_test(__file__, module="s3pathlib.core.copy", preview=False) 140 | -------------------------------------------------------------------------------- /tests/core/test_core_exists.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from s3pathlib.core import S3Path 5 | from s3pathlib.tests import run_cov_test 6 | from s3pathlib.tests.mock import BaseTest 7 | 8 | 9 | class ExistsAPIMixin(BaseTest): 10 | module = "core.exists" 11 | 12 | def _test_exists(self): 13 | # s3 bucket 14 | assert S3Path(self.bucket).exists() is True 15 | 16 | # have access but not exists 17 | assert S3Path(f"{self.bucket}-not-exists").exists() is False 18 | 19 | # doesn't have access 20 | if self.use_mock is False: 21 | with pytest.raises(Exception): 22 | assert S3Path("asdf").exists() is False 23 | 24 | # s3 object 25 | s3path_file = self.s3dir_root.joinpath("file.txt") 26 | self.s3_client.put_object( 27 | Bucket=s3path_file.bucket, 28 | Key=s3path_file.key, 29 | Body=b"a", 30 | ) 31 | assert s3path_file.exists() is True 32 | 33 | with pytest.raises(Exception): 34 | s3path_file.ensure_not_exists() 35 | 36 | s3path_empty_object = self.s3dir_root.joinpath("empty.txt") 37 | self.s3_client.put_object( 38 | Bucket=s3path_empty_object.bucket, 39 | Key=s3path_empty_object.key, 40 | Body=b"", 41 | ) 42 | assert s3path_empty_object.exists() is True 43 | 44 | assert S3Path(self.bucket, "this-never-gonna-exists.exe").exists() is False 45 | 46 | # s3 directory 47 | assert s3path_file.parent.exists() is True 48 | 49 | assert S3Path(self.bucket, "this-never-gonna-exists/").exists() is False 50 | 51 | # void path 52 | p = S3Path() 53 | with pytest.raises(TypeError): 54 | p.exists() 55 | 56 | # relative path 57 | p = S3Path.make_relpath("folder", "file.txt") 58 | with pytest.raises(TypeError): 59 | p.exists() 60 | 61 | # soft folder 62 | s3path_soft_folder_file = self.s3dir_root.joinpath("soft_folder", "file.txt") 63 | self.s3_client.put_object( 64 | Bucket=s3path_soft_folder_file.bucket, 65 | Key=s3path_soft_folder_file.key, 66 | Body=b"a", 67 | ) 68 | assert s3path_soft_folder_file.parent.exists() is True 69 | assert s3path_soft_folder_file.exists() is True 70 | 71 | # hard folder 72 | dir_hard_folder = self.s3dir_root.joinpath("hard_folder/") 73 | self.s3_client.put_object( 74 | Bucket=dir_hard_folder.bucket, 75 | Key=dir_hard_folder.key, 76 | Body=b"", 77 | ) 78 | s3path_hard_folder_file = self.s3dir_root.joinpath("hard_folder", "file.txt") 79 | self.s3_client.put_object( 80 | Bucket=s3path_hard_folder_file.bucket, 81 | Key=s3path_hard_folder_file.key, 82 | Body=b"a", 83 | ) 84 | assert dir_hard_folder.exists() is True 85 | assert s3path_hard_folder_file.exists() is True 86 | 87 | # empty folder 88 | dir_empty_folder = self.s3dir_root.joinpath("empty_folder/") 89 | self.s3_client.put_object( 90 | Bucket=dir_empty_folder.bucket, 91 | Key=dir_empty_folder.key, 92 | Body=b"", 93 | ) 94 | assert dir_empty_folder.exists() is True 95 | 96 | def test(self): 97 | self._test_exists() 98 | 99 | 100 | class Test(ExistsAPIMixin): 101 | use_mock = False 102 | 103 | 104 | class TestUseMock(ExistsAPIMixin): 105 | use_mock = True 106 | 107 | 108 | if __name__ == "__main__": 109 | run_cov_test(__file__, module="s3pathlib.core.exists", preview=False) 110 | -------------------------------------------------------------------------------- /tests/core/test_core_filterable_property.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from s3pathlib.core.filterable_property import FilterableProperty 4 | from s3pathlib.tests import run_cov_test 5 | 6 | 7 | class User: 8 | def __init__(self, name: str): 9 | self.name = name 10 | 11 | @FilterableProperty 12 | def username(self) -> str: 13 | return self.name 14 | 15 | 16 | class TestFilterableProperty: 17 | def test(self): 18 | user = User(name="alice") 19 | func = User.username == "alice" 20 | assert func(user) is True 21 | assert func(User(name="Bob")) is False 22 | 23 | 24 | if __name__ == "__main__": 25 | run_cov_test(__file__, module="s3pathlib.core.filterable_property", preview=False) 26 | -------------------------------------------------------------------------------- /tests/core/test_core_is_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from s3pathlib.core import S3Path 6 | from s3pathlib.tests import run_cov_test 7 | 8 | 9 | class TestIsTestAPIMixin: 10 | def _test_type_test(self): 11 | """ 12 | Test if the instance is a ... 13 | 14 | - is_dir() 15 | - is_file() 16 | - is_bucket() 17 | - is_void() 18 | """ 19 | # s3 object 20 | p = S3Path("bucket", "file.txt") 21 | assert p.is_void() is False 22 | assert p.is_dir() is False 23 | assert p.is_file() is True 24 | assert p.is_bucket() is False 25 | 26 | p.ensure_file() 27 | with pytest.raises(Exception): 28 | p.ensure_not_file() 29 | with pytest.raises(Exception): 30 | p.ensure_dir() 31 | p.ensure_not_dir() 32 | 33 | # s3 directory 34 | p = S3Path("bucket", "folder/") 35 | assert p.is_void() is False 36 | assert p.is_dir() is True 37 | assert p.is_file() is False 38 | assert p.is_bucket() is False 39 | assert p.is_delete_marker() is False 40 | 41 | with pytest.raises(Exception): 42 | p.ensure_file() 43 | p.ensure_not_file() 44 | p.ensure_dir() 45 | with pytest.raises(Exception): 46 | p.ensure_not_dir() 47 | 48 | # s3 bucket 49 | p = S3Path("bucket") 50 | assert p.is_void() is False 51 | assert p.is_dir() is True 52 | assert p.is_file() is False 53 | assert p.is_bucket() is True 54 | assert p.is_delete_marker() is False 55 | 56 | # void path 57 | p = S3Path() 58 | assert p.is_void() is True 59 | assert p.is_dir() is False 60 | assert p.is_file() is False 61 | assert p.is_bucket() is False 62 | 63 | def test(self): 64 | self._test_type_test() 65 | 66 | 67 | if __name__ == "__main__": 68 | run_cov_test(__file__, module="s3pathlib.core.is_test", preview=False) 69 | -------------------------------------------------------------------------------- /tests/core/test_core_iter_object_versions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import time 4 | from pathlib_mate import Path 5 | 6 | from s3pathlib.core import S3Path 7 | from s3pathlib.tests import run_cov_test 8 | from s3pathlib.tests.mock import BaseTest 9 | 10 | 11 | dir_here = Path.dir_here(__file__) 12 | 13 | 14 | class IterObjectsAPIMixin(BaseTest): 15 | module = "core.iter_object_versions" 16 | 17 | def _test_list_object_versions(self): 18 | # prepare test data 19 | s3path = S3Path( 20 | self.s3dir_root_with_versioning, "list_object_versions", "file.txt" 21 | ) 22 | # clear existing data 23 | s3path.delete(is_hard_delete=True) 24 | 25 | v1 = s3path.write_text("v1").version_id 26 | time.sleep(1) 27 | s3path.delete() 28 | time.sleep(1) 29 | v2 = s3path.write_text("v2").version_id 30 | time.sleep(1) 31 | v3 = s3path.write_text("v3").version_id 32 | 33 | s3path_list = s3path.list_object_versions().all() 34 | versions = [s3path.version_id for s3path in s3path_list] 35 | assert versions[0] == v3 36 | assert versions[1] == v2 37 | assert versions[3] == v1 38 | 39 | assert [s3path.is_delete_marker() for s3path in s3path_list] == [ 40 | False, # v3 41 | False, # v2 42 | True, # delete marker 43 | False # v1 44 | ] 45 | assert s3path_list[2].etag is None 46 | assert s3path_list[2].size == 0 47 | 48 | 49 | def test(self): 50 | self._test_list_object_versions() 51 | 52 | 53 | class Test(IterObjectsAPIMixin): 54 | use_mock = False 55 | 56 | 57 | class TestUseMock(IterObjectsAPIMixin): 58 | use_mock = True 59 | 60 | 61 | if __name__ == "__main__": 62 | run_cov_test(__file__, module="s3pathlib.core.iter_object_versions", preview=False) 63 | -------------------------------------------------------------------------------- /tests/core/test_core_joinpath.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from s3pathlib.core import S3Path 5 | from s3pathlib.tests import run_cov_test 6 | 7 | 8 | class TestJoinPathAPIMixin: 9 | def test_joinpath(self): 10 | # ------ 11 | p = S3Path("bucket") 12 | assert p.joinpath("file.txt").uri == "s3://bucket/file.txt" 13 | assert p.joinpath("/file.txt").uri == "s3://bucket/file.txt" 14 | assert p.joinpath("folder/").uri == "s3://bucket/folder/" 15 | assert p.joinpath("/folder/").uri == "s3://bucket/folder/" 16 | assert p.joinpath("folder", "file.txt").uri == "s3://bucket/folder/file.txt" 17 | 18 | relpath_folder = S3Path("my-bucket", "data", "folder/").relative_to( 19 | S3Path("my-bucket", "data") 20 | ) 21 | assert p.joinpath(relpath_folder).uri == "s3://bucket/folder/" 22 | assert ( 23 | p.joinpath( 24 | "data", 25 | relpath_folder, 26 | "file.txt", 27 | ).uri 28 | == "s3://bucket/data/folder/file.txt" 29 | ) 30 | 31 | # ------ 32 | with pytest.raises(TypeError): 33 | S3Path("bucket1").joinpath(S3Path("bucket2")) 34 | 35 | # ------ 36 | p = S3Path("bucket", "file.txt") 37 | p1 = p.joinpath("/") 38 | assert p1.is_dir() 39 | assert p1.uri == "s3://bucket/file.txt/" 40 | 41 | p2 = p.joinpath("subfolder", "/") 42 | assert p2.is_dir() 43 | assert p2.uri == "s3://bucket/file.txt/subfolder/" 44 | 45 | # ------ 46 | p1 = S3Path("bucket", "folder", "subfolder", "file.txt") 47 | p2 = p1.parent # s3://bucket/folder/subfolder 48 | p3 = p2.parent # s3://bucket/folder 49 | with pytest.raises(TypeError): 50 | p3.joinpath(p1, p2) 51 | 52 | with pytest.raises(TypeError): 53 | p3.joinpath(1) 54 | 55 | def test_divide_operator(self): 56 | bucket = S3Path("bucket") 57 | directory = S3Path("bucket", "folder/") 58 | file = S3Path("bucket", "folder", "file.txt") 59 | relpath = S3Path("bucket", "folder", "file.txt").relative_to(S3Path("bucket")) 60 | void = S3Path() 61 | 62 | assert bucket / "folder/" == directory 63 | assert bucket / "folder" != directory 64 | assert ( 65 | bucket 66 | / [ 67 | "folder/", 68 | ] 69 | == directory 70 | ) 71 | assert bucket / ["folder", "/"] == directory 72 | assert ( 73 | bucket 74 | / [ 75 | "folder", 76 | ] 77 | != directory 78 | ) 79 | 80 | assert bucket / "folder" / "file.txt" == file 81 | assert bucket / "folder/" / "file.txt" == file 82 | assert bucket / "folder/" / "/file.txt" == file 83 | 84 | assert bucket / ["folder", "file.txt"] == file 85 | assert bucket / ["folder/", "file.txt"] == file 86 | assert bucket / ["folder/", "/file.txt"] == file 87 | 88 | assert bucket / relpath == file 89 | 90 | p = file / "/" 91 | assert p.is_dir() 92 | assert p.uri == "s3://bucket/folder/file.txt/" 93 | 94 | p = file / ["subfolder", "/"] 95 | assert p.is_dir() 96 | assert p.uri == "s3://bucket/folder/file.txt/subfolder/" 97 | 98 | root = S3Path("bucket") 99 | rel1 = S3Path("bucket/folder/").relative_to(S3Path("bucket")) 100 | rel2 = S3Path("bucket/folder/file.txt").relative_to(S3Path("bucket/folder/")) 101 | assert root / [rel1, rel2] == S3Path("bucket/folder/file.txt") 102 | assert root / (rel1 / rel2) == S3Path("bucket/folder/file.txt") 103 | 104 | with pytest.raises(TypeError): # relpath cannot / non-relpath 105 | rel1 / root 106 | 107 | with pytest.raises(TypeError): 108 | void / "bucket" 109 | 110 | 111 | if __name__ == "__main__": 112 | run_cov_test(__file__, module="s3pathlib.core.joinpath", preview=False) 113 | -------------------------------------------------------------------------------- /tests/core/test_core_metadata.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from datetime import datetime 4 | 5 | from s3pathlib.core import S3Path 6 | from s3pathlib.utils import md5_binary 7 | from s3pathlib.tests import run_cov_test 8 | from s3pathlib.tests.mock import BaseTest 9 | 10 | 11 | class MetadataAPIMixin(BaseTest): 12 | module = "core.metadata" 13 | 14 | p: S3Path 15 | 16 | @classmethod 17 | def custom_setup_class(cls): 18 | cls.p = S3Path(cls.get_s3dir_root(), "file.txt") 19 | cls.bsm.s3_client.put_object( 20 | Bucket=cls.p.bucket, 21 | Key=cls.p.key, 22 | Body="Hello World!", 23 | Metadata={"creator": "Alice"}, 24 | ) 25 | 26 | def _test_attributes(self): 27 | p = self.p 28 | 29 | assert p.etag == md5_binary("Hello World!".encode("utf-8")) 30 | _ = p.last_modified_at 31 | assert p.size == 12 32 | assert p.size_for_human == "12 B" 33 | assert p._static_version_id is None 34 | assert p.version_id == "null" 35 | assert p.expire_at is None 36 | assert p.metadata == {"creator": "Alice"} 37 | 38 | def _test_clear_cache(self): 39 | p = self.p 40 | 41 | p.clear_cache() 42 | assert p._meta is None 43 | assert len(p.etag) == 32 44 | assert isinstance(p._meta, dict) 45 | 46 | def _test_from_content_dict(self): 47 | s3path = S3Path._from_content_dict( 48 | bucket="bucket", 49 | dct={ 50 | "Key": "file.txt", 51 | "LastModified": datetime(2015, 1, 1), 52 | "ETag": "'string'", 53 | "ChecksumAlgorithm": "SHA256", 54 | "Size": 123, 55 | "StorageClass": "STANDARD", 56 | "Owner": {"DisplayName": "string", "ID": "string"}, 57 | }, 58 | ) 59 | assert s3path.key == "file.txt" 60 | assert s3path.last_modified_at == datetime(2015, 1, 1) 61 | assert s3path.etag == "string" 62 | assert s3path.size == 123 63 | 64 | def _test_object_metadata(self): 65 | s3path = S3Path(self.s3dir_root, "object-metadata.txt") 66 | self.s3_client.put_object( 67 | Bucket=s3path.bucket, 68 | Key=s3path.key, 69 | Body="Hello World!", 70 | ) 71 | assert s3path.metadata == {} # if no user metadata, then it will return {} 72 | 73 | self.s3_client.put_object( 74 | Bucket=s3path.bucket, 75 | Key=s3path.key, 76 | Body="Hello World!", 77 | Metadata={"key": "value"}, 78 | ) 79 | s3path._meta = {"ETag": "abcd"} 80 | assert s3path.metadata == {"key": "value"} 81 | 82 | def test(self): 83 | self._test_attributes() 84 | self._test_clear_cache() 85 | self._test_from_content_dict() 86 | self._test_object_metadata() 87 | 88 | 89 | class Test(MetadataAPIMixin): 90 | use_mock = False 91 | 92 | 93 | class TestUseMock(MetadataAPIMixin): 94 | use_mock = True 95 | 96 | 97 | if __name__ == "__main__": 98 | run_cov_test(__file__, module="s3pathlib.core.metadata", preview=False) 99 | -------------------------------------------------------------------------------- /tests/core/test_core_mutate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from s3pathlib.core import S3Path 5 | from s3pathlib.tests import run_cov_test 6 | 7 | 8 | class TestMutateAPIMixin: 9 | def test_copy(self): 10 | p1 = S3Path() 11 | p2 = p1.copy() 12 | assert p1 is not p2 13 | 14 | def test_change(self): 15 | p = S3Path("bkt", "a", "b", "c.jpg") 16 | 17 | p1 = p.change() 18 | assert p1 == p 19 | assert p1 is not p 20 | 21 | p1 = p.change(new_bucket="bkt1") 22 | assert p1.uri == "s3://bkt1/a/b/c.jpg" 23 | 24 | p1 = p.change(new_abspath="x/y/z.png") 25 | assert p1.uri == "s3://bkt/x/y/z.png" 26 | 27 | p1 = p.change(new_ext=".png") 28 | assert p1.uri == "s3://bkt/a/b/c.png" 29 | 30 | p1 = p.change(new_fname="d") 31 | assert p1.uri == "s3://bkt/a/b/d.jpg" 32 | 33 | p1 = p.change(new_basename="d.png") 34 | assert p1.uri == "s3://bkt/a/b/d.png" 35 | 36 | p1 = p.change(new_basename="d/") 37 | assert p1.uri == "s3://bkt/a/b/d/" 38 | assert p1.is_dir() 39 | 40 | p1 = p.change(new_dirname="d/") 41 | assert p1.uri == "s3://bkt/a/d/c.jpg" 42 | 43 | p1 = p.change(new_dirpath="x/y/") 44 | assert p1.uri == "s3://bkt/x/y/c.jpg" 45 | 46 | p1 = S3Path.make_relpath("a/b/c.jpg") 47 | p2 = p1.change(new_basename="d.png") 48 | assert p2._bucket is None 49 | assert p2._parts == ["a", "b", "d.png"] 50 | assert p2._is_dir is False 51 | 52 | p2 = p1.change(new_basename="d/") 53 | assert p2._bucket is None 54 | assert p2._parts == ["a", "b", "d"] 55 | assert p2._is_dir is True 56 | 57 | with pytest.raises(ValueError): 58 | p1 = S3Path() 59 | p1.change(new_dirpath="x", new_dirname="y") 60 | 61 | with pytest.raises(ValueError): 62 | p.change(new_abspath="x/y/z.png", new_basename="file.txt") 63 | 64 | with pytest.raises(ValueError): 65 | p.change(new_dirpath="x", new_dirname="y") 66 | 67 | with pytest.raises(ValueError): 68 | p.change(new_basename="x", new_fname="y", new_ext=".zip") 69 | 70 | def test_to_dir(self): 71 | p1 = S3Path("bkt", "a/") 72 | p2 = p1.to_dir() 73 | assert p2.is_dir() 74 | assert p2 == p1 75 | assert p2 is not p1 76 | 77 | p3 = S3Path("bkt", "a") 78 | p4 = p3.to_dir() 79 | assert p4.is_dir() 80 | assert p4 == p1 81 | 82 | with pytest.raises(ValueError): 83 | _ = S3Path().to_dir() 84 | 85 | def test_to_file(self): 86 | p1 = S3Path("bkt", "a") 87 | p2 = p1.to_file() 88 | assert p2.is_file() 89 | assert p2 == p1 90 | assert p2 is not p1 91 | 92 | p3 = S3Path("bkt", "a/") 93 | p4 = p3.to_file() 94 | assert p4.is_file() 95 | assert p4 == p1 96 | 97 | with pytest.raises(ValueError): 98 | _ = S3Path().to_file() 99 | 100 | 101 | if __name__ == "__main__": 102 | run_cov_test(__file__, module="s3pathlib.core.mutate", preview=False) 103 | -------------------------------------------------------------------------------- /tests/core/test_core_opener.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import json 4 | import pickle 5 | from s3pathlib.core import S3Path 6 | from s3pathlib.tests import run_cov_test 7 | from s3pathlib.tests.mock import BaseTest 8 | 9 | 10 | class OpenerAPIMixin(BaseTest): 11 | module = "core.open" 12 | 13 | def _test_open(self): 14 | s3path = S3Path(self.s3dir_root, "data.json") 15 | with s3path.open("w") as f: 16 | json.dump({"a": 1}, f) 17 | with s3path.open("r") as f: 18 | assert json.load(f) == {"a": 1} 19 | 20 | s3path = S3Path(self.s3dir_root, "data.pickle") 21 | with s3path.open("wb") as f: 22 | pickle.dump({"a": 1}, f) 23 | with s3path.open("rb") as f: 24 | assert pickle.load(f) == {"a": 1} 25 | 26 | def _test_open_with_additional_kwargs(self): 27 | s3path = S3Path(self.s3dir_root, "log.txt") 28 | 29 | # multi part upload 30 | s3path.delete() 31 | with s3path.open( 32 | "w", 33 | multipart_upload=True, 34 | metadata={"creator": "s3pathlib"}, 35 | tags={"project": "s3pathlib"}, 36 | ) as f: 37 | f.write("hello") 38 | 39 | assert s3path.metadata == {"creator": "s3pathlib"} 40 | assert s3path.get_tags()[1] == {"project": "s3pathlib"} 41 | 42 | # normal upload 43 | s3path.delete() 44 | with s3path.open( 45 | "w", 46 | multipart_upload=False, 47 | metadata={"creator": "s3pathlib"}, 48 | tags={"project": "s3pathlib"}, 49 | ) as f: 50 | f.write("hello") 51 | 52 | assert s3path.metadata == {"creator": "s3pathlib"} 53 | assert s3path.get_tags()[1] == {"project": "s3pathlib"} 54 | 55 | def _test_open_with_versioning(self): 56 | s3path = S3Path(self.s3dir_root_with_versioning, "log.txt") 57 | s3path_v1 = s3path.write_text("v1") 58 | s3path_v2 = s3path.write_text("v2") 59 | with s3path.open("r", version_id=s3path_v1.version_id) as f: 60 | assert f.read() == "v1" 61 | with s3path.open("r", version_id=s3path_v2.version_id) as f: 62 | assert f.read() == "v2" 63 | 64 | def test(self): 65 | self._test_open() 66 | self._test_open_with_additional_kwargs() 67 | self._test_open_with_versioning() 68 | 69 | 70 | class Test(OpenerAPIMixin): 71 | use_mock = False 72 | 73 | 74 | class TestUseMock(OpenerAPIMixin): 75 | use_mock = True 76 | 77 | 78 | if __name__ == "__main__": 79 | run_cov_test(__file__, module="s3pathlib.core.opener", preview=False) 80 | -------------------------------------------------------------------------------- /tests/core/test_core_resolve_s3_client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from s3pathlib.aws import context 4 | from s3pathlib.core.resolve_s3_client import resolve_s3_client 5 | from s3pathlib.tests import run_cov_test 6 | from s3pathlib.tests.mock import BaseTest 7 | 8 | 9 | class ResolveS3Client(BaseTest): 10 | def _test_resolve_s3_client(self): 11 | context.attach_boto_session(boto_ses=self.bsm.boto_ses) 12 | assert context.aws_account_id == self.bsm.aws_account_id 13 | assert context.aws_region == self.bsm.aws_region 14 | s3_client_1 = resolve_s3_client(context, None) 15 | s3_client_2 = resolve_s3_client(context, self.bsm) 16 | assert id(s3_client_1) != id(s3_client_2) 17 | 18 | def test(self): 19 | self._test_resolve_s3_client() 20 | 21 | 22 | class Test(ResolveS3Client): 23 | use_mock = False 24 | 25 | 26 | class TestUseMock(ResolveS3Client): 27 | use_mock = True 28 | 29 | 30 | if __name__ == "__main__": 31 | run_cov_test(__file__, module="s3pathlib.core.resolve_s3_client", preview=False) 32 | -------------------------------------------------------------------------------- /tests/core/test_core_serde.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from s3pathlib.core import S3Path 4 | from s3pathlib.tests import run_cov_test 5 | 6 | 7 | class TestSerdeAPIMixin: 8 | def test_serialization(self): 9 | p1 = S3Path("bucket", "folder", "file.txt") 10 | assert p1.to_dict() == { 11 | "bucket": "bucket", 12 | "parts": ["folder", "file.txt"], 13 | "is_dir": False, 14 | } 15 | p2 = S3Path.from_dict(p1.to_dict()) 16 | assert p1 == p2 17 | assert p1 is not p2 18 | 19 | 20 | if __name__ == "__main__": 21 | run_cov_test(__file__, module="s3pathlib.core.serde", preview=False) 22 | -------------------------------------------------------------------------------- /tests/core/test_core_sync.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys 4 | import pytest 5 | from pathlib_mate import Path 6 | from s3pathlib.core import S3Path 7 | from s3pathlib.tests import run_cov_test 8 | from s3pathlib.tests.mock import BaseTest 9 | 10 | 11 | dir_here = Path.dir_here(__file__) 12 | 13 | 14 | class SyncAPIMixin(BaseTest): 15 | module = "core.sync" 16 | 17 | @pytest.mark.skipif( 18 | sys.platform.startswith("win"), 19 | reason="windows CLI system is different", 20 | ) 21 | def _test_sync(self): 22 | s3path1 = self.s3dir_root.joinpath("dir1").to_dir() 23 | s3path2 = self.s3dir_root.joinpath("dir2").to_dir() 24 | path1 = dir_here.joinpath("dir1") 25 | path2 = dir_here.joinpath("dir2") 26 | path0 = dir_here.joinpath("test_upload_dir") 27 | 28 | s3path1.delete() 29 | s3path2.delete() 30 | path1.remove_if_exists() 31 | path2.remove_if_exists() 32 | 33 | s3path1.sync_from(path0, verbose=False) 34 | assert s3path1.count_objects() == 2 35 | 36 | s3path1.sync_to(s3path2, verbose=False) 37 | assert s3path2.count_objects() == 2 38 | 39 | s3path1.sync_to(s3path2.uri, verbose=False) 40 | assert s3path2.count_objects() == 2 41 | 42 | s3path1.sync_from(s3path2.uri, verbose=False) 43 | assert s3path1.count_objects() == 2 44 | 45 | s3path2.sync_to(path1, verbose=False) 46 | assert path1.file_stat()["file"] == 2 47 | 48 | with pytest.raises(ValueError): 49 | S3Path.sync(path1.abspath, path2.abspath, verbose=False) 50 | 51 | def test(self): 52 | if self.use_mock is False: 53 | self._test_sync() 54 | 55 | 56 | # NOTE: this module should ONLY be tested with MOCK 57 | # DO NOT USE REAL S3 BUCKET 58 | class Test(SyncAPIMixin): 59 | use_mock = False 60 | 61 | 62 | if __name__ == "__main__": 63 | run_cov_test(__file__, module="s3pathlib.core.sync", preview=False) 64 | -------------------------------------------------------------------------------- /tests/core/test_core_tagging.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from s3pathlib.core import S3Path 4 | from s3pathlib.tag import encode_url_query 5 | from s3pathlib.tests import run_cov_test 6 | from s3pathlib.tests.mock import BaseTest 7 | 8 | 9 | class TaggingAPIMixin(BaseTest): 10 | module = "core.tagging" 11 | 12 | def _test_get_put_update(self): 13 | s3path_file = S3Path(self.s3dir_root, "hello.txt") 14 | 15 | self.s3_client.put_object( 16 | Bucket=s3path_file.bucket, 17 | Key=s3path_file.key, 18 | Body=b"Hello World!", 19 | ) 20 | 21 | # tags is empty 22 | assert s3path_file.get_tags()[1] == {} 23 | 24 | # use put_object API 25 | self.s3_client.put_object( 26 | Bucket=s3path_file.bucket, 27 | Key=s3path_file.key, 28 | Body=b"Hello World!", 29 | Tagging=encode_url_query({"k1": "v1"}), 30 | ) 31 | assert s3path_file.get_tags()[1] == {"k1": "v1"} 32 | 33 | # put object tagging is a full replacement 34 | _, tags = s3path_file.put_tags(tags={"k2": "v2", "k3": "v3"}) 35 | assert tags == {"k2": "v2", "k3": "v3"} 36 | assert s3path_file.get_tags()[1] == {"k2": "v2", "k3": "v3"} 37 | 38 | # update object is a "true update" operation 39 | _, tags = s3path_file.update_tags(tags={"k1": "v1", "k2": "v22"}) 40 | assert tags == {"k1": "v1", "k2": "v22", "k3": "v3"} 41 | 42 | assert s3path_file.get_tags()[1] == {"k1": "v1", "k2": "v22", "k3": "v3"} 43 | 44 | def _test_get_put_update_with_versioning(self): 45 | s3path_file = S3Path(self.s3dir_root_with_versioning, "hello.txt") 46 | 47 | self.s3_client.put_object( 48 | Bucket=s3path_file.bucket, 49 | Key=s3path_file.key, 50 | Body=b"Hello World!", 51 | ) 52 | 53 | # tags is empty 54 | v1, tags = s3path_file.get_tags() 55 | assert tags == {} 56 | 57 | # use put_object API 58 | self.s3_client.put_object( 59 | Bucket=s3path_file.bucket, 60 | Key=s3path_file.key, 61 | Body=b"Hello World!", 62 | Tagging=encode_url_query({"k1": "v1"}), 63 | ) 64 | 65 | # version is the new one, tags is the new one 66 | v2, tags = s3path_file.get_tags() 67 | assert v1 != v2 68 | assert tags == {"k1": "v1"} 69 | 70 | # get specific version 71 | v, tags = s3path_file.get_tags(version_id=v1) 72 | assert v == v1 73 | assert tags == {} 74 | 75 | # put tags to the latest version 76 | # put object tagging is a full replacement 77 | _, tags = s3path_file.put_tags(tags={"k2": "v2", "k3": "v3"}) 78 | assert tags == {"k2": "v2", "k3": "v3"} 79 | 80 | # get latest version 81 | v, tags = s3path_file.get_tags() 82 | assert v == v2 83 | assert tags == {"k2": "v2", "k3": "v3"} 84 | 85 | # get older version 86 | v, tags = s3path_file.get_tags(version_id=v1) 87 | assert v == v1 88 | assert tags == {} 89 | 90 | # put tags to the older version 91 | # put object tagging is a full replacement 92 | _, tags = s3path_file.put_tags(tags={"k1": "v1", "k2": "v2"}, version_id=v1) 93 | assert tags == {"k1": "v1", "k2": "v2"} 94 | 95 | # update object is a "true update" operation 96 | _, tags = s3path_file.update_tags(tags={"k1": "v1", "k2": "v22"}) 97 | assert tags == {"k1": "v1", "k2": "v22", "k3": "v3"} 98 | v, tags = s3path_file.get_tags() 99 | assert v == v2 100 | assert tags == {"k1": "v1", "k2": "v22", "k3": "v3"} 101 | 102 | _, tags = s3path_file.update_tags(tags={"k2": "v2", "k3": "v3"}, version_id=v1) 103 | assert tags == {"k1": "v1", "k2": "v2", "k3": "v3"} 104 | v, tags = s3path_file.get_tags(version_id=v1) 105 | assert v == v1 106 | assert tags == {"k1": "v1", "k2": "v2", "k3": "v3"} 107 | 108 | def test(self): 109 | self._test_get_put_update() 110 | self._test_get_put_update_with_versioning() 111 | 112 | 113 | class Test(TaggingAPIMixin): 114 | use_mock = False 115 | 116 | 117 | class TestUseMock(TaggingAPIMixin): 118 | use_mock = True 119 | 120 | 121 | if __name__ == "__main__": 122 | run_cov_test(__file__, module="s3pathlib.core.tagging", preview=False) 123 | -------------------------------------------------------------------------------- /tests/core/test_core_upload.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from pathlib_mate import Path 5 | from s3pathlib.core import S3Path 6 | from s3pathlib.tests import run_cov_test 7 | from s3pathlib.tests.mock import BaseTest 8 | 9 | 10 | dir_here = Path.dir_here(__file__) 11 | 12 | 13 | class UploadAPIMixin(BaseTest): 14 | module = "core.upload" 15 | 16 | def _test_upload_file(self): 17 | # before state 18 | p = S3Path(self.s3dir_root, "upload-file", "test.py") 19 | p.delete() 20 | assert p.exists() is False 21 | 22 | # invoke api 23 | p.upload_file(path=__file__, overwrite=True) 24 | 25 | # after state 26 | assert p.exists() is True 27 | 28 | # raise exception if exists 29 | with pytest.raises(FileExistsError): 30 | p.upload_file(path=__file__, overwrite=False) 31 | 32 | # raise type error if upload to a folder 33 | with pytest.raises(TypeError): 34 | p = S3Path("bucket", "folder/") 35 | p.upload_file("/tmp/file.txt") 36 | 37 | def _test_upload_dir(self): 38 | # before state 39 | p = S3Path(self.s3dir_root, "upload-dir/") 40 | p.delete() 41 | assert p.count_objects() == 0 42 | 43 | # invoke api 44 | dir_to_upload = dir_here.joinpath("test_upload_dir").abspath 45 | p.upload_dir( 46 | local_dir=dir_to_upload, 47 | pattern="**/*.txt", 48 | overwrite=True, 49 | ) 50 | 51 | # after state 52 | assert p.count_objects() == 2 53 | 54 | # raise exception if exists 55 | with pytest.raises(FileExistsError): 56 | p.upload_dir( 57 | local_dir=dir_to_upload, 58 | pattern="**/*.txt", 59 | overwrite=False, 60 | ) 61 | 62 | # raise type error if upload to a folder 63 | with pytest.raises(TypeError): 64 | p = S3Path("bucket", "file.txt") 65 | p.upload_dir("/tmp/folder") 66 | 67 | def test(self): 68 | self._test_upload_file() 69 | self._test_upload_dir() 70 | 71 | 72 | class Test(UploadAPIMixin): 73 | use_mock = False 74 | 75 | 76 | class TestUseMock(UploadAPIMixin): 77 | use_mock = True 78 | 79 | 80 | if __name__ == "__main__": 81 | run_cov_test(__file__, module="s3pathlib.core.upload", preview=False) 82 | -------------------------------------------------------------------------------- /tests/core/test_core_uri.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from s3pathlib.core import S3Path 5 | from s3pathlib.tests import run_cov_test 6 | 7 | 8 | class TestUriAPIMixin: 9 | def test_uri_properties(self): 10 | # s3 object 11 | p = S3Path("bucket", "folder", "file.txt") 12 | assert p.bucket == "bucket" 13 | assert p.key == "folder/file.txt" 14 | assert p.uri == "s3://bucket/folder/file.txt" 15 | assert p.arn == "arn:aws:s3:::bucket/folder/file.txt" 16 | assert ( 17 | p.console_url 18 | == "https://console.aws.amazon.com/s3/object/bucket?prefix=folder/file.txt" 19 | ) 20 | assert ( 21 | p.us_gov_cloud_console_url 22 | == "https://console.amazonaws-us-gov.com/s3/object/bucket?prefix=folder/file.txt" 23 | ) 24 | assert ( 25 | p.s3_select_console_url 26 | == "https://console.aws.amazon.com/s3/buckets/bucket/object/select?prefix=folder/file.txt" 27 | ) 28 | assert ( 29 | p.s3_select_us_gov_cloud_console_url 30 | == "https://console.amazonaws-us-gov.com/s3/buckets/bucket/object/select?prefix=folder/file.txt" 31 | ) 32 | 33 | # s3 directory 34 | p = S3Path("bucket", "folder/") 35 | assert p.bucket == "bucket" 36 | assert p.key == "folder/" 37 | assert p.uri == "s3://bucket/folder/" 38 | assert p.arn == "arn:aws:s3:::bucket/folder/" 39 | assert ( 40 | p.console_url 41 | == "https://console.aws.amazon.com/s3/buckets/bucket?prefix=folder/" 42 | ) 43 | 44 | with pytest.raises(TypeError): 45 | assert p.s3_select_console_url 46 | with pytest.raises(TypeError): 47 | assert p.s3_select_us_gov_cloud_console_url 48 | 49 | # s3 bucket 50 | p = S3Path("bucket") 51 | assert p.bucket == "bucket" 52 | assert p.key == "" 53 | assert p.uri == "s3://bucket/" 54 | assert p.arn == "arn:aws:s3:::bucket" 55 | assert ( 56 | p.console_url 57 | == "https://console.aws.amazon.com/s3/buckets/bucket?tab=objects" 58 | ) 59 | 60 | with pytest.raises(TypeError): 61 | assert p.s3_select_console_url 62 | with pytest.raises(TypeError): 63 | assert p.s3_select_us_gov_cloud_console_url 64 | 65 | # void path 66 | p = S3Path() 67 | assert p.bucket is None 68 | assert p.key == "" 69 | assert p.uri is None 70 | assert p.arn is None 71 | assert p.console_url is None 72 | assert p.us_gov_cloud_console_url is None 73 | 74 | with pytest.raises(TypeError): 75 | assert p.s3_select_console_url 76 | with pytest.raises(TypeError): 77 | assert p.s3_select_us_gov_cloud_console_url 78 | 79 | def test_from_s3_uri(self): 80 | p = S3Path.from_s3_uri("s3://bucket/") 81 | assert p._bucket == "bucket" 82 | assert p._parts == [] 83 | assert p._is_dir is True 84 | 85 | p = S3Path.from_s3_uri("s3://bucket/folder/") 86 | assert p._bucket == "bucket" 87 | assert p._parts == [ 88 | "folder", 89 | ] 90 | assert p._is_dir is True 91 | 92 | p = S3Path.from_s3_uri("s3://bucket/folder/file.txt") 93 | assert p._bucket == "bucket" 94 | assert p._parts == ["folder", "file.txt"] 95 | assert p._is_dir is False 96 | 97 | def test_from_s3_arn(self): 98 | p = S3Path.from_s3_arn("arn:aws:s3:::bucket") 99 | assert p._bucket == "bucket" 100 | assert p._parts == [] 101 | assert p._is_dir is True 102 | 103 | p = S3Path.from_s3_arn("arn:aws:s3:::bucket/") 104 | assert p._bucket == "bucket" 105 | assert p._parts == [] 106 | assert p._is_dir is True 107 | 108 | p = S3Path.from_s3_arn("arn:aws:s3:::bucket/folder/") 109 | assert p._bucket == "bucket" 110 | assert p._parts == [ 111 | "folder", 112 | ] 113 | assert p._is_dir is True 114 | 115 | p = S3Path.from_s3_arn("arn:aws:s3:::bucket/folder/file.txt") 116 | assert p._bucket == "bucket" 117 | assert p._parts == ["folder", "file.txt"] 118 | assert p._is_dir is False 119 | 120 | 121 | if __name__ == "__main__": 122 | run_cov_test(__file__, module="s3pathlib.core.uri", preview=False) 123 | -------------------------------------------------------------------------------- /tests/core/test_iter_objects/README.txt: -------------------------------------------------------------------------------- 1 | test for s3pathlib.utils.iter_objects function 2 | test for s3pathlib.utils.iter_objects function 3 | test for s3pathlib.utils.iter_objects function 4 | test for s3pathlib.utils.iter_objects function 5 | test for s3pathlib.utils.iter_objects function 6 | test for s3pathlib.utils.iter_objects function 7 | test for s3pathlib.utils.iter_objects function 8 | test for s3pathlib.utils.iter_objects function 9 | test for s3pathlib.utils.iter_objects function 10 | test for s3pathlib.utils.iter_objects function -------------------------------------------------------------------------------- /tests/core/test_iter_objects/folder-description.txt: -------------------------------------------------------------------------------- 1 | folder description 2 | -------------------------------------------------------------------------------- /tests/core/test_iter_objects/folder1/1.txt: -------------------------------------------------------------------------------- 1 | test1 -------------------------------------------------------------------------------- /tests/core/test_iter_objects/folder1/2.txt: -------------------------------------------------------------------------------- 1 | test2 2 | test2 -------------------------------------------------------------------------------- /tests/core/test_iter_objects/folder1/3.txt: -------------------------------------------------------------------------------- 1 | test3 2 | test3 3 | test3 -------------------------------------------------------------------------------- /tests/core/test_iter_objects/folder2/4.txt: -------------------------------------------------------------------------------- 1 | test4 2 | test4 3 | test4 4 | test4 -------------------------------------------------------------------------------- /tests/core/test_iter_objects/folder2/5.txt: -------------------------------------------------------------------------------- 1 | test5 2 | test5 3 | test5 4 | test5 5 | test5 -------------------------------------------------------------------------------- /tests/core/test_iter_objects/folder2/6.txt: -------------------------------------------------------------------------------- 1 | test6 2 | test6 3 | test6 4 | test6 5 | test6 6 | test6 -------------------------------------------------------------------------------- /tests/core/test_iter_objects/folder3/7.txt: -------------------------------------------------------------------------------- 1 | test7 2 | test7 3 | test7 4 | test7 5 | test7 6 | test7 7 | test7 8 | -------------------------------------------------------------------------------- /tests/core/test_iter_objects/folder3/8.txt: -------------------------------------------------------------------------------- 1 | test8 2 | test8 3 | test8 4 | test8 5 | test8 6 | test8 7 | test8 8 | test8 9 | -------------------------------------------------------------------------------- /tests/core/test_iter_objects/folder3/9.txt: -------------------------------------------------------------------------------- 1 | test9 2 | test9 3 | test9 4 | test9 5 | test9 6 | test9 7 | test9 8 | test9 9 | test9 10 | -------------------------------------------------------------------------------- /tests/core/test_upload_dir/1.txt: -------------------------------------------------------------------------------- 1 | This is 1 -------------------------------------------------------------------------------- /tests/core/test_upload_dir/subfolder/2.txt: -------------------------------------------------------------------------------- 1 | This is 2 2 | -------------------------------------------------------------------------------- /tests/data/list_objects_folder/README.txt: -------------------------------------------------------------------------------- 1 | test for s3pathlib.utils.iter_objects function 2 | test for s3pathlib.utils.iter_objects function 3 | test for s3pathlib.utils.iter_objects function 4 | test for s3pathlib.utils.iter_objects function 5 | test for s3pathlib.utils.iter_objects function 6 | test for s3pathlib.utils.iter_objects function 7 | test for s3pathlib.utils.iter_objects function 8 | test for s3pathlib.utils.iter_objects function 9 | test for s3pathlib.utils.iter_objects function 10 | test for s3pathlib.utils.iter_objects function -------------------------------------------------------------------------------- /tests/data/list_objects_folder/folder-description.txt: -------------------------------------------------------------------------------- 1 | folder description 2 | -------------------------------------------------------------------------------- /tests/data/list_objects_folder/folder1/1.txt: -------------------------------------------------------------------------------- 1 | test1 -------------------------------------------------------------------------------- /tests/data/list_objects_folder/folder1/2.txt: -------------------------------------------------------------------------------- 1 | test2 2 | test2 -------------------------------------------------------------------------------- /tests/data/list_objects_folder/folder1/3.txt: -------------------------------------------------------------------------------- 1 | test3 2 | test3 3 | test3 -------------------------------------------------------------------------------- /tests/data/list_objects_folder/folder2/4.txt: -------------------------------------------------------------------------------- 1 | test4 2 | test4 3 | test4 4 | test4 -------------------------------------------------------------------------------- /tests/data/list_objects_folder/folder2/5.txt: -------------------------------------------------------------------------------- 1 | test5 2 | test5 3 | test5 4 | test5 5 | test5 -------------------------------------------------------------------------------- /tests/data/list_objects_folder/folder2/6.txt: -------------------------------------------------------------------------------- 1 | test6 2 | test6 3 | test6 4 | test6 5 | test6 6 | test6 -------------------------------------------------------------------------------- /tests/data/list_objects_folder/folder3/7.txt: -------------------------------------------------------------------------------- 1 | test7 2 | test7 3 | test7 4 | test7 5 | test7 6 | test7 7 | test7 8 | -------------------------------------------------------------------------------- /tests/data/list_objects_folder/folder3/8.txt: -------------------------------------------------------------------------------- 1 | test8 2 | test8 3 | test8 4 | test8 5 | test8 6 | test8 7 | test8 8 | test8 9 | -------------------------------------------------------------------------------- /tests/data/list_objects_folder/folder3/9.txt: -------------------------------------------------------------------------------- 1 | test9 2 | test9 3 | test9 4 | test9 5 | test9 6 | test9 7 | test9 8 | test9 9 | test9 10 | -------------------------------------------------------------------------------- /tests/data/upload_dir_folder/1.txt: -------------------------------------------------------------------------------- 1 | This is 1 -------------------------------------------------------------------------------- /tests/data/upload_dir_folder/data1.json: -------------------------------------------------------------------------------- 1 | {"name": "alice"} -------------------------------------------------------------------------------- /tests/data/upload_dir_folder/subfolder/2.txt: -------------------------------------------------------------------------------- 1 | This is 2 2 | -------------------------------------------------------------------------------- /tests/data/upload_dir_folder/subfolder/data2.json: -------------------------------------------------------------------------------- 1 | {"name": "bob"} -------------------------------------------------------------------------------- /tests/test_exc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from s3pathlib import exc 5 | 6 | 7 | def test_ensure_one_and_only_one_not_null(): 8 | value_error_cases = [ 9 | dict(), 10 | dict(a=None, b=None), 11 | dict(a=1, b=1), 12 | ] 13 | good_cases = [ 14 | dict(a=None, b=1), 15 | dict(a=1, b=None), 16 | dict(a=None, b=None, c=1), 17 | ] 18 | for kwargs in value_error_cases: 19 | with pytest.raises(ValueError): 20 | exc.ensure_one_and_only_one_not_none(**kwargs) 21 | for kwargs in good_cases: 22 | exc.ensure_one_and_only_one_not_none(**kwargs) 23 | 24 | 25 | def test_ensure_all_none(): 26 | value_error_cases = [ 27 | dict(), 28 | dict(a=None, b=1), 29 | dict(a=1, b=None), 30 | dict(a=1, b=1), 31 | ] 32 | good_cases = [ 33 | dict(a=None), 34 | dict(a=None, b=None), 35 | ] 36 | for kwargs in value_error_cases: 37 | with pytest.raises(ValueError): 38 | exc.ensure_all_none(**kwargs) 39 | for kwargs in good_cases: 40 | exc.ensure_all_none(**kwargs) 41 | 42 | 43 | if __name__ == "__main__": 44 | import os 45 | 46 | basename = os.path.basename(__file__) 47 | pytest.main([basename, "-s", "--tb=native"]) 48 | -------------------------------------------------------------------------------- /tests/test_marker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from s3pathlib.marker import ( 6 | deprecate_v1, 7 | deprecate_v2, 8 | ) 9 | from s3pathlib.tests import run_cov_test 10 | 11 | 12 | def test_deprecate_v1(): 13 | @deprecate_v1( 14 | version="1.1.1", 15 | message="THIS IS FOR UNITTEST: please use another method instead", 16 | ) 17 | def my_func(a: int, b: int) -> int: 18 | return a + b 19 | 20 | class MyClass: 21 | @deprecate_v1( 22 | version="1.1.1", 23 | message="THIS IS FOR UNITTEST: please use another method instead", 24 | ) 25 | def my_method(self, a: int, b: int): 26 | return a + b 27 | 28 | with pytest.warns(): 29 | assert MyClass().my_method(a=1, b=2) == 3 30 | assert my_func(1, 2) == 3 31 | 32 | 33 | def test_deprecate_v2(): 34 | @deprecate_v2( 35 | version="1.1.1", 36 | message="THIS IS FOR UNITTEST: please use another method instead", 37 | ) 38 | def my_func(a: int, b: int) -> int: 39 | return a + b 40 | 41 | class MyClass: 42 | @deprecate_v2( 43 | version="1.1.1", 44 | message="THIS IS FOR UNITTEST: please use another method instead", 45 | ) 46 | def my_method(self, a: int, b: int): 47 | return a + b 48 | 49 | with pytest.warns(): 50 | assert MyClass().my_method(a=1, b=2) == 3 51 | assert my_func(1, 2) == 3 52 | 53 | 54 | if __name__ == "__main__": 55 | run_cov_test(__file__, "s3pathlib.marker", preview=False) 56 | -------------------------------------------------------------------------------- /tests/test_metadata.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from s3pathlib.metadata import warn_upper_case_in_metadata_key 5 | 6 | 7 | def test_warn_upper_case_in_metadata_key(): 8 | with pytest.warns(UserWarning): 9 | warn_upper_case_in_metadata_key(metadata={"Hello": "World"}) 10 | 11 | 12 | if __name__ == "__main__": 13 | from s3pathlib.tests import run_cov_test 14 | 15 | run_cov_test(__file__, module="s3pathlib.metadata", preview=False) 16 | -------------------------------------------------------------------------------- /tests/test_tag.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from s3pathlib.tag import ( 4 | parse_tags, 5 | encode_tag_set, 6 | encode_url_query, 7 | encode_for_put_object, 8 | encode_for_put_bucket_tagging, 9 | encode_for_put_object_tagging, 10 | ) 11 | 12 | 13 | def test_parse_tags(): 14 | assert parse_tags([{"Key": "Name", "Value": "Alice"}]) == {"Name": "Alice"} 15 | 16 | 17 | def test_encode_tag_set(): 18 | assert encode_tag_set(dict(k1="v1", k2="v2")) == [ 19 | {"Key": "k1", "Value": "v1"}, 20 | {"Key": "k2", "Value": "v2"}, 21 | ] 22 | 23 | 24 | def test_encode_url_query(): 25 | assert encode_url_query(dict(k="v", message="a=b")) == "k=v&message=a%3Db" 26 | 27 | 28 | def test_encode_for_xyz(): 29 | tags = {"k": "v"} 30 | assert encode_for_put_object(tags) == "k=v" 31 | assert encode_for_put_bucket_tagging(tags) == [{"Key": "k", "Value": "v"}] 32 | assert encode_for_put_object_tagging(tags) == [{"Key": "k", "Value": "v"}] 33 | 34 | 35 | if __name__ == "__main__": 36 | from s3pathlib.tests import run_cov_test 37 | 38 | run_cov_test(__file__, module="s3pathlib.tag", preview=False) 39 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | import os 5 | from s3pathlib import utils 6 | from s3pathlib.tests import run_cov_test 7 | 8 | dir_here = os.path.dirname(os.path.abspath(__file__)) 9 | dir_project_root = os.path.dirname(dir_here) 10 | 11 | 12 | def test_split_s3_uri(): 13 | s3_uri = "s3://my-bucket/my-prefix/my-file.zip" 14 | bucket, key = utils.split_s3_uri(s3_uri) 15 | assert bucket == "my-bucket" 16 | assert key == "my-prefix/my-file.zip" 17 | 18 | 19 | def test_join_s3_uri(): 20 | bucket = "my-bucket" 21 | key = "my-prefix/my-file.zip" 22 | s3_uri = utils.join_s3_uri(bucket, key) 23 | assert s3_uri == "s3://my-bucket/my-prefix/my-file.zip" 24 | 25 | 26 | def test_split_parts(): 27 | assert utils.split_parts("a/b/c") == ["a", "b", "c"] 28 | assert utils.split_parts("//a//b//c//") == ["a", "b", "c"] 29 | assert utils.split_parts("") == [] 30 | assert utils.split_parts("////") == [] 31 | 32 | 33 | def test_s3_key_smart_join(): 34 | assert utils.smart_join_s3_key( 35 | parts=["/a/", "b/", "/c"], 36 | is_dir=True, 37 | ) == "a/b/c/" 38 | 39 | assert utils.smart_join_s3_key( 40 | parts=["/a/", "b/", "/c"], 41 | is_dir=False, 42 | ) == "a/b/c" 43 | 44 | assert utils.smart_join_s3_key( 45 | parts=["//a//b//c//"], 46 | is_dir=True, 47 | ) == "a/b/c/" 48 | 49 | assert utils.smart_join_s3_key( 50 | parts=["//a//b//c//"], 51 | is_dir=False, 52 | ) == "a/b/c" 53 | 54 | 55 | def test_make_s3_console_url(): 56 | # object 57 | url = utils.make_s3_console_url("my-bucket", "my-file.zip") 58 | assert "object" in url 59 | 60 | # folder 61 | url = utils.make_s3_console_url("my-bucket", "my-folder/") 62 | assert "bucket" in url 63 | 64 | # uri 65 | url = utils.make_s3_console_url(s3_uri="s3://my-bucket/my-folder/data.json") 66 | assert url == "https://console.aws.amazon.com/s3/object/my-bucket?prefix=my-folder/data.json" 67 | 68 | # s3 bucket root 69 | url = utils.make_s3_console_url(s3_uri="s3://my-bucket/") 70 | assert url == "https://console.aws.amazon.com/s3/buckets/my-bucket?tab=objects" 71 | 72 | # version id 73 | url = utils.make_s3_console_url(s3_uri="s3://my-bucket/my-folder/my-file.zip", version_id="v123") 74 | assert url == "https://console.aws.amazon.com/s3/object/my-bucket?prefix=my-folder/my-file.zip&versionId=v123" 75 | 76 | # us gov cloud 77 | url = utils.make_s3_console_url( 78 | s3_uri="s3://my-bucket/my-folder/data.json", is_us_gov_cloud=True 79 | ) 80 | assert url == "https://console.amazonaws-us-gov.com/s3/object/my-bucket?prefix=my-folder/data.json" 81 | 82 | with pytest.raises(ValueError): 83 | utils.make_s3_console_url(bucket="") 84 | 85 | with pytest.raises(ValueError): 86 | utils.make_s3_console_url(prefix="", s3_uri="") 87 | 88 | 89 | def test_ensure_s3_object(): 90 | utils.ensure_s3_object("path/to/key") 91 | with pytest.raises(Exception): 92 | utils.ensure_s3_object("path/to/dir/") 93 | 94 | 95 | def test_ensure_s3_dir(): 96 | utils.ensure_s3_dir("path/to/dir/") 97 | with pytest.raises(Exception): 98 | utils.ensure_s3_dir("path/to/key") 99 | 100 | 101 | def test_repr_data_size(): 102 | assert utils.repr_data_size(3600000) == "3.43 MB" 103 | 104 | 105 | def test_parse_data_size(): 106 | assert utils.parse_data_size("3.43 MB") == 3596615 107 | assert utils.parse_data_size("2_512.4 MB") == 2634442342 108 | assert utils.parse_data_size("2,512.4 MB") == 2634442342 109 | 110 | 111 | if __name__ == "__main__": 112 | run_cov_test(__file__, "s3pathlib.utils", preview=False) 113 | -------------------------------------------------------------------------------- /tests/test_validate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | from s3pathlib import validate as val 5 | 6 | 7 | def test_validate_s3_bucket(): 8 | test_cases = [ 9 | # bad case 10 | ("", False), 11 | ("a", False), 12 | ("a" * 100, False), 13 | ("bucket@example.com", False), 14 | ("my_bucket", False), 15 | ("-my-bucket", False), 16 | ("my-bucket-", False), 17 | ("my-bucket-", False), 18 | ("192.168.0.1", False), 19 | ("xn--my-bucket", False), 20 | ("my-bucket-s3alias", False), 21 | # good case 22 | ("my-bucket", True), 23 | ] 24 | for bucket, flag in test_cases: 25 | if flag: 26 | val.validate_s3_bucket(bucket) 27 | else: 28 | with pytest.raises(Exception): 29 | val.validate_s3_bucket(bucket) 30 | 31 | 32 | def test_validate_s3_key(): 33 | test_cases = [ 34 | # bad cases 35 | ("a" * 2000, False), 36 | ("%20", False), 37 | # good cases 38 | ( 39 | "", 40 | True, 41 | ), 42 | ( 43 | "abcd", 44 | True, 45 | ), 46 | ] 47 | for key, flag in test_cases: 48 | if flag: 49 | val.validate_s3_key(key) 50 | else: 51 | with pytest.raises(Exception): 52 | val.validate_s3_key(key) 53 | 54 | 55 | def test_validate_s3_uri(): 56 | test_cases = [ 57 | # bad cases 58 | ("bucket/key", False), 59 | ("s3://bucket", False), 60 | ("s3://ab/%20", False), 61 | # good cases 62 | ("s3://bucket/key", True), 63 | ("s3://bucket/folder/file.txt", True), 64 | ("s3://bucket/", True), 65 | ("s3://bucket/folder/", True), 66 | ] 67 | for uri, flag in test_cases: 68 | if flag: 69 | val.validate_s3_uri(uri) 70 | else: 71 | with pytest.raises(Exception): 72 | val.validate_s3_uri(uri) 73 | 74 | 75 | def test_validate_s3_arn(): 76 | test_cases = [ 77 | # bad cases 78 | ("bucket/key", False), 79 | ("s3://bucket", False), 80 | ("arn:aws:s3:::b", False), 81 | ("arn:aws:s3:::{}".format("b" * 100), False), 82 | ("arn:aws:s3:::bucket/%20", False), 83 | # good cases 84 | ("arn:aws:s3:::bucket/key", True), 85 | ("arn:aws:s3:::bucket/folder/file.txt", True), 86 | ("arn:aws:s3:::bucket/", True), 87 | ("arn:aws:s3:::bucket/folder", True), 88 | ("arn:aws:s3:::bucket", True), 89 | ] 90 | for arn, flag in test_cases: 91 | if flag: 92 | val.validate_s3_arn(arn) 93 | else: 94 | with pytest.raises(Exception): 95 | val.validate_s3_arn(arn) 96 | 97 | 98 | if __name__ == "__main__": 99 | from s3pathlib.tests import run_cov_test 100 | 101 | run_cov_test(__file__, module="s3pathlib.validate", preview=False) 102 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox is a generic virtualenv management and test command line tool you can use for: 2 | # checking your package installs correctly with different Python versions and interpreters 3 | # running your tests in each of the environments, configuring your test tool of choice 4 | # acting as a frontend to Continuous Integration servers, greatly reducing boilerplate and merging CI and shell-based testing. 5 | # 6 | # content of: tox.ini , put in same dir as setup.py 7 | # for more info: http://tox.readthedocs.io/en/latest/config.html 8 | [tox] 9 | envlist = py38 10 | 11 | [testenv] 12 | deps = 13 | -rrequirements.txt 14 | -rrequirements-test.txt 15 | 16 | commands = 17 | pip install --editable . 18 | pytest tests 19 | --------------------------------------------------------------------------------