├── .coveragerc ├── .gitignore ├── .isort.cfg ├── .travis.yml ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── setup.cfg ├── setup.py ├── swift ├── __init__.py ├── storage.py └── utils.py ├── tests ├── __init__.py ├── manage.py ├── settings.py ├── tests.py └── utils.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = 3 | swift/ 4 | omit = 5 | tests/ -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | /build/ 3 | /dist/ 4 | /*.egg-info 5 | /*.egg 6 | /.tox/ 7 | /env/ 8 | htmlcov 9 | .coverage 10 | 11 | .idea/ 12 | *.iml 13 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | skip=tests 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | sudo: false 4 | 5 | matrix: 6 | include: 7 | - env: TOXENV=flake8 8 | python: 3.8 9 | - env: TOXENV=isort 10 | python: 3.8 11 | - env: TOXENV=readme 12 | python: 3.8 13 | - env: TOXENV=py38-3.2 14 | python: 3.8 15 | - env: TOXENV=py38-4.1 16 | python: 3.8 17 | - env: TOXENV=py38-4.2 18 | python: 3.8 19 | 20 | install: pip install tox codecov 21 | 22 | script: tox 23 | 24 | after_success: codecov 25 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://img.shields.io/pypi/v/django-storage-swift.svg 2 | :target: https://pypi.python.org/pypi/django-storage-swift 3 | 4 | .. image:: https://travis-ci.org/dennisv/django-storage-swift.svg?branch=master 5 | :target: https://travis-ci.org/dennisv/django-storage-swift 6 | 7 | .. image:: https://codecov.io/gh/dennisv/django-storage-swift/branch/master/graph/badge.svg 8 | :target: https://codecov.io/gh/dennisv/django-storage-swift 9 | 10 | django-storage-swift: a storage layer for OpenStack Swift 11 | ========================================================= 12 | 13 | django-storage-swift allows Django applications to use OpenStack Swift 14 | as a file storage layer. 15 | 16 | Features 17 | -------- 18 | 19 | - Reads/writes files into/out of Swift. 20 | - Automatically derives the correct URL to allow files to be accessed 21 | through a web browser based on information returned from the 22 | authorisation server. 23 | 24 | - Allows you to override the host, port and path as necessary. 25 | - Supports the generation of temporary URLs to restrict access to 26 | files. 27 | 28 | Usage 29 | ----- 30 | 31 | You can install django-storage-swift through pip. To store your media 32 | files on swift, add the following line to your settings.py or 33 | local\_settings.py: 34 | 35 | .. code:: python 36 | 37 | DEFAULT_FILE_STORAGE = 'swift.storage.SwiftStorage' 38 | 39 | To store your static files on swift, add the following line: 40 | 41 | .. code:: python 42 | 43 | STATICFILES_STORAGE = 'swift.storage.StaticSwiftStorage' 44 | 45 | This will use another container. 46 | 47 | Configuring 48 | ----------- 49 | 50 | django-storage-swift recognises the following options. 51 | 52 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 53 | | Option | Default | Description | 54 | +==============================================+================+====================================================================================================================================================+ 55 | | ``SWIFT_AUTH_URL`` | *Required* | The URL for the auth server, e.g. ``http://127.0.0.1:5000/v2.0`` | 56 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 57 | | ``SWIFT_USERNAME`` | *Required* | The username to use to authenticate. | 58 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 59 | | ``SWIFT_KEY``/``SWIFT_PASSWORD`` | *Required* | The key (password) to use to authenticate. | 60 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 61 | | ``SWIFT_AUTH_VERSION`` | None | The version of the authentication protocol to use. If no auth version is defined, a version will be guessed based on auth parameters. | 62 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 63 | | ``SWIFT_TENANT_NAME``/``SWIFT_PROJECT_NAME`` | None | (v2 and v3 auth) The tenant/project name to use when authenticating. | 64 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 65 | | ``SWIFT_TENANT_ID``/``SWIFT_PROJECT_ID`` | None | (v2 and v3 auth) The tenant/project id to use when authenticating. | 66 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 67 | | ``SWIFT_USER_DOMAIN_NAME`` | None | (v3 auth only) The domain name we authenticate to | 68 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 69 | | ``SWIFT_USER_DOMAIN_ID`` | None | (v3 auth only) The domain id we authenticate to | 70 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 71 | | ``SWIFT_PROJECT_DOMAIN_NAME`` | None | (v3 auth only) The domain name our project is located in | 72 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 73 | | ``SWIFT_PROJECT_DOMAIN_ID`` | None | (v3 auth only) The domain id our project is located in | 74 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 75 | | ``SWIFT_REGION_NAME`` | None | OpenStack region if needed. Check with your provider. | 76 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 77 | | ``SWIFT_CONTAINER_NAME`` | None | The container in which to store the files. (``DEFAULT_FILE_STORAGE``) | 78 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 79 | | ``SWIFT_STATIC_CONTAINER_NAME`` | None | Alternate container for storing staticfiles. (``STATICFILES_STORAGE``) | 80 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 81 | | ``SWIFT_AUTO_CREATE_CONTAINER`` | False | Should the container be created if it does not exist? | 82 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 83 | | ``SWIFT_AUTO_CREATE_CONTAINER_PUBLIC`` | False | Set the auto created container as public on creation | 84 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 85 | | ``SWIFT_AUTO_CREATE_CONTAINER_ALLOW_ORIGIN`` | None | Set the container's X-Container-Meta-Access-Control-Allow-Origin value, to support CORS requests. | 86 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 87 | | ``SWIFT_AUTO_BASE_URL`` | True | Query the authentication server for the base URL. | 88 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 89 | | ``SWIFT_BASE_URL`` | None | The base URL from which the files can be retrieved, e.g. ``http://127.0.0.1:8080/``. | 90 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 91 | | ``SWIFT_NAME_PREFIX`` | None | Prefix that gets added to all filenames. | 92 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 93 | | ``SWIFT_USE_TEMP_URLS`` | False | Generate temporary URLs for file access (allows files to be accessed without a permissive ACL). | 94 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 95 | | ``SWIFT_TEMP_URL_KEY`` | None | Temporary URL key --- see `the OpenStack documentation `__. | 96 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 97 | | ``SWIFT_TEMP_URL_DURATION`` | ``30*60`` | How long a temporary URL remains valid, in seconds. | 98 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 99 | | ``SWIFT_EXTRA_OPTIONS`` | ``{}`` | Extra options, eg. { "endpoint\_type": "adminURL" }, which will return adminURL instead publicURL. | 100 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 101 | | ``SWIFT_STATIC_AUTO_BASE_URL`` | True | Query the authentication server for the static base URL. | 102 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 103 | | ``SWIFT_STATIC_BASE_URL`` | None | The base URL from which the static files can be retrieved, e.g. ``http://127.0.0.1:8080/``. | 104 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 105 | | ``SWIFT_STATIC_NAME_PREFIX`` | None | Prefix that gets added to all static filenames. | 106 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 107 | | ``SWIFT_CONTENT_TYPE_FROM_FD`` | False | Determine the files mimetypes from the actual content rather than from their filename (default). | 108 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 109 | | ``SWIFT_FULL_LISTING`` | True | Ensures to get whole directory contents (by default swiftclient limits it to 10000 entries) | 110 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 111 | | ``SWIFT_AUTH_TOKEN_DURATION`` | ``60*60*23`` | How long a token is expected to be valid in seconds. | 112 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 113 | | ``SWIFT_LAZY_CONNECT`` | ``False`` | If ``True`` swift connection will be obtained on first use, if ``False`` it will be obtained during storage instantiation. This can decrease | 114 | | | | startup time if you use many fields that use non-default swift storage. | 115 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 116 | | ``SWIFT_GZIP_CONTENT_TYPES`` | ``[]`` | List of content type that will be compressed eg. ['text/plain', 'application/json'] | 117 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 118 | | ``SWIFT_GZIP_COMPRESSION_LEVEL`` | ``4`` | Gzip compression level from 0 to 9. 0 = no compression, 9 = max compression | 119 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 120 | | ``SWIFT_GZIP_UNKNOWN_CONTENT_TYPE`` | ``False`` | If set to True and the content-type can't be guessed, gzip anyway | 121 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 122 | | ``SWIFT_CACHE_HEADERS`` | False | Headers cache on/off switcher | 123 | +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ 124 | 125 | 126 | SWIFT\_BASE\_URL 127 | ~~~~~~~~~~~~~~~~ 128 | 129 | django-swift-storage will automatically query the authentication server 130 | for the URL where your files can be accessed, which takes the form 131 | ``http://server:port/v1/AUTH_token/``. 132 | 133 | Sometimes you want to override the server and port (for example if 134 | you're developing using `devstack `__ inside 135 | Vagrant). This can be accomplished with ``SWIFT_BASE_URL``. 136 | 137 | The provided value is parsed, and: 138 | 139 | - host and port override any automatically derived values 140 | - any path component is put before derived path components. 141 | 142 | So if your auth server returns 143 | ``http://10.0.2.2:8080/v1/AUTH_012345abcd/`` and you have 144 | ``SWIFT_BASE_URL="http://127.0.0.1:8888/foo"``, the ``url`` function 145 | will a path based on ``http://127.0.0.1:8888/foo/v1/AUTH_012345abcd/``. 146 | 147 | Temporary URLs 148 | ~~~~~~~~~~~~~~ 149 | 150 | Temporary URLs provide a means to grant a user permission to access a 151 | file for a limited time only and without making the entire container 152 | public. 153 | 154 | Temporary URLs work as described in the Swift documentation. (The code 155 | to generate the signatures is heavily based on their implementation.) 156 | They require setup of a key for signing: the process is described in 157 | `the OpenStack 158 | documentation `__. 159 | 160 | Use 161 | --- 162 | 163 | Once installed and configured, use of django-storage-swift should be 164 | automatic and seamless. 165 | 166 | You can verify that swift is indeed being used by running, inside 167 | ``python manage.py shell``: 168 | 169 | .. code:: python 170 | 171 | from django.core.files.storage import default_storage 172 | default_storage.http_conn 173 | 174 | The result should be ``<>`` 175 | 176 | Openstack Keystone/Identity v3 177 | ------------------------------ 178 | 179 | To authenticate with a swift installation using Keystone AUTH and the Identity v3 API, you must also specify either the domain ID or name that your user and project (tenant) belongs to. 180 | 181 | .. code:: python 182 | 183 | SWIFT_AUTH_URL='https://keystoneserver/v3' 184 | SWIFT_AUTH_VERSION='3' 185 | SWIFT_USERNAME='<>' 186 | SWIFT_KEY='<>' 187 | SWIFT_TENANT_NAME='<>' 188 | SWIFT_USER_DOMAIN_NAME='<>' 189 | SWIFT_PROJECT_DOMAIN_NAME='<>' 190 | 191 | Troubleshooting 192 | --------------- 193 | 194 | - **I'm getting permission errors accessing my files**: If you are not 195 | using temporary URLs, you may need to make the container publically 196 | readable. See `this helpful 197 | discussion `__. 198 | If you are using temporary URLs, verify that your key is set 199 | correctly. 200 | - **I'm getting empty or truncated file uploads**: Issues with some content 201 | types may cause an incorrect `content_length` header to be sent with file 202 | uploads, resulting in 0 byte or truncated files. To avoid this, set 203 | `SWIFT_CONTENT_LENGTH_FROM_FD: True`. 204 | 205 | 206 | Quickstart 207 | ---------- 208 | 209 | .. code:: python 210 | 211 | # This was executed on a VM running a SAIO, for example with 212 | # https://github.com/swiftstack/vagrant-swift-all-in-one 213 | 214 | # Create two world-readable containers 215 | swift post -r ".r:*" django 216 | swift post -r ".r:*" django-static 217 | 218 | # A virtualenv to keep installation separated 219 | virtualenv sampleenv 220 | source sampleenv/bin/activate 221 | pip install django-storage-swift 222 | pip install django 223 | 224 | # Create a sample project 225 | django-admin startproject sampleproj 226 | export DJANGO_SETTINGS_MODULE=sampleproj.settings 227 | cd sampleproj/ 228 | 229 | # A few required settings, using SAIO defaults 230 | cat <> sampleproj/settings.py 231 | DEFAULT_FILE_STORAGE='swift.storage.SwiftStorage' 232 | STATICFILES_STORAGE ='swift.storage.StaticSwiftStorage' 233 | SWIFT_AUTH_URL='http://127.0.0.1:8080/auth/v1.0' 234 | SWIFT_USERNAME='test:tester' 235 | SWIFT_KEY='testing' 236 | SWIFT_CONTAINER_NAME='django' 237 | SWIFT_STATIC_CONTAINER_NAME='django-static' 238 | EOF 239 | 240 | # Create the initial DB data 241 | python manage.py migrate 242 | 243 | # This uploads static files to Swift 244 | python manage.py collectstatic --noinput 245 | 246 | # Now open http://127.0.0.1:8000/admin/ in your browser 247 | # Static files like CSS are served by Swift 248 | python manage.py runserver 249 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='django-storage-swift', 5 | version='1.3.0', 6 | description='OpenStack Swift storage backend for Django', 7 | long_description=open('README.rst').read(), 8 | url='https://github.com/dennisv/django-storage-swift', 9 | author='Dennis Vermeulen', 10 | author_email='blacktorn@gmail.com', 11 | license='MIT', 12 | packages=['swift'], 13 | install_requires=[ 14 | 'python-swiftclient>=2.2.0', 15 | 'python-keystoneclient>=0.2.3', 16 | 'six', 17 | 'python-magic>=0.4.10', 18 | ], 19 | zip_safe=False, 20 | classifiers=[ 21 | 'Environment :: Web Environment', 22 | 'Framework :: Django', 23 | 'Intended Audience :: Developers', 24 | 'License :: OSI Approved :: MIT License', 25 | 'Programming Language :: Python', 26 | 'Programming Language :: Python :: 3', 27 | 'Programming Language :: Python :: 3.8', 28 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 29 | 'Topic :: Software Development', 30 | 'Topic :: Software Development :: Libraries :: Application Frameworks', 31 | ], 32 | ) 33 | -------------------------------------------------------------------------------- /swift/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dennisv/django-storage-swift/c694e72191cc3a19dd9b67776de9086d716d7d44/swift/__init__.py -------------------------------------------------------------------------------- /swift/storage.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | import mimetypes 3 | import os 4 | import re 5 | from datetime import datetime 6 | from functools import wraps 7 | from io import BytesIO, UnsupportedOperation 8 | from time import time 9 | 10 | import magic 11 | from django.core.exceptions import ImproperlyConfigured 12 | from django.core.files import File 13 | from django.core.files.storage import Storage 14 | from six.moves.urllib import parse as urlparse 15 | 16 | from swift.utils import setting 17 | 18 | try: 19 | from django.utils.deconstruct import deconstructible 20 | except ImportError: 21 | def deconstructible(arg): 22 | return arg 23 | 24 | try: 25 | import swiftclient 26 | from swiftclient.utils import generate_temp_url 27 | except ImportError: 28 | raise ImproperlyConfigured("Could not load swiftclient library") 29 | 30 | 31 | def validate_settings(backend): 32 | # Check mandatory parameters 33 | if not backend.api_auth_url: 34 | raise ImproperlyConfigured("The SWIFT_AUTH_URL setting is required") 35 | 36 | if not backend.api_username: 37 | raise ImproperlyConfigured("The SWIFT_USERNAME setting is required") 38 | 39 | if not backend.api_key: 40 | raise ImproperlyConfigured("The SWIFT_KEY or SWIFT_PASSWORD setting is required") 41 | 42 | if not backend.container_name: 43 | raise ImproperlyConfigured("No container name defined. Use SWIFT_CONTAINER_NAME \ 44 | or SWIFT_STATIC_CONTAINER_NAME depending on the backend") 45 | 46 | # Detect auth version if not defined 47 | # http://docs.openstack.org/developer/python-swiftclient/cli.html#authentication 48 | if not backend.auth_version: 49 | if (backend.user_domain_name or backend.user_domain_id) and \ 50 | (backend.project_domain_name or backend.project_domain_id): 51 | # Set version 3 if domain and project scoping is defined 52 | backend.auth_version = '3' 53 | else: 54 | if backend.tenant_name or backend.tenant_id: 55 | # Set version 2 if a tenant is defined 56 | backend.auth_version = '2' 57 | else: 58 | # Set version 1 if no tenant is not defined 59 | backend.auth_version = '1' 60 | 61 | # Enforce auth_version into a string (more future proof) 62 | backend.auth_version = str(backend.auth_version) 63 | 64 | # Validate v2 auth parameters 65 | if backend.auth_version == '2': 66 | if not (backend.tenant_name or backend.tenant_id): 67 | raise ImproperlyConfigured("SWIFT_TENANT_ID or SWIFT_TENANT_NAME must \ 68 | be defined when using version 2 auth") 69 | 70 | # Validate v3 auth parameters 71 | if backend.auth_version == '3': 72 | if not (backend.user_domain_name or backend.user_domain_id): 73 | raise ImproperlyConfigured("SWIFT_USER_DOMAIN_NAME or \ 74 | SWIFT_USER_DOMAIN_ID must be defined when using version 3 auth") 75 | 76 | if not (backend.project_domain_name or backend.project_domain_id): 77 | raise ImproperlyConfigured("SWIFT_PROJECT_DOMAIN_NAME or \ 78 | SWIFT_PROJECT_DOMAIN_ID must be defined when using version 3 auth") 79 | 80 | if not (backend.tenant_name or backend.tenant_id): 81 | raise ImproperlyConfigured("SWIFT_PROJECT_ID or SWIFT_PROJECT_NAME must \ 82 | be defined when using version 3 auth") 83 | 84 | # Validate temp_url parameters 85 | if backend.use_temp_urls: 86 | if backend.temp_url_key is None: 87 | raise ImproperlyConfigured("SWIFT_TEMP_URL_KEY must be set when \ 88 | SWIFT_USE_TEMP_URL is True") 89 | 90 | # Encode temp_url_key as bytes 91 | try: 92 | backend.temp_url_key = backend.temp_url_key.encode('ascii') 93 | except UnicodeEncodeError: 94 | raise ImproperlyConfigured("SWIFT_TEMP_URL_KEY must ascii") 95 | 96 | # Misc sanity checks 97 | if not isinstance(backend.os_extra_options, dict): 98 | raise ImproperlyConfigured("SWIFT_EXTRA_OPTIONS must be a dict") 99 | 100 | 101 | def prepend_name_prefix(func): 102 | """ 103 | Decorator that wraps instance methods to prepend the instance's filename 104 | prefix to the beginning of the referenced filename. Must only be used on 105 | instance methods where the first parameter after `self` is `name` or a 106 | comparable parameter of a different name. 107 | """ 108 | @wraps(func) 109 | def prepend_prefix(self, name, *args, **kwargs): 110 | name = self.name_prefix + name 111 | return func(self, name, *args, **kwargs) 112 | return prepend_prefix 113 | 114 | 115 | @deconstructible 116 | class SwiftStorage(Storage): 117 | api_auth_url = setting('SWIFT_AUTH_URL') 118 | api_username = setting('SWIFT_USERNAME') 119 | api_key = setting('SWIFT_KEY') or setting('SWIFT_PASSWORD') 120 | auth_version = setting('SWIFT_AUTH_VERSION') 121 | tenant_name = setting('SWIFT_TENANT_NAME') or setting('SWIFT_PROJECT_NAME') 122 | tenant_id = setting('SWIFT_TENANT_ID') or setting('SWIFT_PROJECT_ID') 123 | user_domain_name = setting('SWIFT_USER_DOMAIN_NAME') 124 | user_domain_id = setting('SWIFT_USER_DOMAIN_ID') 125 | project_domain_name = setting('SWIFT_PROJECT_DOMAIN_NAME') 126 | project_domain_id = setting('SWIFT_PROJECT_DOMAIN_ID') 127 | region_name = setting('SWIFT_REGION_NAME') 128 | container_name = setting('SWIFT_CONTAINER_NAME') 129 | auto_create_container = setting('SWIFT_AUTO_CREATE_CONTAINER', False) 130 | auto_create_container_public = setting( 131 | 'SWIFT_AUTO_CREATE_CONTAINER_PUBLIC', False) 132 | auto_create_container_allow_orgin = setting( 133 | 'SWIFT_AUTO_CREATE_CONTAINER_ALLOW_ORIGIN') 134 | auto_base_url = setting('SWIFT_AUTO_BASE_URL', True) 135 | override_base_url = setting('SWIFT_BASE_URL') 136 | use_temp_urls = setting('SWIFT_USE_TEMP_URLS', False) 137 | temp_url_key = setting('SWIFT_TEMP_URL_KEY') 138 | temp_url_duration = setting('SWIFT_TEMP_URL_DURATION', 30 * 60) 139 | auth_token_duration = setting('SWIFT_AUTH_TOKEN_DURATION', 60 * 60 * 23) 140 | os_extra_options = setting('SWIFT_EXTRA_OPTIONS', {}) 141 | auto_overwrite = setting('SWIFT_AUTO_OVERWRITE', False) 142 | lazy_connect = setting('SWIFT_LAZY_CONNECT', False) 143 | content_type_from_fd = setting('SWIFT_CONTENT_TYPE_FROM_FD', False) 144 | content_length_from_fd = setting('SWIFT_CONTENT_LENGTH_FROM_FD', True) 145 | gzip_content_types = setting('SWIFT_GZIP_CONTENT_TYPES', []) 146 | gzip_unknown_content_type = setting('SWIFT_GZIP_UNKNOWN_CONTENT_TYPE', False) 147 | gzip_compression_level = setting('SWIFT_GZIP_COMPRESSION_LEVEL', 4) 148 | _token_creation_time = 0 149 | _token = '' 150 | _swift_conn = None 151 | _base_url = None 152 | name_prefix = setting('SWIFT_NAME_PREFIX', '') 153 | full_listing = setting('SWIFT_FULL_LISTING', True) 154 | max_retries = setting('SWIFT_MAX_RETRIES', 5) 155 | cache_headers = setting('SWIFT_CACHE_HEADERS', False) 156 | 157 | def __init__(self, **settings): 158 | # check if some of the settings provided as class attributes 159 | # should be overwritten 160 | for name, value in settings.items(): 161 | if hasattr(self, name): 162 | setattr(self, name, value) 163 | 164 | validate_settings(self) 165 | 166 | self.last_headers_name = None 167 | self.last_headers_value = None 168 | 169 | self.os_options = { 170 | 'tenant_id': self.tenant_id, 171 | 'tenant_name': self.tenant_name, 172 | 'user_domain_id': self.user_domain_id, 173 | 'user_domain_name': self.user_domain_name, 174 | 'project_domain_id': self.project_domain_id, 175 | 'project_domain_name': self.project_domain_name, 176 | 'region_name': self.region_name, 177 | } 178 | self.os_options.update(self.os_extra_options) 179 | 180 | if not self.lazy_connect: 181 | self.swift_conn 182 | 183 | @property 184 | def swift_conn(self): 185 | """Get swift connection wrapper""" 186 | if not self._swift_conn: 187 | self._swift_conn = swiftclient.Connection( 188 | authurl=self.api_auth_url, 189 | user=self.api_username, 190 | key=self.api_key, 191 | retries=self.max_retries, 192 | tenant_name=self.tenant_name, 193 | os_options=self.os_options, 194 | auth_version=self.auth_version) 195 | self._check_container() 196 | return self._swift_conn 197 | 198 | def _check_container(self): 199 | """ 200 | Check that container exists; raises exception if not. 201 | """ 202 | try: 203 | self.swift_conn.head_container(self.container_name) 204 | except swiftclient.ClientException: 205 | headers = {} 206 | if self.auto_create_container: 207 | if self.auto_create_container_public: 208 | headers['X-Container-Read'] = '.r:*' 209 | if self.auto_create_container_allow_orgin: 210 | headers['X-Container-Meta-Access-Control-Allow-Origin'] = \ 211 | self.auto_create_container_allow_orgin 212 | self.swift_conn.put_container(self.container_name, 213 | headers=headers) 214 | else: 215 | raise ImproperlyConfigured( 216 | "Container %s does not exist." % self.container_name) 217 | 218 | @property 219 | def base_url(self): 220 | if self._base_url is None: 221 | if self.auto_base_url: 222 | # Derive a base URL based on the authentication information from 223 | # the server, optionally overriding the protocol, host/port and 224 | # potentially adding a path fragment before the auth information. 225 | self._base_url = self.swift_conn.url + '/' 226 | if self.override_base_url is not None: 227 | # override the protocol and host, append any path fragments 228 | split_derived = urlparse.urlsplit(self._base_url) 229 | split_override = urlparse.urlsplit(self.override_base_url) 230 | split_result = [''] * 5 231 | split_result[0:2] = split_override[0:2] 232 | split_result[2] = (split_override[2] + split_derived[2] 233 | ).replace('//', '/') 234 | self._base_url = urlparse.urlunsplit(split_result) 235 | 236 | self._base_url = urlparse.urljoin(self._base_url, 237 | self.container_name) 238 | self._base_url += '/' 239 | else: 240 | self._base_url = self.override_base_url 241 | return self._base_url 242 | 243 | def _open(self, name, mode='rb'): 244 | original_name = name 245 | name = self.name_prefix + name 246 | 247 | headers, content = self.swift_conn.get_object(self.container_name, name) 248 | buf = BytesIO(content) 249 | buf.name = os.path.basename(original_name) 250 | buf.mode = mode 251 | return File(buf) 252 | 253 | def _save(self, name, content, headers=None): 254 | original_name = name 255 | name = self.name_prefix + name 256 | 257 | # Django rewinds file position to the beginning before saving, 258 | # so should we. 259 | # See django.core.files.storage.FileSystemStorage#_save 260 | # and django.core.files.base.File#chunks 261 | try: 262 | content.seek(0) 263 | except (AttributeError, UnsupportedOperation): # pragma: no cover 264 | pass 265 | 266 | if self.content_type_from_fd: 267 | content_type = magic.from_buffer(content.read(1024), mime=True) 268 | # Go back to the beginning of the file 269 | content.seek(0) 270 | else: 271 | content_type = mimetypes.guess_type(name)[0] 272 | 273 | if self.content_length_from_fd: 274 | content_length = content.size 275 | else: 276 | content_length = None 277 | 278 | if content_type in self.gzip_content_types or ( 279 | content_type is None and self.gzip_unknown_content_type): 280 | gz_data = BytesIO() 281 | gzf = gzip.GzipFile(filename=name, 282 | fileobj=gz_data, 283 | mode='wb', 284 | compresslevel=self.gzip_compression_level) 285 | gzf.write(content.file.read()) 286 | gzf.close() 287 | content = gz_data.getvalue() 288 | content_length = None 289 | 290 | if not headers: 291 | headers = {} 292 | headers['Content-Encoding'] = 'gzip' 293 | 294 | self.swift_conn.put_object(self.container_name, 295 | name, 296 | content, 297 | content_length=content_length, 298 | content_type=content_type, 299 | headers=headers) 300 | return original_name 301 | 302 | def get_headers(self, name): 303 | if self.cache_headers: 304 | """ 305 | Optimization : only fetch headers once when several calls are made 306 | requiring information for the same name. 307 | When the caller is collectstatic, this makes a huge difference. 308 | According to my test, we get a *2 speed up. Which makes sense : two 309 | api calls were made.. 310 | """ 311 | if name != self.last_headers_name: 312 | # miss -> update 313 | self.last_headers_value = self.swift_conn.head_object(self.container_name, name) 314 | self.last_headers_name = name 315 | else: 316 | self.last_headers_value = self.swift_conn.head_object(self.container_name, name) 317 | self.last_headers_name = name 318 | 319 | return self.last_headers_value 320 | 321 | @prepend_name_prefix 322 | def exists(self, name): 323 | try: 324 | self.get_headers(name) 325 | except swiftclient.ClientException: 326 | return False 327 | return True 328 | 329 | @prepend_name_prefix 330 | def delete(self, name): 331 | try: 332 | self.swift_conn.delete_object(self.container_name, name) 333 | except swiftclient.ClientException: 334 | pass 335 | 336 | def get_valid_name(self, name): 337 | s = name.strip().replace(' ', '_') 338 | return re.sub(r'(?u)[^-_\w./]', '', s) 339 | 340 | @prepend_name_prefix 341 | def get_available_name(self, name, max_length=None): 342 | """ 343 | Returns a filename that's free on the target storage system, and 344 | available for new content to be written to. 345 | """ 346 | if not self.auto_overwrite: 347 | if max_length is None: 348 | name = super(SwiftStorage, self).get_available_name(name) 349 | else: 350 | name = super(SwiftStorage, self).get_available_name( 351 | name, max_length) 352 | 353 | if self.name_prefix: 354 | # Split out the name prefix so we can just return the bit of 355 | # the name that's relevant upstream, since the prefix will 356 | # be automatically added on subsequent requests anyway. 357 | empty, prefix, final = name.partition(self.name_prefix) 358 | return final 359 | else: 360 | return name 361 | 362 | @prepend_name_prefix 363 | def size(self, name): 364 | return int(self.get_headers(name)['content-length']) 365 | 366 | @prepend_name_prefix 367 | def modified_time(self, name): 368 | return datetime.fromtimestamp( 369 | float(self.get_headers(name)['x-timestamp'])) 370 | 371 | @prepend_name_prefix 372 | def url(self, name): 373 | return self._path(name) 374 | 375 | def _path(self, name): 376 | try: 377 | name = name.encode('utf-8') 378 | except UnicodeDecodeError: 379 | pass 380 | url = urlparse.urljoin(self.base_url, urlparse.quote(name)) 381 | 382 | # Are we building a temporary url? 383 | if self.use_temp_urls: 384 | expires = int(time() + int(self.temp_url_duration)) 385 | path = urlparse.unquote(urlparse.urlsplit(url).path) 386 | tmp_path = generate_temp_url(path, expires, self.temp_url_key, 'GET', absolute=True) 387 | url = urlparse.urljoin(self.base_url, tmp_path) 388 | 389 | return url 390 | 391 | def path(self, name): 392 | raise NotImplementedError 393 | 394 | @prepend_name_prefix 395 | def isdir(self, name): 396 | return '.' not in name 397 | 398 | @prepend_name_prefix 399 | def listdir(self, path): 400 | container = self.swift_conn.get_container( 401 | self.container_name, prefix=path, full_listing=self.full_listing) 402 | files = [] 403 | dirs = [] 404 | for obj in container[1]: 405 | remaining_path = obj['name'][len(path):].split('/') 406 | key = remaining_path[0] if remaining_path[0] else remaining_path[1] 407 | 408 | if not self.isdir(key): 409 | files.append(key) 410 | elif key not in dirs: 411 | dirs.append(key) 412 | 413 | return dirs, files 414 | 415 | @prepend_name_prefix 416 | def makedirs(self, dirs): 417 | self.swift_conn.put_object(self.container_name, 418 | '%s/.' % (self.name_prefix + dirs), 419 | contents='') 420 | 421 | @prepend_name_prefix 422 | def rmtree(self, abs_path): 423 | container = self.swift_conn.get_container(self.container_name) 424 | 425 | for obj in container[1]: 426 | if obj['name'].startswith(abs_path): 427 | self.swift_conn.delete_object(self.container_name, 428 | obj['name']) 429 | 430 | 431 | class StaticSwiftStorage(SwiftStorage): 432 | container_name = setting('SWIFT_STATIC_CONTAINER_NAME', '') 433 | name_prefix = setting('SWIFT_STATIC_NAME_PREFIX', '') 434 | auto_base_url = setting('SWIFT_STATIC_AUTO_BASE_URL', True) 435 | override_base_url = setting('SWIFT_STATIC_BASE_URL') 436 | auto_create_container_public = True 437 | use_temp_urls = False 438 | 439 | def get_available_name(self, name, max_length=None): 440 | """ 441 | When running collectstatic we don't want to return an available name, 442 | we want to return the same name because if the file exists we want to 443 | overwrite it. 444 | """ 445 | return name 446 | -------------------------------------------------------------------------------- /swift/utils.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | def setting(name, default=None): 4 | return getattr(settings, name, default) 5 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dennisv/django-storage-swift/c694e72191cc3a19dd9b67776de9086d716d7d44/tests/__init__.py -------------------------------------------------------------------------------- /tests/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | 4 | if __name__ == "__main__": 5 | sys.path.append('..') 6 | 7 | from django.core.management import execute_from_command_line 8 | 9 | execute_from_command_line(sys.argv) 10 | -------------------------------------------------------------------------------- /tests/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | PROJECT_DIR = os.path.abspath(os.path.dirname(__file__)) 4 | SECRET_KEY = '95mk9r=y^bvver#6e#-169t9brqpcq#&@gjk*#!3lckf )p3' 5 | 6 | DATABASES = { 7 | 'default': { 8 | 'ENGINE': 'django.db.backends.sqlite3', 9 | 'NAME': os.path.join(PROJECT_DIR, 'test.db') 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tests/tests.py: -------------------------------------------------------------------------------- 1 | 2 | # -*- coding: UTF-8 -*- 3 | import hmac 4 | from copy import deepcopy 5 | from django.test import TestCase 6 | from django.core.exceptions import ImproperlyConfigured, SuspiciousFileOperation 7 | from django.core.files.base import ContentFile 8 | from hashlib import sha256 9 | from mock import patch 10 | from .utils import FakeSwift, auth_params, base_url, CONTAINER_CONTENTS, TENANT_ID 11 | from swift import storage 12 | from six.moves.urllib import parse as urlparse 13 | 14 | 15 | class SwiftStorageTestCase(TestCase): 16 | 17 | def default_storage(self, auth_config, exclude=None, **params): 18 | """Instantiate default storage with auth parameters""" 19 | return storage.SwiftStorage(**auth_params(auth_config, exclude=exclude, **params)) 20 | 21 | def static_storage(self, auth_config, exclude=None, **params): 22 | """Instantiate static storage with auth parameters""" 23 | return storage.StaticSwiftStorage(**auth_params(auth_config, exclude=exclude, **params)) 24 | 25 | 26 | @patch('swift.storage.swiftclient', new=FakeSwift) 27 | class AuthTest(SwiftStorageTestCase): 28 | """Test authentication parameters""" 29 | 30 | def test_auth_v1(self): 31 | """Test version 1 authentication""" 32 | self.default_storage('v1') 33 | 34 | def test_auth_v2(self): 35 | """Test version 2 authentication""" 36 | self.default_storage('v2') 37 | 38 | def test_auth_v3(self): 39 | """Test version 3 authentication""" 40 | self.default_storage('v3') 41 | 42 | def test_auth_v1_detect_version(self): 43 | """Test version 1 authentication detection""" 44 | backend = self.default_storage('v1', exclude=['auth_version']) 45 | self.assertEqual(backend.auth_version, '1') 46 | 47 | def test_auth_v2_detect_version(self): 48 | """Test version 2 authentication detection""" 49 | backend = self.default_storage('v2', exclude=['auth_version']) 50 | self.assertEqual(backend.auth_version, '2') 51 | 52 | def test_auth_v3_detect_version(self): 53 | """Test version 3 authentication detection""" 54 | backend = self.default_storage('v3', exclude=['auth_version']) 55 | self.assertEqual(backend.auth_version, '3') 56 | 57 | def test_auth_v2_no_tenant(self): 58 | """Missing tenant in v2 auth""" 59 | with self.assertRaises(ImproperlyConfigured): 60 | self.default_storage('v2', exclude=['tenant_name', 'tenant_id']) 61 | 62 | def test_auth_v3_no_tenant(self): 63 | """Missing tenant in v3 auth""" 64 | with self.assertRaises(ImproperlyConfigured): 65 | self.default_storage('v3', exclude=['tenant_name']) 66 | 67 | def test_auth_v3_no_user_domain(self): 68 | """Missing user_domain in v3 auth""" 69 | with self.assertRaises(ImproperlyConfigured): 70 | self.default_storage('v3', exclude=['user_domain_name', 'user_domain_id']) 71 | 72 | def test_auth_v3_no_project_domain(self): 73 | """Missing project_domain in v3 auth""" 74 | with self.assertRaises(ImproperlyConfigured): 75 | self.default_storage('v3', exclude=['project_domain_name', 'project_domain_id']) 76 | 77 | def test_auth_v3_int_auth_version(self): 78 | """Auth version converts into a string""" 79 | backend = self.default_storage('v3', auth_version=3) 80 | self.assertEqual(backend.auth_version, '3') 81 | 82 | 83 | @patch('swift.storage.swiftclient', new=FakeSwift) 84 | class MandatoryParamsTest(SwiftStorageTestCase): 85 | 86 | def test_instantiate_default(self): 87 | """Instantiate default backend with no parameters""" 88 | with self.assertRaises(ImproperlyConfigured): 89 | storage.SwiftStorage() 90 | 91 | def test_instantiate_static(self): 92 | """Instantiate static backend with no parameters""" 93 | with self.assertRaises(ImproperlyConfigured): 94 | storage.StaticSwiftStorage() 95 | 96 | def test_mandatory_auth_url(self): 97 | """Test ImproperlyConfigured if api_auth_url is missing""" 98 | with self.assertRaises(ImproperlyConfigured): 99 | self.default_storage('v3', exclude=['api_auth_url']) 100 | 101 | def test_mandatory_username(self): 102 | """Test ImproperlyConfigured if api_auth_url is missing""" 103 | with self.assertRaises(ImproperlyConfigured): 104 | self.default_storage('v3', exclude=['api_username']) 105 | 106 | def test_mandatory_container_name(self): 107 | """Test ImproperlyConfigured if container_name is missing""" 108 | with self.assertRaises(ImproperlyConfigured): 109 | self.default_storage('v3', exclude=['container_name']) 110 | 111 | def test_mandatory_static_container_name(self): 112 | """Test ImproperlyConfigured if container_name is missing""" 113 | with self.assertRaises(ImproperlyConfigured): 114 | self.static_storage('v3', exclude=['container_name']) 115 | 116 | def test_mandatory_password(self): 117 | """Test ImproperlyConfigured if api_key is missing""" 118 | with self.assertRaises(ImproperlyConfigured): 119 | self.default_storage('v3', exclude=['api_key']) 120 | 121 | 122 | @patch('swift.storage.swiftclient', new=FakeSwift) 123 | class ConfigTest(SwiftStorageTestCase): 124 | 125 | def test_missing_container(self): 126 | """Raise if container don't exist""" 127 | with self.assertRaises(ImproperlyConfigured): 128 | self.default_storage('v3', container_name='idontexist') 129 | 130 | def test_delete_nonexisting_file(self): 131 | """Deleting non-existing file is silently ignored""" 132 | backend = self.default_storage('v3') 133 | backend.delete("idontexist.something") 134 | 135 | def test_auto_base_url(self): 136 | """Automatically resolve base url""" 137 | backend = self.default_storage('v3', auto_base_url=True) 138 | self.assertEqual(backend.base_url, base_url(container="container")) 139 | 140 | def test_override_base_url_no_auto(self): 141 | """Test overriding base url without auto base url""" 142 | url = 'http://localhost:8080/test/' 143 | backend = self.default_storage('v3', 144 | auto_base_url=False, 145 | override_base_url=url) 146 | self.assertEqual(backend.base_url, url) 147 | 148 | def test_override_base_url_auto(self): 149 | """Test overriding base url with auto""" 150 | url = 'http://localhost:8080' 151 | backend = self.default_storage('v3', 152 | auto_base_url=True, 153 | override_base_url=url) 154 | self.assertTrue(backend.base_url.startswith(url)) 155 | storage_url = '{}/v1/AUTH_{}/{}/'.format(url, TENANT_ID, "container") 156 | self.assertEqual(backend.base_url, storage_url) 157 | 158 | def test_illegal_extra_opts(self): 159 | """extra_opts should always be a dict""" 160 | with self.assertRaises(ImproperlyConfigured): 161 | self.default_storage('v3', os_extra_options="boom!") 162 | 163 | @patch.object(FakeSwift.Connection, '__init__', return_value=None) 164 | def test_override_lazy_connect(self, mock_swift_init): 165 | """Test setting lazy_connect delays connection creation""" 166 | backend = self.default_storage('v3', lazy_connect=True) 167 | assert not mock_swift_init.called 168 | self.assertFalse(backend.exists('warez/some_random_movie.mp4')) 169 | assert mock_swift_init.called 170 | 171 | 172 | # @patch('swift.storage.swiftclient', new=FakeSwift) 173 | # class TokenTest(SwiftStorageTestCase): 174 | 175 | # def test_get_token(self): 176 | # """Renewing token""" 177 | # backend = self.default_storage('v3', auth_token_duration=0) 178 | # backend.get_token() 179 | 180 | # def test_set_token(self): 181 | # """Set token manually""" 182 | # backend = self.default_storage('v3', auth_token_duration=0) 183 | # backend.set_token('token') 184 | 185 | 186 | @patch('swift.storage.swiftclient', new=FakeSwift) 187 | class CreateContainerTest(SwiftStorageTestCase): 188 | 189 | def test_auto_create_container(self): 190 | """Auth create container""" 191 | self.default_storage( 192 | 'v3', 193 | auto_create_container=True, 194 | auto_create_container_public=True, 195 | auto_create_container_allow_orgin=True, 196 | container_name='new') 197 | 198 | 199 | @patch('swift.storage.swiftclient', new=FakeSwift) 200 | class BackendTest(SwiftStorageTestCase): 201 | 202 | @patch('swift.storage.swiftclient', new=FakeSwift) 203 | def setUp(self): 204 | self.backend = self.default_storage('v3') 205 | 206 | def test_url(self): 207 | """Get url for a resource""" 208 | name = 'images/test.png' 209 | url = self.backend.url(name) 210 | self.assertEqual(url, base_url(container=self.backend.container_name, path=name)) 211 | 212 | def test_url_unicode_name(self): 213 | """Get url for a resource with unicode filename""" 214 | name = u'images/test终端.png' 215 | url = self.backend.url(name) 216 | self.assertEqual(url, base_url(container=self.backend.container_name, path=name)) 217 | 218 | def test_object_size(self): 219 | """Test getting object size""" 220 | size = self.backend.size('images/test.png') 221 | self.assertEqual(size, 4096) 222 | 223 | def test_modified_time(self): 224 | """Test getting modified time of an object""" 225 | self.backend.modified_time('images/test.png') 226 | 227 | def test_object_exists(self): 228 | """Test for the existence of an object""" 229 | exists = self.backend.exists('images/test.png') 230 | self.assertTrue(exists) 231 | 232 | def test_object_dont_exists(self): 233 | """Test for the existence of an non-existent object""" 234 | exists = self.backend.exists('warez/some_random_movie.mp4') 235 | self.assertFalse(exists) 236 | 237 | def test_get_headers_chache1(self): 238 | self.backend.cache_headers = True; 239 | headers = self.backend.get_headers('images/test.png') 240 | self.assertEqual('fcfc6539ce4e545ce58bafeeac3303a7', headers['hash']) 241 | 242 | def test_get_headers_chache2(self): 243 | self.backend.cache_headers = False; 244 | headers = self.backend.get_headers('images/test.png') 245 | self.assertEqual('fcfc6539ce4e545ce58bafeeac3303a7', headers['hash']) 246 | 247 | def test_listdir(self): 248 | """List root in container""" 249 | dirs, files = self.backend.listdir('') 250 | self.assertListEqual(dirs, ['images', 'css', 'js']) 251 | self.assertListEqual(files, ['root.txt']) 252 | 253 | @patch('tests.utils.FakeSwift.objects', new=deepcopy(CONTAINER_CONTENTS)) 254 | def test_rmtree(self): 255 | """Remove folder in storage""" 256 | backend = self.default_storage('v3') 257 | backend.rmtree('images') 258 | dirs, files = self.backend.listdir('') 259 | self.assertListEqual(dirs, ['css', 'js']) 260 | self.assertListEqual(files, ['root.txt']) 261 | 262 | @patch('tests.utils.FakeSwift.objects', new=deepcopy(CONTAINER_CONTENTS)) 263 | def test_mkdirs(self): 264 | """Make directory/pseudofolder in backend""" 265 | backend = self.default_storage('v3') 266 | backend.makedirs('downloads') 267 | dirs, files = self.backend.listdir('') 268 | self.assertListEqual(dirs, ['images', 'css', 'js', 'downloads']) 269 | self.assertListEqual(files, ['root.txt']) 270 | 271 | @patch('tests.utils.FakeSwift.objects', new=deepcopy(CONTAINER_CONTENTS)) 272 | def test_delete_object(self): 273 | """Delete an object""" 274 | backend = self.default_storage('v3') 275 | backend.delete('root.txt') 276 | dirs, files = self.backend.listdir('') 277 | self.assertListEqual(dirs, ['images', 'css', 'js']) 278 | self.assertListEqual(files, []) 279 | 280 | @patch('tests.utils.FakeSwift.objects', new=deepcopy(CONTAINER_CONTENTS)) 281 | def test_save(self): 282 | """Save an object""" 283 | backend = self.default_storage('v3') 284 | content_file = ContentFile("Hello world!") 285 | name = backend.save("test.txt", content_file) 286 | dirs, files = self.backend.listdir('') 287 | self.assertEqual(files.count(name), 1) 288 | 289 | @patch('tests.utils.FakeSwift.objects', new=deepcopy(CONTAINER_CONTENTS)) 290 | @patch('gzip.GzipFile') 291 | def test_save_gzip(self, gzip_mock): 292 | """Save an object""" 293 | backend = self.default_storage('v3') 294 | backend.gzip_content_types = ['text/plain'] 295 | content_file = ContentFile(b'Hello world!') 296 | name = backend.save('testgz.txt', content_file) 297 | dirs, files = self.backend.listdir('') 298 | self.assertEqual(files.count(name), 1) 299 | self.assertTrue(gzip_mock.called) 300 | 301 | @patch('tests.utils.FakeSwift.objects', new=deepcopy(CONTAINER_CONTENTS)) 302 | def test_content_type_from_fd(self): 303 | """Test content_type detection on save""" 304 | backend = self.default_storage('v3', content_type_from_fd=True) 305 | backend.save("test.txt", ContentFile("Some random data")) 306 | 307 | def test_save_non_rewound(self): 308 | """Save file with position not at the beginning""" 309 | content = dict(orig=b"Hello world!") 310 | content_file = ContentFile(content['orig']) 311 | content_file.seek(5) 312 | 313 | def mocked_put_object(cls, url, token, container, name=None, 314 | contents=None, content_length=None, *args, **kwargs): 315 | content['saved'] = contents.read() 316 | content['size'] = content_length 317 | 318 | with patch('tests.utils.FakeSwift.put_object', new=classmethod(mocked_put_object)): 319 | self.backend.save('test.txt', content_file) 320 | self.assertEqual(content['saved'], content['orig']) 321 | self.assertEqual(content['size'], len(content['orig'])) 322 | 323 | def test_no_content_length_from_fd(self): 324 | """Test disabling content_length_from_fd on save""" 325 | backend = self.default_storage('v3', content_length_from_fd=False) 326 | content = dict(orig="Hello world!") 327 | content_file = ContentFile("") 328 | content_file.write(content['orig']) 329 | 330 | def mocked_put_object(cls, url, token, container, name=None, 331 | contents=None, content_length=None, *args, **kwargs): 332 | content['saved'] = contents.read() 333 | content['size'] = content_length 334 | 335 | with patch('tests.utils.FakeSwift.put_object', new=classmethod(mocked_put_object)): 336 | backend.save('test.txt', content_file) 337 | self.assertEqual(content['saved'], content['orig']) 338 | self.assertIsNone(content['size']) 339 | 340 | def test_open(self): 341 | """Attempt to open a object""" 342 | file = self.backend._open('root.txt') 343 | self.assertEqual(file.name, 'root.txt') 344 | data = file.read() 345 | self.assertEqual(len(data), 4096) 346 | 347 | def test_get_available_name_nonexist(self): 348 | """Available name for non-existent object""" 349 | object = 'images/doesnotexist.png' 350 | name = self.backend.get_available_name(object) 351 | self.assertEqual(name, object) 352 | 353 | def test_get_available_name_max_length_nonexist(self): 354 | """Available name with max_length for non-existent object""" 355 | object = 'images/doesnotexist.png' 356 | name = self.backend.get_available_name(object, len(object)) 357 | self.assertEqual(name, object) 358 | with self.assertRaises(SuspiciousFileOperation): 359 | name = self.backend.get_available_name(object, 16) 360 | 361 | def test_get_available_name_exist(self): 362 | """Available name for existing object""" 363 | object = 'images/test.png' 364 | name = self.backend.get_available_name(object) 365 | self.assertNotEqual(name, object) 366 | 367 | def test_get_available_name_max_length_exist(self): 368 | """Available name with max_length for existing object""" 369 | object = 'images/test.png' 370 | name = self.backend.get_available_name(object, 32) 371 | self.assertNotEqual(name, object) 372 | 373 | def test_get_available_name_prefix(self): 374 | """Available name with prefix""" 375 | object = 'test.png' 376 | # This will add the prefix, then get_avail will remove it again 377 | backend = self.default_storage('v3', name_prefix="prefix-") 378 | name = backend.get_available_name(object) 379 | self.assertEqual(object, name) 380 | 381 | def test_get_available_name_static(self): 382 | """Static's get_available_name should be unmodified""" 383 | static = self.static_storage('v3') 384 | object = "test.txt" 385 | name = static.get_available_name(object) 386 | self.assertEqual(name, object) 387 | 388 | def test_get_valid_name(self): 389 | name = self.backend.get_valid_name("A @#!file.txt") 390 | self.assertEqual(name, "A_file.txt") 391 | 392 | def test_path(self): 393 | """path is not implemented""" 394 | with self.assertRaises(NotImplementedError): 395 | self.backend.path("test.txt") 396 | 397 | 398 | @patch('swift.storage.swiftclient', new=FakeSwift) 399 | class TemporaryUrlTest(SwiftStorageTestCase): 400 | 401 | def assert_valid_temp_url(self, name): 402 | url = self.backend.url(name) 403 | split_url = urlparse.urlsplit(url) 404 | query_params = urlparse.parse_qs(split_url[3]) 405 | split_base_url = urlparse.urlsplit(base_url(container=self.backend.container_name, path=name)) 406 | 407 | # ensure scheme, netloc, and path are same as to non-temporary URL 408 | self.assertEqual(split_base_url[0:2], split_url[0:2]) 409 | 410 | # ensure query string contains signature and expiry 411 | self.assertIn('temp_url_sig', query_params) 412 | self.assertIn('temp_url_expires', query_params) 413 | 414 | def assert_valid_signature(self, path): 415 | """Validate temp-url signature""" 416 | backend = self.default_storage('v3', use_temp_urls=True, temp_url_key='Key') 417 | url = backend.url(path) 418 | url_parsed = urlparse.urlsplit(url) 419 | params = urlparse.parse_qs(url_parsed.query) 420 | msg = "{}\n{}\n{}".format("GET", params['temp_url_expires'][0], urlparse.unquote(url_parsed.path)) 421 | sig = hmac.new(backend.temp_url_key, msg.encode('utf-8'), sha256).hexdigest() 422 | self.assertEqual(params['temp_url_sig'][0], sig) 423 | 424 | def test_signature(self): 425 | self.assert_valid_signature("test/test.txt") 426 | self.assert_valid_signature("test/file with spaces.txt") 427 | 428 | def test_temp_url_key_required(self): 429 | """Must set temp_url_key when use_temp_urls=True""" 430 | with self.assertRaises(ImproperlyConfigured): 431 | self.backend = self.default_storage('v3', use_temp_urls=True) 432 | 433 | def test_temp_url_key(self): 434 | """Get temporary url using string key""" 435 | self.backend = self.default_storage('v3', use_temp_urls=True, temp_url_key='Key') 436 | self.assert_valid_temp_url('images/test.png') 437 | 438 | def test_temp_url_key_unicode(self): 439 | """temp_url_key must be ascii""" 440 | with self.assertRaises(ImproperlyConfigured): 441 | self.backend = self.default_storage('v3', use_temp_urls=True, temp_url_key=u'aあä') 442 | 443 | def test_temp_url_key_unicode_latin(self): 444 | """Get temporary url using a unicode key which can be ascii-encoded""" 445 | self.backend = self.default_storage('v3', use_temp_urls=True, temp_url_key=u'Key') 446 | self.assert_valid_temp_url('images/test.png') 447 | 448 | def test_temp_url_unicode_name(self): 449 | """temp_url file name can be unicode.""" 450 | self.backend = self.default_storage('v3', use_temp_urls=True, temp_url_key=u'Key') 451 | self.assert_valid_temp_url('images/aあä.png') 452 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | from six.moves.urllib import parse as urlparse 3 | 4 | BASE_URL = 'https://objects.example.com/v1' 5 | AUTH_URL = 'https://auth.example.com' 6 | TENANT_ID = '11223344556677889900aabbccddeeff' 7 | TOKEN = 'auth_token' 8 | 9 | AUTH_PARAMETERS = { 10 | 'v1': { 11 | 'api_auth_url': 'https://objects.example.com', 12 | 'api_username': 'user', 13 | 'api_key': 'auth_key', 14 | 'auth_version': '1', 15 | 'container_name': "container", 16 | }, 17 | 'v2': { 18 | 'api_auth_url': 'https://objects.example.com', 19 | 'api_username': 'user', 20 | 'api_key': 'auth_key', 21 | 'tenant_name': 'tenant', 22 | 'tenant_id': 'tenant', 23 | 'auth_version': '2', 24 | 'container_name': "container" 25 | }, 26 | 'v3': { 27 | 'api_auth_url': 'https://objects.example.com', 28 | 'api_username': 'user', 29 | 'api_key': 'auth_key', 30 | 'auth_version': '3', 31 | 'user_domain_name': 'domain', 32 | 'user_domain_id': 'domain', 33 | 'project_domain_name': 'domain', 34 | 'project_domain_id': 'domain', 35 | 'tenant_name': 'project', 36 | 'container_name': "container" 37 | } 38 | } 39 | 40 | 41 | def auth_params(auth_config, exclude=None, **kwargs): 42 | """Appends auth parameters""" 43 | params = deepcopy(AUTH_PARAMETERS[auth_config]) 44 | if exclude: 45 | for name in exclude: 46 | del params[name] 47 | params.update(kwargs) 48 | return params 49 | 50 | 51 | def create_object(path, content_type='image/png', bytes=4096, 52 | hash='fcfc6539ce4e545ce58bafeeac3303a7', 53 | last_modified='2016-08-27T23:12:22.993170'): 54 | """Creates a fake swift object""" 55 | return { 56 | 'hash': hash, 57 | 'last_modified': last_modified, 58 | 'name': path, 59 | 'content_type': content_type, 60 | 'bytes': bytes, 61 | } 62 | 63 | # Files stored in the backend by default 64 | CONTAINER_FILES = [ 65 | 'root.txt', 66 | 'images/test.png', 67 | 'css/test.css', 68 | 'js/test.js', 69 | ] 70 | 71 | CONTAINER_CONTENTS = [create_object(path) for path in CONTAINER_FILES] 72 | 73 | 74 | def base_url(container=None, path=None): 75 | if container: 76 | try: 77 | path = urlparse.quote(path.encode('utf-8')) 78 | except (UnicodeDecodeError, AttributeError): 79 | pass 80 | return "{}/{}/{}".format(base_url(), container, path or '') 81 | return "{}/AUTH_{}".format(BASE_URL, TENANT_ID) 82 | 83 | 84 | class ClientException(Exception): 85 | pass 86 | 87 | 88 | class FakeSwift(object): 89 | ClientException = ClientException 90 | objects = CONTAINER_CONTENTS 91 | containers = ['container'] 92 | 93 | class Connection(object): 94 | service_token = None 95 | def __init__(self, authurl=None, user=None, key=None, retries=5, 96 | preauthurl=None, preauthtoken=None, snet=False, 97 | starting_backoff=1, max_backoff=64, tenant_name=None, 98 | os_options=None, auth_version="1", cacert=None, 99 | insecure=False, cert=None, cert_key=None, 100 | ssl_compression=True, retry_on_ratelimit=False, 101 | timeout=None, session=None): 102 | pass 103 | 104 | def _retry(self, reset_func, func, *args, **kwargs): 105 | self.url, self.token = self.get_auth() 106 | self.http_conn = None 107 | return func(self.url, self.token, *args, 108 | service_token=self.service_token, **kwargs) 109 | 110 | def get_auth(self): 111 | return base_url(), TOKEN 112 | 113 | def head_container(self, container, headers=None): 114 | return self._retry(None, FakeSwift.head_container, container, 115 | headers=headers) 116 | 117 | def put_container(self, container, headers=None, response_dict=None, 118 | query_string=None): 119 | return self._retry(None, FakeSwift.put_container, container, 120 | headers=headers, response_dict=response_dict, 121 | query_string=query_string) 122 | 123 | def head_object(self, container, obj, headers=None): 124 | return self._retry(None, FakeSwift.head_object, container, obj, 125 | headers=headers) 126 | 127 | def get_object(self, container, obj, resp_chunk_size=None, 128 | query_string=None, response_dict=None, headers=None): 129 | return self._retry(None, FakeSwift.get_object, container, obj, 130 | resp_chunk_size=resp_chunk_size, 131 | query_string=query_string, 132 | response_dict=response_dict, headers=headers) 133 | 134 | def get_container(self, container, marker=None, limit=None, prefix=None, 135 | delimiter=None, end_marker=None, path=None, 136 | full_listing=False, headers=None, query_string=None): 137 | return self._retry(None, FakeSwift.get_container, container, 138 | marker=marker, limit=limit, prefix=prefix, 139 | delimiter=delimiter, end_marker=end_marker, 140 | path=path, full_listing=full_listing, 141 | headers=headers, query_string=query_string) 142 | 143 | def delete_object(self, container, obj, query_string=None, 144 | response_dict=None, headers=None): 145 | return self._retry(None, FakeSwift.delete_object, container, obj, 146 | query_string=query_string, 147 | response_dict=response_dict, headers=headers) 148 | 149 | def put_object(self, container, obj, contents, content_length=None, 150 | etag=None, chunk_size=None, content_type=None, 151 | headers=None, query_string=None, response_dict=None): 152 | return self._retry(None, FakeSwift.put_object, container, obj, 153 | contents, content_length=content_length, 154 | etag=etag, chunk_size=chunk_size, 155 | content_type=content_type, headers=headers, 156 | query_string=query_string, 157 | response_dict=response_dict) 158 | 159 | @classmethod 160 | def get_auth(cls, auth_url, user, passwd, **kwargs): 161 | return base_url(), TOKEN 162 | 163 | @classmethod 164 | def http_connection(cls, *args, **kwargs): 165 | return FakeHttpConn() 166 | 167 | @classmethod 168 | def head_container(cls, url, token, container, **kwargs): 169 | if container not in FakeSwift.containers: 170 | raise ClientException 171 | 172 | @classmethod 173 | def put_container(cls, url, token, container, **kwargs): 174 | if container not in cls.containers: 175 | cls.containers.append(container) 176 | 177 | @classmethod 178 | def head_object(cls, url, token, container, name, **kwargs): 179 | for obj in FakeSwift.objects: 180 | if obj['name'] == name: 181 | object = deepcopy(obj) 182 | object['content-length'] = obj['bytes'] 183 | object['x-timestamp'] = '123456789' 184 | return object 185 | raise FakeSwift.ClientException 186 | 187 | @classmethod 188 | def get_object(cls, url, token, container, name, **kwargs): 189 | return None, bytearray(4096) 190 | 191 | @classmethod 192 | def get_container(cls, storage_url, token, container, **kwargs): 193 | """Returns a tuple: Response headers, list of objects""" 194 | return None, FakeSwift.objects 195 | 196 | @classmethod 197 | def delete_object(cls, url, token, container, name, **kwargs): 198 | for obj in FakeSwift.objects: 199 | if obj['name'] == name: 200 | FakeSwift.objects.remove(obj) 201 | return 202 | raise cls.ClientException 203 | 204 | @classmethod 205 | def put_object(cls, url, token, container, name=None, contents=None, 206 | http_conn=None, content_type=None, content_length=None, 207 | headers=None, **kwargs): 208 | if not name: 209 | raise ValueError("Attempting to add an object with no name/path") 210 | FakeSwift.objects.append(create_object(name)) 211 | 212 | 213 | class FakeHttpConn(object): 214 | pass 215 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = true 3 | args_are_paths = false 4 | envlist = 5 | flake8, 6 | isort, 7 | readme, 8 | py38-{3.2,4.1,4.2} 9 | 10 | [testenv] 11 | usedevelop = true 12 | basepython = 13 | py38: python3.8 14 | 15 | deps = 16 | coverage 17 | mock>=2.0.0 18 | 3.2: Django>=3.2,<3.3 19 | 4.1: Django>=4.1,<4.2 20 | 4.2: Django>=4.2,<4.3 21 | master: https://github.com/django/django/archive/master.tar.gz 22 | setenv = 23 | DJANGO_SETTINGS_MODULE=settings 24 | allowlist_externals = 25 | cd 26 | commands = 27 | cd tests && {envpython} -R -Wonce {envbindir}/coverage run manage.py test -v2 {posargs} 28 | coverage report 29 | [testenv:flake8] 30 | usedevelop = false 31 | basepython = python3.8 32 | commands = flake8 33 | deps = flake8 34 | 35 | [testenv:isort] 36 | usedevelop = false 37 | basepython = python3.8 38 | commands = isort --check-only --diff swift tests 39 | deps = isort==5.12.0 40 | 41 | [testenv:readme] 42 | usedevelop = false 43 | basepython = python3.8 44 | commands = python setup.py check -r -s 45 | deps = readme_renderer 46 | 47 | [flake8] 48 | show-source = True 49 | max-line-length = 100 50 | exclude = .env, env, .tox, tests 51 | --------------------------------------------------------------------------------