├── .gitignore ├── .readthedocs.yml ├── .travis.yml ├── LICENSE ├── Makefile ├── README.rst ├── django_selectel_storage ├── __init__.py ├── apps.py ├── compat.py ├── exceptions.py ├── selectel.py ├── storage.py └── utils.py ├── docs ├── Makefile ├── source │ ├── changelog.rst │ ├── conf.py │ ├── configuration.rst │ ├── index.rst │ ├── install.rst │ └── introduction.rst └── usage.rst ├── poetry.lock ├── pyproject.toml ├── setup.cfg ├── setup.py └── tests ├── conftest.py ├── storage_api_methods ├── test_delete.py ├── test_exists.py ├── test_list_dir.py ├── test_open.py ├── test_save.py ├── test_size.py └── test_url.py ├── test_selectel.py └── test_utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .devdata.env 3 | .idea/ 4 | *.pyc 5 | *.coverage 6 | htmlcov 7 | *.egg/ 8 | *.egg-info/ 9 | build 10 | dist 11 | .tox 12 | .pytest_cache 13 | .python-version 14 | docs/build 15 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sphinx: 4 | configuration: docs/source/conf.py 5 | 6 | formats: all 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | dist: xenial 3 | 4 | matrix: 5 | include: 6 | - { python: "2.7", env: TOXENV=py27-dj111 } 7 | 8 | - { python: "3.5", env: TOXENV=py35-dj111 } 9 | - { python: "3.5", env: TOXENV=py35-dj20 } 10 | - { python: "3.5", env: TOXENV=py35-dj21 } 11 | - { python: "3.5", env: TOXENV=py35-dj22 } 12 | 13 | - { python: "3.6", env: TOXENV=py36-dj111 } 14 | - { python: "3.6", env: TOXENV=py36-dj20 } 15 | - { python: "3.6", env: TOXENV=py36-dj21 } 16 | - { python: "3.6", env: TOXENV=py36-dj22 } 17 | - { python: "3.6", env: TOXENV=py36-dj30 } 18 | 19 | 20 | - { python: "3.7", env: TOXENV=py37-dj111 } 21 | - { python: "3.7", env: TOXENV=py37-dj20 } 22 | - { python: "3.7", env: TOXENV=py37-dj21 } 23 | - { python: "3.7", env: TOXENV=py37-dj22 } 24 | - { python: "3.7", env: TOXENV=py37-dj30 } 25 | 26 | - { python: "3.8", env: TOXENV=py38-dj111 } 27 | - { python: "3.8", env: TOXENV=py38-dj20 } 28 | - { python: "3.8", env: TOXENV=py38-dj21 } 29 | - { python: "3.8", env: TOXENV=py38-dj22 } 30 | - { python: "3.8", env: TOXENV=py38-dj30 } 31 | 32 | 33 | script: 34 | - tox -- --isort --flake8 --cov=. --cov-config setup.cfg --cov-report term-missing --cov-report term:skip-covered --cov-append --cov-branch 35 | 36 | 37 | install: 38 | - pip install -q tox tox-travis poetry tox-pyenv coveralls 39 | 40 | after_success: 41 | - coveralls 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 Mikhail Porokhovnichenko 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: check 2 | check: 3 | poetry build 4 | twine check dist/* 5 | twine upload --repository-url https://test.pypi.org/legacy/ dist/* 6 | 7 | .PHONY: release 8 | release: 9 | make check 10 | twine upload dist/* 11 | 12 | .PHONY: push 13 | push: 14 | git push origin master --tags 15 | 16 | 17 | .PHONY: patch 18 | patch: 19 | echo "Making a patch release" 20 | poetry run bump2version patch 21 | 22 | 23 | .PHONY: minor 24 | minor: 25 | echo "Making a minor release" 26 | poetry run bump2version minor 27 | 28 | 29 | .PHONY: major 30 | major: 31 | echo "Making a MAJOR release" 32 | poetry run bump2version major 33 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | django-selectel-storage 3 | ======================= 4 | 5 | 6 | 7 | .. image:: https://badge.fury.io/py/django-selectel-storage.svg 8 | :target: https://badge.fury.io/py/django-selectel-storage 9 | 10 | .. image:: https://img.shields.io/pypi/l/django-selectel-storage 11 | :target: https://raw.githubusercontent.com/marazmiki/django-selectel-storage/master/LICENSE 12 | :alt: The project license 13 | 14 | .. image:: https://travis-ci.org/marazmiki/django-selectel-storage.svg?branch=master 15 | :target: https://travis-ci.org/marazmiki/django-selectel-storage 16 | :alt: Travis CI build status 17 | 18 | .. image:: https://coveralls.io/repos/marazmiki/django-selectel-storage/badge.svg?branch=master 19 | :target: https://coveralls.io/r/marazmiki/django-selectel-storage?branch=master 20 | :alt: Code coverage percentage 21 | 22 | .. image:: https://pypip.in/wheel/django-selectel-storage/badge.svg 23 | :target: https://pypi.python.org/pypi/django-selectel-storage/ 24 | :alt: Wheel Status 25 | 26 | .. image:: https://img.shields.io/pypi/pyversions/django-selectel-storage.svg 27 | :target: https://img.shields.io/pypi/pyversions/django-selectel-storage.svg 28 | :alt: Supported Python versions 29 | 30 | .. image:: https://img.shields.io/pypi/djversions/django-selectel-storage.svg 31 | :target: https://pypi.org/project/django-selectel-storage/ 32 | :alt: Supported Django versions 33 | 34 | .. image:: https://readthedocs.org/projects/django-selectel-storage/badge/?version=latest 35 | :target: https://django-ulogin.readthedocs.io/ru/latest/?badge=latest 36 | :alt: Documentation Status 37 | 38 | .. image:: https://api.codacy.com/project/badge/Grade/f143275acdf249328a4968b62a94e100 39 | :alt: Codacy Badge 40 | :target: https://app.codacy.com/manual/marazmiki/django-selectel-storage?utm_source=github.com&utm_medium=referral&utm_content=marazmiki/django-selectel-storage&utm_campaign=Badge_Grade_Dashboard 41 | 42 | 43 | This application allows you easily save media and static files into Selectel cloud storage. 44 | 45 | 46 | Installation 47 | ------------ 48 | 49 | 1. Install the package 50 | 51 | .. code:: bash 52 | 53 | pip install django-selectel-storage 54 | 55 | 56 | 2. Add to your settings module: 57 | 58 | .. code:: python 59 | 60 | DEFAULT_FILE_STORAGE = 'django_selectel_storage.storage.SelectelStorage' 61 | SELECTEL_STORAGES = { 62 | 'default': { 63 | 'USERNAME': 'xxxx_user1', 64 | 'PASSWORD': 'secret', 65 | 'CONTAINER_NAME': 'bucket', 66 | }, 67 | 'yet-another-schema': { 68 | 'USERNAME': 'yyyy_user2', 69 | 'PASSWORD': 'mystery', 70 | 'CONTAINER_NAME': 'box', 71 | 72 | }, 73 | } 74 | 75 | Please see details in the `documentation `_. 76 | -------------------------------------------------------------------------------- /django_selectel_storage/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '1.0.2' 2 | 3 | default_app_config = ( 4 | 'django_selectel_storage.apps.' 5 | 'DjangoSelectelStorageAppConfig' 6 | ) 7 | -------------------------------------------------------------------------------- /django_selectel_storage/apps.py: -------------------------------------------------------------------------------- 1 | from django import apps 2 | from django.conf import settings 3 | from django.core.checks import Tags, Warning, register 4 | 5 | HINT = ( 6 | "Since 1.0, to improve the experience of multiple containers in the " 7 | "same project using, the settings format has been changed. For now, " 8 | "you should not use SELECTEL_USERNAME, SELECTEL_PASSWORD, " 9 | "SELECTEL_CONTAINER_NAME etc. settings. Consider using " 10 | "a SELECTEL_STORAGE dictionary instead, like DATABASES or CACHES ones. " 11 | ) 12 | 13 | 14 | class DjangoSelectelStorageAppConfig(apps.AppConfig): 15 | name = 'django_selectel_storage' 16 | 17 | 18 | @register(Tags.compatibility) 19 | def settings_compat_check(app_configs, **kwargs): 20 | 21 | if any(( 22 | hasattr(settings, 'SELECTEL_USERNAME'), 23 | hasattr(settings, 'SELECTEL_PASSWORD'), 24 | hasattr(settings, 'SELECTEL_CONTAINER_NAME'), 25 | )): 26 | return [ 27 | Warning( 28 | msg='Obsolete config format', 29 | hint=HINT, 30 | id='django_selectel_storage.W001', 31 | ) 32 | ] 33 | return [] 34 | -------------------------------------------------------------------------------- /django_selectel_storage/compat.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | PY2 = sys.version_info.major == 2 4 | PY3 = sys.version_info.major == 3 5 | 6 | try: 7 | TEXT_TYPE = unicode 8 | except NameError: 9 | TEXT_TYPE = str 10 | 11 | 12 | def b(s): 13 | return s.encode('latin-1') if PY3 else s 14 | -------------------------------------------------------------------------------- /django_selectel_storage/exceptions.py: -------------------------------------------------------------------------------- 1 | class SelectelException(ValueError): 2 | pass 3 | 4 | 5 | class InvalidSchema(SelectelException): 6 | pass 7 | 8 | 9 | class EmptyUsername(SelectelException): 10 | pass 11 | 12 | 13 | class EmptyPassword(SelectelException): 14 | pass 15 | 16 | 17 | class EmptyContainerName(SelectelException): 18 | pass 19 | -------------------------------------------------------------------------------- /django_selectel_storage/selectel.py: -------------------------------------------------------------------------------- 1 | import io 2 | from datetime import datetime, timedelta 3 | from logging import getLogger as get_logger 4 | 5 | import requests 6 | from django.utils.module_loading import import_string 7 | 8 | from .compat import TEXT_TYPE 9 | 10 | log = get_logger('selectel') 11 | now = datetime.now 12 | 13 | 14 | # get, put_stream, remove, info, list 15 | # info -- content-length 16 | class Auth: 17 | THRESHOLD = 300 18 | AUTH_URL = 'https://auth.selcdn.ru/' 19 | 20 | def __init__(self, config, requests): 21 | self.config = config 22 | self.requests = requests 23 | self.token = '' 24 | self.storage_url = '' 25 | self.expires = now() 26 | 27 | @property 28 | def user(self): 29 | return self.config['USERNAME'] 30 | 31 | @property 32 | def key(self): 33 | return self.config['PASSWORD'] 34 | 35 | @property 36 | def container_name(self): 37 | return self.config['CONTAINER'] 38 | 39 | def build_url(self, key): 40 | self.renew_if_need() 41 | return '{storage_url}/{name}/{key}'.format( 42 | storage_url=self.storage_url.rstrip('/'), 43 | name=self.container_name.strip('/'), 44 | key=key.lstrip('/'), 45 | ) 46 | 47 | def is_expired(self): 48 | return (self.expires - now()).total_seconds() < self.THRESHOLD 49 | 50 | def renew_if_need(self): 51 | if self.is_expired() or not self.storage_url: 52 | self.authenticate() 53 | 54 | def update_expires(self, delta): 55 | self.expires = now() + timedelta(seconds=int(delta)) 56 | 57 | def update_auth_token(self, new_token): 58 | self.requests.headers.update({'X-Auth-Token': new_token}) 59 | self.token = new_token 60 | 61 | def authenticate(self): 62 | log.debug('Need to authenticate with %s:%s', self.user, self.key) 63 | resp = self.requests.get(self.AUTH_URL, headers={ 64 | 'X-Auth-User': self.user, 65 | 'X-Auth-Key': self.key 66 | }) 67 | if resp.status_code != 204: 68 | log.debug('Got an unexpected response from auth: %s', resp.content) 69 | raise Exception("Selectel: Unexpected status code: %s" % 70 | resp.status_code) 71 | self.storage_url = resp.headers['X-Storage-Url'] 72 | self.update_expires(resp.headers['X-Expire-Auth-Token']) 73 | self.update_auth_token(resp.headers['X-Auth-Token']) 74 | 75 | def perform_request(self, http_method, key, 76 | raise_exception=False, **kwargs): 77 | self.renew_if_need() 78 | resp = getattr(self.requests, http_method)( 79 | self.build_url(key), 80 | **kwargs 81 | ) # type: requests.Response 82 | if resp.status_code == 401: 83 | log.debug('Got an unexpected 401 error, reauthenticate.') 84 | self.authenticate() 85 | return self.perform_request(http_method, key, raise_exception, 86 | **kwargs) 87 | if raise_exception: 88 | resp.raise_for_status() 89 | return resp 90 | 91 | 92 | class Container: 93 | def __init__(self, config): 94 | self.config = config 95 | self.requests = self.get_requests() # type: requests.Session 96 | self.auth = Auth(self.config, self.requests) 97 | 98 | @property 99 | def name(self): 100 | return self.auth.container_name 101 | 102 | def get_requests(self): 103 | return import_string(self.config.get( 104 | 'REQUESTS_FACTORY', 105 | 'django_selectel_storage.utils.requests_factory' 106 | ))(self.config) 107 | 108 | def build_url(self, key): 109 | return self.auth.build_url(key) 110 | 111 | def perform_request(self, http_method, key, 112 | raise_exception=False, **kwargs): 113 | return self.auth.perform_request(http_method, key, 114 | raise_exception=raise_exception, 115 | **kwargs) 116 | 117 | def open(self, key): 118 | return self.perform_request('get', key, raise_exception=True, 119 | stream=True).raw 120 | 121 | def save(self, key, content, metadata=None): 122 | self.perform_request('put', key, data=content, raise_exception=True) 123 | return key 124 | 125 | def delete(self, key): 126 | self.perform_request('delete', key, raise_exception=False) 127 | 128 | def size(self, key): 129 | try: 130 | resp = self.perform_request('head', key, raise_exception=True) 131 | return int(resp.headers['Content-Length']) 132 | except requests.exceptions.HTTPError: 133 | raise IOError('Failed to get file size of {}'.format(key)) 134 | 135 | def exists(self, key): 136 | return self.perform_request('head', key).status_code == 200 137 | 138 | def list(self, key): 139 | return { 140 | x['name']: x for x in self.perform_request('get', '', params={ 141 | 'format': 'json', 142 | 'prefix': key 143 | }).json() 144 | } 145 | 146 | def send_me_file(self, filename, size): 147 | headers = { 148 | 'Content-Type': 'x-storage/sendmefile+inplace', 149 | 'X-Object-Meta-Sendmefile-Max-Size': TEXT_TYPE(size), 150 | 'X-Object-Meta-Sendmefile-Disable-Web': 'yes', 151 | 'X-Object-Meta-Sendmefile-Allow-Overwrite': 'no', 152 | 'X-Object-Meta-Sendmefile-Ignore-Filename': 'yes', 153 | 'X-Filename': '/' + TEXT_TYPE(filename), 154 | } 155 | self.perform_request('put', filename, data=io.BytesIO(), 156 | headers=headers, raise_exception=True) 157 | return self.build_url(filename) 158 | -------------------------------------------------------------------------------- /django_selectel_storage/storage.py: -------------------------------------------------------------------------------- 1 | from django.core.files import base, storage 2 | 3 | from .selectel import Container 4 | from .utils import read_config 5 | 6 | 7 | class SelectelStorage(storage.Storage): 8 | def __init__(self, *args, **kwargs): 9 | self.config = read_config(args, kwargs) 10 | self.container = Container(self.config) 11 | 12 | def _open(self, name, mode='rb'): 13 | return base.ContentFile(self.container.open(name).read()) 14 | 15 | def _save(self, name, content): 16 | self.container.save(name, content, metadata=None) 17 | return name 18 | 19 | def delete(self, name): 20 | self.container.delete(name) 21 | 22 | def exists(self, name): 23 | return self.container.exists(name) 24 | 25 | def listdir(self, path): 26 | dirs, files = set(), set() 27 | results = self.container.list(path) 28 | for key, metadata in results.items(): 29 | bits = key[len(path):].lstrip('/').split('/') 30 | (dirs if len(bits) > 1 else files).add(bits[0]) 31 | return list(dirs), list(files) 32 | 33 | def size(self, name): 34 | return self.container.size(name) 35 | 36 | def url(self, name): 37 | if not self.config.get('CUSTOM_DOMAIN'): 38 | return self.container.build_url(name) 39 | else: 40 | custom_domain = self.config['CUSTOM_DOMAIN'].rstrip('/') 41 | if not custom_domain.lower().startswith(('http://', 'https://')): 42 | custom_domain = 'https://{0}'.format(custom_domain) 43 | return '{custom_domain}/{name}'.format( 44 | custom_domain=custom_domain, 45 | name=name.strip('/'), 46 | ) 47 | 48 | def save_with_metadata(self, name, content, metadata=None): 49 | self.container.save(name, content, metadata=metadata) 50 | return name 51 | 52 | 53 | class SelectelStaticStorage(SelectelStorage): 54 | pass 55 | -------------------------------------------------------------------------------- /django_selectel_storage/utils.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from django.conf import settings 3 | 4 | from .exceptions import ( 5 | EmptyContainerName, 6 | EmptyPassword, 7 | EmptyUsername, 8 | InvalidSchema, 9 | SelectelException, 10 | ) 11 | 12 | try: 13 | from urllib import parse as urlparse # PY3 14 | except ImportError: 15 | import urlparse # PY2 16 | 17 | 18 | MAX_RETRIES = 3 19 | POOL_CONNECTIONS = 50 20 | POOL_MAXSIZE = 50 21 | 22 | KNOWN_OPTS = ( 23 | 'max_retries', 24 | 'pool_conns', 25 | 'pool_maxsize' 26 | ) 27 | 28 | 29 | def extract(opts, key, default=None, cast=None): 30 | if key not in opts: 31 | return default 32 | return cast(opts[key][0]) if cast is not None else opts[key][0] 33 | 34 | 35 | def empty_username(): 36 | raise EmptyUsername('An username should be given to access the ' 37 | 'Selectel cloud storage') 38 | 39 | 40 | def empty_password(): 41 | raise EmptyPassword('A password should be given to access the' 42 | 'Selectel cloud storage') 43 | 44 | 45 | def empty_container_name(): 46 | raise EmptyContainerName('Looks like both username and password are ' 47 | 'given, but container name seems to be empty') 48 | 49 | 50 | def parse_dsn(string): 51 | bits = urlparse.urlparse(string) # type: urlparse.ParseResult 52 | 53 | if bits.scheme.lower() != 'selectel': 54 | raise InvalidSchema('The only supported schema here is "selectel://"') 55 | 56 | if not bits.hostname: 57 | empty_container_name() 58 | 59 | if bits.hostname and bits.netloc.count(':') == 1: 60 | if bits.password is None and bits.username is None: 61 | empty_container_name() 62 | 63 | if not bits.password: 64 | empty_password() 65 | 66 | if not bits.username: 67 | empty_username() 68 | 69 | if not bits.username: 70 | empty_username() 71 | 72 | if not bits.password: 73 | empty_password() 74 | 75 | username = bits.username 76 | password = bits.password 77 | 78 | custom_domain = bits.hostname 79 | container = bits.path.strip('/') 80 | 81 | if bits.path in ['', '/']: 82 | custom_domain = None 83 | container = bits.hostname 84 | 85 | options = urlparse.parse_qs(bits.query) 86 | 87 | return { 88 | 'USERNAME': username, 89 | 'PASSWORD': password, 90 | 'CONTAINER': container, 91 | 'CUSTOM_DOMAIN': custom_domain, 92 | 'OPTIONS': { 93 | 'MAX_RETRIES': extract( 94 | opts=options, 95 | key='max_retries', 96 | cast=int, 97 | default=MAX_RETRIES 98 | ), 99 | 'POOL_CONNECTIONS': extract( 100 | opts=options, 101 | key='pool_conns', 102 | cast=int, 103 | default=POOL_CONNECTIONS 104 | ), 105 | 'POOL_MAXSIZE': extract( 106 | opts=options, 107 | key='pool_maxsize', 108 | cast=int, 109 | default=POOL_MAXSIZE 110 | ), 111 | 'REQUESTS_FACTORY': extract( 112 | opts=options, 113 | key='requests_factory', 114 | cast=str, 115 | default='django_selectel_storage.utils.requests_factory' 116 | ) 117 | } 118 | } 119 | 120 | 121 | def read_config(args, kwargs): 122 | detected_schema = None 123 | 124 | # Case 1: Storage() with no args and kwargs 125 | if not args and not kwargs: 126 | detected_schema = 'default' 127 | 128 | # Case 2: the only positional argument: a data source string 129 | # like Storage("selectel://user:pass@name") or a schema name 130 | # like Storage("schema-name") 131 | if len(args) == 1 and not kwargs: 132 | try: 133 | return parse_dsn(args[0]) 134 | except SelectelException: 135 | detected_schema = args[0] 136 | 137 | # Case 3: Using the "default" schema when there is not "dsn=" keyword arg. 138 | if kwargs.get('dsn') is None and all(( 139 | 'container' not in kwargs, 140 | 'username' not in kwargs, 141 | 'password' not in kwargs 142 | )): 143 | detected_schema = args[0] if args else 'default' 144 | 145 | # Case 4: Storage(storage='default') default schema, the same as 1) 146 | if 'storage' in kwargs: 147 | detected_schema = kwargs['storage'] 148 | 149 | if detected_schema is not None: 150 | return settings.SELECTEL_STORAGES[detected_schema] 151 | 152 | # 5. Storage(dsn='selectel://user:pass@container') 153 | if 'dsn' in kwargs: 154 | return parse_dsn(kwargs['dsn']) 155 | 156 | if all(( 157 | 'container' in kwargs, 158 | 'username' in kwargs, 159 | 'password' in kwargs 160 | )): 161 | dsn_format = 'selectel://{username}:{password}@{container}' 162 | if 'custom_domain'in kwargs: 163 | dsn_format = ( 164 | 'selectel://{username}:{password}@{custom_domain}/{container}' 165 | ) 166 | if 'options' in kwargs: 167 | dsn_format += '/?' 168 | return parse_dsn(dsn_format.format(**kwargs)) 169 | # 7. Storage(container='container', user='...', password='', options=None) 170 | 171 | 172 | def requests_factory(config): 173 | return requests.Session() 174 | 175 | # def get_requests_adapter(self, **kwargs): 176 | # return requests.adapters.HTTPAdapter( 177 | # max_retries=setting('SELECTEL_MAX_RETRIES', 178 | # MAX_RETRIES), 179 | # pool_connections=setting('SELECTEL_POOL_CONNECTIONS', 180 | # POOL_CONNECTIONS), 181 | # pool_maxsize=setting('SELECTEL_POOL_MAXSIZE', 182 | # POOL_MAXSIZE)) 183 | # 184 | # def setup_requests_adapter(self, **kwargs): 185 | # adapter = self.get_requests_adapter(**kwargs) 186 | # self.mount_requests_adapter('http://', adapter) 187 | # self.mount_requests_adapter('https://', adapter) 188 | # 189 | # def mount_requests_adapter(self, prefix, adapter): 190 | # self.container.storage.session.mount(prefix, adapter) 191 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/source/changelog.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 1.x 5 | --- 6 | 7 | 1.0.2 8 | ----- 9 | 10 | * A new feature ``sendmefile`` implementation to make direct upload to a container. 11 | * Fixed an issue when authentication gets disappeared in some cases (#10, thanks to `@idealatom `_ for PR); 12 | * Fixed some misspells in docs; 13 | 14 | 1.0.1 15 | ----- 16 | 17 | * Fixed an issue when accessing a container without explicit authentication (thanks to `Alexey Kotenko `_ for reporting) 18 | * Added ``requests`` as a required dependency. 19 | 20 | 1.0 21 | ~~~ 22 | 23 | 24 | * Added support for Python ``3.6``, ``3.7`` and ``3.8``; in the opposite, dropped support for old ones (``3.4`` and lower) 25 | * Dropped support for older ``Django`` versions (the oldest one we supporting is ``1.10``) 26 | * A new configuration format allowing create a number of different schemas; 27 | * Get rid off the 3rd party dependency: ``selectel-api``; 28 | * Using ``tox`` and ``pytest`` utilities when developing and testing; 29 | * Using ``poetry`` as package management and deploying tool, so ``setup.py`` is no longer needed; 30 | * All the development utils configs (such as ``.rccoverage``, ``tox.ini`` and so on) also moved in the only ``setup.cfg`` file 31 | * License is MIT 32 | 33 | 34 | 0.3x 35 | ---- 36 | 37 | 0.3.1 38 | ~~~~~ 39 | 40 | Released at 2016-03-13 41 | 42 | * Fix static storage settings 43 | 44 | 0.3.0 45 | ~~~~~ 46 | 47 | Released at 2016-02-07 48 | 49 | * Drop older python verions support (vv 3.2, 3.3) 50 | * Drop older Django version support (1.6) 51 | * Use tox for testing 52 | 53 | 54 | 0.2x 55 | ---- 56 | 57 | 0.2.2 58 | ~~~~~ 59 | 60 | 2015-05-23 61 | 62 | * Update django head versions 63 | 64 | 0.2.1 65 | ~~~~~ 66 | 67 | 2015-05-02 68 | 69 | * Update head django version 70 | 71 | 72 | 0.2.0 73 | ~~~~~ 74 | 75 | 2015-04-04 76 | 77 | * 5ac502a 2014-10-03 | init? [Mikhail Porokhovnichenko] 78 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | 3 | import sys 4 | 5 | sys.path.insert(0, '../..') 6 | 7 | from django_selectel_storage import __version__ # isort:skip noqa: E731 8 | 9 | project = 'django-selectel-storage' 10 | copyright = '2018, Mikhail Porokhovnichenko' 11 | author = 'Mikhail Porokhovnichenko' 12 | 13 | # The short X.Y version 14 | version = __version__ 15 | 16 | release = __version__ 17 | 18 | 19 | # -- General configuration --------------------------------------------------- 20 | 21 | extensions = [ 22 | 'sphinx.ext.autodoc', 23 | 'sphinx.ext.doctest', 24 | ] 25 | 26 | templates_path = ['_templates'] 27 | 28 | source_suffix = '.rst' 29 | 30 | master_doc = 'index' 31 | 32 | # The language for content autogenerated by Sphinx. Refer to documentation 33 | # for a list of supported languages. 34 | # 35 | # This is also used if you do content translation via gettext catalogs. 36 | # Usually you set "language" from the command line for these cases. 37 | language = None 38 | 39 | # List of patterns, relative to source directory, that match files and 40 | # directories to ignore when looking for source files. 41 | # This pattern also affects html_static_path and html_extra_path. 42 | exclude_patterns = [] 43 | 44 | # The name of the Pygments (syntax highlighting) style to use. 45 | pygments_style = None 46 | 47 | 48 | # -- Options for HTML output ------------------------------------------------- 49 | 50 | # The theme to use for HTML and HTML Help pages. See the documentation for 51 | # a list of builtin themes. 52 | # 53 | html_theme = 'alabaster' 54 | 55 | # Theme options are theme-specific and customize the look and feel of a theme 56 | # further. For a list of options available for each theme, see the 57 | # documentation. 58 | # 59 | # html_theme_options = {} 60 | 61 | # Add any paths that contain custom static files (such as style sheets) here, 62 | # relative to this directory. They are copied after the builtin static files, 63 | # so a file named "default.css" will overwrite the builtin "default.css". 64 | html_static_path = ['_static'] 65 | 66 | # Custom sidebar templates, must be a dictionary that maps document names 67 | # to template names. 68 | # 69 | # The default sidebars (for documents that don't match any pattern) are 70 | # defined by theme itself. Builtin themes are using these templates by 71 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 72 | # 'searchbox.html']``. 73 | # 74 | # html_sidebars = {} 75 | 76 | 77 | # -- Options for HTMLHelp output --------------------------------------------- 78 | 79 | # Output file base name for HTML help builder. 80 | htmlhelp_basename = 'django-selectel-storagedoc' 81 | 82 | 83 | # -- Options for manual page output ------------------------------------------ 84 | 85 | # One entry per manual page. List of tuples 86 | # (source start file, name, description, authors, manual section). 87 | man_pages = [ 88 | (master_doc, 89 | 'django-selectel-storage', 90 | 'django-selectel-storage Documentation', 91 | [author], 1) 92 | ] 93 | 94 | 95 | # -- Options for Epub output ------------------------------------------------- 96 | 97 | # Bibliographic Dublin Core info. 98 | epub_title = project 99 | 100 | # The unique identifier of the text. This can be a ISBN number 101 | # or the project homepage. 102 | # 103 | # epub_identifier = '' 104 | 105 | # A unique identification for the text. 106 | # 107 | # epub_uid = '' 108 | 109 | # A list of files that should not be packed into the epub file. 110 | epub_exclude_files = ['search.html'] 111 | 112 | # -- Extension configuration ------------------------------------------------- 113 | -------------------------------------------------------------------------------- /docs/source/configuration.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | Here, we assume you already: 5 | 6 | * `Have registered `_ an account on Selectel, 7 | * `Spent some money `_ there, 8 | * `Created a couple of containers `_ and got credentials for it. 9 | 10 | if you haven't yet, consider doing that right now. 11 | 12 | 13 | A minimal installation 14 | ---------------------- 15 | 16 | To begin using Selectel storage as your storage, you should configure 17 | your containers credentials: 18 | 19 | .. code:: python 20 | 21 | SELECTEL_STORAGES = { 22 | 'default': { 23 | 'USERNAME': 'xxxx_user', 24 | 'PASSWORD': 'p455w0rd', 25 | 'CONTAINER': 'my-data', 26 | }, 27 | 'yet_another_storage': 'selectel://xxxx_user:p455w0rd2@another-container', 28 | } 29 | 30 | As you already guessed, the dictionary means you can create as much configurations 31 | as you want. Here, we have two schemas: ``default`` and ``yet_another_storage``. 32 | 33 | 34 | **You're free to use any names for schema but strongly recommended to declare 35 | a schema with name ``default``.** 36 | 37 | Also, as you already guessed again, you can specify credentials either: 38 | 39 | * As a dictionary with ``USERNAME``, ``PASSWORD`` or ``CONTAINER`` keys, or 40 | * a URL-like string with schema ``selectel://`` 41 | 42 | Please see all available configuration options below. 43 | 44 | .. attention:: 45 | Credentials in examples above are hardcoded. It really sucks, don't 46 | do that in real life. Instead, you can store these variables in the 47 | environment and extract it out of there, for example, via awesome 48 | `python-decouple `_ package. 49 | 50 | 51 | Using django-selectel-storage as a default backend 52 | -------------------------------------------------- 53 | 54 | If you want to use the storage by default, consider adding to your ``settings.py``: 55 | 56 | .. code:: python 57 | 58 | DEFAULT_FILE_STORAGE = 'django_selectel_storage.storage.SelectelStorage' 59 | 60 | in this case, the ``default`` schema will be implicitly used. Please make sure 61 | you defined it before. 62 | 63 | 64 | Storage initialization 65 | ---------------------- 66 | 67 | If you want to explicitly instantiate a storage, you should write something like that: 68 | 69 | .. code:: python 70 | 71 | # models.py 72 | from django.db import models 73 | 74 | class Post(models.Model): 75 | photo = models.ImageField(storage=storage_instance) 76 | 77 | Wondering, what ``storage_instance`` is? Well, it would be one of: 78 | 79 | .. code:: python 80 | 81 | from django_selectel_storage.storage import SelectelStorage 82 | 83 | storage_instance = SelectelStorage() 84 | storage_instance = SelectelStorage('default') 85 | storage_instance = SelectelStorage(storage='default') 86 | 87 | Please pay attention: the last three lines of code do the same thing: creates 88 | a ``SelectelStorage`` instance with using ``default`` schema. Of course, you 89 | can choose any different schema (if you've defined it of course): 90 | 91 | .. code:: python 92 | 93 | storage_instance = SelectelStorage('yet_another_storage') 94 | storage_instance = SelectelStorage(storage='yet_another_storage') 95 | 96 | 97 | We even create an instance via DSN (it's useful somewhere in ``./manage.py shell``, please don't do that in production): 98 | 99 | .. code:: python 100 | 101 | storage_instance = SelectelStorage('selectel://user:password@container_name/') 102 | storage_instance = SelectelStorage(storage='selectel://user:password@container_name/') 103 | 104 | An interesting gotcha here: if you specify a DSN (a string begins with ``selectel://``) as a first positional argument, it would 105 | act like a dsn rather as ``storage=`` 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. 2 | 3 | django-selectel-storage 4 | ####################### 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | :caption: Contents: 9 | 10 | introduction 11 | install 12 | configuration 13 | changelog 14 | 15 | 16 | Indices and tables 17 | ================== 18 | 19 | * :ref:`genindex` 20 | * :ref:`modindex` 21 | * :ref:`search` 22 | -------------------------------------------------------------------------------- /docs/source/install.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | 5 | Get a stable version 6 | -------------------- 7 | 8 | with ``pip``: 9 | 10 | .. code:: bash 11 | 12 | $ pip install django-selectel-storage 13 | 14 | or via `pipenv `_: 15 | 16 | .. code:: bash 17 | 18 | $ pipenv install django-selectel-storage 19 | 20 | 21 | or even via `poetry `_: 22 | 23 | .. code:: bash 24 | 25 | $ poetry add django-selectel-storage 26 | 27 | 28 | Get a developer version 29 | ----------------------- 30 | 31 | Just install it from github also via ``pip`` 32 | 33 | .. code:: bash 34 | 35 | $ pip install -e git+https://github.com/marazmiki/django-selectel-storage#egg=django-selectel-storage 36 | 37 | You even get a code of mandatory git revision: 38 | 39 | .. code:: bash 40 | 41 | $ pip install -e git+https://github.com/marazmiki/django-selectel-storage@{REV_HASH}#egg=django-selectel-storage 42 | 43 | -------------------------------------------------------------------------------- /docs/source/introduction.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | What is "django-selectel-storage"? 5 | ---------------------------------- 6 | 7 | It's an implementation of Django Storage API to seamless store all the data inside the Selectel Cloud Storage. 8 | 9 | What is "Django"? 10 | ----------------- 11 | 12 | It's funny. Of course, you know Django. And know about storage API in Django. Nothing to say here. 13 | 14 | 15 | What is "Selectel"? 16 | ------------------- 17 | 18 | Selectel is one of the leading Russian internet hosting operators. Besides a huge number of useful services it provided, would like to say a couple words about Cloud Storage. 19 | It does the same as Amazon's S3, but simpler and cheaper... a bit. 20 | 21 | Why "Selectel" cloud? 22 | --------------------- 23 | 24 | Well, clouds are so cute. Always use it. Also, if your business is located in Russia, you should store sensitive privacy data in Russia, according to some stupid legal acts. Probably, Selectel Cloud Storage is the best choice, because it's relatively cheap and reliable. 25 | 26 | 27 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | 5 | When using Django, in most cases, you don't need to use storage API 6 | directly (of course, you can although), only through models, when 7 | using ``FileField`` and ``ImageField.`` Like that: 8 | 9 | .. code:: python 10 | 11 | # models.py 12 | class Photo(models.Model): 13 | file = models.ImageField() 14 | 15 | # somewhere else. Here, "photo" is a Photo instance. 16 | # you of course get it. 17 | with open('photo.jpg', 'rb') as fp: 18 | photo = Photo.objects.first() 19 | photo.save() 20 | 21 | 22 | This code works regardless of a storage backend 23 | 24 | 25 | Instancing 26 | ---------- 27 | 28 | 29 | .. code:: python 30 | 31 | class Photo(models.Model): 32 | photo = models.ImageField(storage=selectel_storage) 33 | 34 | "But wait, what is ``selectel_storage`` here?" –– you ask. It's a good question. 35 | 36 | 37 | 38 | .. code:: python 39 | 40 | # or 41 | photo = models.ImageField(storage=SelectelStorage(schema='default')) 42 | 43 | # or even: 44 | photo = models.ImageField(storage=SelectelStorage( 45 | username='', 46 | password='', 47 | container='', 48 | )) 49 | 50 | photo = models.ImageField(storage=SelectelStorage( 51 | 'selectel://user:password@container-name/' 52 | )) 53 | 54 | 55 | If you want to use all your data in the cloud, the better choice to make 56 | the selectel storage default one (see configuration). In this case, you 57 | can define your model this way: 58 | 59 | 60 | .. code:: python 61 | 62 | from django.db import models 63 | 64 | class Photo(models.Model): 65 | photo = models.ImageField() # no storage instance here 66 | -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- 1 | [[package]] 2 | category = "dev" 3 | description = "Version-bump your software with a single command!" 4 | name = "bump2version" 5 | optional = false 6 | python-versions = "*" 7 | version = "0.5.10" 8 | 9 | [[package]] 10 | category = "main" 11 | description = "Python package for providing Mozilla's CA Bundle." 12 | name = "certifi" 13 | optional = false 14 | python-versions = "*" 15 | version = "2020.4.5.1" 16 | 17 | [[package]] 18 | category = "main" 19 | description = "Universal encoding detector for Python 2 and 3" 20 | name = "chardet" 21 | optional = false 22 | python-versions = "*" 23 | version = "3.0.4" 24 | 25 | [[package]] 26 | category = "main" 27 | description = "Internationalized Domain Names in Applications (IDNA)" 28 | name = "idna" 29 | optional = false 30 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 31 | version = "2.9" 32 | 33 | [[package]] 34 | category = "main" 35 | description = "Python HTTP for Humans." 36 | name = "requests" 37 | optional = false 38 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" 39 | version = "2.23.0" 40 | 41 | [package.dependencies] 42 | certifi = ">=2017.4.17" 43 | chardet = ">=3.0.2,<4" 44 | idna = ">=2.5,<3" 45 | urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" 46 | 47 | [package.extras] 48 | security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] 49 | socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] 50 | 51 | [[package]] 52 | category = "main" 53 | description = "HTTP library with thread-safe connection pooling, file post, and more." 54 | name = "urllib3" 55 | optional = false 56 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" 57 | version = "1.25.9" 58 | 59 | [package.extras] 60 | brotli = ["brotlipy (>=0.6.0)"] 61 | secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"] 62 | socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] 63 | 64 | [metadata] 65 | content-hash = "07ff98130dfbc14818efb5912fec1072209513db513ab153d4db970cc7d7f3c3" 66 | python-versions = "^2.7 || ^3.5 || ^3.6 || ^3.7 || ^3.8" 67 | 68 | [metadata.files] 69 | bump2version = [ 70 | {file = "bump2version-0.5.10-py2.py3-none-any.whl", hash = "sha256:185abfd0d8321ec5059424d8b670aa82f7385948ff7ddd986981b4ed04dc819a"}, 71 | ] 72 | certifi = [ 73 | {file = "certifi-2020.4.5.1-py2.py3-none-any.whl", hash = "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304"}, 74 | {file = "certifi-2020.4.5.1.tar.gz", hash = "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"}, 75 | ] 76 | chardet = [ 77 | {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, 78 | {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, 79 | ] 80 | idna = [ 81 | {file = "idna-2.9-py2.py3-none-any.whl", hash = "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"}, 82 | {file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"}, 83 | ] 84 | requests = [ 85 | {file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"}, 86 | {file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"}, 87 | ] 88 | urllib3 = [ 89 | {file = "urllib3-1.25.9-py2.py3-none-any.whl", hash = "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"}, 90 | {file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"}, 91 | ] 92 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "django-selectel-storage" 3 | version = "1.0.2" 4 | description = "A Django storage backend allowing you to easily save user-generated and static files inside Selectel Cloud storage rather than a local filesystem, as Django does by default." 5 | authors = [ 6 | "Mikhail Porokhovnichenko " 7 | ] 8 | license = "MIT" 9 | readme = "README.rst" 10 | homepage = "https://github.com/marazmiki/django-selectel-storage" 11 | repository = "https://github.com/marazmiki/django-selectel-storage" 12 | keywords = ["django", "selectel", "storage", "remote storage", "http"] 13 | 14 | classifiers = [ 15 | "Development Status :: 5 - Production/Stable", 16 | "Environment :: Web Environment", 17 | "Framework :: Django", 18 | "Framework :: Django :: 1.11", 19 | "Framework :: Django :: 2.0", 20 | "Framework :: Django :: 2.1", 21 | "Framework :: Django :: 2.2", 22 | "Framework :: Django :: 3.0", 23 | "License :: OSI Approved :: BSD License", 24 | "Programming Language :: Python :: 2", 25 | "Programming Language :: Python :: 2.7", 26 | "Programming Language :: Python :: 3", 27 | "Programming Language :: Python :: 3.5", 28 | "Programming Language :: Python :: 3.6", 29 | "Programming Language :: Python :: 3.7", 30 | "Programming Language :: Python :: 3.8", 31 | "Topic :: Internet :: WWW/HTTP", 32 | "Topic :: Internet :: WWW/HTTP :: Dynamic Content", 33 | ] 34 | 35 | 36 | 37 | [tool.poetry.dependencies] 38 | python = "^2.7 || ^3.5 || ^3.6 || ^3.7 || ^3.8" 39 | requests = "^2.23.0" 40 | 41 | [tool.poetry.dev-dependencies] 42 | bump2version = "^0" 43 | 44 | [build-system] 45 | requires = ["poetry>=0.12"] 46 | build-backend = "poetry.masonry.api" 47 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 1.0.2 3 | commit = True 4 | tag = True 5 | 6 | [metadata] 7 | name = django-selectel-storage 8 | version = 1.0.0 9 | license = MIT 10 | license_file = LICENSE 11 | long_description = file: README.rst 12 | long_description_content_type = text/x-rst 13 | 14 | [options] 15 | install_requires = 16 | requests 17 | packages = find: 18 | 19 | [bdist_wheel] 20 | universal = 1 21 | 22 | [tox:tox] 23 | isolated_build = True 24 | envlist = 25 | py27-dj{111} 26 | py3{5,6,7,8}-dj{111,20,21,22,30} 27 | basepython = 28 | py27: python2.7 29 | py35: python3.5 30 | py36: python3.6 31 | py37: python3.7 32 | py38: python3.8 33 | 34 | [tox:.package] 35 | basepython = python3 36 | 37 | [testenv] 38 | commands = 39 | poetry install 40 | pytest -s {posargs} 41 | whitelist_externals = 42 | poetry 43 | pytest 44 | passenv = 45 | SELECTEL_USERNAME 46 | SELECTEL_CONTAINER_NAME 47 | SELECTEL_PASSWORD 48 | deps = 49 | pytest 50 | pytest-django 51 | pytest-cov 52 | pytest-isort 53 | pytest-flake8 54 | dj111: django>=1.11,<2 55 | dj20: django>=2.0,<2.1 56 | dj21: django>=2.1,<2.2 57 | dj22: django>=2.2,<2.3 58 | dj30: django>=3.0,<3.1 59 | 60 | [coverage:run] 61 | source = django_selectel_storage 62 | branch = True 63 | omit = 64 | .tox/* 65 | 66 | [coverage:report] 67 | exclude_lines = 68 | if __name__ == .__main__.: 69 | omit = 70 | .tox/* 71 | 72 | [isort] 73 | multi_line_output = 3 74 | include_trailing_comma = true 75 | sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER 76 | default_section = FIRSTPARTY 77 | 78 | [bumpversion:file:setup.cfg] 79 | search = version = {current_version} 80 | replace = version = {new_version} 81 | 82 | [bumpversion:file:pyproject.toml] 83 | search = version = "{current_version}" 84 | replace = version = "{new_version}" 85 | 86 | [bumpversion:file:django_selectel_storage/__init__.py] 87 | search = __version__ = '{current_version}' 88 | replace = __version__ = '{new_version}' 89 | 90 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | setup() 6 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import uuid 3 | 4 | import pytest 5 | 6 | 7 | def pytest_configure(): 8 | """ 9 | Create an MVP of a django test project 10 | """ 11 | from django.conf import settings 12 | 13 | username = os.getenv('SELECTEL_USERNAME') 14 | password = os.getenv('SELECTEL_PASSWORD') 15 | container = os.getenv('SELECTEL_CONTAINER_NAME') 16 | 17 | settings.configure( 18 | INSTALLED_APPS=['django_selectel_storage'], 19 | DATABASES={ 20 | 'default': { 21 | 'ENGINE': 'django.db.backends.sqlite3', 22 | 'NAME': ':MEMORY:' 23 | } 24 | }, 25 | SELECTEL_STORAGES={ 26 | 'default': { 27 | 'USERNAME': username, 28 | 'PASSWORD': password, 29 | 'CONTAINER': container, 30 | }, 31 | 'customized': { 32 | 'USERNAME': username, 33 | 'PASSWORD': password, 34 | 'CONTAINER': container, 35 | }, 36 | 'static': 'selectel://user' 37 | }, 38 | ) 39 | 40 | 41 | @pytest.fixture(autouse=True) 42 | def autouse_db(db): 43 | """ 44 | Makes your test environment to initialize a database 45 | connection automatically for each test case 46 | """ 47 | pass 48 | 49 | 50 | @pytest.fixture 51 | def selectel_storage(settings): 52 | """ 53 | Creates a ``SelectelStorage`` instance to be used in test cases 54 | """ 55 | from django_selectel_storage import storage 56 | return storage.SelectelStorage() 57 | 58 | 59 | @pytest.fixture 60 | def create_file(selectel_storage): 61 | """ 62 | Creates a file with a unique prefix in the Selectel Cloud Storage 63 | container and then deletes it (the file) after a test case finished 64 | """ 65 | from django.core.files.base import ContentFile 66 | from django_selectel_storage.compat import PY3, TEXT_TYPE 67 | 68 | created_records = [] 69 | 70 | def file_creator(filename, content=b'', prefix=''): 71 | if all(( 72 | PY3, 73 | isinstance(content, TEXT_TYPE) 74 | )): 75 | content = content.encode('UTF-8') 76 | container = str(uuid.uuid4()) 77 | key = os.path.join(prefix.lstrip('/') or container, filename) 78 | selectel_storage.save(key, ContentFile(content, key)) 79 | created_records.append(key) 80 | return key 81 | 82 | yield file_creator 83 | 84 | for key in created_records: 85 | selectel_storage.delete(key) 86 | 87 | 88 | @pytest.fixture 89 | def empty_gif(): 90 | """A 1x1 GIF image""" 91 | return ( 92 | b'\x47\x49\x46\x38\x39\x61\x01\x00\x01\x00\xf0\x01\x00' 93 | b'\xff\xff\xff\x00\x00\x00\x21\xf9\x04\x01\x0a\x00\x00' 94 | b'\x00\x2c\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02' 95 | b'\x44\x01\x00\x3b' 96 | ) 97 | 98 | 99 | @pytest.fixture 100 | def lazy_fox(): 101 | """Just a text string""" 102 | return 'The *quick* brown fox jumps over the lazy dog' 103 | -------------------------------------------------------------------------------- /tests/storage_api_methods/test_delete.py: -------------------------------------------------------------------------------- 1 | import os 2 | import uuid 3 | 4 | 5 | def test_not_failed_when_deleting_a_non_existing_file(selectel_storage): 6 | non_existing_file = os.path.join(str(uuid.uuid4()), 'non-exists.txt') 7 | selectel_storage.delete(non_existing_file) 8 | 9 | 10 | def test_delete_actually_works(selectel_storage, create_file): 11 | existing_file = create_file('DELETE_ME.txt', 'Please, delete me!') 12 | 13 | assert selectel_storage.exists(existing_file) 14 | 15 | selectel_storage.delete(existing_file) 16 | 17 | assert not selectel_storage.exists(existing_file) 18 | -------------------------------------------------------------------------------- /tests/storage_api_methods/test_exists.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | 4 | def test_exists_returns_false_when_the_file_does_not_exist(selectel_storage): 5 | non_existing_file = '{0}/non-exist.txt'.format(uuid.uuid4()) 6 | assert not selectel_storage.exists(non_existing_file) 7 | 8 | 9 | def test_exists_returns_true_when_the_file_exists( 10 | selectel_storage, 11 | create_file 12 | ): 13 | existing_file = create_file('exists.txt', 'Yup, it\'s exists!') 14 | assert selectel_storage.exists(existing_file) 15 | -------------------------------------------------------------------------------- /tests/storage_api_methods/test_list_dir.py: -------------------------------------------------------------------------------- 1 | import os 2 | import uuid 3 | 4 | import pytest 5 | 6 | 7 | def test_listdir_not_found(selectel_storage): 8 | assert selectel_storage.listdir('_this_dir_does_not_exist/') == ([], []) 9 | 10 | 11 | @pytest.mark.parametrize( 12 | argnames='var_name, expected_items', 13 | argvalues=[ 14 | pytest.param('files', {'file.img', 'hello.pdf'}, id='files'), 15 | pytest.param('dirs', {'hello'}, id='dirs - it\'s always empty'), 16 | ] 17 | ) 18 | def test_listdir_works( 19 | selectel_storage, 20 | create_file, 21 | var_name, 22 | expected_items 23 | ): 24 | prefix = '{0}/listdir'.format(uuid.uuid4()) 25 | root = 'test-list/' 26 | 27 | for f in ['file.img', 'hello.pdf', 'hello/image.png', 'hello/text.txt']: 28 | create_file(root + f, prefix=prefix) 29 | 30 | dirs, files = selectel_storage.listdir(os.path.join(prefix, root)) 31 | assert {_ for _ in locals()[var_name]} == expected_items 32 | -------------------------------------------------------------------------------- /tests/storage_api_methods/test_open.py: -------------------------------------------------------------------------------- 1 | def test_get_zero_length_str(selectel_storage, create_file): 2 | file = create_file('zero.txt', content='') 3 | content = selectel_storage._open(file, 'r') 4 | assert content.read() == b'' 5 | 6 | 7 | def test_get_zero_length_byte(selectel_storage, create_file): 8 | file = create_file('zero.txt', content=b'') 9 | 10 | content = selectel_storage._open(file, 'r') 11 | assert content.read() == b'' 12 | 13 | 14 | def test_get_binary_file(selectel_storage, create_file, empty_gif): 15 | file = create_file('empty.gif', content=empty_gif) 16 | assert selectel_storage._open(file, 'rb').read() == empty_gif 17 | 18 | 19 | def test_get_text_mode(selectel_storage, create_file, lazy_fox): 20 | file = create_file('file.txt', content=lazy_fox) 21 | assert selectel_storage._open(file, 'r').read() == lazy_fox.encode('UTF-8') 22 | -------------------------------------------------------------------------------- /tests/storage_api_methods/test_save.py: -------------------------------------------------------------------------------- 1 | import io 2 | import tempfile 3 | import uuid 4 | 5 | import pytest 6 | from django.core.files import uploadedfile 7 | 8 | from django_selectel_storage.compat import b 9 | 10 | 11 | @pytest.fixture 12 | def file_id(): 13 | return str(uuid.uuid4()) 14 | 15 | 16 | @pytest.fixture 17 | def simple_file(): 18 | def inner(filename, content): 19 | fp = tempfile.NamedTemporaryFile(delete=False) 20 | fp.write(content) 21 | fp.flush() 22 | return fp 23 | return inner 24 | 25 | 26 | @pytest.fixture 27 | def in_memory_file(): 28 | def inner(filename, content): 29 | return uploadedfile.InMemoryUploadedFile( 30 | file=io.BytesIO(content), 31 | field_name='test_field', 32 | name='_save_new_file.txt', 33 | content_type='text/plain', 34 | size=0, 35 | charset='utf8' 36 | ) 37 | return inner 38 | 39 | 40 | @pytest.fixture 41 | def temporary_uploaded_file(): 42 | def inner(filename, content): 43 | fp = uploadedfile.TemporaryUploadedFile( 44 | name=filename + '.tempfile', 45 | content_type='text/plain', 46 | size=0, 47 | charset='utf8', 48 | ) 49 | fp.write(content) 50 | fp.flush() 51 | return fp 52 | return inner 53 | 54 | 55 | parametrize = pytest.mark.parametrize( 56 | argnames='fixture_name', 57 | argvalues=[ 58 | 'simple_file', 59 | 'in_memory_file', 60 | 'temporary_uploaded_file', 61 | ] 62 | ) 63 | 64 | 65 | @parametrize 66 | def test_save_fileobj(selectel_storage, request, file_id, fixture_name): 67 | filename = 'save_new_file_{0}.txt'.format(file_id) 68 | content = b('test content one') 69 | 70 | fileobj = request.getfixturevalue(fixture_name)(filename, content) 71 | selectel_storage.save(filename, fileobj) 72 | 73 | with selectel_storage.open(filename) as f: 74 | f.seek(0) 75 | assert f.read() == content 76 | 77 | 78 | @parametrize 79 | def test_save_seeked_fileobj(selectel_storage, request, file_id, fixture_name): 80 | filename = 'save_new_file_{0}.txt'.format(file_id) 81 | content = b('test content one') 82 | 83 | fileobj = request.getfixturevalue(fixture_name)(filename, content) 84 | fileobj.seek(0) 85 | 86 | selectel_storage.save(filename, fileobj) 87 | 88 | with selectel_storage.open(filename) as f: 89 | assert f.read() == content 90 | -------------------------------------------------------------------------------- /tests/storage_api_methods/test_size.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def test_error_when_trying_to_get_size_of_non_existing_file(selectel_storage): 5 | with pytest.raises(IOError): 6 | selectel_storage.size('non-exists.txt') 7 | 8 | 9 | def test_zero_size_file(selectel_storage, create_file): 10 | file = create_file('zero_size.txt', content='') 11 | assert selectel_storage.size(file) == 0 12 | 13 | 14 | def test_size_binary_file(selectel_storage, create_file, empty_gif): 15 | file = create_file('empty.gif', content=empty_gif) 16 | assert selectel_storage.size(file) == len(empty_gif) 17 | 18 | 19 | def test_size_text_file(selectel_storage, create_file, lazy_fox): 20 | file = create_file('README.md', content=lazy_fox) 21 | assert selectel_storage.size(file) == len(lazy_fox) 22 | -------------------------------------------------------------------------------- /tests/storage_api_methods/test_url.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from django_selectel_storage import storage 6 | 7 | 8 | @pytest.fixture 9 | def url(): 10 | return os.getenv('SELECTEL_CONTAINER_URL', 'http://127.0.0.1/') 11 | 12 | 13 | @pytest.mark.parametrize( 14 | argnames='appendix, arg', 15 | argvalues=[ 16 | ('/index.html', 'index.html'), 17 | ], 18 | ) 19 | def test_url(selectel_storage, appendix, arg): 20 | assert selectel_storage.url('').rstrip('/') + appendix \ 21 | == selectel_storage.url(arg) 22 | 23 | 24 | @pytest.mark.parametrize( 25 | argnames='appendix', 26 | argvalues=[ 27 | '', 28 | '/', 29 | '////' 30 | ], 31 | ids=[ 32 | 'no trailing slashes', 33 | 'one trailing slash', 34 | 'multiple trailing slashes', 35 | ] 36 | ) 37 | def test_get_base_url_trailing_slashes(url, settings, appendix): 38 | settings.SELECTEL_STORAGES['default']['CUSTOM_DOMAIN'] = url 39 | sel_storage = storage.SelectelStorage() 40 | assert url == sel_storage.url('' + appendix) 41 | 42 | 43 | @pytest.mark.parametrize('custom_domain, expected_url', [ 44 | pytest.param('one.com', 'https://one.com/photo.jpg', id='no schema'), 45 | pytest.param('http://two.com', 'http://two.com/photo.jpg', id='http'), 46 | pytest.param('https://three/', 'https://three/photo.jpg', id='https'), 47 | ]) 48 | def test_get_base_url_custom_domain(custom_domain, expected_url, settings): 49 | settings.SELECTEL_STORAGES['default']['CUSTOM_DOMAIN'] = custom_domain 50 | assert expected_url == storage.SelectelStorage().url('photo.jpg') 51 | -------------------------------------------------------------------------------- /tests/test_selectel.py: -------------------------------------------------------------------------------- 1 | import io 2 | import uuid 3 | 4 | import pytest 5 | import requests 6 | 7 | from django_selectel_storage import selectel, utils 8 | 9 | 10 | @pytest.mark.parametrize( 11 | argnames='delta, expected', 12 | argvalues=[ 13 | (-10, True), 14 | (0, True), 15 | (10, True), 16 | (selectel.Auth.THRESHOLD + 10, False), 17 | ], 18 | ids=[ 19 | 'a negative timedelta', 20 | 'a nullable timedelta', 21 | 'a positive timedelta less than a thresh hold value', 22 | 'a positive timedelta greather than a thresh hold value' 23 | ] 24 | ) 25 | def test_auth_is_expired(delta, expected): 26 | auth = selectel.Auth(requests=None, config=None) 27 | auth.update_expires(delta) 28 | assert auth.is_expired() is expected 29 | 30 | 31 | @pytest.mark.parametrize( 32 | argnames='delta, expected', 33 | argvalues=[ 34 | (-10, True), 35 | (0, True), 36 | (10, True), 37 | (selectel.Auth.THRESHOLD + 10, False), 38 | (None, True), 39 | ], 40 | ids=[ 41 | 'a negative timedelta', 42 | 'a nullable timedelta', 43 | 'a positive timedelta less than a thresh hold value', 44 | 'a positive timedelta greater than a thresh hold value', 45 | 'empty storage_url' 46 | ] 47 | ) 48 | def test_auth_renew_if_expired(delta, expected, monkeypatch): 49 | def patched_authenticate(self): 50 | raise SystemError() 51 | 52 | monkeypatch.setattr(selectel.Auth, 'authenticate', patched_authenticate) 53 | 54 | auth = selectel.Auth(requests=None, config=None) 55 | auth.storage_url = 'https://example.com/' 56 | 57 | if delta is None: 58 | auth.storage_url = '' 59 | else: 60 | auth.update_expires(delta) 61 | 62 | if expected: 63 | with pytest.raises(SystemError): 64 | auth.renew_if_need() 65 | else: 66 | auth.renew_if_need() 67 | 68 | 69 | def test_build_url(monkeypatch): 70 | def patched_authenticate(self): 71 | self.storage_url = 'https://example.com/' 72 | self.update_expires(100500) 73 | 74 | auth = selectel.Auth(requests=None, config={ 75 | 'CONTAINER': 'bucket' 76 | }) 77 | monkeypatch.setattr(selectel.Auth, 'authenticate', patched_authenticate) 78 | 79 | auth.renew_if_need() 80 | assert ( 81 | auth.build_url('index.html') == 'https://example.com/bucket/index.html' 82 | ) 83 | 84 | 85 | def test_send_me_file(): 86 | container = selectel.Container( 87 | config=utils.read_config([], {}) 88 | ) 89 | identifier = str(uuid.uuid4()) 90 | 91 | filename = 'send_me_file-' + identifier + '.txt' 92 | contents = b'This is content of ' + filename.encode() 93 | 94 | public_upload_link = container.send_me_file(filename, len(contents)) 95 | 96 | assert public_upload_link == container.build_url(filename) 97 | 98 | resp = requests.put(public_upload_link, io.BytesIO(contents)) 99 | assert resp.status_code == 201 100 | 101 | resp = requests.get(public_upload_link) 102 | assert resp.content == contents 103 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pytest import param as p 3 | 4 | from django_selectel_storage import exceptions as exs 5 | from django_selectel_storage.storage import SelectelStorage 6 | from django_selectel_storage.utils import parse_dsn 7 | 8 | OPTS = { 9 | 'MAX_RETRIES': 3, 10 | 'POOL_CONNECTIONS': 50, 11 | 'POOL_MAXSIZE': 50, 12 | 'REQUESTS_FACTORY': 'django_selectel_storage.utils.requests_factory', 13 | } 14 | 15 | 16 | @pytest.mark.parametrize('dsn, exc', [ 17 | p('s3://', exs.InvalidSchema, 18 | id='invalid schema'), 19 | p('selectel://', exs.EmptyContainerName, 20 | id='empty credentials'), 21 | p('selectel://container', exs.EmptyUsername, 22 | id='no username and password provided'), 23 | p('selectel://user@container', exs.EmptyPassword, 24 | id='no password provided'), 25 | p('selectel://user:@container', exs.EmptyPassword, 26 | id='an empty password provided'), 27 | p('selectel://:password@container', exs.EmptyUsername, 28 | id='an empty username provided'), 29 | p('selectel://user:password', exs.EmptyContainerName, 30 | id='a container name not provided'), 31 | p('selectel://user:password@', exs.EmptyContainerName, 32 | id='a container name is empty'), 33 | ]) 34 | def test_parse_dsn_exceptions(dsn, exc): 35 | with pytest.raises(exc) as actual_ex: 36 | parse_dsn(dsn) 37 | assert isinstance(actual_ex, exc) 38 | 39 | 40 | @pytest.mark.parametrize('dsn, expected_pairs', [ 41 | p('selectel://usr:pwd@container', { 42 | 'USERNAME': 'usr', 43 | 'PASSWORD': 'pwd', 44 | 'CONTAINER': 'container'}, 45 | id='no trailing slashes'), 46 | p('selectel://john:example@container/', { 47 | 'USERNAME': 'john', 48 | 'PASSWORD': 'example', 49 | 'CONTAINER': 'container'}, 50 | id='there is a trailing slash'), 51 | p('selectel://john:example@custom.domain.com/my-container', { 52 | 'USERNAME': 'john', 53 | 'PASSWORD': 'example', 54 | 'CONTAINER': 'my-container', 55 | 'CUSTOM_DOMAIN': 'custom.domain.com' 56 | }, id='a custom domain given') 57 | ]) 58 | def test_parse_dsn(dsn, expected_pairs): 59 | results = parse_dsn(dsn) 60 | for k, v in expected_pairs.items(): 61 | assert results[k] == v 62 | 63 | 64 | @pytest.mark.parametrize('args, kwargs, expected', [ 65 | pytest.param([], {}, 'default', 66 | id='no args presented'), 67 | pytest.param(['default'], {}, 'default', 68 | id='the only positional arg looking like a schema name'), 69 | pytest.param(['customized'], {}, 'customized', 70 | id='the only positional arg is a non-default schema name'), 71 | pytest.param([], {'storage': 'default'}, 'default', 72 | id='the only keyword argument storage='), 73 | # pytest.param(['selectel://user:pass@container'], {}, 'default', 74 | # id='the only positional arg looking like a DSN'), 75 | pytest.param([], {'dsn': 'selectel://user:pass@container'}, { 76 | 'CONTAINER': 'container', 77 | 'USERNAME': 'user', 78 | 'PASSWORD': 'pass', 79 | 'CUSTOM_DOMAIN': None, 80 | 'OPTIONS': OPTS, 81 | }, 82 | id='the keyword argument dsn='), 83 | pytest.param([], {'dsn': 'selectel://user:pass@ex.com/container'}, { 84 | 'CONTAINER': 'container', 85 | 'USERNAME': 'user', 86 | 'PASSWORD': 'pass', 87 | 'CUSTOM_DOMAIN': 'ex.com', 88 | 'OPTIONS': OPTS, 89 | }, 90 | id='the keyword argument dsn= with a custom domain'), 91 | pytest.param([], {'container': 'container-mega', 92 | 'username': 'user', 93 | 'password': 'pass'}, { 94 | 'CONTAINER': 'container-mega', 95 | 'USERNAME': 'user', 96 | 'PASSWORD': 'pass', 97 | 'CUSTOM_DOMAIN': None, 98 | 'OPTIONS': OPTS, 99 | }, 100 | id='a few keyword arguments'), 101 | ]) 102 | def test_parse_config(settings, args, kwargs, expected): 103 | settings.SELECTEL_STORAGES = { 104 | 'default': { 105 | 'USERNAME': 'john', 106 | 'PASSWORD': 'secret', 107 | 'CONTAINER': 'container-1', 108 | }, 109 | 'customized': { 110 | 'USERNAME': 'jane', 111 | 'PASSWORD': 'secret2', 112 | 'CONTAINER': 'container-2', 113 | }, 114 | 'static': 'selectel://user' 115 | } 116 | storage = SelectelStorage(*args, **kwargs) 117 | conf = storage.config 118 | 119 | if isinstance(expected, str): 120 | assert conf == settings.SELECTEL_STORAGES[expected] 121 | else: 122 | assert conf == expected 123 | 124 | # 1. Storage() # no args 125 | # 2. Storage('default') # the same as storage=default 126 | # 3. Storage(storage='default') # default schema, the same as 1) 127 | # 4. Storage('selectel://user:pass@container') # the same as 5) 128 | # 5. Storage(dsn='selectel://user:pass@container') 129 | # 6. Storage(dsn='selectel://user:pass@example.com/container') 130 | # 7. Storage(container='container', user='...', password='', options=None) 131 | --------------------------------------------------------------------------------