├── localdevstorage ├── tests │ ├── __init__.py │ ├── test_settings.py │ └── test_http.py ├── models.py ├── __init__.py ├── base.py ├── http.py └── sftp.py ├── .gitignore ├── MANIFEST.in ├── setup.cfg ├── .coveragerc ├── CHANGES.rst ├── .travis.yml ├── LICENSE ├── setup.py └── README.rst /localdevstorage/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | .coverage 4 | htmlcov -------------------------------------------------------------------------------- /localdevstorage/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | -------------------------------------------------------------------------------- /localdevstorage/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __version__ = '0.5' 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include CHANGES.rst 4 | include MANIFEST.in 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pytest] 2 | DJANGO_SETTINGS_MODULE = localdevstorage.tests.test_settings 3 | 4 | [bdist_wheel] 5 | universal=1 6 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = 3 | localdevstorage 4 | branch = False 5 | # sftp backend is unsupported / untested 6 | omit = 7 | localdevstorage/tests/* 8 | localdevstorage/sftp.py 9 | 10 | [report] 11 | exclude_lines = 12 | __version__ = 13 | pragma: no cover 14 | def __repr__ 15 | raise AssertionError 16 | raise NotImplementedError 17 | if __name__ == .__main__.: -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 0.1 5 | --- 6 | 7 | First release with HTTP backend 8 | 9 | 0.2 10 | --- 11 | 12 | * SFTP backend, code mostly lifted from the SFTP backend from django-storages 13 | * fixed a bug that made it impossible to upload new files when using the dev storage 14 | 15 | 0.3 16 | --- 17 | * use the right way (tm) to calculate the URL of the remote file. This includes 18 | a switch from ``LOCALDEVSTORAGE_HTTP_FALLBACK_URL`` to 19 | ``LOCALDEVSTORAGE_HTTP_FALLBACK_DOMAIN``. The former will be removed in a 20 | future version, probably 0.4. 21 | 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - 3.4 5 | - 3.3 6 | - 2.7 7 | - 2.6 8 | 9 | sudo: false 10 | 11 | env: 12 | matrix: 13 | - DJANGO=1.6.11 14 | - DJANGO=1.7.8 15 | - DJANGO=1.8.1 16 | 17 | install: 18 | - pip install Django==$DJANGO pytest pytest-django requests responses pytest-cov cov-core coverage python-coveralls 19 | 20 | script: 21 | python setup.py test -a "--cov-config .coveragerc --cov localdevstorage" 22 | 23 | after_success: coveralls --config_file=.coveragerc 24 | 25 | matrix: 26 | exclude: 27 | - python: 2.6 28 | env: DJANGO=1.7.8 29 | - python: 2.6 30 | env: DJANGO=1.8.1 31 | - python: 3.3 32 | env: DJANGO=1.6.11 33 | - python: 3.4 34 | env: DJANGO=1.6.11 -------------------------------------------------------------------------------- /localdevstorage/tests/test_settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | TEST_DIR = os.path.abspath(os.path.dirname(__file__)) 4 | MEDIA_ROOT = TEST_DIR 5 | 6 | DATABASES = { 7 | 'default': { 8 | 'ENGINE': 'django.db.backends.sqlite3', 9 | 'NAME': ':memory:', 10 | } 11 | } 12 | 13 | INSTALLED_APPS = [ 14 | 'django.contrib.staticfiles', 15 | ] 16 | 17 | STATICFILES_FINDERS = [ 18 | 'django.contrib.staticfiles.finders.FileSystemFinder', 19 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 20 | ] 21 | 22 | STATIC_URL = '/static/' 23 | 24 | 25 | STATIC_ROOT = os.path.join(TEST_DIR, 'static') 26 | 27 | TEMPLATE_DIRS = ( 28 | # Specifically choose a name that will not be considered 29 | # by app_directories loader, to make sure each test uses 30 | # a specific template without considering the others. 31 | os.path.join(TEST_DIR, 'test_templates'), 32 | ) 33 | 34 | 35 | SECRET_KEY = "test" 36 | 37 | PASSWORD_HASHERS = ( 38 | 'django.contrib.auth.hashers.UnsaltedMD5PasswordHasher', 39 | ) 40 | 41 | MIDDLEWARE_CLASSES = [] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, Benjamin Wohlwend 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of django-localdevstorage nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL JONAS OBRIST BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /localdevstorage/base.py: -------------------------------------------------------------------------------- 1 | import os 2 | from django.core.files.storage import FileSystemStorage 3 | 4 | try: 5 | FileNotFoundError 6 | except: 7 | FileNotFoundError = IOError 8 | 9 | 10 | class BaseStorage(FileSystemStorage): 11 | def _open(self, name, mode='rb'): 12 | try: 13 | return super(BaseStorage, self)._open(name, mode) 14 | except FileNotFoundError: 15 | if 'w' in mode: # if writing, make sure the parent structure exists 16 | self._ensure_directory(name) 17 | 18 | try: 19 | try: 20 | f = self._get(name) 21 | except IOError: 22 | # if the underlying file doesn't exist, no matter. 23 | pass 24 | else: 25 | # if it does, write the contents locally 26 | self._write(f, name) 27 | 28 | except Exception: 29 | pass 30 | return super(BaseStorage, self)._open(name, mode) 31 | 32 | def _exists_locally(self, name): 33 | return super(BaseStorage, self).exists(name) 34 | 35 | def exists(self, name): 36 | if self._exists_locally(name): 37 | return True 38 | return self._exists_upstream(name) 39 | 40 | def _ensure_directory(self, name): 41 | dirname = os.path.dirname(self.path(name)) 42 | if not os.path.exists(dirname): 43 | os.makedirs(dirname) 44 | 45 | def _write(self, filelike, name): 46 | self._ensure_directory(name) 47 | f = open(self.path(name), mode='wb') 48 | f.write(filelike.read()) 49 | 50 | def _fetch_local(self, name, force=False): 51 | if self._exists_locally(name) and not force: 52 | return 53 | 54 | return self._write(self._get(name), name) 55 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from setuptools import setup, find_packages 4 | from localdevstorage import __version__ as version 5 | 6 | from setuptools.command.test import test as TestCommand 7 | 8 | 9 | class PyTest(TestCommand): 10 | user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] 11 | 12 | def initialize_options(self): 13 | TestCommand.initialize_options(self) 14 | self.pytest_args = [] 15 | 16 | def finalize_options(self): 17 | TestCommand.finalize_options(self) 18 | self.test_args = [] 19 | self.test_suite = True 20 | 21 | def run_tests(self): 22 | #import here, cause outside the eggs aren't loaded 23 | import pytest 24 | errno = pytest.main(self.pytest_args) 25 | sys.exit(errno) 26 | 27 | 28 | setup( 29 | name = 'django-localdevstorage', 30 | version = version, 31 | description = 'A Django storage backend for local development that downloads files from the live site on the fly.', 32 | author = 'Benjamin Wohlwend', 33 | author_email = 'piquadrat@gmail.com', 34 | url = 'https://github.com/piquadrat/django-localdevstorage', 35 | packages = find_packages(), 36 | zip_safe=False, 37 | include_package_data = True, 38 | install_requires=[ 39 | 'Django>=1.2', 40 | 'requests>=1.0', 41 | ], 42 | tests_require=['pytest', 'pytest-django', 'responses'], 43 | cmdclass={'test': PyTest}, 44 | classifiers = [ 45 | "Development Status :: 3 - Alpha", 46 | "Framework :: Django", 47 | "Intended Audience :: Developers", 48 | "License :: OSI Approved :: BSD License", 49 | "Operating System :: OS Independent", 50 | "Programming Language :: Python :: 2.6", 51 | "Programming Language :: Python :: 2.7", 52 | "Programming Language :: Python :: 3.3", 53 | "Programming Language :: Python :: 3.4", 54 | "Topic :: Software Development", 55 | "Topic :: Internet :: WWW/HTTP :: Dynamic Content", 56 | ] 57 | ) 58 | -------------------------------------------------------------------------------- /localdevstorage/tests/test_http.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | from django.conf import settings 5 | from django.test import TestCase 6 | 7 | from localdevstorage.http import HttpStorage 8 | 9 | import responses 10 | 11 | DOMAIN = 'https://example.com' 12 | 13 | 14 | class HttpStorageTest(TestCase): 15 | def tearDown(self): 16 | test_dir = os.path.join(settings.TEST_DIR, 'foo') 17 | if os.path.exists(test_dir): 18 | shutil.rmtree(test_dir) 19 | 20 | 21 | @responses.activate 22 | def test_http_fallback(self): 23 | responses.add( 24 | responses.GET, 25 | DOMAIN + '/foo/test_fallback', 26 | body='foo', status=200, 27 | content_type='text/plain' 28 | ) 29 | storage = HttpStorage(fallback_domain=DOMAIN) 30 | f = storage.open('foo/test_fallback') 31 | self.assertIn('foo', str(f.read())) 32 | assert len(responses.calls) == 1 33 | 34 | @responses.activate 35 | def test_exists(self): 36 | responses.add( 37 | responses.HEAD, 38 | DOMAIN + '/foo/test_exists', 39 | body='foo', status=200, 40 | content_type='text/plain' 41 | ) 42 | responses.add( 43 | responses.HEAD, 44 | DOMAIN + '/foo/test_does_not_exist', 45 | body='foo', status=404, 46 | content_type='text/plain' 47 | ) 48 | storage = HttpStorage(fallback_domain=DOMAIN) 49 | self.assertTrue(storage.exists('foo/test_exists')) 50 | self.assertFalse(storage.exists('foo/test_does_not_exist')) 51 | 52 | @responses.activate 53 | def test_http_auth(self): 54 | responses.add( 55 | responses.GET, 56 | DOMAIN + '/foo/test_auth', 57 | body='foo', status=200, 58 | content_type='text/plain' 59 | ) 60 | with self.settings( 61 | LOCALDEVSTORAGE_HTTP_PASSWORD='pw', 62 | LOCALDEVSTORAGE_HTTP_USERNAME='user' 63 | ): 64 | storage = HttpStorage(fallback_domain='https://example.com') 65 | f = storage.open('foo/test_auth') 66 | self.assertIn('Authorization', responses.calls[0].request.headers) 67 | -------------------------------------------------------------------------------- /localdevstorage/http.py: -------------------------------------------------------------------------------- 1 | try: 2 | from urllib.parse import urljoin 3 | except ImportError: 4 | # Python 2 fallbacks 5 | from urlparse import urljoin 6 | try: 7 | FileNotFoundError 8 | except NameError: 9 | FileNotFoundError = IOError 10 | from io import BytesIO 11 | import requests 12 | import warnings 13 | 14 | from django.core.exceptions import ImproperlyConfigured 15 | from django.conf import settings 16 | from django.utils.encoding import filepath_to_uri 17 | 18 | from localdevstorage.base import BaseStorage 19 | 20 | 21 | class HttpStorage(BaseStorage): 22 | def __init__(self, location=None, base_url=None, fallback_url=None, fallback_domain=None): 23 | self.fallback_url = fallback_url or getattr(settings, 'LOCALDEVSTORAGE_HTTP_FALLBACK_URL', None) 24 | if self.fallback_url: 25 | warnings.warn('fallback_url and LOCALDEVSTORAGE_HTTP_FALLBACK_URL have been replaced by fallback_domain and LOCALDEVSTORAGE_HTTP_FALLBACK_DOMAIN, respectively, and will be removed in a future release.') 26 | self.fallback_domain = fallback_domain or getattr(settings, 'LOCALDEVSTORAGE_HTTP_FALLBACK_DOMAIN', None) 27 | if not (self.fallback_url or self.fallback_domain): 28 | raise ImproperlyConfigured('please define LOCALDEVSTORAGE_HTTP_FALLBACK_DOMAIN in your settings') 29 | self.session = requests.Session() 30 | username = getattr(settings, 'LOCALDEVSTORAGE_HTTP_PASSWORD', None) 31 | password = getattr(settings, 'LOCALDEVSTORAGE_HTTP_USERNAME', None) 32 | if username and password: 33 | self.session.auth = (username, password) 34 | super(BaseStorage, self).__init__(location, base_url) 35 | 36 | def _exists_upstream(self, name): 37 | try: 38 | response = self.session.head(self._path(name)) 39 | return response.status_code == 200 40 | except FileNotFoundError: 41 | return False 42 | 43 | def _url(self, name): 44 | return urljoin('/', filepath_to_uri(name)) 45 | 46 | def _path(self, name): 47 | if self.fallback_domain: 48 | return urljoin(self.fallback_domain, self._url(name)) 49 | return self.fallback_url + name 50 | 51 | def _get(self, name): 52 | response = self.session.get(self._path(name)) 53 | if response.status_code != 200: 54 | raise FileNotFoundError() 55 | return BytesIO(response.content) 56 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | django-localdevstorage 3 | ====================== 4 | 5 | .. image:: https://travis-ci.org/piquadrat/django-localdevstorage.svg?branch=master 6 | :target: http://travis-ci.org/piquadrat/django-localdevstorage 7 | 8 | .. image:: https://coveralls.io/repos/piquadrat/django-localdevstorage/badge.svg?branch=master 9 | :target: https://coveralls.io/r/piquadrat/django-localdevstorage?branch=master 10 | 11 | .. image:: https://img.shields.io/pypi/v/django-localdevstorage.svg 12 | :target: https://pypi.python.org/pypi/django-localdevstorage/ 13 | 14 | django-localdevstorage is a set of storage backends that helps during 15 | development. Instead of having to copy all user generated media from 16 | the live site for local development, the storage backends provided by 17 | django-localdevstorage will download media files that are not available 18 | locally "on demand". 19 | 20 | Installation 21 | ============ 22 | 23 | Set one of the provided storage backends in your ``settings.py``. These 24 | are: 25 | 26 | * HTTP: ``DEFAULT_FILE_STORAGE = 'localdevstorage.http.HttpStorage'`` 27 | * (more will follow) 28 | 29 | .. note:: 30 | If you use `django-filer`_ 0.9 or higher, you have to make sure that 31 | ``localdevstorage`` is *not* used as the thumbnail storage, e.g. by 32 | adding this to your settings:: 33 | 34 | FILER_STORAGES = { 35 | 'public': { 36 | 'thumbnails': { 37 | 'ENGINE': 'django.core.files.storage.FileSystemStorage', 38 | 'OPTIONS': {}, 39 | 'THUMBNAIL_OPTIONS': { 40 | 'base_dir': 'filer_public_thumbnails', 41 | }, 42 | }, 43 | }, 44 | } 45 | 46 | 47 | HTTP 48 | ---- 49 | 50 | Set the fallback domain that should be used to fetch missing files. This 51 | is usually the protocol (http or https) and the domain your live site:: 52 | 53 | LOCALDEVSTORAGE_HTTP_FALLBACK_DOMAIN = 'http://www.example.com/' 54 | 55 | .. note:: 56 | Earlier versions of this library used ``LOCALDEVSTORAGE_HTTP_FALLBACK_URL``. 57 | While this still works, it is recommended to update your settings to the 58 | new name. ``LOCALDEVSTORAGE_HTTP_FALLBACK_URL`` will be removed in a future 59 | version. 60 | 61 | If your server is secured with HTTP basic auth, you can provide a username and 62 | password:: 63 | 64 | LOCALDEVSTORAGE_HTTP_USERNAME = 'foo' 65 | LOCALDEVSTORAGE_HTTP_PASSWORD = 'bar' 66 | 67 | SFTP 68 | ---- 69 | 70 | There are three settings that need to be configured for the SFTP backend: 71 | 72 | * ``LOCALDEVSTORAGE_SFTP_USER`` 73 | * ``LOCALDEVSTORAGE_SFTP_HOST`` 74 | * ``LOCALDEVSTORAGE_SFTP_ROOT_PATH``: this should be the ``MEDIA_ROOT`` 75 | on the remote machine in most cases. 76 | 77 | .. note:: 78 | * The SFTP backend is much slower than the HTTP backend, which you should 79 | use in most cases. The SFTP backend is only really useful if your 80 | media files are not directly accessible through unauthenticated HTTP. 81 | * because the SFTP backend can't prompt for a password, make sure that 82 | a connection can be established through public key exchange. 83 | 84 | .. warning:: 85 | Although we took special care not to do anything destructive on the 86 | remote machine, bugs in our code or in upstream libraries can always 87 | happen. This alone should be reason enough to use the HTTP backend in 88 | almost all cases, since it is, by definition, read only. 89 | 90 | Caveats 91 | ======= 92 | 93 | * Since django-localdevstorage extends a Django storage backend 94 | (``FileSystemStorage`` to be precise), only code that uses Django's 95 | file storage abstraction works with django-localdevstorage. Code that 96 | bypasses Django and accesses files directly will not benefit. 97 | 98 | 99 | .. _django-filer: https://github.com/stefanfoulis/django-filer 100 | -------------------------------------------------------------------------------- /localdevstorage/sftp.py: -------------------------------------------------------------------------------- 1 | # This is mostly lifted from django-storages' sftp backend: Their license: 2 | # 3 | # SFTP storage backend for Django. 4 | # Author: Brent Tubbs 5 | # License: MIT 6 | # 7 | # Modeled on the FTP storage by Rafal Jonca 8 | from __future__ import print_function 9 | try: 10 | import ssh 11 | except ImportError: 12 | import paramiko as ssh 13 | 14 | import os 15 | import posixpath 16 | import warnings 17 | 18 | from django.conf import settings 19 | from django.core.files.base import File 20 | try: 21 | from io import StringIO 22 | except ImportError: 23 | # Python 2 fallbacks 24 | from cStringIO import StringIO 25 | 26 | from localdevstorage.base import BaseStorage 27 | 28 | 29 | class SftpStorage(BaseStorage): 30 | def __init__(self, location=None, base_url=None, user=None, host=None, root_path=None): 31 | warnings.warn( 32 | 'The SFTP backend is unsupported and untested. ' 33 | 'Usage is not recommended!' 34 | ) 35 | self._host = host or settings.LOCALDEVSTORAGE_SFTP_HOST 36 | self._root_path = root_path or settings.LOCALDEVSTORAGE_SFTP_ROOT_PATH 37 | 38 | # if present, settings.SFTP_STORAGE_PARAMS should be a dict with params 39 | # matching the keyword arguments to paramiko.SSHClient().connect(). So 40 | # you can put username/password there. Or you can omit all that if 41 | # you're using keys. 42 | self._params = getattr(settings, 'SFTP_STORAGE_PARAMS', {}) 43 | self._params['username'] = user or settings.LOCALDEVSTORAGE_SFTP_USER 44 | 45 | # for now it's all posix paths. Maybe someday we'll support figuring 46 | # out if the remote host is windows. 47 | self._pathmod = posixpath 48 | super(SftpStorage, self).__init__(location, base_url) 49 | 50 | def _connect(self): 51 | self._ssh = ssh.SSHClient() 52 | 53 | # automatically add host keys from current user. 54 | self._ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts"))) 55 | 56 | # and automatically add new host keys for hosts we haven't seen before. 57 | self._ssh.set_missing_host_key_policy(ssh.AutoAddPolicy()) 58 | 59 | try: 60 | self._ssh.connect(self._host, **self._params) 61 | except ssh.AuthenticationException as e: 62 | raise 63 | except Exception as e: 64 | print(e) 65 | 66 | if not hasattr(self, '_sftp'): 67 | self._sftp = self._ssh.open_sftp() 68 | 69 | @property 70 | def sftp(self): 71 | """Lazy SFTP connection""" 72 | if not hasattr(self, '_sftp'): 73 | self._connect() 74 | return self._sftp 75 | 76 | def _get(self, name): 77 | try: 78 | return SFTPStorageFile(name, self, 'rb') 79 | except IOError: 80 | pass 81 | 82 | def _exists_upstream(self, name): 83 | try: 84 | f = SFTPStorageFile(name, self, 'rb') 85 | f.close() 86 | return True 87 | except Exception: 88 | return False 89 | 90 | def _read(self, name): 91 | remote_path = self._remote_path(name) 92 | return self.sftp.open(remote_path, 'rb') 93 | 94 | def _remote_path(self, name): 95 | return self._join(self._root_path, name) 96 | 97 | def _join(self, *args): 98 | # Use the path module for the remote host type to join a path together 99 | return self._pathmod.join(*args) 100 | 101 | 102 | class SFTPStorageFile(File): 103 | def __init__(self, name, storage, mode): 104 | self._name = name 105 | self._storage = storage 106 | self._mode = mode 107 | self._is_dirty = False 108 | self.file = StringIO() 109 | self._is_read = False 110 | 111 | @property 112 | def size(self): 113 | if not hasattr(self, '_size'): 114 | self._size = self._storage.size(self._name) 115 | return self._size 116 | 117 | def read(self, num_bytes=None): 118 | if not self._is_read: 119 | self.file = self._storage._read(self._name) 120 | self._is_read = True 121 | 122 | return self.file.read(num_bytes) 123 | 124 | def write(self, content): 125 | raise NotImplementedError 126 | 127 | def close(self): 128 | if self._is_dirty: 129 | self._storage._save(self._name, self.file.getvalue()) 130 | self.file.close() 131 | --------------------------------------------------------------------------------