├── offsite_storage ├── __init__.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── collectstatic.py ├── storages │ ├── __init__.py │ └── s3.py └── settings.py ├── .gitignore ├── README.md ├── setup.py └── LICENSE /offsite_storage/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /offsite_storage/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /offsite_storage/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /offsite_storage/storages/__init__.py: -------------------------------------------------------------------------------- 1 | from .s3 import CachedS3FilesStorage, S3MediaStorage 2 | -------------------------------------------------------------------------------- /offsite_storage/management/commands/collectstatic.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.contrib.staticfiles.management.commands import collectstatic 4 | 5 | from ... import settings 6 | 7 | 8 | class Command(collectstatic.Command): 9 | 10 | def set_options(self, **options): 11 | super(Command, self).set_options(**options) 12 | self.ignore_patterns += settings.IGNORE_FILES 13 | -------------------------------------------------------------------------------- /offsite_storage/settings.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | AWS_ACCESS_KEY_ID = getattr(settings, 'AWS_ACCESS_KEY_ID') 5 | AWS_SECRET_ACCESS_KEY = getattr(settings, 'AWS_SECRET_ACCESS_KEY') 6 | AWS_STATIC_BUCKET_NAME = getattr(settings, 'AWS_STATIC_BUCKET_NAME') 7 | 8 | AWS_MEDIA_ACCESS_KEY_ID = getattr( 9 | settings, 'AWS_MEDIA_ACCESS_KEY_ID', AWS_ACCESS_KEY_ID) 10 | AWS_MEDIA_SECRET_ACCESS_KEY = getattr( 11 | settings, 'AWS_MEDIA_SECRET_ACCESS_KEY', AWS_SECRET_ACCESS_KEY) 12 | AWS_MEDIA_BUCKET_NAME = getattr(settings, 'AWS_MEDIA_BUCKET_NAME') 13 | 14 | AWS_S3_ENDPOINT = getattr( 15 | settings, 'AWS_S3_ENDPOINT', 's3.amazonaws.com') 16 | AWS_HOST_URL = 'https://%%(bucket_name)s.%s/' % AWS_S3_ENDPOINT 17 | AWS_POLICY = 'public-read' 18 | 19 | IGNORE_FILES = getattr(settings, 'OFFSITE_STORAGE_IGNORE_FILES', 20 | ['*.less', '*.scss', '*.txt', 'components']) 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | 56 | # IDE 57 | .idea 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # django-offsite-storage 2 | Cloud static and media file storage suitable for app containers 3 | 4 | 5 | ## Installation 6 | 7 | Install the package via `pip`: 8 | 9 | pip install django-offsite-storage 10 | 11 | 12 | Then add `'offsite_storage'` to your `INSTALLED_APPS` just before `'django.contrib.staticfiles'`. 13 | 14 | `django-offsite-storage` overrides default `collectstatic` command to exclude unnecessary files (`less`, `scss`) from the process. 15 | 16 | ## Amazon S3 17 | 18 | Set following settings in your project's settings: 19 | 20 | * AWS_ACCESS_KEY_ID 21 | * AWS_SECRET_ACCESS_KEY 22 | * AWS_STATIC_BUCKET_NAME 23 | * STATICFILES_STORAGE = 'offsite_storage.storages.CachedS3FilesStorage' 24 | 25 | If you intend to use an external storage for your `media files`, set the following settings: 26 | 27 | * AWS_MEDIA_BUCKET_NAME 28 | * DEFAULT_FILE_STORAGE = 'offsite_storage.storages.S3MediaStorage' 29 | * THUMBNAIL_DEFAULT_STORAGE = DEFAULT_FILE_STORAGE 30 | 31 | In the case when you use a different s3 account for your media files, set the following settings: 32 | 33 | * AWS_MEDIA_ACCESS_KEY_ID 34 | * AWS_MEDIA_SECRET_ACCESS_KEY 35 | 36 | To force boto to use a particular S3 host set AWS_S3_ENDPOINT (default is `s3.amazonaws.com`). 37 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | from setuptools import setup, find_packages 3 | 4 | 5 | setup( 6 | name='django-offsite-storage', 7 | author='Mirumee Software', 8 | author_email='hello@mirumee.com', 9 | description='Cloud static and media file storage suitable for app containers', 10 | license='MIT', 11 | version='0.0.10', 12 | url='https://github.com/mirumee/django-offsite-storage', 13 | packages=find_packages(), 14 | include_package_data=True, 15 | install_requires=[ 16 | 'boto>=2.36.0', 17 | 'Django>=1.7'], 18 | classifiers=[ 19 | 'Environment :: Web Environment', 20 | 'Intended Audience :: Developers', 21 | 'License :: OSI Approved :: BSD License', 22 | 'Operating System :: OS Independent', 23 | 'Programming Language :: Python', 24 | 'Programming Language :: Python :: 2.7', 25 | 'Programming Language :: Python :: 3.2', 26 | 'Programming Language :: Python :: 3.3', 27 | 'Topic :: Internet :: WWW/HTTP', 28 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 29 | 'Topic :: Software Development :: Libraries :: Application Frameworks', 30 | 'Topic :: Software Development :: Libraries :: Python Modules'], 31 | zip_safe=False) 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Mirumee Labs 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of django-offsite-storage nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /offsite_storage/storages/s3.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | from collections import OrderedDict 3 | import logging 4 | import mimetypes 5 | from tempfile import TemporaryFile 6 | 7 | from boto.s3.connection import S3Connection 8 | from django.contrib.staticfiles.storage import ( 9 | ManifestStaticFilesStorage, StaticFilesStorage) 10 | from django.core.exceptions import ImproperlyConfigured 11 | from django.core.files import File 12 | 13 | from .. import settings 14 | 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | class CachedS3FilesStorage(ManifestStaticFilesStorage): 19 | 20 | bucket_name = settings.AWS_STATIC_BUCKET_NAME 21 | 22 | def __init__(self, *args, **kwargs): 23 | base_url = settings.AWS_HOST_URL % {'bucket_name': self.bucket_name} 24 | super(CachedS3FilesStorage, self).__init__( 25 | base_url=base_url, *args, **kwargs) 26 | 27 | def post_process(self, paths, dry_run=False, **options): 28 | try: 29 | aws_keys = ( 30 | settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) 31 | except AttributeError: 32 | raise ImproperlyConfigured( 33 | 'Static collection requires ' 34 | 'AWS_ACCESS_KEY and AWS_SECRET_ACCESS_KEY.') 35 | conn = S3Connection(*aws_keys, host=settings.AWS_S3_ENDPOINT) 36 | bucket = conn.get_bucket(self.bucket_name) 37 | 38 | bucket_files = [key.name for key in bucket.list()] 39 | 40 | post_process_generator = super( 41 | CachedS3FilesStorage, self).post_process( 42 | paths, dry_run=False, **options) 43 | self.hashed_files = OrderedDict() 44 | for name, hashed_name, processed in post_process_generator: 45 | if not hashed_name: 46 | logger.warning( 47 | "Hashed name does not exist. Processed file '%s'. %s", 48 | name, processed) 49 | continue 50 | hash_key = self.hash_key(name) 51 | self.hashed_files[hash_key] = hashed_name 52 | processed = False 53 | if hashed_name not in bucket_files: 54 | file_key = bucket.new_key(hashed_name) 55 | mime_type, encoding = mimetypes.guess_type(name) 56 | headers = { 57 | 'Content-Type': mime_type or 'application/octet-stream', 58 | 'Cache-Control': 'max-age=%d' % (3600 * 24 * 365,)} 59 | 60 | with self.open(hashed_name) as hashed_file: 61 | file_key.set_contents_from_file( 62 | hashed_file, policy=settings.AWS_POLICY, 63 | replace=False, headers=headers) 64 | processed = True 65 | yield name, hashed_name, processed 66 | self.save_manifest() 67 | 68 | def hashed_name(self, name, content=None): 69 | try: 70 | return super(CachedS3FilesStorage, self).hashed_name( 71 | name, content=content) 72 | except ValueError: 73 | logger.warning(u'%s does not exist', name) 74 | return name 75 | 76 | 77 | class S3MediaStorage(StaticFilesStorage): 78 | 79 | bucket_name = settings.AWS_MEDIA_BUCKET_NAME 80 | _bucket = None 81 | 82 | @property 83 | def bucket(self): 84 | if not self._bucket: 85 | self._bucket = self._get_bucket() 86 | return self._bucket 87 | 88 | def _get_bucket(self): 89 | try: 90 | aws_keys = ( 91 | settings.AWS_MEDIA_ACCESS_KEY_ID, 92 | settings.AWS_MEDIA_SECRET_ACCESS_KEY) 93 | except AttributeError: 94 | raise ImproperlyConfigured( 95 | 'Static collection requires ' 96 | 'AWS_MEDIA_ACCESS_KEY_ID and AWS_MEDIA_SECRET_ACCESS_KEY.') 97 | conn = S3Connection(*aws_keys) 98 | return conn.get_bucket(self.bucket_name) 99 | 100 | def _save(self, name, content): 101 | file_key = self.bucket.new_key(name) 102 | mime_type, encoding = mimetypes.guess_type(name) 103 | headers = { 104 | 'Content-Type': mime_type or 'application/octet-stream', 105 | 'Cache-Control': 'max-age=%d' % (3600 * 24 * 365,)} 106 | 107 | file_key.set_contents_from_file( 108 | content, headers=headers, policy=settings.AWS_POLICY, rewind=True) 109 | 110 | return name 111 | 112 | def _open(self, name, mode='rb'): 113 | temp_file = TemporaryFile() 114 | self.bucket.get_key(name).get_file(temp_file) 115 | media_file = File(temp_file) 116 | media_file.seek(0) 117 | return media_file 118 | 119 | def url(self, name): 120 | host = settings.AWS_HOST_URL % {'bucket_name': self.bucket_name} 121 | return host + name.split('?')[0] 122 | 123 | def exists(self, name): 124 | return self.bucket.get_key(name, validate=True) 125 | 126 | def listdir(self, name): 127 | raise NotImplementedError() 128 | 129 | def modified_time(self, name): 130 | raise NotImplementedError() 131 | 132 | def size(self, name): 133 | key = self.bucket.get_key(name) 134 | return key.size 135 | 136 | def path(self, name): 137 | raise NotImplementedError() 138 | --------------------------------------------------------------------------------