├── project ├── __init__.py ├── wsgi.py ├── urls.py └── settings.py ├── storage ├── __init__.py ├── tests.py ├── apps.py ├── urls.py ├── admin.py ├── views.py └── models.py ├── .env.example ├── requirements.txt ├── Dockerfile ├── .gitignore ├── manage.py ├── docker-compose.yml ├── LICENSE └── README.md /project/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /storage/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | SECRET_KEY=put-random-secret-string-here 2 | -------------------------------------------------------------------------------- /storage/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /storage/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class StorageConfig(AppConfig): 5 | name = 'storage' 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | django==3.2.23 2 | awssig==0.5.0 3 | gunicorn==20.1.0 4 | whitenoise==6.2.0 5 | dj-database-url==0.5.0 6 | psycopg2-binary==2.9.2 7 | -------------------------------------------------------------------------------- /storage/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from storage import views 4 | 5 | urlpatterns = [ 6 | path('/', views.main, name='public_link'), 7 | ] 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10.5 2 | 3 | ENV PYTHONUNBUFFERED 1 4 | RUN mkdir /app 5 | WORKDIR /app 6 | ADD requirements.txt /app/ 7 | 8 | RUN pip install -U pip 9 | 10 | RUN pip install -r requirements.txt 11 | 12 | ADD . /app/ 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | notes.txt 2 | *.sqlite3 3 | __pycache__/ 4 | media/ 5 | .env 6 | .vscode/ 7 | venv/ 8 | 9 | 10 | # Ignore Jupyter notebook files 11 | .ipynb_checkpoints/ 12 | *.ipynb 13 | 14 | # Local deployment donfiguration 15 | _deploy.sh 16 | -------------------------------------------------------------------------------- /project/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for project project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /storage/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from .models import Blob, Bucket 4 | 5 | 6 | class BlobAdmin(admin.ModelAdmin): 7 | list_display = ['bucket', 'path', 'content_type', 'size', 'created_on'] 8 | readonly_fields = ['bucket', 'path', 'content_type', 'size'] 9 | exclude = ['file'] 10 | list_filter = ['created_on', 'updated_on'] 11 | search_fields = ['path'] 12 | 13 | 14 | class BucketAdmin(admin.ModelAdmin): 15 | list_display = ['name', 'access_key_id', 'size'] 16 | search_fields = ['name'] 17 | 18 | admin.site.register(Blob, BlobAdmin) 19 | admin.site.register(Bucket, BucketAdmin) 20 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings') 9 | try: 10 | from django.core.management import execute_from_command_line 11 | except ImportError as exc: 12 | raise ImportError( 13 | "Couldn't import Django. Are you sure it's installed and " 14 | "available on your PYTHONPATH environment variable? Did you " 15 | "forget to activate a virtual environment?" 16 | ) from exc 17 | execute_from_command_line(sys.argv) 18 | 19 | 20 | if __name__ == '__main__': 21 | main() 22 | -------------------------------------------------------------------------------- /project/urls.py: -------------------------------------------------------------------------------- 1 | """project URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/2.2/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.contrib import admin 17 | from django.urls import include, path 18 | from django.conf.urls.static import static 19 | 20 | urlpatterns = [ 21 | path('admin/', admin.site.urls), 22 | path('', include('storage.urls')), 23 | ] 24 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | app: 4 | build: . 5 | command: bash -c "python ./manage.py migrate --noinput 6 | && python manage.py collectstatic --noinput 7 | && gunicorn project.wsgi:application --bind :8000" 8 | volumes: 9 | - .:/app 10 | environment: 11 | - DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres 12 | - SECRET_KEY=${SECRET_KEY} 13 | ports: 14 | - "8000:8000" 15 | depends_on: 16 | postgres: 17 | condition: service_healthy 18 | volumes: 19 | - media-data:/app/media 20 | 21 | postgres: 22 | image: postgres:14.4 23 | volumes: 24 | - postgres-data:/var/lib/postgresql 25 | environment: 26 | - POSTGRES_PASSWORD=postgres 27 | user: postgres 28 | healthcheck: 29 | test: ["CMD-SHELL", "pg_isready"] 30 | interval: 10s 31 | timeout: 5s 32 | retries: 5 33 | 34 | volumes: 35 | postgres-data: 36 | driver: local 37 | media-data: 38 | driver: local 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Sergey Lyapustin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Your Own Amazon S3 Server 2 | 3 | This application replicate basic Amazon S3 functionality: 4 | - Ability to have multiple Buckets with own credentials 5 | - Upload files with public read-only access 6 | 7 | Main purpose of that application is to store Media and Static files of Django applications outside of the application instance. 8 | That is especially useful if your deploy application to the cloud providers (for ex. Heroku) which does not offer storage option. 9 | 10 | ## Deployment 11 | 12 | - Create an `.env` file based on the `.env.example` 13 | - Build and run app via Docker: `docker-compose up -d` (if you on Apple Silicon, you need to do `export DOCKER_DEFAULT_PLATFORM=linux/amd64` first) 14 | - Create an Administrator account: `docker-compose exec app python ./manage.py createsuperuser` 15 | 16 | ## Using in your Application 17 | 18 | In order to use that storage at your Django projects you may need use `django-storages` package, in that case some extra settings required. 19 | ```python 20 | # Usual AWS S3 Configuration 21 | AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] 22 | AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] 23 | AWS_STORAGE_BUCKET_NAME = os.environ['AWS_STORAGE_BUCKET_NAME'] 24 | 25 | # Some extra setings 26 | AWS_S3_ENDPOINT_URL='https://your.app.instance.com' # Your App Endpoint 27 | AWS_QUERYSTRING_AUTH = False 28 | AWS_DEFAULT_ACL='public-read' 29 | ``` 30 | 31 | ### Demo project 32 | 33 | Demo project, which utilize that app available [here](https://github.com/slyapustin/django-classified-demo). 34 | -------------------------------------------------------------------------------- /storage/views.py: -------------------------------------------------------------------------------- 1 | from django.core.files.base import ContentFile 2 | from django.http import Http404, HttpResponse, QueryDict 3 | from django.shortcuts import get_object_or_404, render 4 | from django.views.decorators.csrf import csrf_exempt 5 | 6 | from .models import Blob, Bucket 7 | 8 | # https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html 9 | 10 | 11 | @csrf_exempt 12 | def main(request, bucket_name, path): 13 | bucket = get_object_or_404(Bucket, name=bucket_name) 14 | if request.method == 'PUT': 15 | if not bucket.verify_request(request): 16 | return HttpResponse(status=403) 17 | 18 | blob = Blob.objects.filter(bucket=bucket, path=path).first() 19 | if not blob: 20 | blob = Blob( 21 | bucket=bucket, 22 | path=path 23 | ) 24 | blob.content_type = request.META.get('CONTENT_TYPE', '') 25 | blob.size = request.META.get('CONTENT_LENGTH', '0') 26 | blob.file.save(f'{bucket.name}/{path}', ContentFile(request.body)) 27 | blob.save() 28 | return HttpResponse('') 29 | elif request.method == 'GET': 30 | # TODO check that https://djangosnippets.org/snippets/365/ 31 | blob = Blob.objects.filter(bucket=bucket, path=path).first() 32 | if not blob: 33 | raise Http404() 34 | return HttpResponse(blob.file.read(), content_type=blob.content_type) 35 | elif request.method == 'HEAD': 36 | if Blob.objects.filter(bucket=bucket, path=path).exists(): 37 | return HttpResponse('') 38 | else: 39 | raise Http404() 40 | else: 41 | return HttpResponse('') 42 | -------------------------------------------------------------------------------- /storage/models.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | import string 4 | 5 | import awssig 6 | from django.core.validators import MinLengthValidator, validate_slug 7 | from django.db import models 8 | from django.db.models import Sum 9 | from django.template.defaultfilters import filesizeformat 10 | from django.urls import reverse 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class Bucket(models.Model): 16 | # https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html 17 | name = models.CharField(validators=[MinLengthValidator(3), validate_slug], max_length=63, unique=True) 18 | access_key_id = models.CharField(max_length=16, default='', blank=True) 19 | secret_key = models.CharField(max_length=128, default='', blank=True) 20 | 21 | def __str__(self): 22 | return self.name 23 | 24 | def verify_request(self, request): 25 | headers={ 26 | "X-Amz-Date": request.META.get('HTTP_X_AMZ_DATE', ''), 27 | "x-amz-date": request.META.get('HTTP_X_AMZ_DATE', ''), 28 | "authorization": request.META.get('HTTP_AUTHORIZATION', ''), 29 | 'content-md5': request.META.get('HTTP_CONTENT_MD5', ''), 30 | 'content-type': request.META.get('CONTENT_TYPE', ''), 31 | 'host': request.META.get('HTTP_HOST', ''), 32 | 'x-amz-acl': request.META.get('HTTP_X_AMZ_ACL', ''), 33 | 'x-amz-content-sha256': request.META.get('HTTP_X_AMZ_CONTENT_SHA256', '') 34 | } 35 | v = awssig.AWSSigV4Verifier( 36 | request_method=request.method, 37 | uri_path=request.META.get('PATH_INFO', ''), 38 | query_string=request.META.get('QUERY_STRING', ''), 39 | headers=headers, 40 | body=request.body, 41 | region="us-east-1", 42 | service="s3", 43 | key_mapping={self.access_key_id: self.secret_key}, 44 | timestamp_mismatch=None) 45 | try: 46 | v.verify() 47 | return True 48 | except awssig.InvalidSignatureError as e: 49 | logger.warning('Invalid signature: %s', e) 50 | except Exception as e: 51 | logger.error('Unable to verify request: %s', e) 52 | 53 | return False 54 | 55 | @property 56 | def size(self): 57 | return filesizeformat(self.blobs.aggregate(Sum('size')).get('size__sum', 0)) 58 | 59 | def save(self, *args, **kwargs): 60 | if not self.access_key_id: 61 | self.access_key_id = ''.join(random.choice( 62 | string.ascii_letters + string.digits) for i in range(16)) 63 | if not self.secret_key: 64 | self.secret_key = ''.join(random.choice( 65 | string.ascii_letters + string.digits) for i in range(32)) 66 | 67 | super().save(*args, **kwargs) 68 | 69 | 70 | class Blob(models.Model): 71 | bucket = models.ForeignKey(Bucket, on_delete=models.PROTECT, related_name='blobs') 72 | path = models.CharField(max_length=512) 73 | file = models.FileField() 74 | 75 | # TODO move extra meta fields to the PostgreSQL JSONField 76 | content_type = models.CharField(max_length=128, default='') 77 | size = models.IntegerField(default=0) 78 | 79 | created_on = models.DateTimeField(auto_now_add=True, null=True, blank=True) 80 | updated_on = models.DateTimeField(auto_now=True, null=True, blank=True) 81 | 82 | def __str__(self): 83 | return self.path 84 | 85 | def get_absolute_url(self): 86 | return reverse( 87 | 'public_link', 88 | kwargs={ 89 | 'bucket_name': self.bucket.name, 90 | 'path': self.path 91 | } 92 | ) 93 | -------------------------------------------------------------------------------- /project/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for project project. 3 | 4 | Generated by 'django-admin startproject' using Django 2.2.3. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.2/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/2.2/ref/settings/ 11 | """ 12 | 13 | import os 14 | import dj_database_url 15 | 16 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 17 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 18 | 19 | 20 | # Quick-start development settings - unsuitable for production 21 | # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ 22 | 23 | # SECURITY WARNING: keep the secret key used in production secret! 24 | SECRET_KEY = os.environ['SECRET_KEY'] 25 | 26 | # SECURITY WARNING: don't run with debug turned on in production! 27 | DEBUG = os.environ.get('DEBUG', 'False') == 'True' 28 | 29 | ALLOWED_HOSTS = ['*',] 30 | 31 | DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' 32 | 33 | # Application definition 34 | 35 | INSTALLED_APPS = [ 36 | 'django.contrib.admin', 37 | 'django.contrib.auth', 38 | 'django.contrib.contenttypes', 39 | 'django.contrib.sessions', 40 | 'django.contrib.messages', 41 | 'django.contrib.staticfiles', 42 | 43 | 'storage' 44 | ] 45 | 46 | MIDDLEWARE = [ 47 | 'whitenoise.middleware.WhiteNoiseMiddleware', 48 | 'django.middleware.security.SecurityMiddleware', 49 | 'django.contrib.sessions.middleware.SessionMiddleware', 50 | 'django.middleware.common.CommonMiddleware', 51 | 'django.middleware.csrf.CsrfViewMiddleware', 52 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 53 | 'django.contrib.messages.middleware.MessageMiddleware', 54 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 55 | ] 56 | 57 | ROOT_URLCONF = 'project.urls' 58 | 59 | TEMPLATES = [ 60 | { 61 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 62 | 'DIRS': [], 63 | 'APP_DIRS': True, 64 | 'OPTIONS': { 65 | 'context_processors': [ 66 | 'django.template.context_processors.debug', 67 | 'django.template.context_processors.request', 68 | 'django.contrib.auth.context_processors.auth', 69 | 'django.contrib.messages.context_processors.messages', 70 | ], 71 | }, 72 | }, 73 | ] 74 | 75 | WSGI_APPLICATION = 'project.wsgi.application' 76 | 77 | 78 | # Database 79 | # https://docs.djangoproject.com/en/2.2/ref/settings/#databases 80 | 81 | DATABASES = { 82 | 'default': { 83 | 'ENGINE': 'django.db.backends.sqlite3', 84 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 85 | } 86 | } 87 | 88 | # This will configure DB based on the `DATABASE_URL` env variable. 89 | DATABASES['default'].update(dj_database_url.config()) 90 | 91 | # Password validation 92 | # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators 93 | 94 | AUTH_PASSWORD_VALIDATORS = [ 95 | { 96 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 97 | }, 98 | { 99 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 100 | }, 101 | { 102 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 103 | }, 104 | { 105 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 106 | }, 107 | ] 108 | 109 | 110 | # Internationalization 111 | # https://docs.djangoproject.com/en/2.2/topics/i18n/ 112 | 113 | LANGUAGE_CODE = 'en-us' 114 | 115 | TIME_ZONE = 'UTC' 116 | 117 | USE_I18N = True 118 | 119 | USE_L10N = True 120 | 121 | USE_TZ = True 122 | 123 | 124 | # Static files (CSS, JavaScript, Images) 125 | # https://docs.djangoproject.com/en/2.2/howto/static-files/ 126 | 127 | STATIC_URL = '/static/' 128 | STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' 129 | STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') 130 | 131 | MEDIA_URL = '/media/' 132 | MEDIA_ROOT = os.path.join(BASE_DIR, 'media') 133 | 134 | SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') 135 | --------------------------------------------------------------------------------