├── tests ├── __init__.py ├── media │ └── media-test.txt ├── static │ ├── static-test.txt │ ├── admin │ │ └── admin-test.txt │ └── appdir │ │ └── appdir-test.txt ├── templates │ ├── flatpage.html │ └── humanize.html ├── i18n_urls.py ├── test_commands.py ├── namespaced_sub_urls.py ├── no_namespaced_urls.py ├── test_interface.py ├── namespaced_urls.py ├── settings.py ├── test_static.py ├── test_redirects.py ├── urls.py └── test_renderer.py ├── MANIFEST.in ├── .github ├── FUNDING.yml └── workflows │ └── ci.yaml ├── django_distill ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── distill-test-publish.py │ │ ├── distill-local.py │ │ └── distill-publish.py ├── errors.py ├── __init__.py ├── distill.py ├── publisher.py ├── backends │ ├── google_storage.py │ ├── amazon_s3.py │ ├── __init__.py │ └── microsoft_azure_storage.py └── renderer.py ├── requirements.txt ├── setup.cfg ├── Pipfile ├── run-tests.py ├── SECURITY.md ├── CODE_OF_CONDUCT.md ├── .gitignore ├── LICENSE ├── setup.py ├── CONTRIBUTING.md └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [meeb] 2 | -------------------------------------------------------------------------------- /django_distill/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/media/media-test.txt: -------------------------------------------------------------------------------- 1 | media 2 | -------------------------------------------------------------------------------- /tests/static/static-test.txt: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | django 2 | requests 3 | -------------------------------------------------------------------------------- /tests/static/admin/admin-test.txt: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /django_distill/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/static/appdir/appdir-test.txt: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /tests/templates/flatpage.html: -------------------------------------------------------------------------------- 1 | {{ flatpage.title }}{{ flatpage.content }} 2 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | requests = "*" 10 | Django = "*" 11 | -------------------------------------------------------------------------------- /django_distill/errors.py: -------------------------------------------------------------------------------- 1 | class DistillError(Exception): 2 | pass 3 | 4 | 5 | class DistillWarning(RuntimeWarning): 6 | pass 7 | 8 | 9 | class DistillPublishError(Exception): 10 | pass 11 | -------------------------------------------------------------------------------- /tests/templates/humanize.html: -------------------------------------------------------------------------------- 1 | {% load humanize %} 2 | 7 | -------------------------------------------------------------------------------- /run-tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | import os 5 | import sys 6 | import django 7 | from django.conf import settings 8 | from django.test.utils import get_runner 9 | 10 | 11 | if __name__ == '__main__': 12 | os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.settings' 13 | django.setup() 14 | TestRunner = get_runner(settings) 15 | test_runner = TestRunner(verbosity=2) 16 | failures = test_runner.run_tests(['tests']) 17 | sys.exit(bool(failures)) 18 | -------------------------------------------------------------------------------- /tests/i18n_urls.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | from django_distill import distill_path 3 | 4 | 5 | app_name = 'i18n' 6 | 7 | 8 | def test_url_i18n_view(request): 9 | return HttpResponse(b'test', content_type='application/octet-stream') 10 | 11 | 12 | def test_no_param_func(): 13 | return None 14 | 15 | 16 | urlpatterns = [ 17 | 18 | distill_path('sub-url-with-i18n-prefix', 19 | test_url_i18n_view, 20 | name='test-url-i18n', 21 | distill_func=test_no_param_func), 22 | 23 | ] 24 | -------------------------------------------------------------------------------- /tests/test_commands.py: -------------------------------------------------------------------------------- 1 | from importlib import import_module 2 | from django.test import TestCase 3 | 4 | 5 | class DjangoDistillCommandTestSuite(TestCase): 6 | 7 | def test_command_imports_distill_local(self): 8 | import_module('django_distill.management.commands.distill-local') 9 | 10 | def test_command_imports_distill_publish(self): 11 | import_module('django_distill.management.commands.distill-publish') 12 | 13 | def test_command_imports_distill_test_publish(self): 14 | import_module('django_distill.management.commands.distill-test-publish') 15 | -------------------------------------------------------------------------------- /tests/namespaced_sub_urls.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | from django_distill import distill_path 3 | 4 | 5 | app_name = 'sub-urls' 6 | 7 | 8 | def test_url_in_deep_namespace_view(request): 9 | return HttpResponse(b'test', content_type='application/octet-stream') 10 | 11 | 12 | def test_no_param_func(): 13 | return None 14 | 15 | 16 | urlpatterns = [ 17 | 18 | distill_path('sub-url-in-sub-namespace', 19 | test_url_in_deep_namespace_view, 20 | name='test_url_in_namespace', 21 | distill_func=test_no_param_func, 22 | distill_file='test_url_in_sub_namespace'), 23 | 24 | ] 25 | -------------------------------------------------------------------------------- /tests/no_namespaced_urls.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | from django_distill import distill_path 3 | 4 | 5 | app_name = 'no-namespaced-urls' 6 | 7 | 8 | def test_url_in_no_namespace_view(request): 9 | return HttpResponse(b'test', content_type='application/octet-stream') 10 | 11 | 12 | def test_no_param_func(): 13 | return None 14 | 15 | 16 | urlpatterns = [ 17 | 18 | distill_path('sub-url-in-no-namespace', 19 | test_url_in_no_namespace_view, 20 | name='test_url_in_no_namespace', 21 | distill_func=test_no_param_func, 22 | distill_file='test_url_in_no_namespace'), 23 | 24 | ] 25 | -------------------------------------------------------------------------------- /tests/test_interface.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django.conf import settings 3 | import django_distill 4 | 5 | 6 | class DjangoDistillInterfaceTestSuite(TestCase): 7 | 8 | def test_import(self): 9 | assert hasattr(django_distill, 'distill_url') 10 | assert callable(django_distill.distill_url) 11 | if settings.HAS_RE_PATH: 12 | assert hasattr(django_distill, 'distill_re_path') 13 | assert callable(django_distill.distill_re_path) 14 | if settings.HAS_PATH: 15 | assert hasattr(django_distill, 'distill_path') 16 | assert callable(django_distill.distill_path) 17 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Typically, only the latest version of `django-distill` will be supported for 6 | security issues. Please upgrade to the latest release first before reporting 7 | any security issues. 8 | 9 | | Version | Supported | 10 | | ------- | ------------------ | 11 | | 3.2.4 | :white_check_mark: | 12 | | < 3.2.4 | :x: | 13 | 14 | ## Reporting a Vulnerability 15 | 16 | `django-distill` is a static site generator, as such any security issues are 17 | going to be inherently limited in scope. Please feel free to create a public 18 | GitHub issue if you encounter any unexpected or potentially security related 19 | issues. 20 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Run Django tests for django-distill 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: 8 | - master 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy3.9', 'pypy3.10'] 16 | steps: 17 | - uses: actions/checkout@v4 18 | - name: Install Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v5 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | pip install -r requirements.txt 26 | - name: Run Django tests 27 | run: ./run-tests.py 28 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | The maintainers of this project will treat everyone equally and fairly 4 | and expect anyone interacting with the project to do the same. Use common 5 | sense, that's about it. 6 | 7 | All interactions that are a net positive for the project are encouraged. 8 | Any negative actions or behaviour deemed innapropriate will be ignored 9 | or deleted. 10 | 11 | This code of conduct applies to this repository and related GitHub hosted 12 | services as well as emails addressed to the maintainers concerning this 13 | project. 14 | 15 | We've never had an issue with conduct or contributor behaviour, however 16 | it appears mandatory to now have a code of conduct and various services 17 | and sites won't stop bugging us about it. 18 | 19 | If anything really bad occurs the worst that will happen is we delete 20 | some comments or a few issues. 21 | -------------------------------------------------------------------------------- /tests/namespaced_urls.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | from django_distill import distill_path 3 | from django.urls import path, include 4 | 5 | 6 | app_name = 'namespaced-urls' 7 | 8 | 9 | def test_url_in_namespace_view(request): 10 | return HttpResponse(b'test', content_type='application/octet-stream') 11 | 12 | 13 | def test_no_param_func(): 14 | return None 15 | 16 | 17 | urlpatterns = [ 18 | 19 | distill_path('sub-url-in-namespace', 20 | test_url_in_namespace_view, 21 | name='test_url_in_namespace', 22 | distill_func=test_no_param_func, 23 | distill_file='test_url_in_namespace'), 24 | path('path/sub-namespace/', 25 | include('tests.namespaced_sub_urls', namespace='sub_test_namespace')), 26 | # Uncomment to trigger a DistillError for including the same sub-urls more than 27 | # once in a single project which is unsupported 28 | #path('path/sub-namespace1/', 29 | # include('tests.namespaced_sub_urls', namespace='sub_test_namespace1')), 30 | 31 | ] 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | # IDE 60 | .vscode 61 | .idea 62 | 63 | # Pipenv lockfile 64 | Pipfile.lock 65 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020 meeb@meeb.org 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /django_distill/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '3.2.7' 2 | 3 | 4 | from django import __version__ as django_version 5 | from django_distill.errors import DistillError 6 | from django_distill.distill import urls_to_distill 7 | from django_distill.renderer import generate_urls 8 | 9 | 10 | try: 11 | from django_distill.distill import distill_url 12 | except ImportError: 13 | def distill_url(*args, **kwargs): 14 | err = ('Your installed version of Django ({}) does not supprt ' 15 | 'django.urls.url or django.conf.urls.url, use ' 16 | 'django_distill.distill_re_path or django_distill.distill_path') 17 | raise DistillError(err.format(django_version)) 18 | 19 | 20 | try: 21 | from django_distill.distill import distill_re_path 22 | except ImportError: 23 | def distill_re_path(*args, **kwargs): 24 | err = ('Your installed version of Django ({}) does not supprt ' 25 | 'django.urls.re_path, please upgrade') 26 | raise DistillError(err.format(django_version)) 27 | 28 | 29 | try: 30 | from django_distill.distill import distill_path 31 | except ImportError: 32 | def distill_path(*args, **kwargs): 33 | err = ('Your installed version of Django ({}) does not supprt ' 34 | 'django.urls.path, please upgrade') 35 | raise DistillError(err.format(django_version)) 36 | 37 | 38 | def distilled_urls(): 39 | return generate_urls(urls_to_distill) 40 | -------------------------------------------------------------------------------- /tests/settings.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | 4 | BASE_DIR = Path(__file__).resolve().parent.parent 5 | 6 | 7 | try: 8 | from django.urls import path 9 | HAS_PATH = True 10 | except ImportError: 11 | HAS_PATH = False 12 | 13 | 14 | try: 15 | from django.urls import re_path 16 | HAS_RE_PATH = True 17 | except ImportError: 18 | HAS_RE_PATH = False 19 | 20 | 21 | SECRET_KEY = 'test' 22 | 23 | 24 | ROOT_URLCONF = 'tests.urls' 25 | 26 | 27 | MIDDLEWARE = ['django.contrib.sessions.middleware.SessionMiddleware'] 28 | 29 | 30 | DATABASES = { 31 | 'default': { 32 | 'ENGINE': 'django.db.backends.sqlite3', 33 | 'NAME': BASE_DIR / 'test.sqlite3', 34 | } 35 | } 36 | 37 | 38 | INSTALLED_APPS = [ 39 | 'django.contrib.sites', 40 | 'django.contrib.flatpages', 41 | 'django.contrib.sessions', 42 | 'django.contrib.sitemaps', 43 | 'django.contrib.humanize', 44 | 'django.contrib.redirects', 45 | 'django_distill', 46 | 'tests', 47 | ] 48 | 49 | 50 | TEMPLATES = [ 51 | { 52 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 53 | 'DIRS': [BASE_DIR / 'tests' / 'templates'], 54 | 'APP_DIRS': True, 55 | }, 56 | ] 57 | 58 | 59 | SITE_ID = 1 60 | 61 | 62 | STATIC_URL = '/static/' 63 | STATIC_ROOT = BASE_DIR / 'tests' / 'static' 64 | MEDIA_URL = '/media/' 65 | MEDIA_ROOT = BASE_DIR / 'tests' / 'media' 66 | 67 | 68 | LANGUAGE_CODE = 'en' 69 | USE_I18N = True 70 | -------------------------------------------------------------------------------- /django_distill/distill.py: -------------------------------------------------------------------------------- 1 | from django_distill.errors import DistillError 2 | 3 | 4 | urls_to_distill = [] 5 | 6 | 7 | def _distill_url(func, *a, **k): 8 | distill_func = k.get('distill_func') 9 | if distill_func: 10 | del k['distill_func'] 11 | else: 12 | distill_func = lambda: None 13 | distill_file = k.get('distill_file') 14 | distill_status_codes = k.get('distill_status_codes') 15 | if distill_file: 16 | del k['distill_file'] 17 | if distill_status_codes: 18 | del k['distill_status_codes'] 19 | else: 20 | distill_status_codes = (200,) 21 | name = k.get('name') 22 | if not name: 23 | raise DistillError('Distill function provided with no name') 24 | if not callable(distill_func): 25 | err = 'Distill function not callable: {}' 26 | raise DistillError(err.format(distill_func)) 27 | url = func(*a, **k) 28 | urls_to_distill.append((url, distill_func, distill_file, distill_status_codes, 29 | name, a, k)) 30 | return url 31 | 32 | 33 | try: 34 | from django.conf.urls import url 35 | def distill_url(*a, **k): 36 | return _distill_url(url, *a, **k) 37 | except ImportError: 38 | try: 39 | from django.urls import url 40 | def distill_url(*a, **k): 41 | return _distill_url(url, *a, **k) 42 | except ImportError: 43 | pass 44 | 45 | 46 | try: 47 | from django.urls import path 48 | def distill_path(*a, **k): 49 | return _distill_url(path, *a, **k) 50 | except ImportError: 51 | pass 52 | 53 | 54 | try: 55 | from django.urls import re_path 56 | def distill_re_path(*a, **k): 57 | return _distill_url(re_path, *a, **k) 58 | except ImportError: 59 | pass 60 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from setuptools import setup, find_packages 4 | 5 | 6 | version = '3.2.7' 7 | 8 | 9 | with open('README.md', 'rt') as f: 10 | long_description = f.read() 11 | 12 | 13 | with open('requirements.txt', 'rt') as f: 14 | requirements = tuple(f.read().split()) 15 | 16 | 17 | setup( 18 | name = 'django-distill', 19 | version = version, 20 | url = 'https://github.com/meeb/django-distill', 21 | author = 'https://github.com/meeb', 22 | author_email = 'meeb@meeb.org', 23 | description = 'Static site renderer and publisher for Django.', 24 | long_description = long_description, 25 | long_description_content_type = 'text/markdown', 26 | license = 'MIT', 27 | include_package_data = True, 28 | install_requires = requirements, 29 | extras_require = { 30 | 'amazon': ['boto3'], 31 | 'google': ['google-api-python-client', 'google-cloud-storage'], 32 | 'microsoft': ['azure-storage-blob'], 33 | }, 34 | packages = find_packages(), 35 | classifiers = [ 36 | 'Development Status :: 5 - Production/Stable', 37 | 'Environment :: Web Environment', 38 | 'Intended Audience :: Developers', 39 | 'License :: OSI Approved :: MIT License', 40 | 'Operating System :: OS Independent', 41 | 'Programming Language :: Python', 42 | 'Programming Language :: Python :: 3', 43 | 'Programming Language :: Python :: 3.5', 44 | 'Programming Language :: Python :: 3.6', 45 | 'Topic :: Internet :: WWW/HTTP', 46 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 47 | 'Topic :: Software Development :: Libraries :: Application Frameworks', 48 | 'Topic :: Software Development :: Libraries :: Python Modules', 49 | ], 50 | keywords = ['django', 'distill', 'static', 'website', 'jamstack', 's3', 51 | 'amazon s3', 'aws', 'amazon', 'google', 'microsoft', 52 | 'google cloud', 'google cloud storage', 'azure', 53 | 'azure storage', 'azure blob storage'], 54 | ) 55 | -------------------------------------------------------------------------------- /tests/test_static.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | from tempfile import TemporaryDirectory 4 | from django.test import TestCase 5 | from django.conf import settings 6 | from django_distill.renderer import copy_static_and_media_files 7 | 8 | 9 | def null(*args, **kwargs): 10 | pass 11 | 12 | 13 | class DjangoDistillStaticFilesTestSuite(TestCase): 14 | 15 | def test_copying_static_files(self): 16 | # Test default behavior 17 | with TemporaryDirectory() as tempdir: 18 | copy_static_and_media_files(tempdir, null) 19 | test_media_file_path = str(Path(tempdir) / 'media' / 'media-test.txt') 20 | self.assertTrue(os.path.exists(test_media_file_path)) 21 | test_static_file_path = str(Path(tempdir) / 'static' / 'static-test.txt') 22 | self.assertTrue(os.path.exists(test_static_file_path)) 23 | # Test admin dir is copied with SKIP_ADMIN_DIRS set to False 24 | settings.DISTILL_SKIP_ADMIN_DIRS = False 25 | with TemporaryDirectory() as tempdir: 26 | copy_static_and_media_files(tempdir, null) 27 | test_media_file_path = str(Path(tempdir) / 'media' / 'media-test.txt') 28 | self.assertTrue(os.path.exists(test_media_file_path)) 29 | test_static_file_path = str(Path(tempdir) / 'static' / 'static-test.txt') 30 | self.assertTrue(os.path.exists(test_static_file_path)) 31 | test_admin_file_path = str(Path(tempdir) / 'static' / 'admin' / 'admin-test.txt') 32 | self.assertTrue(os.path.exists(test_admin_file_path)) 33 | test_appdir_file_path = str(Path(tempdir) / 'static' / 'appdir' / 'appdir-test.txt') 34 | self.assertTrue(os.path.exists(test_appdir_file_path)) 35 | # Test static app dirs are skipped with DISTILL_SKIP_STATICFILES_DIRS populated 36 | settings.DISTILL_SKIP_STATICFILES_DIRS = ['appdir'] 37 | with TemporaryDirectory() as tempdir: 38 | copy_static_and_media_files(tempdir, null) 39 | test_appdir_file_path = str(Path(tempdir) / 'static' / 'appdir' / 'appdir-test.txt') 40 | self.assertFalse(os.path.exists(test_appdir_file_path)) 41 | -------------------------------------------------------------------------------- /django_distill/publisher.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures import ThreadPoolExecutor 2 | from django_distill.errors import DistillPublishError 3 | 4 | 5 | def publish_dir(backend, stdout, verify=True, parallel_publish=1, ignore_remote_content=False): 6 | stdout('Authenticating') 7 | backend.authenticate() 8 | stdout('Getting file indexes') 9 | remote_files = set() if ignore_remote_content else backend.list_remote_files() 10 | local_files = backend.list_local_files() 11 | to_upload = set() 12 | to_delete = set() 13 | local_files_r = set() 14 | # check local files to upload 15 | for f in local_files: 16 | remote_f = backend.remote_path(f) 17 | local_files_r.add(remote_f) 18 | if remote_f not in remote_files: 19 | # local file not present remotely, upload it 20 | to_upload.add(f) 21 | else: 22 | # file is present remotely, check its hash 23 | if not backend.compare_file(f, remote_f): 24 | stdout(f'File stale (hash different): {remote_f}') 25 | to_upload.add(f) 26 | else: 27 | stdout(f'File fresh: {remote_f}') 28 | # check for remote files to delete 29 | for f in remote_files: 30 | if f not in local_files_r: 31 | to_delete.add(f) 32 | with ThreadPoolExecutor(max_workers=parallel_publish) as executor: 33 | # upload any new or changed files 34 | executor.map(lambda f: _publish_file(backend, f, verify, stdout), to_upload) 35 | # Call any final checks that may be needed by the backend 36 | stdout('Final checks') 37 | backend.final_checks() 38 | # delete any orphan files 39 | executor.map(lambda f: _delete_file(backend, f, stdout), to_delete) 40 | 41 | 42 | def _publish_file(backend, f, verify, stdout): 43 | remote_f = backend.remote_path(f) 44 | stdout(f'Publishing: {f} -> {remote_f}') 45 | backend.upload_file(f, remote_f) 46 | if verify: 47 | url = backend.remote_url(f) 48 | stdout(f'Verifying: {url}') 49 | if not backend.check_file(f, url): 50 | err = f'Remote file {url} failed hash check' 51 | raise DistillPublishError(err) 52 | 53 | 54 | def _delete_file(backend, f, stdout): 55 | stdout(f'Deleting remote: {f}') 56 | backend.delete_remote_file(f) 57 | -------------------------------------------------------------------------------- /django_distill/backends/google_storage.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import warnings 4 | from base64 import b64decode 5 | from binascii import hexlify 6 | 7 | 8 | try: 9 | from googleapiclient import discovery 10 | from google.cloud import storage 11 | except ImportError: 12 | name = 'django_distill.backends.google_storage' 13 | pipm = 'google-api-python-client google-cloud-storage' 14 | sys.stdout.write('{} backend requires {}:\n'.format(name, pipm)) 15 | sys.stdout.write('$ pip install django-distill[google]{}\n\n'.format(pipm)) 16 | raise 17 | 18 | 19 | from django_distill.errors import DistillPublishError 20 | from django_distill.backends import BackendBase 21 | 22 | 23 | class GoogleCloudStorageBackend(BackendBase): 24 | ''' 25 | Publisher for Google Cloud Storage. Implements the BackendBase. 26 | ''' 27 | 28 | REQUIRED_OPTIONS = ('ENGINE', 'BUCKET') 29 | 30 | def account_username(self): 31 | return 32 | 33 | def account_container(self): 34 | return self.options.get('BUCKET', '') 35 | 36 | def authenticate(self): 37 | credentials_file = self.options.get('JSON_CREDENTIALS', '') 38 | if credentials_file: 39 | if not os.path.exists(credentials_file): 40 | err = 'Credentials file does not exist: {}' 41 | raise DistillPublishError(err.format(credentials_file)) 42 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials_file 43 | 44 | bucket = self.account_container() 45 | self.d['connection'] = storage.Client() 46 | self.d['bucket'] = self.d['connection'].get_bucket(bucket) 47 | 48 | def list_remote_files(self): 49 | rtn = set() 50 | for b in self.d['bucket'].list_blobs(): 51 | rtn.add(b.name) 52 | return rtn 53 | 54 | def delete_remote_file(self, remote_name): 55 | b = self.d['bucket'].get_blob(remote_name) 56 | return b.delete() 57 | 58 | def compare_file(self, local_name, remote_name): 59 | b = self.d['bucket'].get_blob(remote_name) 60 | local_hash = self._get_local_file_hash(local_name) 61 | remote_hash = str(hexlify(b64decode(b.md5_hash)).decode()) 62 | return local_hash == remote_hash 63 | 64 | def upload_file(self, local_name, remote_name): 65 | b = self.d['bucket'].blob(remote_name) 66 | b.upload_from_filename(local_name) 67 | b.make_public() 68 | return True 69 | 70 | def create_remote_dir(self, remote_dir_name): 71 | # not required for Google Storage buckets 72 | return True 73 | 74 | def remote_path(self, local_name): 75 | truncated_path = super().remote_path(local_name) 76 | # Replace \ for /, Google Cloud Files API docs state they handle both \ and / 77 | # as directory separators but really make sure we're only using / in blob names 78 | return truncated_path.replace('\\', '/') 79 | 80 | 81 | backend_class = GoogleCloudStorageBackend 82 | -------------------------------------------------------------------------------- /django_distill/backends/amazon_s3.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import mimetypes 3 | 4 | try: 5 | import boto3 6 | except ImportError: 7 | name = 'django_distill.backends.amazon_s3' 8 | pipm = 'boto3' 9 | sys.stdout.write('{} backend requires {}:\n'.format(name, pipm)) 10 | sys.stdout.write('$ pip install django-distill[amazon]{}\n\n'.format(pipm)) 11 | raise 12 | 13 | from django_distill.errors import DistillPublishError 14 | from django_distill.backends import BackendBase 15 | 16 | 17 | class AmazonS3Backend(BackendBase): 18 | ''' 19 | Publisher for Amazon S3. Implements the BackendBase. 20 | ''' 21 | 22 | REQUIRED_OPTIONS = ('ENGINE', 'PUBLIC_URL', 'BUCKET') 23 | 24 | def _get_object(self, name): 25 | bucket = self.account_container() 26 | return self.d['connection'].head_object(Bucket=bucket, Key=name) 27 | 28 | def account_username(self): 29 | return self.options.get('ACCESS_KEY_ID', '') 30 | 31 | def account_container(self): 32 | return self.options.get('BUCKET', '') 33 | 34 | def authenticate(self, calling_format=None): 35 | access_key_id = self.account_username() 36 | secret_access_key = self.options.get('SECRET_ACCESS_KEY', '') 37 | endpoint_url = self.options.get('ENDPOINT_URL', None) 38 | bucket = self.account_container() 39 | if access_key_id and secret_access_key: 40 | self.d['connection'] = boto3.client('s3', aws_access_key_id=access_key_id, 41 | aws_secret_access_key=secret_access_key, 42 | endpoint_url=endpoint_url) 43 | else: 44 | self.d['connection'] = boto3.client('s3') 45 | self.d['bucket'] = bucket 46 | 47 | def list_remote_files(self): 48 | rtn = set() 49 | response = self.d['connection'].list_objects_v2(Bucket=self.d['bucket']) 50 | if 'Contents' in response: 51 | for obj in response['Contents']: 52 | rtn.add(obj['Key']) 53 | return rtn 54 | 55 | def delete_remote_file(self, remote_name): 56 | self.d['connection'].delete_object(Bucket=self.d['bucket'], Key=remote_name) 57 | 58 | def compare_file(self, local_name, remote_name): 59 | obj = self._get_object(remote_name) 60 | local_hash = self._get_local_file_hash(local_name) 61 | return local_hash == obj['ETag'][1:-1] 62 | 63 | def upload_file(self, local_name, remote_name): 64 | default_content_type = self.options.get('DEFAULT_CONTENT_TYPE', 'application/octet-stream') 65 | content_type, _ = mimetypes.guess_type(local_name) 66 | if content_type is None: 67 | content_type = default_content_type 68 | extra_args = {'ContentType': content_type} 69 | self.d['connection'].upload_file(local_name, self.d['bucket'], remote_name, ExtraArgs=extra_args) 70 | 71 | def create_remote_dir(self, remote_dir_name): 72 | # not required for S3 buckets 73 | return True 74 | 75 | backend_class = AmazonS3Backend 76 | -------------------------------------------------------------------------------- /tests/test_redirects.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | from tempfile import TemporaryDirectory 4 | from django.test import TestCase 5 | from django.contrib.redirects.models import Redirect 6 | from django_distill.renderer import render_static_redirect, render_redirects 7 | 8 | 9 | def null(*args, **kwargs): 10 | pass 11 | 12 | 13 | class DjangoDistillRedirectsTestSuite(TestCase): 14 | 15 | def test_template(self): 16 | test_url = 'https://example.com/' 17 | test_template = render_static_redirect(test_url) 18 | expected_template = ('\n' 19 | '\n' 20 | '\n' 21 | '\n' 22 | '\n' 23 | 'Redirecting to https://example.com/\n' 24 | '\n' 25 | '\n' 26 | '\n' 27 | '

Redirecting to https://example.com/

\n' 28 | '

If you are not automatically redirected please click this link

\n' 29 | '') 30 | self.assertEqual(test_template, expected_template.encode()) 31 | 32 | def test_redirects(self): 33 | # Create some test redirects 34 | Redirect.objects.create(site_id=1, old_path='/redirect-from1/', new_path='/redirect-to1/') 35 | Redirect.objects.create(site_id=1, old_path='/redirect-from2/', new_path='/redirect-to2/') 36 | Redirect.objects.create(site_id=1, old_path='/redirect-from3/', new_path='/redirect-to3/') 37 | Redirect.objects.create(site_id=1, old_path='/redirect-from4/test.html', new_path='/redirect-to4/test.html') 38 | Redirect.objects.create(site_id=1, old_path='/redirect-from5/noslash', new_path='/redirect-to5/noslash') 39 | Redirect.objects.create(site_id=1, old_path='/redirect-from6/deep/redirect/path/', new_path='/redirect-to6/') 40 | # Render the redirect templates 41 | with TemporaryDirectory() as tempdir: 42 | render_redirects(tempdir, null) 43 | # Test the redirect templates exist 44 | for redirect in Redirect.objects.all(): 45 | redirect_path = redirect.old_path.lstrip('/') 46 | if redirect_path.lower().endswith('.html'): 47 | test_file_path = str(Path(tempdir) / redirect_path) 48 | else: 49 | test_file_path = str(Path(tempdir) / redirect_path / 'index.html') 50 | self.assertTrue(os.path.exists(test_file_path)) 51 | with open(test_file_path, 'rb') as f: 52 | test_file_contents = f.read() 53 | expected_file_contents = render_static_redirect(redirect.new_path) 54 | self.assertEqual(test_file_contents, expected_file_contents) 55 | -------------------------------------------------------------------------------- /django_distill/management/commands/distill-test-publish.py: -------------------------------------------------------------------------------- 1 | import os 2 | from binascii import hexlify 3 | from tempfile import NamedTemporaryFile 4 | from django.conf import settings 5 | from django.core.management.base import (BaseCommand, CommandError) 6 | from django_distill.backends import get_backend 7 | 8 | 9 | class Command(BaseCommand): 10 | 11 | help = 'Tests a distill publishing target' 12 | 13 | def add_arguments(self, parser): 14 | parser.add_argument('publish_target_name', nargs='?', type=str) 15 | 16 | def handle(self, *args, **options): 17 | publish_target_name = options.get('publish_target_name') 18 | if not publish_target_name: 19 | publish_target_name = 'default' 20 | publish_targets = getattr(settings, 'DISTILL_PUBLISH', {}) 21 | publish_target = publish_targets.get(publish_target_name) 22 | if type(publish_target) != dict: 23 | e = 'Invalid publish target name: "{}"'.format(publish_target_name) 24 | e += ', check your settings.DISTILL_PUBLISH values' 25 | raise CommandError(e) 26 | publish_engine = publish_target.get('ENGINE') 27 | if not publish_engine: 28 | e = 'Publish target {} has no ENGINE'.format(publish_target_name) 29 | raise CommandError(e) 30 | self.stdout.write('') 31 | self.stdout.write('You have requested to test a publishing target:') 32 | self.stdout.write('') 33 | self.stdout.write(' Name: {}'.format(publish_target_name)) 34 | self.stdout.write(' Engine: {}'.format(publish_engine)) 35 | self.stdout.write('') 36 | ans = input('Type \'yes\' to continue, or \'no\' to cancel: ') 37 | if ans.lower() == 'yes': 38 | self.stdout.write('') 39 | self.stdout.write('Testing publishing target...') 40 | else: 41 | raise CommandError('Testing publishing target cancelled.') 42 | self.stdout.write('') 43 | self.stdout.write('Connecting to backend engine') 44 | backend_class = get_backend(publish_engine) 45 | random_file = NamedTemporaryFile(delete=False) 46 | random_str = hexlify(os.urandom(16)) 47 | random_file.write(random_str) 48 | random_file.close() 49 | backend = backend_class(os.path.dirname(random_file.name), 50 | publish_target) 51 | self.stdout.write('Authenticating') 52 | backend.authenticate() 53 | remote_file_name = os.path.basename(random_file.name) 54 | self.stdout.write('Uploading test file: {}'.format(random_file.name)) 55 | backend.upload_file(random_file.name, remote_file_name) 56 | url = backend.remote_url(random_file.name) 57 | self.stdout.write('Verifying remote test file: {}'.format(url)) 58 | if backend.check_file(random_file.name, url): 59 | self.stdout.write('File uploaded correctly, file hash is correct') 60 | else: 61 | msg = 'File error, remote file hash differs from local hash' 62 | self.stderr.write(msg) 63 | self.stdout.write('Final checks') 64 | backend.final_checks() 65 | self.stdout.write('Deleting remote test file') 66 | backend.delete_remote_file(remote_file_name) 67 | if os.path.exists(random_file.name): 68 | self.stdout.write('Deleting local test file') 69 | os.unlink(random_file.name) 70 | self.stdout.write('') 71 | self.stdout.write('Backend testing complete.') 72 | -------------------------------------------------------------------------------- /django_distill/management/commands/distill-local.py: -------------------------------------------------------------------------------- 1 | import os 2 | from shutil import rmtree 3 | from django.core.management.base import (BaseCommand, CommandError) 4 | from django.conf import settings 5 | from django_distill.distill import urls_to_distill 6 | from django_distill.renderer import (run_collectstatic, render_to_dir, 7 | copy_static_and_media_files, render_redirects) 8 | from django_distill.errors import DistillError 9 | 10 | 11 | class Command(BaseCommand): 12 | 13 | help = 'Generates a static local site using distill' 14 | 15 | def add_arguments(self, parser): 16 | parser.add_argument('output_dir', nargs='?', type=str) 17 | parser.add_argument('--collectstatic', dest='collectstatic', action='store_true') 18 | parser.add_argument('--quiet', dest='quiet', action='store_true') 19 | parser.add_argument('--force', dest='force', action='store_true') 20 | parser.add_argument('--exclude-staticfiles', dest='exclude_staticfiles', action='store_true') 21 | parser.add_argument('--generate-redirects', dest='generate_redirects', action='store_true') 22 | parser.add_argument('--parallel-render', dest='parallel_render', type=int, default=1) 23 | 24 | def _quiet(self, *args, **kwargs): 25 | pass 26 | 27 | def handle(self, *args, **options): 28 | output_dir = options.get('output_dir') 29 | collectstatic = options.get('collectstatic') 30 | quiet = options.get('quiet') 31 | force = options.get('force') 32 | exclude_staticfiles = options.get('exclude_staticfiles') 33 | generate_redirects = options.get('generate_redirects') 34 | parallel_render = options.get('parallel_render') 35 | if quiet: 36 | stdout = self._quiet 37 | else: 38 | stdout = self.stdout.write 39 | if not output_dir: 40 | output_dir = getattr(settings, 'DISTILL_DIR', None) 41 | if not output_dir: 42 | e = 'Usage: ./manage.py distill-local [directory]' 43 | raise CommandError(e) 44 | if collectstatic: 45 | run_collectstatic(stdout) 46 | if not exclude_staticfiles and not os.path.isdir(settings.STATIC_ROOT): 47 | e = 'Static source directory does not exist, run collectstatic' 48 | raise CommandError(e) 49 | output_dir = os.path.abspath(os.path.expanduser(output_dir)) 50 | stdout('') 51 | stdout('You have requested to create a static version of') 52 | stdout('this site into the output path directory:') 53 | stdout('') 54 | stdout(' Source static path: {}'.format(settings.STATIC_ROOT)) 55 | stdout(' Distill output path: {}'.format(output_dir)) 56 | stdout('') 57 | if os.path.isdir(output_dir): 58 | stdout('Distill output directory exists, clean up?') 59 | stdout('This will delete and recreate all files in the output dir') 60 | stdout('') 61 | if force: 62 | ans = 'yes' 63 | else: 64 | ans = input('Type \'yes\' to continue, or \'no\' to cancel: ') 65 | if ans.lower() == 'yes': 66 | stdout('Recreating output directory...') 67 | rmtree(output_dir) 68 | os.makedirs(output_dir) 69 | else: 70 | raise CommandError('Distilling site cancelled.') 71 | else: 72 | if force: 73 | ans = 'yes' 74 | else: 75 | ans = input('Does not exist, create it? (YES/no): ') 76 | if ans.lower() == 'yes': 77 | stdout('Creating directory...') 78 | os.makedirs(output_dir) 79 | else: 80 | raise CommandError('Aborting...') 81 | stdout('') 82 | stdout('Generating static site into directory: {}'.format(output_dir)) 83 | try: 84 | render_to_dir(output_dir, urls_to_distill, stdout, parallel_render=parallel_render) 85 | if not exclude_staticfiles: 86 | copy_static_and_media_files(output_dir, stdout) 87 | except DistillError as err: 88 | raise CommandError(str(err)) from err 89 | stdout('') 90 | if generate_redirects: 91 | stdout('Generating redirects') 92 | render_redirects(output_dir, stdout) 93 | stdout('') 94 | stdout('Site generation complete.') 95 | -------------------------------------------------------------------------------- /django_distill/management/commands/distill-publish.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | from django.conf import settings 4 | from django.core.management.base import (BaseCommand, CommandError) 5 | from django_distill.backends import get_backend 6 | from django_distill.distill import urls_to_distill 7 | from django_distill.errors import DistillError 8 | from django_distill.renderer import (run_collectstatic, render_to_dir, 9 | copy_static_and_media_files, render_redirects) 10 | from django_distill.publisher import publish_dir 11 | 12 | 13 | class Command(BaseCommand): 14 | 15 | help = 'Distills a site into a temporary local directory then publishes it' 16 | 17 | def add_arguments(self, parser): 18 | parser.add_argument('publish_target_name', nargs='?', type=str) 19 | parser.add_argument('--collectstatic', dest='collectstatic', 20 | action='store_true') 21 | parser.add_argument('--quiet', dest='quiet', action='store_true') 22 | parser.add_argument('--force', dest='force', action='store_true') 23 | parser.add_argument('--exclude-staticfiles', dest='exclude_staticfiles', 24 | action='store_true') 25 | parser.add_argument('--skip-verify', dest='skip_verify', action='store_true') 26 | parser.add_argument('--ignore-remote-content', dest='ignore_remote_content', action='store_true') 27 | parser.add_argument('--parallel-publish', dest='parallel_publish', type=int, default=1) 28 | parser.add_argument('--generate-redirects', dest='generate_redirects', action='store_true') 29 | parser.add_argument('--parallel-render', dest='parallel_render', type=int, default=1) 30 | 31 | def _quiet(self, *args, **kwargs): 32 | pass 33 | 34 | def handle(self, *args, **options): 35 | publish_target_name = options.get('publish_target_name') 36 | if not publish_target_name: 37 | publish_target_name = 'default' 38 | publish_targets = getattr(settings, 'DISTILL_PUBLISH', {}) 39 | publish_target = publish_targets.get(publish_target_name) 40 | if type(publish_target) != dict: 41 | e = 'Invalid publish target name: "{}"'.format(publish_target_name) 42 | e += ', check your settings.DISTILL_PUBLISH values' 43 | raise CommandError(e) 44 | publish_engine = publish_target.get('ENGINE') 45 | if not publish_engine: 46 | e = 'Publish target {} has no ENGINE'.format(publish_target_name) 47 | raise CommandError(e) 48 | collectstatic = options.get('collectstatic') 49 | exclude_staticfiles = options.get('exclude_staticfiles') 50 | skip_verify = options.get('skip_verify', False) 51 | parallel_publish = options.get('parallel_publish') 52 | ignore_remote_content = options.get('ignore_remote_content', False) 53 | quiet = options.get('quiet') 54 | force = options.get('force') 55 | generate_redirects = options.get('generate_redirects') 56 | parallel_render = options.get('parallel_render') 57 | if quiet: 58 | stdout = self._quiet 59 | else: 60 | stdout = self.stdout.write 61 | with tempfile.TemporaryDirectory() as output_dir: 62 | if not output_dir.endswith(os.sep): 63 | output_dir += os.sep 64 | backend_class = get_backend(publish_engine) 65 | backend = backend_class(output_dir, publish_target) 66 | username = backend.account_username() 67 | container = backend.account_container() 68 | stdout('') 69 | stdout('You have requested to distill and publish this site') 70 | stdout('to the following target:') 71 | stdout('') 72 | stdout(' Settings name: {}'.format(publish_target_name)) 73 | stdout(' Engine: {}'.format(publish_engine)) 74 | stdout(' Username: {}'.format(username)) 75 | stdout(' Container: {}'.format(container)) 76 | stdout('') 77 | if collectstatic: 78 | run_collectstatic(stdout) 79 | if not exclude_staticfiles and not os.path.isdir(settings.STATIC_ROOT): 80 | e = 'Static source directory does not exist, run collectstatic' 81 | raise CommandError(e) 82 | if force: 83 | ans = 'yes' 84 | else: 85 | ans = input('Type \'yes\' to continue, or \'no\' to cancel: ') 86 | if ans.lower() == 'yes': 87 | pass 88 | else: 89 | raise CommandError('Publishing site cancelled.') 90 | self.stdout.write('') 91 | msg = 'Generating static site into directory: {}' 92 | stdout(msg.format(output_dir)) 93 | try: 94 | render_to_dir(output_dir, urls_to_distill, stdout, parallel_render=parallel_render) 95 | if not exclude_staticfiles: 96 | copy_static_and_media_files(output_dir, stdout) 97 | except DistillError as err: 98 | raise CommandError(str(err)) from err 99 | stdout('') 100 | if generate_redirects: 101 | stdout('Generating redirects') 102 | render_redirects(output_dir, stdout) 103 | stdout('') 104 | stdout('Publishing site') 105 | backend.index_local_files() 106 | publish_dir(backend, stdout, not skip_verify, parallel_publish, ignore_remote_content) 107 | stdout('') 108 | stdout('Site generation and publishing complete.') 109 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to django-distill 2 | 3 | Thank you for showing an interest in contributing to `django-distill`! 4 | 5 | 6 | ## Code of Conduct 7 | 8 | This project and everyone participating in it is governed by the [CONTRIBUTING.md Code of Conduct](blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. 9 | 10 | 11 | ## Questions 12 | 13 | > If you want to ask a question, we assume that you have read the available [Documentation](blob/master/README.md). 14 | 15 | Before you ask a question please look for existing [Issues](/issues) that might help you. In case you have found a suitable issue and still need clarification you can ask your question in the existing issue. 16 | 17 | If you then still feel the need to ask a question and need clarification please: 18 | 19 | - Open an [Issue](/issues/new). 20 | - Provide as much context as you can about what you're running into. 21 | - Provide project and platform versions (Python, Django, etc), depending on what seems relevant. 22 | 23 | We review issues regularly. 24 | 25 | 26 | ## Contributing 27 | 28 | > ### Legal Notice 29 | > When contributing to this project you must agree that you have authored all of the content of your contribution, that you have the necessary rights to the content and that the content you contribute may be provided under the project license (MIT). 30 | 31 | 32 | ### Reporting Bugs 33 | 34 | 35 | #### Before Submitting a Bug Report 36 | 37 | A good bug report shouldn't leave others needing to chase you up for more information. We ask you to investigate carefully, collect information and describe the issue in detail in your report. 38 | 39 | - Make sure that you are using the latest version. 40 | - Determine if your bug is really a bug and not an error on your side (for example using incompatible environment components/versions). 41 | - Make sure that you have read the [documentation](blob/master/README.md). 42 | - To see if other users have experienced (and potentially already solved) the same issue you are having, check if there is not already a bug report existing for your bug or error in the [issues](issues). 43 | - Also make sure to search the internet to see if users outside of the GitHub community have discussed the issue. 44 | - Collect information about the bug such as stack traces, OS and platform version, versions of Python and Django: 45 | - Can you reliably reproduce the issue? Can you reproduce this issue with the previous version? 46 | 47 | 48 | #### Submitting a Bug Report 49 | 50 | > You must never report security related issues which contain sensitive information, vulnerabilities or bugs including sensitive information to the issue tracker, or elsewhere in public. 51 | > In the extremely unlikely event you do find a critical security issue with a static site generator you can send an email to with the details 52 | 53 | 54 | We use GitHub issues to track bugs and features. If you run into an issue with the project: 55 | 56 | - Open an [Issue](/issues/new). 57 | - Explain the behavior you would expect and the actual behavior. 58 | - Please provide as much context as possible and describe the *reproduction steps* that someone else can follow to recreate the issue on their own. This may include a bare bones example of your code. 59 | - Provide the information you collected in the previous section. 60 | 61 | Once the issue is created: 62 | 63 | - The project team will label the issue accordingly. 64 | - A team member will try to reproduce the issue with your provided steps. 65 | - If the team is able to reproduce the issue the issue will be left to be implemented or resolved 66 | 67 | 68 | ### Suggesting Enhancements 69 | 70 | This section guides you through submitting an enhancement suggestion for CONTRIBUTING.md, **including completely new features and minor improvements to existing functionality**. Following these guidelines will help maintainers and the community to understand your suggestion and find related suggestions. 71 | 72 | 73 | #### Before Submitting an Enhancement 74 | 75 | - Make sure that you are using the latest version. 76 | - Read the [documentation](blob/master/README.md) carefully and find out if the functionality is already covered. 77 | - Perform a [search](/issues) to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one. 78 | - Find out whether your idea fits with the scope and aims of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Keep in mind that we want features that will be useful to the majority of people using `django-distill` and not just a small subset. 79 | 80 | 81 | #### Submitting an Enhancement Suggestion 82 | 83 | Enhancement suggestions are tracked as [GitHub issues](/issues). 84 | 85 | - Use a **clear and descriptive title** for the issue to identify the suggestion. 86 | - Provide a **step-by-step description of the suggested enhancement** in as many details as possible. 87 | - **Describe the current behavior** and **explain which behavior you expected to see instead** and why. At this point you can also tell which alternatives do not work for you. 88 | - **Explain why this enhancement would be useful** to most CONTRIBUTING.md users. 89 | 90 | 91 | ### Code Contribution 92 | 93 | Sensible and well formatted, easy to follow and concise pull requests are welcome. We may not merge your requested feature but it will be reviewed and discussed. 94 | 95 | 96 | 97 | ### Improving The Documentation 98 | 99 | Sensible and well formatted, easy to follow and concise amendments to the documentation submitted as pull requests are welcome. 100 | 101 | 102 | 103 | ## Maintainers 104 | 105 | `django-distill` is not currently looking for additional maintainers or administrators, however contributions are welcome. 106 | -------------------------------------------------------------------------------- /django_distill/backends/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import warnings 4 | import mimetypes 5 | from hashlib import md5 6 | from binascii import hexlify 7 | import requests 8 | from urllib.parse import urlsplit, urlunsplit 9 | from django_distill.errors import DistillPublishError 10 | from django_distill.renderer import filter_dirs 11 | 12 | 13 | class BackendBase(object): 14 | ''' 15 | Generic base class for all backends, mostly an interface / template. 16 | ''' 17 | 18 | REQUIRED_OPTIONS = () 19 | 20 | def __init__(self, source_dir, options): 21 | if not source_dir.endswith(os.sep): 22 | source_dir += os.sep 23 | self.source_dir = source_dir 24 | self.options = options 25 | self.local_files = set() 26 | self.local_dirs = set() 27 | self.remote_files = set() 28 | self.remote_url_parts = urlsplit(options.get('PUBLIC_URL', '')) 29 | self.d = {} 30 | self._validate_options() 31 | 32 | def _validate_options(self): 33 | for o in self.REQUIRED_OPTIONS: 34 | if o not in self.options: 35 | e = 'Missing required settings value for this backend: {}' 36 | raise DistillPublishError(e.format(o)) 37 | 38 | def index_local_files(self): 39 | for root, dirs, files in os.walk(self.source_dir): 40 | dirs[:] = filter_dirs(dirs) 41 | for d in dirs: 42 | self.local_dirs.add(os.path.join(root, d)) 43 | for f in files: 44 | self.local_files.add(os.path.join(root, f)) 45 | 46 | def _get_local_file_hash(self, file_path, digest_func=md5, chunk=1048576): 47 | # md5 is used by Amazon S3, Rackspace Cloud Files and Google Storage 48 | if not self._file_exists(file_path): 49 | return None 50 | digest = digest_func() 51 | with open(file_path, 'rb') as f: 52 | while True: 53 | data = f.read(chunk) 54 | if not data: 55 | break 56 | digest.update(data) 57 | return digest.hexdigest() 58 | 59 | def _get_url_hash(self, url, digest_func=md5, chunk=1024): 60 | # CDN cache buster 61 | url += '?' + hexlify(os.urandom(16)).decode('utf-8') 62 | request = requests.get(url, stream=True) 63 | if request.status_code == 404: 64 | return False 65 | digest = digest_func() 66 | for block in request.iter_content(chunk_size=chunk): 67 | if block: 68 | digest.update(block) 69 | return digest.hexdigest() 70 | 71 | def _file_exists(self, file_path): 72 | return os.path.isfile(file_path) 73 | 74 | def local_file_mimetype(self, local_name): 75 | try: 76 | return mimetypes.guess_type(local_name)[0] 77 | except Exception: 78 | return 'application/octet-stream' 79 | 80 | def remote_url(self, local_name): 81 | if local_name[:len(self.source_dir)] != self.source_dir: 82 | raise DistillPublishError('File {} is not in source dir {}'.format( 83 | local_name, self.source_dir)) 84 | truncated = local_name[len(self.source_dir):] 85 | remote_file = '/'.join(truncated.split(os.sep)) 86 | remote_uri = self.remote_url_parts.path + remote_file 87 | return urlunsplit((self.remote_url_parts.scheme, 88 | self.remote_url_parts.netloc, remote_uri, '', '')) 89 | 90 | def list_local_dirs(self): 91 | return self.local_dirs 92 | 93 | def list_local_files(self): 94 | return self.local_files 95 | 96 | def check_file(self, local_name, url): 97 | if not self._file_exists(local_name): 98 | raise DistillPublishError('File does not exist: {}'.format( 99 | local_name)) 100 | local_hash = self._get_local_file_hash(local_name) 101 | remote_hash = self._get_url_hash(url) 102 | return local_hash == remote_hash 103 | 104 | def final_checks(self): 105 | pass 106 | 107 | def remote_path(self, local_name): 108 | return local_name[len(self.source_dir):] 109 | 110 | def account_username(self): 111 | raise NotImplementedError('account_username must be implemented') 112 | 113 | def account_container(self): 114 | raise NotImplementedError('account_container must be implemented') 115 | 116 | def authenticate(self): 117 | raise NotImplementedError('authenticate must be implemented') 118 | 119 | def list_remote_files(self): 120 | raise NotImplementedError('list_remote_files must be implemented') 121 | 122 | def delete_remote_file(self, remote_name): 123 | raise NotImplementedError('delete_remote_file must be implemented') 124 | 125 | def compare_file(self, local_name, remote_name): 126 | raise NotImplementedError('compare_file must be implemented') 127 | 128 | def upload_file(self, local_name, remote_name): 129 | raise NotImplementedError('upload_file must be implemented') 130 | 131 | def create_remote_dir(self, remote_dir_name): 132 | raise NotImplementedError('create_remote_dir must be implemented') 133 | 134 | 135 | def get_backend(engine): 136 | with warnings.catch_warnings(): 137 | warnings.simplefilter('ignore') 138 | try: 139 | backend = __import__(engine, globals(), locals(), 140 | ['backend_class']) 141 | except ImportError as e: 142 | err = 'Failed to import backend engine: {} ({})\n' 143 | sys.stderr.write(err.format(engine, e)) 144 | raise 145 | module = getattr(backend, 'backend_class') 146 | if not module: 147 | raise ImportError('Backend engine has no backend_class attribute') 148 | return module 149 | -------------------------------------------------------------------------------- /django_distill/backends/microsoft_azure_storage.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import warnings 4 | from urllib.parse import urlsplit, urlunsplit, quote_plus 5 | from time import sleep 6 | from base64 import b64decode 7 | from binascii import hexlify 8 | 9 | 10 | try: 11 | from azure.storage.blob import BlobServiceClient, ContentSettings 12 | except ImportError: 13 | name = 'django_distill.backends.azure_storage' 14 | pipm = 'azure-storage-blob' 15 | sys.stdout.write('{} backend requires {}:\n'.format(name, pipm)) 16 | sys.stdout.write('$ pip install django-distill[microsoft]{}\n\n'.format(pipm)) 17 | raise 18 | 19 | 20 | from django_distill.errors import DistillPublishError 21 | from django_distill.backends import BackendBase 22 | 23 | 24 | class AzureBlobStorateBackend(BackendBase): 25 | ''' 26 | Publisher for Azure Blob Storage. Implements the BackendBase. Azure 27 | static websites in containers are relatively slow to make the files 28 | available via the public URL. To work around this uploaded files 29 | are cached and then verified in a loop at the end with up to 30 | RETRY_ATTEMPTS attempts with a delay of SLEEP_BETWEEN_RETRIES seconds 31 | between each attempt. 32 | ''' 33 | 34 | REQUIRED_OPTIONS = ('ENGINE', 'CONNECTION_STRING') 35 | RETRY_ATTEMPTS = 30 36 | SLEEP_BETWEEN_RETRIES = 3 37 | 38 | def account_username(self): 39 | return 40 | 41 | def account_container(self): 42 | return '$web' 43 | 44 | def connection_string(self): 45 | return self.options.get('CONNECTION_STRING', '') 46 | 47 | def _get_blob(self, name): 48 | return self.d['connection'].get_blob_client( 49 | container=self.account_container(), 50 | blob=name 51 | ) 52 | 53 | def _get_blob_url(self, blob): 54 | blob_parts = urlsplit(blob.url) 55 | prefix = '/{}/'.format(quote_plus(self.account_container())) 56 | path = blob_parts.path 57 | if path.startswith(prefix): 58 | path = path[len(prefix):] 59 | parts = (self.remote_url_parts.scheme, self.remote_url_parts.netloc, 60 | path, None, None) 61 | return urlunsplit(parts) 62 | 63 | def authenticate(self): 64 | self.d['connection'] = BlobServiceClient.from_connection_string( 65 | conn_str=self.connection_string() 66 | ) 67 | 68 | def list_remote_files(self): 69 | container = self.d['connection'].get_container_client( 70 | container=self.account_container() 71 | ) 72 | rtn = set() 73 | for obj in container.list_blobs(): 74 | rtn.add(obj.name) 75 | return rtn 76 | 77 | def delete_remote_file(self, remote_name): 78 | container = self.d['connection'].get_container_client( 79 | container=self.account_container() 80 | ) 81 | return container.delete_blob(remote_name) 82 | 83 | def check_file(self, local_name, url): 84 | # Azure uploads are checked in bulk at the end of the uploads, do 85 | # nothing here 86 | return True 87 | 88 | def compare_file(self, local_name, remote_name): 89 | blob = self._get_blob(remote_name) 90 | properties = blob.get_blob_properties() 91 | content_settings = properties.content_settings 92 | content_md5 = properties.get('content_settings', {}).get('content_md5') 93 | if not content_md5: 94 | return False 95 | local_hash = self._get_local_file_hash(local_name) 96 | remote_hash = str(hexlify(bytes(content_md5)).decode()) 97 | return local_hash == remote_hash 98 | 99 | def upload_file(self, local_name, remote_name): 100 | blob = self._get_blob(remote_name) 101 | mimetype = self.local_file_mimetype(local_name) 102 | content_settings = ContentSettings(content_type=mimetype) 103 | with open(local_name, 'rb') as data: 104 | result = blob.upload_blob( 105 | data, overwrite=True, content_settings=content_settings) 106 | if result: 107 | actual_url = self._get_blob_url(blob) 108 | self.d.setdefault('azure_uploads_to_check', []).append( 109 | (local_name, remote_name, actual_url) 110 | ) 111 | return result 112 | 113 | def _check_file(self, local_name, actual_url): 114 | # Azure specific patched check_file with retries to account for Azure 115 | # being slow 116 | err = ('Failed to upload local file "{}" blob to Azure container at ' 117 | 'URL "{}" not available over the public URL after {} attempts') 118 | local_hash = self._get_local_file_hash(local_name) 119 | for i in range(self.RETRY_ATTEMPTS): 120 | remote_hash = self._get_url_hash(actual_url) 121 | if not remote_hash: 122 | sleep(self.SLEEP_BETWEEN_RETRIES) 123 | continue 124 | if local_hash == remote_hash: 125 | return True 126 | DistillPublishError(err.format(local_name, actual_url, i + 1)) 127 | 128 | def final_checks(self): 129 | # Iterate over any cached files to check and verify they have been 130 | # uploaded correctly. 131 | to_check = self.d.setdefault('azure_uploads_to_check', []) 132 | for (local_name, remote_name, actual_url) in to_check: 133 | # Verify the upload, this may require retries 134 | self._check_file(local_name, actual_url) 135 | # If we reached here no DistillPublishError was raised 136 | return True 137 | 138 | def create_remote_dir(self, remote_dir_name): 139 | # not required for Azure Blob Storage containers 140 | return True 141 | 142 | 143 | backend_class = AzureBlobStorateBackend 144 | -------------------------------------------------------------------------------- /tests/urls.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from django.conf import settings 3 | from django.http import HttpResponse 4 | from django.urls import include, path, reverse 5 | from django.conf.urls.i18n import i18n_patterns 6 | from django.shortcuts import render 7 | from django.utils import timezone 8 | from django.contrib.flatpages.views import flatpage as flatpage_view 9 | from django.contrib.sitemaps import Sitemap 10 | from django.contrib.sitemaps.views import sitemap 11 | from django.apps import apps as django_apps 12 | from django_distill import distill_path, distill_re_path 13 | 14 | 15 | class TestStaticViewSitemap(Sitemap): 16 | 17 | priority = 0.5 18 | changefreq = 'daily' 19 | 20 | def items(self): 21 | return ['path-sitemap'] 22 | 23 | def location(self, item): 24 | return reverse(item) 25 | 26 | 27 | sitemap_dict = { 28 | 'static': TestStaticViewSitemap, 29 | } 30 | 31 | 32 | def test_no_param_view(request): 33 | return HttpResponse(b'test', content_type='application/octet-stream') 34 | 35 | 36 | def test_positional_param_view(request, param): 37 | if not isinstance(param, str): 38 | param = str(param) 39 | return HttpResponse(b'test' + param.encode(), 40 | content_type='application/octet-stream') 41 | 42 | 43 | def test_named_param_view(request, param=None): 44 | if not isinstance(param, str): 45 | param = str(param) 46 | return HttpResponse(b'test' + param.encode(), 47 | content_type='application/octet-stream') 48 | 49 | 50 | def test_session_view(request): 51 | request.session['test'] = 'test' 52 | return HttpResponse(b'test', content_type='application/octet-stream') 53 | 54 | 55 | def test_broken_view(request): 56 | # Trigger a normal Python exception when rendering 57 | a = 1 / 0 58 | 59 | 60 | def test_http404_view(request): 61 | response = HttpResponse(b'404', content_type='application/octet-stream') 62 | response.status_code = 404 63 | return response 64 | 65 | 66 | def test_humanize_view(request): 67 | now = timezone.now() 68 | one_hour_ago = now - timedelta(hours=1) 69 | nineteen_hours_ago = now - timedelta(hours=19) 70 | return render(request, 'humanize.html', { 71 | 'now': now, 72 | 'one_hour_ago': one_hour_ago, 73 | 'nineteen_hours_ago': nineteen_hours_ago 74 | }) 75 | 76 | 77 | def test_no_param_func(): 78 | return None 79 | 80 | 81 | def test_positional_param_func(): 82 | return ('12345', '67890') 83 | 84 | 85 | def test_named_param_func(): 86 | return [{'param': 'test'}] 87 | 88 | 89 | def test_flatpages_func(): 90 | Site = django_apps.get_model('sites.Site') 91 | current_site = Site.objects.get_current() 92 | flatpages = current_site.flatpage_set.filter(registration_required=False) 93 | for flatpage in flatpages: 94 | yield {'url': flatpage.url} 95 | 96 | 97 | def test_request_has_resolver_match(request): 98 | return HttpResponse(request.resolver_match.func.__name__) 99 | 100 | 101 | urlpatterns = [ 102 | 103 | path('path/namespace1/', 104 | include('tests.namespaced_urls', namespace='test_namespace')), 105 | path('path/no-namespace/', 106 | include('tests.no_namespaced_urls')), 107 | 108 | ] 109 | 110 | 111 | urlpatterns += i18n_patterns( 112 | path('path/i18n/', include('tests.i18n_urls')), 113 | ) 114 | 115 | 116 | if settings.HAS_RE_PATH: 117 | urlpatterns += [ 118 | 119 | distill_re_path(r'^re_path/$', 120 | test_no_param_view, 121 | name='re_path-no-param', 122 | distill_func=test_no_param_func, 123 | distill_file='test'), 124 | distill_re_path(r'^re_path-no-func/$', 125 | test_no_param_view, 126 | name='re_path-no-param-no-func', 127 | distill_file='test'), 128 | distill_re_path(r'^re_path/([\d]+)$', 129 | test_positional_param_view, 130 | name='re_path-positional-param', 131 | distill_func=test_positional_param_func), 132 | distill_re_path(r'^re_path/x/([\d]+)$', 133 | test_positional_param_view, 134 | name='re_path-positional-param-custom', 135 | distill_func=test_positional_param_func, 136 | distill_file="re_path/x/{}.html"), 137 | distill_re_path(r'^re_path/(?P[\w]+)$', 138 | test_named_param_view, 139 | name='re_path-named-param', 140 | distill_func=test_named_param_func), 141 | distill_re_path(r'^re_path/x/(?P[\w]+)$', 142 | test_named_param_view, 143 | name='re_path-named-param-custom', 144 | distill_func=test_named_param_func, 145 | distill_file="re_path/x/{param}.html"), 146 | distill_re_path(r'^re_path/broken$', 147 | test_broken_view, 148 | name='re_path-broken', 149 | distill_func=test_no_param_func), 150 | distill_re_path(r'^re_path/ignore-sessions$', 151 | test_session_view, 152 | name='re_path-ignore-sessions', 153 | distill_func=test_no_param_func), 154 | distill_re_path(r'^re_path/404$', 155 | test_http404_view, 156 | name='re_path-404', 157 | distill_status_codes=(404,), 158 | distill_func=test_no_param_func), 159 | distill_re_path(r'^re_path/flatpage(?P.+)$', 160 | flatpage_view, 161 | name='re_path-flatpage', 162 | distill_func=test_flatpages_func), 163 | 164 | ] 165 | 166 | 167 | if settings.HAS_PATH: 168 | urlpatterns += [ 169 | 170 | distill_path('path/', 171 | test_no_param_view, 172 | name='path-no-param', 173 | distill_func=test_no_param_func, 174 | distill_file='test'), 175 | distill_path('path-no-func/', 176 | test_no_param_view, 177 | name='path-no-param-no-func', 178 | distill_file='test'), 179 | distill_path('path/', 180 | test_positional_param_view, 181 | name='path-positional-param', 182 | distill_func=test_positional_param_func), 183 | distill_path('path/x/', 184 | test_positional_param_view, 185 | name='path-positional-param-custom', 186 | distill_func=test_positional_param_func, 187 | distill_file="path/x/{}.html"), 188 | distill_path('path/', 189 | test_named_param_view, 190 | name='path-named-param', 191 | distill_func=test_named_param_func), 192 | distill_path('path/x/', 193 | test_named_param_view, 194 | name='path-named-param-custom', 195 | distill_func=test_named_param_func, 196 | distill_file="path/x/{param}.html"), 197 | distill_path('path/broken', 198 | test_broken_view, 199 | name='path-broken', 200 | distill_func=test_no_param_func), 201 | distill_path('path/ignore-sessions', 202 | test_session_view, 203 | name='path-ignore-sessions', 204 | distill_func=test_no_param_func), 205 | distill_path('path/404', 206 | test_http404_view, 207 | name='path-404', 208 | distill_status_codes=(404,), 209 | distill_func=test_no_param_func), 210 | distill_path('path/flatpage', 211 | flatpage_view, 212 | name='path-flatpage', 213 | distill_func=test_flatpages_func), 214 | distill_path('path/test-sitemap', 215 | sitemap, 216 | {'sitemaps': sitemap_dict}, 217 | name='path-sitemap' 218 | ), 219 | distill_path(route='path/kwargs', 220 | view=test_no_param_view, 221 | name='test-kwargs'), 222 | distill_path('path/humanize', 223 | test_humanize_view, 224 | name='test-humanize'), 225 | distill_path( 226 | "path/has-resolver-match", 227 | test_request_has_resolver_match, 228 | name="test-has-resolver-match", 229 | ), 230 | 231 | ] 232 | -------------------------------------------------------------------------------- /django_distill/renderer.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import inspect 3 | import logging 4 | import os 5 | import types 6 | from shutil import copy2 7 | from concurrent.futures import ThreadPoolExecutor 8 | from django.utils.translation import activate as activate_lang 9 | from django.conf import settings, global_settings 10 | from django.urls import include as include_urls, get_resolver 11 | from django.core.exceptions import ImproperlyConfigured, MiddlewareNotUsed 12 | from django.utils.module_loading import import_string 13 | from django.test import RequestFactory 14 | from django.test.client import ClientHandler 15 | from django.urls import reverse, ResolverMatch 16 | from django.urls.exceptions import NoReverseMatch 17 | from django.core.management import call_command 18 | from django_distill.errors import DistillError 19 | 20 | 21 | logger = logging.getLogger(__name__) 22 | urlconf = get_resolver() 23 | 24 | 25 | def iter_resolved_urls(url_patterns, namespace_path=[]): 26 | url_patterns_resolved = [] 27 | for entry in url_patterns: 28 | if hasattr(entry, 'url_patterns'): 29 | if getattr(entry, 'namespace', None) is not None: 30 | url_patterns_resolved += iter_resolved_urls( 31 | entry.url_patterns, namespace_path + [entry.namespace]) 32 | else: 33 | url_patterns_resolved += iter_resolved_urls( 34 | entry.url_patterns, namespace_path) 35 | else: 36 | url_patterns_resolved.append((namespace_path, entry)) 37 | return url_patterns_resolved 38 | 39 | 40 | def load_namespace_map(): 41 | namespace_map = {} 42 | for (namespaces, url) in iter_resolved_urls(urlconf.url_patterns): 43 | if namespaces: 44 | nspath = ':'.join(namespaces) 45 | if url in namespace_map: 46 | raise DistillError(f'Ambiguous namespace for URL "{url}" in namespace ' 47 | f'"{nspath}", Distill does not support the same Django ' 48 | f'app being include()ed more than once in the same ' 49 | f'project') 50 | else: 51 | namespace_map[url] = nspath 52 | return namespace_map 53 | 54 | 55 | class DistillHandler(ClientHandler): 56 | ''' 57 | Overload ClientHandler's resolve_request(...) to return the already known 58 | and pre-resolved view method. Also overwrite any session handling middleware 59 | with the dummy session handler. 60 | ''' 61 | 62 | def __init__(self, *a, **k): 63 | self.view_func = lambda x: x 64 | self.view_uri_args = () 65 | self.view_uri_kwargs = {} 66 | self.view_args = () 67 | k['enforce_csrf_checks'] = False 68 | super().__init__(*a, *k) 69 | 70 | def set_view(self, view_func, view_uri_args, view_uri_kwargs, view_args): 71 | self.view_func = view_func 72 | self.view_uri_args = view_uri_args 73 | self.view_uri_kwargs = view_uri_kwargs 74 | self.view_args = view_args 75 | 76 | def resolve_request(self, request): 77 | for arg in self.view_args: 78 | self.view_uri_kwargs.update(**arg) 79 | request.resolver_match = ResolverMatch( 80 | self.view_func, 81 | self.view_uri_args, 82 | self.view_uri_kwargs, 83 | ) 84 | return request.resolver_match 85 | 86 | def load_middleware(self, is_async=False): 87 | ''' 88 | Replaces the standard BaseHandler.load_middleware(). This method is 89 | identical apart from not trapping all exceptions. We actually want the 90 | real Python exceptions to be raised here. 91 | ''' 92 | self._view_middleware = [] 93 | self._template_response_middleware = [] 94 | self._exception_middleware = [] 95 | get_response = self._get_response_async if is_async else self._get_response 96 | handler = get_response 97 | handler_is_async = is_async 98 | for middleware_path in reversed(settings.MIDDLEWARE): 99 | middleware = import_string(middleware_path) 100 | middleware_can_sync = getattr(middleware, 'sync_capable', True) 101 | middleware_can_async = getattr(middleware, 'async_capable', False) 102 | if not middleware_can_sync and not middleware_can_async: 103 | raise RuntimeError( 104 | 'Middleware %s must have at least one of ' 105 | 'sync_capable/async_capable set to True.' % middleware_path 106 | ) 107 | elif not handler_is_async and middleware_can_sync: 108 | middleware_is_async = False 109 | else: 110 | middleware_is_async = middleware_can_async 111 | try: 112 | # Adapt handler, if needed. 113 | adapted_handler = self.adapt_method_mode( 114 | middleware_is_async, 115 | handler, 116 | handler_is_async, 117 | debug=settings.DEBUG, 118 | name='middleware %s' % middleware_path, 119 | ) 120 | mw_instance = middleware(adapted_handler) 121 | except MiddlewareNotUsed as exc: 122 | if settings.DEBUG: 123 | if str(exc): 124 | logger.debug('MiddlewareNotUsed(%r): %s', middleware_path, exc) 125 | else: 126 | logger.debug('MiddlewareNotUsed: %r', middleware_path) 127 | continue 128 | else: 129 | handler = adapted_handler 130 | if mw_instance is None: 131 | raise ImproperlyConfigured( 132 | 'Middleware factory %s returned None.' % middleware_path 133 | ) 134 | if hasattr(mw_instance, 'process_view'): 135 | self._view_middleware.insert( 136 | 0, 137 | self.adapt_method_mode(is_async, mw_instance.process_view), 138 | ) 139 | if hasattr(mw_instance, 'process_template_response'): 140 | self._template_response_middleware.append( 141 | self.adapt_method_mode( 142 | is_async, mw_instance.process_template_response 143 | ), 144 | ) 145 | if hasattr(mw_instance, 'process_exception'): 146 | self._exception_middleware.append( 147 | self.adapt_method_mode(False, mw_instance.process_exception), 148 | ) 149 | handler = mw_instance 150 | handler_is_async = middleware_is_async 151 | handler = self.adapt_method_mode(is_async, handler, handler_is_async) 152 | self._middleware_chain = handler 153 | 154 | 155 | class DistillRender(object): 156 | ''' 157 | Renders a complete static site from all urls registered with 158 | distill_url() and then copies over all static media. 159 | ''' 160 | 161 | def __init__(self, urls_to_distill, parallel_render=1): 162 | self.urls_to_distill = urls_to_distill 163 | self.parallel_render = parallel_render 164 | self.namespace_map = load_namespace_map() 165 | # set allowed hosts to '*', static rendering shouldn't care about the hostname 166 | settings.ALLOWED_HOSTS = ['*'] 167 | 168 | def render_file(self, view_name, status_codes, view_args, view_kwargs): 169 | view_details = [] 170 | for params in self.urls_to_distill: 171 | if view_name == params[4]: 172 | view_details = params 173 | break 174 | if not view_details: 175 | raise DistillError(f'No view exists with the name: {view_name}') 176 | url, distill_func, file_name, status_codes, view_name, a, k = view_details 177 | if view_kwargs: 178 | uri = self.generate_uri(url, view_name, view_kwargs) 179 | args = view_kwargs 180 | else: 181 | uri = self.generate_uri(url, view_name, view_args) 182 | args = view_args 183 | render = self.render_view(uri, status_codes, args, a, k) 184 | file_name = self._get_filename(file_name, uri, args) 185 | return uri, file_name, render 186 | 187 | def render_all_urls(self, do_render=True): 188 | 189 | def _render(item): 190 | rtn = [] 191 | for lang in self.get_langs(): 192 | activate_lang(lang) 193 | url, view_name, param_set, status_codes, file_name_base, a, k, do_render = item 194 | uri = self.generate_uri(url, view_name, param_set) 195 | if do_render: 196 | render = self.render_view(uri, status_codes, param_set, a, k) 197 | else: 198 | render = None 199 | file_name = self._get_filename(file_name_base, uri, param_set) 200 | rtn.append((uri, file_name, render)) 201 | return rtn 202 | 203 | to_render = [] 204 | for url, distill_func, file_name_base, status_codes, view_name, a, k in self.urls_to_distill: 205 | for param_set in self.get_uri_values(distill_func, view_name): 206 | if not param_set: 207 | param_set = () 208 | elif self._is_str(param_set): 209 | param_set = (param_set,) 210 | to_render.append((url, view_name, param_set, status_codes, file_name_base, a, k, do_render)) 211 | with ThreadPoolExecutor(max_workers=self.parallel_render) as executor: 212 | results = executor.map(_render, to_render) 213 | for i18n_result in results: 214 | for uri, file_name, render in i18n_result: 215 | yield uri, file_name, render 216 | 217 | def render(self, view_name=None, status_codes=None, view_args=None, view_kwargs=None): 218 | if view_name: 219 | if not status_codes: 220 | status_codes = (200,) 221 | if not view_args: 222 | view_args = () 223 | if not view_kwargs: 224 | view_kwargs = {} 225 | return self.render_file(view_name, status_codes, view_args, view_kwargs) 226 | else: 227 | return self.render_all_urls() 228 | 229 | def urls(self): 230 | for (uri, file_name, _) in self.render_all_urls(do_render=False): 231 | yield uri, file_name 232 | 233 | def get_langs(self): 234 | langs = [] 235 | LANGUAGE_CODE = str(getattr(settings, 'LANGUAGE_CODE', 'en')) 236 | GLOBAL_LANGUAGES = list(getattr(global_settings, 'LANGUAGES', [])) 237 | try: 238 | LANGUAGES = list(getattr(settings, 'LANGUAGES', [])) 239 | except (ValueError, TypeError, AttributeError): 240 | LANGUAGES = [] 241 | try: 242 | DISTILL_LANGUAGES = list(getattr(settings, 'DISTILL_LANGUAGES', [])) 243 | except (ValueError, TypeError, AttributeError) as e: 244 | DISTILL_LANGUAGES = [] 245 | if LANGUAGES != GLOBAL_LANGUAGES: 246 | for lang_code, lang_name in LANGUAGES: 247 | langs.append(lang_code) 248 | if LANGUAGE_CODE not in DISTILL_LANGUAGES and LANGUAGE_CODE not in langs: 249 | langs.append(LANGUAGE_CODE) 250 | for lang in DISTILL_LANGUAGES: 251 | langs.append(lang) 252 | return langs 253 | 254 | def _get_filename(self, file_name, uri, param_set): 255 | if file_name is not None: 256 | if isinstance(param_set, dict): 257 | return file_name.format(**param_set) 258 | else: 259 | return file_name.format(*param_set) 260 | elif uri.endswith('/'): 261 | # rewrite URIs ending with a slash to ../index.html 262 | if uri.startswith('/'): 263 | uri = uri[1:] 264 | return uri + 'index.html' 265 | else: 266 | return None 267 | 268 | def _is_str(self, s): 269 | return isinstance(s, str) 270 | 271 | def get_uri_values(self, func, view_name): 272 | fullargspec = inspect.getfullargspec(func) 273 | try: 274 | if 'view_name' in fullargspec.args: 275 | v = func(view_name) 276 | else: 277 | v = func() 278 | except Exception as e: 279 | raise DistillError('Failed to call distill function: {}'.format(e)) 280 | if not v: 281 | return (None,) 282 | elif isinstance(v, (list, tuple)): 283 | return v 284 | elif isinstance(v, types.GeneratorType): 285 | return list(v) 286 | else: 287 | err = 'Distill function returned an invalid type: {}' 288 | raise DistillError(err.format(type(v))) 289 | 290 | def generate_uri(self, url, view_name, param_set): 291 | namespace = self.namespace_map.get(url, '') 292 | view_name_ns = namespace + ':' + view_name if namespace else view_name 293 | if isinstance(param_set, (list, tuple)): 294 | try: 295 | uri = reverse(view_name, args=param_set) 296 | except NoReverseMatch: 297 | uri = reverse(view_name_ns, args=param_set) 298 | elif isinstance(param_set, dict): 299 | try: 300 | uri = reverse(view_name, kwargs=param_set) 301 | except NoReverseMatch: 302 | uri = reverse(view_name_ns, kwargs=param_set) 303 | else: 304 | err = 'Distill function returned an invalid type: {}' 305 | raise DistillError(err.format(type(param_set))) 306 | return uri 307 | 308 | def render_view(self, uri, status_codes, param_set, args, kwargs={}): 309 | view_path, view_func = None, None 310 | try: 311 | view_path, view_func = args[0], args[1] 312 | except IndexError: 313 | try: 314 | view_path, view_func = kwargs['route'], kwargs['view'] 315 | except KeyError: 316 | pass 317 | if view_path is None or view_func is None: 318 | raise DistillError(f'Invalid view arguments, args:{args}, kwargs:{kwargs}') 319 | # Default status_codes to (200,) if they are invalid or not set 320 | if not isinstance(status_codes, (tuple, list)): 321 | status_codes = (200,) 322 | for status_code in status_codes: 323 | if not isinstance(status_code, int): 324 | status_codes = (200,) 325 | break 326 | view_args = args[2:] if len(args) > 2 else () 327 | request_factory = RequestFactory() 328 | request = request_factory.get(uri) 329 | handler = DistillHandler() 330 | handler.load_middleware() 331 | if isinstance(param_set, dict): 332 | a, k = (), param_set 333 | else: 334 | a, k = param_set, {} 335 | try: 336 | handler.set_view(view_func, a, k, view_args) 337 | response = handler.get_response(request) 338 | except Exception as err: 339 | e = 'Failed to render view "{}": {}'.format(uri, err) 340 | raise DistillError(e) from err 341 | if response.status_code not in status_codes: 342 | err = 'View returned an invalid status code: {} (expected one of {})' 343 | raise DistillError(err.format(response.status_code, status_codes)) 344 | return response 345 | 346 | 347 | def copy_static(dir_from, dir_to): 348 | # we need to ignore some static dirs such as 'admin' so this is a 349 | # little more complex than a straight shutil.copytree() 350 | if not dir_from.endswith(os.sep): 351 | dir_from = dir_from + os.sep 352 | if not dir_to.endswith(os.sep): 353 | dir_to = dir_to + os.sep 354 | for root, dirs, files in os.walk(dir_from): 355 | dirs[:] = filter_dirs(dirs) 356 | for f in files: 357 | from_path = os.path.join(root, f) 358 | base_path = from_path[len(dir_from):] 359 | to_path = os.path.join(dir_to, base_path) 360 | to_path_dir = os.path.dirname(to_path) 361 | if not os.path.isdir(to_path_dir): 362 | os.makedirs(to_path_dir) 363 | copy2(from_path, to_path) 364 | yield from_path, to_path 365 | 366 | 367 | def copy_static_and_media_files(output_dir, stdout): 368 | static_url = str(settings.STATIC_URL) 369 | static_root = str(settings.STATIC_ROOT) 370 | static_url = static_url[1:] if static_url.startswith('/') else static_url 371 | static_output_dir = os.path.join(output_dir, static_url) 372 | for file_from, file_to in copy_static(static_root, static_output_dir): 373 | stdout('Copying static: {} -> {}'.format(file_from, file_to)) 374 | media_url = str(settings.MEDIA_URL) 375 | media_root = str(settings.MEDIA_ROOT) 376 | if media_root: 377 | media_url = media_url[1:] if media_url.startswith('/') else media_url 378 | media_output_dir = os.path.join(output_dir, media_url) 379 | for file_from, file_to in copy_static(media_root, media_output_dir): 380 | stdout('Copying media: {} -> {}'.format(file_from, file_to)) 381 | return True 382 | 383 | 384 | def run_collectstatic(stdout): 385 | stdout('Distill is running collectstatic...') 386 | call_command('collectstatic', '--noinput') 387 | stdout('') 388 | stdout('collectstatic complete, continuing...') 389 | 390 | 391 | def filter_dirs(dirs): 392 | DISTILL_SKIP_ADMIN_DIRS = bool(getattr(settings, 'DISTILL_SKIP_ADMIN_DIRS', True)) 393 | _ignore_dirs = [] 394 | if DISTILL_SKIP_ADMIN_DIRS: 395 | _ignore_dirs = [ 396 | 'admin', 397 | 'grappelli', 398 | 'unfold', 399 | ] 400 | try: 401 | DISTILL_SKIP_STATICFILES_DIRS = list(getattr(settings, 'DISTILL_SKIP_STATICFILES_DIRS', [])) 402 | except (ValueError, TypeError): 403 | DISTILL_SKIP_STATICFILES_DIRS = [] 404 | for d in DISTILL_SKIP_STATICFILES_DIRS: 405 | if isinstance(d, str) and os.sep not in d: 406 | _ignore_dirs.append(d) 407 | return [d for d in dirs if d not in _ignore_dirs] 408 | 409 | 410 | def load_urls(stdout=None): 411 | if stdout: 412 | stdout('Loading site URLs') 413 | for url in urlconf.url_patterns: 414 | include_urls(url) 415 | 416 | 417 | def get_filepath(output_dir, file_name, page_uri): 418 | if file_name: 419 | local_uri = file_name 420 | full_path = os.path.join(output_dir, file_name) 421 | else: 422 | local_uri = page_uri 423 | if page_uri.startswith('/'): 424 | page_uri = page_uri[1:] 425 | page_path = page_uri.replace('/', os.sep) 426 | full_path = os.path.join(output_dir, page_path) 427 | return full_path, local_uri 428 | 429 | 430 | def write_file(full_path, content): 431 | try: 432 | dirname = os.path.dirname(full_path) 433 | if not os.path.isdir(dirname): 434 | os.makedirs(dirname) 435 | with open(full_path, 'wb') as f: 436 | f.write(content) 437 | except IOError as e: 438 | if e.errno == errno.EISDIR: 439 | err = ('Output path: {} is a directory! Try adding a ' 440 | '"distill_file" arg to your distill_url()') 441 | raise DistillError(err.format(full_path)) 442 | else: 443 | raise 444 | 445 | 446 | def get_renderer(urls_to_distill, parallel_render=1): 447 | import_path = getattr(settings, 'DISTILL_RENDERER', None) 448 | if import_path: 449 | render_cls = import_string(import_path) 450 | else: 451 | render_cls = DistillRender 452 | return render_cls(urls_to_distill, parallel_render) 453 | 454 | 455 | def render_to_dir(output_dir, urls_to_distill, stdout, parallel_render=1): 456 | load_urls(stdout) 457 | renderer = get_renderer(urls_to_distill, parallel_render) 458 | for page_uri, file_name, http_response in renderer.render(): 459 | full_path, local_uri = get_filepath(output_dir, file_name, page_uri) 460 | content = http_response.content 461 | mime = http_response.get('Content-Type') 462 | renamed = ' (renamed from "{}")'.format(page_uri) if file_name else '' 463 | msg = 'Rendering page: {} -> {} ["{}", {} bytes] {}' 464 | stdout(msg.format(local_uri, full_path, mime, len(content), renamed)) 465 | write_file(full_path, content) 466 | return True 467 | 468 | 469 | def render_single_file(output_dir, view_name, *args, **kwargs): 470 | from django_distill.distill import urls_to_distill 471 | status_codes = None 472 | if 'status_codes' in kwargs: 473 | status_codes = kwargs['status_codes'] 474 | del kwargs['status_codes'] 475 | view_name = str(view_name) 476 | if not status_codes: 477 | status_codes = (200,) 478 | load_urls() 479 | renderer = get_renderer(urls_to_distill) 480 | page_uri, file_name, http_response = renderer.render( 481 | view_name, status_codes, args, kwargs) 482 | full_path, local_uri = get_filepath(output_dir, file_name, page_uri) 483 | content = http_response.content 484 | write_file(full_path, content) 485 | return True 486 | 487 | 488 | def generate_urls(urls_to_distill): 489 | load_urls() 490 | renderer = get_renderer(urls_to_distill) 491 | return renderer.urls() 492 | 493 | 494 | def render_static_redirect(destination_url): 495 | redir = [] 496 | redir.append(f'') 497 | redir.append(f'') 498 | redir.append(f'') 499 | redir.append(f'') 500 | redir.append(f'') 501 | redir.append(f'Redirecting to {destination_url}') 502 | redir.append(f'') 503 | redir.append(f'') 504 | redir.append(f'') 505 | redir.append(f'

Redirecting to {destination_url}

') 506 | redir.append(f'

If you are not automatically redirected please click this link

') 507 | redir.append(f'') 508 | return '\n'.join(redir).encode() 509 | 510 | 511 | def render_redirects(output_dir, stdout): 512 | from django.contrib.redirects.models import Redirect 513 | for redirect in Redirect.objects.all(): 514 | redirect_path = redirect.old_path.lstrip('/') 515 | if redirect_path.lower().endswith('.html'): 516 | redirect_file = redirect_path 517 | else: 518 | redirect_file = os.path.join(redirect_path, 'index.html') 519 | full_path, local_uri = get_filepath(output_dir, redirect_file, redirect_file) 520 | content = render_static_redirect(redirect.new_path) 521 | msg = 'Rendering redirect: {} -> {}' 522 | stdout(msg.format(local_uri, redirect.new_path)) 523 | write_file(full_path, content) 524 | return True 525 | -------------------------------------------------------------------------------- /tests/test_renderer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import tempfile 4 | from datetime import datetime, timedelta 5 | from unittest.mock import patch 6 | from django.test import TestCase, override_settings 7 | from django.conf import settings 8 | from django.contrib.flatpages.models import FlatPage 9 | from django.apps import apps as django_apps 10 | from django.utils import timezone 11 | from django.utils.translation import activate as activate_lang 12 | from django_distill.distill import urls_to_distill 13 | from django_distill.renderer import DistillRender, render_to_dir, render_single_file, get_renderer 14 | from django_distill.errors import DistillError 15 | from django_distill import distilled_urls 16 | 17 | 18 | class CustomRender(DistillRender): 19 | pass 20 | 21 | 22 | class DjangoDistillRendererTestSuite(TestCase): 23 | 24 | def setUp(self): 25 | self.renderer = DistillRender(urls_to_distill) 26 | # Create a few test flatpages 27 | Site = django_apps.get_model('sites.Site') 28 | current_site = Site.objects.get_current() 29 | page1 = FlatPage() 30 | page1.url = '/flat/page1.html' 31 | page1.title = 'flatpage1' 32 | page1.content = 'flatpage1' 33 | page1.template_name = 'flatpage.html' 34 | page1.save() 35 | page1.sites.add(current_site) 36 | page2 = FlatPage() 37 | page2.url = '/flat/page2.html' 38 | page2.title = 'flatpage2' 39 | page2.content = 'flatpage2' 40 | page2.template_name = 'flatpage.html' 41 | page2.save() 42 | page2.sites.add(current_site) 43 | 44 | def _get_view(self, name): 45 | for u in urls_to_distill: 46 | if u[4] == name: 47 | return u 48 | return False 49 | 50 | def _skip(self, what): 51 | sys.stdout.write('Missing {}, skipping test... '.format(what)) 52 | sys.stdout.flush() 53 | 54 | def test_is_str(self): 55 | self.assertTrue(self.renderer._is_str('a')) 56 | self.assertFalse(self.renderer._is_str(None)) 57 | self.assertFalse(self.renderer._is_str(1)) 58 | self.assertFalse(self.renderer._is_str([])) 59 | self.assertFalse(self.renderer._is_str(())) 60 | self.assertFalse(self.renderer._is_str({})) 61 | self.assertFalse(self.renderer._is_str({'a':'a'})) 62 | self.assertFalse(self.renderer._is_str(object())) 63 | 64 | def test_get_uri_values(self): 65 | test = () 66 | check = self.renderer.get_uri_values(lambda: test, None) 67 | self.assertEqual(check, (None,)) 68 | test = ('a',) 69 | check = self.renderer.get_uri_values(lambda: test, None) 70 | self.assertEqual(check, test) 71 | test = (('a',),) 72 | check = self.renderer.get_uri_values(lambda: test, None) 73 | self.assertEqual(check, test) 74 | test = [] 75 | check = self.renderer.get_uri_values(lambda: test, None) 76 | self.assertEqual(check, (None,)) 77 | test = ['a'] 78 | check = self.renderer.get_uri_values(lambda: test, None) 79 | self.assertEqual(check, test) 80 | test = [['a']] 81 | check = self.renderer.get_uri_values(lambda: test, None) 82 | self.assertEqual(check, test) 83 | for invalid in ('a', 1, b'a', {'s'}, {'a':'a'}, object()): 84 | with self.assertRaises(DistillError): 85 | self.renderer.get_uri_values(lambda: invalid, None) 86 | 87 | def test_re_path_no_param(self): 88 | if not settings.HAS_RE_PATH: 89 | self._skip('django.urls.re_path') 90 | return 91 | view = self._get_view('re_path-no-param') 92 | assert view 93 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 94 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 95 | if not param_set: 96 | param_set = () 97 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 98 | self.assertEqual(uri, '/re_path/') 99 | render = self.renderer.render_view(uri, status_codes, param_set, args) 100 | self.assertEqual(render.content, b'test') 101 | 102 | def test_re_path_positional_param(self): 103 | if not settings.HAS_RE_PATH: 104 | self._skip('django.urls.re_path') 105 | return 106 | view = self._get_view('re_path-positional-param') 107 | assert view 108 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 109 | param_sets = self.renderer.get_uri_values(view_func, view_name) 110 | for param_set in param_sets: 111 | param_set = (param_set,) 112 | first_value = param_set[0] 113 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 114 | self.assertEqual(uri, f'/re_path/{first_value}') 115 | render = self.renderer.render_view(uri, status_codes, param_set, args) 116 | self.assertEqual(render.content, b'test' + first_value.encode()) 117 | 118 | def test_re_path_named_param(self): 119 | if not settings.HAS_RE_PATH: 120 | self._skip('django.urls.re_path') 121 | return 122 | view = self._get_view('re_path-named-param') 123 | assert view 124 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 125 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 126 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 127 | self.assertEqual(uri, '/re_path/test') 128 | render = self.renderer.render_view(uri, status_codes, param_set, args) 129 | self.assertEqual(render.content, b'testtest') 130 | 131 | def test_re_broken(self): 132 | if not settings.HAS_RE_PATH: 133 | self._skip('django.urls.re_path') 134 | return 135 | view = self._get_view('re_path-broken') 136 | assert view 137 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 138 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 139 | if not param_set: 140 | param_set = () 141 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 142 | self.assertEqual(uri, '/re_path/broken') 143 | with self.assertRaises(DistillError): 144 | self.renderer.render_view(uri, status_codes, param_set, args) 145 | 146 | def test_path_no_param(self): 147 | if not settings.HAS_PATH: 148 | self._skip('django.urls.path') 149 | return 150 | view = self._get_view('path-no-param') 151 | assert view 152 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 153 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 154 | if not param_set: 155 | param_set = () 156 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 157 | self.assertEqual(uri, '/path/') 158 | render = self.renderer.render_view(uri, status_codes, param_set, args) 159 | self.assertEqual(render.content, b'test') 160 | 161 | def test_path_positional_param(self): 162 | if not settings.HAS_PATH: 163 | self._skip('django.urls.path') 164 | return 165 | view = self._get_view('path-positional-param') 166 | assert view 167 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 168 | param_sets = self.renderer.get_uri_values(view_func, view_name) 169 | for param_set in param_sets: 170 | param_set = (param_set,) 171 | first_value = param_set[0] 172 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 173 | self.assertEqual(uri, f'/path/{first_value}') 174 | render = self.renderer.render_view(uri, status_codes, param_set, args) 175 | self.assertEqual(render.content, b'test' + first_value.encode()) 176 | 177 | def test_path_named_param(self): 178 | if not settings.HAS_PATH: 179 | self._skip('django.urls.path') 180 | return 181 | view = self._get_view('path-named-param') 182 | assert view 183 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 184 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 185 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 186 | self.assertEqual(uri, '/path/test') 187 | render = self.renderer.render_view(uri, status_codes, param_set, args) 188 | self.assertEqual(render.content, b'testtest') 189 | 190 | def test_path_broken(self): 191 | if not settings.HAS_PATH: 192 | self._skip('django.urls.path') 193 | return 194 | view = self._get_view('path-broken') 195 | assert view 196 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 197 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 198 | if not param_set: 199 | param_set = () 200 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 201 | self.assertEqual(uri, '/path/broken') 202 | with self.assertRaises(DistillError): 203 | self.renderer.render_view(uri, status_codes, param_set, args) 204 | 205 | def test_render_paths(self): 206 | def _blackhole(_): 207 | pass 208 | expected_files = ( 209 | ('test',), 210 | ('re_path', '12345'), 211 | ('re_path', 'test'), 212 | ('re_path', 'x', '12345.html'), 213 | ('re_path', 'x', 'test.html'), 214 | ) 215 | with tempfile.TemporaryDirectory() as tmpdirname: 216 | with self.assertRaises(DistillError): 217 | render_to_dir(tmpdirname, urls_to_distill, _blackhole) 218 | written_files = [] 219 | for (root, dirs, files) in os.walk(tmpdirname): 220 | for f in files: 221 | filepath = os.path.join(root, f) 222 | written_files.append(filepath) 223 | for expected_file in expected_files: 224 | filepath = os.path.join(tmpdirname, *expected_file) 225 | self.assertIn(filepath, written_files) 226 | 227 | @patch.object(CustomRender, "render_view", side_effect=CustomRender.render_view, autospec=True) 228 | @override_settings(DISTILL_RENDERER="tests.test_renderer.CustomRender") 229 | def test_render_paths_custom_renderer(self, render_view_spy): 230 | def _blackhole(_): 231 | pass 232 | expected_files = ( 233 | ('test',), 234 | ('re_path', '12345'), 235 | ('re_path', 'test'), 236 | ) 237 | with tempfile.TemporaryDirectory() as tmpdirname: 238 | with self.assertRaises(DistillError): 239 | render_to_dir(tmpdirname, urls_to_distill, _blackhole) 240 | written_files = [] 241 | for (root, dirs, files) in os.walk(tmpdirname): 242 | for f in files: 243 | filepath = os.path.join(root, f) 244 | written_files.append(filepath) 245 | for expected_file in expected_files: 246 | filepath = os.path.join(tmpdirname, *expected_file) 247 | self.assertIn(filepath, written_files) 248 | #self.assertEqual(render_view_spy.call_count, 34) 249 | 250 | def test_sessions_are_ignored(self): 251 | if settings.HAS_PATH: 252 | view = self._get_view('path-ignore-sessions') 253 | assert view 254 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 255 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 256 | if not param_set: 257 | param_set = () 258 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 259 | self.assertEqual(uri, '/path/ignore-sessions') 260 | render = self.renderer.render_view(uri, status_codes, param_set, args) 261 | self.assertEqual(render.content, b'test') 262 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 263 | if settings.HAS_RE_PATH: 264 | view = self._get_view('re_path-ignore-sessions') 265 | assert view 266 | view_url, iew_func, file_name, status_codes, view_name, args, kwargs = view 267 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 268 | if not param_set: 269 | param_set = () 270 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 271 | self.assertEqual(uri, '/re_path/ignore-sessions') 272 | render = self.renderer.render_view(uri, status_codes, param_set, args) 273 | self.assertEqual(render.content, b'test') 274 | 275 | def test_custom_status_codes(self): 276 | if settings.HAS_PATH: 277 | view = self._get_view('path-404') 278 | assert view 279 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 280 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 281 | if not param_set: 282 | param_set = () 283 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 284 | self.assertEqual(uri, '/path/404') 285 | render = self.renderer.render_view(uri, status_codes, param_set, args) 286 | self.assertEqual(render.content, b'404') 287 | self.assertEqual(render.status_code, 404) 288 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 289 | if settings.HAS_RE_PATH: 290 | view = self._get_view('re_path-404') 291 | assert view 292 | view_url, iew_func, file_name, status_codes, view_name, args, kwargs = view 293 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 294 | if not param_set: 295 | param_set = () 296 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 297 | self.assertEqual(uri, '/re_path/404') 298 | render = self.renderer.render_view(uri, status_codes, param_set, args) 299 | self.assertEqual(render.content, b'404') 300 | self.assertEqual(render.status_code, 404) 301 | 302 | def test_contrib_flatpages(self): 303 | if settings.HAS_PATH: 304 | view = self._get_view('path-flatpage') 305 | assert view 306 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 307 | param_set = self.renderer.get_uri_values(view_func, view_name) 308 | for param in param_set: 309 | page_url = param['url'] 310 | uri = self.renderer.generate_uri(view_url, view_name, param) 311 | self.assertEqual(uri, f'/path/flatpage{page_url}') 312 | render = self.renderer.render_view(uri, status_codes, param, args) 313 | flatpage = FlatPage.objects.get(url=page_url) 314 | expected = f'{flatpage.title}{flatpage.content}\n' 315 | self.assertEqual(render.content, expected.encode()) 316 | self.assertEqual(render.status_code, 200) 317 | if settings.HAS_RE_PATH: 318 | view = self._get_view('re_path-flatpage') 319 | assert view 320 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 321 | param_set = self.renderer.get_uri_values(view_func, view_name) 322 | for param in param_set: 323 | page_url = param['url'] 324 | uri = self.renderer.generate_uri(view_url, view_name, param) 325 | self.assertEqual(uri, f'/re_path/flatpage{page_url}') 326 | render = self.renderer.render_view(uri, status_codes, param, args) 327 | flatpage = FlatPage.objects.get(url=page_url) 328 | expected = f'{flatpage.title}{flatpage.content}\n' 329 | self.assertEqual(render.content, expected.encode()) 330 | self.assertEqual(render.status_code, 200) 331 | 332 | def test_contrib_sitemaps(self): 333 | view = self._get_view('path-sitemap') 334 | assert view 335 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 336 | param_set = () 337 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 338 | self.assertEqual(uri, '/path/test-sitemap') 339 | render = self.renderer.render_view(uri, status_codes, param_set, args) 340 | expected_content = ( 341 | b'\n' 342 | b'\n' 344 | b'http://example.com/path/test-sitemap' 345 | b'daily' 346 | b'0.5\n\n' 347 | ) 348 | self.assertEqual(render.content, expected_content) 349 | self.assertEqual(render.status_code, 200) 350 | 351 | def test_render_single_file(self): 352 | expected_files = ( 353 | ('path', '12345'), 354 | ('path', 'test'), 355 | ) 356 | with tempfile.TemporaryDirectory() as tmpdirname: 357 | render_single_file(tmpdirname, 'path-positional-param', 12345) 358 | render_single_file(tmpdirname, 'path-named-param', param='test') 359 | written_files = [] 360 | for (root, dirs, files) in os.walk(tmpdirname): 361 | for f in files: 362 | filepath = os.path.join(root, f) 363 | written_files.append(filepath) 364 | for expected_file in expected_files: 365 | filepath = os.path.join(tmpdirname, *expected_file) 366 | self.assertIn(filepath, written_files) 367 | 368 | def test_i18n(self): 369 | if not settings.USE_I18N: 370 | self._skip('settings.USE_I18N') 371 | return 372 | settings.DISTILL_LANGUAGES = [ 373 | 'en', 374 | 'fr', 375 | 'de', 376 | ] 377 | expected = {} 378 | for lang_code, lang_name in settings.DISTILL_LANGUAGES: 379 | expected[lang_code] = f'/{lang_code}/path/i18n/sub-url-with-i18n-prefix' 380 | view = self._get_view('test-url-i18n') 381 | assert view 382 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 383 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 384 | if not param_set: 385 | param_set = () 386 | for lang_code, path in expected.items(): 387 | activate_lang(lang_code) 388 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 389 | self.assertEqual(uri, path) 390 | render = self.renderer.render_view(uri, status_codes, param_set, args) 391 | self.assertEqual(render.content, b'test') 392 | # Render the test URLs and confirm the expected language URI prefixes are present 393 | def _blackhole(_): 394 | pass 395 | expected_files = ( 396 | ('test',), 397 | ('re_path', '12345'), 398 | ('re_path', 'test'), 399 | ('re_path', 'x', '12345.html'), 400 | ('re_path', 'x', 'test.html'), 401 | ('en', 'path', 'i18n', 'sub-url-with-i18n-prefix'), 402 | ('fr', 'path', 'i18n', 'sub-url-with-i18n-prefix'), 403 | ('de', 'path', 'i18n', 'sub-url-with-i18n-prefix'), 404 | ) 405 | with tempfile.TemporaryDirectory() as tmpdirname: 406 | with self.assertRaises(DistillError): 407 | render_to_dir(tmpdirname, urls_to_distill, _blackhole, parallel_render=8) 408 | written_files = [] 409 | for (root, dirs, files) in os.walk(tmpdirname): 410 | for f in files: 411 | filepath = os.path.join(root, f) 412 | written_files.append(filepath) 413 | for expected_file in expected_files: 414 | filepath = os.path.join(tmpdirname, *expected_file) 415 | self.assertIn(filepath, written_files) 416 | settings.DISTILL_LANGUAGES = [] 417 | 418 | def test_kwargs(self): 419 | if not settings.HAS_PATH: 420 | self._skip('django.urls.path') 421 | return 422 | view = self._get_view('test-kwargs') 423 | assert view 424 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 425 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 426 | if not param_set: 427 | param_set = () 428 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 429 | self.assertEqual(uri, '/path/kwargs') 430 | render = self.renderer.render_view(uri, status_codes, param_set, args, kwargs) 431 | self.assertEqual(render.content, b'test') 432 | 433 | def test_humanize(self): 434 | if not settings.HAS_PATH: 435 | self._skip('django.urls.path') 436 | return 437 | view = self._get_view('test-humanize') 438 | assert view 439 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 440 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 441 | if not param_set: 442 | param_set = () 443 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 444 | self.assertEqual(uri, '/path/humanize') 445 | now = timezone.now() 446 | one_hour_ago = now - timedelta(hours=1) 447 | nineteen_hours_ago = now - timedelta(hours=19) 448 | render = self.renderer.render_view(uri, status_codes, param_set, args, kwargs) 449 | content = render.content 450 | expected = b'\n'.join([ 451 | b'', 452 | b'
    ', 453 | b'
  • test
    • ', 454 | b'
  • one hour ago naturaltime: an hour ago
    • ', 455 | b'
  • nineteen hours ago naturaltime: 19\xc2\xa0hours ago
    • ', 456 | b'
', 457 | b'', 458 | ]) 459 | self.assertEqual(render.content, expected) 460 | 461 | def test_request_has_resolver_match(self): 462 | view = self._get_view('test-has-resolver-match') 463 | assert view 464 | view_url, view_func, file_name, status_codes, view_name, args, kwargs = view 465 | param_set = self.renderer.get_uri_values(view_func, view_name)[0] 466 | if not param_set: 467 | param_set = () 468 | uri = self.renderer.generate_uri(view_url, view_name, param_set) 469 | render = self.renderer.render_view(uri, status_codes, param_set, args, kwargs) 470 | self.assertEqual(render.content, b"test_request_has_resolver_match") 471 | 472 | def test_parallel_rendering(self): 473 | def _blackhole(_): 474 | pass 475 | expected_files = ( 476 | ('test',), 477 | ('re_path', '12345'), 478 | ('re_path', 'test'), 479 | ('re_path', 'x', '12345.html'), 480 | ('re_path', 'x', 'test.html'), 481 | ) 482 | with tempfile.TemporaryDirectory() as tmpdirname: 483 | with self.assertRaises(DistillError): 484 | render_to_dir(tmpdirname, urls_to_distill, _blackhole, parallel_render=8) 485 | written_files = [] 486 | for (root, dirs, files) in os.walk(tmpdirname): 487 | for f in files: 488 | filepath = os.path.join(root, f) 489 | written_files.append(filepath) 490 | for expected_file in expected_files: 491 | filepath = os.path.join(tmpdirname, *expected_file) 492 | self.assertIn(filepath, written_files) 493 | 494 | def test_generate_urls(self): 495 | urls = distilled_urls() 496 | generated_urls = [] 497 | for url, file_name in urls: 498 | generated_urls.append(url) 499 | expected_urls = ( 500 | '/path/namespace1/sub-url-in-namespace', 501 | '/path/namespace1/path/sub-namespace/sub-url-in-sub-namespace', 502 | '/path/no-namespace/sub-url-in-no-namespace', 503 | '/en/path/i18n/sub-url-with-i18n-prefix', 504 | '/re_path/', 505 | '/re_path-no-func/', 506 | '/re_path/12345', 507 | '/re_path/67890', 508 | '/re_path/x/12345', 509 | '/re_path/x/67890', 510 | '/re_path/test', 511 | '/re_path/x/test', 512 | '/re_path/broken', 513 | '/re_path/ignore-sessions', 514 | '/re_path/404', 515 | '/re_path/flatpage/flat/page1.html', 516 | '/re_path/flatpage/flat/page2.html', 517 | '/path/', 518 | '/path-no-func/', 519 | '/path/12345', 520 | '/path/67890', 521 | '/path/x/12345', 522 | '/path/x/67890', 523 | '/path/test', 524 | '/path/x/test', 525 | '/path/broken', 526 | '/path/ignore-sessions', 527 | '/path/404', 528 | '/path/flatpage/flat/page1.html', 529 | '/path/flatpage/flat/page2.html', 530 | '/path/test-sitemap', 531 | '/path/kwargs', 532 | '/path/humanize', 533 | '/path/has-resolver-match' 534 | ) 535 | self.assertEqual(sorted(generated_urls), sorted(expected_urls)) 536 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # django-distill 2 | 3 | `django-distill` now has a website. Read more at: 4 | 5 | ## :link: https://django-distill.com/ 6 | 7 | `django-distill` is a minimal configuration static site generator and publisher 8 | for Django. Most Django versions are supported, however up to date versions are 9 | advised including the Django 3.x releases. `django-distill` as of the 1.7 release 10 | only supports Python 3. Python 2 support has been dropped. If you require Python 2 11 | support please pin `django-distill` to version 1.6 in your requirements.txt or 12 | Pipfile. Python 3.6 or above is advised. 13 | 14 | `django-distill` extends existing Django sites with the ability to export 15 | fully functional static sites. It is suitable for sites such as blogs that have 16 | a mostly static front end but you still want to use a CMS to manage the 17 | content. 18 | 19 | `django-distill` iterates over URLs in your Django project using easy to write 20 | iterable functions to yield the parameters for whatever pages you want to save 21 | as static HTML. These static files can be automatically uploaded to a bucket-style 22 | remote container such as Amazon S3, Googe Cloud Files, Microsoft Azure Storage, 23 | or, written to a local directory as a fully working local static version of 24 | your project. The site generation, or distillation process, can be easily 25 | integrated into CI/CD workflows to auto-deploy static sites on commit. 26 | `django-distill` can be defined as an extension to Django to make Django 27 | projects compatible with "Jamstack"-style site architecture. 28 | 29 | `django-distill` plugs directly into the existing Django framework without the 30 | need to write custom renderers or other more verbose code. You can also integrate 31 | `django-distill` with existing dynamic sites and just generate static pages for 32 | a small subsection of pages rather than the entire site. 33 | 34 | For static files on CDNs you can use the following 'cache buster' library to 35 | allow for fast static media updates when pushing changes: 36 | 37 | [:link: meeb/django-cachekiller](https://github.com/meeb/django-cachekiller) 38 | 39 | There is a complete example site that creates a static blog and uses 40 | `django-distill` with `django-cachekiller` via continuous deployment on Netlify 41 | available here: 42 | 43 | [:link: meeb/django-distill-example](https://github.com/meeb/django-distill-example) 44 | 45 | 46 | # Installation 47 | 48 | Install from pip: 49 | 50 | ```bash 51 | $ pip install django-distill 52 | ``` 53 | 54 | Add `django_distill` to your `INSTALLED_APPS` in your `settings.py`: 55 | 56 | ```python 57 | INSTALLED_APPS = [ 58 | # ... other apps here ... 59 | 'django_distill', 60 | ] 61 | ``` 62 | 63 | That's it. 64 | 65 | 66 | # Limitations 67 | 68 | `django-distill` generates static pages and therefore only views which allow 69 | `GET` requests that return an `HTTP 200` status code are supported. 70 | 71 | It is assumed you are using URI parameters such as `/blog/123-abc` and not 72 | querystring parameters such as `/blog?post_id=123&title=abc`. Querystring 73 | parameters do not make sense for static page generation for obvious reasons. 74 | 75 | Static media files such as images and style sheets are copied from your static 76 | media directory defined in `STATIC_ROOT`. This means that you will want to run 77 | `./manage.py collectstatic` **before** you run `./manage.py distill-local` 78 | if you have made changes to static media. `django-distill` doesn't chain this 79 | request by design, however you can enable it with the `--collectstatic` 80 | argument. 81 | 82 | 83 | # Usage 84 | 85 | Assuming you have an existing Django project, edit a `urls.py` to include the 86 | `distill_path` function which replaces Django's standard `path` function and 87 | supports the new keyword arguments `distill_func` and `distill_file`. 88 | 89 | The `distill_func` argument should be provided with a function or callable 90 | class that returns an iterable or `None`. 91 | 92 | The `distill_file` argument is entirely optional and allows you to override the 93 | URL that would otherwise be generated from the reverse of the URL regex. This 94 | allows you to rename URLs like `/example` to any other name like 95 | `example.html`. As of v0.8 any URIs ending in a slash `/` are automatically 96 | modified to end in `/index.html`. You can use format string parameters in the 97 | `distill_file` to customise the file name, arg values from the URL will be 98 | substituted in, for example `{}` for positional args or `{param_name}` for 99 | named args. 100 | 101 | An example distill setup for a theoretical blogging app would be: 102 | 103 | ```python 104 | # Replaces the standard django.conf.path, identical syntax 105 | from django_distill import distill_path 106 | 107 | # Views and models from a theoretical blogging app 108 | from blog.views import PostIndex, PostView, PostYear 109 | from blog.models import Post 110 | 111 | def get_index(): 112 | # The index URI path, '', contains no parameters, named or otherwise. 113 | # You can simply just return nothing here. 114 | return None 115 | 116 | def get_all_blogposts(): 117 | # This function needs to return an iterable of dictionaries. Dictionaries 118 | # are required as the URL this distill function is for has named parameters. 119 | # You can just export a small subset of values here if you wish to 120 | # limit what pages will be generated. 121 | for post in Post.objects.all(): 122 | yield {'blog_id': post.id, 'blog_title': post.title} 123 | 124 | def get_years(): 125 | # You can also just return an iterable containing static strings if the 126 | # URL only has one argument and you are using positional URL parameters: 127 | return (2014, 2015) 128 | # This is really just shorthand for ((2014,), (2015,)) 129 | 130 | urlpatterns = ( 131 | # e.g. / the blog index 132 | distill_path('', 133 | PostIndex.as_view(), 134 | name='blog-index', 135 | # Note that for paths which have no paramters 136 | # distill_func is optional 137 | distill_func=get_index, 138 | # '' is not a valid file name! override it to index.html 139 | distill_file='index.html'), 140 | # e.g. /post/123-some-post-title using named parameters 141 | distill_path('post/-.html', 142 | PostView.as_view(), 143 | name='blog-post', 144 | distill_func=get_all_blogposts), 145 | # e.g. /posts-by-year/2015 using positional parameters 146 | # url ends in / so file path will have /index.html appended 147 | distill_path('posts-by-year//', 148 | PostYear.as_view(), 149 | name='blog-year', 150 | distill_func=get_years), 151 | ) 152 | ``` 153 | 154 | Your site will still function identically with the above changes. Internally 155 | the `distill_func` and `distill_file` parameters are removed and the URL is 156 | passed back to Django for normal processing. This has no runtime performance 157 | impact as this happens only once upon starting the application. 158 | 159 | If your path has no URI paramters, such as `/` or `/some-static-url` you do 160 | not have to specify the `distill_func` parameter if you don't want to. As for 161 | paths with no parameters the `distill_func` always returns `None`, this is set 162 | as the default behaviour for `distill_func`s. 163 | 164 | You can use the `distill_re_path` function as well, which replaces the default 165 | `django.urls.re_path` function. Its usage is identical to the above: 166 | 167 | ```python 168 | from django_distill import distill_re_path 169 | 170 | urlpatterns = ( 171 | distill_re_path(r'some/regex' 172 | SomeOtherView.as_view(), 173 | name='url-other-view', 174 | distill_func=some_other_func), 175 | ) 176 | 177 | ``` 178 | 179 | If you are using an older version of Django in the 1.x series you can use the 180 | `distill_url` function instead which replaces the `django.conf.urls.url` or 181 | `django.urls.url` functions. Its usage is identical to the above: 182 | 183 | ```python 184 | from django_distill import distill_url 185 | 186 | urlpatterns = ( 187 | distill_url(r'some/regex' 188 | SomeView.as_view(), 189 | name='url-view', 190 | distill_func=some_func), 191 | ) 192 | ``` 193 | 194 | ### Parameters in file names 195 | 196 | You can use standard Python string formatting in `distill_file` as well to enable 197 | you to change the output file path for a file if you wish. Note this does not 198 | update the URL used by Django so if you use this make sure your `path` pattern 199 | matches the `distill_file` pattern or your links might not work in Django. An 200 | example: 201 | 202 | ```python 203 | # Override file path with parameters. Values are taken from the URL pattern 204 | urlpatterns = ( 205 | distill_path('post/-.html', 206 | PostView.as_view(), 207 | name='blog-post', 208 | distill_func=get_all_blogposts, 209 | distill_file="post/{blog_id}-{blog_title}.html" 210 | ) 211 | ``` 212 | 213 | ### Non-standard status codes 214 | 215 | All views rendered by `django-distill` into static pages must return an HTTP 200 status 216 | code. If for any reason you need to render a view which does not return an HTTP 200 217 | status code, for example you also want to statically generate a 404 page which has a 218 | view which (correctly) returns an HTTP 404 status code you can use the 219 | `distill_status_codes` optional argument to a view. For example: 220 | 221 | ```python 222 | from django_distill import distill_url 223 | 224 | urlpatterns = ( 225 | distill_url(r'some/regex' 226 | SomeView.as_view(), 227 | name='url-view', 228 | distill_status_codes=(200, 404), 229 | distill_func=some_func), 230 | ) 231 | ``` 232 | 233 | The optional `distill_status_codes` argument accepts a tuple of status codes as integers 234 | which are permitted for the view to return without raising an error. By default this is 235 | set to `(200,)` but you can override it if you need to for your site. 236 | 237 | ### Tracking Django's URL function support 238 | 239 | `django-distill` will mirror whatever your installed version of Django supports, 240 | therefore at some point the `distill_url` function will cease working in the future 241 | when Django 2.x itself depreciates the `django.conf.urls.url` and `django.urls.url` 242 | functions. You can use `distill_re_path` as a drop-in replacement. It is advisable to 243 | use `distill_path` or `distill_re_path` if you're building a new site now. 244 | 245 | 246 | ### Internationalization 247 | 248 | Internationalization is only supported for URLs, page content is unable to be 249 | dynamically translated. By default your site will be generated using the 250 | `LANGUAGE_CODE` value in your `settings.py`. If you also set `settings.USE_I18N` to 251 | `True` then set other language codes in your `settings.DISTILL_LANGUAGES` value and register 252 | URLs with `i18n_patterns(...)` then your site will be generated in multiple languges. 253 | This assumes your multi-language site works as expected before adding `django-distill`. 254 | 255 | For example if you set `settings.LANGUAGE_CODE = 'en'` your site will be 256 | generated in one language. 257 | 258 | If you have something like this in your `settings.py` instead: 259 | 260 | ```python 261 | USE_I18N = True 262 | 263 | DISTILL_LANGUAGES = [ 264 | 'en', 265 | 'fr', 266 | 'de', 267 | ] 268 | ``` 269 | 270 | While also using `i18n_patterns`in your `urls.py` like so: 271 | 272 | ```python 273 | from django.conf.urls.i18n import i18n_patterns 274 | from django_distill import distill_path 275 | 276 | urlpatterns = i18n_patterns( 277 | distill_path('some-file.html', 278 | SomeView.as_view(), 279 | name='i18n-view', 280 | distill_func=some_func 281 | ) 282 | ) 283 | ``` 284 | 285 | Then your views will be generaged as `/en/some-file.html`, `/fr/some-file.html` 286 | and `/de/some-file.html`. These URLs should work (and be translated) by your 287 | site already. `django-distill` doesn't do any translation magic, it just 288 | calls the URLs with the language code prefix. 289 | 290 | **Note** While the default suggested method is to use `settings.DISTILL_LANGUAGES` 291 | to keep things seperate `django-distill` will also check `settings.LANGUAGES` for 292 | language codes. 293 | 294 | 295 | ### Sitemaps 296 | 297 | You may need to generate a list of all the URLs registered with `django-distill`. 298 | For example, you have a statically generated blog with a few hundred pages and 299 | you want to list all of the URLs easily in a `sitemap.xml` or other similar list 300 | of all URLs. You could wrap your sitemap view in `distill_path` then replicate 301 | all of your URL generation logic by importing your views `distill_func`s from 302 | your `urls.py` and generating these all manually, but given this is quite a hassle 303 | there's a built-in helper to generate all your URLs that will be distilled for you. 304 | 305 | ```python 306 | from django_distill import distilled_urls 307 | 308 | for uri, file_name in distilled_urls(): 309 | # URI is the generated, complete URI for the page 310 | print(uri) # for example: /blog/my-post-123/ 311 | # file_name is the actual file name on disk, this may be None or a string 312 | print(file_name) # for example: /blog/my-post-123/index.html 313 | ``` 314 | 315 | **Note** that `distilled_urls()` will only return URLs after all of your URLs 316 | in `urls.py` have been loaded with `distill_path(...)`. 317 | 318 | 319 | # The `distill-local` command 320 | 321 | Once you have wrapped the URLs you want to generate statically you can now 322 | generate a complete functioning static site with: 323 | 324 | ```bash 325 | $ ./manage.py distill-local [optional /path/to/export/directory] 326 | ``` 327 | 328 | Under the hood this simply iterates all URLs registered with `distill_url` and 329 | generates the pages for them using parts of the Django testing framework to 330 | spoof requests. Once the site pages have been rendered then files from the 331 | `STATIC_ROOT` are copied over. Existing files with the same name are replaced in 332 | the target directory and orphan files are deleted. 333 | 334 | `distill-local` supports the following optional arguments: 335 | 336 | `--collectstatic`: Automatically run `collectstatic` on your site before 337 | rendering, this is just a shortcut to save you typing an extra command. 338 | 339 | `--quiet`: Disable all output other than asking confirmation questions. 340 | 341 | `--force`: Assume 'yes' to all confirmation questions. 342 | 343 | `--exclude-staticfiles`: Do not copy any static files at all, only render output from 344 | Django views. 345 | 346 | `--parallel-render [number of threads]`: Render files in parallel on multiple 347 | threads, this can speed up rendering. Defaults to `1` thread. 348 | 349 | `--generate-redirects`: Attempt to generate static redirects stored in the 350 | `django.contrib.redirects` app. If you have a redirect from `/old/` to `/new/` using 351 | this flag will create a static HTML `` 352 | style redirect at `/old/index.html` to `/new/`. 353 | 354 | **Note** If any of your views contain a Python error then rendering will fail 355 | then the stack trace will be printed to the terminal and the rendering command 356 | will exit with a status code of 1. 357 | 358 | 359 | # The `distill-publish` command 360 | 361 | ```bash 362 | $ ./manage.py distill-publish [optional destination here] 363 | ``` 364 | 365 | If you have configured at least one publishing destination (see below) you can 366 | use the `distill-publish` command to publish the site to a remote location. 367 | 368 | This will perform a full synchronisation, removing any remote files that are no 369 | longer present in the generated static site and uploading any new or changed 370 | files. The site will be built into a temporary directory locally first when 371 | publishing which is deleted once the site has been published. Each file will be 372 | checked that it has been published correctly by requesting it via the 373 | `PUBLIC_URL`. 374 | 375 | `distill-publish` supports the following optional arguments: 376 | 377 | `--collectstatic`: Automatically run `collectstatic` on your site before 378 | rendering, this is just a shortcut to save you typing an extra command. 379 | 380 | `--quiet`: Disable all output other than asking confirmation questions. 381 | 382 | `--force`: Assume 'yes' to all confirmation questions. 383 | 384 | `--exclude-staticfiles`: Do not copy any static files at all, only render output from 385 | Django views. 386 | 387 | `--skip-verify`: Do not test if files are correctly uploaded on the server. 388 | 389 | `--ignore-remote-content`: Do not fetch the list of remote files. It means that all 390 | files will be uploaded, and no existing remote file will be deleted. This can be 391 | useful if you have a lot of files on the remote server, and you know that you want 392 | to update most of them, and you don't care if old files remain on the server. 393 | 394 | `--parallel-publish [number of threads]`: Publish files in parallel on multiple 395 | threads, this can speed up publishing. Defaults to `1` thread. 396 | 397 | `--parallel-render [number of threads]`: Render files in parallel on multiple 398 | threads, this can speed up rendering. Defaults to `1` thread. 399 | 400 | `--generate-redirects`: Attempt to generate static redirects stored in the 401 | `django.contrib.redirects` app. If you have a redirect from `/old/` to `/new/` using 402 | this flag will create a static HTML `` 403 | style redirect at `/old/index.html` to `/new/`. 404 | 405 | **Note** that this means if you use `--force` and `--quiet` that the output 406 | directory will have all files not part of the site export deleted without any 407 | confirmation. 408 | 409 | **Note** If any of your views contain a Python error then rendering will fail 410 | then the stack trace will be printed to the terminal and the rendering command 411 | will exit with a status code of 1. 412 | 413 | 414 | # The `distill-test-publish` command 415 | 416 | ```bash 417 | $ ./manage.py distill-test-publish [optional destination here] 418 | ``` 419 | 420 | This will connect to your publishing target, authenticate to it, upload a 421 | randomly named file, verify it exists on the `PUBLIC_URL` and then delete it 422 | again. Use this to check your publishing settings are correct. 423 | 424 | `distill-test-publish` has no arguments. 425 | 426 | 427 | # Optional configuration settings 428 | 429 | You can set the following optional `settings.py` variables: 430 | 431 | **DISTILL_DIR**: string, default directory to export to: 432 | 433 | ```python 434 | DISTILL_DIR = '/path/to/export/directory' 435 | ``` 436 | 437 | **DISTILL_PUBLISH**: dictionary, like Django's `settings.DATABASES`, supports 438 | `default`: 439 | 440 | ```python 441 | DISTILL_PUBLISH = { 442 | 'default': { 443 | ... options ... 444 | }, 445 | 'some-other-target': { 446 | ... options ... 447 | }, 448 | } 449 | ``` 450 | 451 | **DISTILL_SKIP_ADMIN_DIRS**: bool, defaults to `True` 452 | 453 | ```python 454 | DISTILL_SKIP_ADMIN_DIRS = True 455 | ``` 456 | 457 | Set `DISTILL_SKIP_ADMIN_DIRS` to `False` if you want `django-distill` to also copy over 458 | static files in the `static/admin` directory. Usually, these are not required or 459 | desired for statically generated sites. The default behaviour is to skip static admin 460 | files. 461 | 462 | 463 | **DISTILL_SKIP_STATICFILES_DIRS**: list, defaults to `[]` 464 | 465 | ```python 466 | DISTILL_SKIP_STATICFILES_DIRS = ['some_dir'] 467 | ``` 468 | 469 | Set `DISTILL_SKIP_STATICFILES_DIRS` to a list of directory names you want `django-distill` 470 | to ignore directories in your defined `static/` directory. You can use this to ignore 471 | copying directories containing files from apps you're not using that get bundled into your 472 | `static/` directory by `collect-static`. For example if you set `DISTILL_SKIP_STATICFILES_DIRS` 473 | to `['some_dir']` the static files directory `static/some_dir` would be skipped. 474 | 475 | 476 | **DISTILL_LANGUAGES**: list, defaults to `[]` 477 | 478 | ```python 479 | DISTILL_LANGUAGES = [ 480 | 'en', 481 | 'fr', 482 | 'de', 483 | ] 484 | ``` 485 | 486 | Set `DISTILL_LANGUAGES` to a list of language codes to attempt to render URLs with. 487 | See the "Internationalization" section for more details. 488 | 489 | 490 | # Developing locally with HTTPS 491 | 492 | If you are using a local development environment which has HTTPS support you may need 493 | to add `SECURE_SSL_REDIRECT = False` to your `settings.py` to prevent a `CommandError` 494 | being raised when a request returns a 301 redirect instead of the expected HTTP/200 495 | response code. 496 | 497 | 498 | # Writing single files 499 | 500 | As of `django-distill` version `3.0.0` you can use the 501 | `django_distill.renderer.render_single_file` method to write out a single file 502 | to disk using `django_distill`. This is useful for writing out single files to disk, 503 | for example, you have a Django site which has some static files in a directory 504 | written by `django_distill` but the rest of the site is a normal dynamic Django site. 505 | You can update a static HTML file every time a model instance is saved. You can 506 | use single file writing with signals to achieve this. For example: 507 | 508 | ```python 509 | # in models.py 510 | from django.db.models.signals import post_save 511 | from django.dispatch import receiver 512 | from django_distill.renderer import render_single_file 513 | 514 | @receiver(post_save, sender=SomeBlogPostModel) 515 | def write_blog_post_static_file_post_save(sender, **kwargs): 516 | render_single_file( 517 | '/path/to/output/directory', 518 | 'blog-post-view-name', 519 | blog_id=sender.pk, 520 | blog_slug=sender.slug 521 | ) 522 | ``` 523 | 524 | The syntax for `render_single_file` is similar to Django's `url.reverse`. The full 525 | usage interface is: 526 | 527 | ```python 528 | render_single_file( 529 | '/path/to/output/directory', 530 | 'view-name-set-in-urls-py', 531 | *view_args, 532 | **view_kwargs 533 | ) 534 | ``` 535 | 536 | For example, if you had a blog post URL defined as: 537 | 538 | ```python 539 | # in urls.py 540 | distill_path('post/_.html', 541 | PostView.as_view(), 542 | name='blog-post', 543 | distill_func=get_all_blogposts), 544 | ``` 545 | 546 | Your usage would be: 547 | 548 | ```python 549 | render_single_file( 550 | '/path/to/output/directory', 551 | 'blog-post', 552 | blog_id=123, 553 | blog_slug='blog-title-slug', 554 | ) 555 | ``` 556 | 557 | which would write out the contents of `/post/123_blog-title-slug.html` into 558 | `/path/to/output/directory` as the file 559 | `/path/to/output/directory/post/123_blog-title-slug.html`. Note any required 560 | sub-directories (`/path/to/output/directory/post` in this example) will be 561 | automatically created if they don't already exist. All `django-distill` rules 562 | apply, such as URLs ending in `/` will be saved as `/index.html` to make sense 563 | for a physical file on disk. 564 | 565 | Also note that `render_single_file` can only be imported and used into an 566 | initialised Django project. 567 | 568 | 569 | # Publishing targets 570 | 571 | You can automatically publish sites to various supported remote targets through 572 | backends just like how you can use MySQL, SQLite, PostgreSQL etc. with 573 | Django by changing the backend database engine. Currently the engines supported 574 | by `django-distill` are: 575 | 576 | **django_distill.backends.amazon_s3**: Publish to an Amazon S3 bucket. Requires 577 | the Python library `boto3` (`$ pip install django-distill[amazon]`). The bucket 578 | must already exist (use the AWS control panel). Options: 579 | 580 | ```python 581 | 'some-s3-container': { 582 | 'ENGINE': 'django_distill.backends.amazon_s3', 583 | 'PUBLIC_URL': 'http://.../', 584 | 'ACCESS_KEY_ID': '...', 585 | 'SECRET_ACCESS_KEY': '...', 586 | 'BUCKET': '...', 587 | 'ENDPOINT_URL': 'https://.../', # Optional, set to use a different S3 endpoint 588 | 'DEFAULT_CONTENT_TYPE': 'application/octet-stream', # Optional 589 | }, 590 | ``` 591 | 592 | **django_distill.backends.google_storage**: Publish to a Google Cloud Storage 593 | bucket. Requires the Python libraries `google-api-python-client` and 594 | `google-cloud-storage` 595 | (`$ pip install django-distill[google]`). The bucket 596 | must already exist and be set up to host a public static website (use the 597 | Google Cloud control panel). Options: 598 | 599 | ```python 600 | 'some-google-storage-bucket': { 601 | 'ENGINE': 'django_distill.backends.google_storage', 602 | 'PUBLIC_URL': 'https://storage.googleapis.com/[bucket.name.here]/', 603 | 'BUCKET': '[bucket.name.here]', 604 | 'JSON_CREDENTIALS': '/path/to/some/credentials.json', 605 | }, 606 | ``` 607 | 608 | Note that `JSON_CREDENTIALS` is optional; if it is not specified, the google libraries 609 | will try other authentication methods, in the search order described here: 610 | https://cloud.google.com/docs/authentication/application-default-credentials (e.g. the 611 | `GOOGLE_APPLICATION_CREDENTIALS` environment variable, attached service account, etc). 612 | 613 | 614 | **django_distill.backends.microsoft_azure_storage**: Publish to a Microsoft 615 | Azure Blob Storage container. Requires the Python library 616 | `azure-storage-blob` (`$ pip install django-distill[microsoft]`). The storage 617 | account must already exist and be set up to host a public static website 618 | (use the Microsoft Azure control panel). Options: 619 | 620 | ```python 621 | 'some-microsoft-storage-account': { 622 | 'ENGINE': 'django_distill.backends.microsoft_azure_storage', 623 | 'PUBLIC_URL': 'https://[storage-account-name]...windows.net/', 624 | 'CONNECTION_STRING': '...', 625 | }, 626 | ``` 627 | 628 | Note that each Azure storage account supports one static website using the 629 | magic container `$web` which is where `django-distill` will attempt to 630 | publish your site. 631 | 632 | 633 | # Tests 634 | 635 | There is a minimal test suite, you can run it by cloing this repository, 636 | installing the required dependancies in `requirements.txt` then execuiting: 637 | 638 | ```bash 639 | # ./run-tests.py 640 | ``` 641 | 642 | 643 | # Contributing 644 | 645 | All properly formatted and sensible pull requests, issues and comments are 646 | welcome. 647 | --------------------------------------------------------------------------------