├── .flake8 ├── .gitignore ├── .travis.yml ├── README.rst ├── requirements.txt ├── runtests.py ├── setup.py ├── tests ├── __init__.py ├── app │ ├── migrations │ │ ├── 0001_initial.py │ │ └── __init__.py │ ├── models.py │ ├── settings.py │ └── urls.py ├── base.py ├── import_data │ ├── documents │ │ └── hello-world.txt │ └── images │ │ ├── floral.jpeg │ │ ├── hidden doge sleeping whippet.jpg │ │ ├── lazors.jpg │ │ ├── stretmch.jpg │ │ └── suunn.jpg ├── test_documents.py ├── test_images.py ├── test_pages.py ├── test_settings.py └── test_sites.py ├── tox.ini └── wagtailimporter ├── __init__.py ├── management ├── __init__.py └── commands │ ├── __init__.py │ └── import_pages.py └── serializer.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = migrations 3 | # ignore = E501 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | /dist/ 3 | /wagtailimporter.egg-info/ 4 | 5 | __pycache__ 6 | .tox/ 7 | .eggs/ 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | cache: pip 3 | 4 | # Matrix of build options 5 | matrix: 6 | include: 7 | - python: 3.5 8 | - python: 3.6 9 | - python: 3.6 10 | env: TOXENV=pylint,flake8 11 | 12 | install: 13 | - pip install --upgrade pip wheel tox setuptools 14 | 15 | script: 16 | - | 17 | : ${TOXENV:=$( tox --listenvs | grep "py${TRAVIS_PYTHON_VERSION/./}" | tr "\n" "," )} ; 18 | tox -e "${TOXENV}" 19 | 20 | deploy: 21 | provider: pypi 22 | user: danni 23 | password: 24 | secure: bZSMxZfYqcqe62cKEGyx7yruPMk46VssYEL8ROkoXCZjnXCVr8HmNHf0rzpIybmiH2FDLX46ip2Ftx0qc6NDszJLxKGnAgURBw8CrEgcooBFjFLKMRFIrR3vUg1eXl18boHoAsW3W/2urpbMEe1hSijZpA7ESRP2oxs/1M0RdBJgm+lhhwjT9xyMm54BJJ/MzuURqS6T7EXQQEs5B4RACAIKqHPVClCu0mSl1XaDzzOXLeZ2S+SmZ+x70pCSBfSyn7Vnn9YhosuS8blEWDjV4eBDXrnaFa3SKRxRx55dtjVomHAdE6sNrMNpzM356g6+92CGBd/xdkeHuualQTTliIl2GcUKEvLgYMZbThsvMhQT3zfRuAZZamexhtTqMho29bHMszzkudEMdkfKZxLi5GzdwNTCD98bz8TDi2yQXiKHOAREI/SE3OU3tZALUiHhVRNXQ1AbggwalZ9j90ALsUBLGoW1yc1XoLeqTgCCqVJZlbhYxrGPnliXtaAzOzvgZ/1Mtuhk2ZKc3jLQ41g2kfKDnxkl+aHrwvq/WlByxHj0RaTtezSUwyjz2A+WOIRqB2UwgpVg8KN5ONAMP17q/DRIRuWt0WvukIjmfM74TUDHbhePoJYsi5nLr/q8a4iUv+Gp/Qy6ECnbZrZTO53ObhJX+3op+VAurbMORQ0H8FU= 25 | on: 26 | tags: true 27 | all_branches: true 28 | python: 3.5 29 | condition: '"$TRAVIS_TAG" = "$(python setup.py --version)"' 30 | 31 | # vim: sw=2 32 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Wagtail Page Importer 2 | ===================== 3 | 4 | Adds the ability to Wagtail to import/update pages from a Yaml file. 5 | 6 | Installing 7 | ---------- 8 | 9 | :: 10 | 11 | INSTALLED_APPS = ( 12 | ... 13 | 'wagtailimporter', 14 | ... 15 | ) 16 | 17 | Usage 18 | ----- 19 | 20 | :: 21 | 22 | ./manage.py import_pages [ [ ... ] ...] 23 | 24 | File format 25 | ----------- 26 | 27 | Create a Yaml file/s for your pages. 28 | 29 | :: 30 | 31 | url: /site 32 | type: home.homepage 33 | 34 | # Your fields go here 35 | title: Welcome to my site! 36 | 37 | --- 38 | 39 | url: /site/about-us 40 | type: basic.basicpage 41 | 42 | title: About Us 43 | # This is a stream field 44 | body: 45 | - type: heading 46 | value: About Us 47 | - type: paragraph 48 | value: | 49 |

50 | Rich text block. 51 |

52 | 53 | - type: linkbutton 54 | # Yaml object to reference another page on the site 55 | link: !page 56 | url: /site/contact-us 57 | text: Contact us 58 | 59 | # This is rich text 60 | tagline: | 61 |
We are super great!
62 | 63 | --- 64 | 65 | !custom_snippet 66 | slug: my-snippet 67 | title: My Snippet 68 | 69 | Foreign Object References 70 | ------------------------- 71 | 72 | Instead of having to resolve foreign keys (yuck!) you can pass Yaml objects 73 | to create references, e.g. `!page` takes arguments that are used to reference 74 | a page (by path `url`). 75 | 76 | Builtin reference types: 77 | 78 | * ``!page`` 79 | 80 | Takes a `url` parameter to another page (must be already present). 81 | 82 | * ``!image`` 83 | 84 | Takes a `file` parameter to an image (either in `MEDIA/original/images`. 85 | 86 | Can also accept other `Image` related parameters such as `title`. 87 | 88 | * ``!document`` 89 | 90 | Takes a `file` parameter to a document (e.g. a PDF) - not to be confused with a yaml 'document'. 91 | 92 | Also accepts a `title` paramater (recommended for a better admin experience) 93 | 94 | * ``!site`` 95 | 96 | Lookup a `Site` by its `hostname`. 97 | 98 | Can also create sites if you specify `root_page` (as a `!page`). 99 | 100 | :: 101 | 102 | !site 103 | hostname: localhost 104 | site_name: My Site 105 | root_page: !page 106 | url: /my-site 107 | is_default_site: true 108 | 109 | * All registered Wagtail settings, using ``!app.mysetting`` 110 | 111 | :: 112 | 113 | !app.socialmedialinks: 114 | site: !site 115 | hostname: localhost 116 | facebook_url: https://www.facebook.com/squareweave/ 117 | 118 | You can also create your own for your models: 119 | 120 | :: 121 | 122 | import wagtailimporter.serializer 123 | 124 | class Toplevel(wagtailimporter.serializer.GetOrCreateForeignObject): 125 | """A reference to a toplevel""" 126 | yaml_tag = '!toplevel' 127 | model = TopLevel 128 | 129 | The following base classes are provided: 130 | 131 | * `GetForeignObject` 132 | 133 | Calls `get` on an object defined by `lookup_keys`. 134 | 135 | * `GetOrCreateForeignObject` (inherits from `GetForeignObject`) 136 | 137 | Calls `get_or_create` as above. 138 | 139 | * `GetOrCreateClusterableForeignObject` (inherits from `GetForeignObject`) 140 | 141 | Calls `get` or creates a new, unsaved object 142 | (useful for `ClusterableModel` related classes). 143 | 144 | For example: 145 | 146 | :: 147 | 148 | url: /my/page 149 | type: some.type 150 | related_pages: 151 | - !relatedpage 152 | page: !page 153 | url: /my/other/page 154 | 155 | :: 156 | 157 | class RelatedPage(Orderable): 158 | """A related page.""" 159 | 160 | parent = ParentalKey(SiteSettings, related_name='related_pages') 161 | page = models.ForeignKey('wagtailcore.Page', 162 | null=True, blank=True, 163 | on_delete=models.CASCADE, 164 | related_name='+') 165 | 166 | panels = [ 167 | PageChooserPanel('page'), 168 | ] 169 | 170 | 171 | class RelatedPageTag(GetOrCreateClusterableForeignObject): 172 | 173 | model = RelatedPage 174 | yaml_tag = '!relatedpage' 175 | lookup_keys = ('page',) 176 | 177 | 178 | Importing snippets 179 | ------------------ 180 | 181 | Foreign object references can also be used to create and import snippets. 182 | 183 | :: 184 | 185 | !custom_snippet 186 | slug: my-snippet 187 | title: My Snippet 188 | 189 | :: 190 | 191 | import wagtailimporter.serializer 192 | 193 | class MySnippet(wagtailimporter.serializer.GetOrCreateForeignObject): 194 | """Creates a snippet""" 195 | yaml_tag = '!my-snippet' 196 | model = MySnippet 197 | 198 | lookup_keys = ('slug',) 199 | 200 | License 201 | ------- 202 | 203 | Copyright (c) 2016, Squareweave Pty Ltd 204 | 205 | All rights reserved. 206 | 207 | Redistribution and use in source and binary forms, with or without 208 | modification, are permitted provided that the following conditions are met: 209 | 210 | * Redistributions of source code must retain the above copyright 211 | notice, this list of conditions and the following disclaimer. 212 | * Redistributions in binary form must reproduce the above copyright 213 | notice, this list of conditions and the following disclaimer in the 214 | documentation and/or other materials provided with the distribution. 215 | * Neither the name of the Squareweave nor the 216 | names of its contributors may be used to endorse or promote products 217 | derived from this software without specific prior written permission. 218 | 219 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 220 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 221 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 222 | DISCLAIMED. IN NO EVENT SHALL SQUAREWEAVE BE LIABLE FOR ANY 223 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 224 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 225 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 226 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 227 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 228 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 229 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | wagtail >= 4.0.0 2 | pyyaml 3 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Set up the test environment and run the tests for wagtailimporter.""" 3 | 4 | import os 5 | import sys 6 | import warnings 7 | 8 | 9 | def run(): 10 | """Run the tests.""" 11 | from django.core.management import execute_from_command_line 12 | os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.app.settings' 13 | os.environ.setdefault('DATABASE_NAME', ':memory:') 14 | warnings.filterwarnings('always', module='wagtailnews', category=DeprecationWarning) 15 | warnings.filterwarnings('always', module='wagtailnews', category=PendingDeprecationWarning) 16 | execute_from_command_line([sys.argv[0], 'test'] + sys.argv[1:]) 17 | 18 | 19 | if __name__ == '__main__': 20 | run() 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | setuptools configuration for wagtailimporter. 3 | """ 4 | 5 | from setuptools import find_packages, setup 6 | 7 | with open('README.rst') as readme, \ 8 | open('requirements.txt') as requirements: 9 | setup( 10 | name="wagtailimporter", 11 | use_scm_version=True, 12 | description="Wagtail module to load pages from Yaml", 13 | long_description=readme.read(), 14 | url='https://github.com/squareweave/wagtailimporter', 15 | author='Squareweave', 16 | author_email='team@squareweave.com.au', 17 | license='BSD', 18 | classifiers=[ 19 | 'Development Status :: 4 - Beta', 20 | 'Intended Audience :: Developers', 21 | 'Programming Language :: Python', 22 | 'Programming Language :: Python :: 3', 23 | 'Programming Language :: Python :: 3.7', 24 | 'Programming Language :: Python :: 3.8', 25 | 'Programming Language :: Python :: 3.9', 26 | 'Programming Language :: Python :: 3.10', 27 | 'Programming Language :: Python :: 3.11', 28 | 'Programming Language :: Python :: 3.12', 29 | 'Framework :: Django :: 3.2', 30 | 'Framework :: Django :: 4', 31 | 'Framework :: Django :: 4.0', 32 | 'Framework :: Django :: 4.1', 33 | 'Framework :: Django :: 4.2', 34 | 'Framework :: Django :: 5.0', 35 | 'Framework :: Wagtail', 36 | 'Framework :: Wagtail :: 4', 37 | 'Framework :: Wagtail :: 5', 38 | 'Framework :: Wagtail :: 6', 39 | ], 40 | packages=find_packages(), 41 | include_package_data=True, 42 | install_requires=requirements.readlines(), 43 | setup_requires=[ 44 | 'setuptools_scm', 45 | ], 46 | ) 47 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squareweave/wagtailimporter/c54d86d478634d462f326866aa5c736585531643/tests/__init__.py -------------------------------------------------------------------------------- /tests/app/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2018-05-07 04:32 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | initial = True 10 | 11 | dependencies = [ 12 | ('wagtailcore', '0040_page_draft_title'), 13 | ('wagtailimages', '0019_delete_filter'), 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='BasicPage', 19 | fields=[ 20 | ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.Page')), 21 | ('body', models.TextField(blank=True)), 22 | ], 23 | options={ 24 | 'abstract': False, 25 | }, 26 | bases=('wagtailcore.page',), 27 | ), 28 | migrations.CreateModel( 29 | name='BasicSetting', 30 | fields=[ 31 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 32 | ('text', models.TextField(blank=True)), 33 | ('cute_dog', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='wagtailimages.Image')), 34 | ('site', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.Site')), 35 | ], 36 | options={ 37 | 'abstract': False, 38 | }, 39 | ), 40 | migrations.CreateModel( 41 | name='ForeignKeyPage', 42 | fields=[ 43 | ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.Page')), 44 | ('image', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='wagtailimages.Image')), 45 | ('other_page', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailcore.Page')), 46 | ], 47 | options={ 48 | 'abstract': False, 49 | }, 50 | bases=('wagtailcore.page',), 51 | ), 52 | ] 53 | -------------------------------------------------------------------------------- /tests/app/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squareweave/wagtailimporter/c54d86d478634d462f326866aa5c736585531643/tests/app/migrations/__init__.py -------------------------------------------------------------------------------- /tests/app/models.py: -------------------------------------------------------------------------------- 1 | """ 2 | Models for testing wagtailimporter. 3 | """ 4 | from django.db import models 5 | from wagtail.contrib.settings.models import BaseSiteSetting, register_setting 6 | from wagtail.models import Page 7 | 8 | 9 | class BasicPage(Page): 10 | """The simplest page type.""" 11 | body = models.TextField(blank=True) 12 | 13 | 14 | class ForeignKeyPage(Page): 15 | """A page with some foreign keys to other things.""" 16 | other_page = models.ForeignKey( 17 | Page, related_name='+', 18 | null=True, blank=True, default=None, on_delete=models.SET_NULL) 19 | image = models.ForeignKey( 20 | 'wagtailimages.Image', 21 | null=True, blank=True, default=None, on_delete=models.SET_NULL) 22 | 23 | 24 | @register_setting 25 | class BasicSetting(BaseSiteSetting): 26 | """The simplest setting.""" 27 | text = models.TextField(blank=True) 28 | cute_dog = models.ForeignKey( 29 | 'wagtailimages.Image', 30 | null=True, blank=True, default=None, on_delete=models.SET_NULL) 31 | -------------------------------------------------------------------------------- /tests/app/settings.py: -------------------------------------------------------------------------------- 1 | """Settings for wagtailimporter tests""" 2 | import os 3 | 4 | INSTALLED_APPS = [ 5 | 'wagtailimporter', 6 | 'tests.app', 7 | 8 | 'taggit', 9 | 'modelcluster', 10 | 11 | 'wagtail', 12 | 'wagtail.admin', 13 | 'wagtail.users', 14 | 'wagtail.sites', 15 | 'wagtail.snippets', 16 | 'wagtail.search', 17 | 'wagtail.documents', 18 | 'wagtail.images', 19 | 'wagtail.contrib.routable_page', 20 | 21 | 22 | 'django.contrib.admin', 23 | 'django.contrib.auth', 24 | 'django.contrib.contenttypes', 25 | 'django.contrib.sessions', 26 | 'django.contrib.staticfiles', 27 | 'django.contrib.messages' 28 | ] 29 | 30 | ALLOWED_HOSTS = ['localhost'] 31 | 32 | SECRET_KEY = 'not a secret' 33 | 34 | ROOT_URLCONF = 'tests.app.urls' 35 | 36 | DATABASES = { 37 | 'default': { 38 | 'ENGINE': 'django.db.backends.sqlite3', 39 | 'NAME': os.environ.get('DATABASE_NAME', ':memory:'), 40 | }, 41 | } 42 | 43 | WAGTAIL_SITE_NAME = 'Wagtail Importer' 44 | WAGTAILADMIN_BASE_URL = "https://example.com/" 45 | 46 | DEBUG = True 47 | 48 | USE_TZ = True 49 | TIME_ZONE = 'Australia/Hobart' 50 | 51 | MIDDLEWARE = [ 52 | 'django.contrib.sessions.middleware.SessionMiddleware', 53 | 'django.middleware.common.CommonMiddleware', 54 | 'django.middleware.csrf.CsrfViewMiddleware', 55 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 56 | 'django.contrib.messages.middleware.MessageMiddleware', 57 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 58 | 59 | 'django.contrib.sites.middleware.CurrentSiteMiddleware', 60 | ] 61 | 62 | TEMPLATES = [ 63 | { 64 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 65 | 'APP_DIRS': True, 66 | 'OPTIONS': { 67 | 'context_processors': [ 68 | 'django.contrib.auth.context_processors.auth', 69 | 'django.template.context_processors.debug', 70 | 'django.template.context_processors.i18n', 71 | 'django.template.context_processors.media', 72 | 'django.template.context_processors.static', 73 | 'django.template.context_processors.tz', 74 | 'django.template.context_processors.request', 75 | 'django.contrib.messages.context_processors.messages' 76 | ], 77 | }, 78 | }, 79 | ] 80 | 81 | STATIC_ROOT = os.path.join(os.path.dirname(__file__), 'static') 82 | STATIC_URL = '/static/' 83 | 84 | DEFAULT_AUTO_FIELD = "django.db.models.AutoField" 85 | -------------------------------------------------------------------------------- /tests/app/urls.py: -------------------------------------------------------------------------------- 1 | """URL config.""" 2 | from django.urls import include, path 3 | from wagtail.admin import urls as wagtailadmin_urls 4 | from wagtail import urls as wagtail_urls 5 | 6 | urlpatterns = [ 7 | path(r'admin/', include(wagtailadmin_urls)), 8 | path(r'', include(wagtail_urls)), 9 | ] 10 | -------------------------------------------------------------------------------- /tests/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | Base test class for testing the import_pages command 3 | """ 4 | 5 | import os 6 | from contextlib import contextmanager 7 | from pathlib import Path 8 | from tempfile import NamedTemporaryFile, TemporaryDirectory 9 | 10 | from django.core.management import call_command 11 | from django.test import override_settings 12 | 13 | 14 | class ImporterTestCaseMixin: 15 | """ 16 | TestCase mixin for testing the import_pages management command. 17 | """ 18 | def run_import(self, yaml, 19 | **kwargs): 20 | """ 21 | Run an import with the supplied YAML document. ``stdout`` and 22 | ``stderr`` are silenced by default, pass in an ``io.StringIO`` instance 23 | as a kwarg if you want to collect either. 24 | """ 25 | with NamedTemporaryFile('w+', dir=str(self.get_import_dir())) as temp: 26 | temp.write(yaml) 27 | temp.seek(0) 28 | 29 | kwargs.setdefault('stdout', open(os.devnull, 'w')) # noqa: E501 pylint: disable=consider-using-with,unspecified-encoding 30 | kwargs.setdefault('stderr', open(os.devnull, 'w')) # noqa: E501 pylint: disable=consider-using-with,unspecified-encoding 31 | call_command('import_pages', temp.name, **kwargs) 32 | 33 | def get_import_dir(self): 34 | """Where to run the import from.""" 35 | return Path(__file__).parent / 'import_data' 36 | 37 | 38 | @contextmanager 39 | def fresh_media_root(**kwargs): 40 | """ 41 | Run a test with a fresh, empty MEDIA_ROOT temporary directory. 42 | """ 43 | with TemporaryDirectory(**kwargs) as media_root: 44 | with override_settings(MEDIA_ROOT=media_root): 45 | yield 46 | -------------------------------------------------------------------------------- /tests/import_data/documents/hello-world.txt: -------------------------------------------------------------------------------- 1 | Hello, world! 2 | -------------------------------------------------------------------------------- /tests/import_data/images/floral.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squareweave/wagtailimporter/c54d86d478634d462f326866aa5c736585531643/tests/import_data/images/floral.jpeg -------------------------------------------------------------------------------- /tests/import_data/images/hidden doge sleeping whippet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squareweave/wagtailimporter/c54d86d478634d462f326866aa5c736585531643/tests/import_data/images/hidden doge sleeping whippet.jpg -------------------------------------------------------------------------------- /tests/import_data/images/lazors.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squareweave/wagtailimporter/c54d86d478634d462f326866aa5c736585531643/tests/import_data/images/lazors.jpg -------------------------------------------------------------------------------- /tests/import_data/images/stretmch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squareweave/wagtailimporter/c54d86d478634d462f326866aa5c736585531643/tests/import_data/images/stretmch.jpg -------------------------------------------------------------------------------- /tests/import_data/images/suunn.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squareweave/wagtailimporter/c54d86d478634d462f326866aa5c736585531643/tests/import_data/images/suunn.jpg -------------------------------------------------------------------------------- /tests/test_documents.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test importing Wagtail pages. 3 | """ 4 | import textwrap 5 | from pathlib import Path 6 | 7 | from django.conf import settings 8 | from django.test import TestCase 9 | from wagtail.documents.models import Document 10 | 11 | from .base import ImporterTestCaseMixin, fresh_media_root 12 | 13 | 14 | class TestDocumentImport(ImporterTestCaseMixin, TestCase): 15 | """Test importing Wagtail documents.""" 16 | 17 | @fresh_media_root() 18 | def test_basic_import(self): 19 | """Test importing a single document.""" 20 | doc = textwrap.dedent( 21 | """ 22 | !document 23 | title: Annual report 24 | file: hello-world.txt 25 | """ 26 | ) 27 | self.run_import(doc) 28 | 29 | document = Document.objects.get() 30 | self.assertEqual(document.title, "Annual report") 31 | self.assertEqual(document.file.name, "documents/hello-world.txt") 32 | 33 | # Check the file imported correctly 34 | original_filename = self.get_import_dir() / 'documents/hello-world.txt' 35 | with document.file as imported: 36 | with open(str(original_filename), 'rb') as original: 37 | self.assertEqual(imported.read(), original.read()) 38 | 39 | @fresh_media_root() 40 | def test_update(self): 41 | """Test updating an document.""" 42 | doc = textwrap.dedent( 43 | """ 44 | !document 45 | title: Annual report 46 | file: hello-world.txt 47 | """ 48 | ) 49 | self.run_import(doc) 50 | document = Document.objects.get() 51 | self.assertEqual(document.title, "Annual report") 52 | self.assertEqual(document.file.name, 'documents/hello-world.txt') 53 | 54 | doc = textwrap.dedent( 55 | """ 56 | !document 57 | title: Annual report 2018 58 | file: hello-world.txt 59 | """ 60 | ) 61 | self.run_import(doc) 62 | 63 | document.refresh_from_db() 64 | # The title should have updated 65 | self.assertEqual(document.title, "Annual report 2018") 66 | # The filename should not have changed 67 | self.assertEqual(document.file.name, 'documents/hello-world.txt') 68 | # Check that there is still only a single document in MEDIA_ROOT 69 | file_path = Path(settings.MEDIA_ROOT) / Path(document.file.name) 70 | directory = file_path.parent 71 | self.assertEqual( 72 | list(directory.iterdir()), # pylint:disable=no-member 73 | [file_path]) 74 | -------------------------------------------------------------------------------- /tests/test_images.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test importing Wagtail pages. 3 | """ 4 | import textwrap 5 | from pathlib import Path 6 | 7 | from django.conf import settings 8 | from django.test import TestCase 9 | from wagtail.images.models import Image 10 | 11 | from .base import ImporterTestCaseMixin, fresh_media_root 12 | 13 | 14 | class TestImageImport(ImporterTestCaseMixin, TestCase): 15 | """Test importing Wagtail images.""" 16 | 17 | @fresh_media_root() 18 | def test_basic_import(self): 19 | """Test importing a single image.""" 20 | doc = textwrap.dedent( 21 | """ 22 | !image 23 | title: Floral doge 24 | file: floral.jpeg 25 | """ 26 | ) 27 | self.run_import(doc) 28 | 29 | image = Image.objects.get() 30 | self.assertEqual(image.title, "Floral doge") 31 | self.assertEqual(image.file.name, 'original_images/images/floral.jpeg') 32 | 33 | # Check the file imported correctly 34 | original_filename = self.get_import_dir() / 'images/floral.jpeg' 35 | with image.file as imported: 36 | with open(str(original_filename), 'rb') as original: 37 | self.assertEqual(imported.read(), original.read()) 38 | 39 | @fresh_media_root() 40 | def test_update(self): 41 | """Test updating an image.""" 42 | doc = textwrap.dedent( 43 | """ 44 | !image 45 | title: Floral doge 46 | file: lazors.jpg 47 | """ 48 | ) 49 | self.run_import(doc) 50 | image = Image.objects.get() 51 | self.assertEqual(image.title, "Floral doge") 52 | self.assertEqual(image.file.name, 'original_images/images/lazors.jpg') 53 | 54 | doc = textwrap.dedent( 55 | """ 56 | !image 57 | title: Dog and flowers 58 | file: lazors.jpg 59 | """ 60 | ) 61 | self.run_import(doc) 62 | 63 | image.refresh_from_db() 64 | # The title should have updated 65 | self.assertEqual(image.title, "Dog and flowers") 66 | # The filename should not have changed 67 | self.assertEqual(image.file.name, 'original_images/images/lazors.jpg') 68 | # Check that there is still only a single image in MEDIA_ROOT 69 | file_path = Path(settings.MEDIA_ROOT) / Path(image.file.name) 70 | directory = file_path.parent 71 | self.assertEqual( 72 | list(directory.iterdir()), # pylint:disable=no-member 73 | [file_path]) 74 | 75 | @fresh_media_root() 76 | def test_spaces_in_filename(self): 77 | """Test importing an images with spaces in the filename.""" 78 | doc = textwrap.dedent( 79 | """ 80 | !image 81 | title: Hidden doge, sleeping whippet 82 | file: "hidden doge sleeping whippet.jpg" 83 | """ 84 | ) 85 | self.run_import(doc) 86 | filename = Path('images/hidden doge sleeping whippet.jpg') 87 | 88 | image = Image.objects.get() 89 | self.assertEqual(Path(image.file.name), 90 | Path('original_images') / filename) 91 | -------------------------------------------------------------------------------- /tests/test_pages.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test importing Wagtail pages. 3 | """ 4 | import textwrap 5 | 6 | from django.test import TestCase 7 | 8 | from .app.models import BasicPage, ForeignKeyPage 9 | from .base import ImporterTestCaseMixin 10 | 11 | 12 | class TestPageImport(ImporterTestCaseMixin, TestCase): 13 | """Test importing Wagtail pages.""" 14 | def test_basic_import(self): 15 | """Test importing some simple nested pages.""" 16 | doc = textwrap.dedent( 17 | """ 18 | url: /page/ 19 | type: app.basicpage 20 | title: Basic import 21 | show_in_menus: true 22 | body: Hello, world! 23 | 24 | --- 25 | 26 | url: /page/child/ 27 | type: app.basicpage 28 | title: Child page 29 | """ 30 | ) 31 | self.run_import(doc) 32 | 33 | self.assertEqual(BasicPage.objects.count(), 2) 34 | 35 | page = BasicPage.objects.get(url_path='/page/') 36 | self.assertEqual(page.title, "Basic import") 37 | self.assertEqual(page.slug, 'page') 38 | self.assertTrue(page.show_in_menus) 39 | self.assertEqual(page.body, "Hello, world!") 40 | 41 | child = BasicPage.objects.child_of(page).get() 42 | self.assertEqual(child.url_path, '/page/child/') 43 | self.assertEqual(child.slug, 'child') 44 | self.assertEqual(child.title, "Child page") 45 | 46 | def test_update(self): 47 | """ 48 | Test updating an existing page with new fields. 49 | """ 50 | doc = textwrap.dedent( 51 | """ 52 | url: /page/ 53 | type: app.basicpage 54 | title: Initial title 55 | """ 56 | ) 57 | self.run_import(doc) 58 | 59 | page = BasicPage.objects.get() 60 | self.assertEqual(page.title, "Initial title") 61 | 62 | doc = textwrap.dedent( 63 | """ 64 | url: /page/ 65 | type: app.basicpage 66 | title: New title 67 | """ 68 | ) 69 | self.run_import(doc) 70 | 71 | page.refresh_from_db() 72 | self.assertEqual(page.title, "New title") 73 | 74 | def test_page_foreign_key(self): 75 | """Test importing some simple nested pages.""" 76 | doc = textwrap.dedent( 77 | """ 78 | url: /target/ 79 | type: app.basicpage 80 | title: Target page 81 | 82 | --- 83 | 84 | url: /source/ 85 | type: app.foreignkeypage 86 | title: Source page 87 | other_page: !page { url: /target/ } 88 | """ 89 | ) 90 | self.run_import(doc) 91 | 92 | target = BasicPage.objects.get() 93 | source = ForeignKeyPage.objects.get() 94 | 95 | self.assertEqual(source.other_page.specific, target) 96 | 97 | def test_deferred_foreign_key(self): 98 | """Test importing some simple nested pages.""" 99 | doc = textwrap.dedent( 100 | """ 101 | url: /parent/ 102 | type: app.foreignkeypage 103 | title: Parent page 104 | 105 | --- 106 | 107 | url: /parent/child/ 108 | type: app.basicpage 109 | title: Child page 110 | 111 | --- 112 | 113 | url: /parent/ 114 | type: app.foreignkeypage 115 | other_page: !page { url: /parent/child/ } 116 | """ 117 | ) 118 | self.run_import(doc) 119 | 120 | parent = ForeignKeyPage.objects.get() 121 | child = BasicPage.objects.child_of(parent).get() 122 | 123 | self.assertEqual(parent.other_page.specific, child) 124 | 125 | def test_other_pages_untouched(self): 126 | """Test that pages not part of the import are untouched.""" 127 | doc = textwrap.dedent( 128 | """ 129 | url: /page/ 130 | type: app.basicpage 131 | title: Basic page 132 | """ 133 | ) 134 | self.run_import(doc) 135 | 136 | page = BasicPage.objects.get() 137 | page.title = "New title" 138 | page.save() 139 | 140 | child = page.add_child(instance=BasicPage( 141 | title="Child page", 142 | slug="child", 143 | )) 144 | 145 | self.run_import(doc) 146 | 147 | page.refresh_from_db() 148 | self.assertEqual(page.title, "Basic page") 149 | self.assertTrue(BasicPage.objects.filter(pk=child.pk).exists()) 150 | -------------------------------------------------------------------------------- /tests/test_settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test importing Wagtail pages. 3 | """ 4 | import textwrap 5 | 6 | from django.test import TestCase 7 | from wagtail.models import Site 8 | from wagtail.images.models import Image 9 | 10 | from .app.models import BasicSetting 11 | from .base import ImporterTestCaseMixin, fresh_media_root 12 | 13 | 14 | class TestSettingsImport(ImporterTestCaseMixin, TestCase): 15 | """Test importing Wagtail sites.""" 16 | def import_site(self, hostname='example.com'): 17 | """Make a site with a hostname to create settings for.""" 18 | doc = textwrap.dedent( 19 | """ 20 | url: /my-home/ 21 | type: app.basicpage 22 | title: Home page 23 | 24 | --- 25 | 26 | !site 27 | hostname: {hostname} 28 | site_name: "Example website" 29 | root_page: !page 30 | url: /my-home/ 31 | is_default_site: True 32 | """ 33 | ).format(hostname=hostname) 34 | self.run_import(doc) 35 | return Site.objects.get(hostname=hostname) 36 | 37 | def test_basic_import(self): 38 | """Test importing a simple setting.""" 39 | site = self.import_site() 40 | doc = textwrap.dedent( 41 | """ 42 | !app.basicsetting 43 | site: !site { hostname: example.com } 44 | text: Hello, world! 45 | """ 46 | ) 47 | self.run_import(doc) 48 | 49 | setting = BasicSetting.objects.get(site=site) 50 | self.assertEqual(setting.text, "Hello, world!") 51 | 52 | @fresh_media_root() 53 | def test_foreign_keys(self): 54 | """Test importing settings that have foreign keys to other models.""" 55 | site = self.import_site() 56 | doc = textwrap.dedent( 57 | """ 58 | !image 59 | file: suunn.jpg 60 | title: Sunny sun dog 61 | 62 | --- 63 | 64 | !app.basicsetting 65 | site: !site { hostname: example.com } 66 | cute_dog: !image { file: suunn.jpg } 67 | """ 68 | ) 69 | self.run_import(doc) 70 | 71 | setting = BasicSetting.objects.get(site=site) 72 | image = Image.objects.get() 73 | self.assertEqual(setting.cute_dog, image) 74 | 75 | @fresh_media_root() 76 | def test_update(self): 77 | """Test updating a site instance, preserving old fields.""" 78 | site = self.import_site() 79 | doc = textwrap.dedent( 80 | """ 81 | !app.basicsetting 82 | site: !site { hostname: example.com } 83 | text: Hello, world! 84 | """ 85 | ) 86 | self.run_import(doc) 87 | 88 | doc = textwrap.dedent( 89 | """ 90 | !image 91 | file: suunn.jpg 92 | title: Sunny sun dog 93 | 94 | --- 95 | 96 | !app.basicsetting 97 | site: !site { hostname: example.com } 98 | cute_dog: !image { file: suunn.jpg } 99 | """ 100 | ) 101 | self.run_import(doc) 102 | 103 | setting = BasicSetting.objects.get(site=site) 104 | image = Image.objects.get() 105 | self.assertEqual(setting.text, "Hello, world!") 106 | self.assertEqual(setting.cute_dog, image) 107 | -------------------------------------------------------------------------------- /tests/test_sites.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test importing Wagtail pages. 3 | """ 4 | import textwrap 5 | 6 | from django.test import TestCase 7 | from wagtail.models import Site 8 | 9 | from .app.models import BasicPage 10 | from .base import ImporterTestCaseMixin 11 | 12 | 13 | class TestSiteImport(ImporterTestCaseMixin, TestCase): 14 | """Test importing Wagtail sites.""" 15 | def test_basic_import(self): 16 | """Test importing some simple nested pages.""" 17 | doc = textwrap.dedent( 18 | """ 19 | url: /my-home/ 20 | type: app.basicpage 21 | title: Home page 22 | 23 | --- 24 | 25 | !site 26 | hostname: example.com 27 | site_name: "Example website" 28 | root_page: !page { url: /my-home/ } 29 | is_default_site: True 30 | """ 31 | ) 32 | self.run_import(doc) 33 | 34 | page = BasicPage.objects.get() 35 | site = Site.objects.get(hostname='example.com') 36 | self.assertEqual(site.root_page.specific, page) 37 | self.assertEqual(site.site_name, "Example website") 38 | 39 | def test_update(self): 40 | """Test updating a site instance.""" 41 | doc = textwrap.dedent( 42 | """ 43 | url: /my-home/ 44 | type: app.basicpage 45 | title: Home page 46 | 47 | --- 48 | 49 | !site 50 | hostname: example.com 51 | site_name: "Example website" 52 | root_page: !page { url: /my-home/ } 53 | is_default_site: True 54 | """ 55 | ) 56 | self.run_import(doc) 57 | 58 | # Check everything imported sensibly 59 | home = BasicPage.objects.get() 60 | site = Site.objects.get(hostname='example.com') 61 | self.assertEqual(site.root_page.specific, home) 62 | self.assertEqual(site.site_name, "Example website") 63 | 64 | doc = textwrap.dedent( 65 | """ 66 | url: /new-home/ 67 | type: app.basicpage 68 | title: New home page 69 | 70 | --- 71 | 72 | !site 73 | hostname: example.com 74 | site_name: "New example website" 75 | root_page: !page { url: /new-home/ } 76 | is_default_site: True 77 | """ 78 | ) 79 | self.run_import(doc) 80 | 81 | # Get new data 82 | new_home = BasicPage.objects.get(url_path='/new-home/') 83 | site.refresh_from_db() 84 | 85 | # The site should have a new home page 86 | self.assertEqual(site.root_page.specific, new_home) 87 | 88 | # Check the old page still exists 89 | self.assertTrue(BasicPage.objects.filter(pk=home.pk).exists()) 90 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skip_missing_interpreters = True 3 | 4 | envlist = 5 | py39-dj32-wt40 6 | py39-dj41-wt40 7 | py310-dj32-wt41 8 | py311-dj41-wt41 9 | py311-dj42-wt50 10 | py311-dj42-wt51 11 | py312-dj42-wt51 12 | py312-dj42-wt52 13 | py312-dj50-wt52 14 | py312-dj50-wt60 15 | flake8 16 | 17 | 18 | [testenv] 19 | commands = python runtests.py {posargs} 20 | 21 | deps = 22 | dj32: django~=3.2.0 23 | dj41: django~=4.1.0 24 | dj42: django~=4.2.0 25 | dj50: django~=5.0.0 26 | wt40: Wagtail~=4.0.0 27 | wt41: Wagtail~=4.1.0 28 | wt50: Wagtail~=5.0.0 29 | wt51: Wagtail~=5.1.0 30 | wt52: Wagtail~=5.2.0 31 | wt60: Wagtail~=6.0.0 32 | 33 | [testenv:flake8] 34 | deps = flake8 35 | basepython = python3 36 | commands = flake8 wagtailimporter/ tests/ 37 | -------------------------------------------------------------------------------- /wagtailimporter/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squareweave/wagtailimporter/c54d86d478634d462f326866aa5c736585531643/wagtailimporter/__init__.py -------------------------------------------------------------------------------- /wagtailimporter/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squareweave/wagtailimporter/c54d86d478634d462f326866aa5c736585531643/wagtailimporter/management/__init__.py -------------------------------------------------------------------------------- /wagtailimporter/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/squareweave/wagtailimporter/c54d86d478634d462f326866aa5c736585531643/wagtailimporter/management/commands/__init__.py -------------------------------------------------------------------------------- /wagtailimporter/management/commands/import_pages.py: -------------------------------------------------------------------------------- 1 | """ 2 | Import pages into Wagtail 3 | """ 4 | import json 5 | import os 6 | from pathlib import Path, PurePosixPath 7 | 8 | import yaml 9 | from django.contrib.contenttypes.models import ContentType 10 | from django.core.exceptions import FieldDoesNotExist 11 | from django.core.management.base import BaseCommand, CommandError 12 | from django.db import transaction 13 | from wagtail.fields import StreamField 14 | from wagtail.models import Page 15 | 16 | from ... import serializer 17 | from ...serializer import normalise 18 | 19 | 20 | class Command(BaseCommand): 21 | """ 22 | Import pages into Wagtail. 23 | """ 24 | 25 | def add_arguments(self, parser): 26 | parser.add_argument('file', nargs='+', type=str) 27 | 28 | @transaction.atomic 29 | def handle(self, *args, **options): 30 | for filename in options['file']: 31 | with open(filename, encoding="utf-8") as file_: 32 | docs = yaml.safe_load_all(file_) 33 | self.stdout.write(f"Reading {filename}") 34 | 35 | cwd = Path.cwd() 36 | try: 37 | os.chdir(str(Path(filename).parent)) 38 | self.import_documents(docs) 39 | finally: 40 | os.chdir(str(cwd)) 41 | 42 | @transaction.atomic 43 | def import_documents(self, docs): 44 | """Import a Yaml file of documents.""" 45 | 46 | for doc in docs: 47 | try: 48 | if isinstance(doc, serializer.GetForeignObject): 49 | self.import_snippet(doc) 50 | else: 51 | self.import_page(doc) 52 | except CommandError as exc: 53 | self.stderr.write(f"Error importing page: {exc}") 54 | 55 | @transaction.atomic 56 | def import_snippet(self, data): 57 | """Import a snippet (which is a GetForeignObject).""" 58 | 59 | obj = data.__to_value__() 60 | self.stdout.write(f"Importing {obj._meta.verbose_name} {obj}") 61 | obj.save() 62 | 63 | @transaction.atomic 64 | def import_page(self, data): 65 | """Import a single wagtail page.""" 66 | model = self.get_page_model_class(data) 67 | self.find_page(model, data) 68 | 69 | def get_page_model_class(self, data): 70 | """ 71 | Get the page model class from the `type' parameter. 72 | """ 73 | 74 | try: 75 | type_ = data.pop('type') 76 | except KeyError as exc: 77 | raise CommandError("Need `type' for page") from exc 78 | 79 | try: 80 | app_label, model = type_.split('.') 81 | return ContentType.objects.get(app_label=app_label, 82 | model=model)\ 83 | .model_class() 84 | except (ValueError, AttributeError) as exc: 85 | raise CommandError("`type' is of form `app.model'") from exc 86 | except ContentType.DoesNotExist as exc: 87 | raise CommandError(f"Unknown page type `{type_}'") from exc 88 | 89 | def find_page(self, model, data): 90 | """ 91 | Find a page by its URL and import its data. 92 | 93 | Data importing has to be done here because often the page can't 94 | be saved until the data is imported (i.e. null fields) 95 | """ 96 | try: 97 | url = PurePosixPath(data.pop('url')) 98 | if not url.is_absolute(): 99 | raise CommandError(f"Path {url} must be absolute") 100 | 101 | except KeyError as exc: 102 | raise CommandError("Need `url' for page") from exc 103 | 104 | try: 105 | page = model.objects.get(url_path=normalise(url)) 106 | self.import_data(page, data) 107 | page.save() 108 | self.stdout.write(f"Updating existing page {url}") 109 | except model.DoesNotExist: 110 | try: 111 | # pylint:disable=no-member 112 | parent = Page.objects.get(url_path=normalise(url.parent)) 113 | except Page.DoesNotExist as exc: 114 | raise CommandError(f"Parent of {url} doesn't exist") from exc 115 | 116 | page = model(slug=url.name) 117 | self.import_data(page, data) 118 | parent.add_child(instance=page) 119 | self.stdout.write(f"Creating new page {url}") 120 | 121 | return page 122 | 123 | def import_data(self, page, data): 124 | """Import the data onto a page.""" 125 | 126 | for key, value in data.items(): 127 | try: 128 | field = page._meta.get_field(key) 129 | 130 | if isinstance(field, StreamField): 131 | value = json.dumps(value, cls=serializer.JSONEncoder) 132 | else: 133 | # Assume we know how to serialise it 134 | pass 135 | 136 | except FieldDoesNotExist: 137 | # This might be a property, just try and set it anyway 138 | pass 139 | 140 | value = serializer.FieldStorable.to_objects(value) 141 | 142 | setattr(page, key, value) 143 | -------------------------------------------------------------------------------- /wagtailimporter/serializer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Objects for YAML serializer/deserializer 3 | """ 4 | import json 5 | import logging 6 | import os 7 | from pathlib import PurePosixPath 8 | 9 | import yaml 10 | from wagtail.contrib.settings.registry import registry 11 | from wagtail.coreutils import string_to_ascii 12 | from wagtail.fields import StreamField 13 | from wagtail.models import Page as WagtailPage 14 | from wagtail.models import Site as WagtailSite 15 | from wagtail.documents.models import Document as WagtailDocument 16 | from wagtail.images.models import Image as WagtailImage 17 | 18 | LOGGER = logging.getLogger(__name__) 19 | 20 | 21 | def normalise(url): 22 | """Normalize URL paths by appending a trailing slash.""" 23 | url = str(url) 24 | 25 | if not url.endswith('/'): 26 | url += '/' 27 | 28 | return url 29 | 30 | 31 | class JSONSerializable: 32 | """Interface to objects which are serializable into JSON.""" 33 | 34 | def __to_json__(self): 35 | """Serialize to JSON.""" 36 | raise NotImplementedError() 37 | 38 | 39 | class FieldStorable: 40 | """Interface to objects which can be stored in Django model fields.""" 41 | 42 | def __to_value__(self): 43 | """Value to store in a Django model.""" 44 | raise NotImplementedError() 45 | 46 | @classmethod 47 | def to_objects(cls, value): 48 | """Convert FieldStorables to objects.""" 49 | if isinstance(value, cls): 50 | value = value.__to_value__() # pylint:disable=no-member 51 | 52 | elif isinstance(value, list): 53 | value = [cls.to_objects(elem) for elem in value] 54 | 55 | elif isinstance(value, dict): 56 | value = { 57 | key: cls.to_objects(elem) 58 | for key, elem in value.items() 59 | } 60 | 61 | else: 62 | pass 63 | 64 | return value 65 | 66 | 67 | class JSONEncoder(json.JSONEncoder): 68 | """ 69 | Extension of the JSON encoder that knows how to encode the YAML objects 70 | """ 71 | 72 | def default(self, o): # pylint:disable=method-hidden 73 | if isinstance(o, JSONSerializable): 74 | return o.__to_json__() 75 | 76 | return super().default(o) 77 | 78 | 79 | class GetForeignObject(FieldStorable, yaml.YAMLObject): 80 | """ 81 | Get a foreign key reference for the provided parameters 82 | """ 83 | 84 | yaml_loader = yaml.SafeLoader 85 | 86 | @property 87 | def model(self): 88 | """Model for this reference.""" 89 | raise NotImplementedError() 90 | 91 | @property 92 | def lookup_keys(self): 93 | """ 94 | Lookup keys. 95 | 96 | Default implementation is all model fields that are defined in 97 | the Yaml block. 98 | """ 99 | 100 | return (field.name for field in self.model._meta.get_fields()) 101 | 102 | def lookup(self): 103 | """ 104 | Generate the lookup 105 | """ 106 | 107 | return { 108 | field: FieldStorable.to_objects(getattr(self, field)) 109 | for field in self.lookup_keys 110 | if hasattr(self, field) 111 | } 112 | 113 | def get_object(self): 114 | """ 115 | Get the object from the database. 116 | """ 117 | return self.model.objects.get(**self.lookup()) 118 | 119 | def __to_value__(self): 120 | obj = self.get_object() 121 | lookup = self.lookup() 122 | 123 | # update the object with any remaining keys 124 | for field in self.model._meta.get_fields(): 125 | 126 | # Skip fields used to find the instance 127 | if field.name in lookup: 128 | continue 129 | 130 | if hasattr(self, field.name): 131 | value = getattr(self, field.name) 132 | 133 | if isinstance(field, StreamField): 134 | value = json.dumps(value, cls=JSONEncoder) 135 | else: 136 | value = FieldStorable.to_objects(value) 137 | 138 | setattr(obj, field.name, value) 139 | 140 | return obj 141 | 142 | 143 | # pylint:disable=abstract-method 144 | class GetOrCreateForeignObject(GetForeignObject): 145 | """ 146 | Get or create a foreign key reference for the provided parameters 147 | """ 148 | 149 | def get_defaults(self): 150 | """ 151 | Defaults to pass when creating an object. 152 | """ 153 | defaults = {} 154 | 155 | for field in self.model._meta.get_fields(): 156 | if hasattr(self, field.name): 157 | value = getattr(self, field.name) 158 | 159 | if isinstance(field, StreamField): 160 | value = json.dumps(value, cls=JSONEncoder) 161 | else: 162 | value = FieldStorable.to_objects(value) 163 | 164 | defaults[field.name] = value 165 | 166 | return defaults 167 | 168 | def get_object(self): 169 | obj, _ = self.model.objects.get_or_create(**self.lookup(), 170 | defaults=self.get_defaults()) 171 | return obj 172 | 173 | 174 | class GetOrCreateClusterableForeignObject(GetForeignObject): 175 | """ 176 | Get or create a foreign key reference for the provided parameters, 177 | assuming the parent of this object is a ClusterableModel. 178 | """ 179 | 180 | def get_object(self): 181 | try: 182 | return super().get_object() 183 | except self.model.DoesNotExist: 184 | return self.model(**self.lookup()) 185 | # pylint:enable=abstract-method 186 | 187 | 188 | class Page(FieldStorable, JSONSerializable, yaml.YAMLObject): 189 | """ 190 | A reference to a page 191 | """ 192 | 193 | yaml_tag = '!page' 194 | yaml_loader = yaml.SafeLoader 195 | 196 | def get_object(self): 197 | """ 198 | Retrieve the page object. 199 | """ 200 | # pylint:disable=no-member 201 | url = PurePosixPath(self.url) 202 | 203 | if not url.is_absolute(): 204 | raise ValueError("URL must be absolute") 205 | 206 | return WagtailPage.objects.only('id').get(url_path=normalise(url)) 207 | 208 | def __to_value__(self): 209 | return self.get_object() 210 | 211 | def __to_json__(self): 212 | return self.get_object().id 213 | 214 | 215 | class Image(JSONSerializable, GetOrCreateForeignObject): 216 | """ 217 | A reference to an image 218 | """ 219 | 220 | yaml_tag = '!image' 221 | yaml_loader = yaml.SafeLoader 222 | model = WagtailImage 223 | 224 | file = None # expected parameter 225 | 226 | def lookup(self): 227 | return {'file': self.db_filename} 228 | 229 | @property 230 | def db_filename(self): 231 | """ 232 | Filename that will be stored into the DB. 233 | 234 | This code is taken from Wagtail. We can't call into the Wagtail code 235 | because it will create unique filenames. 236 | """ 237 | folder_name = "original_images" 238 | filename = f"images/{self.file.replace('/', '-')}" 239 | filename = "".join( 240 | (i if ord(i) < 128 else "_") for i in string_to_ascii(filename) 241 | ) 242 | 243 | full_path = os.path.join(folder_name, filename) 244 | if len(full_path) >= 95: 245 | chars_to_trim = len(full_path) - 94 246 | prefix, extension = os.path.splitext(filename) 247 | filename = prefix[:-chars_to_trim] + extension 248 | full_path = os.path.join(folder_name, filename) 249 | 250 | return full_path 251 | 252 | def get_object(self): 253 | try: 254 | return self.model.objects.get(**self.lookup()) 255 | except self.model.DoesNotExist: 256 | LOGGER.info("Creating file %s...", self.db_filename) 257 | 258 | storage = self.model._meta.get_field('file').storage 259 | filename = self.db_filename 260 | if not storage.exists(filename): 261 | with open(f"images/{self.file}", 'rb') as source: 262 | filename = storage.save(filename, source) 263 | 264 | image = self.model(file=filename) 265 | image.save() # pylint:disable=no-member 266 | return image 267 | 268 | def __to_json__(self): 269 | return self.__to_value__().id 270 | 271 | 272 | class Document(JSONSerializable, GetOrCreateForeignObject): 273 | """ 274 | A reference to a document 275 | """ 276 | 277 | yaml_tag = '!document' 278 | yaml_loader = yaml.SafeLoader 279 | model = WagtailDocument 280 | 281 | # expected parameters 282 | file = None 283 | title = '' 284 | 285 | def lookup(self): 286 | return {'file': self.db_filename} 287 | 288 | @property 289 | def db_filename(self): 290 | """Generate a filename to store in the database for this Document.""" 291 | folder_name = 'documents' 292 | # wagtail code doesn't appear to manipulate document filenames like it 293 | # does for images! 294 | path = os.path.join(folder_name, self.file) 295 | return path 296 | 297 | def get_object(self): 298 | try: 299 | return self.model.objects.get(**self.lookup()) 300 | except self.model.DoesNotExist: 301 | LOGGER.info("Creating file %s...", self.db_filename) 302 | 303 | storage = self.model._meta.get_field('file').storage 304 | filename = self.db_filename 305 | if not storage.exists(filename): 306 | with open(f"documents/{self.file}", 'rb') as source: 307 | filename = storage.save(filename, source) 308 | 309 | doc = self.model(file=filename, title=self.title) 310 | doc.save() # pylint:disable=no-member 311 | return doc 312 | 313 | def __to_json__(self): 314 | return self.__to_value__().id 315 | 316 | 317 | class Site(GetOrCreateForeignObject): 318 | """ 319 | A Wagtail site. 320 | 321 | !site 322 | hostname: localhost 323 | site_name: My Site 324 | root_page: !page 325 | url: /my-site 326 | is_default_site: true 327 | """ 328 | 329 | yaml_tag = '!site' 330 | model = WagtailSite 331 | lookup_keys = ('hostname',) 332 | 333 | 334 | # Make a getter for each registered Wagtail setting, lookup using its app.model 335 | # lowercase dotted model name. 336 | for SomeSetting in registry: 337 | type(SomeSetting.__name__, (GetOrCreateForeignObject,), { 338 | 'yaml_tag': f'!{SomeSetting._meta.label_lower}', 339 | 'model': SomeSetting, 340 | 'lookup_keys': ('site',) 341 | }) 342 | --------------------------------------------------------------------------------