├── tests ├── __init__.py ├── app │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ ├── utils.py │ │ ├── factories.py │ │ ├── test_tasks.py │ │ ├── test_permissions.py │ │ ├── test_views.py │ │ └── test_exporter.py │ ├── admin.py │ └── models.py ├── urls.py └── runtests.py ├── exportdb ├── admin │ └── __init__.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── exportdb.py ├── templates │ └── exportdb │ │ ├── confirm.html │ │ ├── in_progress.html │ │ └── base.html ├── models.py ├── __init__.py ├── permissions.py ├── apps.py ├── urls.py ├── compat.py ├── conf.py ├── tasks.py ├── views.py └── exporter.py ├── requirements ├── test.txt └── base.txt ├── MANIFEST.in ├── .coveragerc ├── tox.ini ├── .gitignore ├── .travis.yml ├── setup.py └── README.rst /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /exportdb/admin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/app/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /exportdb/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /exportdb/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /exportdb/management/commands/exportdb.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | factory-boy==2.5.2 2 | coverage==3.7.1 3 | -------------------------------------------------------------------------------- /exportdb/templates/exportdb/confirm.html: -------------------------------------------------------------------------------- 1 | {% extends "exportdb/base.html" %} 2 | -------------------------------------------------------------------------------- /exportdb/models.py: -------------------------------------------------------------------------------- 1 | from . import permissions # noqa 2 | from . import conf # noqa 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | recursive-include exportdb/static * 3 | recursive-include exportdb/templates * 4 | -------------------------------------------------------------------------------- /requirements/base.txt: -------------------------------------------------------------------------------- 1 | Django>=1.6 2 | django-import-export==0.2.7 3 | celery==3.1.17 4 | django-celery==3.0.17 5 | rules==0.4 6 | -------------------------------------------------------------------------------- /tests/app/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from .models import Book, Category, Author 4 | 5 | admin.site.register(Book) 6 | admin.site.register(Category) 7 | admin.site.register(Author) 8 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = exportdb 4 | 5 | [report] 6 | omit = */migrations/*,*/tests/* 7 | exclude_lines = 8 | pragma: no cover, 9 | noqa 10 | 11 | [html] 12 | directory = cover 13 | -------------------------------------------------------------------------------- /exportdb/__init__.py: -------------------------------------------------------------------------------- 1 | VERSION = (0, 4, 6) 2 | 3 | 4 | def get_version(): 5 | return '.'.join([str(bit) for bit in VERSION]) 6 | 7 | 8 | __version__ = get_version() 9 | 10 | default_app_config = 'exportdb.apps.ExportDBConfig' 11 | -------------------------------------------------------------------------------- /exportdb/permissions.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | import rules 4 | 5 | 6 | @rules.predicate 7 | def permission(*args, **kwargs): 8 | return settings.EXPORTDB_PERMISSION(*args, **kwargs) 9 | 10 | 11 | rules.add_rule('exportdb.can_export', permission) 12 | -------------------------------------------------------------------------------- /exportdb/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.utils.translation import ugettext_lazy as _ 3 | 4 | 5 | class ExportDBConfig(AppConfig): 6 | name = 'exportdb' 7 | verbose_name = _('Export db') 8 | 9 | def ready(self): 10 | from . import conf # noqa 11 | -------------------------------------------------------------------------------- /exportdb/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | from .views import ExportView, ExportPendingView 4 | 5 | 6 | urlpatterns = [ 7 | url(r'^$', ExportView.as_view(), name='exportdb_export'), 8 | url(r'^progress/$', ExportPendingView.as_view(), name='exportdb_progress') 9 | ] 10 | -------------------------------------------------------------------------------- /tests/app/tests/utils.py: -------------------------------------------------------------------------------- 1 | from exportdb.exporter import ExportModelResource 2 | 3 | 4 | class UserResource(ExportModelResource): 5 | custom_attr = 1 6 | 7 | def __init__(self, **kwargs): 8 | super(UserResource, self).__init__(**kwargs) 9 | self.foo = kwargs.get('foo') 10 | 11 | 12 | class GroupResource(ExportModelResource): 13 | custom_attr = 1 14 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{27,34,35}-django{1.6,1.7,1.8,1.9,1.10} 3 | 4 | [testenv] 5 | deps = 6 | django16: Django>=1.6,<1.7 7 | django17: Django>=1.7,<1.8 8 | django18: Django>=1.8,<1.9 9 | django19: Django>=1.9,<1.10 10 | django10: Django>=1.10,<1.11 11 | coverage 12 | coveralls 13 | codecov 14 | commands=coverage run --rcfile={toxinidir}/.coveragerc {toxinidir}/setup.py test 15 | -------------------------------------------------------------------------------- /tests/urls.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.conf.urls import include, url 4 | from django.contrib.staticfiles.urls import staticfiles_urlpatterns 5 | 6 | 7 | from django.contrib import admin 8 | admin.autodiscover() 9 | 10 | 11 | urlpatterns = [ 12 | url(r'^admin/exportdb/', include('exportdb.urls')), 13 | url(r'^admin/', include(admin.site.urls)), 14 | ] 15 | 16 | urlpatterns += staticfiles_urlpatterns() 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.eggs 6 | *.egg-info 7 | dist 8 | build 9 | eggs 10 | parts 11 | bin 12 | var 13 | sdist 14 | develop-eggs 15 | .installed.cfg 16 | setup.cfg 17 | test_media 18 | 19 | # Installer logs 20 | pip-log.txt 21 | 22 | # Unit test / coverage reports 23 | .coverage 24 | .tox 25 | /cover 26 | 27 | # Mac stuff and sqlite files 28 | .DS\_Store 29 | *.sqlite 30 | *.db 31 | 32 | # virtualenv & docs 33 | env 34 | /.env 35 | docs/_build 36 | -------------------------------------------------------------------------------- /exportdb/compat.py: -------------------------------------------------------------------------------- 1 | import django 2 | 3 | try: 4 | from django.utils.module_loading import import_string # noqa 5 | except ImportError: # noqa 6 | from django.utils.module_loading import import_by_path as import_string # noqa 7 | 8 | 9 | try: 10 | # django >= 1.7 11 | from django.apps import apps 12 | get_model = apps.get_model 13 | get_models = apps.get_models 14 | except ImportError: # noqa 15 | # django < 1.7 16 | from django.db.models import get_model, get_models 17 | 18 | 19 | def jquery_in_vendor(): 20 | return django.VERSION >= (1, 9, 0) 21 | -------------------------------------------------------------------------------- /exportdb/conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import posixpath 3 | 4 | from django.conf import settings 5 | 6 | import rules 7 | from appconf import AppConf 8 | 9 | 10 | EXPORT_SUBDIR = 'exports' 11 | 12 | 13 | class ExportDBConf(AppConf): 14 | EXPORT_ROOT = os.path.join(settings.MEDIA_ROOT, EXPORT_SUBDIR) 15 | EXPORT_MEDIA_URL = posixpath.join(settings.MEDIA_URL, EXPORT_SUBDIR) 16 | EXPORT_CONF = {} 17 | 18 | # form used in admin view to confirm export 19 | CONFIRM_FORM = 'django.forms.Form' 20 | 21 | # Who can perform the export 22 | PERMISSION = rules.is_superuser 23 | -------------------------------------------------------------------------------- /tests/app/tests/factories.py: -------------------------------------------------------------------------------- 1 | import factory 2 | import factory.fuzzy 3 | 4 | from ..models import Author, Book, Category 5 | 6 | 7 | class AuthorFactory(factory.django.DjangoModelFactory): 8 | name = factory.fuzzy.FuzzyText(length=8) 9 | 10 | class Meta: 11 | model = Author 12 | 13 | 14 | class BookFactory(factory.django.DjangoModelFactory): 15 | name = factory.fuzzy.FuzzyText(length=15) 16 | author = factory.SubFactory(AuthorFactory) 17 | 18 | class Meta: 19 | model = Book 20 | 21 | 22 | class CategoryFactory(factory.django.DjangoModelFactory): 23 | name = factory.fuzzy.FuzzyText(length=15) 24 | 25 | class Meta: 26 | model = Category 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "3.5" 5 | 6 | install: 7 | - pip install coverage codecov coveralls tox 8 | 9 | script: 10 | - tox 11 | 12 | after_success: 13 | - coveralls 14 | - codecov 15 | 16 | env: 17 | - TOXENV=py27-django1.6 18 | - TOXENV=py27-django1.7 19 | - TOXENV=py27-django1.8 20 | - TOXENV=py27-django1.9 21 | - TOXENV=py27-django1.10 22 | 23 | - TOXENV=py34-django1.6 24 | - TOXENV=py34-django1.7 25 | - TOXENV=py34-django1.8 26 | - TOXENV=py34-django1.9 27 | - TOXENV=py34-django1.10 28 | 29 | - TOXENV=py35-django1.6 30 | - TOXENV=py35-django1.7 31 | - TOXENV=py35-django1.8 32 | - TOXENV=py35-django1.9 33 | - TOXENV=py35-django1.10 34 | -------------------------------------------------------------------------------- /tests/app/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class Author(models.Model): 5 | name = models.CharField(max_length=100) 6 | birthday = models.DateTimeField(auto_now_add=True) 7 | 8 | 9 | class Category(models.Model): 10 | name = models.CharField(max_length=100) 11 | 12 | 13 | class Book(models.Model): 14 | name = models.CharField('Book name', max_length=100) 15 | author = models.ForeignKey(Author, blank=True, null=True) 16 | author_email = models.EmailField('Author email', max_length=75, blank=True) 17 | imported = models.BooleanField(default=False) 18 | published = models.DateField('Published', blank=True, null=True) 19 | price = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) 20 | categories = models.ManyToManyField(Category, blank=True) 21 | -------------------------------------------------------------------------------- /tests/app/tests/test_tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import mock 4 | 5 | from django.test import TestCase 6 | 7 | from tablib import Databook, Dataset 8 | 9 | from exportdb.exporter import Exporter, get_export_models 10 | 11 | 12 | class TaskTests(TestCase): 13 | 14 | @mock.patch('exportdb.exporter.Exporter.export') 15 | @mock.patch('exportdb.tasks.get_resource_for_model') 16 | def test_kwargs_forwarded(self, resource_for_model, mocked_export): 17 | """ 18 | Test that kwargs passed to `export` are forwarded to the ModelResource 19 | for export. 20 | """ 21 | from exportdb.tasks import export 22 | 23 | book = Databook() 24 | book.add_sheet(Dataset()) 25 | mocked_export.return_value = book 26 | 27 | kwargs = {'foo': 'bar', 'baz': 10} 28 | export(Exporter, **kwargs) 29 | 30 | export_models = get_export_models() 31 | for i, export_model in enumerate(export_models): 32 | self.assertEqual( 33 | resource_for_model.call_args_list[i], 34 | mock.call(export_model, foo='bar', baz=10) 35 | ) 36 | -------------------------------------------------------------------------------- /tests/app/tests/test_permissions.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import rules 3 | 4 | from django.contrib.auth.models import User 5 | from django.test import TestCase 6 | 7 | try: 8 | from django.test import override_settings 9 | except ImportError: 10 | from django.test.utils import override_settings 11 | 12 | 13 | permission_mock = mock.Mock(return_value=True) 14 | 15 | 16 | class PermissionTests(TestCase): 17 | def setUp(self): 18 | self.super_user = User.objects.create_superuser('test', 'test@test.com', 'letmein') 19 | self.normal_user = User.objects.create_user('normal', 'normal@test.com', 'letmein') 20 | self.staff_user = User.objects.create_user('staff', 'staf@test.com', 'letmein') 21 | self.staff_user.is_staff = True 22 | self.staff_user.save() 23 | 24 | self.client.login(username='test', password='letmein') 25 | 26 | def test_default_permission(self): 27 | self.assertTrue( 28 | rules.test_rule('exportdb.can_export', self.super_user) 29 | ) 30 | 31 | self.assertFalse( 32 | rules.test_rule('exportdb.can_export', self.normal_user) 33 | ) 34 | 35 | self.assertFalse( 36 | rules.test_rule('exportdb.can_export', self.staff_user) 37 | ) 38 | 39 | @override_settings(EXPORTDB_PERMISSION=permission_mock) 40 | def test_permission_testings(self): 41 | with mock.patch('rules.is_staff', return_value=True): 42 | self.assertTrue( 43 | rules.test_rule('exportdb.can_export', self.normal_user) 44 | ) 45 | permission_mock.assert_called_with(self.normal_user) 46 | -------------------------------------------------------------------------------- /exportdb/tasks.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import posixpath 4 | 5 | from django.conf import settings 6 | from django.utils import timezone 7 | 8 | from celery import shared_task, current_task 9 | 10 | from .exporter import get_export_models, get_resource_for_model 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | @shared_task 16 | def export(exporter_class, format='xlsx', **kwargs): 17 | """ 18 | Generates the export. 19 | 20 | Support for django-tenant-schemas is built in. 21 | """ 22 | tenant = kwargs.pop('tenant', None) 23 | if tenant is not None: 24 | logger.debug('Settings tenant to %s' % tenant) 25 | from django.db import connection 26 | connection.set_tenant(tenant) 27 | 28 | export_root = settings.EXPORTDB_EXPORT_ROOT % tenant.schema_name 29 | else: 30 | export_root = settings.EXPORTDB_EXPORT_ROOT 31 | 32 | filename = u'export-{timestamp}.{ext}'.format( 33 | timestamp=timezone.now().strftime('%Y-%m-%d_%H%M%S'), 34 | ext=format 35 | ) 36 | 37 | models = get_export_models() 38 | resources = [get_resource_for_model(model, **kwargs) for model in models] 39 | exporter = exporter_class(resources) 40 | 41 | logger.info('Exporting resources: %s' % resources) 42 | databook = exporter.export(task=current_task) 43 | export_to = os.path.join(export_root, filename) 44 | if not os.path.exists(export_root): 45 | os.makedirs(export_root) 46 | with open(export_to, 'wb') as outfile: 47 | outfile.write(getattr(databook, format)) 48 | return posixpath.join(settings.EXPORTDB_EXPORT_MEDIA_URL, filename) 49 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from setuptools import setup, find_packages 4 | 5 | import exportdb 6 | 7 | 8 | def read_file(name): 9 | with open(os.path.join(os.path.dirname(__file__), name)) as f: 10 | return f.read() 11 | 12 | 13 | readme = read_file('README.rst') 14 | requirements = [ 15 | 'Django', 16 | 'django-appconf', 17 | 'django-import-export', 18 | 'celery', 19 | 'django-celery', 20 | 'rules', 21 | ] 22 | test_requirements = [ 23 | 'mock', 24 | 'factory-boy', 25 | 'coverage' 26 | ] 27 | 28 | setup( 29 | name='django-exportdb', 30 | version=exportdb.get_version(), 31 | license='MIT', 32 | 33 | # Packaging 34 | packages=find_packages(exclude=('tests', 'tests.*')), 35 | include_package_data=True, 36 | install_requires=requirements, 37 | extras_require={ 38 | 'test': test_requirements, 39 | }, 40 | zip_safe=False, 41 | tests_require=test_requirements, 42 | test_suite='tests.runtests.runtests', 43 | 44 | # PyPI metadata 45 | description='Dump the entire database to xlsx workbook with a sheet per model', 46 | long_description='\n\n'.join([readme]), 47 | author='Maykin Media, Sergei Maertens', 48 | author_email='sergei@maykinmedia.nl', 49 | platforms=['any'], 50 | url='https://github.com/maykinmedia/django-exportdb', 51 | classifiers=[ 52 | 'Development Status :: 3 - Alpha', 53 | 'Framework :: Django :: 1.6', 54 | 'Framework :: Django :: 1.7', 55 | 'Framework :: Django :: 1.8', 56 | 'Intended Audience :: Developers', 57 | 'Operating System :: Unix', 58 | 'Operating System :: MacOS', 59 | 'Operating System :: Microsoft :: Windows', 60 | 'Programming Language :: Python :: 2.7', 61 | 'Programming Language :: Python :: 3.4', 62 | 'Topic :: Software Development :: Libraries :: Application Frameworks' 63 | ] 64 | ) 65 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Django Export DB 2 | ================ 3 | 4 | Export the entire database to an Excel workbook with a sheet per model. 5 | 6 | .. image:: https://travis-ci.org/maykinmedia/django-exportdb.svg?branch=master 7 | :target: https://travis-ci.org/maykinmedia/django-exportdb 8 | 9 | .. image:: https://codecov.io/github/maykinmedia/django-exportdb/coverage.svg?branch=master 10 | :target: https://codecov.io/github/maykinmedia/django-exportdb?branch=master 11 | 12 | .. image:: https://coveralls.io/repos/maykinmedia/django-exportdb/badge.svg 13 | :target: https://coveralls.io/r/maykinmedia/django-exportdb 14 | 15 | .. image:: https://img.shields.io/pypi/v/django-exportdb.svg 16 | :target: https://pypi.python.org/pypi/django-exportdb 17 | 18 | Installation 19 | ------------ 20 | 21 | $ pip install django-exportdb 22 | 23 | Add `exportdb` to INSTALLED_APPS, and make sure that django.contrib.admin is in there as well. 24 | 25 | Add 26 | 27 | url(r'^admin/exportdb/', include('exportdb.urls')) 28 | 29 | to your urls.py, make sure that it comes before url(r'^admin/', ...) if you hook 30 | it into the admin. 31 | 32 | Configuration 33 | ------------- 34 | 35 | EXPORTDB_EXPORT_CONF 36 | Configures what models and fields are exported. Example:: 37 | 38 | EXPORT_CONF = { 39 | 'models': { 40 | 'auth.User': { 41 | 'fields': ('username',), 42 | 'resource_class': 'app.tests.utils.UserResource' 43 | }, 44 | 'auth.Group': { 45 | 'resource_class': 'app.tests.utils.GroupResource' 46 | }, 47 | 'auth.Permission': { 48 | 'fields': ('name',) 49 | } 50 | } 51 | } 52 | EXPORTDB_CONFIRM_FORM 53 | Form shown to confirm the export 54 | EXPORTDB_EXPORT_ROOT 55 | The filesystem path where the exports are stored 56 | EXPORTDB_PERMISSION 57 | Who can access the export. By default only superusers have access. 58 | 59 | To allow all `staff` users to use the export add the following to your settings:: 60 | 61 | EXPORTDB_PERMISSION = rules.is_staff 62 | -------------------------------------------------------------------------------- /exportdb/templates/exportdb/in_progress.html: -------------------------------------------------------------------------------- 1 | {% extends "exportdb/base.html" %}{% load i18n admin_static %} 2 | 3 | {% block extrahead %}{{ block.super }} 4 | 17 | {% endblock %} 18 | 19 | {% block content %} 20 |
21 |
22 |

{% trans "Export in progress" %}

23 | 24 |
25 | {% trans "The following object types are being exported" %} 26 |
    27 | {% for model in models %} 28 |
  • {{ model }}
    • 29 | {% endfor %} 30 |
31 |
32 |
33 | 34 |
35 |
0%
36 |
37 | 38 | 39 | 42 |
43 | 44 | 45 | 76 | 77 | {% endblock content %} 78 | -------------------------------------------------------------------------------- /tests/runtests.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | 5 | BASE_DIR = os.path.abspath(os.path.dirname(__file__)) 6 | 7 | 8 | def runtests(): 9 | from django.conf import settings 10 | settings.configure( 11 | INSTALLED_APPS=[ 12 | 'django.contrib.admin', 13 | 'django.contrib.auth', 14 | 'django.contrib.contenttypes', 15 | 'django.contrib.sessions', 16 | 'django.contrib.sites', 17 | 18 | 'import_export', 19 | 'exportdb', 20 | 21 | 'app', 22 | ], 23 | ROOT_URLCONF='urls', 24 | DEBUG=True, 25 | STATIC_URL='/static/', 26 | SECRET_KEY='&t=qu_de!+ih0gq9a+v3bjd^f@ulb7ioy_!o=gi^k12aebt7+i', 27 | 28 | MEDIA_ROOT=os.path.join(BASE_DIR, 'test_media'), 29 | 30 | TEMPLATES = [{ 31 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 32 | 'DIRS': [], 33 | 'APP_DIRS': True, 34 | 'OPTIONS': { 35 | 'context_processors': [ 36 | 'django.template.context_processors.debug', 37 | 'django.template.context_processors.request', 38 | 'django.contrib.auth.context_processors.auth', 39 | 'django.contrib.messages.context_processors.messages', 40 | ], 41 | }, 42 | }], 43 | 44 | MIDDLEWARE_CLASSES=( 45 | 'django.contrib.sessions.middleware.SessionMiddleware', 46 | 'django.middleware.common.CommonMiddleware', 47 | 'django.middleware.csrf.CsrfViewMiddleware', 48 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 49 | 'django.contrib.messages.middleware.MessageMiddleware', 50 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 51 | ), 52 | 53 | DATABASES={ 54 | 'default': { 55 | 'ENGINE': 'django.db.backends.sqlite3', 56 | 'NAME': os.path.join(os.path.dirname(__file__), 'database.db'), 57 | } 58 | } 59 | ) 60 | 61 | test_dir = os.path.dirname(__file__) 62 | sys.path.insert(0, test_dir) 63 | 64 | import django 65 | from django.test.utils import get_runner 66 | 67 | try: 68 | django.setup() # 1.7 and upwards 69 | except AttributeError: 70 | pass 71 | 72 | TestRunner = get_runner(settings) 73 | test_runner = TestRunner(verbosity=1, interactive=True) 74 | failures = test_runner.run_tests(['app']) 75 | sys.exit(failures) 76 | 77 | 78 | if __name__ == '__main__': 79 | runtests() 80 | -------------------------------------------------------------------------------- /tests/app/tests/test_views.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from pkg_resources import parse_version 4 | 5 | import mock 6 | from datetime import date 7 | 8 | import django 9 | from django import forms 10 | from django.contrib.auth.models import User 11 | from django.core.urlresolvers import reverse 12 | from django.test import TestCase 13 | from django.utils.translation import ugettext as _ 14 | 15 | from exportdb.compat import jquery_in_vendor 16 | from exportdb.exporter import Exporter 17 | 18 | 19 | try: 20 | from django.test import override_settings 21 | except ImportError: 22 | from django.test.utils import override_settings 23 | 24 | 25 | class ExportForm(forms.Form): 26 | from_date = forms.DateField(label='from date') 27 | 28 | 29 | class ViewTests(TestCase): 30 | 31 | confirm_url = reverse('exportdb_export') 32 | 33 | def setUp(self): 34 | self.user = User.objects.create_superuser('test', 'test@test.com', 'letmein') 35 | self.client.login(username='test', password='letmein') 36 | 37 | def test_confirm_page(self): 38 | response = self.client.get(self.confirm_url) 39 | self.assertEqual(response.status_code, 200) 40 | self.assertEqual(set(response.context['models']), set([ 41 | _('session (sessions.Session)'), 42 | _('log entry (admin.LogEntry)'), 43 | _('author (app.Author)'), 44 | _('category (app.Category)'), 45 | _('book (app.Book)'), 46 | _('site (sites.Site)'), 47 | _('permission (auth.Permission)'), 48 | _('group (auth.Group)'), 49 | _('user (auth.User)'), 50 | _('content type (contenttypes.ContentType)') 51 | ])) 52 | 53 | if jquery_in_vendor(): 54 | self.assertContains(response, '/static/admin/js/vendor/jquery/jquery.js', count=1) 55 | else: 56 | self.assertContains(response, '/static/admin/js/jquery.js', count=1) 57 | 58 | self.assertIsInstance(response.context['form'], forms.Form) 59 | 60 | @override_settings(EXPORTDB_CONFIRM_FORM='tests.app.tests.test_views.ExportForm') 61 | def test_custom_confirm_form(self): 62 | response = self.client.get(self.confirm_url) 63 | self.assertEqual(response.status_code, 200) 64 | self.assertEqual(response.context['form'].__class__.__name__, 'ExportForm') 65 | self.assertContains(response, 'from date', count=1) 66 | 67 | @override_settings(EXPORTDB_CONFIRM_FORM='tests.app.tests.test_views.ExportForm', CELERY_ALWAYS_EAGER=True) 68 | def test_form_data_passed_to_exporter(self): 69 | """ 70 | Test that form.cleaned_data is passed to the exporter and exporter resources 71 | """ 72 | class AsyncResult(object): 73 | id = 'foo' 74 | 75 | def post_confirm(): 76 | response = self.client.post(self.confirm_url, {'from_date': date.today()}) 77 | self.assertEqual(response.status_code, 200) 78 | return response 79 | 80 | with mock.patch('exportdb.views.export.delay') as mocked_export: 81 | mocked_export.return_value = AsyncResult() 82 | post_confirm() 83 | 84 | self.assertEqual(mocked_export.call_count, 1) 85 | self.assertEqual(mocked_export.call_args, mock.call( 86 | Exporter, tenant=None, from_date=date.today() 87 | )) 88 | -------------------------------------------------------------------------------- /exportdb/templates/exportdb/base.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base_site.html" %}{% load i18n admin_static %} 2 | 3 | {% block extrahead %}{{ block.super }} 4 | {% if jquery_in_vendor %} 5 | 6 | {% else %} 7 | 8 | {% endif %} 9 | 10 | 11 | 12 | {{ form.media }} 13 | {% endblock %} 14 | 15 | 16 | {% block extrastyle %}{{ block.super }}{% endblock %} 17 | 18 | {% block content %} 19 |
20 |
21 |
22 | 23 | {% if form.errors %} 24 |

25 | {% if form.errors|length == 1 %}{% trans "Please correct the error below." %}{% else %}{% trans "Please correct the errors below." %}{% endif %} 26 |

27 | {{ form.non_field_errors }} 28 | {% endif %} 29 | 30 | {% if form.fields %} 31 | 32 |
33 |

{% trans "Extra fields" %}

34 | {% for field in form %} 35 |
36 | 37 | {{ field.errors }} 38 | {% if field.is_checkbox %} 39 | {{ field }}{{ field.label_tag }} 40 | {% else %} 41 | {{ field.label_tag }} 42 | {% if field.is_readonly %} 43 |

{{ field.contents|linebreaksbr }}

44 | {% else %} 45 | {{ field }} 46 | {% endif %} 47 | {% endif %} 48 | {% if field.help_text %} 49 |

{{ field.help_text|safe }}

50 | {% endif %} 51 |
52 |
53 | {% endfor %} 54 | 55 | 56 | {% endif %} 57 | 58 |
59 |

{% trans "Confirm export" %}

60 | 61 |
62 | {% trans "The following object types will be exported" %} 63 |
    64 | {% for model in models %} 65 |
  • {{ model }}
    • 66 | {% endfor %} 67 |
68 |
69 |
70 | 71 |
72 | 73 | 74 |
75 | 76 |
77 | 78 | 79 | {% endblock %} 80 | -------------------------------------------------------------------------------- /exportdb/views.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pkg_resources import parse_version 3 | 4 | import django 5 | from django.conf import settings 6 | from django.contrib import messages 7 | from django.contrib.auth.decorators import login_required 8 | from django.core.exceptions import PermissionDenied, ImproperlyConfigured 9 | from django.db import connection 10 | from django.http import HttpResponse 11 | from django.utils.decorators import method_decorator 12 | from django.utils.translation import ugettext_lazy as _ 13 | from django.views.generic import FormView, View 14 | 15 | import rules 16 | from celery.result import AsyncResult 17 | 18 | from .compat import import_string, jquery_in_vendor 19 | from .exporter import get_export_models, Exporter 20 | from .tasks import export 21 | 22 | 23 | EXPORTDB_EXPORT_KEY = 'exportdb_export' 24 | 25 | 26 | class ExportPermissionMixin(object): 27 | """ 28 | Check permissions 29 | """ 30 | @method_decorator(login_required) 31 | def dispatch(self, request, *args, **kwargs): 32 | if not rules.test_rule('exportdb.can_export', request.user): 33 | raise PermissionDenied 34 | return super(ExportPermissionMixin, self).dispatch(request, *args, **kwargs) 35 | 36 | 37 | class ExportView(ExportPermissionMixin, FormView): 38 | form_class = None 39 | template_name = 'exportdb/confirm.html' 40 | exporter_class = Exporter 41 | 42 | def get_form_class(self): 43 | """ 44 | Return the form class to use for the confirmation form. 45 | 46 | In the method instead of the attribute, since it's not guaranteed that 47 | the appconf is loaded in Django versions < 1.7. 48 | """ 49 | if self.form_class is None: 50 | self.form_class = import_string(settings.EXPORTDB_CONFIRM_FORM) 51 | return self.form_class 52 | 53 | def get_export_models(self, **kwargs): 54 | kwargs.setdefault('admin_only', False) 55 | try: 56 | return get_export_models(**kwargs) 57 | except ImproperlyConfigured as e: 58 | messages.error(self.request, e.args[0]) 59 | return [] 60 | 61 | def get_exporter_class(self): 62 | return self.exporter_class 63 | 64 | def get_context_data(self, **kwargs): 65 | context = super(ExportView, self).get_context_data(**kwargs) 66 | context['title'] = _('Export database') 67 | context['jquery_in_vendor'] = jquery_in_vendor() 68 | context['models'] = [ 69 | u'{name} ({app}.{model})'.format( 70 | name=model._meta.verbose_name, 71 | app=model._meta.app_label, 72 | model=model.__name__ 73 | ) 74 | for model in self.get_export_models() 75 | ] 76 | return context 77 | 78 | def form_valid(self, form): 79 | # multi-tenant support 80 | tenant = getattr(connection, 'tenant', None) 81 | # start actual export and render the template 82 | async_result = export.delay(self.get_exporter_class(), tenant=tenant, **form.cleaned_data) 83 | self.request.session[EXPORTDB_EXPORT_KEY] = async_result.id 84 | context = self.get_context_data(export_running=True) 85 | self.template_name = 'exportdb/in_progress.html' 86 | return self.render_to_response(context) 87 | 88 | 89 | class ExportPendingView(ExportPermissionMixin, View): 90 | 91 | def json_response(self, data): 92 | return HttpResponse(json.dumps(data), content_type='application/json') 93 | 94 | def get(self, request, *args, **kwargs): 95 | async_result = AsyncResult(request.session.get(EXPORTDB_EXPORT_KEY)) 96 | if not async_result: 97 | return self.json_response({'status': 'FAILURE', 'progress': 0}) 98 | 99 | if async_result.state == 'PROGRESS': 100 | try: 101 | progress = async_result.info['progress'] 102 | if progress > 0.99: 103 | progress = 0.99 104 | except: 105 | progress = 0.01 106 | elif async_result.ready(): 107 | progress = 1 108 | else: 109 | progress = 1 110 | 111 | content = { 112 | 'status': async_result.state, 113 | 'progress': progress, 114 | 'file': async_result.result if async_result.ready() else None 115 | } 116 | return self.json_response(content) 117 | -------------------------------------------------------------------------------- /tests/app/tests/test_exporter.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.contrib.auth.models import User, Group, Permission 4 | from django.test import TestCase 5 | from django.utils.translation import ugettext_lazy as _ 6 | 7 | from import_export.resources import ModelResource 8 | 9 | from exportdb.exporter import get_export_models, get_resource_for_model, Exporter, ExportModelResource 10 | 11 | from ..models import Author, Book, Category 12 | from .factories import AuthorFactory, BookFactory, CategoryFactory 13 | from .utils import UserResource, GroupResource 14 | 15 | try: 16 | from django.test import override_settings 17 | except ImportError: 18 | from django.test.utils import override_settings 19 | 20 | 21 | class ExporterTests(TestCase): 22 | 23 | def test_get_export_models(self): 24 | """ 25 | Test that `get_export_models` returns the expected list of installed models. 26 | """ 27 | from django.contrib.auth.models import User, Group, Permission 28 | from django.contrib.sites.models import Site 29 | from django.contrib.sessions.models import Session 30 | from django.contrib.admin.models import LogEntry 31 | from django.contrib.contenttypes.models import ContentType 32 | 33 | export_models = set(get_export_models(admin_only=False)) 34 | expected_export_models = set([ 35 | User, Group, Permission, Site, Session, LogEntry, ContentType, 36 | Author, Book, Category 37 | ]) 38 | 39 | self.assertEqual(export_models, expected_export_models) 40 | 41 | export_models_admin = set(get_export_models(admin_only=True)) 42 | expected_export_models_admin = set([ 43 | User, Group, Site, 44 | Author, Book, Category 45 | ]) 46 | 47 | self.assertEqual(export_models_admin, expected_export_models_admin) 48 | 49 | def test_get_resource_for_model(self): 50 | resource = get_resource_for_model(Author) 51 | self.assertIsInstance(resource, ModelResource) 52 | self.assertEqual(resource.Meta.model, Author) 53 | 54 | def test_exporter(self): 55 | authors = AuthorFactory.create_batch(3) 56 | categories = CategoryFactory.create_batch(3) 57 | 58 | books = BookFactory.create_batch(2, author=authors[0]) 59 | books += BookFactory.create_batch(3, author=authors[1]) 60 | books += BookFactory.create_batch(4, author=authors[2]) 61 | 62 | books[0].categories = categories[:2] # 2 categories 63 | books[-1].categories = categories[2:] # 1 category 64 | 65 | resources = [get_resource_for_model(model) for model in [Author, Book, Category]] 66 | exporter = Exporter(resources) 67 | book = exporter.export() 68 | 69 | sheets = book.sheets() 70 | self.assertEqual(len(sheets), 3) 71 | titles = set([sheet.title for sheet in sheets]) 72 | expected_titles = set(['authors (app.Author)', 'books (app.Book)', 'categorys (app.Category)']) 73 | self.assertEqual(titles, expected_titles) 74 | 75 | # test that all the data is there as expected 76 | author_sheet, book_sheet, category_sheet = sheets 77 | 78 | self.assertEqual(len(author_sheet), 3) # 3 authors 79 | self.assertEqual(len(author_sheet[0]), 3) # id, name, birthday 80 | 81 | self.assertEqual(len(book_sheet), 9) # 3 authors 82 | # id, name, author_id, author_email, imported, published, price, categories 83 | self.assertEqual(len(book_sheet[0]), 8) 84 | self.assertEqual(book_sheet[0][2], authors[0].id) 85 | self.assertEqual(book_sheet[0][-1], ','.join([str(cat.id) for cat in categories[:2]])) 86 | 87 | self.assertEqual(len(category_sheet), 3) 88 | self.assertEqual(len(category_sheet[0]), 2) # id, name 89 | 90 | 91 | EXPORT_CONF = { 92 | 'models': { 93 | 'auth.User': { 94 | 'fields': ('username',), 95 | 'resource_class': 'app.tests.utils.UserResource', 96 | 'title': _('Custom title'), 97 | }, 98 | 'auth.Group': { 99 | 'resource_class': 'app.tests.utils.GroupResource' 100 | }, 101 | 'auth.Permission': { 102 | 'fields': ('name',) 103 | } 104 | } 105 | } 106 | 107 | 108 | EXPORT_CONF_AUTHORS = { 109 | 'models': { 110 | 'app.Author': { 111 | 'title': _('Author'), 112 | } 113 | } 114 | } 115 | 116 | 117 | @override_settings(EXPORTDB_EXPORT_CONF=EXPORT_CONF) 118 | class ExportResourcesTests(TestCase): 119 | 120 | def test_use_custom_resources(self): 121 | user_resource = get_resource_for_model(User) 122 | self.assertTrue(issubclass(user_resource.__class__, UserResource)) 123 | 124 | group_resource = get_resource_for_model(Group) 125 | self.assertTrue(issubclass(group_resource.__class__, GroupResource)) 126 | 127 | permission_resource = get_resource_for_model(Permission) 128 | self.assertTrue(issubclass(permission_resource.__class__, ExportModelResource)) 129 | 130 | def test_pass_kwargs(self): 131 | kwargs = {'foo': 'bar'} 132 | user_resource = get_resource_for_model(User, **kwargs) 133 | self.assertEqual(user_resource.foo, 'bar') 134 | 135 | def test_custom_dataset_title(self): 136 | user_resource = get_resource_for_model(User) 137 | self.assertEqual(user_resource.title, 'Custom title') 138 | 139 | group_resource = get_resource_for_model(Group) 140 | self.assertIsNone(group_resource.title) 141 | 142 | @override_settings(EXPORTDB_EXPORT_CONF=EXPORT_CONF_AUTHORS) 143 | def test_exporter_custom_title(self): 144 | AuthorFactory.create_batch(3) 145 | 146 | exporter = Exporter([get_resource_for_model(Author)]) 147 | book = exporter.export() 148 | 149 | sheets = book.sheets() 150 | self.assertEqual(len(sheets), 1) 151 | 152 | sheet = sheets[0] 153 | self.assertEqual(sheet.title, 'Author') 154 | -------------------------------------------------------------------------------- /exportdb/exporter.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.conf import settings 4 | from django.contrib import admin 5 | from django.db import connection 6 | from django.db.models.query import QuerySet 7 | from django.core.exceptions import ImproperlyConfigured 8 | from django.utils.encoding import force_text 9 | 10 | from import_export import resources, fields 11 | from tablib import Databook, Dataset 12 | 13 | from .compat import get_model, get_models, import_string 14 | 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class ExportModelResource(resources.ModelResource): 20 | """ 21 | Resource class that ties in better with Celery and progress reporting. 22 | """ 23 | 24 | num_done = 0 25 | title = None 26 | 27 | def __init__(self, **kwargs): 28 | if 'title' in kwargs: 29 | self.title = kwargs.pop('title') 30 | 31 | self.kwargs = kwargs # by default, silently accept all kwargs 32 | 33 | def export(self, queryset=None, task_meta=None): 34 | if queryset is None: 35 | queryset = self.get_queryset() 36 | headers = self.get_export_headers() 37 | data = Dataset(headers=headers) 38 | 39 | if isinstance(queryset, QuerySet): 40 | # Iterate without the queryset cache, to avoid wasting memory when 41 | # exporting large datasets. 42 | iterable = queryset.iterator() 43 | else: 44 | iterable = queryset 45 | 46 | if task_meta is not None: # initialize the total amount accross multiple resources 47 | self.num_done = task_meta['done'] 48 | 49 | for obj in iterable: 50 | data.append(self.export_resource(obj)) 51 | 52 | if task_meta is not None: 53 | self._update_task_state(task_meta) 54 | 55 | logger.debug('Num done: %d' % self.num_done) 56 | 57 | return data 58 | 59 | def _update_task_state(self, task_meta): 60 | total = task_meta['total'] 61 | self.num_done += 1 62 | progress = float(self.num_done) / total 63 | task_meta['task'].update_state( 64 | state='PROGRESS', 65 | meta={'progress': progress, 'model': self.__class__.__name__} 66 | ) 67 | 68 | 69 | def modelresource_factory(model, resource_class=ExportModelResource, **meta_kwargs): 70 | attrs = {'model': model} 71 | 72 | field_names = [] 73 | field_labels = {} 74 | if 'fields' in meta_kwargs: 75 | for field in meta_kwargs['fields']: 76 | if isinstance(field, tuple): 77 | field_names.append(field[0]) 78 | field_labels[field[0]] = field[1] 79 | else: 80 | field_names.append(field) 81 | attrs['fields'] = field_names 82 | attrs['export_order'] = field_names 83 | del meta_kwargs['fields'] 84 | 85 | attrs.update(**meta_kwargs) 86 | 87 | Meta = type(str('Meta'), (object,), attrs) 88 | class_name = model.__name__ + str('Resource') 89 | class_attrs = {'Meta': Meta} 90 | 91 | all_fields = [f.name for f in model._meta.get_fields()] 92 | for field in field_names: 93 | # it's a real field, the meta class deals with this 94 | if field in all_fields and field not in field_labels: 95 | continue 96 | # explicitly add the field 97 | label = field_labels.get(field, field) 98 | class_attrs[field] = fields.Field(attribute=field, column_name=label, readonly=True) 99 | 100 | metaclass = resources.ModelDeclarativeMetaclass 101 | return metaclass(class_name, (resource_class,), class_attrs) 102 | 103 | 104 | def get_export_models(admin_only=False): 105 | """ 106 | Gets a list of models that can be exported. 107 | """ 108 | export_conf = settings.EXPORTDB_EXPORT_CONF.get('models') 109 | if export_conf is None: 110 | if admin_only: 111 | if admin.site._registry == {}: 112 | admin.autodiscover() 113 | return admin.site._registry.keys() 114 | return get_models() 115 | else: 116 | models = [] 117 | not_installed = [] 118 | for model, _ in export_conf.items(): 119 | app_label, model_class_name = model.split('.') 120 | model_class = get_model(app_label, model_class_name) 121 | if model_class is not None: 122 | models.append(model_class) 123 | else: 124 | not_installed.append(model) 125 | 126 | if not_installed: 127 | raise ImproperlyConfigured( 128 | 'The following models can\'t be exported because they haven\'t ' 129 | 'been installed: %s' % u', '.join(not_installed) 130 | ) 131 | return models 132 | 133 | 134 | def get_resource_for_model(model, **kwargs): 135 | """ 136 | Finds or generates the resource to use for :param:`model`. 137 | """ 138 | # TODO: settings to map model to resource 139 | 140 | model_name = u'{app_label}.{name}'.format( 141 | app_label=model._meta.app_label, 142 | name=model.__name__ 143 | ) 144 | resource_class = ExportModelResource 145 | export_conf = settings.EXPORTDB_EXPORT_CONF.get('models') 146 | if export_conf is not None: 147 | model_conf = export_conf.get(model_name) 148 | if model_conf is not None: 149 | 150 | # support custom resource titles 151 | kwargs['title'] = model_conf.get('title') 152 | 153 | # support custom resource classes 154 | if 'resource_class' in model_conf: 155 | resource_class = import_string(model_conf['resource_class']) 156 | 157 | # specify fields to be exported 158 | fields = model_conf.get('fields') 159 | if fields is not None: 160 | # use own factory 161 | return modelresource_factory(model, resource_class=resource_class, fields=fields)(**kwargs) 162 | return resources.modelresource_factory(model, resource_class=resource_class)(**kwargs) 163 | 164 | 165 | class Exporter(object): 166 | 167 | def __init__(self, resources): 168 | self.resources = resources 169 | 170 | def export(self, task=None): 171 | """ 172 | Export the resources to a file. 173 | 174 | :param task: optional celery task. If given, the task state will be 175 | updated. 176 | """ 177 | book = Databook() 178 | 179 | export_kwargs = {} 180 | if task is not None: 181 | total = sum([resource.get_queryset().count() for resource in self.resources]) 182 | export_kwargs['task_meta'] = {'task': task, 'total': total, 'done': 0} 183 | 184 | num_queries_start = len(connection.queries) 185 | 186 | for resource in self.resources: 187 | model = resource.Meta.model 188 | logger.debug('Export kwargs: %s' % export_kwargs) 189 | dataset = resource.export(**export_kwargs) # takes optional queryset argument (select related) 190 | 191 | len_queries = len(connection.queries) 192 | queries = len_queries - num_queries_start 193 | logger.info('Number of objects: %d' % resource.get_queryset().count()) 194 | logger.info('Executed %d queries' % queries) 195 | num_queries_start = len_queries 196 | 197 | if task is not None: 198 | export_kwargs['task_meta']['done'] += dataset.height 199 | 200 | if resource.title is not None: 201 | dataset.title = force_text(resource.title)[:31] # maximum of 31 chars int title 202 | else: 203 | dataset.title = u'{name} ({app}.{model})'.format( 204 | name=model._meta.verbose_name_plural, 205 | app=model._meta.app_label, 206 | model=model.__name__ 207 | )[:31] # maximum of 31 chars int title 208 | 209 | book.add_sheet(dataset) 210 | return book 211 | --------------------------------------------------------------------------------