├── .gitignore ├── .gitlab-ci.yml ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── docker-compose.yml ├── docs ├── do_import.png ├── match_columns.png └── start_import.png ├── manage.py ├── setup.py ├── simple_import ├── __init__.py ├── admin.py ├── forms.py ├── migrations │ ├── 0001_initial.py │ └── __init__.py ├── models.py ├── odsreader.py ├── static │ ├── test_import.csv │ ├── test_import.ods │ └── test_import.xls ├── templates │ └── simple_import │ │ ├── base.html │ │ ├── do_import.html │ │ ├── import.html │ │ ├── match_columns.html │ │ └── match_relations.html ├── tests.py ├── urls.py ├── utils.py └── views.py ├── simple_import_demo ├── __init__.py ├── settings.py ├── urls.py └── wsgi.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | settings_local.py 3 | coverage.xml 4 | django_setuptest-*.egg/ 5 | django_simple_import.egg-info/ 6 | *.egg/ 7 | pep8.txt 8 | import_file/ 9 | .idea 10 | error_file 11 | simple_import/static 12 | build 13 | dist 14 | db.sqlite3 15 | .tox 16 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: python:3.7-slim 2 | 3 | test: 4 | script: 5 | - pip install tox 6 | - tox 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | ENV PYTHONUNBUFFERED 1 3 | 4 | RUN mkdir -p /usr/src/app 5 | WORKDIR /usr/src/app 6 | 7 | COPY setup.py /usr/src/app/ 8 | RUN pip install -e .[ods,xlsx,xls] 9 | 10 | COPY . /usr/src/app 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Burke Software and Consulting LLC 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions 6 | are met: 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 3. The name of the author may not be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 | OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 | IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 | NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 | THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include simple_import/templates * 2 | recursive-exclude simple_import/static * 3 | recursive-exclude simple_import_demo * 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | django-simple-import 2 | ==================== 3 | 4 | An easy to use import tool for Django. django-simple-import aims to keep track of logs and user preferences in the database. 5 | 6 | Project is now stable and in maintenance only mode. If you'd like to add features please fork or take over ownership. 7 | 8 | ![](/docs/start_import.png) 9 | ![](/docs/match_columns.png) 10 | ![](/docs/do_import.png) 11 | 12 | # Changelog 13 | 14 | ## 2.1 15 | 16 | - Add support for Django 2.2 and 3.0 17 | - Move to gitlab and gitlab CI for freedom and consistency with other Burke Software maintained projects 18 | 19 | ## 2.0 20 | 21 | 2.0 adds support for Django 1.9 and 1.10. Support for 1.8 and under is dropped. Support for Python 2 is dropped. 22 | Removed support for django-custom-field 23 | Use 1.x for older environments. 24 | 25 | ## 1.17 26 | 27 | The most apparent changes are 1.7 compatibility and migration to Django's 28 | atomic transactions. Please report any issues. I test against mysql innodb, postgres, and sqlite. 29 | 30 | ## Features 31 | - Supports csv, xls, xlsx, and ods import file 32 | - Save user matches of column headers to fields 33 | - Guess matches 34 | - Create, update, or both 35 | - Allow programmers to define special import methods for custom handling 36 | - Set related objects by any unique field 37 | - Simulate imports before commiting to database 38 | - Undo (create only) imports 39 | - Security checks if user has correct permissions (partially implemented) 40 | 41 | ## Install 42 | 43 | 1. `pip install django-simple-import[ods,xls,xlsx]` for full install or specify which formats you need to support. CSV is supported out of box. 44 | 2. Add 'simple_import' to INSTALLED APPS 45 | 3. Add simple_import to urls.py like 46 | `url(r'^simple_import/', include('simple_import.urls')),` 47 | 4. migrate 48 | 49 | ## Optional Settings 50 | Define allowed methods to be "imported". Example: 51 | 52 | class Foo(models.Model): 53 | ... 54 | def set_bar(self, value): 55 | self.bar = value 56 | simple_import_methods = ('set_bar',) 57 | 58 | ### settings.py 59 | SIMPLE_IMPORT_LAZY_CHOICES: Default True. If enabled simple_import will look up choices when importing. Example: 60 | 61 | choices = ['M', 'Monday'] 62 | 63 | If the spreadsheet value is "Monday" it will set the database value to "M." 64 | 65 | SIMPLE_IMPORT_LAZY_CHOICES_STRIP: Default False. If enabled, simple_import will trip leading/trailing whitespace 66 | from the cell's value before checking for a match. Only relevant when SIMPLE_IMPORT_LAZY_CHOICES is also enabled. 67 | 68 | If you need any help, we do consulting and custom development. Just email us at david at burkesoftware.com. 69 | 70 | 71 | ## Usage 72 | 73 | Go to `/simple_import/start_import/` or use the admin interface. 74 | 75 | The screenshots have a django-grappelli like theme. The base templates have no style and are very basic. 76 | See an example of customization [here](https://github.com/burke-software/django-sis/tree/master/templates/simple_import). 77 | It is often sufficient to simply override `simple_import/templates/base.html`. 78 | 79 | There is also a log of import records. Check out `/admin/simple_import/`. 80 | 81 | ## Odd Things 82 | 83 | Added a special set password property on auth.User to set password. This sets the password instead of just 84 | saving a hash. 85 | 86 | User has some required fields that...aren't really required. Hardcoded to let them pass. 87 | 88 | ### Security 89 | 90 | I'm working on the assumption staff users are trusted. Only users with change permission 91 | to a field will see it as an option. I have not spent much time looking for ways users could 92 | manipulate URLs to run unauthorized imports. Feel free to contribute changes. 93 | All import views do require admin "is staff" permission. 94 | 95 | ## Testing 96 | 97 | If you have [docker-compose](https://docs.docker.com/compose/) and [Docker](https://www.docker.com/) 98 | installed, then just running `docker-compose run --rm app ./manage.py test` will do everything you need to test 99 | the packages. 100 | 101 | Otherwise look at the `.travis.yml` file for test dependencies. 102 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | app: 2 | build: . 3 | volumes: 4 | - .:/usr/src/app 5 | command: ./manage.py runserver 0.0.0.0:8000 6 | ports: 7 | - "8000:8000" -------------------------------------------------------------------------------- /docs/do_import.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/burke-software/django-simple-import/0b3d33c3a6949de22cf0141828bcbd44d0e2d9cb/docs/do_import.png -------------------------------------------------------------------------------- /docs/match_columns.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/burke-software/django-simple-import/0b3d33c3a6949de22cf0141828bcbd44d0e2d9cb/docs/match_columns.png -------------------------------------------------------------------------------- /docs/start_import.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/burke-software/django-simple-import/0b3d33c3a6949de22cf0141828bcbd44d0e2d9cb/docs/start_import.png -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "simple_import_demo.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name = "django-simple-import", 5 | version = "2.1.0", 6 | author = "David Burke", 7 | author_email = "david@burkesoftware.com", 8 | description = ("A Django import tool easy enough your users could use it"), 9 | license = "BSD", 10 | keywords = "django import", 11 | url = "https://gitlab.com/burke-software/django-simple-import", 12 | packages=find_packages(exclude=("simple_import/static","simple_import_demo")), 13 | include_package_data=True, 14 | classifiers=[ 15 | "Development Status :: 5 - Production/Stable", 16 | 'Environment :: Web Environment', 17 | 'Framework :: Django', 18 | 'Programming Language :: Python', 19 | 'Programming Language :: Python :: 3', 20 | 'Intended Audience :: Developers', 21 | 'Intended Audience :: System Administrators', 22 | "License :: OSI Approved :: BSD License", 23 | ], 24 | install_requires=['django'], 25 | extras_require = { 26 | 'xlsx': ["openpyxl"], 27 | 'ods': ["odfpy"], 28 | 'xls': ["xlrd"], 29 | }, 30 | ) 31 | -------------------------------------------------------------------------------- /simple_import/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/burke-software/django-simple-import/0b3d33c3a6949de22cf0141828bcbd44d0e2d9cb/simple_import/__init__.py -------------------------------------------------------------------------------- /simple_import/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from simple_import.models import ImportLog, ColumnMatch 3 | 4 | class ImportLogAdmin(admin.ModelAdmin): 5 | list_display = ('name', 'user', 'date',) 6 | def has_add_permission(self, request): 7 | return False 8 | admin.site.register(ImportLog, ImportLogAdmin) 9 | -------------------------------------------------------------------------------- /simple_import/forms.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.contrib.contenttypes.models import ContentType 3 | 4 | from simple_import.models import ImportLog, ColumnMatch, RelationalMatch 5 | 6 | 7 | class ImportForm(forms.ModelForm): 8 | class Meta: 9 | model = ImportLog 10 | fields = ('name', 'import_file', 'import_type') 11 | model = forms.ModelChoiceField(ContentType.objects.all()) 12 | 13 | 14 | class MatchForm(forms.ModelForm): 15 | class Meta: 16 | model = ColumnMatch 17 | exclude = ['header_position'] 18 | 19 | 20 | class MatchRelationForm(forms.ModelForm): 21 | class Meta: 22 | model = RelationalMatch 23 | widgets = { 24 | 'related_field_name': forms.Select(choices=(('', '---------'),)) 25 | } 26 | fields = "__all__" 27 | -------------------------------------------------------------------------------- /simple_import/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | from django.conf import settings 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('contenttypes', '0001_initial'), 12 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='ColumnMatch', 18 | fields=[ 19 | ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)), 20 | ('column_name', models.CharField(max_length=200)), 21 | ('field_name', models.CharField(blank=True, max_length=255)), 22 | ('default_value', models.CharField(blank=True, max_length=2000)), 23 | ('null_on_empty', models.BooleanField(default=False, help_text='If cell is blank, clear out the field setting it to blank.')), 24 | ('header_position', models.IntegerField(help_text='Annoying way to order the columns to match the header rows')), 25 | ], 26 | ), 27 | migrations.CreateModel( 28 | name='ImportedObject', 29 | fields=[ 30 | ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)), 31 | ('object_id', models.IntegerField()), 32 | ('content_type', models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE)), 33 | ], 34 | ), 35 | migrations.CreateModel( 36 | name='ImportLog', 37 | fields=[ 38 | ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)), 39 | ('name', models.CharField(max_length=255)), 40 | ('date', models.DateTimeField(auto_now_add=True, verbose_name='Date Created')), 41 | ('import_file', models.FileField(upload_to='import_file')), 42 | ('error_file', models.FileField(blank=True, upload_to='error_file')), 43 | ('import_type', models.CharField(choices=[('N', 'Create New Records'), ('U', 'Create and Update Records'), ('O', 'Only Update Records')], max_length=1)), 44 | ('update_key', models.CharField(blank=True, max_length=200)), 45 | ], 46 | ), 47 | migrations.CreateModel( 48 | name='ImportSetting', 49 | fields=[ 50 | ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)), 51 | ('content_type', models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE)), 52 | ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), 53 | ], 54 | ), 55 | migrations.CreateModel( 56 | name='RelationalMatch', 57 | fields=[ 58 | ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)), 59 | ('field_name', models.CharField(max_length=255)), 60 | ('related_field_name', models.CharField(blank=True, max_length=255)), 61 | ('import_log', models.ForeignKey(to='simple_import.ImportLog', on_delete=models.CASCADE)), 62 | ], 63 | ), 64 | migrations.AddField( 65 | model_name='importlog', 66 | name='import_setting', 67 | field=models.ForeignKey(to='simple_import.ImportSetting', editable=False, on_delete=models.CASCADE), 68 | ), 69 | migrations.AddField( 70 | model_name='importlog', 71 | name='user', 72 | field=models.ForeignKey(to=settings.AUTH_USER_MODEL, related_name='simple_import_log', editable=False, on_delete=models.CASCADE), 73 | ), 74 | migrations.AddField( 75 | model_name='importedobject', 76 | name='import_log', 77 | field=models.ForeignKey(to='simple_import.ImportLog', on_delete=models.CASCADE), 78 | ), 79 | migrations.AddField( 80 | model_name='columnmatch', 81 | name='import_setting', 82 | field=models.ForeignKey(to='simple_import.ImportSetting', on_delete=models.CASCADE), 83 | ), 84 | migrations.AlterUniqueTogether( 85 | name='importsetting', 86 | unique_together={('user', 'content_type')}, 87 | ), 88 | migrations.AlterUniqueTogether( 89 | name='columnmatch', 90 | unique_together={('column_name', 'import_setting')}, 91 | ), 92 | ] 93 | -------------------------------------------------------------------------------- /simple_import/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/burke-software/django-simple-import/0b3d33c3a6949de22cf0141828bcbd44d0e2d9cb/simple_import/migrations/__init__.py -------------------------------------------------------------------------------- /simple_import/models.py: -------------------------------------------------------------------------------- 1 | from django.contrib.contenttypes.models import ContentType 2 | from django.conf import settings 3 | from django.contrib.contenttypes.fields import GenericForeignKey 4 | from django.core.exceptions import ValidationError 5 | from django.db import models, transaction 6 | from django.utils.encoding import smart_text 7 | from .utils import get_all_field_names 8 | import csv 9 | import datetime 10 | from io import StringIO 11 | AUTH_USER_MODEL = settings.AUTH_USER_MODEL 12 | 13 | 14 | class ImportSetting(models.Model): 15 | """ Save some settings per user per content type """ 16 | user = models.ForeignKey(AUTH_USER_MODEL, on_delete=models.CASCADE) 17 | content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) 18 | 19 | class Meta(): 20 | unique_together = ('user', 'content_type',) 21 | 22 | 23 | class ColumnMatch(models.Model): 24 | """ Match column names from the user uploaded file to the database """ 25 | column_name = models.CharField(max_length=200) 26 | field_name = models.CharField(max_length=255, blank=True) 27 | import_setting = models.ForeignKey(ImportSetting, on_delete=models.CASCADE) 28 | default_value = models.CharField(max_length=2000, blank=True) 29 | null_on_empty = models.BooleanField( 30 | default=False, 31 | help_text="If cell is blank, clear out the field setting it to blank.") 32 | header_position = models.IntegerField( 33 | help_text="Annoying way to order the columns to match the header rows") 34 | 35 | class Meta: 36 | unique_together = ('column_name', 'import_setting') 37 | 38 | def __str__(self): 39 | return str('{0} {1}'.format(self.column_name, self.field_name)) 40 | 41 | def guess_field(self): 42 | """ Guess the match based on field names 43 | First look for an exact field name match 44 | then search defined alternative names 45 | then normalize the field name and check for match 46 | """ 47 | model = self.import_setting.content_type.model_class() 48 | field_names = get_all_field_names(model) 49 | if self.column_name in field_names: 50 | self.field_name = self.column_name 51 | return 52 | #TODO user defined alt names 53 | normalized_field_name = self.column_name.lower().replace(' ', '_') 54 | if normalized_field_name in field_names: 55 | self.field_name = normalized_field_name 56 | return 57 | # Try verbose name 58 | for field_name in field_names: 59 | field = model._meta.get_field(field_name) 60 | if hasattr(field, 'verbose_name'): 61 | if (field.verbose_name.lower().replace(' ', '_') == normalized_field_name): 62 | self.field_name = field_name 63 | 64 | 65 | class ImportLog(models.Model): 66 | """ A log of all import attempts """ 67 | name = models.CharField(max_length=255) 68 | user = models.ForeignKey( 69 | AUTH_USER_MODEL, editable=False, related_name="simple_import_log", on_delete=models.CASCADE) 70 | date = models.DateTimeField(auto_now_add=True, verbose_name="Date Created") 71 | import_file = models.FileField(upload_to="import_file") 72 | error_file = models.FileField(upload_to="error_file", blank=True) 73 | import_setting = models.ForeignKey(ImportSetting, editable=False, on_delete=models.CASCADE) 74 | import_type_choices = ( 75 | ("N", "Create New Records"), 76 | ("U", "Create and Update Records"), 77 | ("O", "Only Update Records"), 78 | ) 79 | import_type = models.CharField(max_length=1, choices=import_type_choices) 80 | update_key = models.CharField(max_length=200, blank=True) 81 | 82 | def __str__(self): 83 | return str(self.name) 84 | 85 | def clean(self): 86 | filename = str(self.import_file).lower() 87 | if not filename[-3:] in ('xls', 'ods', 'csv', 'lsx'): 88 | raise ValidationError( 89 | 'Invalid file type. Must be xls, xlsx, ods, or csv.') 90 | 91 | @transaction.atomic 92 | def undo(self): 93 | if self.import_type != "N": 94 | raise Exception("Cannot undo this type of import!") 95 | for obj in self.importedobject_set.all(): 96 | if obj.content_object: 97 | obj.content_object.delete() 98 | obj.delete() 99 | 100 | @staticmethod 101 | def is_empty(value): 102 | """ Check `value` for emptiness by first comparing with None and then 103 | by coercing to string, trimming, and testing for zero length """ 104 | return value is None or not len(smart_text(value).strip()) 105 | 106 | def get_matches(self): 107 | """ Get each matching header row to database match 108 | Returns a ColumnMatch queryset""" 109 | header_row = self.get_import_file_as_list(only_header=True) 110 | match_ids = [] 111 | 112 | for i, cell in enumerate(header_row): 113 | # Sometimes we get blank headers, ignore them. 114 | if self.is_empty(cell): 115 | continue 116 | 117 | try: 118 | match = ColumnMatch.objects.get( 119 | import_setting=self.import_setting, 120 | column_name=cell, 121 | ) 122 | except ColumnMatch.DoesNotExist: 123 | match = ColumnMatch( 124 | import_setting=self.import_setting, 125 | column_name=cell, 126 | ) 127 | match.guess_field() 128 | 129 | match.header_position = i 130 | match.save() 131 | 132 | match_ids += [match.id] 133 | 134 | return ColumnMatch.objects.filter( 135 | pk__in=match_ids).order_by('header_position') 136 | 137 | def get_import_file_as_list(self, only_header=False): 138 | file_ext = str(self.import_file).lower()[-3:] 139 | data = [] 140 | 141 | self.import_file.seek(0) 142 | 143 | if file_ext == "xls": 144 | import xlrd 145 | 146 | wb = xlrd.open_workbook(file_contents=self.import_file.read()) 147 | sh1 = wb.sheet_by_index(0) 148 | for rownum in range(sh1.nrows): 149 | row_values = [] 150 | for cell in sh1.row(rownum): 151 | # xlrd is too dumb to just check for dates. So we have to ourselves 152 | # 3 is date 153 | # http://www.lexicon.net/sjmachin/xlrd.html#xlrd.Cell-class 154 | if cell.ctype == 3: 155 | row_values += [datetime.datetime(*xlrd.xldate_as_tuple(cell.value, wb.datemode))] 156 | else: 157 | row_values += [cell.value] 158 | data += [row_values] 159 | if only_header: 160 | break 161 | elif file_ext == "csv": 162 | reader = csv.reader(StringIO(self.import_file.read().decode())) 163 | for row in reader: 164 | data += [row] 165 | if only_header: 166 | break 167 | elif file_ext == "lsx": 168 | from openpyxl.reader.excel import load_workbook 169 | # load_workbook actually accepts a file-like object for the filename param 170 | wb = load_workbook(filename=self.import_file, read_only=True) 171 | sheet = wb.get_active_sheet() 172 | for row in sheet.iter_rows(): 173 | data_row = [] 174 | for cell in row: 175 | data_row += [cell.value] 176 | data += [data_row] 177 | if only_header: 178 | break 179 | elif file_ext == "ods": 180 | from .odsreader import ODSReader 181 | doc = ODSReader(self.import_file) 182 | table = list(doc.SHEETS.items())[0] 183 | 184 | # Remove blank columns that ods files seems to have 185 | blank_columns = [] 186 | for i, header_cell in enumerate(table[1][0]): 187 | if self.is_empty(header_cell): 188 | blank_columns += [i] 189 | # just an overly complicated way to remove these 190 | # indexes from a list 191 | for offset, blank_column in enumerate(blank_columns): 192 | for row in table[1]: 193 | del row[blank_column - offset] 194 | 195 | if only_header: 196 | data += [table[1][0]] 197 | else: 198 | data += table[1] 199 | # Remove blank columns. We use the header row as a unique index. Can't handle blanks. 200 | columns_to_del = [] 201 | for i, header_cell in enumerate(data[0]): 202 | if self.is_empty(header_cell): 203 | columns_to_del += [i] 204 | num_deleted = 0 205 | for column_to_del in columns_to_del: 206 | for row in data: 207 | del row[column_to_del - num_deleted] 208 | num_deleted += 1 209 | if only_header: 210 | return data[0] 211 | return data 212 | 213 | 214 | class RelationalMatch(models.Model): 215 | """Store which unique field is being use to match. 216 | This can be used only to set a FK or one M2M relation 217 | on the import root model. It does not add them. 218 | With Multple rows set to the same field, you could set more 219 | than one per row. 220 | EX Lets say a student has an ID and username and both 221 | are marked as unique in Django orm. The user could reference 222 | that student by either.""" 223 | import_log = models.ForeignKey(ImportLog, on_delete=models.CASCADE) 224 | field_name = models.CharField(max_length=255) # Ex student_number_set 225 | related_field_name = models.CharField(max_length=255, blank=True) # Ex username 226 | 227 | 228 | class ImportedObject(models.Model): 229 | import_log = models.ForeignKey(ImportLog, on_delete=models.CASCADE) 230 | object_id = models.IntegerField() 231 | content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) 232 | content_object = GenericForeignKey('content_type', 'object_id') 233 | -------------------------------------------------------------------------------- /simple_import/odsreader.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011 Marco Conti 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Thanks to grt for the fixes 16 | 17 | import odf.opendocument 18 | from odf.table import * 19 | from odf.text import P 20 | 21 | class ODSReader: 22 | 23 | # loads the file 24 | def __init__(self, file): 25 | self.doc = odf.opendocument.load(file) 26 | self.SHEETS = {} 27 | for sheet in self.doc.spreadsheet.getElementsByType(Table): 28 | self.readSheet(sheet) 29 | 30 | 31 | # reads a sheet in the sheet dictionary, storing each sheet as an array (rows) of arrays (columns) 32 | def readSheet(self, sheet): 33 | name = sheet.getAttribute("name") 34 | rows = sheet.getElementsByType(TableRow) 35 | arrRows = [] 36 | 37 | # for each row 38 | for row in rows: 39 | row_comment = "" 40 | arrCells = [] 41 | cells = row.getElementsByType(TableCell) 42 | 43 | # for each cell 44 | for cell in cells: 45 | # repeated value? 46 | repeat = cell.getAttribute("numbercolumnsrepeated") 47 | if(not repeat): 48 | repeat = 1 49 | 50 | ps = cell.getElementsByType(P) 51 | textContent = "" 52 | 53 | # for each text node 54 | for p in ps: 55 | for n in p.childNodes: 56 | if (n.nodeType == 3): 57 | textContent = textContent + str(n.data) 58 | 59 | if(textContent or textContent == ""): 60 | if(textContent == "" or textContent[0] != "#"): # ignore comments cells 61 | for rr in range(int(repeat)): # repeated? 62 | arrCells.append(textContent) 63 | 64 | else: 65 | row_comment = row_comment + textContent + " "; 66 | 67 | # if row contained something 68 | if(len(arrCells)): 69 | arrRows.append(arrCells) 70 | 71 | #else: 72 | # print "Empty or commented row (", row_comment, ")" 73 | 74 | self.SHEETS[name] = arrRows 75 | 76 | # returns a sheet as an array (rows) of arrays (columns) 77 | def getSheet(self, name): 78 | return self.SHEETS[name] -------------------------------------------------------------------------------- /simple_import/static/test_import.csv: -------------------------------------------------------------------------------- 1 | name,UseR,nothing,import file,import_setting,importtype 2 | Test Import,1,,/tmp/foo.xls,1,N 3 | Something that won't import,1,????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????,hello,1,ERROR 4 | -------------------------------------------------------------------------------- /simple_import/static/test_import.ods: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/burke-software/django-simple-import/0b3d33c3a6949de22cf0141828bcbd44d0e2d9cb/simple_import/static/test_import.ods -------------------------------------------------------------------------------- /simple_import/static/test_import.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/burke-software/django-simple-import/0b3d33c3a6949de22cf0141828bcbd44d0e2d9cb/simple_import/static/test_import.xls -------------------------------------------------------------------------------- /simple_import/templates/simple_import/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {% block simple_import_page_title %}{% endblock %} 6 | 7 | 8 | 9 |

{% block simple_import_title %}{% endblock %}

10 | {% block simple_import_form %}{% endblock %} 11 | 12 | -------------------------------------------------------------------------------- /simple_import/templates/simple_import/do_import.html: -------------------------------------------------------------------------------- 1 | {% extends "simple_import/base.html" %} 2 | 3 | {% block simple_import_page_title %}Import Results{% endblock %} 4 | 5 | {% block simple_import_title %}Import Results{% endblock %} 6 | 7 | {% block simple_import_form %} 8 | {% if success_undo %} 9 |

10 | Import was undone. This is now a simulation, you can run the import again. 11 |

12 | {% endif %} 13 | 14 | {% if create_count %} 15 | Created: {{ create_count }}
16 | {% endif %} 17 | {% if update_count %} 18 | Updated: {{ update_count }}
19 | {% endif %} 20 | 21 | {% if fail_count %} 22 | Failed: {{ fail_count }}
23 | Download failed records 24 | {% endif %} 25 | 26 | 27 | {% if commit %} 28 | {% if import_log.import_type == "N" %} 29 |

30 | It's possible to undo Create only reports. Click here to undo. 31 | If you imported properties that created other records, we can not guarentee those records will be removed. 32 |

33 | {% endif %} 34 | {% else %} 35 |

36 | This was only a simulation. Click here to run the import. 37 |

38 | {% endif %} 39 | {% endblock %} -------------------------------------------------------------------------------- /simple_import/templates/simple_import/import.html: -------------------------------------------------------------------------------- 1 | {% extends "simple_import/base.html" %} 2 | 3 | {% block simple_import_page_title %}Import{% endblock %} 4 | 5 | {% block simple_import_title %}Import{% endblock %} 6 | 7 | {% block simple_import_form %} 8 |
{% csrf_token %} 9 | 10 | {{ form.as_table }} 11 |
12 | 13 |
14 | {% endblock %} -------------------------------------------------------------------------------- /simple_import/templates/simple_import/match_columns.html: -------------------------------------------------------------------------------- 1 | {% extends "simple_import/base.html" %} 2 | 3 | {% block simple_import_page_title %}Match Colums{% endblock %} 4 | 5 | {% block simple_import_title %}Match Columns{% endblock %} 6 | 7 | {% block simple_import_form %} 8 |
{% csrf_token %} 9 | {{ formset.management_form }} 10 | 11 | 16 | {% if errors %} 17 | 22 | {% endif %} 23 | 24 | 25 | 26 | 27 | 28 | 29 | {% if import_log.import_type == "U" or import_log.import_type == "O" %} 30 | 31 | {% endif %} 32 | 33 | 34 | {% for form in formset %} 35 | 36 | 46 | 49 | 52 | 55 | {% if import_log.import_type == "U" or import_log.import_type == "O" %} 56 | 65 | {% endif %} 66 | 67 | 68 | {% endfor %} 69 |
Column HeaderFieldSample DataDefault ValueUpdate KeyClear field on blank cell
37 | {% if form.instance.column_name %} 38 | {{ form.id.as_hidden }} 39 | {{ form.instance.column_name }} 40 | {% else %} 41 | {{ form.column_name }} 42 | {% endif %} 43 | {{ form.column_name.as_hidden }} {{ form.column_name.error }} 44 | {{ form.import_setting.as_hidden }} {{ form.import_setting.error }} 45 | 47 | {{ form.field_name }} {{ form.field_name.error }} 48 | 50 | {{ form.sample }} 51 | 53 | {{ form.default_value }} {{ form.default_value.error }} 54 | 57 | 64 | {{ form.null_on_empty }}
70 | 71 | 72 |
73 | {% endblock %} -------------------------------------------------------------------------------- /simple_import/templates/simple_import/match_relations.html: -------------------------------------------------------------------------------- 1 | {% extends "simple_import/base.html" %} 2 | 3 | {% block simple_import_page_title %}Match Relations and Prepare to Run Import{% endblock %} 4 | 5 | {% block simple_import_title %}Match Relations and Prepare to Run Import{% endblock %} 6 | 7 | {% block simple_import_form %} 8 |
{% csrf_token %} 9 | {{ formset.management_form }} 10 | {% if errors %} 11 | 16 | {% endif %} 17 | {% if existing_matches %} 18 |

19 | Next you need to match how you reference related fields. 20 | As an example let's say you are importing Students. Students can be referred to as their ID (ex. 453) or Username (ex. jstudent). 21 | We need to specify how to reference the student. You may select any unique field. 22 |

23 | 24 | 25 | 26 | 27 | 28 | {% for form in formset %} 29 | 30 | 36 | 39 | 40 | {% endfor %} 41 |
{{ root_model }} FieldUnique Mapping
31 | {{ form.id.as_hidden }} 32 | {{ form.import_log.as_hidden }} 33 | {{ form.field_name.as_hidden }} 34 | {{ form.instance.field_name }} 35 | 37 | {{ form.related_field_name }} 38 |
42 | {% endif %} 43 | 44 | 45 | 46 | 47 |
48 | {% endblock %} -------------------------------------------------------------------------------- /simple_import/tests.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from django.contrib.contenttypes.models import ContentType 4 | from django.urls import reverse 5 | from django.test import TestCase 6 | from .models import * 7 | from django.core.files import File 8 | from django.contrib.auth import get_user_model 9 | User = get_user_model() 10 | 11 | 12 | class SimpleTest(TestCase): 13 | def setUp(self): 14 | user = User.objects.create_user( 15 | 'temporary', 'temporary@gmail.com', 'temporary') 16 | user.is_staff = True 17 | user.is_superuser = True 18 | user.save() 19 | self.user = user 20 | self.client.login(username='temporary', password='temporary') 21 | self.absolute_path = os.path.join( 22 | os.path.dirname(__file__), 'static', 'test_import.xls') 23 | self.import_setting = ImportSetting.objects.create( 24 | user=user, 25 | content_type=ContentType.objects.get_for_model(ImportLog) 26 | ) 27 | with open(self.absolute_path, 'rb') as fp: 28 | self.import_log = ImportLog.objects.create( 29 | name='test', 30 | user=user, 31 | import_file=File(fp), 32 | import_setting=self.import_setting, 33 | import_type='N', 34 | ) 35 | 36 | def test_csv(self): 37 | path = os.path.join( 38 | os.path.dirname(__file__), 'static', 'test_import.csv') 39 | with open(path, 'rb') as fp: 40 | import_log = ImportLog.objects.create( 41 | name="test", 42 | user=self.user, 43 | import_file=File(fp), 44 | import_setting=self.import_setting, 45 | import_type='N', 46 | ) 47 | file_data = import_log.get_import_file_as_list(only_header=True) 48 | self.assertIn('name', file_data) 49 | 50 | def test_ods(self): 51 | path = os.path.join( 52 | os.path.dirname(__file__), 'static', 'test_import.ods') 53 | with open(path, 'rb') as fp: 54 | import_log = ImportLog.objects.create( 55 | name="test", 56 | user=self.user, 57 | import_file=File(fp), 58 | import_setting=self.import_setting, 59 | import_type='N', 60 | ) 61 | file_data = import_log.get_import_file_as_list(only_header=True) 62 | self.assertIn('name', file_data) 63 | 64 | def test_import(self): 65 | """ Make sure we can upload the file and match columns """ 66 | import_log_ct_id = ContentType.objects.get_for_model(ImportLog).id 67 | 68 | self.assertEqual(ImportLog.objects.count(), 1) 69 | 70 | with open(self.absolute_path, 'rb') as fp: 71 | response = self.client.post(reverse('simple_import-start_import'), { 72 | 'name': 'This is a test', 73 | 'import_file': fp, 74 | 'import_type': "N", 75 | 'model': import_log_ct_id}, follow=True) 76 | 77 | self.assertEqual(ImportLog.objects.count(), 2) 78 | 79 | self.assertRedirects(response, reverse('simple_import-match_columns', kwargs={'import_log_id': ImportLog.objects.all()[1].id})) 80 | self.assertContains(response, '

Match Columns

') 81 | # Check matching 82 | self.assertContains(response, '') 86 | # Check Sample Data 87 | self.assertContains(response, '/tmp/foo.xls') 88 | 89 | def test_match_columns(self): 90 | """ Test matching columns view """ 91 | self.assertEqual(ColumnMatch.objects.count(), 0) 92 | 93 | response = self.client.post( 94 | reverse('simple_import-match_columns', kwargs={'import_log_id': self.import_log.id}), { 95 | 'columnmatch_set-TOTAL_FORMS':6, 96 | 'columnmatch_set-INITIAL_FORMS':6, 97 | 'columnmatch_set-MAX_NUM_FORMS':1000, 98 | 'columnmatch_set-0-id':1, 99 | 'columnmatch_set-0-column_name':'name', 100 | 'columnmatch_set-0-import_setting':self.import_setting.id, 101 | 'columnmatch_set-0-field_name':'name', 102 | 'columnmatch_set-1-id':2, 103 | 'columnmatch_set-1-column_name':'UseR', 104 | 'columnmatch_set-1-import_setting':self.import_setting.id, 105 | 'columnmatch_set-1-field_name':'user', 106 | 'columnmatch_set-2-id':3, 107 | 'columnmatch_set-2-column_name':'nothing', 108 | 'columnmatch_set-2-import_setting':self.import_setting.id, 109 | 'columnmatch_set-2-field_name':'', 110 | 'columnmatch_set-3-id':4, 111 | 'columnmatch_set-3-column_name':'import file', 112 | 'columnmatch_set-3-import_setting':self.import_setting.id, 113 | 'columnmatch_set-3-field_name':'import_file', 114 | 'columnmatch_set-4-id':5, 115 | 'columnmatch_set-4-column_name':'import_setting', 116 | 'columnmatch_set-4-import_setting':self.import_setting.id, 117 | 'columnmatch_set-4-field_name':'import_setting', 118 | 'columnmatch_set-5-id':6, 119 | 'columnmatch_set-5-column_name':'importtype', 120 | 'columnmatch_set-5-import_setting':self.import_setting.id, 121 | 'columnmatch_set-5-field_name':'import_type', 122 | }, follow=True) 123 | 124 | self.assertRedirects(response, reverse('simple_import-match_relations', kwargs={'import_log_id': self.import_log.id})) 125 | self.assertContains(response, '

Match Relations and Prepare to Run Import

') 126 | self.assertEqual(ColumnMatch.objects.count(), 6) 127 | 128 | def test_match_relations(self): 129 | """ Test matching relations view """ 130 | self.assertEqual(RelationalMatch.objects.count(), 0) 131 | 132 | response = self.client.post( 133 | reverse('simple_import-match_relations', kwargs={'import_log_id': self.import_log.id}), { 134 | 'relationalmatch_set-TOTAL_FORMS':2, 135 | 'relationalmatch_set-INITIAL_FORMS':2, 136 | 'relationalmatch_set-MAX_NUM_FORMS':1000, 137 | 'relationalmatch_set-0-id':1, 138 | 'relationalmatch_set-0-import_log':self.import_log.id, 139 | 'relationalmatch_set-0-field_name':'user', 140 | 'relationalmatch_set-0-related_field_name':'id', 141 | 'relationalmatch_set-1-id':2, 142 | 'relationalmatch_set-1-import_log':self.import_log.id, 143 | 'relationalmatch_set-1-field_name':'import_setting', 144 | 'relationalmatch_set-1-related_field_name':'id', 145 | }, follow=True) 146 | 147 | self.assertRedirects(response, reverse('simple_import-do_import', kwargs={'import_log_id': self.import_log.id})) 148 | self.assertContains(response, '

Import Results

') 149 | 150 | self.assertEqual(RelationalMatch.objects.count(), 2) 151 | 152 | -------------------------------------------------------------------------------- /simple_import/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | from simple_import import views 3 | 4 | urlpatterns = [ 5 | url('^start_import/$', views.start_import, name='simple_import-start_import'), 6 | url('^match_columns/(?P\d+)/$', views.match_columns, name='simple_import-match_columns'), 7 | url('^match_relations/(?P\d+)/$', views.match_relations, name='simple_import-match_relations'), 8 | url('^do_import/(?P\d+)/$', views.do_import, name='simple_import-do_import'), 9 | ] 10 | -------------------------------------------------------------------------------- /simple_import/utils.py: -------------------------------------------------------------------------------- 1 | from itertools import chain 2 | 3 | 4 | def get_all_field_names(model_class): 5 | return list(set(chain.from_iterable( 6 | (field.name, field.attname) if hasattr(field, 'attname') else (field.name,) 7 | for field in model_class._meta.get_fields() 8 | # For complete backwards compatibility, you may want to exclude 9 | # GenericForeignKey from the results. 10 | if not (field.many_to_one and field.related_model is None) 11 | ))) 12 | 13 | -------------------------------------------------------------------------------- /simple_import/views.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.contrib.contenttypes.models import ContentType 3 | from django.contrib.admin.models import LogEntry, ADDITION, CHANGE 4 | from django.contrib.admin.views.decorators import staff_member_required 5 | from django.contrib import messages 6 | from django.conf import settings 7 | from django.core.exceptions import SuspiciousOperation 8 | from django.urls import reverse 9 | from django.db.models import Q, ForeignKey 10 | from django.db import transaction, IntegrityError 11 | from django.core.exceptions import ObjectDoesNotExist 12 | from django.forms.models import inlineformset_factory 13 | from django.http import HttpResponseRedirect 14 | from django.shortcuts import render, get_object_or_404, redirect 15 | from django.template import RequestContext 16 | import sys 17 | from django.db.models.fields import AutoField, BooleanField 18 | from django.utils.encoding import smart_text 19 | from django.contrib.auth import get_user_model 20 | User = get_user_model() 21 | 22 | from .models import (ImportLog, ImportSetting, ColumnMatch, 23 | ImportedObject, RelationalMatch) 24 | from .forms import ImportForm, MatchForm, MatchRelationForm 25 | from .utils import get_all_field_names 26 | 27 | 28 | def is_foreign_key_id_name(field_name, field_object): 29 | """ Determines if field name is a ForeignKey 30 | ending in "_id" 31 | Used to find FK duplicates in _meta.get_all_field_names 32 | """ 33 | if field_name[-3:] == "_id" and isinstance(field_object, ForeignKey): 34 | return True 35 | 36 | 37 | def validate_match_columns(import_log, model_class, header_row): 38 | """ Perform some basic pre import validation to make sure it's 39 | even possible the import can work 40 | Returns list of errors 41 | """ 42 | errors = [] 43 | column_matches = import_log.import_setting.columnmatch_set.all() 44 | field_names = get_all_field_names(model_class) 45 | for field_name in field_names: 46 | field_object = model_class._meta.get_field(field_name) 47 | model = field_object.model 48 | direct = field_object.concrete 49 | # Skip if update only and skip ptr which suggests it's a django 50 | # inherited field. Also some hard coded ones for Django Auth 51 | if (import_log.import_type != "O" and 52 | field_name[-3:] != "ptr" and 53 | field_name not in ['password', 'date_joined', 'last_login'] and 54 | not is_foreign_key_id_name(field_name, field_object)): 55 | if ((direct and model and not field_object.blank) or 56 | (not getattr(field_object, "blank", True))): 57 | field_matches = column_matches.filter(field_name=field_name) 58 | match_in_header = False 59 | if field_matches: 60 | for field_match in field_matches: 61 | if field_match.column_name.lower() in header_row: 62 | match_in_header = True 63 | if not match_in_header: 64 | errors += [u"{0} is required but is not in your spreadsheet. ".format(field_object.verbose_name.title())] 65 | else: 66 | errors += [u"{0} is required but has no match.".format(field_object.verbose_name.title())] 67 | return errors 68 | 69 | 70 | @staff_member_required 71 | def match_columns(request, import_log_id): 72 | """ View to match import spreadsheet columns with database fields 73 | """ 74 | import_log = get_object_or_404(ImportLog, id=import_log_id) 75 | 76 | if not request.user.is_superuser and import_log.user != request.user: 77 | raise SuspiciousOperation("Non superuser attempting to view other users import") 78 | 79 | # need to generate matches if they don't exist already 80 | existing_matches = import_log.get_matches() 81 | 82 | MatchFormSet = inlineformset_factory(ImportSetting, ColumnMatch, form=MatchForm, extra=0) 83 | 84 | import_data = import_log.get_import_file_as_list() 85 | header_row = [x.lower() for x in import_data[0]] # make all lower 86 | try: 87 | sample_row = import_data[1] 88 | except IndexError: 89 | messages.error(request, 'Error: Spreadsheet was empty.') 90 | return redirect('simple_import-start_import') 91 | 92 | errors = [] 93 | 94 | model_class = import_log.import_setting.content_type.model_class() 95 | field_names = get_all_field_names(model_class) 96 | for field_name in field_names: 97 | field_object = model_class._meta.get_field(field_name) 98 | # We can't add a new AutoField and specify it's value 99 | if import_log.import_type == "N" and isinstance(field_object, AutoField): 100 | field_names.remove(field_name) 101 | 102 | if request.method == 'POST': 103 | formset = MatchFormSet(request.POST, instance=import_log.import_setting) 104 | if formset.is_valid(): 105 | formset.save() 106 | if import_log.import_type in ["U", "O"]: 107 | update_key = request.POST.get('update_key', '') 108 | 109 | if update_key: 110 | field_name = import_log.import_setting.columnmatch_set.get(column_name=update_key).field_name 111 | if field_name: 112 | field_object = model_class._meta.get_field(field_name) 113 | direct = field_object.concrete 114 | 115 | if direct and field_object.unique: 116 | import_log.update_key = update_key 117 | import_log.save() 118 | else: 119 | errors += ['Update key must be unique. Please select a unique field.'] 120 | else: 121 | errors += ['Update key must matched with a column.'] 122 | else: 123 | errors += ['Please select an update key. This key is used to linked records for updating.'] 124 | errors += validate_match_columns( 125 | import_log, 126 | model_class, 127 | header_row) 128 | 129 | all_field_names = [] 130 | for clean_data in formset.cleaned_data: 131 | if clean_data['field_name']: 132 | if clean_data['field_name'] in all_field_names: 133 | errors += ["{0} is duplicated.".format(clean_data['field_name'])] 134 | all_field_names += [clean_data['field_name']] 135 | if not errors: 136 | return HttpResponseRedirect(reverse( 137 | match_relations, 138 | kwargs={'import_log_id': import_log.id})) 139 | else: 140 | formset = MatchFormSet(instance=import_log.import_setting, queryset=existing_matches) 141 | 142 | field_choices = (('', 'Do Not Use'),) 143 | for field_name in field_names: 144 | field_object = model_class._meta.get_field(field_name) 145 | direct = field_object.concrete 146 | m2m = field_object.many_to_many 147 | add = True 148 | 149 | if direct: 150 | field_verbose = field_object.verbose_name 151 | else: 152 | field_verbose = field_name 153 | 154 | if direct and not field_object.blank: 155 | field_verbose += " (Required)" 156 | if direct and field_object.unique: 157 | field_verbose += " (Unique)" 158 | if m2m or isinstance(field_object, ForeignKey): 159 | field_verbose += " (Related)" 160 | elif not direct: 161 | add = False 162 | if is_foreign_key_id_name(field_name, field_object): 163 | add = False 164 | 165 | if add: 166 | field_choices += ((field_name, field_verbose),) 167 | 168 | # Include defined methods 169 | # Model must have a simple_import_methods defined 170 | if hasattr(model_class, 'simple_import_methods'): 171 | for import_method in model_class.simple_import_methods: 172 | field_choices += (("simple_import_method__{0}".format(import_method), 173 | "{0} (Method)".format(import_method)),) 174 | # User model should allow set password 175 | if issubclass(model_class, User): 176 | field_choices += (("simple_import_method__{0}".format('set_password'), 177 | "Set Password (Method)"),) 178 | 179 | for i, form in enumerate(formset): 180 | form.fields['field_name'].widget = forms.Select(choices=(field_choices)) 181 | form.sample = sample_row[i] 182 | 183 | return render( 184 | request, 185 | 'simple_import/match_columns.html', 186 | {'import_log': import_log, 'formset': formset, 'errors': errors}, 187 | ) 188 | 189 | 190 | def get_direct_fields_from_model(model_class): 191 | direct_fields = [] 192 | all_field_names = get_all_field_names(model_class) 193 | for field_name in all_field_names: 194 | field = model_class._meta.get_field(field_name) 195 | direct = field.concrete 196 | m2m = field.many_to_many 197 | if direct and not m2m and field.__class__.__name__ != "ForeignKey": 198 | direct_fields += [field] 199 | return direct_fields 200 | 201 | 202 | @staff_member_required 203 | def match_relations(request, import_log_id): 204 | import_log = get_object_or_404(ImportLog, id=import_log_id) 205 | model_class = import_log.import_setting.content_type.model_class() 206 | matches = import_log.get_matches() 207 | field_names = [] 208 | choice_set = [] 209 | 210 | for match in matches.exclude(field_name=""): 211 | field_name = match.field_name 212 | 213 | if not field_name.startswith('simple_import_method__'): 214 | field = model_class._meta.get_field(field_name) 215 | m2m = field.many_to_many 216 | 217 | if m2m or isinstance(field, ForeignKey): 218 | RelationalMatch.objects.get_or_create( 219 | import_log=import_log, 220 | field_name=field_name) 221 | 222 | field_names.append(field_name) 223 | choices = () 224 | if hasattr(field, 'related'): 225 | try: 226 | parent_model = field.related.parent_model() 227 | except AttributeError: # Django 1.8+ 228 | parent_model = field.related.model 229 | else: 230 | parent_model = field.related_model 231 | for field in get_direct_fields_from_model(parent_model): 232 | if field.unique: 233 | choices += ((field.name, str(field.verbose_name)),) 234 | choice_set += [choices] 235 | 236 | existing_matches = import_log.relationalmatch_set.filter(field_name__in=field_names) 237 | 238 | MatchRelationFormSet = inlineformset_factory( 239 | ImportLog, 240 | RelationalMatch, 241 | form=MatchRelationForm, extra=0) 242 | 243 | if request.method == 'POST': 244 | formset = MatchRelationFormSet(request.POST, instance=import_log) 245 | 246 | if formset.is_valid(): 247 | formset.save() 248 | 249 | url = reverse('simple_import-do_import', 250 | kwargs={'import_log_id': import_log.id}) 251 | 252 | if 'commit' in request.POST: 253 | url += "?commit=True" 254 | 255 | return HttpResponseRedirect(url) 256 | else: 257 | formset = MatchRelationFormSet(instance=import_log) 258 | 259 | for i, form in enumerate(formset.forms): 260 | choices = choice_set[i] 261 | form.fields['related_field_name'].widget = forms.Select(choices=choices) 262 | 263 | return render( 264 | request, 265 | 'simple_import/match_relations.html', 266 | {'formset': formset, 267 | 'existing_matches': existing_matches}, 268 | ) 269 | 270 | 271 | def set_field_from_cell(import_log, new_object, header_row_field_name, cell): 272 | """ Set a field from a import cell. Use referenced fields the field 273 | is m2m or a foreign key. 274 | """ 275 | if not header_row_field_name.startswith('simple_import_method__'): 276 | field = new_object._meta.get_field(header_row_field_name) 277 | m2m = field.many_to_many 278 | if m2m: 279 | new_object.simple_import_m2ms[header_row_field_name] = cell 280 | elif isinstance(field, ForeignKey): 281 | related_field_name = RelationalMatch.objects.get( 282 | import_log=import_log, 283 | field_name=field.name, 284 | ).related_field_name 285 | try: 286 | related_model = field.remote_field.parent_model() 287 | except AttributeError: 288 | related_model = field.remote_field.model 289 | related_object = related_model.objects.get(**{related_field_name:cell}) 290 | setattr(new_object, header_row_field_name, related_object) 291 | elif field.choices and getattr(settings, 'SIMPLE_IMPORT_LAZY_CHOICES', True): 292 | # Trim leading and trailing whitespace from cell values 293 | if getattr(settings, 'SIMPLE_IMPORT_LAZY_CHOICES_STRIP', False): 294 | cell = cell.strip() 295 | # Prefer database values over choices lookup 296 | database_values, verbose_values = zip(*field.choices) 297 | if cell in database_values: 298 | setattr(new_object, header_row_field_name, cell) 299 | elif cell in verbose_values: 300 | for choice in field.choices: 301 | if smart_text(cell) == smart_text(choice[1]): 302 | setattr(new_object, header_row_field_name, choice[0]) 303 | elif isinstance(field, BooleanField): 304 | # Some formats/libraries report booleans as strings 305 | if cell == False or cell == "FALSE": 306 | setattr(new_object, header_row_field_name, False) 307 | else: 308 | setattr(new_object, header_row_field_name, cell) 309 | else: 310 | setattr(new_object, header_row_field_name, cell) 311 | 312 | 313 | def set_method_from_cell(import_log, new_object, header_row_field_name, cell): 314 | """ Run a method from a import cell. 315 | """ 316 | if not header_row_field_name.startswith('simple_import_method__'): 317 | pass 318 | elif header_row_field_name.startswith('simple_import_method__'): 319 | getattr(new_object, header_row_field_name[22:])(cell) 320 | 321 | 322 | @staff_member_required 323 | def do_import(request, import_log_id): 324 | """ Import the data! """ 325 | import_log = get_object_or_404(ImportLog, id=import_log_id) 326 | if import_log.import_type == "N" and 'undo' in request.GET and request.GET['undo'] == "True": 327 | import_log.undo() 328 | return HttpResponseRedirect(reverse( 329 | do_import, 330 | kwargs={'import_log_id': import_log.id}) + '?success_undo=True') 331 | 332 | if 'success_undo' in request.GET and request.GET['success_undo'] == "True": 333 | success_undo = True 334 | else: 335 | success_undo = False 336 | 337 | model_class = import_log.import_setting.content_type.model_class() 338 | import_data = import_log.get_import_file_as_list() 339 | header_row = import_data.pop(0) 340 | header_row_field_names = [] 341 | header_row_default = [] 342 | header_row_null_on_empty = [] 343 | error_data = [header_row + ['Error Type', 'Error Details']] 344 | create_count = 0 345 | update_count = 0 346 | fail_count = 0 347 | if 'commit' in request.GET and request.GET['commit'] == "True": 348 | commit = True 349 | else: 350 | commit = False 351 | 352 | key_column_name = None 353 | if import_log.update_key and import_log.import_type in ["U", "O"]: 354 | key_match = import_log.import_setting.columnmatch_set.get(column_name=import_log.update_key) 355 | key_column_name = key_match.column_name 356 | key_field_name = key_match.field_name 357 | for i, cell in enumerate(header_row): 358 | match = import_log.import_setting.columnmatch_set.get(column_name=cell) 359 | header_row_field_names += [match.field_name] 360 | header_row_default += [match.default_value] 361 | header_row_null_on_empty += [match.null_on_empty] 362 | if key_column_name != None and key_column_name.lower() == cell.lower(): 363 | key_index = i 364 | 365 | with transaction.atomic(): 366 | sid = transaction.savepoint() 367 | for row in import_data: 368 | try: 369 | with transaction.atomic(): 370 | is_created = True 371 | if import_log.import_type == "N": 372 | new_object = model_class() 373 | elif import_log.import_type == "O": 374 | filters = {key_field_name: row[key_index]} 375 | new_object = model_class.objects.get(**filters) 376 | is_created = False 377 | elif import_log.import_type == "U": 378 | filters = {key_field_name: row[key_index]} 379 | new_object = model_class.objects.filter(**filters).first() 380 | if new_object == None: 381 | new_object = model_class() 382 | is_created = False 383 | 384 | new_object.simple_import_m2ms = {} # Need to deal with these after saving 385 | for i, cell in enumerate(row): 386 | if header_row_field_names[i]: # skip blank 387 | if not import_log.is_empty(cell) or header_row_null_on_empty[i]: 388 | set_field_from_cell(import_log, new_object, header_row_field_names[i], cell) 389 | elif header_row_default[i]: 390 | set_field_from_cell(import_log, new_object, header_row_field_names[i], header_row_default[i]) 391 | new_object.save() 392 | 393 | for i, cell in enumerate(row): 394 | if header_row_field_names[i]: # skip blank 395 | if not import_log.is_empty(cell) or header_row_null_on_empty[i]: 396 | set_method_from_cell(import_log, new_object, header_row_field_names[i], cell) 397 | elif header_row_default[i]: 398 | set_method_from_cell(import_log, new_object, header_row_field_names[i], header_row_default[i]) 399 | new_object.save() 400 | 401 | for key in new_object.simple_import_m2ms.keys(): 402 | value = new_object.simple_import_m2ms[key] 403 | m2m = getattr(new_object, key) 404 | m2m_model = type(m2m.model()) 405 | related_field_name = RelationalMatch.objects.get(import_log=import_log, field_name=key).related_field_name 406 | m2m_object = m2m_model.objects.get(**{related_field_name:value}) 407 | m2m.add(m2m_object) 408 | 409 | if is_created: 410 | LogEntry.objects.log_action( 411 | user_id = request.user.pk, 412 | content_type_id = ContentType.objects.get_for_model(new_object).pk, 413 | object_id = new_object.pk, 414 | object_repr = smart_text(new_object), 415 | action_flag = ADDITION 416 | ) 417 | create_count += 1 418 | else: 419 | LogEntry.objects.log_action( 420 | user_id = request.user.pk, 421 | content_type_id = ContentType.objects.get_for_model(new_object).pk, 422 | object_id = new_object.pk, 423 | object_repr = smart_text(new_object), 424 | action_flag = CHANGE 425 | ) 426 | update_count += 1 427 | ImportedObject.objects.create( 428 | import_log = import_log, 429 | object_id = new_object.pk, 430 | content_type = import_log.import_setting.content_type) 431 | except IntegrityError: 432 | exc = sys.exc_info() 433 | error_data += [row + ["Integrity Error", smart_text(exc[1])]] 434 | fail_count += 1 435 | except ObjectDoesNotExist: 436 | exc = sys.exc_info() 437 | error_data += [row + ["No Record Found to Update", smart_text(exc[1])]] 438 | fail_count += 1 439 | except ValueError: 440 | exc = sys.exc_info() 441 | if str(exc[1]).startswith('invalid literal for int() with base 10'): 442 | error_data += [row + ["Incompatible Data - A number was expected, but a character was used", smart_text(exc[1])]] 443 | else: 444 | error_data += [row + ["Value Error", smart_text(exc[1])]] 445 | fail_count += 1 446 | except: 447 | error_data += [row + ["Unknown Error"]] 448 | fail_count += 1 449 | if not commit: 450 | transaction.savepoint_rollback(sid) 451 | 452 | if fail_count: 453 | from io import StringIO 454 | from django.core.files.base import ContentFile 455 | from openpyxl.workbook import Workbook 456 | from openpyxl.writer.excel import save_virtual_workbook 457 | 458 | wb = Workbook() 459 | ws = wb.worksheets[0] 460 | ws.title = "Errors" 461 | filename = 'Errors.xlsx' 462 | for row in error_data: 463 | ws.append(row) 464 | buf = StringIO() 465 | # Not Python 3 compatible 466 | #buf.write(str(save_virtual_workbook(wb))) 467 | import_log.error_file.save(filename, ContentFile(save_virtual_workbook(wb))) 468 | import_log.save() 469 | 470 | return render( 471 | request, 472 | 'simple_import/do_import.html', 473 | { 474 | 'error_data': error_data, 475 | 'create_count': create_count, 476 | 'update_count': update_count, 477 | 'fail_count': fail_count, 478 | 'import_log': import_log, 479 | 'commit': commit, 480 | 'success_undo': success_undo,}, 481 | ) 482 | 483 | 484 | @staff_member_required 485 | def start_import(request): 486 | """ View to create a new import record 487 | """ 488 | if request.method == 'POST': 489 | form = ImportForm(request.POST, request.FILES) 490 | if form.is_valid(): 491 | import_log = form.save(commit=False) 492 | import_log.user = request.user 493 | import_log.import_setting, created = ImportSetting.objects.get_or_create( 494 | user=request.user, 495 | content_type=form.cleaned_data['model'], 496 | ) 497 | import_log.save() 498 | return HttpResponseRedirect(reverse(match_columns, kwargs={'import_log_id': import_log.id})) 499 | else: 500 | form = ImportForm() 501 | if not request.user.is_superuser: 502 | form.fields["model"].queryset = ContentType.objects.filter( 503 | Q(permission__group__user=request.user, permission__codename__startswith="change_") | 504 | Q(permission__user=request.user, permission__codename__startswith="change_")).distinct() 505 | 506 | return render(request, 'simple_import/import.html', {'form':form,}) 507 | 508 | -------------------------------------------------------------------------------- /simple_import_demo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/burke-software/django-simple-import/0b3d33c3a6949de22cf0141828bcbd44d0e2d9cb/simple_import_demo/__init__.py -------------------------------------------------------------------------------- /simple_import_demo/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for simple_import_demo project. 3 | 4 | For more information on this file, see 5 | https://docs.djangoproject.com/en/1.7/topics/settings/ 6 | 7 | For the full list of settings and their values, see 8 | https://docs.djangoproject.com/en/1.7/ref/settings/ 9 | """ 10 | 11 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 12 | import os 13 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 14 | 15 | 16 | # Quick-start development settings - unsuitable for production 17 | # See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/ 18 | 19 | # SECURITY WARNING: keep the secret key used in production secret! 20 | SECRET_KEY = '*ey3vq0_k9nsu-sgyrcs^90i2ie%&akeu&+2pl1$!(8p+ql13m' 21 | 22 | # SECURITY WARNING: don't run with debug turned on in production! 23 | DEBUG = True 24 | 25 | ALLOWED_HOSTS = [] 26 | 27 | 28 | # Application definition 29 | 30 | INSTALLED_APPS = ( 31 | 'django.contrib.admin', 32 | 'django.contrib.auth', 33 | 'django.contrib.contenttypes', 34 | 'django.contrib.sessions', 35 | 'django.contrib.messages', 36 | 'django.contrib.staticfiles', 37 | 'simple_import', 38 | ) 39 | 40 | MIDDLEWARE = [ 41 | 'django.middleware.security.SecurityMiddleware', 42 | 'django.contrib.sessions.middleware.SessionMiddleware', 43 | 'django.middleware.common.CommonMiddleware', 44 | 'django.middleware.csrf.CsrfViewMiddleware', 45 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 46 | 'django.contrib.messages.middleware.MessageMiddleware', 47 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 48 | ] 49 | 50 | ROOT_URLCONF = 'simple_import_demo.urls' 51 | 52 | WSGI_APPLICATION = 'simple_import_demo.wsgi.application' 53 | 54 | 55 | # Database 56 | # https://docs.djangoproject.com/en/1.7/ref/settings/#databases 57 | 58 | DATABASES = { 59 | 'default': { 60 | 'ENGINE': 'django.db.backends.sqlite3', 61 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 62 | } 63 | } 64 | 65 | 66 | TEMPLATES = [ 67 | { 68 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 69 | 'APP_DIRS': True, 70 | 'OPTIONS': { 71 | 'context_processors': [ 72 | 'django.template.context_processors.debug', 73 | 'django.template.context_processors.request', 74 | 'django.contrib.auth.context_processors.auth', 75 | 'django.contrib.messages.context_processors.messages', 76 | ], 77 | }, 78 | }, 79 | ] 80 | 81 | # Internationalization 82 | # https://docs.djangoproject.com/en/1.7/topics/i18n/ 83 | 84 | LANGUAGE_CODE = 'en-us' 85 | 86 | TIME_ZONE = 'UTC' 87 | 88 | USE_I18N = True 89 | 90 | USE_L10N = True 91 | 92 | USE_TZ = True 93 | 94 | 95 | # Static files (CSS, JavaScript, Images) 96 | # https://docs.djangoproject.com/en/1.7/howto/static-files/ 97 | 98 | STATIC_URL = '/static/' 99 | MEDIA_URL = '/media/' 100 | -------------------------------------------------------------------------------- /simple_import_demo/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import include, url 2 | from django.contrib import admin 3 | from django.conf import settings 4 | from django.conf.urls.static import static 5 | 6 | urlpatterns = [ 7 | url(r'^admin/', admin.site.urls), 8 | url(r'^simple_import/', include('simple_import.urls')), 9 | ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) 10 | -------------------------------------------------------------------------------- /simple_import_demo/wsgi.py: -------------------------------------------------------------------------------- 1 | import os 2 | from django.core.wsgi import get_wsgi_application 3 | 4 | os.environ['DJANGO_SETTINGS_MODULE'] = 'simple_import_demo.settings' 5 | application = get_wsgi_application() 6 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | toxworkdir={env:TOX_WORK_DIR:.tox} 3 | envlist = py{37}-django{111,22,30} 4 | 5 | [testenv] 6 | passenv = * 7 | install_command = pip install {opts} {packages} 8 | deps = 9 | django111: django>=1.11,<1.12 10 | django22: django>=2.2,<2.3 11 | django30: django>=3.0,<3.1 12 | -e{toxinidir}[ods,xlsx,xls] 13 | commands = 14 | {envpython} {toxinidir}/manage.py test --noinput 15 | 16 | --------------------------------------------------------------------------------