\d+)/$', views.do_import, name='simple_import-do_import'),
9 | ]
10 |
--------------------------------------------------------------------------------
/simple_import/static/test_import.csv:
--------------------------------------------------------------------------------
1 | name,UseR,nothing,import file,import_setting,importtype
2 | Test Import,1,,/tmp/foo.xls,1,N
3 | Something that won't import,1,????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????,hello,1,ERROR
4 |
--------------------------------------------------------------------------------
/simple_import/forms.py:
--------------------------------------------------------------------------------
1 | from django import forms
2 | from django.contrib.contenttypes.models import ContentType
3 |
4 | from simple_import.models import ImportLog, ColumnMatch, RelationalMatch
5 |
6 |
7 | class ImportForm(forms.ModelForm):
8 | class Meta:
9 | model = ImportLog
10 | fields = ('name', 'import_file', 'import_type')
11 | model = forms.ModelChoiceField(ContentType.objects.all())
12 |
13 |
14 | class MatchForm(forms.ModelForm):
15 | class Meta:
16 | model = ColumnMatch
17 | exclude = ['header_position']
18 |
19 |
20 | class MatchRelationForm(forms.ModelForm):
21 | class Meta:
22 | model = RelationalMatch
23 | widgets = {
24 | 'related_field_name': forms.Select(choices=(('', '---------'),))
25 | }
26 | fields = "__all__"
27 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup, find_packages
2 |
3 | setup(
4 | name = "django-simple-import",
5 | version = "2.1.0",
6 | author = "David Burke",
7 | author_email = "david@burkesoftware.com",
8 | description = ("A Django import tool easy enough your users could use it"),
9 | license = "BSD",
10 | keywords = "django import",
11 | url = "https://gitlab.com/burke-software/django-simple-import",
12 | packages=find_packages(exclude=("simple_import/static","simple_import_demo")),
13 | include_package_data=True,
14 | classifiers=[
15 | "Development Status :: 5 - Production/Stable",
16 | 'Environment :: Web Environment',
17 | 'Framework :: Django',
18 | 'Programming Language :: Python',
19 | 'Programming Language :: Python :: 3',
20 | 'Intended Audience :: Developers',
21 | 'Intended Audience :: System Administrators',
22 | "License :: OSI Approved :: BSD License",
23 | ],
24 | install_requires=['django'],
25 | extras_require = {
26 | 'xlsx': ["openpyxl"],
27 | 'ods': ["odfpy"],
28 | 'xls': ["xlrd"],
29 | },
30 | )
31 |
--------------------------------------------------------------------------------
/simple_import/templates/simple_import/do_import.html:
--------------------------------------------------------------------------------
1 | {% extends "simple_import/base.html" %}
2 |
3 | {% block simple_import_page_title %}Import Results{% endblock %}
4 |
5 | {% block simple_import_title %}Import Results{% endblock %}
6 |
7 | {% block simple_import_form %}
8 | {% if success_undo %}
9 |
10 | Import was undone. This is now a simulation, you can run the import again.
11 |
12 | {% endif %}
13 |
14 | {% if create_count %}
15 | Created: {{ create_count }} Download failed records
24 | {% endif %}
25 |
26 |
27 | {% if commit %}
28 | {% if import_log.import_type == "N" %}
29 |
30 | It's possible to undo Create only reports. Click here to undo.
31 | If you imported properties that created other records, we can not guarentee those records will be removed.
32 |
33 | {% endif %}
34 | {% else %}
35 |
36 | This was only a simulation. Click here to run the import.
37 |
38 | {% endif %}
39 | {% endblock %}
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2013 Burke Software and Consulting LLC
2 | All rights reserved.
3 |
4 | Redistribution and use in source and binary forms, with or without
5 | modification, are permitted provided that the following conditions
6 | are met:
7 | 1. Redistributions of source code must retain the above copyright
8 | notice, this list of conditions and the following disclaimer.
9 | 2. Redistributions in binary form must reproduce the above copyright
10 | notice, this list of conditions and the following disclaimer in the
11 | documentation and/or other materials provided with the distribution.
12 | 3. The name of the author may not be used to endorse or promote products
13 | derived from this software without specific prior written permission.
14 |
15 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 | OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 | IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 | NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 | THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 |
--------------------------------------------------------------------------------
/simple_import/templates/simple_import/match_relations.html:
--------------------------------------------------------------------------------
1 | {% extends "simple_import/base.html" %}
2 |
3 | {% block simple_import_page_title %}Match Relations and Prepare to Run Import{% endblock %}
4 |
5 | {% block simple_import_title %}Match Relations and Prepare to Run Import{% endblock %}
6 |
7 | {% block simple_import_form %}
8 |
48 | {% endblock %}
--------------------------------------------------------------------------------
/simple_import/odsreader.py:
--------------------------------------------------------------------------------
1 | # Copyright 2011 Marco Conti
2 |
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 |
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 |
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | # Thanks to grt for the fixes
16 |
17 | import odf.opendocument
18 | from odf.table import *
19 | from odf.text import P
20 |
21 | class ODSReader:
22 |
23 | # loads the file
24 | def __init__(self, file):
25 | self.doc = odf.opendocument.load(file)
26 | self.SHEETS = {}
27 | for sheet in self.doc.spreadsheet.getElementsByType(Table):
28 | self.readSheet(sheet)
29 |
30 |
31 | # reads a sheet in the sheet dictionary, storing each sheet as an array (rows) of arrays (columns)
32 | def readSheet(self, sheet):
33 | name = sheet.getAttribute("name")
34 | rows = sheet.getElementsByType(TableRow)
35 | arrRows = []
36 |
37 | # for each row
38 | for row in rows:
39 | row_comment = ""
40 | arrCells = []
41 | cells = row.getElementsByType(TableCell)
42 |
43 | # for each cell
44 | for cell in cells:
45 | # repeated value?
46 | repeat = cell.getAttribute("numbercolumnsrepeated")
47 | if(not repeat):
48 | repeat = 1
49 |
50 | ps = cell.getElementsByType(P)
51 | textContent = ""
52 |
53 | # for each text node
54 | for p in ps:
55 | for n in p.childNodes:
56 | if (n.nodeType == 3):
57 | textContent = textContent + str(n.data)
58 |
59 | if(textContent or textContent == ""):
60 | if(textContent == "" or textContent[0] != "#"): # ignore comments cells
61 | for rr in range(int(repeat)): # repeated?
62 | arrCells.append(textContent)
63 |
64 | else:
65 | row_comment = row_comment + textContent + " ";
66 |
67 | # if row contained something
68 | if(len(arrCells)):
69 | arrRows.append(arrCells)
70 |
71 | #else:
72 | # print "Empty or commented row (", row_comment, ")"
73 |
74 | self.SHEETS[name] = arrRows
75 |
76 | # returns a sheet as an array (rows) of arrays (columns)
77 | def getSheet(self, name):
78 | return self.SHEETS[name]
--------------------------------------------------------------------------------
/simple_import/templates/simple_import/match_columns.html:
--------------------------------------------------------------------------------
1 | {% extends "simple_import/base.html" %}
2 |
3 | {% block simple_import_page_title %}Match Colums{% endblock %}
4 |
5 | {% block simple_import_title %}Match Columns{% endblock %}
6 |
7 | {% block simple_import_form %}
8 |
73 | {% endblock %}
--------------------------------------------------------------------------------
/simple_import_demo/settings.py:
--------------------------------------------------------------------------------
1 | """
2 | Django settings for simple_import_demo project.
3 |
4 | For more information on this file, see
5 | https://docs.djangoproject.com/en/1.7/topics/settings/
6 |
7 | For the full list of settings and their values, see
8 | https://docs.djangoproject.com/en/1.7/ref/settings/
9 | """
10 |
11 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
12 | import os
13 | BASE_DIR = os.path.dirname(os.path.dirname(__file__))
14 |
15 |
16 | # Quick-start development settings - unsuitable for production
17 | # See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/
18 |
19 | # SECURITY WARNING: keep the secret key used in production secret!
20 | SECRET_KEY = '*ey3vq0_k9nsu-sgyrcs^90i2ie%&akeu&+2pl1$!(8p+ql13m'
21 |
22 | # SECURITY WARNING: don't run with debug turned on in production!
23 | DEBUG = True
24 |
25 | ALLOWED_HOSTS = []
26 |
27 |
28 | # Application definition
29 |
30 | INSTALLED_APPS = (
31 | 'django.contrib.admin',
32 | 'django.contrib.auth',
33 | 'django.contrib.contenttypes',
34 | 'django.contrib.sessions',
35 | 'django.contrib.messages',
36 | 'django.contrib.staticfiles',
37 | 'simple_import',
38 | )
39 |
40 | MIDDLEWARE = [
41 | 'django.middleware.security.SecurityMiddleware',
42 | 'django.contrib.sessions.middleware.SessionMiddleware',
43 | 'django.middleware.common.CommonMiddleware',
44 | 'django.middleware.csrf.CsrfViewMiddleware',
45 | 'django.contrib.auth.middleware.AuthenticationMiddleware',
46 | 'django.contrib.messages.middleware.MessageMiddleware',
47 | 'django.middleware.clickjacking.XFrameOptionsMiddleware',
48 | ]
49 |
50 | ROOT_URLCONF = 'simple_import_demo.urls'
51 |
52 | WSGI_APPLICATION = 'simple_import_demo.wsgi.application'
53 |
54 |
55 | # Database
56 | # https://docs.djangoproject.com/en/1.7/ref/settings/#databases
57 |
58 | DATABASES = {
59 | 'default': {
60 | 'ENGINE': 'django.db.backends.sqlite3',
61 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
62 | }
63 | }
64 |
65 |
66 | TEMPLATES = [
67 | {
68 | 'BACKEND': 'django.template.backends.django.DjangoTemplates',
69 | 'APP_DIRS': True,
70 | 'OPTIONS': {
71 | 'context_processors': [
72 | 'django.template.context_processors.debug',
73 | 'django.template.context_processors.request',
74 | 'django.contrib.auth.context_processors.auth',
75 | 'django.contrib.messages.context_processors.messages',
76 | ],
77 | },
78 | },
79 | ]
80 |
81 | # Internationalization
82 | # https://docs.djangoproject.com/en/1.7/topics/i18n/
83 |
84 | LANGUAGE_CODE = 'en-us'
85 |
86 | TIME_ZONE = 'UTC'
87 |
88 | USE_I18N = True
89 |
90 | USE_L10N = True
91 |
92 | USE_TZ = True
93 |
94 |
95 | # Static files (CSS, JavaScript, Images)
96 | # https://docs.djangoproject.com/en/1.7/howto/static-files/
97 |
98 | STATIC_URL = '/static/'
99 | MEDIA_URL = '/media/'
100 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | django-simple-import
2 | ====================
3 |
4 | An easy to use import tool for Django. django-simple-import aims to keep track of logs and user preferences in the database.
5 |
6 | Project is now stable and in maintenance only mode. If you'd like to add features please fork or take over ownership.
7 |
8 | 
9 | 
10 | 
11 |
12 | # Changelog
13 |
14 | ## 2.1
15 |
16 | - Add support for Django 2.2 and 3.0
17 | - Move to gitlab and gitlab CI for freedom and consistency with other Burke Software maintained projects
18 |
19 | ## 2.0
20 |
21 | 2.0 adds support for Django 1.9 and 1.10. Support for 1.8 and under is dropped. Support for Python 2 is dropped.
22 | Removed support for django-custom-field
23 | Use 1.x for older environments.
24 |
25 | ## 1.17
26 |
27 | The most apparent changes are 1.7 compatibility and migration to Django's
28 | atomic transactions. Please report any issues. I test against mysql innodb, postgres, and sqlite.
29 |
30 | ## Features
31 | - Supports csv, xls, xlsx, and ods import file
32 | - Save user matches of column headers to fields
33 | - Guess matches
34 | - Create, update, or both
35 | - Allow programmers to define special import methods for custom handling
36 | - Set related objects by any unique field
37 | - Simulate imports before commiting to database
38 | - Undo (create only) imports
39 | - Security checks if user has correct permissions (partially implemented)
40 |
41 | ## Install
42 |
43 | 1. `pip install django-simple-import[ods,xls,xlsx]` for full install or specify which formats you need to support. CSV is supported out of box.
44 | 2. Add 'simple_import' to INSTALLED APPS
45 | 3. Add simple_import to urls.py like
46 | `url(r'^simple_import/', include('simple_import.urls')),`
47 | 4. migrate
48 |
49 | ## Optional Settings
50 | Define allowed methods to be "imported". Example:
51 |
52 | class Foo(models.Model):
53 | ...
54 | def set_bar(self, value):
55 | self.bar = value
56 | simple_import_methods = ('set_bar',)
57 |
58 | ### settings.py
59 | SIMPLE_IMPORT_LAZY_CHOICES: Default True. If enabled simple_import will look up choices when importing. Example:
60 |
61 | choices = ['M', 'Monday']
62 |
63 | If the spreadsheet value is "Monday" it will set the database value to "M."
64 |
65 | SIMPLE_IMPORT_LAZY_CHOICES_STRIP: Default False. If enabled, simple_import will trip leading/trailing whitespace
66 | from the cell's value before checking for a match. Only relevant when SIMPLE_IMPORT_LAZY_CHOICES is also enabled.
67 |
68 | If you need any help, we do consulting and custom development. Just email us at david at burkesoftware.com.
69 |
70 |
71 | ## Usage
72 |
73 | Go to `/simple_import/start_import/` or use the admin interface.
74 |
75 | The screenshots have a django-grappelli like theme. The base templates have no style and are very basic.
76 | See an example of customization [here](https://github.com/burke-software/django-sis/tree/master/templates/simple_import).
77 | It is often sufficient to simply override `simple_import/templates/base.html`.
78 |
79 | There is also a log of import records. Check out `/admin/simple_import/`.
80 |
81 | ## Odd Things
82 |
83 | Added a special set password property on auth.User to set password. This sets the password instead of just
84 | saving a hash.
85 |
86 | User has some required fields that...aren't really required. Hardcoded to let them pass.
87 |
88 | ### Security
89 |
90 | I'm working on the assumption staff users are trusted. Only users with change permission
91 | to a field will see it as an option. I have not spent much time looking for ways users could
92 | manipulate URLs to run unauthorized imports. Feel free to contribute changes.
93 | All import views do require admin "is staff" permission.
94 |
95 | ## Testing
96 |
97 | If you have [docker-compose](https://docs.docker.com/compose/) and [Docker](https://www.docker.com/)
98 | installed, then just running `docker-compose run --rm app ./manage.py test` will do everything you need to test
99 | the packages.
100 |
101 | Otherwise look at the `.travis.yml` file for test dependencies.
102 |
--------------------------------------------------------------------------------
/simple_import/migrations/0001_initial.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | from __future__ import unicode_literals
3 |
4 | from django.db import models, migrations
5 | from django.conf import settings
6 |
7 |
8 | class Migration(migrations.Migration):
9 |
10 | dependencies = [
11 | ('contenttypes', '0001_initial'),
12 | migrations.swappable_dependency(settings.AUTH_USER_MODEL),
13 | ]
14 |
15 | operations = [
16 | migrations.CreateModel(
17 | name='ColumnMatch',
18 | fields=[
19 | ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
20 | ('column_name', models.CharField(max_length=200)),
21 | ('field_name', models.CharField(blank=True, max_length=255)),
22 | ('default_value', models.CharField(blank=True, max_length=2000)),
23 | ('null_on_empty', models.BooleanField(default=False, help_text='If cell is blank, clear out the field setting it to blank.')),
24 | ('header_position', models.IntegerField(help_text='Annoying way to order the columns to match the header rows')),
25 | ],
26 | ),
27 | migrations.CreateModel(
28 | name='ImportedObject',
29 | fields=[
30 | ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
31 | ('object_id', models.IntegerField()),
32 | ('content_type', models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE)),
33 | ],
34 | ),
35 | migrations.CreateModel(
36 | name='ImportLog',
37 | fields=[
38 | ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
39 | ('name', models.CharField(max_length=255)),
40 | ('date', models.DateTimeField(auto_now_add=True, verbose_name='Date Created')),
41 | ('import_file', models.FileField(upload_to='import_file')),
42 | ('error_file', models.FileField(blank=True, upload_to='error_file')),
43 | ('import_type', models.CharField(choices=[('N', 'Create New Records'), ('U', 'Create and Update Records'), ('O', 'Only Update Records')], max_length=1)),
44 | ('update_key', models.CharField(blank=True, max_length=200)),
45 | ],
46 | ),
47 | migrations.CreateModel(
48 | name='ImportSetting',
49 | fields=[
50 | ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
51 | ('content_type', models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE)),
52 | ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
53 | ],
54 | ),
55 | migrations.CreateModel(
56 | name='RelationalMatch',
57 | fields=[
58 | ('id', models.AutoField(serialize=False, verbose_name='ID', auto_created=True, primary_key=True)),
59 | ('field_name', models.CharField(max_length=255)),
60 | ('related_field_name', models.CharField(blank=True, max_length=255)),
61 | ('import_log', models.ForeignKey(to='simple_import.ImportLog', on_delete=models.CASCADE)),
62 | ],
63 | ),
64 | migrations.AddField(
65 | model_name='importlog',
66 | name='import_setting',
67 | field=models.ForeignKey(to='simple_import.ImportSetting', editable=False, on_delete=models.CASCADE),
68 | ),
69 | migrations.AddField(
70 | model_name='importlog',
71 | name='user',
72 | field=models.ForeignKey(to=settings.AUTH_USER_MODEL, related_name='simple_import_log', editable=False, on_delete=models.CASCADE),
73 | ),
74 | migrations.AddField(
75 | model_name='importedobject',
76 | name='import_log',
77 | field=models.ForeignKey(to='simple_import.ImportLog', on_delete=models.CASCADE),
78 | ),
79 | migrations.AddField(
80 | model_name='columnmatch',
81 | name='import_setting',
82 | field=models.ForeignKey(to='simple_import.ImportSetting', on_delete=models.CASCADE),
83 | ),
84 | migrations.AlterUniqueTogether(
85 | name='importsetting',
86 | unique_together={('user', 'content_type')},
87 | ),
88 | migrations.AlterUniqueTogether(
89 | name='columnmatch',
90 | unique_together={('column_name', 'import_setting')},
91 | ),
92 | ]
93 |
--------------------------------------------------------------------------------
/simple_import/tests.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from django.contrib.contenttypes.models import ContentType
4 | from django.urls import reverse
5 | from django.test import TestCase
6 | from .models import *
7 | from django.core.files import File
8 | from django.contrib.auth import get_user_model
9 | User = get_user_model()
10 |
11 |
12 | class SimpleTest(TestCase):
13 | def setUp(self):
14 | user = User.objects.create_user(
15 | 'temporary', 'temporary@gmail.com', 'temporary')
16 | user.is_staff = True
17 | user.is_superuser = True
18 | user.save()
19 | self.user = user
20 | self.client.login(username='temporary', password='temporary')
21 | self.absolute_path = os.path.join(
22 | os.path.dirname(__file__), 'static', 'test_import.xls')
23 | self.import_setting = ImportSetting.objects.create(
24 | user=user,
25 | content_type=ContentType.objects.get_for_model(ImportLog)
26 | )
27 | with open(self.absolute_path, 'rb') as fp:
28 | self.import_log = ImportLog.objects.create(
29 | name='test',
30 | user=user,
31 | import_file=File(fp),
32 | import_setting=self.import_setting,
33 | import_type='N',
34 | )
35 |
36 | def test_csv(self):
37 | path = os.path.join(
38 | os.path.dirname(__file__), 'static', 'test_import.csv')
39 | with open(path, 'rb') as fp:
40 | import_log = ImportLog.objects.create(
41 | name="test",
42 | user=self.user,
43 | import_file=File(fp),
44 | import_setting=self.import_setting,
45 | import_type='N',
46 | )
47 | file_data = import_log.get_import_file_as_list(only_header=True)
48 | self.assertIn('name', file_data)
49 |
50 | def test_ods(self):
51 | path = os.path.join(
52 | os.path.dirname(__file__), 'static', 'test_import.ods')
53 | with open(path, 'rb') as fp:
54 | import_log = ImportLog.objects.create(
55 | name="test",
56 | user=self.user,
57 | import_file=File(fp),
58 | import_setting=self.import_setting,
59 | import_type='N',
60 | )
61 | file_data = import_log.get_import_file_as_list(only_header=True)
62 | self.assertIn('name', file_data)
63 |
64 | def test_import(self):
65 | """ Make sure we can upload the file and match columns """
66 | import_log_ct_id = ContentType.objects.get_for_model(ImportLog).id
67 |
68 | self.assertEqual(ImportLog.objects.count(), 1)
69 |
70 | with open(self.absolute_path, 'rb') as fp:
71 | response = self.client.post(reverse('simple_import-start_import'), {
72 | 'name': 'This is a test',
73 | 'import_file': fp,
74 | 'import_type': "N",
75 | 'model': import_log_ct_id}, follow=True)
76 |
77 | self.assertEqual(ImportLog.objects.count(), 2)
78 |
79 | self.assertRedirects(response, reverse('simple_import-match_columns', kwargs={'import_log_id': ImportLog.objects.all()[1].id}))
80 | self.assertContains(response, 'Match Columns ')
81 | # Check matching
82 | self.assertContains(response, 'import setting (Required) (Related) ')
86 | # Check Sample Data
87 | self.assertContains(response, '/tmp/foo.xls')
88 |
89 | def test_match_columns(self):
90 | """ Test matching columns view """
91 | self.assertEqual(ColumnMatch.objects.count(), 0)
92 |
93 | response = self.client.post(
94 | reverse('simple_import-match_columns', kwargs={'import_log_id': self.import_log.id}), {
95 | 'columnmatch_set-TOTAL_FORMS':6,
96 | 'columnmatch_set-INITIAL_FORMS':6,
97 | 'columnmatch_set-MAX_NUM_FORMS':1000,
98 | 'columnmatch_set-0-id':1,
99 | 'columnmatch_set-0-column_name':'name',
100 | 'columnmatch_set-0-import_setting':self.import_setting.id,
101 | 'columnmatch_set-0-field_name':'name',
102 | 'columnmatch_set-1-id':2,
103 | 'columnmatch_set-1-column_name':'UseR',
104 | 'columnmatch_set-1-import_setting':self.import_setting.id,
105 | 'columnmatch_set-1-field_name':'user',
106 | 'columnmatch_set-2-id':3,
107 | 'columnmatch_set-2-column_name':'nothing',
108 | 'columnmatch_set-2-import_setting':self.import_setting.id,
109 | 'columnmatch_set-2-field_name':'',
110 | 'columnmatch_set-3-id':4,
111 | 'columnmatch_set-3-column_name':'import file',
112 | 'columnmatch_set-3-import_setting':self.import_setting.id,
113 | 'columnmatch_set-3-field_name':'import_file',
114 | 'columnmatch_set-4-id':5,
115 | 'columnmatch_set-4-column_name':'import_setting',
116 | 'columnmatch_set-4-import_setting':self.import_setting.id,
117 | 'columnmatch_set-4-field_name':'import_setting',
118 | 'columnmatch_set-5-id':6,
119 | 'columnmatch_set-5-column_name':'importtype',
120 | 'columnmatch_set-5-import_setting':self.import_setting.id,
121 | 'columnmatch_set-5-field_name':'import_type',
122 | }, follow=True)
123 |
124 | self.assertRedirects(response, reverse('simple_import-match_relations', kwargs={'import_log_id': self.import_log.id}))
125 | self.assertContains(response, 'Match Relations and Prepare to Run Import ')
126 | self.assertEqual(ColumnMatch.objects.count(), 6)
127 |
128 | def test_match_relations(self):
129 | """ Test matching relations view """
130 | self.assertEqual(RelationalMatch.objects.count(), 0)
131 |
132 | response = self.client.post(
133 | reverse('simple_import-match_relations', kwargs={'import_log_id': self.import_log.id}), {
134 | 'relationalmatch_set-TOTAL_FORMS':2,
135 | 'relationalmatch_set-INITIAL_FORMS':2,
136 | 'relationalmatch_set-MAX_NUM_FORMS':1000,
137 | 'relationalmatch_set-0-id':1,
138 | 'relationalmatch_set-0-import_log':self.import_log.id,
139 | 'relationalmatch_set-0-field_name':'user',
140 | 'relationalmatch_set-0-related_field_name':'id',
141 | 'relationalmatch_set-1-id':2,
142 | 'relationalmatch_set-1-import_log':self.import_log.id,
143 | 'relationalmatch_set-1-field_name':'import_setting',
144 | 'relationalmatch_set-1-related_field_name':'id',
145 | }, follow=True)
146 |
147 | self.assertRedirects(response, reverse('simple_import-do_import', kwargs={'import_log_id': self.import_log.id}))
148 | self.assertContains(response, 'Import Results ')
149 |
150 | self.assertEqual(RelationalMatch.objects.count(), 2)
151 |
152 |
--------------------------------------------------------------------------------
/simple_import/models.py:
--------------------------------------------------------------------------------
1 | from django.contrib.contenttypes.models import ContentType
2 | from django.conf import settings
3 | from django.contrib.contenttypes.fields import GenericForeignKey
4 | from django.core.exceptions import ValidationError
5 | from django.db import models, transaction
6 | from django.utils.encoding import smart_text
7 | from .utils import get_all_field_names
8 | import csv
9 | import datetime
10 | from io import StringIO
11 | AUTH_USER_MODEL = settings.AUTH_USER_MODEL
12 |
13 |
14 | class ImportSetting(models.Model):
15 | """ Save some settings per user per content type """
16 | user = models.ForeignKey(AUTH_USER_MODEL, on_delete=models.CASCADE)
17 | content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
18 |
19 | class Meta():
20 | unique_together = ('user', 'content_type',)
21 |
22 |
23 | class ColumnMatch(models.Model):
24 | """ Match column names from the user uploaded file to the database """
25 | column_name = models.CharField(max_length=200)
26 | field_name = models.CharField(max_length=255, blank=True)
27 | import_setting = models.ForeignKey(ImportSetting, on_delete=models.CASCADE)
28 | default_value = models.CharField(max_length=2000, blank=True)
29 | null_on_empty = models.BooleanField(
30 | default=False,
31 | help_text="If cell is blank, clear out the field setting it to blank.")
32 | header_position = models.IntegerField(
33 | help_text="Annoying way to order the columns to match the header rows")
34 |
35 | class Meta:
36 | unique_together = ('column_name', 'import_setting')
37 |
38 | def __str__(self):
39 | return str('{0} {1}'.format(self.column_name, self.field_name))
40 |
41 | def guess_field(self):
42 | """ Guess the match based on field names
43 | First look for an exact field name match
44 | then search defined alternative names
45 | then normalize the field name and check for match
46 | """
47 | model = self.import_setting.content_type.model_class()
48 | field_names = get_all_field_names(model)
49 | if self.column_name in field_names:
50 | self.field_name = self.column_name
51 | return
52 | #TODO user defined alt names
53 | normalized_field_name = self.column_name.lower().replace(' ', '_')
54 | if normalized_field_name in field_names:
55 | self.field_name = normalized_field_name
56 | return
57 | # Try verbose name
58 | for field_name in field_names:
59 | field = model._meta.get_field(field_name)
60 | if hasattr(field, 'verbose_name'):
61 | if (field.verbose_name.lower().replace(' ', '_') == normalized_field_name):
62 | self.field_name = field_name
63 |
64 |
65 | class ImportLog(models.Model):
66 | """ A log of all import attempts """
67 | name = models.CharField(max_length=255)
68 | user = models.ForeignKey(
69 | AUTH_USER_MODEL, editable=False, related_name="simple_import_log", on_delete=models.CASCADE)
70 | date = models.DateTimeField(auto_now_add=True, verbose_name="Date Created")
71 | import_file = models.FileField(upload_to="import_file")
72 | error_file = models.FileField(upload_to="error_file", blank=True)
73 | import_setting = models.ForeignKey(ImportSetting, editable=False, on_delete=models.CASCADE)
74 | import_type_choices = (
75 | ("N", "Create New Records"),
76 | ("U", "Create and Update Records"),
77 | ("O", "Only Update Records"),
78 | )
79 | import_type = models.CharField(max_length=1, choices=import_type_choices)
80 | update_key = models.CharField(max_length=200, blank=True)
81 |
82 | def __str__(self):
83 | return str(self.name)
84 |
85 | def clean(self):
86 | filename = str(self.import_file).lower()
87 | if not filename[-3:] in ('xls', 'ods', 'csv', 'lsx'):
88 | raise ValidationError(
89 | 'Invalid file type. Must be xls, xlsx, ods, or csv.')
90 |
91 | @transaction.atomic
92 | def undo(self):
93 | if self.import_type != "N":
94 | raise Exception("Cannot undo this type of import!")
95 | for obj in self.importedobject_set.all():
96 | if obj.content_object:
97 | obj.content_object.delete()
98 | obj.delete()
99 |
100 | @staticmethod
101 | def is_empty(value):
102 | """ Check `value` for emptiness by first comparing with None and then
103 | by coercing to string, trimming, and testing for zero length """
104 | return value is None or not len(smart_text(value).strip())
105 |
106 | def get_matches(self):
107 | """ Get each matching header row to database match
108 | Returns a ColumnMatch queryset"""
109 | header_row = self.get_import_file_as_list(only_header=True)
110 | match_ids = []
111 |
112 | for i, cell in enumerate(header_row):
113 | # Sometimes we get blank headers, ignore them.
114 | if self.is_empty(cell):
115 | continue
116 |
117 | try:
118 | match = ColumnMatch.objects.get(
119 | import_setting=self.import_setting,
120 | column_name=cell,
121 | )
122 | except ColumnMatch.DoesNotExist:
123 | match = ColumnMatch(
124 | import_setting=self.import_setting,
125 | column_name=cell,
126 | )
127 | match.guess_field()
128 |
129 | match.header_position = i
130 | match.save()
131 |
132 | match_ids += [match.id]
133 |
134 | return ColumnMatch.objects.filter(
135 | pk__in=match_ids).order_by('header_position')
136 |
137 | def get_import_file_as_list(self, only_header=False):
138 | file_ext = str(self.import_file).lower()[-3:]
139 | data = []
140 |
141 | self.import_file.seek(0)
142 |
143 | if file_ext == "xls":
144 | import xlrd
145 |
146 | wb = xlrd.open_workbook(file_contents=self.import_file.read())
147 | sh1 = wb.sheet_by_index(0)
148 | for rownum in range(sh1.nrows):
149 | row_values = []
150 | for cell in sh1.row(rownum):
151 | # xlrd is too dumb to just check for dates. So we have to ourselves
152 | # 3 is date
153 | # http://www.lexicon.net/sjmachin/xlrd.html#xlrd.Cell-class
154 | if cell.ctype == 3:
155 | row_values += [datetime.datetime(*xlrd.xldate_as_tuple(cell.value, wb.datemode))]
156 | else:
157 | row_values += [cell.value]
158 | data += [row_values]
159 | if only_header:
160 | break
161 | elif file_ext == "csv":
162 | reader = csv.reader(StringIO(self.import_file.read().decode()))
163 | for row in reader:
164 | data += [row]
165 | if only_header:
166 | break
167 | elif file_ext == "lsx":
168 | from openpyxl.reader.excel import load_workbook
169 | # load_workbook actually accepts a file-like object for the filename param
170 | wb = load_workbook(filename=self.import_file, read_only=True)
171 | sheet = wb.get_active_sheet()
172 | for row in sheet.iter_rows():
173 | data_row = []
174 | for cell in row:
175 | data_row += [cell.value]
176 | data += [data_row]
177 | if only_header:
178 | break
179 | elif file_ext == "ods":
180 | from .odsreader import ODSReader
181 | doc = ODSReader(self.import_file)
182 | table = list(doc.SHEETS.items())[0]
183 |
184 | # Remove blank columns that ods files seems to have
185 | blank_columns = []
186 | for i, header_cell in enumerate(table[1][0]):
187 | if self.is_empty(header_cell):
188 | blank_columns += [i]
189 | # just an overly complicated way to remove these
190 | # indexes from a list
191 | for offset, blank_column in enumerate(blank_columns):
192 | for row in table[1]:
193 | del row[blank_column - offset]
194 |
195 | if only_header:
196 | data += [table[1][0]]
197 | else:
198 | data += table[1]
199 | # Remove blank columns. We use the header row as a unique index. Can't handle blanks.
200 | columns_to_del = []
201 | for i, header_cell in enumerate(data[0]):
202 | if self.is_empty(header_cell):
203 | columns_to_del += [i]
204 | num_deleted = 0
205 | for column_to_del in columns_to_del:
206 | for row in data:
207 | del row[column_to_del - num_deleted]
208 | num_deleted += 1
209 | if only_header:
210 | return data[0]
211 | return data
212 |
213 |
214 | class RelationalMatch(models.Model):
215 | """Store which unique field is being use to match.
216 | This can be used only to set a FK or one M2M relation
217 | on the import root model. It does not add them.
218 | With Multple rows set to the same field, you could set more
219 | than one per row.
220 | EX Lets say a student has an ID and username and both
221 | are marked as unique in Django orm. The user could reference
222 | that student by either."""
223 | import_log = models.ForeignKey(ImportLog, on_delete=models.CASCADE)
224 | field_name = models.CharField(max_length=255) # Ex student_number_set
225 | related_field_name = models.CharField(max_length=255, blank=True) # Ex username
226 |
227 |
228 | class ImportedObject(models.Model):
229 | import_log = models.ForeignKey(ImportLog, on_delete=models.CASCADE)
230 | object_id = models.IntegerField()
231 | content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
232 | content_object = GenericForeignKey('content_type', 'object_id')
233 |
--------------------------------------------------------------------------------
/simple_import/views.py:
--------------------------------------------------------------------------------
1 | from django import forms
2 | from django.contrib.contenttypes.models import ContentType
3 | from django.contrib.admin.models import LogEntry, ADDITION, CHANGE
4 | from django.contrib.admin.views.decorators import staff_member_required
5 | from django.contrib import messages
6 | from django.conf import settings
7 | from django.core.exceptions import SuspiciousOperation
8 | from django.urls import reverse
9 | from django.db.models import Q, ForeignKey
10 | from django.db import transaction, IntegrityError
11 | from django.core.exceptions import ObjectDoesNotExist
12 | from django.forms.models import inlineformset_factory
13 | from django.http import HttpResponseRedirect
14 | from django.shortcuts import render, get_object_or_404, redirect
15 | from django.template import RequestContext
16 | import sys
17 | from django.db.models.fields import AutoField, BooleanField
18 | from django.utils.encoding import smart_text
19 | from django.contrib.auth import get_user_model
20 | User = get_user_model()
21 |
22 | from .models import (ImportLog, ImportSetting, ColumnMatch,
23 | ImportedObject, RelationalMatch)
24 | from .forms import ImportForm, MatchForm, MatchRelationForm
25 | from .utils import get_all_field_names
26 |
27 |
28 | def is_foreign_key_id_name(field_name, field_object):
29 | """ Determines if field name is a ForeignKey
30 | ending in "_id"
31 | Used to find FK duplicates in _meta.get_all_field_names
32 | """
33 | if field_name[-3:] == "_id" and isinstance(field_object, ForeignKey):
34 | return True
35 |
36 |
37 | def validate_match_columns(import_log, model_class, header_row):
38 | """ Perform some basic pre import validation to make sure it's
39 | even possible the import can work
40 | Returns list of errors
41 | """
42 | errors = []
43 | column_matches = import_log.import_setting.columnmatch_set.all()
44 | field_names = get_all_field_names(model_class)
45 | for field_name in field_names:
46 | field_object = model_class._meta.get_field(field_name)
47 | model = field_object.model
48 | direct = field_object.concrete
49 | # Skip if update only and skip ptr which suggests it's a django
50 | # inherited field. Also some hard coded ones for Django Auth
51 | if (import_log.import_type != "O" and
52 | field_name[-3:] != "ptr" and
53 | field_name not in ['password', 'date_joined', 'last_login'] and
54 | not is_foreign_key_id_name(field_name, field_object)):
55 | if ((direct and model and not field_object.blank) or
56 | (not getattr(field_object, "blank", True))):
57 | field_matches = column_matches.filter(field_name=field_name)
58 | match_in_header = False
59 | if field_matches:
60 | for field_match in field_matches:
61 | if field_match.column_name.lower() in header_row:
62 | match_in_header = True
63 | if not match_in_header:
64 | errors += [u"{0} is required but is not in your spreadsheet. ".format(field_object.verbose_name.title())]
65 | else:
66 | errors += [u"{0} is required but has no match.".format(field_object.verbose_name.title())]
67 | return errors
68 |
69 |
70 | @staff_member_required
71 | def match_columns(request, import_log_id):
72 | """ View to match import spreadsheet columns with database fields
73 | """
74 | import_log = get_object_or_404(ImportLog, id=import_log_id)
75 |
76 | if not request.user.is_superuser and import_log.user != request.user:
77 | raise SuspiciousOperation("Non superuser attempting to view other users import")
78 |
79 | # need to generate matches if they don't exist already
80 | existing_matches = import_log.get_matches()
81 |
82 | MatchFormSet = inlineformset_factory(ImportSetting, ColumnMatch, form=MatchForm, extra=0)
83 |
84 | import_data = import_log.get_import_file_as_list()
85 | header_row = [x.lower() for x in import_data[0]] # make all lower
86 | try:
87 | sample_row = import_data[1]
88 | except IndexError:
89 | messages.error(request, 'Error: Spreadsheet was empty.')
90 | return redirect('simple_import-start_import')
91 |
92 | errors = []
93 |
94 | model_class = import_log.import_setting.content_type.model_class()
95 | field_names = get_all_field_names(model_class)
96 | for field_name in field_names:
97 | field_object = model_class._meta.get_field(field_name)
98 | # We can't add a new AutoField and specify it's value
99 | if import_log.import_type == "N" and isinstance(field_object, AutoField):
100 | field_names.remove(field_name)
101 |
102 | if request.method == 'POST':
103 | formset = MatchFormSet(request.POST, instance=import_log.import_setting)
104 | if formset.is_valid():
105 | formset.save()
106 | if import_log.import_type in ["U", "O"]:
107 | update_key = request.POST.get('update_key', '')
108 |
109 | if update_key:
110 | field_name = import_log.import_setting.columnmatch_set.get(column_name=update_key).field_name
111 | if field_name:
112 | field_object = model_class._meta.get_field(field_name)
113 | direct = field_object.concrete
114 |
115 | if direct and field_object.unique:
116 | import_log.update_key = update_key
117 | import_log.save()
118 | else:
119 | errors += ['Update key must be unique. Please select a unique field.']
120 | else:
121 | errors += ['Update key must matched with a column.']
122 | else:
123 | errors += ['Please select an update key. This key is used to linked records for updating.']
124 | errors += validate_match_columns(
125 | import_log,
126 | model_class,
127 | header_row)
128 |
129 | all_field_names = []
130 | for clean_data in formset.cleaned_data:
131 | if clean_data['field_name']:
132 | if clean_data['field_name'] in all_field_names:
133 | errors += ["{0} is duplicated.".format(clean_data['field_name'])]
134 | all_field_names += [clean_data['field_name']]
135 | if not errors:
136 | return HttpResponseRedirect(reverse(
137 | match_relations,
138 | kwargs={'import_log_id': import_log.id}))
139 | else:
140 | formset = MatchFormSet(instance=import_log.import_setting, queryset=existing_matches)
141 |
142 | field_choices = (('', 'Do Not Use'),)
143 | for field_name in field_names:
144 | field_object = model_class._meta.get_field(field_name)
145 | direct = field_object.concrete
146 | m2m = field_object.many_to_many
147 | add = True
148 |
149 | if direct:
150 | field_verbose = field_object.verbose_name
151 | else:
152 | field_verbose = field_name
153 |
154 | if direct and not field_object.blank:
155 | field_verbose += " (Required)"
156 | if direct and field_object.unique:
157 | field_verbose += " (Unique)"
158 | if m2m or isinstance(field_object, ForeignKey):
159 | field_verbose += " (Related)"
160 | elif not direct:
161 | add = False
162 | if is_foreign_key_id_name(field_name, field_object):
163 | add = False
164 |
165 | if add:
166 | field_choices += ((field_name, field_verbose),)
167 |
168 | # Include defined methods
169 | # Model must have a simple_import_methods defined
170 | if hasattr(model_class, 'simple_import_methods'):
171 | for import_method in model_class.simple_import_methods:
172 | field_choices += (("simple_import_method__{0}".format(import_method),
173 | "{0} (Method)".format(import_method)),)
174 | # User model should allow set password
175 | if issubclass(model_class, User):
176 | field_choices += (("simple_import_method__{0}".format('set_password'),
177 | "Set Password (Method)"),)
178 |
179 | for i, form in enumerate(formset):
180 | form.fields['field_name'].widget = forms.Select(choices=(field_choices))
181 | form.sample = sample_row[i]
182 |
183 | return render(
184 | request,
185 | 'simple_import/match_columns.html',
186 | {'import_log': import_log, 'formset': formset, 'errors': errors},
187 | )
188 |
189 |
190 | def get_direct_fields_from_model(model_class):
191 | direct_fields = []
192 | all_field_names = get_all_field_names(model_class)
193 | for field_name in all_field_names:
194 | field = model_class._meta.get_field(field_name)
195 | direct = field.concrete
196 | m2m = field.many_to_many
197 | if direct and not m2m and field.__class__.__name__ != "ForeignKey":
198 | direct_fields += [field]
199 | return direct_fields
200 |
201 |
202 | @staff_member_required
203 | def match_relations(request, import_log_id):
204 | import_log = get_object_or_404(ImportLog, id=import_log_id)
205 | model_class = import_log.import_setting.content_type.model_class()
206 | matches = import_log.get_matches()
207 | field_names = []
208 | choice_set = []
209 |
210 | for match in matches.exclude(field_name=""):
211 | field_name = match.field_name
212 |
213 | if not field_name.startswith('simple_import_method__'):
214 | field = model_class._meta.get_field(field_name)
215 | m2m = field.many_to_many
216 |
217 | if m2m or isinstance(field, ForeignKey):
218 | RelationalMatch.objects.get_or_create(
219 | import_log=import_log,
220 | field_name=field_name)
221 |
222 | field_names.append(field_name)
223 | choices = ()
224 | if hasattr(field, 'related'):
225 | try:
226 | parent_model = field.related.parent_model()
227 | except AttributeError: # Django 1.8+
228 | parent_model = field.related.model
229 | else:
230 | parent_model = field.related_model
231 | for field in get_direct_fields_from_model(parent_model):
232 | if field.unique:
233 | choices += ((field.name, str(field.verbose_name)),)
234 | choice_set += [choices]
235 |
236 | existing_matches = import_log.relationalmatch_set.filter(field_name__in=field_names)
237 |
238 | MatchRelationFormSet = inlineformset_factory(
239 | ImportLog,
240 | RelationalMatch,
241 | form=MatchRelationForm, extra=0)
242 |
243 | if request.method == 'POST':
244 | formset = MatchRelationFormSet(request.POST, instance=import_log)
245 |
246 | if formset.is_valid():
247 | formset.save()
248 |
249 | url = reverse('simple_import-do_import',
250 | kwargs={'import_log_id': import_log.id})
251 |
252 | if 'commit' in request.POST:
253 | url += "?commit=True"
254 |
255 | return HttpResponseRedirect(url)
256 | else:
257 | formset = MatchRelationFormSet(instance=import_log)
258 |
259 | for i, form in enumerate(formset.forms):
260 | choices = choice_set[i]
261 | form.fields['related_field_name'].widget = forms.Select(choices=choices)
262 |
263 | return render(
264 | request,
265 | 'simple_import/match_relations.html',
266 | {'formset': formset,
267 | 'existing_matches': existing_matches},
268 | )
269 |
270 |
271 | def set_field_from_cell(import_log, new_object, header_row_field_name, cell):
272 | """ Set a field from a import cell. Use referenced fields the field
273 | is m2m or a foreign key.
274 | """
275 | if not header_row_field_name.startswith('simple_import_method__'):
276 | field = new_object._meta.get_field(header_row_field_name)
277 | m2m = field.many_to_many
278 | if m2m:
279 | new_object.simple_import_m2ms[header_row_field_name] = cell
280 | elif isinstance(field, ForeignKey):
281 | related_field_name = RelationalMatch.objects.get(
282 | import_log=import_log,
283 | field_name=field.name,
284 | ).related_field_name
285 | try:
286 | related_model = field.remote_field.parent_model()
287 | except AttributeError:
288 | related_model = field.remote_field.model
289 | related_object = related_model.objects.get(**{related_field_name:cell})
290 | setattr(new_object, header_row_field_name, related_object)
291 | elif field.choices and getattr(settings, 'SIMPLE_IMPORT_LAZY_CHOICES', True):
292 | # Trim leading and trailing whitespace from cell values
293 | if getattr(settings, 'SIMPLE_IMPORT_LAZY_CHOICES_STRIP', False):
294 | cell = cell.strip()
295 | # Prefer database values over choices lookup
296 | database_values, verbose_values = zip(*field.choices)
297 | if cell in database_values:
298 | setattr(new_object, header_row_field_name, cell)
299 | elif cell in verbose_values:
300 | for choice in field.choices:
301 | if smart_text(cell) == smart_text(choice[1]):
302 | setattr(new_object, header_row_field_name, choice[0])
303 | elif isinstance(field, BooleanField):
304 | # Some formats/libraries report booleans as strings
305 | if cell == False or cell == "FALSE":
306 | setattr(new_object, header_row_field_name, False)
307 | else:
308 | setattr(new_object, header_row_field_name, cell)
309 | else:
310 | setattr(new_object, header_row_field_name, cell)
311 |
312 |
313 | def set_method_from_cell(import_log, new_object, header_row_field_name, cell):
314 | """ Run a method from a import cell.
315 | """
316 | if not header_row_field_name.startswith('simple_import_method__'):
317 | pass
318 | elif header_row_field_name.startswith('simple_import_method__'):
319 | getattr(new_object, header_row_field_name[22:])(cell)
320 |
321 |
322 | @staff_member_required
323 | def do_import(request, import_log_id):
324 | """ Import the data! """
325 | import_log = get_object_or_404(ImportLog, id=import_log_id)
326 | if import_log.import_type == "N" and 'undo' in request.GET and request.GET['undo'] == "True":
327 | import_log.undo()
328 | return HttpResponseRedirect(reverse(
329 | do_import,
330 | kwargs={'import_log_id': import_log.id}) + '?success_undo=True')
331 |
332 | if 'success_undo' in request.GET and request.GET['success_undo'] == "True":
333 | success_undo = True
334 | else:
335 | success_undo = False
336 |
337 | model_class = import_log.import_setting.content_type.model_class()
338 | import_data = import_log.get_import_file_as_list()
339 | header_row = import_data.pop(0)
340 | header_row_field_names = []
341 | header_row_default = []
342 | header_row_null_on_empty = []
343 | error_data = [header_row + ['Error Type', 'Error Details']]
344 | create_count = 0
345 | update_count = 0
346 | fail_count = 0
347 | if 'commit' in request.GET and request.GET['commit'] == "True":
348 | commit = True
349 | else:
350 | commit = False
351 |
352 | key_column_name = None
353 | if import_log.update_key and import_log.import_type in ["U", "O"]:
354 | key_match = import_log.import_setting.columnmatch_set.get(column_name=import_log.update_key)
355 | key_column_name = key_match.column_name
356 | key_field_name = key_match.field_name
357 | for i, cell in enumerate(header_row):
358 | match = import_log.import_setting.columnmatch_set.get(column_name=cell)
359 | header_row_field_names += [match.field_name]
360 | header_row_default += [match.default_value]
361 | header_row_null_on_empty += [match.null_on_empty]
362 | if key_column_name != None and key_column_name.lower() == cell.lower():
363 | key_index = i
364 |
365 | with transaction.atomic():
366 | sid = transaction.savepoint()
367 | for row in import_data:
368 | try:
369 | with transaction.atomic():
370 | is_created = True
371 | if import_log.import_type == "N":
372 | new_object = model_class()
373 | elif import_log.import_type == "O":
374 | filters = {key_field_name: row[key_index]}
375 | new_object = model_class.objects.get(**filters)
376 | is_created = False
377 | elif import_log.import_type == "U":
378 | filters = {key_field_name: row[key_index]}
379 | new_object = model_class.objects.filter(**filters).first()
380 | if new_object == None:
381 | new_object = model_class()
382 | is_created = False
383 |
384 | new_object.simple_import_m2ms = {} # Need to deal with these after saving
385 | for i, cell in enumerate(row):
386 | if header_row_field_names[i]: # skip blank
387 | if not import_log.is_empty(cell) or header_row_null_on_empty[i]:
388 | set_field_from_cell(import_log, new_object, header_row_field_names[i], cell)
389 | elif header_row_default[i]:
390 | set_field_from_cell(import_log, new_object, header_row_field_names[i], header_row_default[i])
391 | new_object.save()
392 |
393 | for i, cell in enumerate(row):
394 | if header_row_field_names[i]: # skip blank
395 | if not import_log.is_empty(cell) or header_row_null_on_empty[i]:
396 | set_method_from_cell(import_log, new_object, header_row_field_names[i], cell)
397 | elif header_row_default[i]:
398 | set_method_from_cell(import_log, new_object, header_row_field_names[i], header_row_default[i])
399 | new_object.save()
400 |
401 | for key in new_object.simple_import_m2ms.keys():
402 | value = new_object.simple_import_m2ms[key]
403 | m2m = getattr(new_object, key)
404 | m2m_model = type(m2m.model())
405 | related_field_name = RelationalMatch.objects.get(import_log=import_log, field_name=key).related_field_name
406 | m2m_object = m2m_model.objects.get(**{related_field_name:value})
407 | m2m.add(m2m_object)
408 |
409 | if is_created:
410 | LogEntry.objects.log_action(
411 | user_id = request.user.pk,
412 | content_type_id = ContentType.objects.get_for_model(new_object).pk,
413 | object_id = new_object.pk,
414 | object_repr = smart_text(new_object),
415 | action_flag = ADDITION
416 | )
417 | create_count += 1
418 | else:
419 | LogEntry.objects.log_action(
420 | user_id = request.user.pk,
421 | content_type_id = ContentType.objects.get_for_model(new_object).pk,
422 | object_id = new_object.pk,
423 | object_repr = smart_text(new_object),
424 | action_flag = CHANGE
425 | )
426 | update_count += 1
427 | ImportedObject.objects.create(
428 | import_log = import_log,
429 | object_id = new_object.pk,
430 | content_type = import_log.import_setting.content_type)
431 | except IntegrityError:
432 | exc = sys.exc_info()
433 | error_data += [row + ["Integrity Error", smart_text(exc[1])]]
434 | fail_count += 1
435 | except ObjectDoesNotExist:
436 | exc = sys.exc_info()
437 | error_data += [row + ["No Record Found to Update", smart_text(exc[1])]]
438 | fail_count += 1
439 | except ValueError:
440 | exc = sys.exc_info()
441 | if str(exc[1]).startswith('invalid literal for int() with base 10'):
442 | error_data += [row + ["Incompatible Data - A number was expected, but a character was used", smart_text(exc[1])]]
443 | else:
444 | error_data += [row + ["Value Error", smart_text(exc[1])]]
445 | fail_count += 1
446 | except:
447 | error_data += [row + ["Unknown Error"]]
448 | fail_count += 1
449 | if not commit:
450 | transaction.savepoint_rollback(sid)
451 |
452 | if fail_count:
453 | from io import StringIO
454 | from django.core.files.base import ContentFile
455 | from openpyxl.workbook import Workbook
456 | from openpyxl.writer.excel import save_virtual_workbook
457 |
458 | wb = Workbook()
459 | ws = wb.worksheets[0]
460 | ws.title = "Errors"
461 | filename = 'Errors.xlsx'
462 | for row in error_data:
463 | ws.append(row)
464 | buf = StringIO()
465 | # Not Python 3 compatible
466 | #buf.write(str(save_virtual_workbook(wb)))
467 | import_log.error_file.save(filename, ContentFile(save_virtual_workbook(wb)))
468 | import_log.save()
469 |
470 | return render(
471 | request,
472 | 'simple_import/do_import.html',
473 | {
474 | 'error_data': error_data,
475 | 'create_count': create_count,
476 | 'update_count': update_count,
477 | 'fail_count': fail_count,
478 | 'import_log': import_log,
479 | 'commit': commit,
480 | 'success_undo': success_undo,},
481 | )
482 |
483 |
484 | @staff_member_required
485 | def start_import(request):
486 | """ View to create a new import record
487 | """
488 | if request.method == 'POST':
489 | form = ImportForm(request.POST, request.FILES)
490 | if form.is_valid():
491 | import_log = form.save(commit=False)
492 | import_log.user = request.user
493 | import_log.import_setting, created = ImportSetting.objects.get_or_create(
494 | user=request.user,
495 | content_type=form.cleaned_data['model'],
496 | )
497 | import_log.save()
498 | return HttpResponseRedirect(reverse(match_columns, kwargs={'import_log_id': import_log.id}))
499 | else:
500 | form = ImportForm()
501 | if not request.user.is_superuser:
502 | form.fields["model"].queryset = ContentType.objects.filter(
503 | Q(permission__group__user=request.user, permission__codename__startswith="change_") |
504 | Q(permission__user=request.user, permission__codename__startswith="change_")).distinct()
505 |
506 | return render(request, 'simple_import/import.html', {'form':form,})
507 |
508 |
--------------------------------------------------------------------------------