├── tests ├── __init__.py └── djangoexample │ ├── testapp │ ├── apps.py │ ├── __init__.py │ ├── migrations │ │ ├── __init__.py │ │ └── 0001_initial.py │ ├── admin.py │ ├── urls.py │ ├── forms.py │ ├── templates │ │ └── testapp │ │ │ ├── base.html │ │ │ ├── create.html │ │ │ └── testimport.html │ ├── models.py │ ├── views.py │ ├── importers.py │ ├── settings.py │ └── test_importer.py │ ├── citations.csv │ └── manage.py ├── pytest.ini ├── djangomodelimport ├── caches.py ├── __init__.py ├── utils.py ├── loaders.py ├── widgets.py ├── parsers.py ├── resultset.py ├── formclassbuilder.py ├── forms.py ├── core.py ├── magic.py └── fields.py ├── .gitignore ├── .travis.yml ├── setup.cfg ├── .vscode └── settings.json ├── pyproject.toml ├── README.md └── poetry.lock /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/apps.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | DJANGO_SETTINGS_MODULE = tests.djangoexample.testapp.settings -------------------------------------------------------------------------------- /tests/djangoexample/citations.csv: -------------------------------------------------------------------------------- 1 | id,author,name,metadata_isbn,metadata_doi 2 | ,Fred Johnson,Starburst,ISBN333,doi:111 3 | ,Fred Johnson,Gattica,ISBN666,doi:222 4 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from .models import Author 4 | 5 | 6 | @admin.register(Author) 7 | class AuthorAdmin(admin.ModelAdmin): 8 | pass 9 | -------------------------------------------------------------------------------- /djangomodelimport/caches.py: -------------------------------------------------------------------------------- 1 | class SimpleDictCache(dict): 2 | """A simple cache object keyed by the field name, containing a number of 3 | cached instance loaders or preloaded caches. 4 | """ 5 | 6 | pass 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **~ 2 | **.pyc 3 | **._* 4 | **.DS_Store 5 | **.nfs* 6 | build* 7 | .sconf_temp 8 | .sconsign.dblite 9 | dist 10 | *.egg-info 11 | .eggs 12 | example/testdb.sqlite3 13 | environ/ 14 | examples/example_site/env 15 | **.pem 16 | src/ 17 | .idea/ -------------------------------------------------------------------------------- /djangomodelimport/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import ModelImporter # noqa 2 | from .forms import ImporterModelForm # noqa 3 | from .parsers import ( 4 | BaseImportParser, 5 | TablibCSVImportParser, 6 | TablibXLSXImportParser, 7 | ) # noqa 8 | from .resultset import ImportResultRow, ImportResultSet # noqa 9 | 10 | __version__ = "0.7.5" 11 | -------------------------------------------------------------------------------- /tests/djangoexample/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testapp.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | sys.path.append("../") 11 | execute_from_command_line(sys.argv) 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: required 3 | python: 3.6 4 | 5 | env: 6 | - DJANGO_SETTINGS_MODULE=example.testapp.settings 7 | 8 | install: 9 | - pip3 install -r example/requirements.txt 10 | 11 | script: 12 | - cd example && python manage.py test 13 | 14 | branches: 15 | only: 16 | - master 17 | 18 | notifications: 19 | email: false 20 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/urls.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.urls import path 3 | 4 | from .views import CitationCreateView, TestImportView 5 | 6 | urlpatterns = [ 7 | path(r"^admin/", admin.site.urls), 8 | path(r"^$", TestImportView.as_view(), name="start"), 9 | path(r"^create/$", CitationCreateView.as_view(), name="create"), 10 | ] 11 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E501 3 | max-line-length = 100 4 | 5 | [isort] 6 | profile = black 7 | known_django = django 8 | sections = FUTURE,STDLIB,THIRDPARTY,DJANGO,FIRSTPARTY,LOCALFOLDER 9 | extra_standard_library = tablib,yaml 10 | known_first_party = djangomodelimport 11 | multi_line_output = 3 12 | line_length = 100 13 | include_trailing_comma = True 14 | 15 | [mypy] 16 | show_error_codes = True 17 | warn_return_any = False 18 | check_untyped_defs = True 19 | ignore_missing_imports = True -------------------------------------------------------------------------------- /tests/djangoexample/testapp/forms.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | 3 | from djangomodelimport import JSONField 4 | 5 | from .models import Citation 6 | 7 | 8 | class TestImportForm(forms.Form): 9 | file_upload = forms.FileField() 10 | save = forms.BooleanField(required=False) 11 | 12 | 13 | class CitationForm(forms.ModelForm): 14 | metadata = JSONField() 15 | 16 | class Meta: 17 | fields = [ 18 | "name", 19 | "author", 20 | "metadata", 21 | ] 22 | model = Citation 23 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/templates/testapp/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Django Model Importer 7 | 8 | 9 | 10 | 11 |
12 |

Django Model Importer Example

13 | 14 | {% block content %}{% endblock %} 15 |
16 | 17 | 18 | -------------------------------------------------------------------------------- /djangomodelimport/utils.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import runtime_checkable, Protocol, Iterable 3 | 4 | from django.forms import Field 5 | 6 | 7 | @runtime_checkable 8 | class HasSource(Protocol): 9 | """Matches any object that has a source property""" 10 | 11 | source: str | Iterable[str] 12 | 13 | 14 | @dataclasses.dataclass 15 | class ImportFieldMetadata: 16 | """Describes how a field maps to headers during an import 17 | 18 | For `sources`, they are defined as a list of a group of headers that satisfy the field. 19 | """ 20 | 21 | field: Field 22 | help_text: str = "" 23 | sources: list[list[tuple[str, str]]] = dataclasses.field(default_factory=list) 24 | required: bool = False 25 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/templates/testapp/create.html: -------------------------------------------------------------------------------- 1 | {% extends "testapp/base.html" %} 2 | 3 | {% block content %} 4 |

Create Citation

5 |

This form lets you experiment with the ImporterModelForm Fields through a traditional form interface.

6 | 7 |
8 | {% csrf_token %} 9 | {{ form.as_p }} 10 | 11 |

12 |

13 | 14 | 15 |
16 | 17 | 22 | {% endblock %} 23 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.bracketPairColorization.enabled": true, 3 | "editor.guides.bracketPairs": "active", 4 | "files.trimFinalNewlines": true, 5 | "files.trimTrailingWhitespace": true, 6 | "indentRainbow.colors": [ 7 | "rgba(87, 106, 130, 0.05)", 8 | "rgba(87, 106, 130, 0.1)" 9 | ], 10 | "emmet.includeLanguages": { 11 | "django-html": "html" 12 | }, 13 | "editor.defaultFormatter": "esbenp.prettier-vscode", 14 | "python.formatting.provider": "black", 15 | "[python]": { 16 | "editor.defaultFormatter": "ms-python.python", 17 | "editor.codeActionsOnSave": { 18 | "source.organizeImports": true 19 | } 20 | }, 21 | "python.testing.cwd": "server/src", 22 | "python.testing.pytestEnabled": true, 23 | "window.autoDetectColorScheme": true, 24 | "editor.padding.bottom": 200, 25 | "editor.scrollBeyondLastLine": false, 26 | "editor.formatOnSave": true, 27 | "[django-html]": { 28 | "editor.formatOnSave": false 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "django-model-import" 3 | version = "0.7.7" 4 | description = "A Django library for importing CSVs and other structured data quickly using Django's ModelForm for validation and deserialisation into an instance." 5 | authors = ["Aidan Lister "] 6 | readme = "README.md" 7 | license = "MIT" 8 | homepage = "https://github.com/uptick/django-model-import" 9 | repository = "https://github.com/uptick/django-model-import" 10 | documentation = "https://github.com/uptick/django-model-import/tree/master/tests/djangoexample" 11 | packages = [ 12 | {include = "djangomodelimport"} 13 | ] 14 | 15 | [tool.poetry.dependencies] 16 | python = "^3.10" 17 | python-dateutil = "^2.7.0" 18 | tablib = "^3.0.0" 19 | django = ">=3.2.0,<6.0.0" 20 | 21 | [tool.poetry.group.dev.dependencies] 22 | pytest = "^7.2.0" 23 | pytest-django = "4.5.2" 24 | jsonfield = "^3.1.0" 25 | 26 | [build-system] 27 | requires = ["poetry-core>=1.0.0"] 28 | build-backend = "poetry.core.masonry.api" 29 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/models.py: -------------------------------------------------------------------------------- 1 | from jsonfield import JSONField 2 | 3 | from django.db import models 4 | 5 | 6 | class Author(models.Model): 7 | name = models.CharField(max_length=100) 8 | 9 | def __str__(self): 10 | return self.name 11 | 12 | 13 | class Book(models.Model): 14 | name = models.CharField(max_length=100) 15 | author = models.ForeignKey(Author, on_delete=models.PROTECT) 16 | 17 | def __str__(self): 18 | return self.name 19 | 20 | 21 | class Citation(models.Model): 22 | name = models.CharField(max_length=100) 23 | author = models.ForeignKey(Author, on_delete=models.PROTECT) 24 | 25 | # Add a JSON field (SQLite only supports TextField, but this should work with JSONField or HStoreField) 26 | # Setting blank=True is important here to make sure we're testing for: 27 | # this issue, https://github.com/uptick/django-model-import/issues/9 28 | metadata = JSONField() 29 | 30 | def __str__(self): 31 | return self.name 32 | 33 | 34 | class Contact(models.Model): 35 | name = models.CharField(max_length=100) 36 | email = models.EmailField(max_length=1000) 37 | mobile = models.CharField(max_length=50) 38 | address = models.TextField() 39 | 40 | def __str__(self): 41 | return self.name 42 | 43 | 44 | class Company(models.Model): 45 | name = models.CharField(max_length=100) 46 | primary_contact = models.ForeignKey(Contact, on_delete=models.PROTECT) 47 | 48 | def __str__(self): 49 | return self.name 50 | -------------------------------------------------------------------------------- /djangomodelimport/loaders.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable, Any, TypeVar 2 | 3 | from django.db.models import QuerySet 4 | 5 | T = TypeVar("T") 6 | 7 | 8 | class CachedInstanceLoader(dict): 9 | """A clever cache that queries the database for any missing objects. 10 | 11 | If there's an error, it's only raised against the first item that causes it, then it's 12 | cached for extra speed. 13 | """ 14 | 15 | def __init__( 16 | self, 17 | queryset: QuerySet[T], 18 | to_field: str | Iterable[str], 19 | *args: Any, 20 | **kwargs: Any, 21 | ): 22 | self.queryset = queryset 23 | self.model = queryset.model 24 | self.to_field = to_field 25 | self.multifield = isinstance(to_field, list) or isinstance(to_field, tuple) 26 | 27 | def __getitem__(self, item: str) -> T: 28 | # Attempt to get the currently cached value. 29 | value = super(CachedInstanceLoader, self).__getitem__(item) 30 | 31 | # If the cached value is an error, re-raise 32 | if isinstance(value, Exception): 33 | raise value 34 | 35 | return value 36 | 37 | def __missing__(self, value: str) -> T: 38 | if self.multifield: 39 | params = dict(zip(self.to_field, value)) 40 | else: 41 | params = {self.to_field: value} 42 | 43 | try: 44 | self[value] = inst = self.queryset.get(**params) 45 | except self.model.DoesNotExist as err: 46 | self[value] = err # Further warnings will be re-raised 47 | raise 48 | except self.model.MultipleObjectsReturned as err: 49 | self[value] = err # Further warnings will be re-raised 50 | raise 51 | return inst 52 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/views.py: -------------------------------------------------------------------------------- 1 | from django.urls import reverse 2 | from django.views.generic.edit import CreateView, FormView 3 | 4 | import djangomodelimport 5 | 6 | from .forms import CitationForm, TestImportForm 7 | from .importers import CitationImporter 8 | from .models import Citation 9 | 10 | 11 | class TestImportView(FormView): 12 | template_name = "testapp/testimport.html" 13 | form_class = TestImportForm 14 | 15 | def form_valid(self, form): 16 | thefile = form.cleaned_data["file_upload"] 17 | contents = thefile.read().decode("utf8", "ignore") 18 | 19 | parser = djangomodelimport.TablibCSVImportParser(CitationImporter) 20 | headers, rows = parser.parse(contents) 21 | 22 | importer = djangomodelimport.ModelImporter(CitationImporter) 23 | 24 | commit = form.cleaned_data["save"] 25 | importresult = importer.process(headers, rows, commit=commit) 26 | 27 | context = self.get_context_data(importresult=importresult) 28 | return self.render_to_response(context) 29 | 30 | def get_context_data(self, **kwargs): 31 | ctx = super().get_context_data(**kwargs) 32 | ctx.update( 33 | { 34 | "citations": Citation.objects.all(), 35 | } 36 | ) 37 | return ctx 38 | 39 | 40 | class CitationCreateView(CreateView): 41 | template_name = "testapp/create.html" 42 | form_class = CitationForm 43 | 44 | def get_context_data(self, **kwargs): 45 | ctx = super().get_context_data(**kwargs) 46 | ctx.update( 47 | { 48 | "citations": Citation.objects.all(), 49 | } 50 | ) 51 | return ctx 52 | 53 | def get_success_url(self): 54 | return reverse("create") 55 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/importers.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | 3 | import djangomodelimport 4 | 5 | from .models import Author, Book, Citation, Company, Contact 6 | 7 | 8 | class BookImporter(djangomodelimport.ImporterModelForm): 9 | name = forms.CharField() 10 | author = forms.ModelChoiceField(queryset=Author.objects.all(), to_field_name="name") 11 | 12 | class Meta: 13 | model = Book 14 | fields = ( 15 | "name", 16 | "author", 17 | ) 18 | 19 | 20 | class BookImporterWithCache(djangomodelimport.ImporterModelForm): 21 | name = forms.CharField() 22 | author = djangomodelimport.CachedChoiceField( 23 | queryset=Author.objects.all(), to_field="name" 24 | ) 25 | 26 | class Meta: 27 | model = Book 28 | fields = ( 29 | "name", 30 | "author", 31 | ) 32 | 33 | 34 | class CitationImporter(djangomodelimport.ImporterModelForm): 35 | name = forms.CharField() 36 | author = djangomodelimport.CachedChoiceField( 37 | queryset=Author.objects.all(), to_field="name" 38 | ) 39 | metadata = djangomodelimport.JSONField() 40 | 41 | class Meta: 42 | model = Citation 43 | fields = ( 44 | "name", 45 | "author", 46 | "metadata", 47 | ) 48 | 49 | 50 | class CompanyImporter(djangomodelimport.ImporterModelForm): 51 | primary_contact = djangomodelimport.FlatRelatedField( 52 | queryset=Contact.objects.all(), 53 | fields={ 54 | "contact_name": {"to_field": "name", "required": True}, 55 | "email": {"to_field": "email"}, 56 | "mobile": {"to_field": "mobile"}, 57 | "address": {"to_field": "address"}, 58 | }, 59 | ) 60 | 61 | class Meta: 62 | model = Company 63 | fields = ( 64 | "name", 65 | "primary_contact", 66 | ) 67 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | SECRET_KEY = "cheese" 4 | 5 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 6 | 7 | DATABASE_ENGINE = "sqlite3" 8 | 9 | DATABASES = { 10 | "default": { 11 | "ENGINE": "django.db.backends.sqlite3", 12 | "NAME": os.path.join(BASE_DIR, "testdb.sqlite3"), 13 | } 14 | } 15 | 16 | MIDDLEWARE = [ 17 | "django.middleware.security.SecurityMiddleware", 18 | "django.contrib.sessions.middleware.SessionMiddleware", 19 | "django.middleware.common.CommonMiddleware", 20 | "django.middleware.csrf.CsrfViewMiddleware", 21 | "django.contrib.auth.middleware.AuthenticationMiddleware", 22 | "django.contrib.messages.middleware.MessageMiddleware", 23 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 24 | ] 25 | 26 | INSTALLED_APPS = ( 27 | "django.contrib.admin", 28 | "django.contrib.staticfiles", 29 | "django.contrib.auth", 30 | "django.contrib.contenttypes", 31 | "django.contrib.sessions", 32 | "django.contrib.messages", 33 | "testapp", 34 | ) 35 | 36 | ROOT_URLCONF = "testapp.urls" 37 | 38 | DEBUG = True 39 | 40 | STATIC_ROOT = "./static/" 41 | 42 | STATIC_URL = "/static/" 43 | 44 | TEMPLATES = [ 45 | { 46 | "BACKEND": "django.template.backends.django.DjangoTemplates", 47 | "DIRS": [], 48 | "APP_DIRS": True, 49 | "OPTIONS": { 50 | "context_processors": [ 51 | "django.template.context_processors.debug", 52 | "django.template.context_processors.i18n", 53 | "django.template.context_processors.request", 54 | "django.template.context_processors.static", 55 | "django.contrib.auth.context_processors.auth", 56 | "django.contrib.messages.context_processors.messages", 57 | ], 58 | }, 59 | }, 60 | ] 61 | 62 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 63 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/templates/testapp/testimport.html: -------------------------------------------------------------------------------- 1 | {% extends "testapp/base.html" %} 2 | 3 | {% block content %} 4 |

Upload Citations

5 |
6 | {% csrf_token %} 7 | {{ form.as_p }} 8 | 9 | 10 |
11 | 12 | {% if importresult %} 13 | {% if importresult.get_errors %} 14 |
15 | {% with num_rows=importresult.get_errors|length %} 16 |
{{ num_rows }} row error{{ num_rows|pluralize }} must be resolved before continuing:
17 | {% endwith %} 18 | 29 |
30 | {% else %} 31 |
32 | Data has passed all checks. You may proceed with the import. 33 |
34 | {% endif %} 35 | 36 | 37 | 38 | 39 | 40 | 41 | {% for header in importresult.get_import_headers %} 42 | 43 | {% endfor %} 44 | 45 | 46 | {% for result in importresult.results %} 47 | 48 | 49 | {% if result.is_valid %} 50 | 56 | {% for value in result.get_instance_values %} 57 | 58 | {% endfor %} 59 | {% else %} 60 | 61 | 70 | {% endif %} 71 | 72 | {% endfor %} 73 |
row#status{{ header }}
{{ result.linenumber }} 51 | OKAY 52 | {% if result.instance.pk %} 53 | {{ result.instance.pk }} 54 | {% endif %} 55 | {{ value|default:"-" }}ERROR 62 | {% if result.errors %} 63 |
    64 | {% for field, error in result.errors %} 65 |
  • {{ field }}: {{ error }}
    • 66 | {% endfor %} 67 |
68 | {% endif %} 69 |
74 | {% endif %} 75 | 76 |

Saved Rows

77 | 82 | {% endblock %} 83 | -------------------------------------------------------------------------------- /djangomodelimport/widgets.py: -------------------------------------------------------------------------------- 1 | import operator 2 | 3 | from django import forms 4 | 5 | 6 | class DisplayChoiceWidget(forms.Widget): 7 | """This widget is helpful when the value being uploaded by the customer is the 8 | display choice, not the value. This widget will map the display choice back to the value. 9 | """ 10 | 11 | choices = None 12 | display_to_choice_map = None 13 | 14 | def flip_enum(self, choices): 15 | return dict(zip(dict(choices).values(), dict(choices).keys())) 16 | 17 | def __init__(self, choices, *args, **kwargs): 18 | self.choices = choices 19 | self.display_to_choice_map = self.flip_enum(choices) 20 | return super().__init__(choices, *args, **kwargs) 21 | 22 | def value_from_datadict(self, data, files, name): 23 | """ 24 | Given a dictionary of data and this widget's name, return the value 25 | of this widget or None if it's not provided. 26 | """ 27 | val = data.get(name) 28 | return self.display_to_choice_map.get(val) 29 | 30 | def format_value(self, value): 31 | return self.choices.get(value) 32 | 33 | 34 | class CompositeLookupWidget(forms.Widget): 35 | def __init__(self, source, *args, **kwargs): 36 | self.source = source 37 | super().__init__(*args, **kwargs) 38 | 39 | def value_from_datadict(self, data, files, name): 40 | getter = operator.itemgetter(*self.source) 41 | try: 42 | return getter(data) 43 | except KeyError: 44 | pass 45 | 46 | def value_omitted_from_data(self, data, files, name): 47 | for field_name in self.source: 48 | if field_name not in data: 49 | return True 50 | return False 51 | 52 | 53 | class NamedSourceWidget(forms.Widget): 54 | """This lets you override the column from which to import data.""" 55 | 56 | def __init__(self, source, *args, **kwargs): 57 | self.source = source 58 | super().__init__(*args, **kwargs) 59 | 60 | def value_from_datadict(self, data, files, name): 61 | return data.get(self.source, "") 62 | 63 | def value_omitted_from_data(self, data, files, name): 64 | return self.source not in data 65 | 66 | 67 | class JSONFieldWidget(forms.Widget): 68 | template_name = "django/forms/widgets/textarea.html" 69 | 70 | def render(self, name, value, attrs=None, renderer=None): 71 | return "" 72 | 73 | def value_omitted_from_data(self, data, files, name): 74 | return not any([key.startswith(name) for key in data.keys()]) 75 | 76 | def value_from_datadict(self, data, files, name): 77 | extra_fields = {} 78 | for f in data.keys(): 79 | if f.startswith(name): 80 | new_field = f[len(name) + 1 :] 81 | extra_fields[new_field] = data[f] 82 | return extra_fields 83 | -------------------------------------------------------------------------------- /djangomodelimport/parsers.py: -------------------------------------------------------------------------------- 1 | class BaseImportParser: 2 | def __init__(self, modelvalidator): 3 | """We provide the modelvalidator to get some Meta information about 4 | valid fields, and any soft headings. 5 | """ 6 | self.modelvalidator = modelvalidator 7 | 8 | def get_soft_headings(self): 9 | # Soft headings are used to provide similar heading suggestions 10 | # and look like this: {the field name: [list of other possible names] } 11 | # eg. 12 | # 13 | # soft_headings = { 14 | # 'type': ['Asset Type'], 15 | # } 16 | header_map = {} 17 | if hasattr(self.modelvalidator, "ImporterMeta"): 18 | if hasattr(self.modelvalidator.ImporterMeta, "soft_headings"): 19 | importer_softheadings = self.modelvalidator.ImporterMeta.soft_headings 20 | 21 | for renameto in importer_softheadings: # new column name 22 | for renamefrom in importer_softheadings[ 23 | renameto 24 | ]: # old column name 25 | header_map[renamefrom.lower()] = renameto.lower() 26 | return header_map 27 | 28 | def parse(self, data): 29 | """Parsers should return a tuple containing (headings, data) 30 | 31 | They should also take a dictionary of soft_headings which map 32 | similar names to actual headings. 33 | """ 34 | raise NotImplementedError 35 | 36 | 37 | class TablibBaseImportParser(BaseImportParser): 38 | def __init__(self, *args, **kwargs): 39 | # Inline import, so tablib is only grabbed if/when this Parser is instanciated. 40 | from tablib import Dataset 41 | 42 | self.dataset_class = Dataset 43 | super().__init__(*args, **kwargs) 44 | 45 | 46 | class TablibCSVImportParser(TablibBaseImportParser): 47 | def parse(self, data): 48 | dataset = self.dataset_class() 49 | dataset.csv = data 50 | 51 | header_map = self.get_soft_headings() 52 | 53 | # Make all our headings lowercase and sub in soft headings 54 | for col_id, header in enumerate( 55 | dataset.headers 56 | ): # replace it in headers if found 57 | header_name = header.strip().lower() 58 | dataset.headers[col_id] = header_name 59 | if header_name in header_map.keys(): 60 | dataset.headers[col_id] = header_map[header_name] 61 | 62 | return (dataset.headers, dataset.dict) 63 | 64 | 65 | class TablibXLSXImportParser(TablibBaseImportParser): 66 | def parse(self, data): 67 | dataset = self.dataset_class() 68 | # TODO: This does not currently work, as dataset.xlsx cannot be set. 69 | # http://docs.python-tablib.org/en/latest/api/#tablib.Dataset.xlsx 70 | # We can wait for it to be supported, or in the meantime, use this converter: 71 | # https://github.com/dilshod/xlsx2csv 72 | dataset.xlsx = data # CANNOT SET 73 | return (dataset.headers, dataset.dict) 74 | -------------------------------------------------------------------------------- /djangomodelimport/resultset.py: -------------------------------------------------------------------------------- 1 | class ImportResultSet: 2 | """Hold all imported results.""" 3 | 4 | results = None 5 | headers = None 6 | header_form = None 7 | created = 0 8 | updated = 0 9 | skipped = 0 10 | failed = 0 11 | 12 | def __init__(self, headers, header_form): 13 | self.results = [] 14 | self.headers = headers 15 | self.header_form = header_form 16 | 17 | def __repr__(self): 18 | i = len(self.results) 19 | j = len(self.get_errors()) 20 | k = len(self.get_warnings()) 21 | return f"ImportResultSet ({i} rows, {j} errors, {k} warnings)" 22 | 23 | def append(self, index, row, errors, instance, created, warnings=None): 24 | result_row = ImportResultRow( 25 | self, index, row, errors, instance, created, warnings 26 | ) 27 | self.results.append(result_row) 28 | return result_row 29 | 30 | def get_import_headers(self): 31 | return self.header_form.get_headers(self.headers) 32 | 33 | def get_results(self): 34 | return self.results 35 | 36 | def get_errors(self): 37 | return [ 38 | (row.linenumber, row.errors) for row in self.results if not row.is_valid() 39 | ] 40 | 41 | def get_warnings(self): 42 | return [(row.linenumber, row.warnings) for row in self.results if row.warnings] 43 | 44 | def set_counts( 45 | self, created=created, updated=updated, skipped=skipped, failed=failed 46 | ): 47 | self.created = created 48 | self.updated = updated 49 | self.skipped = skipped 50 | self.failed = failed 51 | 52 | def get_counts(self): 53 | return (self.created, self.updated, self.skipped, self.failed) 54 | 55 | 56 | class ImportResultRow: 57 | """Hold the result of an imported row.""" 58 | 59 | resultset = None 60 | linenumber = None 61 | row = None 62 | errors = None 63 | instance = None 64 | created = None 65 | 66 | def __init__( 67 | self, resultset, linenumber, row, errors, instance, created, warnings=None 68 | ): 69 | self.resultset = resultset 70 | self.linenumber = linenumber 71 | self.row = row 72 | self.errors = errors 73 | self.instance = instance 74 | self.created = created 75 | self.warnings = warnings or [] 76 | 77 | def __repr__(self): 78 | valid_str = "valid" if self.is_valid() else "invalid" 79 | mode_str = "create" if self.created else "update" 80 | res = self.get_instance_values() if self.is_valid() else self.errors 81 | sample = str([(k, v) for k, v in self.row.items()])[:100] 82 | return f"{self.linenumber}. [{valid_str}] [{mode_str}] ... {sample} ... {res}" 83 | 84 | def get_instance_values(self): 85 | return self.resultset.header_form.get_instance_values( 86 | self.instance, self.resultset.get_import_headers() 87 | ) 88 | 89 | def is_valid(self): 90 | return len(self.errors) == 0 91 | 92 | def get_errors(self): 93 | return self.errors 94 | -------------------------------------------------------------------------------- /djangomodelimport/formclassbuilder.py: -------------------------------------------------------------------------------- 1 | from functools import cached_property 2 | from typing import TypeVar, TYPE_CHECKING 3 | 4 | from django.db.models.fields import NOT_PROVIDED 5 | from django.forms import modelform_factory 6 | 7 | from .fields import JSONField, FlatRelatedField 8 | 9 | if TYPE_CHECKING: 10 | from . import ImporterModelForm # NOQA 11 | 12 | _ImporterForm = TypeVar("_ImporterForm", bound="ImporterModelForm") 13 | 14 | 15 | class FormClassBuilder: 16 | """Constructs instances of ImporterModelForm, taking headers into account.""" 17 | 18 | def __init__(self, modelimportformclass: _ImporterForm, headers: list[str]) -> None: 19 | self.headers = headers 20 | self.modelimportformclass = modelimportformclass 21 | self.model = modelimportformclass.Meta.model 22 | 23 | def build_update_form(self) -> _ImporterForm: 24 | return self._get_modelimport_form_class(fields=self.valid_fields) 25 | 26 | def build_create_form(self) -> _ImporterForm: 27 | # Combine valid & required fields; preserving order of valid fields. 28 | form_fields = self.valid_fields + list( 29 | set(self.required_fields) - set(self.valid_fields) 30 | ) 31 | return self._get_modelimport_form_class(fields=form_fields) 32 | 33 | @cached_property 34 | def valid_fields(self) -> list[str]: 35 | """Using the provided headers, prepare a list of valid 36 | fields for this importer. Preserves field ordering as defined by the headers. 37 | """ 38 | 39 | # Get the viable headers for the importer class 40 | form_field_metadata = self.modelimportformclass.get_field_metadata() 41 | 42 | # Check each header combination against the input headers to 43 | # see if they evaluate to a field 44 | valid_present_fields = set() 45 | for field_name, field_meta in form_field_metadata.items(): 46 | if isinstance(field_meta.field, (FlatRelatedField, JSONField)): 47 | # FlatRelatedField: these are a collection of other columns that build a relation on the fly. Always add. 48 | # JSONField: these are provided as FIELDNAME__SOME_DATA, so won't match directly. Just let the whole thing through. 49 | valid_present_fields.add(field_name) 50 | else: 51 | for source in field_meta.sources: 52 | if {key for key, _ in source} <= set(self.headers): 53 | valid_present_fields.add(field_name) 54 | 55 | return list(valid_present_fields) 56 | 57 | @cached_property 58 | def required_fields(self) -> list[str]: 59 | fields = self.model._meta.get_fields() 60 | required_fields = [] 61 | 62 | # Required means `blank` is False and `editable` is True. 63 | for f in fields: 64 | # Note - if the field doesn't have a `blank` attribute it is probably 65 | # a ManyToOne relation (reverse foreign key), which you probably want to ignore. 66 | if ( 67 | getattr(f, "blank", True) is False 68 | and getattr(f, "editable", True) is True 69 | and f.default is NOT_PROVIDED 70 | ): 71 | required_fields.append(f.name) 72 | return required_fields 73 | 74 | def _get_modelimport_form_class(self, fields) -> _ImporterForm: 75 | """Return a modelform for use with this data. 76 | 77 | We use a modelform_factory to dynamically limit the fields on the import, 78 | otherwise the absence of a value can be taken as false for boolean fields, 79 | where as we want the model's default value to kick in. 80 | """ 81 | klass = modelform_factory( 82 | self.model, 83 | form=self.modelimportformclass, 84 | fields=fields, 85 | ) 86 | # Remove fields altogether if they haven't been specified in the import (makes sense for updates). #houseofcards.. 87 | base_fields_to_del = set(klass.base_fields.keys()) - set(fields) 88 | for f in base_fields_to_del: 89 | del klass.base_fields[f] 90 | return klass 91 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11 on 2018-02-19 23:07 3 | from __future__ import unicode_literals 4 | 5 | import jsonfield.fields 6 | 7 | import django.db.models.deletion 8 | from django.db import migrations, models 9 | 10 | 11 | def create_initial_authors(apps, schema): 12 | Author = apps.get_model("testapp.Author") 13 | Author.objects.create( 14 | name="Fred Johnston", 15 | ) 16 | 17 | 18 | class Migration(migrations.Migration): 19 | 20 | initial = True 21 | 22 | dependencies = [] 23 | 24 | operations = [ 25 | migrations.CreateModel( 26 | name="Author", 27 | fields=[ 28 | ( 29 | "id", 30 | models.AutoField( 31 | auto_created=True, 32 | primary_key=True, 33 | serialize=False, 34 | verbose_name="ID", 35 | ), 36 | ), 37 | ("name", models.CharField(max_length=100)), 38 | ], 39 | ), 40 | migrations.CreateModel( 41 | name="Book", 42 | fields=[ 43 | ( 44 | "id", 45 | models.AutoField( 46 | auto_created=True, 47 | primary_key=True, 48 | serialize=False, 49 | verbose_name="ID", 50 | ), 51 | ), 52 | ("name", models.CharField(max_length=100)), 53 | ( 54 | "author", 55 | models.ForeignKey( 56 | on_delete=django.db.models.deletion.PROTECT, to="testapp.Author" 57 | ), 58 | ), 59 | ], 60 | ), 61 | migrations.CreateModel( 62 | name="Citation", 63 | fields=[ 64 | ( 65 | "id", 66 | models.AutoField( 67 | auto_created=True, 68 | primary_key=True, 69 | serialize=False, 70 | verbose_name="ID", 71 | ), 72 | ), 73 | ("name", models.CharField(max_length=100)), 74 | ("metadata", jsonfield.fields.JSONField()), 75 | ( 76 | "author", 77 | models.ForeignKey( 78 | on_delete=django.db.models.deletion.PROTECT, to="testapp.Author" 79 | ), 80 | ), 81 | ], 82 | ), 83 | migrations.CreateModel( 84 | name="Contact", 85 | fields=[ 86 | ( 87 | "id", 88 | models.AutoField( 89 | auto_created=True, 90 | primary_key=True, 91 | serialize=False, 92 | verbose_name="ID", 93 | ), 94 | ), 95 | ("name", models.CharField(max_length=100)), 96 | ("email", models.EmailField(max_length=1000)), 97 | ("mobile", models.CharField(max_length=50)), 98 | ("address", models.TextField()), 99 | ], 100 | ), 101 | migrations.CreateModel( 102 | name="Company", 103 | fields=[ 104 | ( 105 | "id", 106 | models.AutoField( 107 | auto_created=True, 108 | primary_key=True, 109 | serialize=False, 110 | verbose_name="ID", 111 | ), 112 | ), 113 | ("name", models.CharField(max_length=100)), 114 | ( 115 | "primary_contact", 116 | models.ForeignKey( 117 | on_delete=django.db.models.deletion.PROTECT, 118 | to="testapp.Contact", 119 | ), 120 | ), 121 | ], 122 | ), 123 | migrations.RunPython(code=create_initial_authors), 124 | ] 125 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # django-model-import 2 | 3 | [![PyPI version](https://badge.fury.io/py/django-model-import.svg)](https://badge.fury.io/py/django-model-import) 4 | 5 | Django Model Import is a light weight CSV importer built for speed. 6 | 7 | It uses a standard Django `ModelForm` to parse each row, giving you a familiar API to work with 8 | for data validation and model instantiation. In most cases, if you already have a `ModelForm` 9 | for the `ContentType` you are importing you do not need to create an import specific form. 10 | 11 | To present feedback to the end-user running the import you can easily generate a preview 12 | of the imported data by toggling the `commit` parameter. 13 | 14 | It also provides some import optimized fields for ForeignKey's, allowing preloading all 15 | possible values, or caching each lookup as it occurs, or looking up a model where multiple 16 | fields are needed to uniquely identify a resource. 17 | 18 | 19 | ## Installation 20 | 21 | ```bash 22 | poetry add django-model-import 23 | ``` 24 | 25 | 26 | ## Quickstart 27 | 28 | ```python 29 | import djangomodelimport 30 | 31 | class BookImporter(djangomodelimport.ImporterModelForm): 32 | name = forms.CharField() 33 | author = CachedChoiceField(queryset=Author.objects.all(), to_field='name') 34 | 35 | class Meta: 36 | model = Book 37 | fields = ( 38 | 'name', 39 | 'author', 40 | ) 41 | 42 | with default_storage.open('books.csv', 'rb') as fh: 43 | data = fh.read().decode("utf-8") 44 | 45 | # Use tablib 46 | parser = djangomodelimport.TablibCSVImportParser(BookImporter) 47 | headers, rows = parser.parse(data) 48 | 49 | # Process 50 | importer = djangomodelimport.ModelImporter(BookImporter) 51 | preview = importer.process(headers, rows, commit=False) 52 | errors = preview.get_errors() 53 | 54 | if errors: 55 | print(errors) 56 | 57 | importresult = importer.process(headers, rows, commit=True) 58 | for result in importresult.get_results(): 59 | print(result.instance) 60 | ``` 61 | 62 | 63 | ## Composite key lookups 64 | 65 | Often a relationship cannot be referenced via a single unique string. For this we can use 66 | a `CachedChoiceField` with a `CompositeLookupWidget`. The widget looks for the values 67 | under the `type` and `variant` columns in the source CSV, and does a unique lookup 68 | with the field names specified in `to_field`, e.g. `queryset.get(type__name=type, name=variant)`. 69 | 70 | The results of each `get` are cached internally for the remainder of the import minimising 71 | any database access. 72 | 73 | ```python 74 | class AssetImporter(ImporterModelForm): 75 | site = djangomodelimport.CachedChoiceField(queryset=Site.objects.active(), to_field='ref') 76 | type = djangomodelimport.CachedChoiceField(queryset=AssetType.objects.filter(is_active=True), to_field='name') 77 | type_variant = djangomodelimport.CachedChoiceField( 78 | queryset=InspectionItemTypeVariant.objects.filter(is_active=True), 79 | required=False, 80 | widget=djangomodelimport.CompositeLookupWidget(source=('type', 'variant')), 81 | to_field=('type__name', 'name'), 82 | ) 83 | contractor = djangomodelimport.CachedChoiceField(queryset=Contractor.objects.active(), to_field='name') 84 | ``` 85 | 86 | 87 | ## Flat related fields 88 | 89 | Often you'll have a OneToOneField or just a ForeignKey to another model, but you want to be able to 90 | create/update that other model via this one. You can flatten all of the related model's fields onto 91 | this importer using `FlatRelatedField`. 92 | 93 | ```python 94 | class ClientImporter(ImporterModelForm): 95 | primary_contact = FlatRelatedField( 96 | queryset=ContactDetails.objects.all(), 97 | fields={ 98 | 'contact_name': {'to_field': 'name', 'required': True}, 99 | 'email': {'to_field': 'email'}, 100 | 'email_cc': {'to_field': 'email_cc'}, 101 | 'mobile': {'to_field': 'mobile'}, 102 | 'phone_bh': {'to_field': 'phone_bh'}, 103 | 'phone_ah': {'to_field': 'phone_ah'}, 104 | 'fax': {'to_field': 'fax'}, 105 | }, 106 | ) 107 | 108 | class Meta: 109 | model = Client 110 | fields = ( 111 | 'name', 112 | 'ref', 113 | 'is_active', 114 | 'account', 115 | 116 | 'primary_contact', 117 | ) 118 | ``` 119 | 120 | ## Tests 121 | Run tests with `python example/manage.py test testapp` 122 | -------------------------------------------------------------------------------- /djangomodelimport/forms.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from functools import partial 3 | 4 | from django import forms 5 | from django.core.exceptions import NON_FIELD_ERRORS 6 | 7 | from .fields import FlatRelatedField, SourceFieldSwitcher 8 | from .magic import ( 9 | CachedChoiceFieldFormMixin, 10 | FlatRelatedFieldFormMixin, 11 | JSONFieldFormMixin, 12 | SourceFieldSwitcherMixin, 13 | ) 14 | from .utils import HasSource, ImportFieldMetadata 15 | from .widgets import CompositeLookupWidget 16 | 17 | 18 | class ImporterModelForm( 19 | SourceFieldSwitcherMixin, 20 | JSONFieldFormMixin, 21 | FlatRelatedFieldFormMixin, 22 | CachedChoiceFieldFormMixin, 23 | forms.ModelForm, 24 | ): 25 | """Extends the ModelForm to prime our caches and tweaks the validation 26 | routines to ensure we are not doing too many queries with our cached fields. 27 | """ 28 | 29 | def __init__(self, data, caches, author=None, *args, **kwargs) -> None: 30 | self.caches = caches 31 | self.author = author 32 | self._warnings = defaultdict(list) 33 | super().__init__(data, *args, **kwargs) 34 | 35 | def add_warning(self, field: str, warning: str) -> None: 36 | # Mimic django form behaviour for errors 37 | if not field: 38 | field = NON_FIELD_ERRORS 39 | 40 | self._warnings[field].append(warning) 41 | 42 | @property 43 | def warnings(self) -> dict[str, list[str]]: 44 | return dict(self._warnings) 45 | 46 | # This improves preview performance but eliminates validation on uniqueness constraints 47 | # def validate_unique(self): 48 | # pass 49 | 50 | @classmethod 51 | def get_available_headers(cls) -> list[tuple[str, str]]: 52 | """Returns a list of headers available on the import form 53 | 54 | Returned as a tuple of (key, label) 55 | """ 56 | fields = cls.get_field_metadata() 57 | 58 | header_set = {"id"} 59 | headers = [("id", "ID")] 60 | 61 | for field in fields.values(): 62 | for sources in field.sources: 63 | for header, label in sources: 64 | if header not in header_set: 65 | header_set.add(header) 66 | headers.append((header, label)) 67 | 68 | return headers 69 | 70 | @classmethod 71 | def get_field_metadata(cls) -> dict[str, ImportFieldMetadata]: 72 | """Generate a dict of available fields for the ImporterClass""" 73 | # 1) Evaluate Field type: 74 | # - SourceFieldSwitcher: these are a collection of different ways to find a related object. 75 | # - FlatRelatedField: these are a collection of other columns that build a relation on the fly. 76 | 77 | # 2) For each field, check its widget to see what headers it can source from 78 | # - Anything using NamedSourceWidget or has a "source" attr: 79 | # these lookup columns might not match the form field. 80 | # - Anything using CompositeLookupWidget: 81 | # these lookup columns might not always mention the form field target. 82 | # - Fields defined as attributes on the importer, 83 | # but not listed as form fields (eg because they're used for postprocessing). 84 | 85 | # Gather help_texts and verbose_names from the model and importer class 86 | help_texts = getattr(cls.Meta, "help_texts", {}) 87 | model_fields = { 88 | field.name: { 89 | "label": field.verbose_name.title(), 90 | "help_text": field.help_text, 91 | } 92 | for field in cls.Meta.model._meta.fields 93 | } 94 | 95 | for key, value in model_fields.items(): 96 | if value["help_text"] and key not in help_texts: 97 | help_texts[key] = value["help_text"] 98 | 99 | def _get_headers(name, widget) -> list[tuple[str, str]]: 100 | """Evaluate the viable headers added by a field widget""" 101 | found_headers = [] 102 | 103 | match widget: 104 | case CompositeLookupWidget(source=sub_fields): 105 | # Collection of options for how a field is defined 106 | for new_sub_headers in map( 107 | partial( 108 | _get_headers, 109 | widget=None, 110 | ), 111 | sub_fields, 112 | ): 113 | found_headers.extend(new_sub_headers) 114 | case HasSource(source=str(source)): 115 | # Renames the header for a field 116 | # for example the "NamedSourceWidget" 117 | found_headers.extend(_get_headers(source, widget=None)) 118 | case HasSource(source=sources): 119 | # Renames the header for a field 120 | # for example the "NamedSourceWidget" 121 | for source in sources: 122 | found_headers.extend(_get_headers(source, widget=None)) 123 | case _: 124 | # Anything else 125 | found_headers.append( 126 | ( 127 | name, 128 | model_fields.get(name, {}).get( 129 | "label", name.replace("_", " ").title() 130 | ), 131 | ) 132 | ) 133 | 134 | return found_headers 135 | 136 | import_fields: dict[str, ImportFieldMetadata] = {} 137 | 138 | for field_name, field_instance in cls.base_fields.items(): 139 | # Get or create new ImportFieldMetadata 140 | field = ImportFieldMetadata( 141 | field=field_instance, 142 | required=field_instance.required, 143 | help_text=help_texts.get(field_name, ""), 144 | ) 145 | 146 | # Find the header sources for this field 147 | match field_instance: 148 | case SourceFieldSwitcher(fields=switch_fields): 149 | # Containers a collections of ways to assign this field 150 | for switch_field in switch_fields: 151 | field.sources.append( 152 | _get_headers(field_name, switch_field.widget) 153 | ) 154 | case FlatRelatedField(fields=related_fields): 155 | # Defines a way to create related objects from a set of headers 156 | temp_source = [] 157 | for new_fields in map( 158 | partial(_get_headers, widget=field_instance.widget), 159 | related_fields.keys(), 160 | ): 161 | temp_source.extend(new_fields) 162 | field.sources.append(temp_source) 163 | case _: 164 | fields = _get_headers(field_name, field_instance.widget) 165 | field.sources.append(fields) 166 | 167 | # Add the field to the result 168 | import_fields[field_name] = field 169 | 170 | return import_fields 171 | -------------------------------------------------------------------------------- /djangomodelimport/core.py: -------------------------------------------------------------------------------- 1 | from django.db import transaction 2 | 3 | from .caches import SimpleDictCache 4 | from .formclassbuilder import FormClassBuilder 5 | from .resultset import ImportResultSet 6 | 7 | 8 | class ModelImporter: 9 | """A base class which parses and processes a CSV import, and handles the priming of any required caches.""" 10 | 11 | def __init__(self, modelimportformclass) -> None: 12 | """ 13 | @param modelimportformclass The ImporterModelForm class (which extends a simple ModelForm) 14 | """ 15 | self.instances = [] 16 | self.errors = [] 17 | self.modelimportformclass = modelimportformclass 18 | self.model = modelimportformclass.Meta.model 19 | self.update_cache = None 20 | self.update_queryset = None 21 | 22 | def get_for_update(self, pk): 23 | return ( 24 | self.update_cache[pk] 25 | if self.update_cache 26 | else self.update_queryset.get(pk=pk) 27 | ) 28 | 29 | @transaction.atomic 30 | def process( 31 | self, 32 | headers, 33 | rows, 34 | commit=False, 35 | allow_update=True, 36 | allow_insert=True, 37 | limit_to_queryset=None, 38 | author=None, 39 | progress_logger=None, 40 | skip_func=None, 41 | resultset_cls=ImportResultSet, 42 | ): 43 | """Process the data. 44 | 45 | @param limit_to_queryset A queryset which limits the instances which can be updated, and creates a cache of the 46 | updatable records to improve update performance. 47 | """ 48 | # Set up a cache context which will be filled by the Cached fields 49 | caches = SimpleDictCache() 50 | 51 | # Set up an "update" cache to preload any objects which might be updated 52 | if allow_update: 53 | self.update_queryset = ( 54 | limit_to_queryset 55 | if limit_to_queryset is not None 56 | else self.model.objects.all() 57 | ) 58 | # We only build the update_cache if limit_to_queryset is provided, with the assumption that the dataset 59 | # is then not too big. This may not be a valid assumption. 60 | # @todo Could we be smarter about the update cache, e.g. iterate through the source row PKs 61 | self.update_cache = {} 62 | if limit_to_queryset is not None: 63 | for obj in self.update_queryset: 64 | self.update_cache[str(obj.id)] = obj 65 | 66 | formclassbuilder = FormClassBuilder(self.modelimportformclass, headers) 67 | 68 | # Create a Form for rows where we are doing an UPDATE (required fields only relevant if attempting to wipe them). 69 | ModelUpdateForm = formclassbuilder.build_update_form() 70 | 71 | # Create a Form for rows where doing an INSERT (includes required fields). 72 | ModelCreateForm = formclassbuilder.build_create_form() 73 | 74 | # Create form to pass context to the ImportResultSet 75 | # TODO: evaluate this, only added because of FlatRelatedField 76 | header_form = ModelCreateForm(data={}, caches={}, author=author) 77 | importresult = resultset_cls(headers=headers, header_form=header_form) 78 | 79 | sid = transaction.savepoint() 80 | 81 | # Start processing 82 | created = updated = skipped = failed = 0 83 | for i, row in enumerate(rows, start=1): 84 | errors = [] 85 | warnings = [] 86 | instance = None 87 | to_be_created = ( 88 | row.get("id", "") == "" 89 | ) # If ID is blank we are creating a new row, otherwise we are updating 90 | to_be_updated = not to_be_created 91 | to_be_skipped = skip_func(row) if skip_func else False 92 | import_form_class = ModelCreateForm if to_be_created else ModelUpdateForm 93 | 94 | # Evaluate skip first 95 | # So that the import doesn't die for no reason 96 | if to_be_skipped: 97 | skipped += 1 98 | continue 99 | 100 | if to_be_created and not allow_insert: 101 | errors = [("id", ["Creating new rows is not permitted"])] 102 | importresult.append(i, row, errors, instance, to_be_created) 103 | continue 104 | 105 | if to_be_updated and not allow_update: 106 | errors = [("id", ["Updating existing rows is not permitted"])] 107 | importresult.append(i, row, errors, instance, to_be_created) 108 | continue 109 | 110 | if to_be_updated: 111 | try: 112 | instance = self.get_for_update(row["id"]) 113 | except ValueError as e: 114 | # We cannot validate an id's format until we try to fetch it from the DB 115 | if "expected a number" in str(e): 116 | errors = [ 117 | ( 118 | "id", 119 | [ 120 | f'{self.model._meta.verbose_name.title()} {row["id"]} is an invalid format for an ID.' 121 | ], 122 | ) 123 | ] 124 | else: 125 | raise e 126 | except self.model.DoesNotExist: 127 | errors = [ 128 | ( 129 | "id", 130 | [ 131 | f'{self.model._meta.verbose_name.title()} {row["id"]} does not exist.' 132 | ], 133 | ) 134 | ] 135 | except KeyError: 136 | errors = [ 137 | ( 138 | "id", 139 | [ 140 | f'{self.model._meta.verbose_name.title()} {row["id"]} cannot be updated.' 141 | ], 142 | ) 143 | ] 144 | 145 | if not errors: 146 | form = import_form_class( 147 | row, caches=caches, instance=instance, author=author 148 | ) 149 | if form.is_valid(): 150 | try: 151 | with transaction.atomic(): 152 | instance = form.save(commit=commit) 153 | 154 | if to_be_created: 155 | created += 1 156 | if to_be_updated: 157 | updated += 1 158 | except Exception as err: 159 | errors = [(i, repr(err))] 160 | 161 | else: 162 | # TODO: Filter out errors associated with FlatRelatedField 163 | errors = list(form.errors.items()) 164 | 165 | warnings = list(form.warnings.items()) 166 | 167 | if not instance or not instance.pk or errors: 168 | failed += 1 169 | 170 | result_row = importresult.append( 171 | i, row, errors, instance, to_be_created, warnings 172 | ) 173 | if progress_logger: 174 | progress_logger(result_row) 175 | 176 | if commit: 177 | transaction.savepoint_commit(sid) 178 | else: 179 | transaction.savepoint_rollback(sid) 180 | 181 | importresult.set_counts( 182 | created=created, updated=updated, skipped=skipped, failed=failed 183 | ) 184 | return importresult 185 | -------------------------------------------------------------------------------- /djangomodelimport/magic.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import ValidationError 2 | from django.forms import FileField 3 | 4 | from .fields import ( 5 | CachedChoiceField, 6 | FlatRelatedField, 7 | JSONField, 8 | SourceFieldSwitcher, 9 | UseCacheMixin, 10 | ) 11 | from .loaders import CachedInstanceLoader 12 | from .widgets import CompositeLookupWidget, NamedSourceWidget 13 | 14 | """ These mixins hold all the code that relates to our special fields (flat related, json, cached choice) 15 | that just doesn't work without access to the form instance. """ 16 | 17 | 18 | class FlatRelatedFieldFormMixin: 19 | def __init__(self, data, *args, **kwargs): 20 | super().__init__(data, *args, **kwargs) 21 | self.flat_related_mapping = {} 22 | flat_related = {} 23 | 24 | for field, fieldinstance in self.fields.items(): 25 | # For each FlatRelatedField, save a mapping back to the field. 26 | if isinstance(fieldinstance, FlatRelatedField): 27 | flat_related[field] = {} 28 | for f in fieldinstance.fields.keys(): 29 | self.flat_related_mapping[f] = field 30 | 31 | # Tinker with data to combine flat fields into related objects. 32 | new_data = self.data.copy() 33 | for field, value in self.data.items(): 34 | if field in self.flat_related_mapping: 35 | flat_related[self.flat_related_mapping[field]][field] = value 36 | del new_data[field] 37 | 38 | self.flat_data = self.data 39 | self.data = new_data 40 | 41 | for field, values in flat_related.items(): 42 | mapped_values = dict( 43 | (self.fields[field].fields[k]["to_field"], v) for k, v in values.items() 44 | ) 45 | # Get or create the related instance. 46 | if getattr(self.instance, field + "_id") is None: 47 | instance = self.fields[field].model(**mapped_values) 48 | else: 49 | instance = getattr(self.instance, field) 50 | for attr, value in mapped_values.items(): 51 | setattr(instance, attr, value) 52 | 53 | instance.save() # NOTE: This gets fired during preview, but that's ok, since we wrap previews in a big rollback transaction. 54 | self.data[field] = instance 55 | 56 | def get_headers(self, given_headers=None): 57 | headers = [] 58 | for field, fieldinstance in self.fields.items(): 59 | if isinstance(fieldinstance, FlatRelatedField): 60 | headers.extend( 61 | f 62 | for f in fieldinstance.fields.keys() 63 | if given_headers is None or f in given_headers 64 | ) 65 | else: 66 | headers.append(field) 67 | return headers 68 | 69 | def get_instance_values(self, instance, headers): 70 | instance_values = [] 71 | for header in headers: 72 | if header in self.flat_related_mapping: 73 | rel_field_name = self.flat_related_mapping[header] 74 | rel = getattr(instance, rel_field_name) 75 | instance_values.append( 76 | getattr(rel, self.fields[rel_field_name].fields[header]["to_field"]) 77 | ) 78 | else: 79 | try: 80 | instance_values.append(getattr(instance, header)) 81 | except ValueError: 82 | instance_values.append( 83 | "" 84 | ) # trying to access an m2m is not allowed before it has been saved 85 | except AttributeError: 86 | instance_values.append( 87 | "" 88 | ) # trying to access a field that doesn't exist on the model definition, should we check for the field in _meta.exclude? 89 | return instance_values 90 | 91 | # TODO: 92 | # def full_clean(self): 93 | # """ Validate that required fields in FlatRelated fields have been provided. """ 94 | # super().full_clean() 95 | # for field, fieldinstance in self.fields.items(): 96 | # if isinstance(fieldinstance, FlatRelatedField): 97 | # for f in fieldinstance.fields: 98 | # import pdb; pdb.set_trace() 99 | # pass 100 | 101 | 102 | class CachedChoiceFieldFormMixin: 103 | def __init__(self, data, *args, **kwargs): 104 | super().__init__(data, *args, **kwargs) 105 | for field, fieldinstance in self.fields.items(): 106 | # For each CachedInstanceLoader, prime the cache. 107 | if isinstance(fieldinstance, UseCacheMixin): 108 | if field not in self.caches: 109 | self.caches[field] = CachedInstanceLoader( 110 | fieldinstance.queryset, fieldinstance.to_field 111 | ) 112 | fieldinstance.set_cache(self.caches[field]) 113 | 114 | def _get_validation_exclusions(self): 115 | """We need to exclude any CachedChoiceFields from validation, as this 116 | causes a m * n queries where m is the number of relations, n is rows. 117 | """ 118 | exclude = super()._get_validation_exclusions() 119 | for field, fieldinstance in self.fields.items(): 120 | if isinstance(fieldinstance, CachedChoiceField): 121 | exclude.add(field) 122 | return exclude 123 | 124 | 125 | class JSONFieldFormMixin: 126 | def _clean_fields(self): 127 | for name, field in self.fields.items(): 128 | # value_from_datadict() gets the data from the data dictionaries. 129 | # Each widget type knows how to retrieve its own data, because some 130 | # widgets split data over several HTML fields. 131 | if field.disabled: 132 | value = self.get_initial_for_field(field, name) 133 | else: 134 | value = field.widget.value_from_datadict( 135 | self.data, self.files, self.add_prefix(name) 136 | ) 137 | try: 138 | if isinstance(field, FileField): 139 | initial = self.get_initial_for_field(field, name) 140 | value = field.clean(value, initial) 141 | # PATCH 142 | if isinstance(field, JSONField): 143 | initial = getattr(self.instance, name) 144 | value = field.clean(value) 145 | value = dict(initial, **value) # this is the secret sauce. 146 | # ENDPATCH 147 | else: 148 | value = field.clean(value) 149 | self.cleaned_data[name] = value 150 | if hasattr(self, "clean_%s" % name): 151 | value = getattr(self, "clean_%s" % name)() 152 | self.cleaned_data[name] = value 153 | except ValidationError as e: 154 | self.add_error(name, e) 155 | 156 | 157 | class SourceFieldSwitcherMixin: 158 | def __init__(self, data, *args, **kwargs): 159 | """Swap out all `SourceFieldSwitcher` fields for actual fields.""" 160 | for field_name, field_class in self.__class__.base_fields.items(): 161 | if not isinstance(field_class, SourceFieldSwitcher): 162 | continue 163 | for actual_field in field_class.fields: 164 | if isinstance(actual_field.widget, NamedSourceWidget): 165 | lookup = {actual_field.widget.source} 166 | elif isinstance(actual_field.widget, CompositeLookupWidget): 167 | lookup = set(actual_field.widget.source) 168 | else: 169 | lookup = {field_name} 170 | if lookup < set(data.keys()): 171 | self.base_fields[field_name] = actual_field 172 | break 173 | 174 | super().__init__(data=data, *args, **kwargs) 175 | -------------------------------------------------------------------------------- /djangomodelimport/fields.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import re 4 | from typing import Any, Iterable 5 | 6 | from dateutil import parser 7 | from django import forms 8 | from django.db.models import QuerySet 9 | from django.forms import Field 10 | from django.forms.utils import from_current_timezone 11 | 12 | from .widgets import JSONFieldWidget 13 | 14 | 15 | class UseCacheMixin: 16 | instancecache = None 17 | 18 | def set_cache(self, cache): 19 | self.instancecache = cache 20 | 21 | 22 | class FlatRelatedField(forms.Field): 23 | """Will create the related object if it does not yet exist. 24 | 25 | All the magic happens in magic.py in FlatRelatedFieldFormMixin 26 | """ 27 | 28 | def __init__( 29 | self, 30 | queryset: QuerySet, 31 | fields: dict[str, str] = None, 32 | *args: Any, 33 | **kwargs: Any, 34 | ) -> None: 35 | self.queryset = queryset 36 | # TODO: If lookup key is provided, allow using it to look up value instead of only 37 | # retrieving it off the object itself. 38 | self.model = queryset.model 39 | self.fields = fields or {} 40 | # Required is False, because this check gets passed down to the fields on the related instance. 41 | return super().__init__(required=False, *args, **kwargs) 42 | 43 | 44 | class CachedChoiceField(UseCacheMixin, forms.Field): 45 | """Use a CachedChoiceField when you have a large table of choices, but 46 | expect the number of different values that occur to be relatively small. 47 | 48 | If you expect a larger number of different values, you might want to use a 49 | PreloadedChoiceField. 50 | """ 51 | 52 | def __init__( 53 | self, 54 | queryset: QuerySet, 55 | to_field: str | Iterable[str] = None, 56 | none_if_missing: Any = None, 57 | *args: Any, 58 | **kwargs: Any, 59 | ) -> None: 60 | self.queryset = queryset 61 | self.model = queryset.model 62 | self.to_field = to_field 63 | self.none_if_missing = none_if_missing or [] 64 | super().__init__(*args, **kwargs) 65 | 66 | def get_from_cache(self, value: Any) -> Any: 67 | return self.instancecache[value] 68 | 69 | def clean(self, value: Any) -> Any: 70 | value = super().clean(value) 71 | 72 | # Fast fail if no value provided 73 | if not value: 74 | return None 75 | 76 | # Composite lookups are fine to have blank values in them e.g. for a firstname/lastname 77 | # lookup, it's fine to have ('Jenny', ''). 78 | # However, in some situations we need some fields to be set to be able to do the lookup. 79 | # If they are missing then the lookup is blank. 80 | # @todo Think about whether this should be a validation error if self.required is True 81 | if self.none_if_missing: 82 | for field_pos in self.none_if_missing: 83 | if not value[field_pos]: 84 | return None 85 | 86 | # Try and get the value from the loader 87 | try: 88 | return self.get_from_cache(value) 89 | except self.model.DoesNotExist: 90 | raise forms.ValidationError( 91 | "No %s matching '%s'." % (self.model._meta.verbose_name.title(), value) 92 | ) 93 | except self.model.MultipleObjectsReturned: 94 | raise forms.ValidationError( 95 | "Multiple %s matching '%s'. Expected just one." 96 | % (self.model._meta.verbose_name_plural.title(), value) 97 | ) 98 | 99 | 100 | class PreloadedChoiceField(forms.Field): 101 | """This will load all the possible values for this relationship once, 102 | to avoid hitting the database for each relationship in the import. 103 | """ 104 | 105 | def clean(self, value: Any) -> Any: 106 | raise NotImplementedError 107 | 108 | 109 | class DateTimeParserField(forms.DateTimeField): 110 | """A DateTime parser field that does it's best effort to understand. 111 | 112 | Defaults to assuming little endian when there is ambiguity: 113 | - XX/XX/XX -> DD/MM/YY 114 | - XX/XX/XXXX -> DD/MM/YYYY 115 | 116 | Pass in `middle_endian=True` to get: 117 | - XX/XX/XX -> MM/DD/YY 118 | - XX/XX/XXXX -> MM/DD/YYYY 119 | 120 | If year is passed first, will always use big endian: 121 | - XXXX/XX/XX -> YYYY/MM/DD 122 | """ 123 | 124 | def __init__(self, middle_endian: bool = False, *args: Any, **kwargs: Any): 125 | self.middle_endian = middle_endian 126 | super().__init__(*args, **kwargs) 127 | 128 | def to_python(self, value: str) -> datetime.datetime: 129 | value = (value or "").strip() 130 | if value: 131 | try: 132 | dayfirst = ( 133 | not bool(re.match(r"^\d{4}.\d\d?.\d\d?", value)) 134 | and not self.middle_endian 135 | ) 136 | return from_current_timezone(parser.parse(value, dayfirst=dayfirst)) 137 | except (TypeError, ValueError, OverflowError): 138 | raise forms.ValidationError( 139 | self.error_messages["invalid"], code="invalid" 140 | ) 141 | 142 | else: 143 | return None 144 | 145 | 146 | class JSONField(forms.Field): 147 | """This lets you store any fields prefixed by the field name into a JSON blob. 148 | 149 | For example, adding a field: 150 | metadata = JSONField() 151 | 152 | When the row is submitted with data that looks like this: 153 | id name author metadata_rank metadata_score 154 | ----------------------------------------------- 155 | ding bob hello twenty 156 | 157 | This field will return a JSON blob that looks like: 158 | {rank: "hello", score: "twenty"} 159 | """ 160 | 161 | def __init__(self, **kwargs: Any) -> None: 162 | kwargs["widget"] = kwargs.get("widget", JSONFieldWidget) 163 | kwargs["required"] = False 164 | kwargs["initial"] = dict 165 | super().__init__(**kwargs) 166 | 167 | def validate_json( 168 | self, value: str | None, is_serialized: bool = False 169 | ) -> dict[str, Any]: 170 | # if empty 171 | if value is None or value == "" or value == "null": 172 | value = "{}" 173 | 174 | # ensure valid JSON 175 | try: 176 | # convert strings to dictionaries 177 | if isinstance(value, str): 178 | dictionary = json.loads(value) 179 | 180 | # if serialized field, deserialize values 181 | if is_serialized and isinstance(dictionary, dict): 182 | dictionary = dict( 183 | (k, json.loads(v)) for k, v in dictionary.items() 184 | ) # TODO: modify to use field's deserializer 185 | # if not a string we'll check at the next control if it's a dict 186 | else: 187 | dictionary = value 188 | except ValueError as e: 189 | raise forms.ValidationError(("Invalid JSON: {0}").format(e)) 190 | 191 | # ensure is a dictionary 192 | if not isinstance(dictionary, dict): 193 | raise forms.ValidationError( 194 | ("No lists or values allowed, only dictionaries") 195 | ) 196 | 197 | # convert any non string object into string 198 | for key, value in dictionary.items(): 199 | if isinstance(value, dict) or isinstance(value, list): 200 | dictionary[key] = json.dumps(value) 201 | if ( 202 | isinstance(value, bool) 203 | or isinstance(value, int) 204 | or isinstance(value, float) 205 | ): 206 | if not is_serialized: # Only convert if not from serializedfield 207 | dictionary[key] = str(value).lower() 208 | 209 | return dictionary 210 | 211 | def to_python(self, value: str) -> dict[str, Any]: 212 | return self.validate_json(value) 213 | 214 | def render(self, name: str, value: str, attrs: Any = None) -> Any: 215 | # return json representation of a meaningful value 216 | # doesn't show anything for None, empty strings or empty dictionaries 217 | if value and not isinstance(value, str): 218 | value = json.dumps(value, sort_keys=True, indent=4) 219 | return super().render(name, value, attrs) 220 | 221 | 222 | class SourceFieldSwitcher(forms.Field): 223 | fields = None 224 | 225 | def __init__(self, *fields: Field, **kwargs: Any) -> None: 226 | self.fields = fields 227 | super().__init__(**kwargs) 228 | -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. 2 | 3 | [[package]] 4 | name = "asgiref" 5 | version = "3.9.1" 6 | description = "ASGI specs, helper code, and adapters" 7 | optional = false 8 | python-versions = ">=3.9" 9 | groups = ["main", "dev"] 10 | files = [ 11 | {file = "asgiref-3.9.1-py3-none-any.whl", hash = "sha256:f3bba7092a48005b5f5bacd747d36ee4a5a61f4a269a6df590b43144355ebd2c"}, 12 | {file = "asgiref-3.9.1.tar.gz", hash = "sha256:a5ab6582236218e5ef1648f242fd9f10626cfd4de8dc377db215d5d5098e3142"}, 13 | ] 14 | 15 | [package.dependencies] 16 | typing_extensions = {version = ">=4", markers = "python_version < \"3.11\""} 17 | 18 | [package.extras] 19 | tests = ["mypy (>=1.14.0)", "pytest", "pytest-asyncio"] 20 | 21 | [[package]] 22 | name = "attrs" 23 | version = "22.2.0" 24 | description = "Classes Without Boilerplate" 25 | optional = false 26 | python-versions = ">=3.6" 27 | groups = ["dev"] 28 | files = [ 29 | {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, 30 | {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, 31 | ] 32 | 33 | [package.extras] 34 | cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] 35 | dev = ["attrs[docs,tests]"] 36 | docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] 37 | tests = ["attrs[tests-no-zope]", "zope.interface"] 38 | tests-no-zope = ["cloudpickle ; platform_python_implementation == \"CPython\"", "cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990) ; platform_python_implementation == \"CPython\"", "mypy (>=0.971,<0.990) ; platform_python_implementation == \"CPython\"", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version < \"3.11\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version < \"3.11\"", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] 39 | 40 | [[package]] 41 | name = "colorama" 42 | version = "0.4.6" 43 | description = "Cross-platform colored terminal text." 44 | optional = false 45 | python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" 46 | groups = ["dev"] 47 | markers = "sys_platform == \"win32\"" 48 | files = [ 49 | {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, 50 | {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, 51 | ] 52 | 53 | [[package]] 54 | name = "django" 55 | version = "4.1.7" 56 | description = "A high-level Python web framework that encourages rapid development and clean, pragmatic design." 57 | optional = false 58 | python-versions = ">=3.8" 59 | groups = ["main", "dev"] 60 | files = [ 61 | {file = "Django-4.1.7-py3-none-any.whl", hash = "sha256:f2f431e75adc40039ace496ad3b9f17227022e8b11566f4b363da44c7e44761e"}, 62 | {file = "Django-4.1.7.tar.gz", hash = "sha256:44f714b81c5f190d9d2ddad01a532fe502fa01c4cb8faf1d081f4264ed15dcd8"}, 63 | ] 64 | 65 | [package.dependencies] 66 | asgiref = ">=3.5.2,<4" 67 | sqlparse = ">=0.2.2" 68 | tzdata = {version = "*", markers = "sys_platform == \"win32\""} 69 | 70 | [package.extras] 71 | argon2 = ["argon2-cffi (>=19.1.0)"] 72 | bcrypt = ["bcrypt"] 73 | 74 | [[package]] 75 | name = "exceptiongroup" 76 | version = "1.1.0" 77 | description = "Backport of PEP 654 (exception groups)" 78 | optional = false 79 | python-versions = ">=3.7" 80 | groups = ["dev"] 81 | markers = "python_version == \"3.10\"" 82 | files = [ 83 | {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, 84 | {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, 85 | ] 86 | 87 | [package.extras] 88 | test = ["pytest (>=6)"] 89 | 90 | [[package]] 91 | name = "iniconfig" 92 | version = "2.0.0" 93 | description = "brain-dead simple config-ini parsing" 94 | optional = false 95 | python-versions = ">=3.7" 96 | groups = ["dev"] 97 | files = [ 98 | {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, 99 | {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, 100 | ] 101 | 102 | [[package]] 103 | name = "jsonfield" 104 | version = "3.1.0" 105 | description = "A reusable Django field that allows you to store validated JSON in your model." 106 | optional = false 107 | python-versions = ">=3.6" 108 | groups = ["dev"] 109 | files = [ 110 | {file = "jsonfield-3.1.0-py3-none-any.whl", hash = "sha256:df857811587f252b97bafba42e02805e70a398a7a47870bc6358a0308dd689ed"}, 111 | {file = "jsonfield-3.1.0.tar.gz", hash = "sha256:7e4e84597de21eeaeeaaa7cc5da08c61c48a9b64d0c446b2d71255d01812887a"}, 112 | ] 113 | 114 | [package.dependencies] 115 | Django = ">=2.2" 116 | 117 | [[package]] 118 | name = "packaging" 119 | version = "23.0" 120 | description = "Core utilities for Python packages" 121 | optional = false 122 | python-versions = ">=3.7" 123 | groups = ["dev"] 124 | files = [ 125 | {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, 126 | {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, 127 | ] 128 | 129 | [[package]] 130 | name = "pluggy" 131 | version = "1.0.0" 132 | description = "plugin and hook calling mechanisms for python" 133 | optional = false 134 | python-versions = ">=3.6" 135 | groups = ["dev"] 136 | files = [ 137 | {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, 138 | {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, 139 | ] 140 | 141 | [package.extras] 142 | dev = ["pre-commit", "tox"] 143 | testing = ["pytest", "pytest-benchmark"] 144 | 145 | [[package]] 146 | name = "pytest" 147 | version = "7.2.1" 148 | description = "pytest: simple powerful testing with Python" 149 | optional = false 150 | python-versions = ">=3.7" 151 | groups = ["dev"] 152 | files = [ 153 | {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, 154 | {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, 155 | ] 156 | 157 | [package.dependencies] 158 | attrs = ">=19.2.0" 159 | colorama = {version = "*", markers = "sys_platform == \"win32\""} 160 | exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} 161 | iniconfig = "*" 162 | packaging = "*" 163 | pluggy = ">=0.12,<2.0" 164 | tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} 165 | 166 | [package.extras] 167 | testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] 168 | 169 | [[package]] 170 | name = "pytest-django" 171 | version = "4.5.2" 172 | description = "A Django plugin for pytest." 173 | optional = false 174 | python-versions = ">=3.5" 175 | groups = ["dev"] 176 | files = [ 177 | {file = "pytest-django-4.5.2.tar.gz", hash = "sha256:d9076f759bb7c36939dbdd5ae6633c18edfc2902d1a69fdbefd2426b970ce6c2"}, 178 | {file = "pytest_django-4.5.2-py3-none-any.whl", hash = "sha256:c60834861933773109334fe5a53e83d1ef4828f2203a1d6a0fa9972f4f75ab3e"}, 179 | ] 180 | 181 | [package.dependencies] 182 | pytest = ">=5.4.0" 183 | 184 | [package.extras] 185 | docs = ["sphinx", "sphinx-rtd-theme"] 186 | testing = ["Django", "django-configurations (>=2.0)"] 187 | 188 | [[package]] 189 | name = "python-dateutil" 190 | version = "2.8.2" 191 | description = "Extensions to the standard Python datetime module" 192 | optional = false 193 | python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" 194 | groups = ["main"] 195 | files = [ 196 | {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, 197 | {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, 198 | ] 199 | 200 | [package.dependencies] 201 | six = ">=1.5" 202 | 203 | [[package]] 204 | name = "six" 205 | version = "1.16.0" 206 | description = "Python 2 and 3 compatibility utilities" 207 | optional = false 208 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" 209 | groups = ["main"] 210 | files = [ 211 | {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, 212 | {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, 213 | ] 214 | 215 | [[package]] 216 | name = "sqlparse" 217 | version = "0.4.3" 218 | description = "A non-validating SQL parser." 219 | optional = false 220 | python-versions = ">=3.5" 221 | groups = ["main", "dev"] 222 | files = [ 223 | {file = "sqlparse-0.4.3-py3-none-any.whl", hash = "sha256:0323c0ec29cd52bceabc1b4d9d579e311f3e4961b98d174201d5622a23b85e34"}, 224 | {file = "sqlparse-0.4.3.tar.gz", hash = "sha256:69ca804846bb114d2ec380e4360a8a340db83f0ccf3afceeb1404df028f57268"}, 225 | ] 226 | 227 | [[package]] 228 | name = "tablib" 229 | version = "3.3.0" 230 | description = "Format agnostic tabular data library (XLS, JSON, YAML, CSV)" 231 | optional = false 232 | python-versions = ">=3.7" 233 | groups = ["main"] 234 | files = [ 235 | {file = "tablib-3.3.0-py3-none-any.whl", hash = "sha256:f7f2e214a1a68577f2599927a8870495adac0f7f2673b1819130f2060e1914ab"}, 236 | {file = "tablib-3.3.0.tar.gz", hash = "sha256:11e02a6f81d256e0666877d8397972d10302307a54c04fd7157e92faf740cb10"}, 237 | ] 238 | 239 | [package.extras] 240 | all = ["markuppy", "odfpy", "openpyxl (>=2.6.0)", "pandas", "pyyaml", "tabulate", "xlrd", "xlwt"] 241 | cli = ["tabulate"] 242 | html = ["markuppy"] 243 | ods = ["odfpy"] 244 | pandas = ["pandas"] 245 | xls = ["xlrd", "xlwt"] 246 | xlsx = ["openpyxl (>=2.6.0)"] 247 | yaml = ["pyyaml"] 248 | 249 | [[package]] 250 | name = "tomli" 251 | version = "2.0.1" 252 | description = "A lil' TOML parser" 253 | optional = false 254 | python-versions = ">=3.7" 255 | groups = ["dev"] 256 | markers = "python_version == \"3.10\"" 257 | files = [ 258 | {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, 259 | {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, 260 | ] 261 | 262 | [[package]] 263 | name = "typing-extensions" 264 | version = "4.12.2" 265 | description = "Backported and Experimental Type Hints for Python 3.8+" 266 | optional = false 267 | python-versions = ">=3.8" 268 | groups = ["main", "dev"] 269 | markers = "python_version == \"3.10\"" 270 | files = [ 271 | {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, 272 | {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, 273 | ] 274 | 275 | [[package]] 276 | name = "tzdata" 277 | version = "2022.7" 278 | description = "Provider of IANA time zone data" 279 | optional = false 280 | python-versions = ">=2" 281 | groups = ["main", "dev"] 282 | markers = "sys_platform == \"win32\"" 283 | files = [ 284 | {file = "tzdata-2022.7-py2.py3-none-any.whl", hash = "sha256:2b88858b0e3120792a3c0635c23daf36a7d7eeeca657c323da299d2094402a0d"}, 285 | {file = "tzdata-2022.7.tar.gz", hash = "sha256:fe5f866eddd8b96e9fcba978f8e503c909b19ea7efda11e52e39494bad3a7bfa"}, 286 | ] 287 | 288 | [metadata] 289 | lock-version = "2.1" 290 | python-versions = "^3.10" 291 | content-hash = "680c716535877645c8d7707124f937ae3c75c5a6ff097076741849678ed350a4" 292 | -------------------------------------------------------------------------------- /tests/djangoexample/testapp/test_importer.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from testapp.importers import ( 4 | BookImporter, 5 | BookImporterWithCache, 6 | CitationImporter, 7 | CompanyImporter, 8 | ) 9 | from testapp.models import Author, Book, Citation, Company, Contact 10 | 11 | from django.test import TestCase 12 | 13 | from djangomodelimport import DateTimeParserField, ModelImporter, TablibCSVImportParser 14 | 15 | sample_csv_1_books = """id,name,author 16 | ,How to be awesome,Aidan Lister 17 | ,How not to be awesome,Bill 18 | """ 19 | 20 | sample_csv_2_books = """id,name,author 21 | 111,Howdy,Author Joe 22 | 333,Goody,Author Bill 23 | """ 24 | 25 | sample_csv_3_citations = """id,author,name,metadata_isbn,metadata_doi 26 | ,Fred Johnson,Starburst,ISBN333,doi:111 27 | ,Fred Johnson,Gattica,ISBN666,doi:222 28 | """ 29 | 30 | sample_csv_4_citations = """id,author,name,metadata_xxx,metadata_yyy,metadata_doi 31 | 10,Fred Johnson,Starburst,qqqq,www,valid_doi1 32 | 20,Fred Johnson,Gattica,aaa,bbb,valid_doi2 33 | """ 34 | 35 | sample_csv_5_books = """id,name,author 36 | ,How to be awesome,Aidan Lister 37 | ,How to be really awesome,Aidan Lister 38 | ,How to be the best,Aidan Lister 39 | ,How to be great,Aidan Lister 40 | ,How to be so good,Aidan Lister 41 | ,How to be better than that,Aidan Lister 42 | ,How not to be awesome,Bill 43 | """ 44 | 45 | sample_csv_6_companies = """id,name,contact_name,email,mobile,address 46 | ,Microsoft,Aidan,aidan@ms.com,0432 000 000,SomeAdress 47 | """ 48 | 49 | sample_csv_8_books = """id,name 50 | 801,How to be fine 51 | 802,How to be average 52 | 803,How to be the best 53 | 804,How to be awesome 54 | """ 55 | 56 | sample_csv_9_books = """id,name,author,skip 57 | ,How to be fine,Aidan Lister,true 58 | ,How to be average,Aidan Lister, 59 | ,How to be the best,Aidan Lister, 60 | ,How to be awesome,Aidan Lister,true 61 | """ 62 | 63 | 64 | class ImporterTests(TestCase): 65 | def setUp(self): 66 | pass 67 | 68 | def test_importer(self): 69 | Author.objects.create(name="Aidan Lister") 70 | Author.objects.create(name="Bill") 71 | 72 | parser = TablibCSVImportParser(BookImporter) 73 | headers, rows = parser.parse(sample_csv_1_books) 74 | 75 | importer = ModelImporter(BookImporter) 76 | preview = importer.process(headers, rows, commit=False) 77 | 78 | # Make sure there's no errors 79 | errors = preview.get_errors() 80 | self.assertEqual(errors, []) 81 | 82 | importresult = importer.process(headers, rows, commit=True) 83 | res = importresult.get_results() 84 | 85 | # Make sure we get two rows 86 | self.assertEqual(len(res), 2) 87 | self.assertEqual(res[0].instance.author.name, "Aidan Lister") 88 | 89 | def test_importer_no_insert(self): 90 | parser = TablibCSVImportParser(BookImporter) 91 | headers, rows = parser.parse(sample_csv_1_books) 92 | 93 | importer = ModelImporter(BookImporter) 94 | preview = importer.process(headers, rows, allow_insert=False, commit=False) 95 | 96 | # Make sure there's no errors 97 | errors = preview.get_errors() 98 | self.assertEqual(len(errors), 2) 99 | self.assertEqual( 100 | errors[0], (1, [("id", ["Creating new rows is not permitted"])]) 101 | ) 102 | 103 | def test_importer_no_update(self): 104 | a1 = Author.objects.create(name="Aidan Lister") 105 | a2 = Author.objects.create(name="Bill") 106 | 107 | Book.objects.create(id=111, name="Hello", author=a1) 108 | Book.objects.create(id=333, name="Goodbye", author=a2) 109 | 110 | parser = TablibCSVImportParser(BookImporter) 111 | headers, rows = parser.parse(sample_csv_2_books) 112 | 113 | importer = ModelImporter(BookImporter) 114 | preview = importer.process( 115 | headers, 116 | rows, 117 | allow_update=False, 118 | limit_to_queryset=Book.objects.all(), 119 | commit=False, 120 | ) 121 | 122 | # Make sure there's no errors 123 | errors = preview.get_errors() 124 | self.assertEqual(len(errors), 2) 125 | self.assertEqual( 126 | errors[0], (1, [("id", ["Updating existing rows is not permitted"])]) 127 | ) 128 | 129 | def test_importer_limited_queryset(self): 130 | a1 = Author.objects.create(name="Author Joe") 131 | a2 = Author.objects.create(name="Author Bill") 132 | 133 | b1 = Book.objects.create(id=111, name="Hello", author=a1) 134 | Book.objects.create(id=333, name="Goodbye", author=a2) 135 | 136 | parser = TablibCSVImportParser(BookImporter) 137 | headers, rows = parser.parse(sample_csv_2_books) 138 | 139 | importer = ModelImporter(BookImporter) 140 | preview = importer.process( 141 | headers, 142 | rows, 143 | allow_update=True, 144 | limit_to_queryset=Book.objects.filter(id=b1.id), 145 | commit=False, 146 | ) 147 | 148 | # Make sure there's no errors 149 | errors = preview.get_errors() 150 | self.assertEqual(len(errors), 1) 151 | self.assertEqual(errors[0], (2, [("id", ["Book 333 cannot be updated."])])) 152 | 153 | def test_required_fields_on_update(self): 154 | a1 = Author.objects.create(name="Aidan Lister") 155 | a2 = Author.objects.create(name="Maddi T") 156 | 157 | b1 = Book.objects.create(id=801, name="Hello b1", author=a1) 158 | Book.objects.create(id=802, name="Hello b2", author=a1) 159 | Book.objects.create(id=803, name="Hello b3", author=a2) 160 | b4 = Book.objects.create(id=804, name="Hello b4", author=a2) 161 | 162 | parser = TablibCSVImportParser(BookImporter) 163 | headers, rows = parser.parse(sample_csv_8_books) 164 | 165 | importer = ModelImporter(BookImporter) 166 | res = importer.process(headers, rows, allow_update=True, commit=True) 167 | 168 | # Make sure there's no errors 169 | errors = res.get_errors() 170 | self.assertEqual(len(errors), 0) 171 | 172 | # Check we updated properly 173 | b1.refresh_from_db() 174 | b4.refresh_from_db() 175 | self.assertEqual(b1.name, "How to be fine") 176 | self.assertEqual(b1.author.name, "Aidan Lister") 177 | self.assertEqual(b4.name, "How to be awesome") 178 | self.assertEqual(b4.author.name, "Maddi T") 179 | 180 | def test_skip_function(self): 181 | Author.objects.create(name="Aidan Lister") 182 | parser = TablibCSVImportParser(BookImporter) 183 | headers, rows = parser.parse(sample_csv_9_books) 184 | 185 | importer = ModelImporter(BookImporter) 186 | res = importer.process( 187 | headers, 188 | rows, 189 | allow_update=True, 190 | commit=True, 191 | skip_func=lambda row: row.get("skip") == "true", 192 | ) 193 | 194 | # Make sure there's no errors 195 | errors = res.get_errors() 196 | self.assertEqual(len(errors), 0) 197 | 198 | # Check two rows were skipped and two imported 199 | self.assertEqual(res.skipped, 2) 200 | self.assertEqual(res.created, 2) 201 | 202 | 203 | class CachedChoiceFieldTests(TestCase): 204 | def setUp(self): 205 | pass 206 | 207 | def test_import(self): 208 | Author.objects.create(name="Aidan Lister") 209 | Author.objects.create(name="Bill") 210 | 211 | parser = TablibCSVImportParser(BookImporterWithCache) 212 | headers, rows = parser.parse(sample_csv_5_books) 213 | 214 | importer = ModelImporter(BookImporterWithCache) 215 | 216 | # Check for only two queries (one to look up Bill, another to look up Aidan Lister) 217 | # Expected query log: 218 | # SAVEPOINT "s140735624082240_x2" 219 | # SAVEPOINT "s140735624082240_x3" 220 | # SELECT "testapp_author"."id", "testapp_author"."name" FROM "testapp_author" WHERE "testapp_author"."name" = 'Aidan Lister' 221 | # INSERT INTO "testapp_book" ("name", "author_id") VALUES ('How to be awesome', 2) 222 | # INSERT INTO "testapp_book" ("name", "author_id") VALUES ('How to be really awesome', 2) 223 | # INSERT INTO "testapp_book" ("name", "author_id") VALUES ('How to be the best', 2) 224 | # INSERT INTO "testapp_book" ("name", "author_id") VALUES ('How to be great', 2) 225 | # INSERT INTO "testapp_book" ("name", "author_id") VALUES ('How to be so good', 2) 226 | # INSERT INTO "testapp_book" ("name", "author_id") VALUES ('How to be better than that', 2) 227 | # SELECT "testapp_author"."id", "testapp_author"."name" FROM "testapp_author" WHERE "testapp_author"."name" = 'Bill' 228 | # INSERT INTO "testapp_book" ("name", "author_id") VALUES ('How not to be awesome', 3) 229 | # RELEASE SAVEPOINT "s140735624082240_x3" 230 | # RELEASE SAVEPOINT "s140735624082240_x2" 231 | with self.assertNumQueries(13): 232 | importresult = importer.process(headers, rows, commit=True) 233 | 234 | res = importresult.get_results() 235 | 236 | # Make sure there's no errors 237 | errors = importresult.get_errors() 238 | self.assertEqual(errors, []) 239 | 240 | # Make sure we get two rows 241 | self.assertEqual(len(res), 7) 242 | self.assertEqual(res[0].instance.author.name, "Aidan Lister") 243 | 244 | 245 | class JSONFieldTests(TestCase): 246 | def setUp(self): 247 | pass 248 | 249 | def test_import(self): 250 | Author.objects.get_or_create(name="Fred Johnson") 251 | 252 | parser = TablibCSVImportParser(CitationImporter) 253 | headers, rows = parser.parse(sample_csv_3_citations) 254 | 255 | importer = ModelImporter(CitationImporter) 256 | importresult = importer.process(headers, rows, commit=True) 257 | 258 | # Make sure there's no errors 259 | errors = importresult.get_errors() 260 | self.assertEqual(errors, []) 261 | 262 | res = importresult.get_results() 263 | 264 | # Make sure we get two rows 265 | expected_json = {"isbn": "ISBN333", "doi": "doi:111"} 266 | self.assertEqual(len(res), 2) 267 | self.assertEqual(res[0].instance.metadata, expected_json) 268 | 269 | # Really check it worked 270 | cite = Citation.objects.get(pk=res[0].instance.pk) 271 | self.assertEqual(cite.metadata, expected_json) 272 | 273 | def test_fields_get_merged(self): 274 | (author, created) = Author.objects.get_or_create(name="Fred Johnson") 275 | c1 = Citation.objects.create( 276 | id=10, 277 | author=author, 278 | name="Diff1", 279 | metadata={ 280 | "doi": "some doi", 281 | "isbn": "hello", 282 | }, 283 | ) 284 | c2 = Citation.objects.create( 285 | id=20, 286 | author=author, 287 | name="Diff2", 288 | metadata={ 289 | "doi": "another doi", 290 | "isbn": "mate", 291 | }, 292 | ) 293 | 294 | # id,author,name,metadata_xxx,metadata_yyy,metadata_doi 295 | # 10,Fred Johnson,Starburst,qqqq,www,valid_doi1 296 | # 20,Fred Johnson,Gattica,aaa,,valid_doi2 297 | parser = TablibCSVImportParser(CitationImporter) 298 | headers, rows = parser.parse(sample_csv_4_citations) 299 | 300 | importer = ModelImporter(CitationImporter) 301 | importresult = importer.process(headers, rows, commit=True) 302 | 303 | # Make sure there's no errors 304 | errors = importresult.get_errors() 305 | self.assertEqual(errors, []) 306 | 307 | # Check it worked 308 | c1.refresh_from_db() 309 | c1_expected = { 310 | "xxx": "qqqq", 311 | "yyy": "www", 312 | "doi": "valid_doi1", 313 | "isbn": "hello", 314 | } 315 | self.assertDictEqual(c1.metadata, c1_expected) 316 | 317 | c2.refresh_from_db() 318 | c2_expected = { 319 | "xxx": "aaa", 320 | "yyy": "bbb", 321 | "doi": "valid_doi2", 322 | "isbn": "mate", 323 | } 324 | self.assertDictEqual(c2.metadata, c2_expected) 325 | 326 | 327 | class FlatRelatedFieldTests(TestCase): 328 | def setUp(self): 329 | pass 330 | 331 | def test_import(self): 332 | parser = TablibCSVImportParser(CompanyImporter) 333 | headers, rows = parser.parse(sample_csv_6_companies) 334 | 335 | importer = ModelImporter(CompanyImporter) 336 | importresult = importer.process(headers, rows, commit=True) 337 | 338 | # Make sure there's no errors 339 | errors = importresult.get_errors() 340 | self.assertEqual(errors, []) 341 | 342 | org = Company.objects.all().first() 343 | self.assertEqual(org.name, "Microsoft") 344 | self.assertEqual(org.primary_contact.name, "Aidan") 345 | 346 | def test_update(self): 347 | contact = Contact.objects.create( 348 | name="Tapir", email="ziggur@t.com", mobile="5317707" 349 | ) 350 | company = Company.objects.create(name="Okapi", primary_contact=contact) 351 | 352 | headers = ["id", "contact_name", "email"] 353 | rows = [ 354 | { 355 | "id": company.id, 356 | "contact_name": "Foo Fighter Client", 357 | "email": "b@rrow.com", 358 | }, 359 | ] 360 | 361 | importer = ModelImporter(CompanyImporter) 362 | importresult = importer.process(headers, rows, commit=True) 363 | 364 | # Make sure there's no errors 365 | errors = importresult.get_errors() 366 | self.assertEqual(errors, []) 367 | 368 | company.refresh_from_db() 369 | self.assertEqual(company.primary_contact.name, "Foo Fighter Client") 370 | self.assertEqual(company.primary_contact.email, "b@rrow.com") 371 | self.assertEqual( 372 | company.primary_contact.mobile, "5317707" 373 | ) # This one should've stayed the same. 374 | 375 | 376 | class DateTimeParserFieldTests(TestCase): 377 | def setUp(self): 378 | self.ledtf = DateTimeParserField() # Little-endian 379 | self.medtf = DateTimeParserField(middle_endian=True) 380 | 381 | def test_little_endian_parsing(self): 382 | self.assertEqual( 383 | self.ledtf.to_python("01/02/03"), datetime.datetime(2003, 2, 1, 0, 0) 384 | ) 385 | self.assertEqual( 386 | self.ledtf.to_python("01/02/2003"), datetime.datetime(2003, 2, 1, 0, 0) 387 | ) 388 | 389 | def test_middle_endian_parsing(self): 390 | self.assertEqual( 391 | self.medtf.to_python("01/02/03"), datetime.datetime(2003, 1, 2, 0, 0) 392 | ) 393 | self.assertEqual( 394 | self.medtf.to_python("01/02/2003"), datetime.datetime(2003, 1, 2, 0, 0) 395 | ) 396 | 397 | def test_big_endian_parsing(self): 398 | self.assertEqual( 399 | self.ledtf.to_python("2001/02/03"), datetime.datetime(2001, 2, 3, 0, 0) 400 | ) 401 | self.assertEqual( 402 | self.medtf.to_python("2001/02/03"), datetime.datetime(2001, 2, 3, 0, 0) 403 | ) 404 | 405 | self.assertEqual( 406 | self.ledtf.to_python("2018-02-12 17:06:46"), 407 | datetime.datetime(2018, 2, 12, 17, 6, 46), 408 | ) 409 | self.assertEqual( 410 | self.medtf.to_python("2018-02-12 17:06:46"), 411 | datetime.datetime(2018, 2, 12, 17, 6, 46), 412 | ) 413 | --------------------------------------------------------------------------------