├── .coveragerc ├── .coveralls.yml ├── .github └── workflows │ └── django.yml ├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── coverage.rc ├── data_importer ├── __init__.py ├── admin.py ├── core │ ├── __init__.py │ ├── base.py │ ├── default_settings.py │ ├── descriptor.py │ └── exceptions.py ├── django_migrations │ ├── 0001_initial.py │ └── __init__.py ├── forms.py ├── importers │ ├── __init__.py │ ├── base.py │ ├── csv_importer.py │ ├── generic.py │ ├── xls_importer.py │ ├── xlsx_importer.py │ └── xml_importer.py ├── listeners.py ├── models.py ├── readers │ ├── __init__.py │ ├── csv_reader.py │ ├── xls_reader.py │ ├── xlsx_reader.py │ └── xml_reader.py ├── south_migrations │ ├── 0001_initial.py │ ├── 0002_auto__add_field_filehistory_owner.py │ ├── 0003_auto__add_field_filehistory_is_task__add_field_filehistory_status.py │ ├── 0004_auto__del_field_filehistory_content__add_field_filehistory_filename.py │ ├── 0005_auto__add_filehistorylog.py │ └── __init__.py ├── tasks.py ├── templates │ ├── data_importer.html │ └── my_upload.html └── views.py ├── docs ├── Makefile ├── make.bat └── source │ ├── conf.py │ ├── core.rst │ ├── core │ ├── base.rst │ ├── default_settings.rst │ ├── descriptor.rst │ └── exceptions.rst │ ├── forms.rst │ ├── importers.rst │ ├── importers │ ├── base.rst │ ├── csvimporter.rst │ ├── generic.rst │ ├── xlsimporter.rst │ ├── xlsximporter.rst │ └── xmlimporter.rst │ ├── index.rst │ ├── models.rst │ ├── readers │ ├── csvreader.rst │ ├── xlsreader.rst │ ├── xlsxreader.rst │ └── xmlreader.rst │ ├── readme.rst │ └── views.rst ├── example ├── __init__.py ├── django_test_settings.py ├── manage.py ├── models.py ├── templates │ └── index.html ├── urls.py └── views.py ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── data │ ├── Workbook3.csv │ ├── Workbook3.xlsx │ ├── invoice.csv │ ├── person_test.csv │ ├── ptbr_test.xls │ ├── ptbr_test.xlsx │ ├── ptbr_test_mac.csv │ ├── ptbr_test_win.csv │ ├── test.csv │ ├── test.xls │ ├── test.xlsx │ ├── test.xml │ ├── test_invalid_lines.xlsx │ ├── test_json_descriptor.json │ └── test_music.xml ├── test_base.py ├── test_base_model.py ├── test_csv_importer.py ├── test_descriptor.py ├── test_dict_fields.py ├── test_foreignkey.py ├── test_forms.py ├── test_generic_importer.py ├── test_models.py ├── test_settings.py ├── test_tasks.py ├── test_xls_importer.py ├── test_xlsx_importer.py └── test_xml_importer.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = data_importer 3 | omit = */migrations/*, */tests/*, */django_migrations/*, */south_migrations/* 4 | 5 | [report] 6 | # Regexes for lines to exclude from consideration 7 | exclude_lines = 8 | # Have to re-enable the standard pragma 9 | pragma: no cover 10 | 11 | # Don't complain about missing debug-only code: 12 | def __repr__ 13 | def __unicode__ 14 | if self\.debug 15 | import .* 16 | from .* 17 | @property 18 | @staticmethod 19 | @source.setter 20 | class Meta 21 | except ImportError 22 | 23 | # Don't complain if tests don't hit defensive assertion code: 24 | raise AssertionError 25 | raise NotImplementedError 26 | 27 | # Don't complain if non-runnable code isn't run: 28 | if 0: 29 | if __name__ == .__main__.: 30 | 31 | 32 | ignore_errors = True 33 | -------------------------------------------------------------------------------- /.coveralls.yml: -------------------------------------------------------------------------------- 1 | repo_token: pYmoUmt7U5eLpw7LijsqBrLXR7nKfNStt 2 | -------------------------------------------------------------------------------- /.github/workflows/django.yml: -------------------------------------------------------------------------------- 1 | name: Django CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | strategy: 14 | max-parallel: 2 15 | matrix: 16 | python-version: [3.7, 3.8, 3.9] 17 | django-version: [1.11, 2, 3] 18 | 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install django 26 | run: pip install django==${{ matrix.django-version }} 27 | - name: Run Tests 28 | run: | 29 | python setup.py test 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.c 3 | *.pyo 4 | dist/* 5 | example/media/* 6 | .eggs/ 7 | .vagrant 8 | *.pyc 9 | .coverage 10 | .tox/* 11 | *.egg-info/* 12 | cover/* 13 | docs/_build/* 14 | /media 15 | *.zip 16 | README.rst 17 | *.sqlite 18 | .cache/* 19 | htmlcov/* 20 | test.txt 21 | pytestdebug.log 22 | build/* 23 | .venv/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - 3.7 5 | 6 | before_install: 7 | - sudo apt-get install -y libxml2-dev libxslt-dev 8 | 9 | script: python setup.py test 10 | 11 | env: 12 | - DJANGO=1.8 13 | - DJANGO=1.9 14 | - DJANGO=1.10 15 | - DJANGO=1.11 16 | - DJANGO=2 17 | - DJANGO=3 18 | - DJANGO=4 19 | 20 | install: 21 | - export PYTHONPATH=$PWD; 22 | - export DJANGO_SETTINGS_MODULE=django_test_settings; 23 | - pip install -q Django==$DJANGO 24 | 25 | branches: 26 | only: 27 | - master 28 | 29 | after_success: 30 | - coveralls 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 1994-2004 The FreeBSD Project. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE FREEBSD PROJECT "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FREEBSD PROJECT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 10 | 11 | The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. 12 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Make content for developer 2 | help: 3 | @echo "dev Instal requirements" 4 | @echo "test Run suit test" 5 | @echo "coverage Run Coverage" 6 | @echo "clean Remove trash files" 7 | @echo "send_package Send Package to Pypi" 8 | @echo "p3 Create docker with py3" 9 | @echo "p2 Create docker with py2" 10 | @echo "update Update all local packages" 11 | 12 | 13 | dev: 14 | pip install -r requirements.txt 15 | 16 | update: 17 | pip freeze --local | grep -v '^\-e' | cut -d = -f 1 | xargs -n1 pip install -U 18 | pip freeze > requirements.txt 19 | 20 | test: 21 | PYTHONPATH=`pwd` py.test -s -x --nomigrations 22 | 23 | coverage: 24 | rm -rf htmlcov 25 | PYTHONPATH=`pwd` py.test --cov=data_importer --cov-report html 26 | 27 | send_package: 28 | python setup.py build bdist_wheel bdist sdist 29 | twine upload dist/ 30 | 31 | clean: 32 | find . -name '*.pyc' -delete 33 | rm -rf .tox 34 | rm -rf data_importer.egg-info 35 | rm -rf cover 36 | rm -rf README.rst 37 | python setup.py clean --all 38 | 39 | py3: 40 | docker-compose run -d --name py3 py3 41 | 42 | py2: 43 | docker-compose run -d --name py2 py2 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Django Data Importer 2 | ==================== 3 | 4 | [![Latest Version](http://img.shields.io/pypi/v/data-importer.svg)](https://pypi.python.org/pypi/data-importer) 5 | [![Coverage Status](https://coveralls.io/repos/valdergallo/data-importer/badge.png)](https://coveralls.io/r/valdergallo/data-importer) 6 | [![BSD License](http://img.shields.io/badge/license-BSD-yellow.svg)](http://opensource.org/licenses/BSD-3-Clause) 7 | [![PyPi downloads](https://img.shields.io/pypi/dm/data-importer.svg)](https://pypi.python.org/pypi/data-importer) 8 | 9 | 10 | **Django Data Importer** is a tool which allow you to transform easily a `CSV, XML, XLS and XLSX` file into a python object or a django model instance. It is based on the django-style declarative model. 11 | 12 | Features 13 | -------- 14 | 15 | * Support to Django Customer User 16 | * (beta) QuerysetToWorkbook 17 | * Ignore empty line 18 | * Accept custom clean_fields 19 | * Accept post_clean 20 | * Accept post_save 21 | * Accept post_save_all_lines 22 | * Accept pre_clean 23 | * Accept pre_commit 24 | * Accept save with transaction 25 | * Auto generate async importers 26 | * Auto Importer CSV 27 | * Auto Importer XLS 28 | * Auto Importer XLSX 29 | * Auto Importer XML 30 | * Check import status on Django Admin 31 | * Convert text values by default as unicode 32 | * Default FormView 33 | * Django Admin integration to download files in File History 34 | * Easy interface to create Importers 35 | * Easy interface to create Readers 36 | * File History integrated with FormView 37 | * GenericImporter to import files (CSV, XLS, XLSX, XML) 38 | * Get fields from Django Models 39 | * Ignore First Row 40 | * Integrated with Celery 41 | * Integrated with Django Models 42 | * Integrated with Django Models Validators 43 | * Open file to read 44 | * Read source as File, cStringIO, Text, FileField 45 | * Set starting_row 46 | * Set XLS/XLSX importer by sheet_index 47 | * Set XLS/XLSX importer by sheet_name 48 | * Support to user a JSON descriptor with Fields 49 | * Fields as OrderedDict with text position 50 | 51 | 52 | Installation 53 | ------------ 54 | 55 | Use either ``easy_install``: 56 | 57 | easy_install data-importer 58 | 59 | or ``pip``: 60 | 61 | pip install data-importer 62 | 63 | 64 | Settings 65 | -------- 66 | 67 | Customize data_importer decoders 68 | 69 | >**DATA_IMPORTER_EXCEL_DECODER**
70 | > Default value is cp1252 71 | 72 | >**DATA_IMPORTER_DECODER**
73 | > Default value is UTF-8 74 | 75 | 76 | Add support to South Migrations and Django Migrations 77 | 78 | 79 | ``` 80 | SOUTH_MIGRATION_MODULES = { 81 | 'data_importer': 'data_importer.south_migrations', 82 | } 83 | 84 | MIGRATION_MODULES = { 85 | 'data_importer': 'data_importer.django_migrations' 86 | } 87 | ``` 88 | 89 | 90 | Basic example 91 | ------------- 92 | 93 | Consider the following: 94 | 95 | ``` 96 | >>> from data_importer.importers import CSVImporter 97 | >>> class MyCSVImporterModel(CSVImporter): 98 | ... fields = ['name', 'age', 'length'] 99 | ... class Meta: 100 | ... delimiter = ";" 101 | ``` 102 | 103 | You declare a ``MyCSVImporterModel`` which will match to a CSV file like this: 104 | 105 | Anthony;27;1.75 106 | 107 | To import the file or any iterable object, just do: 108 | 109 | ``` 110 | >>> my_csv_list = MyCSVImporterModel(source="my_csv_file_name.csv") 111 | >>> row, first_line = my_csv_list.cleaned_data[0] 112 | >>> first_line['age'] 113 | 27 114 | ``` 115 | 116 | Without an explicit declaration, data and columns are matched in the same 117 | order: 118 | 119 | Anthony --> Column 0 --> Field 0 --> name 120 | 27 --> Column 1 --> Field 1 --> age 121 | 1.75 --> Column 2 --> Field 2 --> length 122 | 123 | 124 | Using Fields as Dict 125 | --------------------- 126 | 127 | You can use diferents ways to define the fields as dicts 128 | 129 | 130 | ``` 131 | >>> class TestMetaDict(XLSImporter): 132 | ... fields = OrderedDict(( 133 | ... ('business_place', 'A'), 134 | ... ('doc_number', 'b'), 135 | ... ('doc_data', 'C'), 136 | ... )) 137 | ``` 138 | 139 | or 140 | 141 | ``` 142 | >>> class TestMetaDict(XLSImporter): 143 | ... fields = OrderedDict(( 144 | ... ('business_place', 0), 145 | ... ('doc_number', 1), 146 | ... ('doc_data', 2), 147 | ... )) 148 | ``` 149 | 150 | or 151 | 152 | ``` 153 | >>> class TestMetaDict(XLSImporter): 154 | ... fields = OrderedDict(( 155 | ... ('business_place', '0'), 156 | ... ('doc_number', 1,) 157 | ... ('doc_data', 'C'), 158 | ... )) 159 | ``` 160 | 161 | Using declaration, data and columns are matched in the same 162 | order: 163 | 164 | New York --> Column 0 --> Field 0 --> business_place 165 | 664736 --> Column 1 --> Field 1 --> doc_number 166 | 2015-01-01 --> Column 2 --> Field 2 --> doc_data 167 | 168 | 169 | Django Model 170 | ------------ 171 | 172 | If you now want to interact with a django model, you just have to add a **Meta.model** option to the class meta. 173 | 174 | ``` 175 | >>> from django.db import models 176 | >>> class MyModel(models.Model): 177 | ... name = models.CharField(max_length=150) 178 | ... age = models.CharField(max_length=150) 179 | ... length = models.CharField(max_length=150) 180 | 181 | >>> from data_importer.importers import CSVImporter 182 | >>> from data_importer.model import MyModel 183 | >>> class MyCSVImporterModel(CSVImporter): 184 | ... class Meta: 185 | ... delimiter = ";" 186 | ... model = MyModel 187 | ``` 188 | 189 | That will automatically match to the following django model. 190 | 191 | *The django model should be imported in the model* 192 | 193 | >**delimiter**
194 | > define the delimiter of the csv file.
195 | > If you do not set one, the sniffer will try yo find one itself. 196 | 197 | >**ignore\_first_line**
198 | > Skip the first line if True. 199 | 200 | >**model**
201 | > If defined, the importer will create an instance of this model. 202 | 203 | >**raise_errors**
204 | > If set to True, an error in a imported line will stop the loading. 205 | 206 | >**exclude**
207 | > Exclude fields from list fields to import 208 | 209 | >**transaction**
210 | > Use transaction to save objects 211 | 212 | 213 | Django XML 214 | ------------ 215 | 216 | If you now want to interact with a django model, you just have to add a **Meta.model** option to the class meta. 217 | 218 | XML file example: 219 | 220 | ``` 221 | 222 | 223 | Rocky Balboa 224 | 40 225 | 1.77 226 | 227 | 228 | Chuck Norris 229 | 73 230 | 1.78 231 | 232 | 233 | 234 | >>> from django.db import models 235 | >>> class MyModel(models.Model): 236 | ... name = models.CharField(max_length=150) 237 | ... age = models.CharField(max_length=150) 238 | ... height = models.CharField(max_length=150) 239 | 240 | >>> from data_importer.importers import XMLImporter 241 | >>> from data_importer.model import MyModel 242 | >>> class MyCSVImporterModel(XMLImporter): 243 | ... root = 'file' 244 | ... class Meta: 245 | ... model = MyModel 246 | ``` 247 | 248 | That will automatically match to the following django model. 249 | 250 | *The django model should be imported in the model* 251 | 252 | > **model**
253 | > If defined, the importer will create an instance of this model. 254 | 255 | >**raise_errors**
256 | > If set to True, an error in a imported line will stop the loading. 257 | 258 | >**exclude**
259 | > Exclude fields from list fields to import 260 | 261 | >**transaction**
262 | > Use transaction to save objects 263 | 264 | 265 | Django XLS/XLSX 266 | ---------------- 267 | 268 | My XLS/XLSX file can be imported too 269 | 270 | 271 | | Header1 | Header2 | Header3 | Header4 272 | | ------- | ------- | ------- | ------- 273 | | Teste 1 | Teste 2 | Teste 3 | Teste 4 274 | | Teste 1 | Teste 2 | Teste 3 | Teste 4 275 | 276 | 277 | 278 | This is my model 279 | 280 | ``` 281 | >>> from django.db import models 282 | >>> class MyModel(models.Model): 283 | ... header1 = models.CharField(max_length=150) 284 | ... header2 = models.CharField(max_length=150) 285 | ... header3 = models.CharField(max_length=150) 286 | ... header4 = models.CharField(max_length=150) 287 | ``` 288 | 289 | This is my class 290 | ``` 291 | >>> from data_importer import XLSImporter 292 | >>> from data_importer.model import MyModel 293 | >>> class MyXLSImporterModel(XLSImporter): 294 | ... class Meta: 295 | ... model = MyModel 296 | ``` 297 | 298 | 299 | If you are using XLSX you will need use `XLSXImporter` to made same importer 300 | ``` 301 | >>> from data_importer import XLSXImporter 302 | >>> from data_importer.model import MyModel 303 | >>> class MyXLSXImporterModel(XLSXImporter): 304 | ... class Meta: 305 | ... model = MyModel 306 | ``` 307 | 308 | 309 | > **ignore\_first_line**
310 | > Skip the first line if True. 311 | 312 | >**model**
313 | > If defined, the importer will create an instance of this model. 314 | 315 | >**raise_errors**
316 | > If set to True, an error in a imported line will stop the loading. 317 | 318 | >**exclude**
319 | > Exclude fields from list fields to import 320 | 321 | >**transaction**
322 | > Use transaction to save objects 323 | 324 | 325 | Descriptor 326 | ---------- 327 | 328 | Using file descriptor to define fields for large models. 329 | 330 | 331 | import_test.json 332 | 333 | ``` 334 | { 335 | 'app_name': 'mytest.Contact', 336 | { 337 | // field name / name on import file or key index 338 | 'name': 'My Name', 339 | 'year': 'My Year', 340 | 'last': 3 341 | } 342 | } 343 | ``` 344 | 345 | model.py 346 | 347 | ``` 348 | class Contact(models.Model): 349 | name = models.CharField(max_length=50) 350 | year = models.CharField(max_length=10) 351 | laster = models.CharField(max_length=5) 352 | phone = models.CharField(max_length=5) 353 | address = models.CharField(max_length=5) 354 | state = models.CharField(max_length=5) 355 | ``` 356 | 357 | importer.py 358 | 359 | ``` 360 | class MyImpoter(BaseImpoter): 361 | class Meta: 362 | config_file = 'import_test.json' 363 | model = Contact 364 | delimiter = ',' 365 | ignore_first_line = True 366 | ``` 367 | 368 | content_file.csv 369 | 370 | ``` 371 | name,year,last 372 | Test,12,1 373 | Test2,13,2 374 | Test3,14,3 375 | ``` 376 | 377 | Default DataImporterForm 378 | ------------------------ 379 | 380 | `DataImporterForm` is one `django.views.generic.edit.FormView` 381 | to **save file** in `FileUpload` and parse content on success. 382 | 383 | Example 384 | ------- 385 | 386 | ``` 387 | class DataImporterCreateView(DataImporterForm): 388 | extra_context = {'title': 'Create Form Data Importer', 389 | 'template_file': 'myfile.csv' 390 | } 391 | importer = MyCSVImporterModel 392 | ``` 393 | 394 | TEST 395 | ---- 396 | 397 | Acentuation with XLS | Excel MAC 2011 | **OK** 398 | ----------------------- | ----------------- | -------- 399 | Acentuation with XLS | Excel WIN 2010 | **OK** 400 | Acentuation with XLSX | Excel MAC 2011 | **OK** 401 | Acentuation with XLSX | Excel WIN 2010 | **OK** 402 | Acentuation with CSV | Excel Win 2010 | **OK** 403 | 404 | 405 | Python | 3.4+ 406 | -------|------ 407 | Python | 2.7+ 408 | Django | 1.8+ 409 | -------------------------------------------------------------------------------- /coverage.rc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = data_importer 3 | omit = */migrations/*, example/* 4 | 5 | [report] 6 | # Regexes for lines to exclude from consideration 7 | exclude_lines = 8 | # Have to re-enable the standard pragma 9 | pragma: no cover 10 | 11 | # Don't complain about missing debug-only code: 12 | def __repr__ 13 | def __unicode__ 14 | if self.debug: 15 | if settings.DEBUG 16 | raise AssertionError 17 | raise NotImplementedError 18 | if self\.debug 19 | import .* 20 | from .* 21 | @property 22 | @staticmethod 23 | @source.setter 24 | class Meta 25 | 26 | # Don't complain if tests don't hit defensive assertion code: 27 | raise AssertionError 28 | raise NotImplementedError 29 | 30 | # Don't complain if non-runnable code isn't run: 31 | if 0: 32 | if __name__ == .__main__.: 33 | 34 | 35 | ignore_errors = True 36 | -------------------------------------------------------------------------------- /data_importer/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | __doc__ = "Data Importer" 3 | __version__ = "3.1.5" 4 | __author__ = "Valder Gallo " 5 | -------------------------------------------------------------------------------- /data_importer/admin.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from django.contrib import admin 3 | from data_importer.models import FileHistory 4 | 5 | 6 | class FileAdmin(admin.ModelAdmin): 7 | list_display = [ 8 | "compose_file_name", 9 | "updated_at", 10 | "owner", 11 | "active", 12 | "is_task", 13 | "status", 14 | "file_link", 15 | ] 16 | list_filter = ["is_task", "active", "status"] 17 | search_fields = ["file_upload"] 18 | 19 | 20 | admin.site.register(FileHistory, FileAdmin) 21 | -------------------------------------------------------------------------------- /data_importer/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valdergallo/data-importer/7d3f1d7209597484e3041768ae01c90b16bcc08f/data_importer/core/__init__.py -------------------------------------------------------------------------------- /data_importer/core/base.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | from . import default_settings 3 | 4 | DATA_IMPORTER_EXCEL_DECODER = default_settings.DATA_IMPORTER_EXCEL_DECODER 5 | DATA_IMPORTER_DECODER = default_settings.DATA_IMPORTER_DECODER 6 | 7 | 8 | def objclass2dict(objclass): 9 | """ 10 | Meta is a objclass on python 2.7 and no have __dict__ attribute. 11 | 12 | This method convert one objclass to one lazy dict without AttributeError 13 | """ 14 | 15 | class Dict(dict): 16 | def __init__(self, data=None): 17 | if data is None: 18 | data = {} 19 | super(Dict, self).__init__(data) 20 | self.__dict__ = dict(self.items()) 21 | 22 | def __getattr__(self, key): 23 | try: 24 | return self.__getattribute__(key) 25 | except AttributeError: 26 | return False 27 | 28 | obj_list = [i for i in dir(objclass) if not str(i).startswith("__")] 29 | obj_values = [] 30 | for objitem in obj_list: 31 | obj_values.append(getattr(objclass, objitem)) 32 | return Dict(zip(obj_list, obj_values)) 33 | 34 | 35 | def convert_alphabet_to_number(letters): 36 | letters = str(letters).lower() 37 | if letters.isdigit(): 38 | return int(letters) 39 | result = "" 40 | for letter in letters: 41 | number = ord(letter) - 96 42 | result += str(number) 43 | # -1 to get zero in list items 44 | return int(result) - 1 45 | 46 | 47 | def reduce_list(key_list, values_list): 48 | new_list = [] 49 | for key in key_list: 50 | try: 51 | v = values_list[key] 52 | except IndexError: 53 | continue 54 | new_list.append(v) 55 | return new_list 56 | -------------------------------------------------------------------------------- /data_importer/core/default_settings.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | try: 5 | from django.conf import settings 6 | except ImportError: 7 | settings = {} 8 | 9 | DATA_IMPORTER_TASK = ( 10 | False # If you need use task data_importer will need install celery 11 | ) 12 | DATA_IMPORTER_QUEUE = "DataImporter" # Celery Default Queue 13 | DATA_IMPORTER_TASK_LOCK_EXPIRE = 60 * 20 # Lock expires in 20 minutes 14 | 15 | DATA_IMPORTER_EXCEL_DECODER = ( 16 | hasattr(settings, "DATA_IMPORTER_EXCEL_DECODER") 17 | and settings.DATA_IMPORTER_EXCEL_DECODER 18 | or "cp1252" 19 | ) 20 | DATA_IMPORTER_DECODER = ( 21 | hasattr(settings, "DATA_IMPORTER_DECODER") 22 | and settings.DATA_IMPORTER_DECODER 23 | or "utf-8" 24 | ) 25 | -------------------------------------------------------------------------------- /data_importer/core/descriptor.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | import os 4 | from data_importer.core.exceptions import InvalidModel, InvalidDescriptor 5 | 6 | try: 7 | import json 8 | except ImportError: 9 | import simplejson as json 10 | 11 | 12 | class ReadDescriptor(object): 13 | def __init__(self, file_name=None, model_name=None): 14 | self.file_name = file_name 15 | self.model_name = model_name 16 | self.source = None 17 | 18 | self.read_file() 19 | 20 | def read_file(self): 21 | "Read json file" 22 | if not os.path.exists(self.file_name): 23 | raise InvalidDescriptor("Invalid JSON File Source") 24 | 25 | read_file = open(self.file_name, "r") 26 | self.source = json.loads(read_file.read()) 27 | 28 | def get_model(self): 29 | "Read model from JSON descriptor" 30 | valid_model = [i for i in self.source if self.model_name in i.get("model")] 31 | if not valid_model: 32 | raise InvalidModel("Model Name does not exist in descriptor") 33 | 34 | return valid_model[0] 35 | 36 | def get_fields(self): 37 | "Get content" 38 | model = self.get_model() 39 | fields = model.get("fields") 40 | if isinstance(fields, dict): 41 | fields = fields.keys() 42 | return fields 43 | -------------------------------------------------------------------------------- /data_importer/core/exceptions.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | 5 | class StopImporter(Exception): 6 | """ 7 | Stop interator and raise error message 8 | """ 9 | 10 | 11 | class UnsuportedFile(Exception): 12 | """ 13 | Unsuported file type 14 | """ 15 | 16 | 17 | class InvalidModel(Exception): 18 | """ 19 | Invalid model in descriptor 20 | """ 21 | 22 | 23 | class InvalidDescriptor(Exception): 24 | """ 25 | Invalid Descriptor File 26 | Descriptor must be one valid JSON 27 | """ 28 | -------------------------------------------------------------------------------- /data_importer/django_migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | from django.conf import settings 6 | import data_importer.models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 13 | ("contenttypes", "0001_initial"), 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name="FileHistory", 19 | fields=[ 20 | ( 21 | "id", 22 | models.AutoField( 23 | verbose_name="ID", 24 | serialize=False, 25 | auto_created=True, 26 | primary_key=True, 27 | ), 28 | ), 29 | ("created_at", models.DateTimeField(auto_now_add=True)), 30 | ("updated_at", models.DateTimeField(auto_now=True)), 31 | ("active", models.BooleanField(default=True, db_index=True)), 32 | ( 33 | "file_upload", 34 | models.FileField( 35 | upload_to=data_importer.models.get_random_filename 36 | ), 37 | ), 38 | ("is_task", models.BooleanField(default=0)), 39 | ( 40 | "status", 41 | models.IntegerField( 42 | default=1, 43 | choices=[ 44 | (1, b"Imported"), 45 | (2, b"Waiting"), 46 | (3, b"Cancelled"), 47 | (-1, b"Error"), 48 | ], 49 | ), 50 | ), 51 | ("object_id", models.PositiveIntegerField(null=True, blank=True)), 52 | ( 53 | "content_type", 54 | models.ForeignKey( 55 | blank=True, 56 | to="contenttypes.ContentType", 57 | null=True, 58 | on_delete=models.CASCADE, 59 | ), 60 | ), 61 | ( 62 | "owner", 63 | models.ForeignKey( 64 | to=settings.AUTH_USER_MODEL, null=True, on_delete=models.CASCADE 65 | ), 66 | ), 67 | ], 68 | options={ 69 | "verbose_name_plural": "File Histories", 70 | }, 71 | bases=(models.Model,), 72 | ), 73 | ] 74 | -------------------------------------------------------------------------------- /data_importer/django_migrations/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django migrations for data_importer app 3 | 4 | This package does not contain South migrations. South migrations can be found 5 | in the ``south_migrations`` package. 6 | """ 7 | 8 | SOUTH_ERROR_MESSAGE = """\n 9 | For South support, customize the SOUTH_MIGRATION_MODULES setting like so: 10 | 11 | SOUTH_MIGRATION_MODULES = { 12 | 'data_importer': 'data_importer.south_migrations', 13 | } 14 | """ 15 | 16 | # Ensure the user is not using Django 1.6 or below with South 17 | try: 18 | from django.db import migrations 19 | except ImportError: 20 | from django.core.exceptions import ImproperlyConfigured 21 | 22 | raise ImproperlyConfigured(SOUTH_ERROR_MESSAGE) 23 | -------------------------------------------------------------------------------- /data_importer/forms.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import os 3 | from django import forms 4 | from data_importer.models import FileHistory 5 | from data_importer.tasks import DataImpoterTask 6 | 7 | try: 8 | import celery 9 | 10 | HAS_CELERY = True 11 | except ImportError: 12 | HAS_CELERY = False 13 | 14 | 15 | class FileUploadForm(forms.ModelForm): 16 | is_task = True 17 | importer = None 18 | 19 | class Meta: 20 | model = FileHistory 21 | fields = ("file_upload",) 22 | -------------------------------------------------------------------------------- /data_importer/importers/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from data_importer.importers.base import BaseImporter 3 | from data_importer.importers.csv_importer import CSVImporter 4 | from data_importer.importers.xls_importer import XLSImporter 5 | from data_importer.importers.xlsx_importer import XLSXImporter 6 | from data_importer.importers.xml_importer import XMLImporter 7 | from data_importer.importers.generic import GenericImporter 8 | from data_importer.core.exceptions import StopImporter 9 | from data_importer.core.exceptions import UnsuportedFile 10 | from data_importer.core.exceptions import InvalidModel 11 | from data_importer.core.exceptions import InvalidDescriptor 12 | 13 | __all__ = ( 14 | "BaseImporter", 15 | "CSVImporter", 16 | "XLSImporter", 17 | "XLSXImporter", 18 | "XMLImporter", 19 | "GenericImporter", 20 | "StopImporter", 21 | "UnsuportedFile", 22 | "InvalidModel", 23 | "InvalidDescriptor", 24 | ) 25 | -------------------------------------------------------------------------------- /data_importer/importers/base.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | import os 4 | import sys 5 | import re 6 | import io 7 | import six 8 | import codecs 9 | from django.db import transaction 10 | from django.core.exceptions import FieldDoesNotExist 11 | from django.core.exceptions import ValidationError 12 | from data_importer.core.descriptor import ReadDescriptor 13 | from data_importer.core.exceptions import StopImporter 14 | from data_importer.core.base import objclass2dict 15 | from data_importer.core.base import DATA_IMPORTER_EXCEL_DECODER 16 | from data_importer.core.base import DATA_IMPORTER_DECODER 17 | from data_importer.core.base import convert_alphabet_to_number 18 | from data_importer.core.base import reduce_list 19 | from collections import OrderedDict 20 | 21 | try: 22 | from django.utils.encoding import force_str 23 | except ImportError: 24 | from django.utils.encoding import force_unicode as force_str 25 | 26 | 27 | class BaseImporter(object): 28 | """ 29 | Base Importer method to create simples importes CSV files. 30 | 31 | set_reader: can be override to create new importers files 32 | """ 33 | 34 | def __new__(cls, **kargs): 35 | """ 36 | Provide custom methods in subclass Meta 37 | """ 38 | if hasattr(cls, "Meta"): 39 | cls.Meta = objclass2dict(cls.Meta) 40 | return super(BaseImporter, cls).__new__(cls) 41 | 42 | def __init__(self, source=None, *args, **kwargs): 43 | self.file_history = None 44 | self._error = [] 45 | self._fields = [] 46 | self._cleaned_data = () 47 | self._reader = None 48 | self._excluded = False 49 | self._readed = False 50 | self._reduce_list = [] 51 | 52 | self.start_fields() 53 | if source: 54 | self.source = source 55 | self.set_reader() 56 | 57 | class Meta: 58 | """Importer configurations""" 59 | 60 | @staticmethod 61 | def to_unicode(bytestr): 62 | """ 63 | Receive string bytestr and try to return a utf-8 string. 64 | """ 65 | if not isinstance(bytestr, str): 66 | return bytestr 67 | 68 | try: 69 | decoded = bytestr.decode( 70 | DATA_IMPORTER_EXCEL_DECODER 71 | ) # default by excel csv 72 | except (UnicodeEncodeError, AttributeError): 73 | decoded = force_str(bytestr, DATA_IMPORTER_DECODER) 74 | 75 | return decoded 76 | 77 | @property 78 | def source(self): 79 | """Return source opened""" 80 | return self._source 81 | 82 | @source.setter 83 | def source(self, source=None, encoding="cp1252"): 84 | """Open source to reader""" 85 | if isinstance(source, io.IOBase): 86 | self._source = source 87 | elif ( 88 | isinstance(source, six.string_types) 89 | and os.path.exists(source) 90 | and source.endswith("csv") 91 | ): 92 | if sys.version_info >= (3, 0): 93 | self._source = codecs.open(source, "rb", encoding=encoding) 94 | else: 95 | self._source = codecs.open(source, "rb") 96 | elif isinstance(source, list): 97 | self._source = source 98 | elif hasattr(source, "file_upload"): # for FileHistory instances 99 | self._source = source.file_upload 100 | self.file_history = source 101 | elif hasattr(source, "file"): 102 | self._source = io.open(source.file.name, "rb") 103 | else: 104 | self._source = source 105 | # raise ValueError('Invalid Source') 106 | 107 | @property 108 | def meta(self): 109 | """Is same to use .Meta""" 110 | if hasattr(self, "Meta"): 111 | return self.Meta 112 | 113 | def start_fields(self): 114 | """ 115 | Initial function to find fields or headers values 116 | This values will be used to process clean and save method 117 | If this method not have fields and have Meta.model this method 118 | will use model fields to populate content without id 119 | """ 120 | if self.Meta.model and not hasattr(self, "fields"): 121 | all_models_fields = [ 122 | i.name for i in self.Meta.model._meta.fields if i.name != "id" 123 | ] 124 | self.fields = all_models_fields 125 | 126 | self.exclude_fields() 127 | 128 | if self.Meta.descriptor: 129 | self.load_descriptor() 130 | 131 | def exclude_fields(self): 132 | """ 133 | Exclude fields from Meta.exclude 134 | """ 135 | # convert dict to fields and filter content 136 | if hasattr(self, "fields") and isinstance(self.fields, dict): 137 | order_dict = OrderedDict(self.fields) 138 | self.fields = list(self.fields) 139 | self._reduce_list = list( 140 | map(convert_alphabet_to_number, order_dict.values()) 141 | ) 142 | 143 | if self.Meta.exclude and not self._excluded: 144 | self._excluded = True 145 | for exclude in self.Meta.exclude: 146 | if exclude in self.fields: 147 | self.fields = list(self.fields) 148 | self.fields.remove(exclude) 149 | 150 | def load_descriptor(self): 151 | """ 152 | Set fields from descriptor file 153 | """ 154 | descriptor = ReadDescriptor(self.Meta.descriptor, 155 | self.Meta.descriptor_model) 156 | self.fields = descriptor.get_fields() 157 | self.exclude_fields() 158 | 159 | @property 160 | def errors(self): 161 | """ 162 | Show errors catch by clean methods 163 | """ 164 | return self._error 165 | 166 | def is_valid(self): 167 | """ 168 | Clear content and return False if have errors 169 | """ 170 | if not self.cleaned_data: 171 | self.cleaned_data 172 | return not any(self._error) 173 | 174 | def set_reader(self): 175 | """ 176 | Method responsable to convert file content into a list with same 177 | values that have fields 178 | 179 | fields: ['myfield1', 'myfield2'] 180 | 181 | response: [['value_myfield1', 'value_myfield2'], 182 | ['value2_myfield1', 'value2_myfield2']] 183 | """ 184 | raise NotImplementedError("No reader implemented") 185 | 186 | def clean_field(self, field_name, value): 187 | """ 188 | User default django field validators to clean content 189 | and run custom validates 190 | """ 191 | if self.Meta.model: 192 | # default django validate field 193 | try: 194 | field = self.Meta.model._meta.get_field(field_name) 195 | field.clean(value, field) 196 | except FieldDoesNotExist: 197 | pass # do nothing if not find this field in model 198 | except Exception as msg: 199 | default_msg = msg.messages[0].replace("This field", "") 200 | new_msg = "Field ({0!s}) {1!s}".format(field.name, default_msg) 201 | raise ValidationError(new_msg) 202 | 203 | clean_function = getattr(self, "clean_{0!s}".format(field_name), False) 204 | 205 | if clean_function: 206 | try: 207 | return clean_function(value) 208 | except Exception as msg: 209 | default_msg = str(msg).replace("This field", "") 210 | new_msg = "Field ({0!s}) {1!s}".format(field_name, default_msg) 211 | raise ValidationError(new_msg) 212 | return value 213 | 214 | def process_row(self, row, values): 215 | """ 216 | Read clean functions from importer and return tupla with row number, 217 | field and value 218 | """ 219 | values_encoded = [self.to_unicode(i) for i in values] 220 | try: 221 | # reduce list 222 | if self._reduce_list: 223 | values_encoded = reduce_list(self._reduce_list, values_encoded) 224 | values = dict(zip(self.fields, values_encoded)) 225 | except TypeError: 226 | raise TypeError("Invalid Line: {0!s}".format(row)) 227 | 228 | has_error = False 229 | 230 | if self.Meta.ignore_empty_lines: 231 | # ignore empty lines 232 | if not any(values.values()): 233 | return None 234 | 235 | for k, v in values.items(): 236 | if self.Meta.raise_errors: 237 | values[k] = self.clean_field(k, v) 238 | else: 239 | try: 240 | values[k] = self.clean_field(k, v) 241 | except StopImporter as e: 242 | raise StopImporter(self.get_error_message(e, row)) 243 | except Exception as e: 244 | self._error.append(self.get_error_message(e, row)) 245 | has_error = True 246 | 247 | if has_error: 248 | return None 249 | 250 | # validate full row data 251 | try: 252 | clean_row_values = self.clean_row(values) 253 | if clean_row_values is not None: 254 | values = clean_row_values 255 | except Exception as e: 256 | self._error.append(self.get_error_message(e, row)) 257 | return None 258 | 259 | return (row, values) 260 | 261 | def get_error_message(self, error, row=None, error_type=None): 262 | messages = "" 263 | 264 | if not error_type: 265 | error_type = "{0!s}".format(type(error).__name__) 266 | 267 | if hasattr(error, "message") and error.message: 268 | messages = "{0!s}".format(error.message) 269 | 270 | if hasattr(error, "messages") and not messages: 271 | if error.messages: 272 | messages = ",".join(error.messages) 273 | 274 | invalid_characters = ["'", '"', "“", "”", "'", '"'] 275 | for i in invalid_characters: 276 | messages = re.sub(i, "", messages) 277 | error_type = re.sub(i, "", error_type) 278 | 279 | if row: 280 | return row, error_type, messages 281 | else: 282 | return error_type, messages 283 | 284 | @property 285 | def cleaned_data(self): 286 | """ 287 | Return tupla with data cleaned 288 | """ 289 | if self._readed: 290 | return self._cleaned_data 291 | 292 | self._readed = True 293 | 294 | try: 295 | self.pre_clean() 296 | except Exception as e: 297 | self._error.append( 298 | self.get_error_message(e, error_type="__pre_clean__") 299 | ) 300 | 301 | try: 302 | self.clean() 303 | except Exception as e: 304 | self._error.append( 305 | self.get_error_message(e, error_type="__clean_all__") 306 | ) 307 | 308 | # create clean content 309 | for data in self._read_file(): 310 | if data: 311 | self._cleaned_data += (data,) 312 | 313 | try: 314 | self.post_clean() 315 | except Exception as e: 316 | self._error.append( 317 | self.get_error_message(e, error_type="__post_clean__") 318 | ) 319 | 320 | return self._cleaned_data 321 | 322 | def pre_clean(self): 323 | """ 324 | Executed before all clean methods 325 | Important: pre_clean dont have cleaned_data content 326 | """ 327 | 328 | def post_clean(self): 329 | """ 330 | Excuted after all clean method 331 | """ 332 | 333 | def clean(self): 334 | """ 335 | Custom clean method 336 | """ 337 | 338 | def clean_row(self, row_values): 339 | """ 340 | Custom clean method for full row data 341 | """ 342 | return row_values 343 | 344 | def pre_commit(self): 345 | """ 346 | Executed before commit multiple register 347 | """ 348 | 349 | def post_save_all_lines(self): 350 | """ 351 | End exection 352 | """ 353 | 354 | def _read_file(self): 355 | """ 356 | Create cleaned_data content 357 | """ 358 | if hasattr(self._reader, "read"): 359 | reader = self._reader.read() 360 | else: 361 | reader = self._reader 362 | 363 | for row, values in enumerate(reader, 1): 364 | if self.Meta.ignore_first_line: 365 | row -= 1 366 | if self.Meta.starting_row and row < self.Meta.starting_row: 367 | pass 368 | elif row < 1: 369 | pass 370 | else: 371 | yield self.process_row(row, values) 372 | 373 | def save(self, instance=None): 374 | """ 375 | Save all contents 376 | DONT override this method 377 | """ 378 | if not instance: 379 | instance = self.Meta.model 380 | 381 | if not instance: 382 | raise AttributeError("Invalid instance model") 383 | 384 | if self.Meta.transaction: 385 | with transaction.atomic(): 386 | for row, data in self.cleaned_data: 387 | record = instance(**data) 388 | record.save() 389 | 390 | try: 391 | self.pre_commit() 392 | except Exception as e: 393 | self._error.append( 394 | self.get_error_message(e, error_type="__pre_commit__") 395 | ) 396 | transaction.rollback() 397 | 398 | try: 399 | transaction.commit() 400 | except Exception as e: 401 | self._error.append( 402 | self.get_error_message(e, error_type="__trasaction__") 403 | ) 404 | transaction.rollback() 405 | 406 | else: 407 | for row, data in self.cleaned_data: 408 | record = instance(**data) 409 | record.save(force_update=False) 410 | 411 | self.post_save_all_lines() 412 | 413 | return True 414 | -------------------------------------------------------------------------------- /data_importer/importers/csv_importer.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from data_importer.importers.base import BaseImporter 4 | from data_importer.readers import CSVReader 5 | 6 | 7 | class CSVImporter(BaseImporter): 8 | def set_reader(self): 9 | delimiter = self.Meta.delimiter or ";" 10 | delimiter = str(delimiter) 11 | self._reader = CSVReader(self, delimiter=delimiter) 12 | -------------------------------------------------------------------------------- /data_importer/importers/generic.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from data_importer.readers.xls_reader import XLSReader 4 | from data_importer.readers.xlsx_reader import XLSXReader 5 | from data_importer.readers.csv_reader import CSVReader 6 | from data_importer.readers.xml_reader import XMLReader 7 | from data_importer.core.exceptions import UnsuportedFile 8 | from .base import BaseImporter 9 | 10 | 11 | class GenericImporter(BaseImporter): 12 | """ 13 | An implementation of BaseImporter that sets the right reader 14 | by file extension. 15 | Probably the best choice for almost all implementation cases 16 | """ 17 | 18 | def set_reader(self): 19 | reader = self.get_reader_class() 20 | 21 | # default importers configurations 22 | extra_values = { 23 | "xlsx": {"data_only": True}, 24 | "xls": { 25 | "sheet_name": self.Meta.sheet_name or None, 26 | "sheet_index": self.Meta.sheet_index or 0, 27 | }, 28 | "csv": {"delimiter": self.Meta.delimiter or ";"}, 29 | "xml": {}, 30 | } 31 | selected_extra_values = extra_values[self.get_source_file_extension()] 32 | self._reader = reader(self, **selected_extra_values) 33 | 34 | def get_reader_class(self): 35 | """ 36 | Gets the right file reader class by source file extension 37 | """ 38 | readers = { 39 | "xls": XLSReader, 40 | "xlsx": XLSXReader, 41 | "xml": XMLReader, 42 | "csv": CSVReader, 43 | } 44 | source_file_extension = self.get_source_file_extension() 45 | 46 | # No reader for invalid extensions 47 | if source_file_extension not in readers.keys(): 48 | raise UnsuportedFile("Unsuported File") 49 | 50 | return readers[source_file_extension] 51 | 52 | def get_source_file_extension(self): 53 | """ 54 | Gets the source file extension. Used to choose the right reader 55 | """ 56 | if hasattr(self.source, "file") and hasattr(self.source.file, "name"): 57 | filename = self.source.file.name # File instances 58 | elif hasattr(self.source, "file_upload"): 59 | if hasattr(self.source.file_upload, "name"): 60 | filename = ( 61 | self.source.file_upload.name 62 | ) # Default DataImporter.models.FileUploadHistory 63 | else: 64 | filename = self.source.file_upload 65 | elif hasattr(self.source, "name"): 66 | filename = self.source.name # Default filename 67 | else: 68 | filename = self.source 69 | 70 | ext = filename.split(".")[-1] 71 | return ext.lower() 72 | -------------------------------------------------------------------------------- /data_importer/importers/xls_importer.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from .base import BaseImporter 4 | from data_importer.readers import XLSReader 5 | 6 | 7 | class XLSImporter(BaseImporter): 8 | def set_reader(self): 9 | """ 10 | [[1,2,3], [2,3,4]] 11 | """ 12 | sheet_by_name = self.Meta.sheet_name or None 13 | sheet_by_index = self.Meta.sheet_index or 0 14 | 15 | self._reader = XLSReader( 16 | self, sheet_name=sheet_by_name, sheet_index=sheet_by_index 17 | ) 18 | -------------------------------------------------------------------------------- /data_importer/importers/xlsx_importer.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from data_importer.importers.base import BaseImporter 4 | from data_importer.readers.xlsx_reader import XLSXReader 5 | 6 | 7 | class XLSXImporter(BaseImporter): 8 | def set_reader(self, data_only=True): 9 | "Read XLSX files" 10 | self._reader = XLSXReader(self, data_only=True) 11 | -------------------------------------------------------------------------------- /data_importer/importers/xml_importer.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from data_importer.importers.base import BaseImporter 4 | from data_importer.readers.xml_reader import XMLReader 5 | 6 | 7 | class XMLImporter(BaseImporter): 8 | """ 9 | Import XML files 10 | """ 11 | 12 | root = "root" 13 | 14 | def set_reader(self): 15 | self._reader = XMLReader(self) 16 | -------------------------------------------------------------------------------- /data_importer/listeners.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from django.db.models.signals import post_delete 3 | from data_importer.models import FileHistory 4 | import os 5 | from django.conf import settings 6 | 7 | 8 | def delete_filefield(sender, instance, **kwargs): 9 | """ 10 | Automatically deleted files when records removed. 11 | """ 12 | has_delete_config = hasattr(settings, "DATA_IMPORTER_HISTORY") 13 | 14 | if has_delete_config and settings.DATA_IMPORTER_HISTORY == False: 15 | if os.path.exists(instance.filename.path): 16 | os.unlink(instance.filename.path) 17 | 18 | 19 | post_delete.connect(delete_filefield, sender=FileHistory) 20 | -------------------------------------------------------------------------------- /data_importer/models.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from django.db import models 3 | import os 4 | import tempfile 5 | import zipfile 6 | from datetime import date 7 | import uuid 8 | import django 9 | from distutils.version import StrictVersion 10 | 11 | try: 12 | from django.core.servers.basehttp import FileWrapper 13 | except ImportError: 14 | from wsgiref.util import FileWrapper 15 | 16 | from django.http import HttpResponse 17 | from django.conf import settings 18 | from django.contrib.contenttypes.models import ContentType 19 | 20 | # port settings.AUTH_USER_MODEL 21 | try: 22 | User = settings.AUTH_USER_MODEL 23 | except AttributeError: 24 | from django.contrib.auth.models import User 25 | 26 | 27 | DJANGO_VERSION = StrictVersion(django.get_version()) 28 | 29 | if DJANGO_VERSION > StrictVersion("1.7"): 30 | from django.contrib.contenttypes.fields import GenericForeignKey # for Django > 1.7 31 | else: 32 | from django.contrib.contenttypes.generic import ( 33 | GenericForeignKey, 34 | ) # for Django < 1.9 35 | 36 | 37 | DATA_IMPORTER_TASK = ( 38 | hasattr(settings, "DATA_IMPORTER_TASK") and settings.DATA_IMPORTER_TASK or 0 39 | ) 40 | 41 | CELERY_STATUS = ( 42 | (1, "Imported"), 43 | (2, "Waiting"), 44 | (3, "Cancelled"), 45 | (-1, "Error"), 46 | ) 47 | 48 | 49 | def get_random_filename(instance, filename): 50 | _, ext = os.path.splitext(filename) 51 | filename = "{0!s}{1!s}".format(str(uuid.uuid4()), ext) 52 | user_dir = "anonymous" 53 | if instance.owner: 54 | user_dir = instance.owner.get_username() 55 | return os.path.join( 56 | "upload_history", user_dir, date.today().strftime("%Y/%m/%d"), filename 57 | ) 58 | 59 | 60 | class FileHistory(models.Model): 61 | created_at = models.DateTimeField(auto_now_add=True) 62 | updated_at = models.DateTimeField(auto_now=True) 63 | active = models.BooleanField(default=True, db_index=True) 64 | file_upload = models.FileField(upload_to=get_random_filename) 65 | owner = models.ForeignKey(User, null=True, on_delete=models.CASCADE) 66 | is_task = models.BooleanField(default=DATA_IMPORTER_TASK) 67 | status = models.IntegerField(choices=CELERY_STATUS, default=1) 68 | 69 | content_type = models.ForeignKey( 70 | ContentType, null=True, blank=True, on_delete=models.CASCADE 71 | ) 72 | object_id = models.PositiveIntegerField(null=True, blank=True) 73 | content_object = GenericForeignKey("content_type", "object_id") 74 | 75 | class Meta: 76 | verbose_name_plural = "File Histories" 77 | 78 | def file_link(self): 79 | _url = self.file_upload.url 80 | return "Download".format(_url) 81 | 82 | file_link.allow_tags = True 83 | 84 | def download_file(self, request): 85 | """ 86 | Send a file through Django without loading the whole file into 87 | memory at once. The FileWrapper will turn the file object into an 88 | iterator for chunks of 8KB. 89 | """ 90 | filename = self.file_upload.path 91 | wrapper = FileWrapper(open(filename, "rb")) 92 | response = HttpResponse(wrapper, content_type="application/force-download") 93 | response["Content-Length"] = os.path.getsize(filename) 94 | return response 95 | 96 | def download_zipfile(self, request): 97 | """ 98 | Create a ZIP file on disk and transmit it in chunks of 8KB, 99 | without loading the whole file into memory. A similar approach can 100 | be used for large dynamic PDF files. 101 | """ 102 | temp = tempfile.TemporaryFile() 103 | archive = zipfile.ZipFile(temp, "w", zipfile.ZIP_DEFLATED) 104 | for index in range(10): 105 | filename = self.filename.path 106 | archive.write(filename, "file{0:d}.txt".format(index)) 107 | archive.close() 108 | wrapper = FileWrapper(temp) 109 | response = HttpResponse(wrapper, content_type="application/zip") 110 | response["Content-Disposition"] = "attachment; filename={0!s}.zip".format( 111 | self.filename 112 | ) 113 | response["Content-Length"] = temp.tell() 114 | temp.seek(0) 115 | return response 116 | 117 | @property 118 | def compose_file_name(self): 119 | basename = os.path.basename(self.file_upload.file.name) 120 | return "{0!s} ({1!s})".format(basename, self.owner) 121 | -------------------------------------------------------------------------------- /data_importer/readers/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from data_importer.readers.csv_reader import CSVReader 3 | from data_importer.readers.xls_reader import XLSReader 4 | from data_importer.readers.xlsx_reader import XLSXReader 5 | from data_importer.readers.xml_reader import XMLReader 6 | 7 | __all__ = ( 8 | "CSVReader", 9 | "XLSReader", 10 | "XLSXReader", 11 | "XMLReader", 12 | ) 13 | -------------------------------------------------------------------------------- /data_importer/readers/csv_reader.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | import csv 4 | 5 | 6 | class CSVReader(object): 7 | def __init__(self, instance, delimiter=";"): 8 | self.instance = instance 9 | self.delimiter = delimiter 10 | 11 | def read(self): 12 | return csv.reader(self.instance.source, delimiter=self.delimiter) 13 | -------------------------------------------------------------------------------- /data_importer/readers/xls_reader.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | import datetime 4 | import xlrd 5 | from django.db.models.fields.files import FieldFile 6 | 7 | # from xlrd/biffh.py 8 | # ( 9 | # XL_CELL_EMPTY, 10 | # XL_CELL_TEXT, 11 | # XL_CELL_NUMBER, 12 | # XL_CELL_DATE, 13 | # XL_CELL_BOOLEAN, 14 | # XL_CELL_ERROR, 15 | # XL_CELL_BLANK, # for use in debugging, gathering stats, etc 16 | # ) = range(7) 17 | 18 | 19 | class XLSReader(object): 20 | def __init__(self, instance, sheet_name=None, sheet_index=0, on_demand=True): 21 | 22 | if isinstance(instance.source, FieldFile): 23 | source = instance.source.path 24 | else: 25 | source = instance.source 26 | 27 | self.workbook = xlrd.open_workbook(source, on_demand=on_demand) 28 | 29 | if sheet_name: 30 | self.worksheet = self.workbook.sheet_by_name(instance.Meta.sheet_name) 31 | else: 32 | self.worksheet = self.workbook.sheet_by_index(sheet_index) 33 | 34 | @staticmethod 35 | def convert_value(item, workbook): 36 | """ 37 | Handle different value types for XLS. Item is a cell object. 38 | """ 39 | # Thx to Augusto C Men to point fast solution for XLS/XLSX dates 40 | if item.ctype == 3: # XL_CELL_DATE: 41 | return datetime.datetime( 42 | *xlrd.xldate_as_tuple(item.value, workbook.datemode) 43 | ) 44 | 45 | if item.ctype == 2: # XL_CELL_NUMBER: 46 | if item.value % 1 == 0: # integers 47 | return int(item.value) 48 | else: 49 | return item.value 50 | 51 | return item.value 52 | 53 | def read(self): 54 | for i in range(0, self.worksheet.nrows): 55 | values = [ 56 | self.convert_value(cell, self.workbook) 57 | for cell in self.worksheet.row(i) 58 | ] 59 | yield values 60 | -------------------------------------------------------------------------------- /data_importer/readers/xlsx_reader.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from openpyxl import load_workbook 4 | 5 | 6 | class XLSXReader(object): 7 | def __init__(self, instance, data_only=True, sheet_index=None): 8 | self.workbook = load_workbook(instance.source, data_only=data_only) 9 | if instance.Meta.sheet_name: 10 | self.worksheet = self.workbook.get_sheet_by_name(instance.Meta.sheet_name) 11 | else: 12 | if sheet_index is None: 13 | sheet_index = instance.Meta.sheet_index or 0 14 | self.worksheet = self.workbook.worksheets[sheet_index] 15 | 16 | def read(self): 17 | for line, row in enumerate(self.worksheet.iter_rows()): 18 | values = [cell.value for cell in row] 19 | yield values 20 | -------------------------------------------------------------------------------- /data_importer/readers/xml_reader.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | import xml.etree.ElementTree as et 4 | 5 | 6 | class XMLReader(object): 7 | def __init__(self, instance): 8 | self.instance = instance 9 | 10 | def read(self): 11 | "Convert XML to Dict" 12 | tree = et.parse(self.instance.source) 13 | elements = tree.findall(self.instance.root) 14 | for elem in elements: 15 | items = list(elem) 16 | content = [i.text for i in items] 17 | yield content 18 | -------------------------------------------------------------------------------- /data_importer/south_migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import datetime 3 | from south.db import db 4 | from south.v2 import SchemaMigration 5 | from django.db import models 6 | 7 | 8 | class Migration(SchemaMigration): 9 | def forwards(self, orm): 10 | # Adding model 'FileHistory' 11 | db.create_table( 12 | "data_importer_filehistory", 13 | ( 14 | ("id", self.gf("django.db.models.fields.AutoField")(primary_key=True)), 15 | ( 16 | "created_at", 17 | self.gf("django.db.models.fields.DateTimeField")( 18 | auto_now_add=True, blank=True 19 | ), 20 | ), 21 | ( 22 | "updated_at", 23 | self.gf("django.db.models.fields.DateTimeField")( 24 | auto_now=True, blank=True 25 | ), 26 | ), 27 | ( 28 | "active", 29 | self.gf("django.db.models.fields.BooleanField")( 30 | default=True, db_index=True 31 | ), 32 | ), 33 | ( 34 | "content", 35 | self.gf("django.db.models.fields.files.FileField")(max_length=100), 36 | ), 37 | ), 38 | ) 39 | db.send_create_signal("data_importer", ["FileHistory"]) 40 | 41 | def backwards(self, orm): 42 | # Deleting model 'FileHistory' 43 | db.delete_table("data_importer_filehistory") 44 | 45 | models = { 46 | "data_importer.filehistory": { 47 | "Meta": {"object_name": "FileHistory"}, 48 | "active": ( 49 | "django.db.models.fields.BooleanField", 50 | [], 51 | {"default": "True", "db_index": "True"}, 52 | ), 53 | "content": ( 54 | "django.db.models.fields.files.FileField", 55 | [], 56 | {"max_length": "100"}, 57 | ), 58 | "created_at": ( 59 | "django.db.models.fields.DateTimeField", 60 | [], 61 | {"auto_now_add": "True", "blank": "True"}, 62 | ), 63 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 64 | "updated_at": ( 65 | "django.db.models.fields.DateTimeField", 66 | [], 67 | {"auto_now": "True", "blank": "True"}, 68 | ), 69 | } 70 | } 71 | 72 | complete_apps = ["data_importer"] 73 | -------------------------------------------------------------------------------- /data_importer/south_migrations/0002_auto__add_field_filehistory_owner.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import datetime 3 | from south.db import db 4 | from south.v2 import SchemaMigration 5 | from django.db import models 6 | 7 | 8 | class Migration(SchemaMigration): 9 | def forwards(self, orm): 10 | # Adding field 'FileHistory.owner' 11 | db.add_column( 12 | "data_importer_filehistory", 13 | "owner", 14 | self.gf("django.db.models.fields.related.ForeignKey")( 15 | to=orm["auth.User"], null=True 16 | ), 17 | keep_default=False, 18 | ) 19 | 20 | def backwards(self, orm): 21 | # Deleting field 'FileHistory.owner' 22 | db.delete_column("data_importer_filehistory", "owner_id") 23 | 24 | models = { 25 | "auth.group": { 26 | "Meta": {"object_name": "Group"}, 27 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 28 | "name": ( 29 | "django.db.models.fields.CharField", 30 | [], 31 | {"unique": "True", "max_length": "80"}, 32 | ), 33 | "permissions": ( 34 | "django.db.models.fields.related.ManyToManyField", 35 | [], 36 | { 37 | "to": u"orm['auth.Permission']", 38 | "symmetrical": "False", 39 | "blank": "True", 40 | }, 41 | ), 42 | }, 43 | "auth.permission": { 44 | "Meta": { 45 | "ordering": "('content_type__app_label', 'content_type__model', 'codename')", 46 | "unique_together": "(('content_type', 'codename'),)", 47 | "object_name": "Permission", 48 | }, 49 | "codename": ( 50 | "django.db.models.fields.CharField", 51 | [], 52 | {"max_length": "100"}, 53 | ), 54 | "content_type": ( 55 | "django.db.models.fields.related.ForeignKey", 56 | [], 57 | {"to": u"orm['contenttypes.ContentType']"}, 58 | ), 59 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 60 | "name": ("django.db.models.fields.CharField", [], {"max_length": "50"}), 61 | }, 62 | "auth.user": { 63 | "Meta": {"object_name": "User"}, 64 | "date_joined": ( 65 | "django.db.models.fields.DateTimeField", 66 | [], 67 | {"default": "datetime.datetime.now"}, 68 | ), 69 | "email": ( 70 | "django.db.models.fields.EmailField", 71 | [], 72 | {"max_length": "75", "blank": "True"}, 73 | ), 74 | "first_name": ( 75 | "django.db.models.fields.CharField", 76 | [], 77 | {"max_length": "30", "blank": "True"}, 78 | ), 79 | "groups": ( 80 | "django.db.models.fields.related.ManyToManyField", 81 | [], 82 | {"to": u"orm['auth.Group']", "symmetrical": "False", "blank": "True"}, 83 | ), 84 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 85 | "is_active": ( 86 | "django.db.models.fields.BooleanField", 87 | [], 88 | {"default": "True"}, 89 | ), 90 | "is_staff": ( 91 | "django.db.models.fields.BooleanField", 92 | [], 93 | {"default": "False"}, 94 | ), 95 | "is_superuser": ( 96 | "django.db.models.fields.BooleanField", 97 | [], 98 | {"default": "False"}, 99 | ), 100 | "last_login": ( 101 | "django.db.models.fields.DateTimeField", 102 | [], 103 | {"default": "datetime.datetime.now"}, 104 | ), 105 | "last_name": ( 106 | "django.db.models.fields.CharField", 107 | [], 108 | {"max_length": "30", "blank": "True"}, 109 | ), 110 | "password": ( 111 | "django.db.models.fields.CharField", 112 | [], 113 | {"max_length": "128"}, 114 | ), 115 | "user_permissions": ( 116 | "django.db.models.fields.related.ManyToManyField", 117 | [], 118 | { 119 | "to": u"orm['auth.Permission']", 120 | "symmetrical": "False", 121 | "blank": "True", 122 | }, 123 | ), 124 | "username": ( 125 | "django.db.models.fields.CharField", 126 | [], 127 | {"unique": "True", "max_length": "30"}, 128 | ), 129 | }, 130 | "contenttypes.contenttype": { 131 | "Meta": { 132 | "ordering": "('name',)", 133 | "unique_together": "(('app_label', 'model'),)", 134 | "object_name": "ContentType", 135 | "db_table": "'django_content_type'", 136 | }, 137 | "app_label": ( 138 | "django.db.models.fields.CharField", 139 | [], 140 | {"max_length": "100"}, 141 | ), 142 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 143 | "model": ("django.db.models.fields.CharField", [], {"max_length": "100"}), 144 | "name": ("django.db.models.fields.CharField", [], {"max_length": "100"}), 145 | }, 146 | "data_importer.filehistory": { 147 | "Meta": {"object_name": "FileHistory"}, 148 | "active": ( 149 | "django.db.models.fields.BooleanField", 150 | [], 151 | {"default": "True", "db_index": "True"}, 152 | ), 153 | "content": ( 154 | "django.db.models.fields.files.FileField", 155 | [], 156 | {"max_length": "100"}, 157 | ), 158 | "created_at": ( 159 | "django.db.models.fields.DateTimeField", 160 | [], 161 | {"auto_now_add": "True", "blank": "True"}, 162 | ), 163 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 164 | "owner": ( 165 | "django.db.models.fields.related.ForeignKey", 166 | [], 167 | {"to": u"orm['auth.User']", "null": "True"}, 168 | ), 169 | "updated_at": ( 170 | "django.db.models.fields.DateTimeField", 171 | [], 172 | {"auto_now": "True", "blank": "True"}, 173 | ), 174 | }, 175 | } 176 | 177 | complete_apps = ["data_importer"] 178 | -------------------------------------------------------------------------------- /data_importer/south_migrations/0003_auto__add_field_filehistory_is_task__add_field_filehistory_status.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import datetime 3 | from south.db import db 4 | from south.v2 import SchemaMigration 5 | from django.db import models 6 | 7 | 8 | class Migration(SchemaMigration): 9 | def forwards(self, orm): 10 | # Adding field 'FileHistory.is_task' 11 | db.add_column( 12 | "data_importer_filehistory", 13 | "is_task", 14 | self.gf("django.db.models.fields.BooleanField")(default=False), 15 | keep_default=False, 16 | ) 17 | 18 | # Adding field 'FileHistory.status' 19 | db.add_column( 20 | "data_importer_filehistory", 21 | "status", 22 | self.gf("django.db.models.fields.IntegerField")(default=1), 23 | keep_default=False, 24 | ) 25 | 26 | def backwards(self, orm): 27 | # Deleting field 'FileHistory.is_task' 28 | db.delete_column("data_importer_filehistory", "is_task") 29 | 30 | # Deleting field 'FileHistory.status' 31 | db.delete_column("data_importer_filehistory", "status") 32 | 33 | models = { 34 | "auth.group": { 35 | "Meta": {"object_name": "Group"}, 36 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 37 | "name": ( 38 | "django.db.models.fields.CharField", 39 | [], 40 | {"unique": "True", "max_length": "80"}, 41 | ), 42 | "permissions": ( 43 | "django.db.models.fields.related.ManyToManyField", 44 | [], 45 | { 46 | "to": u"orm['auth.Permission']", 47 | "symmetrical": "False", 48 | "blank": "True", 49 | }, 50 | ), 51 | }, 52 | "auth.permission": { 53 | "Meta": { 54 | "ordering": "('content_type__app_label', 'content_type__model', 'codename')", 55 | "unique_together": "(('content_type', 'codename'),)", 56 | "object_name": "Permission", 57 | }, 58 | "codename": ( 59 | "django.db.models.fields.CharField", 60 | [], 61 | {"max_length": "100"}, 62 | ), 63 | "content_type": ( 64 | "django.db.models.fields.related.ForeignKey", 65 | [], 66 | {"to": u"orm['contenttypes.ContentType']"}, 67 | ), 68 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 69 | "name": ("django.db.models.fields.CharField", [], {"max_length": "50"}), 70 | }, 71 | "auth.user": { 72 | "Meta": {"object_name": "User"}, 73 | "date_joined": ( 74 | "django.db.models.fields.DateTimeField", 75 | [], 76 | {"default": "datetime.datetime.now"}, 77 | ), 78 | "email": ( 79 | "django.db.models.fields.EmailField", 80 | [], 81 | {"max_length": "75", "blank": "True"}, 82 | ), 83 | "first_name": ( 84 | "django.db.models.fields.CharField", 85 | [], 86 | {"max_length": "30", "blank": "True"}, 87 | ), 88 | "groups": ( 89 | "django.db.models.fields.related.ManyToManyField", 90 | [], 91 | {"to": u"orm['auth.Group']", "symmetrical": "False", "blank": "True"}, 92 | ), 93 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 94 | "is_active": ( 95 | "django.db.models.fields.BooleanField", 96 | [], 97 | {"default": "True"}, 98 | ), 99 | "is_staff": ( 100 | "django.db.models.fields.BooleanField", 101 | [], 102 | {"default": "False"}, 103 | ), 104 | "is_superuser": ( 105 | "django.db.models.fields.BooleanField", 106 | [], 107 | {"default": "False"}, 108 | ), 109 | "last_login": ( 110 | "django.db.models.fields.DateTimeField", 111 | [], 112 | {"default": "datetime.datetime.now"}, 113 | ), 114 | "last_name": ( 115 | "django.db.models.fields.CharField", 116 | [], 117 | {"max_length": "30", "blank": "True"}, 118 | ), 119 | "password": ( 120 | "django.db.models.fields.CharField", 121 | [], 122 | {"max_length": "128"}, 123 | ), 124 | "user_permissions": ( 125 | "django.db.models.fields.related.ManyToManyField", 126 | [], 127 | { 128 | "to": u"orm['auth.Permission']", 129 | "symmetrical": "False", 130 | "blank": "True", 131 | }, 132 | ), 133 | "username": ( 134 | "django.db.models.fields.CharField", 135 | [], 136 | {"unique": "True", "max_length": "30"}, 137 | ), 138 | }, 139 | "contenttypes.contenttype": { 140 | "Meta": { 141 | "ordering": "('name',)", 142 | "unique_together": "(('app_label', 'model'),)", 143 | "object_name": "ContentType", 144 | "db_table": "'django_content_type'", 145 | }, 146 | "app_label": ( 147 | "django.db.models.fields.CharField", 148 | [], 149 | {"max_length": "100"}, 150 | ), 151 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 152 | "model": ("django.db.models.fields.CharField", [], {"max_length": "100"}), 153 | "name": ("django.db.models.fields.CharField", [], {"max_length": "100"}), 154 | }, 155 | "data_importer.filehistory": { 156 | "Meta": {"object_name": "FileHistory"}, 157 | "active": ( 158 | "django.db.models.fields.BooleanField", 159 | [], 160 | {"default": "True", "db_index": "True"}, 161 | ), 162 | "content": ( 163 | "django.db.models.fields.files.FileField", 164 | [], 165 | {"max_length": "100"}, 166 | ), 167 | "created_at": ( 168 | "django.db.models.fields.DateTimeField", 169 | [], 170 | {"auto_now_add": "True", "blank": "True"}, 171 | ), 172 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 173 | "is_task": ( 174 | "django.db.models.fields.BooleanField", 175 | [], 176 | {"default": "False"}, 177 | ), 178 | "owner": ( 179 | "django.db.models.fields.related.ForeignKey", 180 | [], 181 | {"to": u"orm['auth.User']", "null": "True"}, 182 | ), 183 | "status": ("django.db.models.fields.IntegerField", [], {"default": "1"}), 184 | "updated_at": ( 185 | "django.db.models.fields.DateTimeField", 186 | [], 187 | {"auto_now": "True", "blank": "True"}, 188 | ), 189 | }, 190 | } 191 | 192 | complete_apps = ["data_importer"] 193 | -------------------------------------------------------------------------------- /data_importer/south_migrations/0004_auto__del_field_filehistory_content__add_field_filehistory_filename.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import datetime 3 | from south.db import db 4 | from south.v2 import SchemaMigration 5 | from django.db import models 6 | 7 | 8 | class Migration(SchemaMigration): 9 | def forwards(self, orm): 10 | db.rename_column("data_importer_filehistory", "content", "filename") 11 | 12 | def backwards(self, orm): 13 | db.rename_column("data_importer_filehistory", "filename", "content") 14 | 15 | models = { 16 | "auth.group": { 17 | "Meta": {"object_name": "Group"}, 18 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 19 | "name": ( 20 | "django.db.models.fields.CharField", 21 | [], 22 | {"unique": "True", "max_length": "80"}, 23 | ), 24 | "permissions": ( 25 | "django.db.models.fields.related.ManyToManyField", 26 | [], 27 | { 28 | "to": u"orm['auth.Permission']", 29 | "symmetrical": "False", 30 | "blank": "True", 31 | }, 32 | ), 33 | }, 34 | "auth.permission": { 35 | "Meta": { 36 | "ordering": "('content_type__app_label', 'content_type__model', 'codename')", 37 | "unique_together": "(('content_type', 'codename'),)", 38 | "object_name": "Permission", 39 | }, 40 | "codename": ( 41 | "django.db.models.fields.CharField", 42 | [], 43 | {"max_length": "100"}, 44 | ), 45 | "content_type": ( 46 | "django.db.models.fields.related.ForeignKey", 47 | [], 48 | {"to": u"orm['contenttypes.ContentType']"}, 49 | ), 50 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 51 | "name": ("django.db.models.fields.CharField", [], {"max_length": "50"}), 52 | }, 53 | "auth.user": { 54 | "Meta": {"object_name": "User"}, 55 | "date_joined": ( 56 | "django.db.models.fields.DateTimeField", 57 | [], 58 | {"default": "datetime.datetime.now"}, 59 | ), 60 | "email": ( 61 | "django.db.models.fields.EmailField", 62 | [], 63 | {"max_length": "75", "blank": "True"}, 64 | ), 65 | "first_name": ( 66 | "django.db.models.fields.CharField", 67 | [], 68 | {"max_length": "30", "blank": "True"}, 69 | ), 70 | "groups": ( 71 | "django.db.models.fields.related.ManyToManyField", 72 | [], 73 | {"to": u"orm['auth.Group']", "symmetrical": "False", "blank": "True"}, 74 | ), 75 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 76 | "is_active": ( 77 | "django.db.models.fields.BooleanField", 78 | [], 79 | {"default": "True"}, 80 | ), 81 | "is_staff": ( 82 | "django.db.models.fields.BooleanField", 83 | [], 84 | {"default": "False"}, 85 | ), 86 | "is_superuser": ( 87 | "django.db.models.fields.BooleanField", 88 | [], 89 | {"default": "False"}, 90 | ), 91 | "last_login": ( 92 | "django.db.models.fields.DateTimeField", 93 | [], 94 | {"default": "datetime.datetime.now"}, 95 | ), 96 | "last_name": ( 97 | "django.db.models.fields.CharField", 98 | [], 99 | {"max_length": "30", "blank": "True"}, 100 | ), 101 | "password": ( 102 | "django.db.models.fields.CharField", 103 | [], 104 | {"max_length": "128"}, 105 | ), 106 | "user_permissions": ( 107 | "django.db.models.fields.related.ManyToManyField", 108 | [], 109 | { 110 | "to": u"orm['auth.Permission']", 111 | "symmetrical": "False", 112 | "blank": "True", 113 | }, 114 | ), 115 | "username": ( 116 | "django.db.models.fields.CharField", 117 | [], 118 | {"unique": "True", "max_length": "30"}, 119 | ), 120 | }, 121 | "contenttypes.contenttype": { 122 | "Meta": { 123 | "ordering": "('name',)", 124 | "unique_together": "(('app_label', 'model'),)", 125 | "object_name": "ContentType", 126 | "db_table": "'django_content_type'", 127 | }, 128 | "app_label": ( 129 | "django.db.models.fields.CharField", 130 | [], 131 | {"max_length": "100"}, 132 | ), 133 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 134 | "model": ("django.db.models.fields.CharField", [], {"max_length": "100"}), 135 | "name": ("django.db.models.fields.CharField", [], {"max_length": "100"}), 136 | }, 137 | "data_importer.filehistory": { 138 | "Meta": {"object_name": "FileHistory"}, 139 | "active": ( 140 | "django.db.models.fields.BooleanField", 141 | [], 142 | {"default": "True", "db_index": "True"}, 143 | ), 144 | "created_at": ( 145 | "django.db.models.fields.DateTimeField", 146 | [], 147 | {"auto_now_add": "True", "blank": "True"}, 148 | ), 149 | "filename": ( 150 | "django.db.models.fields.files.FileField", 151 | [], 152 | {"max_length": "100"}, 153 | ), 154 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 155 | "is_task": ( 156 | "django.db.models.fields.BooleanField", 157 | [], 158 | {"default": "False"}, 159 | ), 160 | "owner": ( 161 | "django.db.models.fields.related.ForeignKey", 162 | [], 163 | {"to": u"orm['auth.User']", "null": "True"}, 164 | ), 165 | "status": ("django.db.models.fields.IntegerField", [], {"default": "1"}), 166 | "updated_at": ( 167 | "django.db.models.fields.DateTimeField", 168 | [], 169 | {"auto_now": "True", "blank": "True"}, 170 | ), 171 | }, 172 | } 173 | 174 | complete_apps = ["data_importer"] 175 | -------------------------------------------------------------------------------- /data_importer/south_migrations/0005_auto__add_filehistorylog.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import datetime 3 | from south.db import db 4 | from south.v2 import SchemaMigration 5 | from django.db import models 6 | 7 | 8 | class Migration(SchemaMigration): 9 | def forwards(self, orm): 10 | # Adding model 'FileHistoryLog' 11 | db.create_table( 12 | "data_importer_filehistorylog", 13 | ( 14 | ("id", self.gf("django.db.models.fields.AutoField")(primary_key=True)), 15 | ( 16 | "filehistory", 17 | self.gf("django.db.models.fields.related.ForeignKey")( 18 | to=orm["data_importer.FileHistory"] 19 | ), 20 | ), 21 | ( 22 | "created_at", 23 | self.gf("django.db.models.fields.DateTimeField")( 24 | auto_now_add=True, blank=True 25 | ), 26 | ), 27 | ("log", self.gf("django.db.models.fields.CharField")(max_length=255)), 28 | ), 29 | ) 30 | db.send_create_signal("data_importer", ["FileHistoryLog"]) 31 | 32 | def backwards(self, orm): 33 | # Deleting model 'FileHistoryLog' 34 | db.delete_table("data_importer_filehistorylog") 35 | 36 | models = { 37 | "auth.group": { 38 | "Meta": {"object_name": "Group"}, 39 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 40 | "name": ( 41 | "django.db.models.fields.CharField", 42 | [], 43 | {"unique": "True", "max_length": "80"}, 44 | ), 45 | "permissions": ( 46 | "django.db.models.fields.related.ManyToManyField", 47 | [], 48 | { 49 | "to": u"orm['auth.Permission']", 50 | "symmetrical": "False", 51 | "blank": "True", 52 | }, 53 | ), 54 | }, 55 | "auth.permission": { 56 | "Meta": { 57 | "ordering": "('content_type__app_label', 'content_type__model', 'codename')", 58 | "unique_together": "(('content_type', 'codename'),)", 59 | "object_name": "Permission", 60 | }, 61 | "codename": ( 62 | "django.db.models.fields.CharField", 63 | [], 64 | {"max_length": "100"}, 65 | ), 66 | "content_type": ( 67 | "django.db.models.fields.related.ForeignKey", 68 | [], 69 | {"to": u"orm['contenttypes.ContentType']"}, 70 | ), 71 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 72 | "name": ("django.db.models.fields.CharField", [], {"max_length": "50"}), 73 | }, 74 | "auth.user": { 75 | "Meta": {"object_name": "User"}, 76 | "date_joined": ( 77 | "django.db.models.fields.DateTimeField", 78 | [], 79 | {"default": "datetime.datetime.now"}, 80 | ), 81 | "email": ( 82 | "django.db.models.fields.EmailField", 83 | [], 84 | {"max_length": "75", "blank": "True"}, 85 | ), 86 | "first_name": ( 87 | "django.db.models.fields.CharField", 88 | [], 89 | {"max_length": "30", "blank": "True"}, 90 | ), 91 | "groups": ( 92 | "django.db.models.fields.related.ManyToManyField", 93 | [], 94 | {"to": u"orm['auth.Group']", "symmetrical": "False", "blank": "True"}, 95 | ), 96 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 97 | "is_active": ( 98 | "django.db.models.fields.BooleanField", 99 | [], 100 | {"default": "True"}, 101 | ), 102 | "is_staff": ( 103 | "django.db.models.fields.BooleanField", 104 | [], 105 | {"default": "False"}, 106 | ), 107 | "is_superuser": ( 108 | "django.db.models.fields.BooleanField", 109 | [], 110 | {"default": "False"}, 111 | ), 112 | "last_login": ( 113 | "django.db.models.fields.DateTimeField", 114 | [], 115 | {"default": "datetime.datetime.now"}, 116 | ), 117 | "last_name": ( 118 | "django.db.models.fields.CharField", 119 | [], 120 | {"max_length": "30", "blank": "True"}, 121 | ), 122 | "password": ( 123 | "django.db.models.fields.CharField", 124 | [], 125 | {"max_length": "128"}, 126 | ), 127 | "user_permissions": ( 128 | "django.db.models.fields.related.ManyToManyField", 129 | [], 130 | { 131 | "to": u"orm['auth.Permission']", 132 | "symmetrical": "False", 133 | "blank": "True", 134 | }, 135 | ), 136 | "username": ( 137 | "django.db.models.fields.CharField", 138 | [], 139 | {"unique": "True", "max_length": "30"}, 140 | ), 141 | }, 142 | "contenttypes.contenttype": { 143 | "Meta": { 144 | "ordering": "('name',)", 145 | "unique_together": "(('app_label', 'model'),)", 146 | "object_name": "ContentType", 147 | "db_table": "'django_content_type'", 148 | }, 149 | "app_label": ( 150 | "django.db.models.fields.CharField", 151 | [], 152 | {"max_length": "100"}, 153 | ), 154 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 155 | "model": ("django.db.models.fields.CharField", [], {"max_length": "100"}), 156 | "name": ("django.db.models.fields.CharField", [], {"max_length": "100"}), 157 | }, 158 | "data_importer.filehistory": { 159 | "Meta": {"object_name": "FileHistory"}, 160 | "active": ( 161 | "django.db.models.fields.BooleanField", 162 | [], 163 | {"default": "True", "db_index": "True"}, 164 | ), 165 | "created_at": ( 166 | "django.db.models.fields.DateTimeField", 167 | [], 168 | {"auto_now_add": "True", "blank": "True"}, 169 | ), 170 | "filename": ( 171 | "django.db.models.fields.files.FileField", 172 | [], 173 | {"max_length": "100"}, 174 | ), 175 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 176 | "is_task": ( 177 | "django.db.models.fields.BooleanField", 178 | [], 179 | {"default": "False"}, 180 | ), 181 | "owner": ( 182 | "django.db.models.fields.related.ForeignKey", 183 | [], 184 | {"to": u"orm['auth.User']", "null": "True"}, 185 | ), 186 | "status": ("django.db.models.fields.IntegerField", [], {"default": "1"}), 187 | "updated_at": ( 188 | "django.db.models.fields.DateTimeField", 189 | [], 190 | {"auto_now": "True", "blank": "True"}, 191 | ), 192 | }, 193 | "data_importer.filehistorylog": { 194 | "Meta": {"object_name": "FileHistoryLog"}, 195 | "created_at": ( 196 | "django.db.models.fields.DateTimeField", 197 | [], 198 | {"auto_now_add": "True", "blank": "True"}, 199 | ), 200 | "filehistory": ( 201 | "django.db.models.fields.related.ForeignKey", 202 | [], 203 | {"to": u"orm['data_importer.FileHistory']"}, 204 | ), 205 | "id": ("django.db.models.fields.AutoField", [], {"primary_key": "True"}), 206 | "log": ("django.db.models.fields.CharField", [], {"max_length": "255"}), 207 | }, 208 | } 209 | 210 | complete_apps = ["data_importer"] 211 | -------------------------------------------------------------------------------- /data_importer/south_migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valdergallo/data-importer/7d3f1d7209597484e3041768ae01c90b16bcc08f/data_importer/south_migrations/__init__.py -------------------------------------------------------------------------------- /data_importer/tasks.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import os 3 | 4 | try: 5 | from celery import Task 6 | except ImportError: 7 | Task = object 8 | 9 | from django.core.cache import cache 10 | from data_importer.core import default_settings 11 | from django.core.mail import EmailMessage 12 | from django.utils.safestring import mark_safe 13 | 14 | try: 15 | from django.conf import settings 16 | except ImportError as e: 17 | settings = None 18 | 19 | try: 20 | LOCK_EXPIRE = settings.DATA_IMPORTER_TASK_LOCK_EXPIRE 21 | except AttributeError: 22 | LOCK_EXPIRE = default_settings.DATA_IMPORTER_TASK_LOCK_EXPIRE 23 | 24 | try: 25 | DATA_IMPORTER_QUEUE = settings.DATA_IMPORTER_QUEUE 26 | except AttributeError: 27 | DATA_IMPORTER_QUEUE = default_settings.DATA_IMPORTER_QUEUE 28 | 29 | acquire_lock = lambda lock_id: cache.add(lock_id, "true", LOCK_EXPIRE) 30 | release_lock = lambda lock_id: cache.delete(lock_id) 31 | 32 | 33 | class DataImpoterTask(Task): 34 | """ 35 | This tasks is executed by Celery. 36 | """ 37 | 38 | name = "data_importer_task" 39 | queue = DATA_IMPORTER_QUEUE 40 | time_limit = 60 * 15 41 | 42 | @staticmethod 43 | def send_email(subject="[Data Importer] was processed", body="", owner=None): 44 | email = EmailMessage( 45 | subject=subject, 46 | body=body, 47 | to=[owner.email], 48 | headers={"Content-Type": "text/plain"}, 49 | ) 50 | email.send() 51 | 52 | def run( 53 | self, 54 | importer=None, 55 | source="", 56 | owner=None, 57 | message="", 58 | send_email=True, 59 | **kwargs 60 | ): 61 | if not importer: 62 | return 63 | 64 | self.parser = importer(source=source) 65 | 66 | lock_id = "{0!s}-lock".format((self.name)) 67 | 68 | if acquire_lock(lock_id): 69 | """ 70 | If parser use raise_errors the error message will raise 71 | and logged by celery 72 | """ 73 | # validate content 74 | self.parser.is_valid() 75 | # save valid values 76 | self.parser.save() 77 | 78 | message += "\n" 79 | 80 | if owner and owner.email and self.parser.errors: 81 | message += mark_safe(self.parser.errors) 82 | elif owner and owner.email and not self.parser.errors: 83 | message = "Your file was imported with sucess" 84 | 85 | if hasattr(owner, "email") and send_email: 86 | self.send_email(body=message, owner=owner) 87 | 88 | release_lock(lock_id) 89 | else: 90 | return 0 91 | -------------------------------------------------------------------------------- /data_importer/templates/data_importer.html: -------------------------------------------------------------------------------- 1 | {% comment %}Include this to your upload page template{% endcomment %} 2 |

{{ title }}

3 | 4 | {% if messages %} 5 | 10 | {% endif %} 11 | 12 | download template file 13 | 14 |
{% csrf_token %} 15 | {{ form.as_p }} 16 | 17 |
18 | -------------------------------------------------------------------------------- /data_importer/templates/my_upload.html: -------------------------------------------------------------------------------- 1 | {% extends "index.html" %} 2 | 3 | {% block contents %} 4 | 5 |

My Uploads

6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | {% for file in files %} 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | {% endfor %} 27 | 28 |
Uploaded FileTypeUpload timeStart execution timeFinish execution time
{{ file.uploaded_file }}{{ file.state }}{{ file.upload_timestamp }}{{ file.start_execution_timestamp }}{{ file.finish_execution_timestamp }}
29 | 30 | {% endblock %} 31 | -------------------------------------------------------------------------------- /data_importer/views.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from data_importer.forms import FileUploadForm 3 | from data_importer.models import FileHistory 4 | from django.contrib import messages 5 | from data_importer.tasks import DataImpoterTask 6 | from django.contrib.contenttypes.models import ContentType 7 | 8 | try: 9 | from django.views.generic.edit import FormView 10 | except ImportError: 11 | """ 12 | Add suport to django 1.2 13 | """ 14 | try: 15 | from django.views.generic.create_update import create_object as FormView 16 | except ImportError: 17 | raise ImportError("Django must be version 1.2 or greater") 18 | 19 | 20 | class DataImporterForm(FormView): 21 | """ 22 | Usage example 23 | ============== 24 | 25 | class DataImporterCreateView(DataImporterForm): 26 | extra_context = {'title': 'Create Form Data Importer', 27 | 'template_file': 'myfile.csv' 28 | } 29 | importer = MyImporterModel 30 | """ 31 | 32 | model = FileHistory 33 | template_name = "data_importer.html" 34 | form_class = FileUploadForm 35 | task = DataImpoterTask() 36 | importer = None 37 | is_task = True 38 | success_url = "." 39 | extra_context = { 40 | "title": "Form Data Importer", 41 | "template_file": "myfile.csv", 42 | "success_message": "File uploaded successfully", 43 | } 44 | 45 | def get_context_data(self, **kwargs): 46 | context = super(DataImporterForm, self).get_context_data(**kwargs) 47 | context.update(self.extra_context) 48 | return context 49 | 50 | def form_valid(self, form, owner=None): 51 | if self.request.user.id: 52 | owner = self.request.user 53 | 54 | if self.importer.Meta.model: 55 | content_type = ContentType.objects.get_for_model(self.importer.Meta.model) 56 | file_history, _ = FileHistory.objects.get_or_create( 57 | file_upload=form.cleaned_data["file_upload"], 58 | owner=owner, 59 | content_type=content_type, 60 | ) 61 | 62 | if not self.is_task or not hasattr(self.task, "delay"): 63 | self.task.run( 64 | importer=self.importer, 65 | source=file_history, 66 | owner=owner, 67 | send_email=False, 68 | ) 69 | if self.task.parser.errors: 70 | messages.error(self.request, self.task.parser.errors) 71 | else: 72 | messages.success( 73 | self.request, 74 | self.extra_context.get( 75 | "success_message", "File uploaded successfully" 76 | ), 77 | ) 78 | else: 79 | self.task.delay(importer=self.importer, source=file_history, owner=owner) 80 | if owner: 81 | messages.info( 82 | self.request, 83 | "When importer was finished one email will send to: {0!s}".format( 84 | owner.email 85 | ), 86 | ) 87 | else: 88 | messages.info(self.request, "Importer task in queue") 89 | 90 | return super(DataImporterForm, self).form_valid(form) 91 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/DjangoDataImporter.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/DjangoDataImporter.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/DjangoDataImporter" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/DjangoDataImporter" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\DjangoDataImporter.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\DjangoDataImporter.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # 3 | # Django Data Importer documentation build configuration file, created by 4 | # sphinx-quickstart on Sun Mar 31 13:42:50 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | # If extensions (or modules to document with autodoc) are in another directory, 15 | # add these directories to sys.path here. If the directory is relative to the 16 | # documentation root, use os.path.abspath to make it absolute, like shown here. 17 | 18 | # -- General configuration ----------------------------------------------------- 19 | 20 | # If your documentation needs a minimal Sphinx version, state it here. 21 | # needs_sphinx = '1.0' 22 | 23 | # Add any Sphinx extension module names here, as strings. They can be extensions 24 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 25 | import os 26 | 27 | extensions = [ 28 | "sphinx.ext.autodoc", 29 | "sphinx.ext.todo", 30 | "sphinx.ext.coverage", 31 | "sphinx.ext.pngmath", 32 | "sphinx.ext.mathjax", 33 | "sphinx.ext.ifconfig", 34 | "sphinx.ext.viewcode", 35 | ] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ["_templates"] 39 | 40 | # The suffix of source filenames. 41 | source_suffix = ".rst" 42 | 43 | # Github configuration 44 | # 'sphinxtogithub', 45 | sphinx_to_github = True 46 | sphinx_to_github_verbose = True 47 | 48 | # The encoding of source files. 49 | # source_encoding = 'utf-8-sig' 50 | 51 | # The master toctree document. 52 | master_doc = "index" 53 | 54 | # General information about the project. 55 | project = u"Django Data Importer" 56 | copyright = u"2013, Valder Gallo" 57 | 58 | # The version info for the project you're documenting, acts as replacement for 59 | # |version| and |release|, also used in various other places throughout the 60 | # built documents. 61 | # 62 | # The short X.Y version. 63 | 64 | # The language for content autogenerated by Sphinx. Refer to documentation 65 | # for a list of supported languages. 66 | # language = None 67 | 68 | # There are two options for replacing |today|: either, you set today to some 69 | # non-false value, then it is used: 70 | # today = '' 71 | # Else, today_fmt is used as the format for a strftime call. 72 | # today_fmt = '%B %d, %Y' 73 | 74 | # List of patterns, relative to source directory, that match files and 75 | # directories to ignore when looking for source files. 76 | exclude_patterns = [] 77 | 78 | # The reST default role (used for this markup: `text`) to use for all documents. 79 | # default_role = None 80 | 81 | # If true, '()' will be appended to :func: etc. cross-reference text. 82 | # add_function_parentheses = True 83 | 84 | # If true, the current module name will be prepended to all description 85 | # unit titles (such as .. function::). 86 | # add_module_names = True 87 | 88 | # If true, sectionauthor and moduleauthor directives will be shown in the 89 | # output. They are ignored by default. 90 | # show_authors = False 91 | 92 | # The name of the Pygments (syntax highlighting) style to use. 93 | pygments_style = "sphinx" 94 | 95 | # A list of ignored prefixes for module index sorting. 96 | # modindex_common_prefix = [] 97 | 98 | # If true, keep warnings as "system message" paragraphs in the built documents. 99 | # keep_warnings = False 100 | 101 | 102 | # -- Options for HTML output --------------------------------------------------- 103 | 104 | # The theme to use for HTML and HTML Help pages. See the documentation for 105 | # a list of builtin themes. 106 | on_rtd = os.environ.get("READTHEDOCS", False) 107 | 108 | # if not on_rtd: # only import and set the theme if we're building docs locally 109 | if on_rtd: # only import and set the theme if we're building docs locally 110 | # import sphinx_rtd_theme 111 | html_theme = "sphinx_rtd_theme" 112 | # html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 113 | else: 114 | html_theme = "default" 115 | 116 | # Theme options are theme-specific and customize the look and feel of a theme 117 | # further. For a list of options available for each theme, see the 118 | # documentation. 119 | # html_theme_options = {} 120 | 121 | # Add any paths that contain custom themes here, relative to this directory. 122 | # html_theme_path = [] 123 | 124 | # The name for this set of Sphinx documents. If None, it defaults to 125 | # " v documentation". 126 | # html_title = None 127 | 128 | # A shorter title for the navigation bar. Default is the same as html_title. 129 | # html_short_title = None 130 | 131 | # The name of an image file (relative to this directory) to place at the top 132 | # of the sidebar. 133 | # html_logo = None 134 | 135 | # The name of an image file (within the static path) to use as favicon of the 136 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 137 | # pixels large. 138 | # html_favicon = None 139 | 140 | # Add any paths that contain custom static files (such as style sheets) here, 141 | # relative to this directory. They are copied after the builtin static files, 142 | # so a file named "default.css" will overwrite the builtin "default.css". 143 | html_static_path = ["_static"] 144 | 145 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 146 | # using the given strftime format. 147 | # html_last_updated_fmt = '%b %d, %Y' 148 | 149 | # If true, SmartyPants will be used to convert quotes and dashes to 150 | # typographically correct entities. 151 | # html_use_smartypants = True 152 | 153 | # Custom sidebar templates, maps document names to template names. 154 | # html_sidebars = {} 155 | 156 | # Additional templates that should be rendered to pages, maps page names to 157 | # template names. 158 | # html_additional_pages = {} 159 | 160 | # If false, no module index is generated. 161 | # html_domain_indices = True 162 | 163 | # If false, no index is generated. 164 | # html_use_index = True 165 | 166 | # If true, the index is split into individual pages for each letter. 167 | # html_split_index = False 168 | 169 | # If true, links to the reST sources are added to the pages. 170 | # html_show_sourcelink = True 171 | 172 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 173 | # html_show_sphinx = True 174 | 175 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 176 | # html_show_copyright = True 177 | 178 | # If true, an OpenSearch description file will be output, and all pages will 179 | # contain a tag referring to it. The value of this option must be the 180 | # base URL from which the finished HTML is served. 181 | # html_use_opensearch = '' 182 | 183 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 184 | # html_file_suffix = None 185 | 186 | # Output file base name for HTML help builder. 187 | htmlhelp_basename = "DjangoDataImporterdoc" 188 | 189 | 190 | # -- Options for LaTeX output -------------------------------------------------- 191 | 192 | latex_elements = { 193 | # The paper size ('letterpaper' or 'a4paper'). 194 | #'papersize': 'letterpaper', 195 | # The font size ('10pt', '11pt' or '12pt'). 196 | #'pointsize': '10pt', 197 | # Additional stuff for the LaTeX preamble. 198 | #'preamble': '', 199 | } 200 | 201 | # Grouping the document tree into LaTeX files. List of tuples 202 | # (source start file, target name, title, author, documentclass [howto/manual]). 203 | latex_documents = [ 204 | ( 205 | "index", 206 | "DjangoDataImporter.tex", 207 | u"Django Data Importer Documentation", 208 | u"Valder Gallo", 209 | "manual", 210 | ), 211 | ] 212 | 213 | # The name of an image file (relative to this directory) to place at the top of 214 | # the title page. 215 | # latex_logo = None 216 | 217 | # For "manual" documents, if this is true, then toplevel headings are parts, 218 | # not chapters. 219 | # latex_use_parts = False 220 | 221 | # If true, show page references after internal links. 222 | # latex_show_pagerefs = False 223 | 224 | # If true, show URL addresses after external links. 225 | # latex_show_urls = False 226 | 227 | # Documents to append as an appendix to all manuals. 228 | # latex_appendices = [] 229 | 230 | # If false, no module index is generated. 231 | # latex_domain_indices = True 232 | 233 | 234 | # -- Options for manual page output -------------------------------------------- 235 | 236 | # One entry per manual page. List of tuples 237 | # (source start file, name, description, authors, manual section). 238 | man_pages = [ 239 | ( 240 | "index", 241 | "djangodataimporter", 242 | u"Django Data Importer Documentation", 243 | [u"Valder Gallo"], 244 | 1, 245 | ) 246 | ] 247 | 248 | # If true, show URL addresses after external links. 249 | # man_show_urls = False 250 | 251 | 252 | # -- Options for Texinfo output ------------------------------------------------ 253 | 254 | # Grouping the document tree into Texinfo files. List of tuples 255 | # (source start file, target name, title, author, 256 | # dir menu entry, description, category) 257 | texinfo_documents = [ 258 | ( 259 | "index", 260 | "DjangoDataImporter", 261 | u"Django Data Importer Documentation", 262 | u"Valder Gallo", 263 | "DjangoDataImporter", 264 | "One line description of project.", 265 | "Miscellaneous", 266 | ), 267 | ] 268 | 269 | # Documents to append as an appendix to all manuals. 270 | # texinfo_appendices = [] 271 | 272 | # If false, no module index is generated. 273 | # texinfo_domain_indices = True 274 | 275 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 276 | # texinfo_show_urls = 'footnote' 277 | 278 | # If true, do not generate a @detailmenu in the "Top" node's menu. 279 | # texinfo_no_detailmenu = False 280 | 281 | 282 | # --- Django configuration ------------------------------------------------- 283 | 284 | import os 285 | import sys 286 | import inspect 287 | from django.utils.html import strip_tags 288 | from django.utils.encoding import force_text 289 | 290 | BASEDIR = os.path.abspath(os.path.dirname(__file__) + "../example/") 291 | sys.path.append(BASEDIR) 292 | 293 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") 294 | 295 | import data_importer 296 | 297 | version = data_importer.__version__ 298 | # The full version, including alpha/beta/rc tags. 299 | release = data_importer.__version__ 300 | 301 | 302 | def process_docstring(app, what, name, obj, options, lines): 303 | # This causes import errors if left outside the function 304 | from django.db import models 305 | 306 | # Only look at objects that inherit from Django's base model class 307 | if inspect.isclass(obj) and issubclass(obj, models.Model): 308 | # Grab the field list from the meta class 309 | fields = obj._meta._fields() 310 | 311 | for field in fields: 312 | # Decode and strip any html out of the field's help text 313 | help_text = strip_tags(force_text(field.help_text)) 314 | 315 | # Decode and capitalize the verbose name, for use if there isn't 316 | # any help text 317 | verbose_name = force_text(field.verbose_name).capitalize() 318 | 319 | if help_text: 320 | # Add the model field to the end of the docstring as a param 321 | # using the help text as the description 322 | lines.append(u":param {0!s}: {1!s}".format(field.attname, help_text)) 323 | else: 324 | # Add the model field to the end of the docstring as a param 325 | # using the verbose name as the description 326 | lines.append(u":param {0!s}: {1!s}".format(field.attname, verbose_name)) 327 | 328 | # Add the field's type to the docstring 329 | lines.append( 330 | u":type {0!s}: {1!s}".format(field.attname, type(field).__name__) 331 | ) 332 | 333 | # Return the extended docstring 334 | return lines 335 | 336 | 337 | def setup(app): 338 | # Register the docstring processor with sphinx 339 | app.connect("autodoc-process-docstring", process_docstring) 340 | -------------------------------------------------------------------------------- /docs/source/core.rst: -------------------------------------------------------------------------------- 1 | Core 2 | ==== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | core/base 8 | core/default_settings 9 | core/exceptions 10 | core/descriptor 11 | -------------------------------------------------------------------------------- /docs/source/core/base.rst: -------------------------------------------------------------------------------- 1 | Base 2 | ==== 3 | 4 | .. automodule:: data_importer.core.base 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | -------------------------------------------------------------------------------- /docs/source/core/default_settings.rst: -------------------------------------------------------------------------------- 1 | Default Settings 2 | ================ 3 | 4 | Customize data_importer decoders 5 | 6 | **DATA_IMPORTER_EXCEL_DECODER** 7 | Default value is cp1252 8 | 9 | **DATA_IMPORTER_DECODER** 10 | Default value is UTF-8 11 | 12 | **DATA_IMPORTER_TASK** 13 | Need Celery installed to set importers as Task 14 | `default value is False` 15 | 16 | **DATA_IMPORTER_QUEUE** 17 | Set Celery Queue in DataImpoter Tasks 18 | `default value is DataImporter` 19 | 20 | **DATA_IMPORTER_TASK_LOCK_EXPIRE** 21 | Set task expires time 22 | `default value is 60 * 20` 23 | -------------------------------------------------------------------------------- /docs/source/core/descriptor.rst: -------------------------------------------------------------------------------- 1 | Descriptors 2 | =========== 3 | 4 | .. automodule:: data_importer.core.descriptor 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | -------------------------------------------------------------------------------- /docs/source/core/exceptions.rst: -------------------------------------------------------------------------------- 1 | Exceptions 2 | ========== 3 | 4 | .. automodule:: data_importer.core.exceptions 5 | :members: 6 | -------------------------------------------------------------------------------- /docs/source/forms.rst: -------------------------------------------------------------------------------- 1 | Data Importer Forms 2 | =================== 3 | 4 | Is one simple django.ModelForm with content to upload content 5 | 6 | :Parameters: 7 | * content (`FileField`) File uploaded 8 | 9 | 10 | FileUploadForm 11 | -------------- 12 | 13 | .. automodule:: data_importer.forms 14 | :members: 15 | :undoc-members: 16 | :inherited-members: 17 | -------------------------------------------------------------------------------- /docs/source/importers.rst: -------------------------------------------------------------------------------- 1 | Importers 2 | ========= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | importers/xlsimporter 8 | importers/xlsximporter 9 | importers/xmlimporter 10 | importers/generic 11 | -------------------------------------------------------------------------------- /docs/source/importers/base.rst: -------------------------------------------------------------------------------- 1 | BaseImporter 2 | ============ 3 | 4 | .. automodule:: data_importer.importers.base 5 | :members: 6 | :inherited-members: 7 | -------------------------------------------------------------------------------- /docs/source/importers/csvimporter.rst: -------------------------------------------------------------------------------- 1 | CSVImporter 2 | =========== 3 | 4 | .. automodule:: data_importer.importers.csv_importer 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | -------------------------------------------------------------------------------- /docs/source/importers/generic.rst: -------------------------------------------------------------------------------- 1 | GenericImporter 2 | =============== 3 | 4 | .. automodule:: data_importer.importers.generic 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | -------------------------------------------------------------------------------- /docs/source/importers/xlsimporter.rst: -------------------------------------------------------------------------------- 1 | XLSImporter 2 | =========== 3 | 4 | .. automodule:: data_importer.importers.xls_importer 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | -------------------------------------------------------------------------------- /docs/source/importers/xlsximporter.rst: -------------------------------------------------------------------------------- 1 | XLSImporter 2 | =========== 3 | 4 | .. automodule:: data_importer.importers.xlsx_importer 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | -------------------------------------------------------------------------------- /docs/source/importers/xmlimporter.rst: -------------------------------------------------------------------------------- 1 | XMLImporter 2 | =========== 3 | 4 | .. automodule:: data_importer.importers.xml_importer 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Django Data Importer documentation master file, created by 2 | sphinx-quickstart on Sun Mar 31 13:42:50 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Django Data Importer's documentation! 7 | ================================================ 8 | 9 | Data importer documentation help 10 | 11 | Contents: 12 | 13 | * `Create Issues`_ 14 | * `Report Bugs`_ 15 | * `GitHub Project`_ 16 | 17 | .. toctree:: 18 | :maxdepth: 2 19 | 20 | readme 21 | core 22 | readers 23 | importers 24 | forms 25 | models 26 | views 27 | 28 | 29 | 30 | Indices and tables 31 | ================== 32 | 33 | * :ref:`genindex` 34 | * :ref:`modindex` 35 | * :ref:`search` 36 | 37 | .. _Report Bugs: https://github.com/valdergallo/data-importer/issues 38 | .. _Create Issues: https://github.com/valdergallo/data-importer/issues 39 | .. _GitHub Project: https://github.com/valdergallo/data-importer 40 | -------------------------------------------------------------------------------- /docs/source/models.rst: -------------------------------------------------------------------------------- 1 | Data Importer Models 2 | ==================== 3 | 4 | FileHistory 5 | ----------- 6 | 7 | .. automodule:: data_importer.models 8 | :members: 9 | :inherited-members: 10 | -------------------------------------------------------------------------------- /docs/source/readers/csvreader.rst: -------------------------------------------------------------------------------- 1 | CSVReader 2 | ========= 3 | 4 | .. automodule:: data_importer.readers.csv_importer 5 | :members: 6 | :inherited-members: 7 | -------------------------------------------------------------------------------- /docs/source/readers/xlsreader.rst: -------------------------------------------------------------------------------- 1 | XLSReader 2 | ========= 3 | 4 | .. automodule:: data_importer.readers.xls_importer 5 | :members: 6 | :inherited-members: 7 | -------------------------------------------------------------------------------- /docs/source/readers/xlsxreader.rst: -------------------------------------------------------------------------------- 1 | XLSXReader 2 | ========== 3 | 4 | .. automodule:: data_importer.readers.xlsx_importer 5 | :members: 6 | :inherited-members: 7 | -------------------------------------------------------------------------------- /docs/source/readers/xmlreader.rst: -------------------------------------------------------------------------------- 1 | XMLReader 2 | ========== 3 | 4 | .. automodule:: data_importer.readers.xml_importer 5 | :members: 6 | :inherited-members: 7 | -------------------------------------------------------------------------------- /docs/source/readme.rst: -------------------------------------------------------------------------------- 1 | Django Data Importer 2 | ==================== 3 | 4 | **Django Data Importer** is a tool which allow you to transform easily a CSV, XML, XLS and XLSX file into a python object or a django model instance. It is based on the django-style declarative model. 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | Documentation and usage 10 | ----------------------- 11 | 12 | Read docs online in Read the Docs: 13 | 14 | https://django-data-importer.readthedocs.org/ 15 | 16 | 17 | You can generate everything at the above site in your local folder by:: 18 | 19 | $ cd doc 20 | $ make html 21 | $ open _build/html/index.html # Or your preferred web browser 22 | 23 | 24 | 25 | Installation 26 | ------------ 27 | 28 | Use either ``easy_install``:: 29 | 30 | easy_install data-importer 31 | 32 | or ``pip``:: 33 | 34 | pip install data-importer 35 | 36 | 37 | Settings 38 | -------- 39 | 40 | Customize data_importer decoders 41 | 42 | **DATA_IMPORTER_EXCEL_DECODER** 43 | Default value is cp1252 44 | 45 | **DATA_IMPORTER_DECODER** 46 | Default value is UTF-8 47 | 48 | Migrations 49 | ---------- 50 | 51 | **Django 1.7+** 52 | MIGRATION_MODULES = {'data_importer': 'data_importer.django_migrations'} 53 | 54 | 55 | Basic example 56 | ------------- 57 | 58 | Consider the following: 59 | 60 | >>> from data_importer.importers import CSVImporter 61 | >>> class MyCSVImporterModel(CSVImporter): 62 | ... fields = ['name', 'age', 'length'] 63 | ... class Meta: 64 | ... delimiter = ";" 65 | 66 | You declare a ``MyCSVImporterModel`` which will match to a CSV file like this: 67 | 68 | Anthony;27;1.75 69 | 70 | To import the file or any iterable object, just do: 71 | 72 | >>> my_csv_list = MyCSVImporterModel(source="my_csv_file_name.csv") 73 | >>> row, first_line = my_csv_list.cleaned_data[0] 74 | >>> first_line['age'] 75 | 27 76 | 77 | Without an explicit declaration, data and columns are matched in the same 78 | order:: 79 | 80 | Anthony --> Column 0 --> Field 0 --> name 81 | 27 --> Column 1 --> Field 1 --> age 82 | 1.75 --> Column 2 --> Field 2 --> length 83 | 84 | Django Model 85 | ------------ 86 | 87 | If you now want to interact with a django model, you just have to add a **Meta.model** option to the class meta. 88 | 89 | >>> from django.db import models 90 | >>> class MyModel(models.Model): 91 | ... name = models.CharField(max_length=150) 92 | ... age = models.CharField(max_length=150) 93 | ... length = models.CharField(max_length=150) 94 | 95 | >>> from data_importer.importers import CSVImporter 96 | >>> from data_importer.model import MyModel 97 | >>> class MyCSVImporterModel(CSVImporter): 98 | ... class Meta: 99 | ... delimiter = ";" 100 | ... model = MyModel 101 | 102 | That will automatically match to the following django model. 103 | 104 | *The django model should be imported in the model* 105 | 106 | .. py:class:: Meta 107 | 108 | **delimiter** 109 | define the delimiter of the csv file. 110 | If you do not set one, the sniffer will try yo find one itself. 111 | 112 | **ignore_first_line** 113 | Skip the first line if True. 114 | 115 | **model** 116 | If defined, the importer will create an instance of this model. 117 | 118 | **raise_errors** 119 | If set to True, an error in a imported line will stop the loading. 120 | 121 | **exclude** 122 | Exclude fields from list fields to import 123 | 124 | **transaction** `(beta) not tested` 125 | Use transaction to save objects 126 | 127 | **ignore_empty_lines** 128 | Not validate empty lines 129 | 130 | 131 | Django XML 132 | ------------ 133 | 134 | If you now want to interact with a django model, you just have to add a **Meta.model** option to the class meta. 135 | 136 | XML file example: 137 | 138 | .. code-block:: guess 139 | 140 | 141 | 142 | Rocky Balboa 143 | 40 144 | 1.77 145 | 146 | 147 | Chuck Norris 148 | 73 149 | 1.78 150 | 151 | 152 | 153 | >>> from django.db import models 154 | >>> class MyModel(models.Model): 155 | ... name = models.CharField(max_length=150) 156 | ... age = models.CharField(max_length=150) 157 | ... height = models.CharField(max_length=150) 158 | 159 | >>> from data_importer.importers import XMLImporter 160 | >>> from data_importer.model import MyModel 161 | >>> class MyCSVImporterModel(XMLImporter): 162 | ... root = 'file' 163 | ... class Meta: 164 | ... model = MyModel 165 | 166 | That will automatically match to the following django model. 167 | 168 | *The django model should be imported in the model* 169 | 170 | 171 | 172 | .. py:class:: Meta 173 | 174 | **model** 175 | If defined, the importer will create an instance of this model. 176 | 177 | **raise_errors** 178 | If set to True, an error in a imported line will stop the loading. 179 | 180 | **exclude** 181 | Exclude fields from list fields to import 182 | 183 | **transaction** `(beta) not tested` 184 | Use transaction to save objects 185 | 186 | 187 | Django XLS/XLSX 188 | ---------------- 189 | 190 | My XLS/XLSX file can be imported too 191 | 192 | +---------+---------+---------+---------+ 193 | | Header1 | Header2 | Header3 | Header4 | 194 | +=========+=========+=========+=========+ 195 | | Teste 1 | Teste 2 | Teste 3 | Teste 4 | 196 | +---------+---------+---------+---------+ 197 | | Teste 1 | Teste 2 | Teste 3 | Teste 4 | 198 | +---------+---------+---------+---------+ 199 | 200 | 201 | This is my model 202 | 203 | >>> from django.db import models 204 | >>> class MyModel(models.Model): 205 | ... header1 = models.CharField(max_length=150) 206 | ... header2 = models.CharField(max_length=150) 207 | ... header3 = models.CharField(max_length=150) 208 | ... header4 = models.CharField(max_length=150) 209 | 210 | This is my class 211 | 212 | >>> from data_importer.importers import XLSImporter 213 | >>> from data_importer.model import MyModel 214 | >>> class MyXLSImporterModel(XLSImporter): 215 | ... class Meta: 216 | ... model = MyModel 217 | 218 | If you are using XLSX you will need use XLSXImporter to made same importer 219 | 220 | >>> from data_importer.importers import XLSXImporter 221 | >>> from data_importer.model import MyModel 222 | >>> class MyXLSXImporterModel(XLSXImporter): 223 | ... class Meta: 224 | ... model = MyModel 225 | 226 | .. py:class:: Meta 227 | 228 | **ignore_first_line** 229 | Skip the first line if True. 230 | 231 | **model** 232 | If defined, the importer will create an instance of this model. 233 | 234 | **raise_errors** 235 | If set to True, an error in a imported line will stop the loading. 236 | 237 | **exclude** 238 | Exclude fields from list fields to import 239 | 240 | **transaction** `(beta) not tested` 241 | Use transaction to save objects 242 | 243 | 244 | Descriptor 245 | ---------- 246 | 247 | Using file descriptor to define fields for large models. 248 | 249 | 250 | import_test.json 251 | 252 | .. code-block:: javascript 253 | 254 | { 255 | 'app_name': 'mytest.Contact', 256 | { 257 | // field name / name on import file or key index 258 | 'name': 'My Name', 259 | 'year': 'My Year', 260 | 'last': 3 261 | } 262 | } 263 | 264 | 265 | model.py 266 | 267 | .. code-block:: python 268 | 269 | class Contact(models.Model): 270 | name = models.CharField(max_length=50) 271 | year = models.CharField(max_length=10) 272 | laster = models.CharField(max_length=5) 273 | phone = models.CharField(max_length=5) 274 | address = models.CharField(max_length=5) 275 | state = models.CharField(max_length=5) 276 | 277 | 278 | importer.py 279 | 280 | .. code-block:: python 281 | 282 | class MyImpoter(BaseImpoter): 283 | class Meta: 284 | config_file = 'import_test.json' 285 | model = Contact 286 | delimiter = ',' 287 | ignore_first_line = True 288 | 289 | 290 | content_file.csv 291 | 292 | .. code-block:: guest 293 | 294 | name,year,last 295 | Test,12,1 296 | Test2,13,2 297 | Test3,14,3 298 | 299 | 300 | TEST 301 | ---- 302 | 303 | +-----------------------+--------------------+-----+ 304 | |Acentuation with XLS | Excel MAC 2011 | OK | 305 | +-----------------------+--------------------+-----+ 306 | |Acentuation with XLS | Excel WIN 2010 | OK | 307 | +-----------------------+--------------------+-----+ 308 | |Acentuation with XLSX | Excel MAC 2011 | OK | 309 | +-----------------------+--------------------+-----+ 310 | |Acentuation with XLSX | Excel WIN 2010 | OK | 311 | +-----------------------+--------------------+-----+ 312 | |Acentuation with CSV | Excel Win 2010 | OK | 313 | +-----------------------+--------------------+-----+ 314 | 315 | ----------------------------------------------------------- 316 | 317 | :Python: python 2.7 318 | :Django: 1.6; 1.7 319 | -------------------------------------------------------------------------------- /docs/source/views.rst: -------------------------------------------------------------------------------- 1 | Views 2 | ===== 3 | 4 | DataImporterForm 5 | ---------------- 6 | 7 | Is a mixin of `django.views.generic.edit.FormView` with default template and form 8 | to upload files and importent content. 9 | 10 | :Parameters: 11 | :model: Model where the file will be save 12 | 13 | *By default this values is FileHistory* 14 | :template_name: Template name to be used with FormView 15 | 16 | *By default is data_importer/data_importer.html* 17 | :form_class: Form that will be used to upload file 18 | 19 | *By default this value is FileUploadForm* 20 | :task: Task that will be used to parse file imported 21 | 22 | *By default this value is DataImpoterTask* 23 | 24 | :importer: Must be one data_importer.importers class that will be used to validate data. 25 | 26 | :is_task: Use importer in async mode. 27 | 28 | :success_url: Redirect to success page after importer 29 | 30 | :extra_context: Set extra context values in template 31 | 32 | 33 | Usage example 34 | ------------- 35 | 36 | .. code-block:: python 37 | 38 | class DataImporterCreateView(DataImporterForm): 39 | extra_context = {'title': 'Create Form Data Importer', 40 | 'template_file': 'myfile.csv'} 41 | importer = MyImporterModel -------------------------------------------------------------------------------- /example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valdergallo/data-importer/7d3f1d7209597484e3041768ae01c90b16bcc08f/example/__init__.py -------------------------------------------------------------------------------- /example/django_test_settings.py: -------------------------------------------------------------------------------- 1 | SECRET_KEY = 1 2 | 3 | INSTALLED_APPS = ( 4 | # default 5 | "django.contrib.auth", 6 | "django.contrib.contenttypes", 7 | "django.contrib.sessions", 8 | # extra 9 | "data_importer", 10 | "example", 11 | "tests", 12 | ) 13 | 14 | DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": "test.sqlite"}} 15 | 16 | TEMPLATES = [ 17 | { 18 | "BACKEND": "django.template.backends.django.DjangoTemplates", 19 | "DIRS": [], 20 | "APP_DIRS": True, 21 | }, 22 | ] 23 | 24 | ROOT_URLCONF = "example.urls" 25 | 26 | EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" 27 | -------------------------------------------------------------------------------- /example/manage.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | if __name__ == "__main__": 5 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") 6 | 7 | from django.core.management import execute_from_command_line 8 | 9 | execute_from_command_line(sys.argv) 10 | -------------------------------------------------------------------------------- /example/models.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from django.db import models 3 | 4 | 5 | class Person(models.Model): 6 | first_name = models.CharField(max_length=255) 7 | last_name = models.CharField(max_length=255) 8 | age = models.CharField(max_length=10) 9 | 10 | def __unicode__(self): 11 | return self.first_name 12 | 13 | 14 | class PersonFile(models.Model): 15 | filefield = models.FileField(upload_to="test") 16 | 17 | def __unicode__(self): 18 | return self.filefield 19 | 20 | 21 | class Mercado(models.Model): 22 | item = models.CharField(max_length=50) 23 | qtde = models.IntegerField(default=0) 24 | 25 | def __unicode__(self): 26 | return self.item 27 | 28 | 29 | class Invoice(models.Model): 30 | name = models.CharField(max_length=50) 31 | sales_date = models.DateField() 32 | price = models.FloatField() 33 | 34 | def __unicode__(self): 35 | return self.name 36 | 37 | 38 | class ItemInvoice(models.Model): 39 | invoice = models.ForeignKey(Invoice, on_delete=models.CASCADE) 40 | name = models.CharField(max_length=50) 41 | 42 | def __unicode__(self): 43 | return self.name 44 | -------------------------------------------------------------------------------- /example/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {{ title|default:"Example" }} 5 | 6 | 7 | {% block contents %} 8 | 9 | {% endblock %} 10 | 11 | 12 | -------------------------------------------------------------------------------- /example/urls.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from django.conf.urls import patterns, include, url 3 | from django.conf import settings 4 | from django.conf.urls.static import static 5 | 6 | # Uncomment the next two lines to enable the admin: 7 | from django.contrib import admin 8 | 9 | admin.autodiscover() 10 | 11 | from example.views import DataImporterCreateView 12 | 13 | urlpatterns = ( 14 | patterns( 15 | "", 16 | url(r"^$", DataImporterCreateView.as_view(), name="data_importer"), 17 | # Uncomment the admin/doc line below to enable admin documentation: 18 | url(r"^admin/doc/", include("django.contrib.admindocs.urls")), 19 | # Uncomment the next line to enable the admin: 20 | url(r"^admin/", include(admin.site.urls)), 21 | ) 22 | + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) 23 | ) 24 | -------------------------------------------------------------------------------- /example/views.py: -------------------------------------------------------------------------------- 1 | from data_importer.views import DataImporterForm 2 | from data_importer.importers import CSVImporter 3 | from example.models import Person 4 | 5 | 6 | class ExampleCSVImporter(CSVImporter): 7 | class Meta: 8 | model = Person 9 | delimiter = ";" 10 | ignore_first_line = True 11 | 12 | 13 | class DataImporterCreateView(DataImporterForm): 14 | extra_context = { 15 | "title": "Create Form Data Importer", 16 | "template_file": "myfile.csv", 17 | "success_message": "File uploaded successfully", 18 | } 19 | importer = ExampleCSVImporter 20 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | asgiref==3.5.0 2 | attrs==21.4.0 3 | backports.zoneinfo==0.2.1 4 | coverage==6.3.2 5 | data-importer==3.1.5 6 | distlib==0.3.4 7 | Django==2.2.27 8 | et-xmlfile==1.1.0 9 | filelock==3.6.0 10 | flake8==4.0.1 11 | iniconfig==1.1.1 12 | mccabe==0.6.1 13 | mock==4.0.3 14 | openpyxl==3.0.9 15 | packaging==21.3 16 | platformdirs==2.5.1 17 | pluggy==1.0.0 18 | py==1.11.0 19 | pycodestyle==2.8.0 20 | pyflakes==2.4.0 21 | pyparsing==3.0.7 22 | pytest==7.0.1 23 | pytest-cov==3.0.0 24 | pytest-django==3.10.0 25 | six==1.16.0 26 | sqlparse==0.4.4 27 | toml==0.10.2 28 | tomli==2.0.1 29 | tox==3.24.5 30 | tox-pyenv==1.1.0 31 | virtualenv==20.13.3 32 | xlrd==2.0.1 33 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file=README.rst 3 | 4 | [tool:pytest] 5 | DJANGO_SETTINGS_MODULE=example.django_test_settings 6 | django_find_project=false -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import os 3 | import sys 4 | from setuptools import setup, find_packages 5 | from setuptools.command.test import test as TestCommand 6 | import data_importer 7 | 8 | 9 | def readme(): 10 | try: 11 | os.system("pandoc --from=markdown --to=rst README.md -o README.rst") 12 | with open("README.rst") as f: 13 | return f.read() 14 | except Exception: 15 | return """**Django Data Importer** is a tool which allow you to transform easily a CSV, XML, XLS and XLSX file into a python object or a django model instance. It is based on the django-style declarative model. Read More: (https://django-data-importer.readthedocs.io/en/latest/)""" 16 | 17 | 18 | class PyTest(TestCommand): 19 | def finalize_options(self): 20 | TestCommand.finalize_options(self) 21 | self.test_args = ["data_importer", "tests", "--cov=data_importer", "-vrsx"] 22 | self.test_suite = True 23 | 24 | def run_tests(self): 25 | # import here, cause outside the eggs aren't loaded 26 | import pytest 27 | 28 | errno = pytest.main(self.test_args) 29 | sys.exit(errno) 30 | 31 | 32 | setup( 33 | name="data-importer", 34 | url="https://github.com/valdergallo/data-importer", 35 | download_url="https://github.com/valdergallo/data-importer/tarball/{0!s}/".format( 36 | data_importer.__version__ 37 | ), 38 | author="valdergallo", 39 | author_email="valdergallo@gmail.com", 40 | keywords="Django Data Importer XLS XLSX CSV XML", 41 | description="Simple library to easily import data with Django", 42 | license="BSD", 43 | long_description=readme(), 44 | classifiers=[ 45 | "Framework :: Django", 46 | "Operating System :: OS Independent", 47 | "Topic :: Utilities", 48 | ], 49 | version=data_importer.__version__, 50 | install_requires=[ 51 | "django>=1.4", 52 | "openpyxl>=2.4.0", 53 | "xlrd>=1.0.0", 54 | ], 55 | tests_require=[ 56 | "pytest>=3.0.2", 57 | "pytest-cov>=2.3.1", 58 | "pytest-django>=2.9.1", 59 | "mock>=2.0.0", 60 | "django>=1.4", 61 | "six>=1.10", 62 | "openpyxl>=2.4.0", 63 | "xlrd>=1.0.0", 64 | ], 65 | cmdclass={"test": PyTest}, 66 | zip_safe=False, 67 | platforms="any", 68 | package_dir={"": "."}, 69 | packages=find_packages( 70 | ".", exclude=["tests", "*.tests", "docs", "example", "media"] 71 | ), 72 | package_data={"": ["templates/data_importer.html", "templates/my_upload.html"]}, 73 | ) 74 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # python 2 | -------------------------------------------------------------------------------- /tests/data/Workbook3.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valdergallo/data-importer/7d3f1d7209597484e3041768ae01c90b16bcc08f/tests/data/Workbook3.xlsx -------------------------------------------------------------------------------- /tests/data/invoice.csv: -------------------------------------------------------------------------------- 1 | name;data;price 2 | test;2014-01-01;23,98 3 | -------------------------------------------------------------------------------- /tests/data/person_test.csv: -------------------------------------------------------------------------------- 1 | first_name;last_name;age 2 | Eldo;Rock;28 3 | Jose;Aldo;18 4 | Carlos;Nunas;29 5 | -------------------------------------------------------------------------------- /tests/data/ptbr_test.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valdergallo/data-importer/7d3f1d7209597484e3041768ae01c90b16bcc08f/tests/data/ptbr_test.xls -------------------------------------------------------------------------------- /tests/data/ptbr_test.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valdergallo/data-importer/7d3f1d7209597484e3041768ae01c90b16bcc08f/tests/data/ptbr_test.xlsx -------------------------------------------------------------------------------- /tests/data/ptbr_test_mac.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valdergallo/data-importer/7d3f1d7209597484e3041768ae01c90b16bcc08f/tests/data/ptbr_test_mac.csv -------------------------------------------------------------------------------- /tests/data/ptbr_test_win.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valdergallo/data-importer/7d3f1d7209597484e3041768ae01c90b16bcc08f/tests/data/ptbr_test_win.csv -------------------------------------------------------------------------------- /tests/data/test.csv: -------------------------------------------------------------------------------- 1 | coisa;test;new 2 | 1;test;l1 3 | 2;test;l2 4 | -------------------------------------------------------------------------------- /tests/data/test.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valdergallo/data-importer/7d3f1d7209597484e3041768ae01c90b16bcc08f/tests/data/test.xls -------------------------------------------------------------------------------- /tests/data/test.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valdergallo/data-importer/7d3f1d7209597484e3041768ae01c90b16bcc08f/tests/data/test.xlsx -------------------------------------------------------------------------------- /tests/data/test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Rocky Balboa 4 | 40 5 | 1.77 6 | 7 | 8 | Chuck Norris 9 | 73 10 | 1.78 11 | 12 | 13 | -------------------------------------------------------------------------------- /tests/data/test_json_descriptor.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model": "mytest.Contact", 4 | "fields":[ 5 | "name", 6 | "year", 7 | "last" 8 | ] 9 | }, 10 | { 11 | "model": "mytest.State", 12 | "fields":[ 13 | "name", 14 | "abbr" 15 | ] 16 | } 17 | ] 18 | -------------------------------------------------------------------------------- /tests/data/test_music.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | some filename.mp3 4 | Gogo (after 3.0) 5 | 131 6 | 7 | 8 | another filename.mp3 9 | iTunes 10 | 128 11 | 12 | 13 | -------------------------------------------------------------------------------- /tests/test_base.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from io import StringIO 4 | from data_importer.importers import BaseImporter 5 | from data_importer.importers import CSVImporter 6 | from data_importer.importers.base import objclass2dict 7 | from data_importer.importers.base import convert_alphabet_to_number 8 | from data_importer.importers.base import reduce_list 9 | from django.test import TestCase 10 | from unittest import skipIf 11 | import data_importer 12 | import django 13 | 14 | source_content = StringIO("header1,header2\ntest1,1\ntest2,2\ntest3,3\ntest4,4") 15 | 16 | 17 | class TestBaseImportMeta(TestCase): 18 | def setUp(self): 19 | class MyCSVImporter(CSVImporter): 20 | fields = [ 21 | "test_field", 22 | "test2_field", 23 | "test3_field", 24 | ] 25 | 26 | class Meta: 27 | exclude = ["test2_field", "test3_field"] 28 | 29 | self.importer = MyCSVImporter(source=None) 30 | 31 | def test_meta_values(self): 32 | self.assertEqual( 33 | self.importer.Meta.get("exclude"), ["test2_field", "test3_field"] 34 | ) 35 | 36 | def test_get_author(self): 37 | self.assertEqual( 38 | data_importer.__author__, "Valder Gallo " 39 | ) 40 | 41 | def test_get_doc(self): 42 | self.assertEqual(data_importer.__doc__, "Data Importer") 43 | 44 | def test_get_meta_higher_doc(self): 45 | self.assertEqual(BaseImporter.Meta.__doc__, "Importer configurations") 46 | 47 | def test_get_meta_lower_doc(self): 48 | self.assertEqual(BaseImporter.meta.__doc__, "Is same to use .Meta") 49 | 50 | def test_raise_error_in_process_row(self): 51 | row = ["test1", "test2", "test3"] 52 | values = ["test1", "test2", "test3", "test3", "test3", "test3"] 53 | base = BaseImporter() 54 | base.fields = [] 55 | self.assertRaises(TypeError, base.process_row(row, values), "Invalid Line: 2") 56 | 57 | def test_ignore_empty_lines(self): 58 | base = BaseImporter() 59 | row = ["test1", "test2", "test3"] 60 | values = [False, False, False] 61 | base.fields = row 62 | base.Meta.ignore_empty_lines = True 63 | 64 | self.assertFalse(base.process_row(row, values)) 65 | 66 | def test_private_values(self): 67 | base = CSVImporter() 68 | 69 | self.assertFalse(base._error) 70 | self.assertFalse(base._cleaned_data) 71 | self.assertFalse(base._fields) 72 | self.assertFalse(base._reader) 73 | self.assertFalse(base._excluded) 74 | self.assertFalse(base._readed) 75 | 76 | def test_meta_class_values(self): 77 | self.assertEqual(self.importer.Meta.exclude, ["test2_field", "test3_field"]) 78 | 79 | def test_meta_silent_attributeerror_values(self): 80 | self.assertFalse(self.importer.Meta.test) 81 | 82 | def test_fields(self): 83 | self.assertEqual( 84 | list(self.importer.fields), 85 | [ 86 | "test_field", 87 | ], 88 | ) 89 | 90 | def test_objclass2dict(self): 91 | class Meta: 92 | test_1 = 1 93 | test_2 = 2 94 | test_3 = 3 95 | 96 | return_dict = objclass2dict(Meta) 97 | self.assertEqual(return_dict, {"test_1": 1, "test_2": 2, "test_3": 3}) 98 | 99 | 100 | class ImportersTests(TestCase): 101 | def test_xls_importers(self): 102 | import data_importer 103 | 104 | self.assertTrue(data_importer.importers.XLSImporter) 105 | 106 | def test_xlsx_importers(self): 107 | import data_importer 108 | 109 | self.assertTrue(data_importer.importers.XLSXImporter) 110 | 111 | def test_base_importers(self): 112 | import data_importer 113 | 114 | self.assertTrue(data_importer.importers.XMLImporter) 115 | 116 | 117 | class TestClassObjToLazyDict(TestCase): 118 | def setUp(self): 119 | from data_importer.importers.base import objclass2dict 120 | 121 | class MyTestClass: 122 | test_field = "test" 123 | 124 | self.testclass = objclass2dict(MyTestClass) 125 | 126 | def test_get_object(self): 127 | self.assertTrue(self.testclass.test_field) 128 | 129 | def test_get_false_without_raises(self): 130 | self.assertFalse(self.testclass.not_raise_error) 131 | 132 | 133 | class TestReadContent(TestCase): 134 | def setUp(self): 135 | class MyCSVImporter(CSVImporter): 136 | fields = [ 137 | "test_field", 138 | "test2_field", 139 | "test3_field", 140 | ] 141 | 142 | class Meta: 143 | exclude = ["test3_field"] 144 | delimiter = "," 145 | raise_errors = True 146 | 147 | def clean_test_field(self, value): 148 | return str(value).upper() 149 | 150 | self.source_content = source_content 151 | self.source_content.seek(0) 152 | self.importer = MyCSVImporter(source=self.source_content) 153 | 154 | def test_read_content(self): 155 | self.assertTrue(self.importer.is_valid(), self.importer.errors) 156 | 157 | def test_meta_lower(self): 158 | self.assertEqual(self.importer.meta.delimiter, ",") 159 | 160 | def test_is_valid(self): 161 | self.assertTrue(self.importer.is_valid()) 162 | 163 | def test_read_content_first_line(self): 164 | self.assertEqual( 165 | self.importer.cleaned_data[0], 166 | (1, {"test2_field": "header2", "test_field": "HEADER1"}), 167 | self.importer.cleaned_data[0], 168 | ) 169 | 170 | def test_errors(self): 171 | self.assertFalse(self.importer.errors) 172 | self.assertFalse(self.importer._error) 173 | 174 | def test_start_fields(self): 175 | self.importer.start_fields() 176 | self.assertEqual(self.importer.fields, ["test_field", "test2_field"]) 177 | 178 | def test_raise_error_on_clean(self): 179 | class MyCSVImporterClean(CSVImporter): 180 | fields = [ 181 | "test", 182 | ] 183 | 184 | def clean_test(self, value): 185 | value.coisa = 1 186 | 187 | importer_error = MyCSVImporterClean( 188 | source=[ 189 | "test1", 190 | ] 191 | ) 192 | 193 | self.assertFalse(importer_error.is_valid()) 194 | # test get row 195 | self.assertEqual(importer_error.errors[0][0], 1) 196 | # test get error type 197 | self.assertEqual(importer_error.errors[0][1], "ValidationError") 198 | # test get error message 199 | self.assertIn("object has no attribute coisa", importer_error.errors[0][2]) 200 | 201 | def test_read_content_skip_first_line(self): 202 | class MyCSVImporter(CSVImporter): 203 | fields = [ 204 | "test_field", 205 | "test_number_field", 206 | "test3_field", 207 | ] 208 | 209 | class Meta: 210 | exclude = ["test3_field"] 211 | delimiter = "," 212 | raise_errors = True 213 | ignore_first_line = True 214 | 215 | def clean_test_field(self, value): 216 | return str(value).upper() 217 | 218 | self.source_content.seek(0) 219 | importer = MyCSVImporter(source=self.source_content) 220 | self.assertTrue(importer.is_valid(), importer.errors) 221 | self.assertEqual( 222 | importer.cleaned_data[0], 223 | (1, {"test_number_field": "1", "test_field": "TEST1"}), 224 | importer.cleaned_data[0], 225 | ) 226 | 227 | def test_exclude_with_tupla(self): 228 | class MyCSVImporter(CSVImporter): 229 | fields = [ 230 | "test_field", 231 | "test_number_field", 232 | "test3_field", 233 | ] 234 | 235 | class Meta: 236 | exclude = ("test3_field",) 237 | delimiter = "," 238 | raise_errors = True 239 | ignore_first_line = True 240 | 241 | def clean_test_field(self, value): 242 | return str(value).upper() 243 | 244 | self.source_content.seek(0) 245 | importer = MyCSVImporter(source=self.source_content) 246 | self.assertTrue(importer.is_valid(), importer.errors) 247 | self.assertEqual( 248 | importer.cleaned_data[0], 249 | (1, {"test_number_field": "1", "test_field": "TEST1"}), 250 | importer.cleaned_data[0], 251 | ) 252 | 253 | 254 | class MyBaseImporter(BaseImporter): 255 | fields = ("name", "value") 256 | 257 | class Meta: 258 | delimiter = "," 259 | 260 | 261 | class BaseImporterTest(TestCase): 262 | @skipIf(django.VERSION < (1, 4), "not supported in this library version") 263 | def test_raise_not_implemented(self): 264 | with self.assertRaises(NotImplementedError): 265 | instance = MyBaseImporter(source=source_content) 266 | instance.set_reader() 267 | 268 | 269 | class TestConvertLetterToNumber(TestCase): 270 | def test_convert_text_lower_to_number(self): 271 | r = convert_alphabet_to_number("a") 272 | self.assertEqual(r, 0) 273 | 274 | def test_convert_text_upper_to_number(self): 275 | r = convert_alphabet_to_number("C") 276 | self.assertEqual(r, 2) 277 | 278 | def test_convert_worlds_to_number(self): 279 | r = convert_alphabet_to_number("ACDC") 280 | self.assertEqual(r, 1342) 281 | 282 | 283 | class TestReduceList(TestCase): 284 | def test_reduce_list(self): 285 | list_values = [1, 2, 3, 4, 5, 6] 286 | list_key = [1, 3, 6] 287 | reduced = reduce_list(list_key, list_values) 288 | self.assertEqual(reduced, [2, 4]) 289 | 290 | def test_reduce_list_two(self): 291 | list_values = [1, 2, 3, 4, 5, 6] 292 | list_key = [0, 2, 3] 293 | reduced = reduce_list(list_key, list_values) 294 | self.assertEqual(reduced, [1, 3, 4]) 295 | -------------------------------------------------------------------------------- /tests/test_base_model.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from django.test import TestCase 4 | from data_importer.importers import CSVImporter 5 | import os 6 | from django.core.files import File as DjangoFile 7 | from example.models import Person 8 | from example.models import Mercado 9 | from example.models import PersonFile 10 | from example.models import Invoice 11 | from distutils.version import StrictVersion 12 | import django 13 | import io 14 | 15 | 16 | LOCAL_DIR = os.path.dirname(__file__) 17 | 18 | 19 | person_content = """first_name,last_name,age\ntest_first_name_1,test_last_name_1,age1\ntest_first_name_2,test_last_name_2,age2\ntest_first_name_3,test_last_name_3,age3""" 20 | 21 | 22 | class TestBaseWithModel(TestCase): 23 | def setUp(self): 24 | class TestMeta(CSVImporter): 25 | class Meta: 26 | model = Person 27 | delimiter = "," 28 | raise_errors = True 29 | ignore_first_line = True 30 | 31 | self.importer = TestMeta(source=person_content.split("\n")) 32 | 33 | def test_get_fields_from_model(self): 34 | self.assertEqual(self.importer.fields, ["first_name", "last_name", "age"]) 35 | 36 | def test_values_is_valid(self): 37 | self.assertTrue(self.importer.is_valid()) 38 | 39 | def test_cleaned_data_content(self): 40 | content = { 41 | "first_name": "test_first_name_1", 42 | "last_name": "test_last_name_1", 43 | "age": "age1", 44 | } 45 | self.assertEqual(self.importer.cleaned_data[0], (1, content)) 46 | 47 | def test_source_importer_file(self): 48 | base = CSVImporter(source=io.open("test.txt", "w")) 49 | self.assertEqual(type(base._source), io.TextIOWrapper, type(base._source)) 50 | 51 | def test_source_importer_list(self): 52 | base = CSVImporter(source=["test1", "test2"]) 53 | self.assertEqual(type(base._source), list, type(base._source)) 54 | 55 | def test_source_importer_django_file(self): 56 | person = PersonFile() 57 | person.filefield = DjangoFile(io.open("test.txt", "w")) 58 | 59 | base = CSVImporter(source=person.filefield) 60 | self.assertEqual(type(base._source), io.BufferedReader, type(base._source)) 61 | 62 | def test_save_data_content(self): 63 | for row, data in self.importer.cleaned_data: 64 | instace = Person(**data) 65 | instace.save() 66 | self.assertTrue(instace.id) 67 | 68 | def tearDown(self): 69 | try: 70 | os.remove("test.txt") 71 | except Exception: 72 | pass 73 | 74 | 75 | class TestPTBRCSVImporter(TestCase): 76 | def setUp(self): 77 | class TestMeta(CSVImporter): 78 | class Meta: 79 | ignore_first_line = True 80 | delimiter = ";" 81 | model = Mercado 82 | 83 | self.csv_file = os.path.join(LOCAL_DIR, "data/ptbr_test_win.csv") 84 | self.importer = TestMeta(source=self.csv_file) 85 | 86 | def test_values_is_valid(self): 87 | self.assertTrue(self.importer.is_valid()) 88 | 89 | def test_count_rows(self): 90 | self.assertEqual(len(self.importer.cleaned_data), 4) 91 | 92 | def test_cleaned_data_content(self): 93 | content = { 94 | "item": "Caça", 95 | "qtde": "1", 96 | } 97 | 98 | self.assertEqual( 99 | self.importer.cleaned_data[0], (1, content), self.importer.cleaned_data[0] 100 | ) 101 | 102 | content = { 103 | "item": "Amanhã", 104 | "qtde": "2", 105 | } 106 | 107 | self.assertEqual( 108 | self.importer.cleaned_data[1], (2, content), self.importer.cleaned_data 109 | ) 110 | 111 | content = { 112 | "item": "Qüanto", 113 | "qtde": "3", 114 | } 115 | 116 | self.assertEqual( 117 | self.importer.cleaned_data[2], (3, content), self.importer.cleaned_data 118 | ) 119 | 120 | content = { 121 | "item": "Será", 122 | "qtde": "4", 123 | } 124 | 125 | self.assertEqual( 126 | self.importer.cleaned_data[3], (4, content), self.importer.cleaned_data 127 | ) 128 | 129 | 130 | class TestModelValidator(TestCase): 131 | def setUp(self): 132 | class TestMeta(CSVImporter): 133 | class Meta: 134 | ignore_first_line = True 135 | delimiter = ";" 136 | model = Invoice 137 | 138 | self.csv_file = os.path.join(LOCAL_DIR, "data/invoice.csv") 139 | self.importer = TestMeta(source=self.csv_file) 140 | 141 | def test_values_is_valid(self): 142 | self.assertFalse(self.importer.is_valid()) 143 | 144 | def test_errors_values(self): 145 | self.importer.is_valid() 146 | # DJANGO_VERSION = StrictVersion(django.get_version()) 147 | error = [(1, "ValidationError", "Field (price) 23,98 value must be a float.")] 148 | self.assertEqual(self.importer.errors, error, self.importer.cleaned_data) 149 | -------------------------------------------------------------------------------- /tests/test_csv_importer.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from django.test import TestCase 4 | from data_importer.importers.base import objclass2dict 5 | from data_importer.importers import CSVImporter 6 | from io import StringIO 7 | import data_importer 8 | 9 | source_content = StringIO("header1,header2\ntest1,1\ntest2,2\ntest3,3\ntest4,4") 10 | 11 | 12 | class TestBaseImportMeta(TestCase): 13 | def setUp(self): 14 | class TestMeta(CSVImporter): 15 | fields = [ 16 | "test_field", 17 | "test2_field", 18 | "test3_field", 19 | ] 20 | 21 | class Meta: 22 | exclude = ["test2_field", "test3_field"] 23 | 24 | self.importer = TestMeta(source=None) 25 | 26 | def test_meta_values(self): 27 | self.assertEqual( 28 | self.importer.Meta.get("exclude"), ["test2_field", "test3_field"] 29 | ) 30 | 31 | def test_get_author(self): 32 | self.assertEqual( 33 | data_importer.__author__, "Valder Gallo " 34 | ) 35 | 36 | def test_get_doc(self): 37 | self.assertEqual(data_importer.__doc__, "Data Importer") 38 | 39 | def test_private_values(self): 40 | base = CSVImporter() 41 | 42 | self.assertFalse(base._error) 43 | self.assertFalse(base._cleaned_data) 44 | self.assertFalse(base._fields) 45 | self.assertFalse(base._reader) 46 | self.assertFalse(base._excluded) 47 | self.assertFalse(base._readed) 48 | 49 | def test_meta_class_values(self): 50 | self.assertEqual(self.importer.Meta.exclude, ["test2_field", "test3_field"]) 51 | 52 | def test_meta_silent_attributeerror_values(self): 53 | self.assertFalse(self.importer.Meta.test) 54 | 55 | def test_fields(self): 56 | self.assertEqual( 57 | list(self.importer.fields), 58 | [ 59 | "test_field", 60 | ], 61 | ) 62 | 63 | def test_objclass2dict(self): 64 | class Meta: 65 | test_1 = 1 66 | test_2 = 2 67 | test_3 = 3 68 | 69 | return_dict = objclass2dict(Meta) 70 | self.assertEqual(return_dict, {"test_1": 1, "test_2": 2, "test_3": 3}) 71 | 72 | 73 | class TestImporters(TestCase): 74 | def test_xls_importers(self): 75 | import data_importer 76 | 77 | self.assertTrue(data_importer.importers.XLSImporter) 78 | 79 | def test_xlsx_importers(self): 80 | import data_importer 81 | 82 | self.assertTrue(data_importer.importers.XLSXImporter) 83 | 84 | def test_base_importers(self): 85 | import data_importer 86 | 87 | self.assertTrue(data_importer.importers.XMLImporter) 88 | 89 | 90 | class TestClassObjToLazyDict(TestCase): 91 | def setUp(self): 92 | from data_importer.importers.base import objclass2dict 93 | 94 | class MyTestClass: 95 | test_field = "test" 96 | 97 | self.testclass = objclass2dict(MyTestClass) 98 | 99 | def test_get_object(self): 100 | self.assertTrue(self.testclass.test_field) 101 | 102 | def test_get_false_without_raises(self): 103 | self.assertFalse(self.testclass.not_raise_error) 104 | 105 | 106 | class TestReadContent(TestCase): 107 | def setUp(self): 108 | class TestMeta(CSVImporter): 109 | fields = [ 110 | "test_field", 111 | "test2_field", 112 | "test3_field", 113 | ] 114 | 115 | class Meta: 116 | exclude = ["test3_field"] 117 | delimiter = "," 118 | raise_errors = True 119 | 120 | def clean_test_field(self, value): 121 | return str(value).upper() 122 | 123 | self.source_content = source_content 124 | self.source_content.seek(0) 125 | self.importer = TestMeta(source=self.source_content) 126 | 127 | def test_read_content(self): 128 | self.assertTrue(self.importer.is_valid(), self.importer.errors) 129 | 130 | def test_meta_lower(self): 131 | self.assertEqual(self.importer.meta.delimiter, ",") 132 | 133 | def test_is_valid(self): 134 | self.assertTrue(self.importer.is_valid()) 135 | 136 | def test_read_content_first_line(self): 137 | self.assertEqual( 138 | self.importer.cleaned_data[0], 139 | (1, {"test2_field": "header2", "test_field": "HEADER1"}), 140 | self.importer.cleaned_data[0], 141 | ) 142 | 143 | def test_errors(self): 144 | self.assertFalse(self.importer.errors) 145 | self.assertFalse(self.importer._error) 146 | 147 | def test_start_fields(self): 148 | self.importer.start_fields() 149 | self.assertEqual(self.importer.fields, ["test_field", "test2_field"]) 150 | 151 | def test_raise_error_on_clean(self): 152 | class TestMetaClean(CSVImporter): 153 | fields = [ 154 | "test", 155 | ] 156 | 157 | def clean_test(self, value): 158 | value.coisa = 1 159 | 160 | importer_error = TestMetaClean( 161 | source=[ 162 | "test1", 163 | ] 164 | ) 165 | 166 | self.assertFalse(importer_error.is_valid()) 167 | # test get row 168 | self.assertEqual(importer_error.errors[0][0], 1) 169 | # test get error type 170 | self.assertEqual(importer_error.errors[0][1], "ValidationError") 171 | # test get error message 172 | self.assertIn("object has no attribute coisa", importer_error.errors[0][2]) 173 | 174 | def test_read_content_skip_first_line(self): 175 | class TestMeta(CSVImporter): 176 | fields = [ 177 | "test_field", 178 | "test_number_field", 179 | "test3_field", 180 | ] 181 | 182 | class Meta: 183 | exclude = ["test3_field"] 184 | delimiter = "," 185 | raise_errors = True 186 | ignore_first_line = True 187 | 188 | def clean_test_field(self, value): 189 | return str(value).upper() 190 | 191 | self.source_content.seek(0) 192 | importer = TestMeta(source=self.source_content) 193 | self.assertTrue(importer.is_valid(), importer.errors) 194 | self.assertEqual( 195 | importer.cleaned_data[0], 196 | (1, {"test_number_field": "1", "test_field": "TEST1"}), 197 | importer.cleaned_data[0], 198 | ) 199 | 200 | def test_error_not_is_cleaned_data(self): 201 | class TestMetaClean(CSVImporter): 202 | fields = [ 203 | "test", 204 | ] 205 | 206 | def clean_test(self, value): 207 | value.coisa = 1 208 | 209 | importer_error = TestMetaClean( 210 | source=[ 211 | "test1", 212 | ] 213 | ) 214 | self.assertFalse(importer_error.is_valid()) 215 | self.assertEqual(importer_error.cleaned_data, ()) 216 | -------------------------------------------------------------------------------- /tests/test_descriptor.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | import os 4 | from django.test import TestCase 5 | from data_importer.core.descriptor import ReadDescriptor 6 | from data_importer.core.descriptor import InvalidDescriptor 7 | from data_importer.core.descriptor import InvalidModel 8 | from data_importer.importers.base import BaseImporter 9 | 10 | 11 | BASEDIR = os.path.dirname(__file__) 12 | JSON_FILE = os.path.abspath(os.path.join(BASEDIR, "data/test_json_descriptor.json")) 13 | 14 | 15 | class ReadDescriptorTestCase(TestCase): 16 | def setUp(self): 17 | self.descriptor = ReadDescriptor(file_name=JSON_FILE, model_name="Contact") 18 | 19 | def test_readed_file(self): 20 | self.assertTrue(self.descriptor.source) 21 | 22 | def test_get_fields(self): 23 | self.assertEqual(self.descriptor.get_fields(), ["name", "year", "last"]) 24 | 25 | def test_invalid_model(self): 26 | descriptor = ReadDescriptor(file_name=JSON_FILE, model_name="TestInvalidModel") 27 | self.assertRaises(InvalidModel, lambda: descriptor.get_model()) 28 | 29 | def test_invalid_file(self): 30 | self.assertRaises( 31 | InvalidDescriptor, 32 | lambda: ReadDescriptor( 33 | file_name="invalid_file.er", model_name="TestInvalidModel" 34 | ), 35 | ) 36 | 37 | 38 | class MyBaseImport(BaseImporter): 39 | class Meta: 40 | delimiter = ";" 41 | ignore_first_line = True 42 | descriptor = JSON_FILE 43 | descriptor_model = "Contact" 44 | 45 | def set_reader(self): 46 | return 47 | 48 | 49 | class TestDescriptionUsingBaseImporter(TestCase): 50 | def setUp(self): 51 | self.importer = MyBaseImport(source=None) 52 | 53 | def test_get_fields(self): 54 | self.assertEqual(self.importer.fields, ["name", "year", "last"]) 55 | -------------------------------------------------------------------------------- /tests/test_dict_fields.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from io import StringIO 4 | from data_importer.importers import CSVImporter 5 | from django.test import TestCase 6 | from collections import OrderedDict 7 | 8 | 9 | source_content = StringIO( 10 | "header1,header2,header3\ntest1,1,exclude1\ntest2,2,exclude2\ntest3,3,exclude3\ntest4,4,exclude4" 11 | ) 12 | 13 | 14 | class TestReadContent(TestCase): 15 | def setUp(self): 16 | class TestMeta(CSVImporter): 17 | fields = OrderedDict( 18 | ( 19 | ("test_field", 0), 20 | ("test2_field", 1), 21 | ("test3_field", 2), 22 | ) 23 | ) 24 | 25 | class Meta: 26 | exclude = ["test3_field"] 27 | delimiter = "," 28 | raise_errors = True 29 | 30 | def clean_test_field(self, value): 31 | return str(value).upper() 32 | 33 | self.source_content = source_content 34 | self.source_content.seek(0) 35 | self.importer = TestMeta(source=self.source_content) 36 | 37 | def test_read_content(self): 38 | self.assertTrue(self.importer.is_valid(), self.importer.errors) 39 | 40 | def test_meta_lower(self): 41 | self.assertEqual(self.importer.meta.delimiter, ",") 42 | 43 | def test_is_valid(self): 44 | self.assertTrue(self.importer.is_valid()) 45 | 46 | def test_read_content_first_line(self): 47 | self.assertEqual( 48 | self.importer.cleaned_data[0], 49 | (1, OrderedDict({"test_field": "HEADER1", "test2_field": "header2"})), 50 | ) 51 | 52 | def test_errors(self): 53 | self.assertFalse(self.importer.errors) 54 | self.assertFalse(self.importer._error) 55 | 56 | def test_start_fields(self): 57 | self.importer.start_fields() 58 | self.assertEqual(self.importer.fields, ["test_field", "test2_field"]) 59 | 60 | def test_raise_error_on_clean(self): 61 | class TestMetaClean(CSVImporter): 62 | fields = [ 63 | "test", 64 | ] 65 | 66 | def clean_test(self, value): 67 | value.coisa = 1 68 | 69 | importer_error = TestMetaClean( 70 | source=[ 71 | "test1", 72 | ] 73 | ) 74 | 75 | self.assertFalse(importer_error.is_valid()) 76 | # test get row 77 | self.assertEqual(importer_error.errors[0][0], 1) 78 | # test get error type 79 | self.assertEqual(importer_error.errors[0][1], "ValidationError") 80 | # test get error message 81 | self.assertIn("object has no attribute coisa", importer_error.errors[0][2]) 82 | 83 | def test_read_content_skip_first_line(self): 84 | class TestMeta(CSVImporter): 85 | fields = OrderedDict( 86 | ( 87 | ("test_field", "A"), 88 | ("test_number_field", "B"), 89 | ("test3_field", "c"), 90 | ) 91 | ) 92 | 93 | class Meta: 94 | exclude = ["test3_field"] 95 | delimiter = "," 96 | raise_errors = True 97 | ignore_first_line = True 98 | 99 | def clean_test_field(self, value): 100 | return str(value).upper() 101 | 102 | self.source_content.seek(0) 103 | importer = TestMeta(source=self.source_content) 104 | self.assertTrue(importer.is_valid(), importer.errors) 105 | should_be = ( 106 | 1, 107 | OrderedDict([("test_field", "TEST1"), ("test_number_field", "1")]), 108 | ) 109 | self.assertEqual(importer.cleaned_data[0], should_be) 110 | 111 | def test_exclude_with_tupla(self): 112 | class TestMeta(CSVImporter): 113 | fields = OrderedDict( 114 | ( 115 | ("test_field", 0), 116 | ("test_number_field", "b"), 117 | ("test3_field", "c"), 118 | ) 119 | ) 120 | 121 | class Meta: 122 | exclude = ("test3_field",) 123 | delimiter = "," 124 | raise_errors = True 125 | ignore_first_line = True 126 | 127 | def clean_test_field(self, value): 128 | return str(value).upper() 129 | 130 | self.source_content.seek(0) 131 | importer = TestMeta(source=self.source_content) 132 | self.assertTrue(importer.is_valid(), importer.errors) 133 | should_be = ( 134 | 1, 135 | OrderedDict([("test_field", "TEST1"), ("test_number_field", "1")]), 136 | ) 137 | self.assertEqual(importer.cleaned_data[0], should_be) 138 | -------------------------------------------------------------------------------- /tests/test_foreignkey.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | import os 4 | from django.test import TestCase 5 | from data_importer.importers import CSVImporter 6 | from example.models import ItemInvoice 7 | 8 | 9 | LOCAL_DIR = os.path.dirname(__file__) 10 | 11 | 12 | person_content = ( 13 | """invoice,name\n1,test_last_name_1\n2,test_last_name_2\n3,test_last_name_3""" 14 | ) 15 | 16 | 17 | class TestForeignKeyWithModel(TestCase): 18 | def setUp(self): 19 | class TestMeta(CSVImporter): 20 | class Meta: 21 | model = ItemInvoice 22 | delimiter = "," 23 | raise_errors = False 24 | ignore_first_line = True 25 | 26 | self.importer = TestMeta(source=person_content.split("\n")) 27 | 28 | def test_get_fields_from_model(self): 29 | self.assertEqual(self.importer.fields, ["invoice", "name"]) 30 | 31 | def test_values_is_valid(self): 32 | self.assertFalse(self.importer.is_valid()) 33 | -------------------------------------------------------------------------------- /tests/test_forms.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from django.test import TestCase 4 | from mock import Mock 5 | import sys 6 | from data_importer import forms 7 | from imp import reload 8 | 9 | 10 | class TestFileUploadForm(TestCase): 11 | def setUp(self): 12 | self.form = forms.FileUploadForm() 13 | 14 | def test_invalid_form(self): 15 | self.assertFalse(self.form.is_valid()) 16 | 17 | def test_default_importer(self): 18 | self.assertEqual(self.form.importer, None) 19 | 20 | def test_default_importer_task(self): 21 | self.assertEqual(self.form.is_task, True) 22 | 23 | 24 | class TestTaskImporter(TestCase): 25 | def test_celery_importer(self): 26 | sys.modules["celery"] = Mock() 27 | reload(forms) 28 | self.assertEqual(forms.HAS_CELERY, True) 29 | -------------------------------------------------------------------------------- /tests/test_generic_importer.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | import mock 4 | import os 5 | import django 6 | from django.test import TestCase 7 | from unittest import skipIf 8 | from data_importer.importers.generic import GenericImporter 9 | from data_importer.readers.xls_reader import XLSReader 10 | from data_importer.readers.xlsx_reader import XLSXReader 11 | from data_importer.readers.csv_reader import CSVReader 12 | from data_importer.readers.xml_reader import XMLReader 13 | from data_importer.core.exceptions import UnsuportedFile 14 | from data_importer.models import FileHistory 15 | from example.models import Invoice 16 | 17 | 18 | LOCAL_DIR = os.path.dirname(__file__) 19 | 20 | 21 | class TestGenericImporterSetup(TestCase): 22 | def setUp(self): 23 | self.xls_file = os.path.join(LOCAL_DIR, "data/test.xls") 24 | self.xlsx_file = os.path.join(LOCAL_DIR, "data/test.xlsx") 25 | self.csv_file = os.path.join(LOCAL_DIR, "data/test.csv") 26 | self.xml_file = os.path.join(LOCAL_DIR, "data/test.xml") 27 | self.unsuported_file = os.path.join(LOCAL_DIR, "data/test_json_descriptor.json") 28 | 29 | def test_xls_reader_set(self): 30 | importer = GenericImporter(source=self.xls_file) 31 | self.assertEqual(importer.get_reader_class(), XLSReader) 32 | 33 | def test_xlsx_reader_set(self): 34 | importer = GenericImporter(source=self.xlsx_file) 35 | self.assertEqual(importer.get_reader_class(), XLSXReader) 36 | 37 | def test_csv_reader_set(self): 38 | importer = GenericImporter(source=self.csv_file) 39 | self.assertEqual(importer.get_reader_class(), CSVReader) 40 | 41 | def test_xml_reader_set(self): 42 | importer = GenericImporter(source=self.xml_file) 43 | self.assertEqual(importer.get_reader_class(), XMLReader) 44 | 45 | def test_getting_source_file_extension(self): 46 | importer = GenericImporter(source=self.csv_file) 47 | self.assertEqual(importer.get_source_file_extension(), "csv") 48 | 49 | @skipIf(django.VERSION < (1, 4), "not supported in this library version") 50 | def test_unsuported_raise_error_message(self): 51 | with self.assertRaises(UnsuportedFile): 52 | GenericImporter(source=self.unsuported_file) 53 | 54 | def test_import_with_file_instance(self): 55 | file_instance = open(self.csv_file) 56 | importer = GenericImporter(source=file_instance) 57 | self.assertEqual(importer.get_source_file_extension(), "csv") 58 | 59 | def test_import_with_model_instance(self): 60 | file_mock = mock.MagicMock(spec=FileHistory, name="FileHistoryMock") 61 | file_mock.file_upload = "/media/test.csv" 62 | 63 | importer = GenericImporter(source=file_mock) 64 | self.assertEqual(importer.get_source_file_extension(), "csv") 65 | 66 | 67 | class CustomerDataImporter(GenericImporter): 68 | class Meta: 69 | model = Invoice 70 | ignore_first_line = True 71 | 72 | 73 | # class TestGenericImporterBehavior(TestCase): 74 | 75 | # def setUp(self): 76 | # self.xls_file = os.path.join(LOCAL_DIR, 'data/test_invalid_lines.xlsx') 77 | 78 | # def test_xlsx_is_not_valid(self): 79 | # instance = CustomerDataImporter(source=self.xls_file) 80 | # self.assertFalse(instance.is_valid()) 81 | 82 | # def test_save_lines_without_errors(self): 83 | # instance = CustomerDataImporter(source=self.xls_file) 84 | # instance.save() 85 | 86 | # count_invoices = Invoice.objects.count() 87 | # self.assertEqual(count_invoices, 6, ('error', count_invoices)) 88 | 89 | # def test_get_three_errors(self): 90 | # instance = CustomerDataImporter(source=self.xls_file) 91 | # instance.is_valid() 92 | # self.assertEqual(len(instance.errors), 3) 93 | -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from data_importer import models 4 | from django.test import TestCase 5 | import mock 6 | from datetime import date 7 | from imp import reload 8 | 9 | 10 | def fake_uuid4(): 11 | return "fake_uuid4" 12 | 13 | 14 | class TestDataImporterModels(TestCase): 15 | @mock.patch("uuid.uuid4", fake_uuid4) 16 | def test_get_random_filename_without_owner(self): 17 | try: 18 | reload(models) 19 | except RuntimeError: 20 | return True 21 | 22 | instance = models.FileHistory() 23 | filename = "test_file.xls" 24 | today = date.today().strftime("%Y/%m/%d") 25 | rand_filename = models.get_random_filename(instance, filename) 26 | 27 | expected_name = "upload_history/anonymous/{0!s}/fake_uuid4.xls".format(today) 28 | self.assertEqual(rand_filename, expected_name) 29 | -------------------------------------------------------------------------------- /tests/test_settings.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from django.test import TestCase 4 | from data_importer.importers.base import ( 5 | DATA_IMPORTER_EXCEL_DECODER, 6 | DATA_IMPORTER_DECODER, 7 | ) 8 | 9 | 10 | class SettingsDataImporterContentTest(TestCase): 11 | def test_data_importer_excel_decoder(self): 12 | self.assertEqual(DATA_IMPORTER_EXCEL_DECODER, "cp1252") 13 | 14 | def test_data_importer_decoder(self): 15 | self.assertEqual(DATA_IMPORTER_DECODER, "utf-8") 16 | -------------------------------------------------------------------------------- /tests/test_tasks.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | import os 4 | from django.test import TestCase 5 | from data_importer.tasks import DataImpoterTask 6 | from django.contrib.auth.models import User 7 | from example.models import Person 8 | from data_importer.importers import CSVImporter 9 | from django.core import mail 10 | 11 | LOCAL_DIR = os.path.dirname(__file__) 12 | 13 | 14 | class MyCSVImporter(CSVImporter): 15 | class Meta: 16 | model = Person 17 | delimiter = ";" 18 | ignore_first_line = True 19 | 20 | 21 | class DataImporterTaskTest(TestCase): 22 | def setUp(self): 23 | owner, _ = User.objects.get_or_create(username="test", email="test@test.com") 24 | source = os.path.join(LOCAL_DIR, "data/person_test.csv") 25 | self.importer = MyCSVImporter 26 | 27 | self.task = DataImpoterTask() 28 | self.task.run(importer=self.importer, source=source, owner=owner) 29 | 30 | def test_task_run(self): 31 | created_person = Person.objects.filter( 32 | first_name="Eldo", last_name="Rock", age="28" 33 | ).exists() 34 | self.assertTrue(created_person) 35 | 36 | def test_task_create_all(self): 37 | self.assertEqual(Person.objects.all().count(), 3) 38 | 39 | def test_send_email(self): 40 | outbox = mail.outbox[0] 41 | self.assertEqual(outbox.body, "Your file was imported with sucess") 42 | self.assertEqual(outbox.to, ["test@test.com"]) 43 | -------------------------------------------------------------------------------- /tests/test_xls_importer.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from django.test import TestCase 4 | from data_importer.importers.xls_importer import XLSImporter 5 | import os 6 | import datetime 7 | from django.db import models 8 | 9 | LOCAL_DIR = os.path.dirname(__file__) 10 | 11 | 12 | # class TestDictXLSImportMeta(TestCase): 13 | # def setUp(self): 14 | # class TestMeta(XLSImporter): 15 | # fields = [ 16 | # 'business_place', 17 | # 'doc_number', 18 | # 'doc_data', 19 | # ] 20 | 21 | # class Meta: 22 | # ignore_first_line = True 23 | 24 | # self.xls_file = os.path.join(LOCAL_DIR, 'data/test.xls') 25 | # self.importer = TestMeta(source=self.xls_file) 26 | 27 | # class TestMetaDict(XLSImporter): 28 | # fields = { 29 | # 'business_place': 'A', 30 | # 'doc_number': 'B', 31 | # 'doc_data': 'C', 32 | # } 33 | 34 | # class Meta: 35 | # ignore_first_line = True 36 | 37 | # self.importer_dict = TestMetaDict(source=self.xls_file) 38 | 39 | # def test_file_path(self): 40 | # self.assertTrue(os.path.exists(self.xls_file)) 41 | 42 | # def test_values_is_valid(self): 43 | # self.assertTrue(self.importer.is_valid()) 44 | # self.assertTrue(self.importer_dict.is_valid()) 45 | 46 | # def test_count_rows(self): 47 | # self.assertEqual(len(self.importer.cleaned_data), 9) 48 | # self.assertEqual(len(self.importer_dict.cleaned_data), 9) 49 | 50 | # def test_cleaned_data_content(self): 51 | # content = { 52 | # 'doc_number': 10000000, 53 | # 'business_place': 'SP', 54 | # 'doc_data': datetime.datetime(1982, 11, 1, 0, 0) 55 | # } 56 | 57 | # self.assertEqual(self.importer.cleaned_data[0], (1, content), self.importer.cleaned_data) 58 | # self.assertEqual(self.importer_dict.cleaned_data[0], (1, content), self.importer_dict.cleaned_data) 59 | 60 | 61 | class InvoiceXLS(models.Model): 62 | business_place = models.CharField(max_length=2) 63 | doc_number = models.IntegerField() 64 | doc_data = models.DateTimeField(max_length=10) 65 | 66 | def save(self, *args, **kwargs): 67 | return self.full_clean() == None 68 | 69 | 70 | class TestModelXLSImporter(TestCase): 71 | def setUp(self): 72 | class TestMeta(XLSImporter): 73 | class Meta: 74 | ignore_first_line = True 75 | model = InvoiceXLS 76 | 77 | self.xls_file = os.path.join(LOCAL_DIR, "data/test.xls") 78 | self.importer = TestMeta(source=self.xls_file) 79 | 80 | class TestMetaDict(XLSImporter): 81 | class Meta: 82 | ignore_first_line = True 83 | model = InvoiceXLS 84 | 85 | self.importer_dict = TestMetaDict(source=self.xls_file) 86 | 87 | def test_values_is_valid(self): 88 | self.assertTrue(self.importer.is_valid()) 89 | self.assertTrue(self.importer_dict.is_valid()) 90 | 91 | def test_count_rows(self): 92 | self.assertEqual(len(self.importer.cleaned_data), 9) 93 | self.assertEqual(len(self.importer_dict.cleaned_data), 9) 94 | 95 | def test_cleaned_data_content(self): 96 | content = { 97 | "doc_number": 10000000, 98 | "business_place": "SP", 99 | "doc_data": datetime.datetime(1982, 11, 1, 0, 0), 100 | } 101 | 102 | self.assertEqual( 103 | self.importer.cleaned_data[0], (1, content), self.importer.cleaned_data 104 | ) 105 | self.assertEqual( 106 | self.importer_dict.cleaned_data[0], 107 | (1, content), 108 | self.importer_dict.cleaned_data, 109 | ) 110 | 111 | def test_save_data(self): 112 | for row, data in self.importer.cleaned_data: 113 | instace = InvoiceXLS(**data) 114 | self.assertTrue(instace.save()) 115 | for row, data in self.importer_dict.cleaned_data: 116 | instace = InvoiceXLS(**data) 117 | self.assertTrue(instace.save()) 118 | 119 | def test_save_importer(self): 120 | self.assertTrue(self.importer.save()) 121 | self.assertTrue(self.importer_dict.save()) 122 | 123 | 124 | class MercadoXLS(models.Model): 125 | item = models.CharField(max_length=50) 126 | qtde = models.IntegerField(default=0) 127 | 128 | def save(self, *args, **kwargs): 129 | return self.full_clean() == None 130 | 131 | 132 | class TestPTBRXLSImporter(TestCase): 133 | def setUp(self): 134 | class TestMeta(XLSImporter): 135 | class Meta: 136 | ignore_first_line = True 137 | model = MercadoXLS 138 | 139 | self.xls_file = os.path.join(LOCAL_DIR, "data/ptbr_test.xls") 140 | self.importer = TestMeta(source=self.xls_file) 141 | 142 | def test_values_is_valid(self): 143 | self.assertTrue(self.importer.is_valid()) 144 | 145 | def test_count_rows(self): 146 | self.assertEqual(len(self.importer.cleaned_data), 4) 147 | 148 | def test_cleaned_data_content(self): 149 | content = { 150 | "item": "Caça", 151 | "qtde": 1, 152 | } 153 | 154 | self.assertEqual( 155 | self.importer.cleaned_data[0], (1, content), self.importer.cleaned_data 156 | ) 157 | 158 | content = { 159 | "item": "Amanhã", 160 | "qtde": 2, 161 | } 162 | 163 | self.assertEqual( 164 | self.importer.cleaned_data[1], (2, content), self.importer.cleaned_data 165 | ) 166 | 167 | content = { 168 | "item": "Qüanto", 169 | "qtde": 3, 170 | } 171 | 172 | self.assertEqual( 173 | self.importer.cleaned_data[2], (3, content), self.importer.cleaned_data 174 | ) 175 | 176 | content = { 177 | "item": "Será", 178 | "qtde": 4, 179 | } 180 | 181 | self.assertEqual( 182 | self.importer.cleaned_data[3], (4, content), self.importer.cleaned_data 183 | ) 184 | -------------------------------------------------------------------------------- /tests/test_xlsx_importer.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | from django.test import TestCase 4 | from data_importer.importers.xlsx_importer import XLSXImporter 5 | import os 6 | import datetime 7 | from django.db import models 8 | 9 | LOCAL_DIR = os.path.dirname(__file__) 10 | 11 | 12 | class TestXLSImportMeta(TestCase): 13 | def setUp(self): 14 | class TestMeta(XLSXImporter): 15 | fields = [ 16 | "business_place", 17 | "doc_number", 18 | "doc_data", 19 | ] 20 | 21 | class Meta: 22 | ignore_first_line = True 23 | 24 | self.xls_file = os.path.join(LOCAL_DIR, "data/test.xlsx") 25 | self.importer = TestMeta(source=self.xls_file) 26 | 27 | def test_file_path(self): 28 | self.assertTrue(os.path.exists(self.xls_file)) 29 | 30 | def test_values_is_valid(self): 31 | self.assertTrue(self.importer.is_valid()) 32 | 33 | def test_count_rows(self): 34 | self.assertEqual(len(self.importer.cleaned_data), 9) 35 | 36 | def test_cleaned_data_content(self): 37 | content = { 38 | "doc_number": 1000000, 39 | "business_place": "SP", 40 | "doc_data": datetime.datetime(1982, 11, 1, 0, 0), 41 | } 42 | 43 | self.assertEqual( 44 | self.importer.cleaned_data[0], (1, content), self.importer.cleaned_data 45 | ) 46 | 47 | 48 | class InvoiceXLSX(models.Model): 49 | business_place = models.CharField(max_length=2) 50 | doc_number = models.IntegerField() 51 | doc_data = models.DateTimeField(max_length=10) 52 | 53 | def save(self, *args, **kwargs): 54 | return self.full_clean() == None 55 | 56 | 57 | class TestModelXLSImporter(TestCase): 58 | def setUp(self): 59 | class TestMeta(XLSXImporter): 60 | class Meta: 61 | ignore_first_line = True 62 | model = InvoiceXLSX 63 | 64 | self.xls_file = os.path.join(LOCAL_DIR, "data/test.xlsx") 65 | self.importer = TestMeta(source=self.xls_file) 66 | 67 | def test_values_is_valid(self): 68 | self.assertTrue(self.importer.is_valid()) 69 | 70 | def test_count_rows(self): 71 | self.assertEqual(len(self.importer.cleaned_data), 9) 72 | 73 | def test_cleaned_data_content(self): 74 | content = { 75 | "doc_number": 1000000, 76 | "business_place": "SP", 77 | "doc_data": datetime.datetime(1982, 11, 1, 0, 0), 78 | } 79 | 80 | self.assertEqual( 81 | self.importer.cleaned_data[0], (1, content), self.importer.cleaned_data 82 | ) 83 | 84 | def test_save_data(self): 85 | for row, data in self.importer.cleaned_data: 86 | instace = InvoiceXLSX(**data) 87 | self.assertTrue(instace.save()) 88 | 89 | def test_save_importer(self): 90 | self.assertTrue(self.importer.save()) 91 | 92 | 93 | class MercadoXLSX(models.Model): 94 | item = models.CharField(max_length=50) 95 | qtde = models.IntegerField(default=0) 96 | 97 | def save(self, *args, **kwargs): 98 | return self.full_clean() == None 99 | 100 | 101 | class TestPTBRXLSImporter(TestCase): 102 | def setUp(self): 103 | class TestMeta(XLSXImporter): 104 | class Meta: 105 | ignore_first_line = True 106 | model = MercadoXLSX 107 | 108 | self.xls_file = os.path.join(LOCAL_DIR, "data/ptbr_test.xlsx") 109 | self.importer = TestMeta(source=self.xls_file) 110 | 111 | def test_values_is_valid(self): 112 | self.assertTrue(self.importer.is_valid()) 113 | 114 | def test_count_rows(self): 115 | self.assertEqual(len(self.importer.cleaned_data), 4) 116 | 117 | def test_cleaned_data_content(self): 118 | content = { 119 | "item": "Caça", 120 | "qtde": 1, 121 | } 122 | 123 | self.assertEqual( 124 | self.importer.cleaned_data[0], (1, content), self.importer.cleaned_data 125 | ) 126 | 127 | content = { 128 | "item": "Amanhã", 129 | "qtde": 2, 130 | } 131 | 132 | self.assertEqual( 133 | self.importer.cleaned_data[1], (2, content), self.importer.cleaned_data 134 | ) 135 | 136 | content = { 137 | "item": "Qüanto", 138 | "qtde": 3, 139 | } 140 | 141 | self.assertEqual( 142 | self.importer.cleaned_data[2], (3, content), self.importer.cleaned_data 143 | ) 144 | 145 | content = { 146 | "item": "Será", 147 | "qtde": 4, 148 | } 149 | 150 | self.assertEqual( 151 | self.importer.cleaned_data[3], (4, content), self.importer.cleaned_data 152 | ) 153 | -------------------------------------------------------------------------------- /tests/test_xml_importer.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import unicode_literals 3 | import os 4 | from django.test import TestCase 5 | from django.db import models 6 | from data_importer.importers.xml_importer import XMLImporter 7 | 8 | LOCAL_DIR = os.path.dirname(__file__) 9 | 10 | sxml = os.path.join(LOCAL_DIR, "data/test_music.xml") 11 | 12 | 13 | class Musics(models.Model): 14 | name = models.CharField(max_length=100) 15 | encoder = models.CharField(max_length=100) 16 | bitrate = models.CharField(max_length=100) 17 | 18 | def save(self, *args, **kwargs): 19 | return self.full_clean() is None 20 | 21 | 22 | class TestXMLImporter(TestCase): 23 | def setUp(self): 24 | class TestXML(XMLImporter): 25 | root = "file" 26 | fields = ["name", "encoder", "bitrate"] 27 | 28 | self.importer = TestXML(source=sxml) 29 | 30 | def test_read_content(self): 31 | self.assertEqual( 32 | self.importer.cleaned_data[0], 33 | ( 34 | 1, 35 | { 36 | "bitrate": "131", 37 | "name": "some filename.mp3", 38 | "encoder": "Gogo (after 3.0)", 39 | }, 40 | ), 41 | ) 42 | 43 | def test_values_is_valid(self): 44 | self.assertTrue(self.importer.is_valid()) 45 | 46 | 47 | class TestXMLCleanImporter(TestCase): 48 | def setUp(self): 49 | class TestXML(XMLImporter): 50 | root = "file" 51 | fields = ["name", "encoder", "bitrate"] 52 | 53 | def clean_name(self, value): 54 | return str(value).upper() 55 | 56 | self.importer = TestXML(source=sxml) 57 | 58 | def test_read_content(self): 59 | self.assertEqual( 60 | self.importer.cleaned_data[0], 61 | ( 62 | 1, 63 | { 64 | "bitrate": "131", 65 | "name": "SOME FILENAME.MP3", 66 | "encoder": "Gogo (after 3.0)", 67 | }, 68 | ), 69 | ) 70 | 71 | def test_values_is_valid(self): 72 | self.assertTrue(self.importer.is_valid()) 73 | 74 | 75 | class TestXMLModelImporter(TestCase): 76 | def setUp(self): 77 | class TestXML(XMLImporter): 78 | root = "file" 79 | 80 | class Meta: 81 | model = Musics 82 | 83 | def clean_name(self, value): 84 | return str(value).upper() 85 | 86 | self.importer = TestXML(source=sxml) 87 | 88 | def test_model_fields(self): 89 | self.assertEqual(self.importer.fields, ["name", "encoder", "bitrate"]) 90 | 91 | def test_read_content(self): 92 | content = { 93 | "bitrate": "131", 94 | "encoder": "Gogo (after 3.0)", 95 | "name": "SOME FILENAME.MP3", 96 | } 97 | self.assertEqual(self.importer.cleaned_data[0], (1, content)) 98 | 99 | def test_values_is_valid(self): 100 | self.assertTrue(self.importer.is_valid()) 101 | 102 | def test_save_data(self): 103 | for row, data in self.importer.cleaned_data: 104 | instace = Musics(**data) 105 | self.assertTrue(instace.save()) 106 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | skipsdist = true 8 | envlist = py3k-django{18,19,20,21,30} 9 | 10 | 11 | [testenv] 12 | # virtualenv not works in shared folder 13 | envdir = {homedir}/.tox/{envname} 14 | 15 | setenv = VIRTUAL_ENV={envdir} 16 | PYTHONPATH={toxinidir} 17 | 18 | commands = pytest 19 | 20 | 21 | # py3k =========================== 22 | 23 | [testenv:py3k-django18] 24 | basepython=python3.7 25 | deps= 26 | django==1.8 27 | pytest>=3.0.2 28 | pytest-cov>=2.3.1 29 | pytest-django==3.10.0 30 | mock>=2.0.0 31 | six>=1.10 32 | openpyxl>=2.4.0 33 | xlrd>=1.0.0 34 | 35 | [testenv:py3k-django19] 36 | basepython=python3.7 37 | deps= 38 | django==1.9 39 | pytest>=3.0.2 40 | pytest-cov>=2.3.1 41 | pytest-django==3.10.0 42 | mock>=2.0.0 43 | six>=1.10 44 | openpyxl>=2.4.0 45 | xlrd>=1.0.0 46 | 47 | [testenv:py3k-django20] 48 | basepython=python3.8 49 | deps= 50 | django==2.0 51 | pytest>=3.0.2 52 | pytest-cov>=2.3.1 53 | pytest-django==3.10.0 54 | mock>=2.0.0 55 | six>=1.10 56 | openpyxl>=2.4.0 57 | xlrd>=1.0.0 58 | 59 | [testenv:py3k-django21] 60 | basepython=python3.8 61 | deps= 62 | django==2.1 63 | pytest>=3.0.2 64 | pytest-cov>=2.3.1 65 | pytest-django>=2.9.1 66 | mock>=2.0.0 67 | six>=1.10 68 | openpyxl>=2.4.0 69 | xlrd>=1.0.0 70 | 71 | [testenv:py3k-django30] 72 | basepython=python3.8 73 | deps= 74 | django==3.0 75 | pytest>=3.0.2 76 | pytest-cov>=2.3.1 77 | pytest-django>=2.9.1 78 | mock>=2.0.0 79 | six>=1.10 80 | openpyxl>=2.4.0 81 | xlrd>=1.0.0 82 | --------------------------------------------------------------------------------