├── .editorconfig ├── .env ├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ └── checks.yml ├── .gitignore ├── .gitlint ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── .vscode ├── launch.json ├── recommended_settings.json └── tasks.json ├── AUTHORS.rst ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── README.md ├── docker-compose.yaml ├── docs ├── _static │ └── images │ │ ├── action-bands-openapi.png │ │ ├── bands-openapi.png │ │ ├── export-status.png │ │ ├── filters-openapi.png │ │ ├── force_import_admin.png │ │ ├── force_import_results.png │ │ ├── import-job-admin.png │ │ └── start_api.png ├── api_admin.rst ├── api_drf.rst ├── api_models.rst ├── authors.rst ├── conf.py ├── contributing.rst ├── extensions.rst ├── getting_started.rst ├── history.rst ├── index.rst ├── installation.rst └── migrate_from_original_import_export.rst ├── import_export_extensions ├── __init__.py ├── admin │ ├── __init__.py │ ├── forms │ │ ├── __init__.py │ │ ├── export_job_admin_form.py │ │ ├── import_admin_form.py │ │ └── import_job_admin_form.py │ ├── mixins │ │ ├── __init__.py │ │ ├── base_mixin.py │ │ ├── export_mixin.py │ │ ├── import_export_mixin.py │ │ ├── import_mixin.py │ │ └── types.py │ ├── model_admins │ │ ├── __init__.py │ │ ├── export_job_admin.py │ │ ├── import_job_admin.py │ │ └── mixins.py │ └── widgets.py ├── api │ ├── __init__.py │ ├── mixins │ │ ├── __init__.py │ │ ├── common.py │ │ ├── export_mixins.py │ │ └── import_mixins.py │ ├── serializers │ │ ├── __init__.py │ │ ├── export_job.py │ │ ├── import_job.py │ │ ├── import_job_details.py │ │ └── progress.py │ └── views │ │ ├── __init__.py │ │ ├── export_job.py │ │ └── import_job.py ├── apps.py ├── fields.py ├── forms.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_alter_exportjob_export_status.py │ ├── 0003_importjob_skip_parse_step.py │ ├── 0004_alter_exportjob_created_by_and_more.py │ ├── 0005_importjob_force_import.py │ ├── 0006_importjob_input_errors_file.py │ ├── 0007_alter_exportjob_result_alter_importjob_result.py │ ├── 0008_alter_exportjob_id_alter_importjob_id.py │ ├── 0009_alter_exportjob_data_file_alter_importjob_data_file.py │ └── __init__.py ├── models │ ├── __init__.py │ ├── core.py │ ├── export_job.py │ ├── import_job.py │ └── tools.py ├── resources.py ├── results.py ├── signals.py ├── static │ └── import_export_extensions │ │ ├── css │ │ ├── admin │ │ │ ├── admin.css │ │ │ └── import_result_diff.css │ │ └── widgets │ │ │ └── progress_bar.css │ │ └── js │ │ ├── admin │ │ └── admin.js │ │ └── widgets │ │ └── progress_bar.js ├── tasks.py ├── templates │ └── admin │ │ └── import_export_extensions │ │ ├── celery_export_results.html │ │ ├── celery_export_status.html │ │ ├── celery_import_results.html │ │ ├── celery_import_status.html │ │ ├── import_job_results.html │ │ └── progress_bar.html ├── utils.py └── widgets.py ├── invocations ├── __init__.py ├── ci.py ├── docs.py └── project.py ├── poetry.lock ├── pyproject.toml ├── tasks.py └── test_project ├── __init__.py ├── celery_app.py ├── conftest.py ├── fake_app ├── __init__.py ├── admin.py ├── api │ ├── __init__.py │ └── views.py ├── apps.py ├── factories.py ├── filters.py ├── migrations │ ├── 0001_initial.py │ └── __init__.py ├── models.py ├── resources.py └── signals.py ├── manage.py ├── settings.py ├── tests ├── __init__.py ├── conftest.py ├── integration_tests │ ├── __init__.py │ ├── test_admin │ │ ├── __init__.py │ │ ├── test_changelist_view.py │ │ ├── test_export │ │ │ ├── __init__.py │ │ │ ├── test_admin_actions.py │ │ │ ├── test_admin_class.py │ │ │ ├── test_celery_endpoints.py │ │ │ └── test_export.py │ │ └── test_import │ │ │ ├── __init__.py │ │ │ ├── test_admin_actions.py │ │ │ ├── test_admin_class.py │ │ │ ├── test_celery_endpoints.py │ │ │ └── test_import.py │ ├── test_api │ │ ├── __init__.py │ │ ├── test_export.py │ │ └── test_import.py │ └── test_resources.py ├── test_export_job.py ├── test_fields.py ├── test_forms.py ├── test_import_job │ ├── __init__.py │ ├── test_cancel.py │ ├── test_import_data.py │ └── test_parse_data.py ├── test_resources.py ├── test_result.py ├── test_utils.py ├── test_views.py └── test_widgets.py ├── urls.py └── wsgi.py /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | charset = utf-8 6 | trim_trailing_whitespace = true 7 | insert_final_newline = true 8 | indent_style = space 9 | indent_size = 2 10 | 11 | [*.py] 12 | indent_size = 4 13 | 14 | [*.txt] 15 | indent_style = tab 16 | indent_size = 4 17 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | # this docker-compose environment file 2 | # https://docs.docker.com/compose/env-file/#environment-file 3 | COMPOSE_PROJECT_NAME=django-import-export-extensions 4 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * nikita.azanov@saritasa.com andrey.otto@saritasa.com stanislav.khlud@saritasa.com egor.toryshak@saritasa.com 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file 2 | version: 2 3 | updates: 4 | - package-ecosystem: github-actions 5 | directory: "/" 6 | schedule: 7 | interval: monthly 8 | time: "00:00" 9 | groups: 10 | github-actions: 11 | patterns: 12 | - "*" 13 | - package-ecosystem: pip 14 | directory: "/" 15 | schedule: 16 | interval: weekly 17 | time: "00:00" 18 | groups: 19 | pip: 20 | patterns: 21 | - "*" 22 | allow: 23 | - dependency-name: "*" 24 | dependency-type: "all" 25 | -------------------------------------------------------------------------------- /.github/workflows/checks.yml: -------------------------------------------------------------------------------- 1 | name: Run tests and style checks 2 | on: 3 | pull_request: 4 | push: 5 | branches: [main] 6 | jobs: 7 | test: 8 | runs-on: ubuntu-latest 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | python-version: 13 | - "3.10" 14 | - "3.11" 15 | - "3.12" 16 | - "3.13" 17 | django-version: 18 | - ">=4.2,<4.3" 19 | - ">=5.0,<5.3" 20 | name: Python ${{ matrix.python-version }} - Django ${{ matrix.django-version }} 21 | steps: 22 | - name: Check out repository code 23 | uses: actions/checkout@v4 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v5 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | - name: Install Poetry 29 | uses: snok/install-poetry@v1 30 | with: 31 | version: latest 32 | virtualenvs-create: true 33 | virtualenvs-in-project: true 34 | installer-parallel: true 35 | - name: Cache poetry dependencies 36 | id: cached-poetry-dependencies 37 | uses: actions/cache@v4 38 | with: 39 | path: .venv 40 | key: ${{ runner.os }}-${{ matrix.python-version }}-poetry-${{ hashFiles('**/poetry.lock') }} 41 | - name: Cache pre-commit 42 | uses: actions/cache@v4 43 | with: 44 | path: ~/.cache/pre-commit 45 | key: ${{ runner.os }}-${{ matrix.python-version }}-pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }} 46 | - name: Install local dependencies 47 | if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' 48 | run: poetry install --no-interaction 49 | - name: Install Django 50 | run: | 51 | poetry run pip install 'django${{ matrix.django-version }}' 52 | - name: Prepare env 53 | run: | 54 | poetry run inv ci.prepare 55 | - name: Run checks ${{ matrix.python-version }} 56 | run: poetry run inv pre-commit.run-hooks 57 | - name: Coveralls Upload 58 | uses: coverallsapp/github-action@v2 59 | with: 60 | github-token: ${{ secrets.GITHUB_TOKEN }} 61 | flag-name: Python${{ matrix.python-version }} - Django${{ matrix.django-version }} 62 | parallel: true 63 | coveralls-finish: 64 | needs: test 65 | if: ${{ always() }} 66 | runs-on: ubuntu-latest 67 | steps: 68 | - name: Coveralls Finished 69 | uses: coverallsapp/github-action@v2 70 | with: 71 | github-token: ${{ secrets.GITHUB_TOKEN }} 72 | parallel-finished: true 73 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | coverage.lcov 47 | *.cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # virtualenv 85 | .venv 86 | venv/ 87 | ENV/ 88 | 89 | # Spyder project settings 90 | .spyderproject 91 | .spyproject 92 | 93 | # Rope project settings 94 | .ropeproject 95 | 96 | # mkdocs documentation 97 | /site 98 | 99 | # mypy 100 | .mypy_cache/ 101 | 102 | # ruff 103 | .ruff_cache/ 104 | 105 | # IDE settings 106 | .vscode/* 107 | !.vscode/recommended_settings.json 108 | !.vscode/tasks.json 109 | !.vscode/launch.json 110 | !.vscode/extensions.json 111 | !.vscode/boilerplate-words.txt 112 | !.vscode/project-related-words.txt 113 | !.vscode/cspell.json 114 | .idea/ 115 | 116 | # Test files 117 | test_project/media 118 | test_project/static 119 | test_project/db.sqlite3 120 | 121 | # Database dumps 122 | *.sql 123 | 124 | # MacOS files 125 | .DS_Store 126 | -------------------------------------------------------------------------------- /.gitlint: -------------------------------------------------------------------------------- 1 | # Gitlint config: https://jorisroovers.com/gitlint/configuration/ 2 | 3 | # Example of commit: 4 | # Add checks to prevent inconsistent data 5 | # 6 | # Explanation, reasons, notes 7 | 8 | [general] 9 | ignore=body-is-missing 10 | ignore-merge-commits=true 11 | 12 | [title-max-length] 13 | line-length=50 14 | 15 | [title-min-length] 16 | min-length=3 17 | 18 | # Make sure that commit title starts with uppercase letter 19 | [title-match-regex] 20 | regex=\A[A-Z] 21 | 22 | [body-max-line-length] 23 | line-length=79 24 | 25 | [body-min-length] 26 | min-length=1 27 | 28 | # Ignore rules if it's work in progress commit 29 | [ignore-by-title] 30 | regex=^(WIP|wip) 31 | 32 | # Ignore certain lines in a commit body that match a regex. 33 | # E.g. Ignore all lines that start with 'Co-authored-By' 34 | [ignore-body-lines] 35 | regex=^Co-authored-by 36 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: end-of-file-fixer 6 | - id: trailing-whitespace 7 | - id: debug-statements 8 | - id: check-merge-conflict 9 | - id: detect-private-key 10 | - repo: https://github.com/astral-sh/ruff-pre-commit 11 | rev: v0.9.1 12 | hooks: 13 | - id: ruff 14 | args: [--fix] 15 | - id: ruff-format 16 | - repo: https://github.com/google/yamlfmt 17 | rev: v0.15.0 18 | hooks: 19 | - id: yamlfmt 20 | - repo: https://github.com/asottile/add-trailing-comma 21 | rev: v3.1.0 22 | hooks: 23 | - id: add-trailing-comma 24 | - repo: https://github.com/jorisroovers/gitlint 25 | rev: v0.19.1 26 | hooks: 27 | - id: gitlint 28 | - repo: https://github.com/rtts/djhtml 29 | rev: 3.0.7 30 | hooks: 31 | - id: djhtml 32 | args: ["--tabwidth=2"] 33 | - id: djcss 34 | args: ["--tabwidth=2"] 35 | - id: djjs 36 | args: ["--tabwidth=2"] 37 | - repo: local 38 | hooks: 39 | - id: check_new_migrations 40 | name: check for new migrations 41 | entry: inv django.check-new-migrations 42 | language: system 43 | pass_filenames: false 44 | types: [file] 45 | stages: [pre-push] 46 | - id: tests 47 | name: run tests 48 | entry: inv pytest.run --params="--numprocesses auto --create-db --cov=." 49 | language: system 50 | pass_filenames: false 51 | types: [python] 52 | stages: [pre-push] 53 | - id: mypy 54 | name: mypy 55 | entry: inv mypy.run 56 | language: system 57 | pass_filenames: false 58 | types: [file] 59 | stages: [pre-push] 60 | - id: package_installation_verify 61 | name: verify package can be installed 62 | entry: pip install --dry-run . 63 | language: system 64 | pass_filenames: false 65 | types: [python] 66 | stages: [pre-push] 67 | - id: doc_build_verify 68 | name: verify that docs could be build 69 | entry: inv docs.build 70 | language: system 71 | pass_filenames: false 72 | stages: [pre-push] 73 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | version: 2 5 | # Set the version of Python and other tools you might need 6 | build: 7 | os: ubuntu-22.04 8 | tools: 9 | python: "3.12" 10 | jobs: 11 | # Job for install dependencies with poetry 12 | # Docs: https://docs.readthedocs.io/en/stable/build-customization.html#install-dependencies-with-poetry 13 | post_create_environment: 14 | # https://python-poetry.org/docs/#installing-manually 15 | - python -m pip install poetry 16 | post_install: 17 | # Install dependencies with 'docs' dependency group 18 | # https://python-poetry.org/docs/managing-dependencies/#dependency-groups 19 | - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs 20 | # Build documentation in the docs/ directory with Sphinx 21 | sphinx: 22 | configuration: docs/conf.py 23 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Python: Django", 6 | "type": "debugpy", 7 | "request": "launch", 8 | "program": "${workspaceFolder}/test_project/manage.py", 9 | "preLaunchTask": "Launch containers and wait for DB", 10 | "args": [ 11 | "runserver_plus", 12 | "localhost:8000", 13 | ], 14 | "django": true, 15 | "justMyCode": false 16 | }, 17 | { 18 | "name": "Python: Django With SQL Logs", 19 | "type": "debugpy", 20 | "request": "launch", 21 | "program": "${workspaceFolder}/test_project/manage.py", 22 | "preLaunchTask": "Launch containers and wait for DB", 23 | "args": [ 24 | "runserver_plus", 25 | "localhost:8000", 26 | "--print-sql" 27 | ], 28 | "django": true, 29 | "justMyCode": false 30 | }, 31 | { 32 | "name": "Python: Celery", 33 | "type": "debugpy", 34 | "request": "launch", 35 | "module": "celery", 36 | "preLaunchTask": "Launch containers and wait for DB", 37 | "args": [ 38 | "--app", 39 | "test_project.celery_app.app", 40 | "worker", 41 | "--beat", 42 | "--scheduler=django", 43 | "--loglevel=info", 44 | ], 45 | "justMyCode": false 46 | }, 47 | { 48 | "name": "Python: Debug Tests", 49 | "type": "debugpy", 50 | "request": "launch", 51 | "program": "${file}", 52 | "purpose": ["debug-test"], 53 | "console": "integratedTerminal", 54 | "justMyCode": false 55 | }, 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /.vscode/recommended_settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "files.exclude": { 3 | "**/__pycache__": true, 4 | "**/.pytest_cache": true, 5 | "**/.mypy_cache": true, 6 | "**/.ruff_cache": true, 7 | "**/htmlcov": true, 8 | }, 9 | 10 | "editor.rulers": [79], 11 | 12 | "editor.bracketPairColorization.enabled": true, 13 | 14 | "python.analysis.typeCheckingMode": "off", 15 | 16 | "python.analysis.inlayHints.functionReturnTypes": true, 17 | "mypy.enabled": false, 18 | 19 | "python.testing.unittestEnabled": false, 20 | "python.testing.pytestEnabled": true, 21 | 22 | "[python]": { 23 | "editor.formatOnSave": true, 24 | "editor.defaultFormatter": "charliermarsh.ruff" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "Launch containers and wait for DB", 6 | "type": "shell", 7 | "command": "inv django.wait-for-database", 8 | "problemMatcher": [], 9 | "group": { 10 | "kind": "build", 11 | "isDefault": false 12 | } 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Author 6 | ------ 7 | 8 | Saritasa, LLC 9 | 10 | Contributors 11 | ------------ 12 | 13 | * TheSuperiorStanislav (Stanislav Khlud) 14 | * yalef (Romaschenko Vladislav) 15 | * NikAzanov (Nikita Azanov) 16 | * ron8mcr (Roman Gorbil) 17 | * Eg0ra (Egor Toryshak) 18 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Contributing 5 | ============ 6 | 7 | Contributions are welcome, and they are greatly appreciated! Every little bit 8 | helps, and credit will always be given. 9 | 10 | You can contribute in many ways: 11 | 12 | Types of Contributions 13 | ---------------------- 14 | 15 | Report Bugs 16 | ~~~~~~~~~~~ 17 | 18 | Report bugs at https://github.com/saritasa-nest/django-import-export-extensions/issues. 19 | 20 | If you are reporting a bug, please include: 21 | 22 | * Any details about your local setup that might be helpful in troubleshooting. 23 | * Detailed steps to reproduce the bug. 24 | 25 | Fix Bugs 26 | ~~~~~~~~ 27 | 28 | Look through the GitHub issues for bugs. Anything tagged with "bug" and "help 29 | wanted" is open to whoever wants to implement it. 30 | 31 | Implement Features 32 | ~~~~~~~~~~~~~~~~~~ 33 | 34 | Look through the GitHub issues for features. Anything tagged with "enhancement" 35 | and "help wanted" is open to whoever wants to implement it. 36 | 37 | Write Documentation 38 | ~~~~~~~~~~~~~~~~~~~ 39 | 40 | ``django-import-export-extensions`` could always use more documentation, whether as part of the 41 | official ``django-import-export-extensions`` docs, in docstrings, or even on the web in blog posts, 42 | articles, and such. 43 | 44 | Submit Feedback 45 | ~~~~~~~~~~~~~~~ 46 | 47 | The best way to send feedback is to file an issue at https://github.com/saritasa-nest/django-import-export-extensions/issues. 48 | 49 | If you are proposing a feature: 50 | 51 | * Explain in detail how it would work. 52 | * Keep the scope as narrow as possible, to make it easier to implement. 53 | * Remember that this is a volunteer-driven project, and that contributions 54 | are welcome :) 55 | 56 | Get Started! 57 | ------------ 58 | 59 | Ready to contribute? Here's how to set up `django-import-export-extensions` for local development. 60 | 61 | 1. Fork the `django-import-export-extensions` repo on GitHub. 62 | 2. Clone your fork locally:: 63 | 64 | git clone git@github.com:your_name_here/django-import-export-extensions.git 65 | 66 | 3. Setup virtual environment: 67 | 68 | Using pyenv:: 69 | 70 | pyenv install 3.13 71 | pyenv shell $(pyenv latest 3.13) 72 | poetry config virtualenvs.in-project true 73 | source .venv/bin/activate && poetry install 74 | 75 | Using uv:: 76 | 77 | uv venv --python 3.13 --prompt django-import-export-extensions --seed 78 | poetry config virtualenvs.in-project true 79 | source .venv/bin/activate && poetry install 80 | 81 | 4. Create a branch for local development:: 82 | 83 | git checkout -b name-of-your-bugfix-or-feature 84 | 85 | Now you can make your changes locally. 86 | 87 | 5. When you're done making changes, check that your changes pass linters and the 88 | tests:: 89 | 90 | inv pre-commit.run-hooks 91 | 92 | 6. Commit your changes and push your branch to GitHub:: 93 | 94 | git add . 95 | git commit -m "Your detailed description of your changes." 96 | git push origin name-of-your-bugfix-or-feature 97 | 98 | 7. Submit a pull request through the GitHub website. 99 | 100 | Starting test project 101 | --------------------- 102 | 103 | To check your changes, you can run test_project: 104 | 105 | 1. Set up aliases for docker hosts in ``/etc/hosts``:: 106 | 107 | inv ci.prepare 108 | 109 | or specify values required for database and redis in the ``.env`` file. 110 | Example:: 111 | 112 | DB_HOST=localhost 113 | REDIS_HOST=localhost 114 | 115 | 2. Run the project and go to ``localhost:8000`` page in browser to check whether 116 | it was started:: 117 | 118 | inv django.run 119 | 120 | .. note:: 121 | To run import/export in background, change `CELERY_TASK_ALWAYS_EAGER `_ 122 | to ``False`` and start celery with:: 123 | 124 | inv celery.run 125 | 126 | Pull Request Guidelines 127 | ----------------------- 128 | 129 | Before you submit a pull request, check that it meets these guidelines: 130 | 131 | 1. The pull request should include tests. 132 | 2. If the pull request adds functionality, the docs should be updated. Put 133 | your new functionality into a function with a docstring, and add the 134 | feature to the list in README.md. 135 | 3. The pull request should work for each supported Python version, and for PyPy. Check 136 | github actions status, verify that all checks have been passed. 137 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022, Saritasa 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Django-import-export-extensions 2 | 3 | [![PyPI - Python Versions](https://img.shields.io/pypi/pyversions/django-import-export-extensions)](https://pypi.org/project/django-import-export-extensions/) 4 | [![PyPI - Django Versions](https://img.shields.io/pypi/frameworkversions/django/django-import-export-extensions)](https://pypi.org/project/django-import-export-extensions/) 5 | ![PyPI](https://img.shields.io/pypi/v/django-import-export-extensions) 6 | 7 | [![Build status on Github](https://github.com/saritasa-nest/django-import-export-extensions/actions/workflows/checks.yml/badge.svg)](https://github.com/saritasa-nest/django-import-export-extensions/actions/workflows/checks.yml) 8 | [![Test coverage](https://coveralls.io/repos/github/saritasa-nest/django-import-export-extensions/badge.svg?branch=main)](https://coveralls.io/github/saritasa-nest/django-import-export-extensions?branch=main) 9 | [![Documentation Status](https://readthedocs.org/projects/django-import-export-extensions/badge/?version=latest)](https://django-import-export-extensions.readthedocs.io/en/latest/?badge=latest) 10 | 11 | ![PyPI Downloads](https://static.pepy.tech/badge/django-import-export-extensions/month) 12 | 13 | ## Links 14 | 15 | - [Documentation]() 16 | - [GitHub]() 17 | - [PyPI]() 18 | - [Contibuting]() 19 | - [History](https://django-import-export-extensions.readthedocs.io/en/stable/history.html) 20 | 21 | ## Description 22 | 23 | `django-import-export-extensions` extends the functionality of 24 | [django-import-export](https://github.com/django-import-export/django-import-export/) 25 | adding the following features: 26 | 27 | - Import/export resources in the background via Celery 28 | - Manage import/export jobs via Django Admin 29 | - DRF integration that allows to work with import/export jobs via API 30 | - Support [drf-spectacular](https://github.com/tfranzel/drf-spectacular) generated API schema 31 | - Additional fields and widgets (FileWidget, IntermediateManyToManyWidget, IntermediateManyToManyField) 32 | 33 | ## Installation 34 | 35 | To install `django-import-export-extensions`, run this command in your 36 | terminal: 37 | 38 | ```sh 39 | pip install django-import-export-extensions 40 | ``` 41 | 42 | Add `import_export` and `import_export_extensions` to `INSTALLED_APPS` 43 | 44 | ```python 45 | # settings.py 46 | INSTALLED_APPS = ( 47 | ..., 48 | "import_export", 49 | "import_export_extensions", 50 | ) 51 | ``` 52 | 53 | Run `migrate` command to create ImportJob/ExportJob models and 54 | `collectstatic` to let Django collect package static files to use in the 55 | admin. 56 | 57 | ```sh 58 | python manage.py migrate 59 | python manage.py collectstatic 60 | ``` 61 | 62 | ## Usage 63 | 64 | Prepare resource for your model 65 | 66 | ```python 67 | # apps/books/resources.py 68 | from import_export_extensions.resources import CeleryModelResource 69 | 70 | from .. import models 71 | 72 | 73 | class BookResource(CeleryModelResource): 74 | 75 | class Meta: 76 | model = models.Book 77 | ``` 78 | 79 | Use `CeleryImportExportMixin` class and set `resource_classes` in admin 80 | model to import/export via Django Admin 81 | 82 | ```python 83 | # apps/books/admin.py 84 | from django.contrib import admin 85 | 86 | from import_export_extensions.admin import CeleryImportExportMixin 87 | 88 | from .. import resources 89 | 90 | 91 | @admin.register(models.Book) 92 | class BookAdmin(CeleryImportExportMixin, admin.ModelAdmin): 93 | resource_classes = [resources.BookResource] 94 | ``` 95 | 96 | Prepare view sets to import/export via API 97 | 98 | ``` python 99 | # apps/books/api/views.py 100 | from .. import resources 101 | 102 | from import_export_extensions.api import views 103 | 104 | 105 | class BookExportViewSet(views.ExportJobViewSet): 106 | resource_class = resources.BookResource 107 | 108 | 109 | class BookImportViewSet(views.ImportJobViewSet): 110 | resource_class = resources.BookResource 111 | ``` 112 | 113 | Don't forget to [configure 114 | Celery](https://docs.celeryq.dev/en/stable/django/first-steps-with-django.html) 115 | if you want to run import/export in background 116 | 117 | ## License 118 | 119 | - Free software: MIT license 120 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | redis: 3 | image: redis:latest 4 | ports: 5 | - "6379:6379" 6 | postgres: 7 | image: postgres:latest 8 | ports: 9 | - "5432:5432" 10 | healthcheck: 11 | test: ["CMD-SHELL", "pg_isready -h postgres -t 5 -U ${COMPOSE_PROJECT_NAME}-user || false"] 12 | interval: 1s 13 | timeout: 5s 14 | retries: 10 15 | environment: 16 | - POSTGRES_DB=${COMPOSE_PROJECT_NAME}-dev 17 | - POSTGRES_USER=${COMPOSE_PROJECT_NAME}-user 18 | - POSTGRES_PASSWORD=testpass 19 | -------------------------------------------------------------------------------- /docs/_static/images/action-bands-openapi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/docs/_static/images/action-bands-openapi.png -------------------------------------------------------------------------------- /docs/_static/images/bands-openapi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/docs/_static/images/bands-openapi.png -------------------------------------------------------------------------------- /docs/_static/images/export-status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/docs/_static/images/export-status.png -------------------------------------------------------------------------------- /docs/_static/images/filters-openapi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/docs/_static/images/filters-openapi.png -------------------------------------------------------------------------------- /docs/_static/images/force_import_admin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/docs/_static/images/force_import_admin.png -------------------------------------------------------------------------------- /docs/_static/images/force_import_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/docs/_static/images/force_import_results.png -------------------------------------------------------------------------------- /docs/_static/images/import-job-admin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/docs/_static/images/import-job-admin.png -------------------------------------------------------------------------------- /docs/_static/images/start_api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/docs/_static/images/start_api.png -------------------------------------------------------------------------------- /docs/api_admin.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | Admin 3 | ===== 4 | 5 | .. automodule:: import_export_extensions.admin.model_admins.export_job_admin 6 | :members: 7 | 8 | .. automodule:: import_export_extensions.admin.model_admins.import_job_admin 9 | :members: 10 | 11 | .. automodule:: import_export_extensions.admin.model_admins.mixins 12 | :members: 13 | 14 | .. automodule:: import_export_extensions.admin.forms.export_job_admin_form 15 | :members: 16 | 17 | .. automodule:: import_export_extensions.admin.forms.import_job_admin_form 18 | :members: 19 | 20 | .. automodule:: import_export_extensions.admin.forms.import_admin_form 21 | :members: 22 | 23 | .. automodule:: import_export_extensions.admin.mixins.types 24 | :members: 25 | 26 | .. automodule:: import_export_extensions.admin.mixins.export_mixin 27 | :members: 28 | 29 | .. automodule:: import_export_extensions.admin.mixins.import_export_mixin 30 | :members: 31 | 32 | .. automodule:: import_export_extensions.admin.mixins.import_mixin 33 | :members: 34 | 35 | .. automodule:: import_export_extensions.admin.widgets 36 | :members: 37 | -------------------------------------------------------------------------------- /docs/api_drf.rst: -------------------------------------------------------------------------------- 1 | ==================== 2 | API (Rest Framework) 3 | ==================== 4 | 5 | .. autoclass:: import_export_extensions.api.ImportJobViewSet 6 | :members: 7 | 8 | .. autoclass:: import_export_extensions.api.ExportJobViewSet 9 | :members: 10 | 11 | .. autoclass:: import_export_extensions.api.ImportJobForUserViewSet 12 | :members: 13 | 14 | .. autoclass:: import_export_extensions.api.ExportJobForUserViewSet 15 | :members: 16 | 17 | .. autoclass:: import_export_extensions.api.BaseImportJobViewSet 18 | :members: 19 | 20 | .. autoclass:: import_export_extensions.api.BaseExportJobViewSet 21 | :members: 22 | 23 | .. autoclass:: import_export_extensions.api.BaseImportJobForUserViewSet 24 | :members: 25 | 26 | .. autoclass:: import_export_extensions.api.BaseExportJobForUserViewSet 27 | :members: 28 | 29 | .. autoclass:: import_export_extensions.api.LimitQuerySetToCurrentUserMixin 30 | :members: 31 | 32 | .. autoclass:: import_export_extensions.api.ImportStartActionMixin 33 | :members: 34 | 35 | .. autoclass:: import_export_extensions.api.ExportStartActionMixin 36 | :members: 37 | 38 | .. autoclass:: import_export_extensions.api.CreateExportJob 39 | :members: create, validate 40 | 41 | .. autoclass:: import_export_extensions.api.CreateImportJob 42 | :members: create, validate 43 | 44 | .. autoclass:: import_export_extensions.api.ExportJobSerializer 45 | :members: 46 | 47 | .. autoclass:: import_export_extensions.api.ImportJobSerializer 48 | :members: 49 | 50 | .. autoclass:: import_export_extensions.api.ProgressSerializer 51 | :members: 52 | 53 | .. attribute:: info 54 | :type: ProgressInfoSerializer 55 | :value: {"current": 0, "total": 0} 56 | 57 | Shows current and total imported/exported values 58 | 59 | .. autoclass:: import_export_extensions.api.ProgressInfoSerializer 60 | :members: 61 | 62 | .. attribute:: current 63 | :type: int 64 | :value: 0 65 | 66 | Shows number of imported/exported objects 67 | 68 | .. attribute:: total 69 | :type: int 70 | :value: 0 71 | 72 | Shows total objects to import/export 73 | -------------------------------------------------------------------------------- /docs/api_models.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | Models 3 | ====== 4 | 5 | .. autoclass:: import_export_extensions.models.ExportJob 6 | :members: 7 | 8 | .. autoclass:: import_export_extensions.models.ImportJob 9 | :members: 10 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pathlib 3 | import sys 4 | 5 | import django 6 | 7 | # If extensions (or modules to document with autodoc) are in another directory, 8 | # add these directories to sys.path here. If the directory is relative to the 9 | # documentation root, use os.path.abspath to make it absolute, like shown here. 10 | sys.path.append(str(pathlib.Path.cwd())) 11 | sys.path.append(str(pathlib.Path("..").resolve())) 12 | sys.path.append(str(pathlib.Path("../test_project").resolve())) 13 | os.environ["DJANGO_SETTINGS_MODULE"] = "settings" 14 | 15 | django.setup() 16 | 17 | # -- General configuration --------------------------------------------- 18 | 19 | # Add any Sphinx extension module names here, as strings. They can be 20 | # extensions coming with Sphinx (named "sphinx.ext.*") or your custom ones. 21 | extensions = [ 22 | "sphinx.ext.autodoc", 23 | "sphinx.ext.autosectionlabel", 24 | "sphinx.ext.intersphinx", 25 | ] 26 | 27 | # Add any paths that contain templates here, relative to this directory. 28 | templates_path = ["_templates"] 29 | 30 | # The suffix(es) of source filenames. 31 | source_suffix = ".rst" 32 | 33 | # The master toctree document. 34 | master_doc = "index" 35 | 36 | # General information about the project. 37 | project = "django-import-export-extensions" 38 | copyright = "2022-2024, Saritasa" 39 | author = "Saritasa" 40 | 41 | # The language for content autogenerated by Sphinx. Refer to documentation 42 | # for a list of supported languages. 43 | # 44 | # This is also used if you do content translation via gettext catalogs. 45 | # Usually you set "language" from the command line for these cases. 46 | language = "en" 47 | 48 | # List of patterns, relative to source directory, that match files and 49 | # directories to ignore when looking for source files. 50 | # This patterns also effect to html_static_path and html_extra_path 51 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 52 | 53 | # The name of the Pygments (syntax highlighting) style to use. 54 | pygments_style = "sphinx" 55 | 56 | # If true, `todo` and `todoList` produce output, else they produce nothing. 57 | todo_include_todos = False 58 | 59 | 60 | # -- Options for HTML output ------------------------------------------- 61 | 62 | # The theme to use for HTML and HTML Help pages. See the documentation for 63 | # a list of builtin themes. 64 | # 65 | html_theme = "furo" 66 | 67 | # Add any paths that contain custom static files (such as style sheets) here, 68 | # relative to this directory. They are copied after the builtin static files, 69 | # so a file named "default.css" will overwrite the builtin "default.css". 70 | html_static_path = ["_static"] 71 | 72 | 73 | # -- Options for HTMLHelp output --------------------------------------- 74 | 75 | # Output file base name for HTML help builder. 76 | htmlhelp_basename = "django-import-export-extensions" 77 | 78 | 79 | # -- Options for LaTeX output ------------------------------------------ 80 | 81 | # Grouping the document tree into LaTeX files. List of tuples 82 | # (source start file, target name, title, author, documentclass 83 | # [howto, manual, or own class]). 84 | latex_documents = [ 85 | ( 86 | master_doc, 87 | "django-import-export-extensions.tex", 88 | "django-import-export-extensions Documentation", 89 | author, 90 | "manual", 91 | ), 92 | ] 93 | 94 | 95 | # -- Options for manual page output ------------------------------------ 96 | 97 | # One entry per manual page. List of tuples 98 | # (source start file, name, description, authors, manual section). 99 | man_pages = [ 100 | ( 101 | master_doc, 102 | "django-import-export-extensions", 103 | "django-import-export-extensions Documentation", 104 | [author], 105 | 1, 106 | ), 107 | ] 108 | 109 | 110 | # -- Options for Texinfo output ----------------------------------------------- 111 | 112 | # Grouping the document tree into Texinfo files. List of tuples 113 | # (source start file, target name, title, author, 114 | # dir menu entry, description, category) 115 | texinfo_documents = [ 116 | ( 117 | "index", 118 | "django-import-export-extensions", 119 | "django-import-export-extensions Documentation", 120 | author, 121 | "django-import-export-extensions", 122 | "Utils for import/export data for Django", 123 | "Miscellaneous", 124 | ), 125 | ] 126 | 127 | # Documents to append as an appendix to all manuals. 128 | texinfo_appendices = [] 129 | 130 | # intersphinx documentation 131 | intersphinx_mapping = { 132 | "tablib": ("https://tablib.readthedocs.io/en/stable/", None), 133 | "django-import-export": ( 134 | "https://django-import-export.readthedocs.io/en/latest/", 135 | None, 136 | ), 137 | } 138 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | Django import/export extensions 3 | =============================== 4 | 5 | django-import-export-extensions is a Django application and library based on 6 | `django-import-export` library that provided extended features. 7 | 8 | **Features:** 9 | 10 | * Import/export :ref:`resources` in the background via Celery 11 | * Manage import/export :ref:`jobs` via Django Admin 12 | * :ref:`DRF integration` that allows to work with import/export jobs via API 13 | * Support `drf-spectacular `_ generated API schema 14 | * Additional :ref:`fields` and :ref:`widgets` (FileWidget, IntermediateManyToManyWidget, IntermediateManyToManyField) 15 | 16 | .. toctree:: 17 | :maxdepth: 2 18 | :caption: User Guide 19 | 20 | installation 21 | getting_started 22 | extensions 23 | migrate_from_original_import_export 24 | authors 25 | history 26 | 27 | .. toctree:: 28 | :maxdepth: 2 29 | :caption: API Documentation 30 | 31 | api_admin 32 | api_models 33 | api_drf 34 | 35 | .. toctree:: 36 | :maxdepth: 2 37 | :caption: Developers 38 | 39 | contributing 40 | 41 | 42 | Indices and tables 43 | ================== 44 | * :ref:`genindex` 45 | * :ref:`modindex` 46 | * :ref:`search` 47 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============================== 4 | Installation and configuration 5 | ============================== 6 | 7 | 8 | Stable release 9 | -------------- 10 | 11 | To install ``django-import-export-extensions``, run the following command in your terminal: 12 | 13 | Using pip: 14 | 15 | .. code-block:: shell 16 | 17 | pip install django-import-export-extensions 18 | 19 | Using uv: 20 | 21 | .. code-block:: shell 22 | 23 | uv pip install django-import-export-extensions 24 | 25 | Using poetry: 26 | 27 | .. code-block:: shell 28 | 29 | poetry add django-import-export-extensions 30 | 31 | This is the preferred installation method, 32 | as it will always install the most recent stable release of ``django-import-export-extensions``. 33 | 34 | Next, add ``import_export`` and ``import_export_extensions`` to your ``INSTALLED_APPS`` setting: 35 | 36 | .. code-block:: python 37 | 38 | # settings.py 39 | INSTALLED_APPS = [ 40 | ... 41 | "import_export", 42 | "import_export_extensions", 43 | ] 44 | 45 | Finally, run the ``migrate`` and ``collectstatic`` commands: 46 | 47 | * ``migrate``: Creates the ImportJob and ExportJob models. 48 | * ``collectstatic``: Allows Django to collect static files for use in the admin interface. 49 | 50 | .. code-block:: shell 51 | 52 | python manage.py migrate 53 | python manage.py collectstatic 54 | 55 | 56 | Celery 57 | ------ 58 | 59 | To use background import/export, you need to 60 | `set up Celery `_. 61 | Once Celery is set up, no additional configuration is required. 62 | 63 | 64 | Settings 65 | ------------- 66 | 67 | You can configure the following settings in your Django settings file: 68 | 69 | ``IMPORT_EXPORT_MAX_DATASET_ROWS`` 70 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 71 | 72 | Defines the maximum number of rows allowed in a file for import, helping to avoid memory overflow. 73 | The default value is 10,000. If the file exceeds this limit, a ``ValueError`` exception 74 | will be raised during the import process. 75 | 76 | ``MIME_TYPES_MAP`` 77 | ~~~~~~~~~~~~~~~~~~ 78 | 79 | Mapping file extensions to mime types to import files. 80 | By default, it uses the `mimetypes.types_map `_ 81 | from Python's mimetypes module. 82 | 83 | ``STATUS_UPDATE_ROW_COUNT`` 84 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 85 | 86 | Defines the number of rows after import/export of which the task status is 87 | updated. This helps to increase the speed of import/export. The default value 88 | is 100. This parameter can be specified separately for each resource by adding 89 | ``status_update_row_count`` to its ``Meta``. 90 | 91 | ``DRF_EXPORT_DJANGO_FILTERS_BACKEND`` 92 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 93 | 94 | Specifies filter backend class for ``django-filters`` in export action. 95 | Default: ``django_filters.rest_framework.DjangoFilterBackend`` 96 | 97 | ``DRF_EXPORT_ORDERING_BACKEND`` 98 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 99 | 100 | Specifies filter backend class for ``ordering`` in export action. 101 | Default: ``rest_framework.filters.OrderingFilter`` 102 | 103 | Settings from django-import-export 104 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 105 | Additionally, the package supports settings from the original django-import-export package. 106 | For full details on these settings, refer to the `official documentation `_. 107 | 108 | **Note**: The only setting that does not affect functionality in this package is ``IMPORT_EXPORT_TMP_STORAGE_CLASS``, 109 | as the storage is not used in the implementation of ``CeleryImportAdminMixin``. 110 | 111 | Picking storage 112 | ~~~~~~~~~~~~~~~ 113 | 114 | To use different storage for import/export jobs you can use `STORAGES `. 115 | from django. 116 | 117 | .. code-block:: python 118 | 119 | STORAGES = { 120 | "default": { 121 | "BACKEND": "django.core.files.storage.filesystem.FileSystemStorage", 122 | }, 123 | "staticfiles": { 124 | "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", 125 | }, 126 | # Use this to specify custom storage for package 127 | "django_import_export_extensions": { 128 | "BACKEND": "django.core.files.storage.filesystem.FileSystemStorage", 129 | }, 130 | } 131 | -------------------------------------------------------------------------------- /docs/migrate_from_original_import_export.rst: -------------------------------------------------------------------------------- 1 | ==================================================== 2 | Migrate from original `django-import-export` package 3 | ==================================================== 4 | 5 | If you're already using ``django-import-export`` and want to take advantage of 6 | ``django-import-export-extensions`` for background import/export, the transition is simple. First, 7 | install the package by following the :ref:`the installation guide`. 8 | Then, all you need to do is update the base classes for your resource and admin models. 9 | 10 | Migrate resources 11 | ---------------------------------------------------------------- 12 | 13 | To enable import/export via Celery, simply replace the base resource classes from the original package 14 | with ``CeleryResource`` or ``CeleryModelResource`` from ``django-import-export-extensions``: 15 | 16 | .. code-block:: diff 17 | :emphasize-lines: 2,6,13 18 | 19 | - from import_export import resources 20 | + from import_export_extensions import resources 21 | 22 | class SimpleResource( 23 | - resources.Resource, 24 | + resources.CeleryResource, 25 | ): 26 | """Simple resource.""" 27 | 28 | 29 | class BookResource( 30 | - resources.ModelResource, 31 | + resources.CeleryModelResource, 32 | ): 33 | """Resource class for `Book` model.""" 34 | 35 | class Meta: 36 | model = Book 37 | 38 | Migrate admin models 39 | -------------------- 40 | 41 | Then you also need to change admin mixins to use celery import/export via Django Admin: 42 | 43 | .. code-block:: diff 44 | :emphasize-lines: 4,10,11 45 | 46 | from django.contrib import admin 47 | 48 | - from import_export.admin import ImportExportModelAdmin 49 | + from import_export_extensions.admin import CeleryImportExportMixin 50 | 51 | from . import resources 52 | 53 | class BookAdmin( 54 | - ImportExportModelAdmin, 55 | + CeleryImportExportMixin, 56 | + admin.ModelAdmin, 57 | ): 58 | """Resource class for `Book` model.""" 59 | 60 | resource_classes = ( 61 | resources.BookResource, 62 | ) 63 | 64 | 65 | If you only need import (or export) functionality, you can use ``CeleryImportAdminMixin`` 66 | (``CeleryExportAdminMixin``) instead of ``CeleryImportExportMixin``. 67 | 68 | If you only need import (or export) functionality, you can use the ``CeleryImportAdminMixin`` 69 | (or ``CeleryExportAdminMixin``) instead of the ``CeleryImportExportMixin``. 70 | 71 | Migrate custom import/export 72 | ---------------------------- 73 | 74 | Background import/export is implemented using the ``ImportJob`` and ``ExportJob`` models. 75 | As a result, calling the simple ``resource.export()`` will not trigger a Celery task — it behaves 76 | exactly like the original ``Resource.export()`` method. To initiate background import/export, 77 | you need to create instances of the import/export job: 78 | 79 | .. code-block:: python 80 | :linenos: 81 | 82 | >>> from .resources import BandResource 83 | >>> from import_export.formats import base_formats 84 | >>> from import_export_extensions.models import ExportJob 85 | >>> file_format = base_formats.CSV 86 | >>> file_format_path = f"{file_format.__module__}.{file_format.__name__}" 87 | >>> export_job = ExportJob.objects.create( 88 | resource_path=BandResource.class_path, 89 | file_format_path=file_format_path 90 | ) 91 | >>> export_job.export_status 92 | 'CREATED' 93 | 94 | You can check the current status of the job using the ``export_status`` (or ``import_status``) 95 | property of the model. Additionally, the ``progress`` property provides information about the total 96 | number of rows and the number of rows that have been completed. 97 | 98 | .. code-block:: python 99 | :linenos: 100 | :emphasize-lines: 2,4 101 | 102 | >>> export_job.refresh_from_db() 103 | >>> export_job.export_status 104 | 'EXPORTING' 105 | >>> export_job.progress 106 | {'state': 'EXPORTING', 'info': {'current': 53, 'total': 100}} 107 | >>> export_job.refresh_from_db() 108 | >>> export_job.export_status 109 | 'EXPORTED' 110 | >>> export_job.data_file.path 111 | '../media/import_export_extensions/export/3dfb7510-5593-4dc6-9d7d-bbd907cd3eb6/Artists-2020-02-22.csv' 112 | 113 | Other configuration 114 | ------------------- 115 | 116 | You may need to configure `MEDIA_URL `_ in your 117 | project settings, otherwise you may see a 404 error when attempting to download exported files. 118 | -------------------------------------------------------------------------------- /import_export_extensions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/import_export_extensions/__init__.py -------------------------------------------------------------------------------- /import_export_extensions/admin/__init__.py: -------------------------------------------------------------------------------- 1 | from .mixins import ( 2 | CeleryExportAdminMixin, 3 | CeleryImportAdminMixin, 4 | CeleryImportExportMixin, 5 | ) 6 | from .model_admins import ExportJobAdmin, ImportJobAdmin 7 | -------------------------------------------------------------------------------- /import_export_extensions/admin/forms/__init__.py: -------------------------------------------------------------------------------- 1 | from .export_job_admin_form import ExportJobAdminForm 2 | from .import_admin_form import ForceImportForm 3 | from .import_job_admin_form import ImportJobAdminForm 4 | -------------------------------------------------------------------------------- /import_export_extensions/admin/forms/export_job_admin_form.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.urls import reverse 3 | 4 | from ... import models 5 | from ..widgets import ProgressBarWidget 6 | 7 | 8 | class ExportJobAdminForm(forms.ModelForm): 9 | """Admin form for `ExportJob` model. 10 | 11 | Adds custom `export_progressbar` field that displays current export 12 | progress using AJAX requests to specified endpoint. Fields widget is 13 | defined in `__init__` method. 14 | 15 | """ 16 | 17 | export_progressbar = forms.Field( 18 | label="Export progress", 19 | required=False, 20 | ) 21 | 22 | def __init__( 23 | self, 24 | instance: models.ExportJob, 25 | *args, 26 | **kwargs, 27 | ): 28 | """Provide `export_progressbar` widget the `ExportJob` instance.""" 29 | super().__init__(*args, instance=instance, **kwargs) 30 | url_name = "admin:export_job_progress" 31 | self.fields["export_progressbar"].widget = ProgressBarWidget( 32 | job=instance, 33 | url=reverse(url_name, args=(instance.id,)), 34 | ) 35 | 36 | class Meta: 37 | fields = ( 38 | "export_status", 39 | "resource_path", 40 | "file_format_path", 41 | "data_file", 42 | "resource_kwargs", 43 | "traceback", 44 | "error_message", 45 | "result", 46 | "export_task_id", 47 | "export_started", 48 | "export_finished", 49 | "created_by", 50 | "created", 51 | "modified", 52 | ) 53 | -------------------------------------------------------------------------------- /import_export_extensions/admin/forms/import_admin_form.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | 3 | from import_export import forms as base_forms 4 | 5 | 6 | class ForceImportForm(base_forms.ImportForm): 7 | """Import form with `force_import` option.""" 8 | 9 | force_import = forms.BooleanField( 10 | required=False, 11 | initial=False, 12 | ) 13 | -------------------------------------------------------------------------------- /import_export_extensions/admin/forms/import_job_admin_form.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.urls import reverse 3 | 4 | from ... import models 5 | from ..widgets import ProgressBarWidget 6 | 7 | 8 | class ImportJobAdminForm(forms.ModelForm): 9 | """Admin form for ``ImportJob`` model. 10 | 11 | Adds custom `import_progressbar` field that displays current import 12 | progress using AJAX requests to specified endpoint. Fields widget is 13 | defined in `__init__` method. 14 | 15 | """ 16 | 17 | import_progressbar = forms.Field( 18 | required=False, 19 | ) 20 | 21 | def __init__( 22 | self, 23 | instance: models.ImportJob, 24 | *args, 25 | **kwargs, 26 | ): 27 | """Provide `import_progressbar` widget the ``ImportJob`` instance.""" 28 | super().__init__(*args, instance=instance, **kwargs) 29 | url_name = "admin:import_job_progress" 30 | self.fields["import_progressbar"].label = ( 31 | "Import progress" if 32 | instance.import_status == models.ImportJob.ImportStatus.IMPORTING 33 | else "Parsing progress" 34 | ) 35 | self.fields["import_progressbar"].widget = ProgressBarWidget( 36 | job=instance, 37 | url=reverse(url_name, args=(instance.id,)), 38 | ) 39 | 40 | class Meta: 41 | fields = ( 42 | "import_status", 43 | "resource_path", 44 | "data_file", 45 | "resource_kwargs", 46 | "traceback", 47 | "error_message", 48 | "result", 49 | "parse_task_id", 50 | "import_task_id", 51 | "parse_finished", 52 | "import_started", 53 | "import_finished", 54 | "created_by", 55 | "created", 56 | "modified", 57 | ) 58 | -------------------------------------------------------------------------------- /import_export_extensions/admin/mixins/__init__.py: -------------------------------------------------------------------------------- 1 | from .export_mixin import CeleryExportAdminMixin 2 | from .import_export_mixin import CeleryImportExportMixin 3 | from .import_mixin import CeleryImportAdminMixin 4 | -------------------------------------------------------------------------------- /import_export_extensions/admin/mixins/base_mixin.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from import_export import admin as import_export_admin 4 | 5 | from . import types 6 | 7 | 8 | class BaseCeleryImportExportAdminMixin( 9 | import_export_admin.ImportExportMixinBase, 10 | ): 11 | """Extend base mixin with common logic for import/export.""" 12 | 13 | @property 14 | def model_info(self) -> types.ModelInfo: 15 | """Get info of model.""" 16 | return types.ModelInfo( 17 | meta=self.model._meta, 18 | ) 19 | 20 | def get_context_data(self, **kwargs) -> dict[str, typing.Any]: 21 | """Get context data.""" 22 | return {} 23 | -------------------------------------------------------------------------------- /import_export_extensions/admin/mixins/import_export_mixin.py: -------------------------------------------------------------------------------- 1 | from .export_mixin import CeleryExportAdminMixin 2 | from .import_mixin import CeleryImportAdminMixin 3 | 4 | 5 | class CeleryImportExportMixin( 6 | CeleryImportAdminMixin, 7 | CeleryExportAdminMixin, 8 | ): 9 | """Import and export mixin.""" 10 | 11 | # template for change_list view 12 | import_export_change_list_template = ( 13 | "admin/import_export/change_list_import_export.html" 14 | ) 15 | -------------------------------------------------------------------------------- /import_export_extensions/admin/mixins/types.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | from django.db.models.options import Options 4 | 5 | from import_export.formats import base_formats 6 | 7 | from ... import resources 8 | 9 | ResourceObj = resources.CeleryResource | resources.CeleryModelResource 10 | ResourceType = type[ResourceObj] 11 | FormatType = type[base_formats.Format] 12 | 13 | 14 | @dataclasses.dataclass 15 | class ModelInfo: 16 | """Contain base info about imported model.""" 17 | 18 | meta: Options 19 | 20 | @property 21 | def name(self) -> str: 22 | """Get name of model.""" 23 | return self.meta.model_name 24 | 25 | @property 26 | def app_label(self): 27 | """App label of model.""" 28 | return self.meta.app_label 29 | 30 | @property 31 | def app_model_name(self) -> str: 32 | """Return url name.""" 33 | return f"{self.app_label}_{self.name}" 34 | -------------------------------------------------------------------------------- /import_export_extensions/admin/model_admins/__init__.py: -------------------------------------------------------------------------------- 1 | from .export_job_admin import ExportJobAdmin 2 | from .import_job_admin import ImportJobAdmin 3 | -------------------------------------------------------------------------------- /import_export_extensions/admin/model_admins/mixins.py: -------------------------------------------------------------------------------- 1 | 2 | from django.contrib.contenttypes.models import ContentType 3 | from django.core.handlers.wsgi import WSGIRequest 4 | from django.utils.module_loading import import_string 5 | from django.utils.translation import gettext_lazy as _ 6 | 7 | from ...models.core import BaseJob 8 | 9 | 10 | class BaseImportExportJobAdminMixin: 11 | """Mixin provides common methods for ImportJob and ExportJob admins.""" 12 | 13 | def has_add_permission( 14 | self, 15 | request: WSGIRequest, 16 | *args, 17 | **kwargs, 18 | ) -> bool: 19 | """Import/Export Jobs should not be created using this interface.""" 20 | return False 21 | 22 | def has_delete_permission( 23 | self, 24 | request: WSGIRequest, 25 | *args, 26 | **kwargs, 27 | ) -> bool: 28 | """Import/Export Jobs should not be deleted using this interface. 29 | 30 | Instead, admins must cancel jobs. 31 | 32 | """ 33 | return False 34 | 35 | def _model(self, obj: BaseJob) -> str: 36 | """Add `model` field of import/export job.""" 37 | try: 38 | resource_class = import_string(obj.resource_path) 39 | model = resource_class.Meta.model._meta.verbose_name_plural 40 | # In case resource has no Meta or model we need to catch AttributeError 41 | except (ImportError, AttributeError): # pragma: no cover 42 | model = _("Unknown") 43 | return model 44 | 45 | def get_from_content_type( 46 | self, 47 | obj: BaseJob, 48 | ) -> ContentType | None: # pragma: no cover 49 | """Shortcut to get object from content_type.""" 50 | content_type = obj.resource_kwargs.get("content_type") 51 | obj_id = obj.resource_kwargs.get("object_id") 52 | 53 | if content_type and obj_id: 54 | content_type = ContentType.objects.get(id=content_type) 55 | return content_type.model_class().objects.filter(id=obj_id).first() 56 | return None 57 | -------------------------------------------------------------------------------- /import_export_extensions/admin/widgets.py: -------------------------------------------------------------------------------- 1 | 2 | from django import forms 3 | from django.template.loader import render_to_string 4 | 5 | 6 | class ProgressBarWidget(forms.Widget): 7 | """Widget for progress bar field. 8 | 9 | Value for `progress_bar` element is changed using JS code. 10 | 11 | """ 12 | 13 | template_name = "admin/import_export_extensions/progress_bar.html" 14 | 15 | def __init__(self, *args, **kwargs): 16 | """Get ``ImportJob`` or ``ExportJob`` instance from kwargs. 17 | 18 | ``ImportJob`` or ``ExportJob`` instance is used 19 | to render hidden element in `render` method. 20 | 21 | """ 22 | self.job = kwargs.pop("job") 23 | self.url = kwargs.pop("url") 24 | super().__init__(*args, **kwargs) 25 | 26 | def render(self, *args, **kwargs) -> str: 27 | """Render HTML5 `progress` element. 28 | 29 | Additionally, method provides hidden `import_job_url` and 30 | `export_job_url` value that is used in `js/admin/progress_bar.js` 31 | to send GET requests. 32 | 33 | """ 34 | return render_to_string(self.template_name, {"job_url": self.url}) 35 | 36 | class Media: 37 | """Class with custom assets for widget.""" 38 | 39 | css = dict( 40 | all=("import_export_extensions/css/widgets/progress_bar.css",), 41 | ) 42 | js = ( 43 | "admin/js/jquery.init.js", 44 | "import_export_extensions/js/widgets/progress_bar.js", 45 | ) 46 | -------------------------------------------------------------------------------- /import_export_extensions/api/__init__.py: -------------------------------------------------------------------------------- 1 | from .mixins import ( 2 | ExportStartActionMixin, 3 | ImportStartActionMixin, 4 | LimitQuerySetToCurrentUserMixin, 5 | ) 6 | from .serializers import ( 7 | CreateExportJob, 8 | CreateImportJob, 9 | ExportJobSerializer, 10 | ImportJobSerializer, 11 | ProgressInfoSerializer, 12 | ProgressSerializer, 13 | ) 14 | from .views import ( 15 | BaseExportJobForUserViewSet, 16 | BaseExportJobViewSet, 17 | BaseImportJobForUserViewSet, 18 | BaseImportJobViewSet, 19 | ExportJobForUserViewSet, 20 | ExportJobViewSet, 21 | ImportJobForUserViewSet, 22 | ImportJobViewSet, 23 | ) 24 | -------------------------------------------------------------------------------- /import_export_extensions/api/mixins/__init__.py: -------------------------------------------------------------------------------- 1 | from .common import LimitQuerySetToCurrentUserMixin 2 | from .export_mixins import ExportStartActionMixin 3 | from .import_mixins import ImportStartActionMixin 4 | -------------------------------------------------------------------------------- /import_export_extensions/api/mixins/common.py: -------------------------------------------------------------------------------- 1 | class LimitQuerySetToCurrentUserMixin: 2 | """Make queryset to return only current user jobs.""" 3 | 4 | def get_queryset(self): 5 | """Return user's jobs.""" 6 | if self.action in ( 7 | getattr(self, "import_action", ""), 8 | getattr(self, "export_action", ""), 9 | ): 10 | # To make it consistent and for better support of drf-spectacular 11 | return super().get_queryset() # pragma: no cover 12 | return ( 13 | super() 14 | .get_queryset() 15 | .filter(created_by_id=getattr(self.request.user, "pk", None)) 16 | ) 17 | -------------------------------------------------------------------------------- /import_export_extensions/api/mixins/export_mixins.py: -------------------------------------------------------------------------------- 1 | import collections.abc 2 | import contextlib 3 | import typing 4 | 5 | from django.conf import settings 6 | from django.utils import module_loading 7 | 8 | from rest_framework import ( 9 | decorators, 10 | request, 11 | response, 12 | status, 13 | ) 14 | 15 | from ... import resources 16 | from .. import serializers 17 | 18 | 19 | class ExportStartActionMixin: 20 | """Mixin which adds start export action.""" 21 | 22 | resource_class: type[resources.CeleryModelResource] 23 | export_action = "start_export_action" 24 | export_action_name = "export" 25 | export_action_url = "export" 26 | export_detail_serializer_class = serializers.ExportJobSerializer 27 | export_ordering: collections.abc.Sequence[str] = () 28 | export_ordering_fields: collections.abc.Sequence[str] = () 29 | export_open_api_description = ( 30 | "This endpoint creates export job and starts it. " 31 | "To monitor progress use detail endpoint for jobs to fetch state of " 32 | "job. Once it's status is `EXPORTED`, you can download file." 33 | ) 34 | 35 | def __init_subclass__(cls) -> None: 36 | super().__init_subclass__() 37 | # Skip if it is has no resource_class specified 38 | if not hasattr(cls, "resource_class"): 39 | return 40 | filter_backends = [ 41 | module_loading.import_string( 42 | settings.DRF_EXPORT_DJANGO_FILTERS_BACKEND, 43 | ), 44 | ] 45 | if cls.export_ordering_fields: 46 | filter_backends.append( 47 | module_loading.import_string( 48 | settings.DRF_EXPORT_ORDERING_BACKEND, 49 | ), 50 | ) 51 | 52 | def start_export_action( 53 | self: "ExportStartActionMixin", 54 | request: request.Request, 55 | *args, 56 | **kwargs, 57 | ) -> response.Response: 58 | return self.start_export(request) 59 | 60 | setattr(cls, cls.export_action, start_export_action) 61 | decorators.action( 62 | methods=["POST"], 63 | url_name=cls.export_action_name, 64 | url_path=cls.export_action_url, 65 | detail=False, 66 | queryset=cls.resource_class.get_model_queryset(), 67 | serializer_class=cls().get_export_create_serializer_class(), 68 | filterset_class=getattr( 69 | cls.resource_class, 70 | "filterset_class", 71 | None, 72 | ), 73 | filter_backends=filter_backends, 74 | ordering=cls.export_ordering, 75 | ordering_fields=cls.export_ordering_fields, 76 | )(getattr(cls, cls.export_action)) 77 | # Correct specs of drf-spectacular if it is installed 78 | with contextlib.suppress(ImportError): 79 | from drf_spectacular import utils 80 | 81 | utils.extend_schema_view( 82 | **{ 83 | cls.export_action: utils.extend_schema( 84 | description=cls.export_open_api_description, 85 | filters=True, 86 | responses={ 87 | status.HTTP_201_CREATED: cls().get_export_detail_serializer_class(), # noqa: E501 88 | }, 89 | ), 90 | }, 91 | )(cls) 92 | 93 | def get_queryset(self): 94 | """Return export model queryset on export action. 95 | 96 | For better openapi support and consistency. 97 | 98 | """ 99 | if self.action == self.export_action: 100 | return self.resource_class.get_model_queryset() # pragma: no cover 101 | return super().get_queryset() 102 | 103 | def get_export_detail_serializer_class(self): 104 | """Get serializer which will be used show details of export job.""" 105 | return self.export_detail_serializer_class 106 | 107 | def get_export_create_serializer_class(self): 108 | """Get serializer which will be used to start export job.""" 109 | return serializers.get_create_export_job_serializer( 110 | self.resource_class, 111 | ) 112 | 113 | def get_export_resource_kwargs(self) -> dict[str, typing.Any]: 114 | """Provide extra arguments to resource class.""" 115 | return {} 116 | 117 | def get_serializer(self, *args, **kwargs): 118 | """Provide resource kwargs to serializer class.""" 119 | if self.action == self.export_action: 120 | kwargs.setdefault( 121 | "resource_kwargs", 122 | self.get_export_resource_kwargs(), 123 | ) 124 | return super().get_serializer(*args, **kwargs) 125 | 126 | def start_export(self, request: request.Request) -> response.Response: 127 | """Validate request data and start ExportJob.""" 128 | ordering = request.query_params.get("ordering", "") 129 | if ordering: 130 | ordering = ordering.split(",") 131 | serializer = self.get_serializer( 132 | data=request.data, 133 | ordering=ordering, 134 | filter_kwargs=request.query_params, 135 | ) 136 | serializer.is_valid(raise_exception=True) 137 | export_job = serializer.save() 138 | return response.Response( 139 | data=self.get_export_detail_serializer_class()( 140 | instance=export_job, 141 | ).data, 142 | status=status.HTTP_201_CREATED, 143 | ) 144 | -------------------------------------------------------------------------------- /import_export_extensions/api/mixins/import_mixins.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import typing 3 | 4 | from rest_framework import ( 5 | decorators, 6 | request, 7 | response, 8 | status, 9 | ) 10 | 11 | from ... import resources 12 | from .. import serializers 13 | 14 | 15 | class ImportStartActionMixin: 16 | """Mixin which adds start import action.""" 17 | 18 | resource_class: type[resources.CeleryModelResource] 19 | import_action = "start_import_action" 20 | import_action_name = "import" 21 | import_action_url = "import" 22 | import_detail_serializer_class = serializers.ImportJobSerializer 23 | import_open_api_description = ( 24 | "This endpoint creates import job and starts it. " 25 | "To monitor progress use detail endpoint for jobs to fetch state of " 26 | "job. Once it's status is `PARSED`, you can confirm import and data " 27 | "should start importing. When status `INPUT_ERROR` or `PARSE_ERROR` " 28 | "it means data failed validations and can't be imported. " 29 | "When status is `IMPORTED`, it means data is in system and " 30 | "job is completed." 31 | ) 32 | 33 | def __init_subclass__(cls) -> None: 34 | super().__init_subclass__() 35 | # Skip if it is has no resource_class specified 36 | if not hasattr(cls, "resource_class"): 37 | return 38 | 39 | def start_import_action( 40 | self: "ImportStartActionMixin", 41 | request: request.Request, 42 | *args, 43 | **kwargs, 44 | ) -> response.Response: 45 | return self.start_import(request) 46 | 47 | setattr(cls, cls.import_action, start_import_action) 48 | decorators.action( 49 | methods=["POST"], 50 | url_name=cls.import_action_name, 51 | url_path=cls.import_action_url, 52 | detail=False, 53 | queryset=cls.resource_class.get_model_queryset(), 54 | serializer_class=cls().get_import_create_serializer_class(), 55 | )(getattr(cls, cls.import_action)) 56 | # Correct specs of drf-spectacular if it is installed 57 | with contextlib.suppress(ImportError): 58 | from drf_spectacular import utils 59 | 60 | utils.extend_schema_view( 61 | **{ 62 | cls.import_action: utils.extend_schema( 63 | description=cls.import_open_api_description, 64 | filters=True, 65 | responses={ 66 | status.HTTP_201_CREATED: cls().get_import_detail_serializer_class(), # noqa: E501 67 | }, 68 | ), 69 | }, 70 | )(cls) 71 | 72 | def get_queryset(self): 73 | """Return import model queryset on import action. 74 | 75 | For better openapi support and consistency. 76 | 77 | """ 78 | if self.action == self.import_action: 79 | return self.resource_class.get_model_queryset() # pragma: no cover 80 | return super().get_queryset() 81 | 82 | def get_import_detail_serializer_class(self): 83 | """Get serializer which will be used show details of import job.""" 84 | return self.import_detail_serializer_class 85 | 86 | def get_import_create_serializer_class(self): 87 | """Get serializer which will be used to start import job.""" 88 | return serializers.get_create_import_job_serializer( 89 | self.resource_class, 90 | ) 91 | 92 | def get_import_resource_kwargs(self) -> dict[str, typing.Any]: 93 | """Provide extra arguments to resource class.""" 94 | return {} 95 | 96 | def get_serializer(self, *args, **kwargs): 97 | """Provide resource kwargs to serializer class.""" 98 | if self.action == self.import_action: 99 | kwargs.setdefault( 100 | "resource_kwargs", 101 | self.get_import_resource_kwargs(), 102 | ) 103 | return super().get_serializer(*args, **kwargs) 104 | 105 | def start_import(self, request: request.Request) -> response.Response: 106 | """Validate request data and start ImportJob.""" 107 | serializer = self.get_serializer(data=request.data) 108 | serializer.is_valid(raise_exception=True) 109 | 110 | import_job = serializer.save() 111 | 112 | return response.Response( 113 | data=self.get_import_detail_serializer_class()( 114 | instance=import_job, 115 | ).data, 116 | status=status.HTTP_201_CREATED, 117 | ) 118 | -------------------------------------------------------------------------------- /import_export_extensions/api/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | from .export_job import ( 2 | CreateExportJob, 3 | ExportJobSerializer, 4 | get_create_export_job_serializer, 5 | ) 6 | from .import_job import ( 7 | CreateImportJob, 8 | ImportJobSerializer, 9 | get_create_import_job_serializer, 10 | ) 11 | from .progress import ProgressInfoSerializer, ProgressSerializer 12 | -------------------------------------------------------------------------------- /import_export_extensions/api/serializers/export_job.py: -------------------------------------------------------------------------------- 1 | import collections.abc 2 | import typing 3 | 4 | from rest_framework import request, serializers 5 | 6 | from celery import states 7 | 8 | from ... import models, resources 9 | from .progress import ProgressSerializer 10 | 11 | 12 | class ExportProgressSerializer(ProgressSerializer): 13 | """Serializer to show ExportJob progress.""" 14 | 15 | state = serializers.ChoiceField( 16 | choices=[ 17 | *models.ExportJob.ExportStatus.values, 18 | states.PENDING, 19 | states.STARTED, 20 | states.SUCCESS, 21 | ], 22 | ) 23 | 24 | 25 | class ExportJobSerializer(serializers.ModelSerializer): 26 | """Serializer to show information about export job.""" 27 | 28 | progress = ExportProgressSerializer() 29 | 30 | class Meta: 31 | model = models.ExportJob 32 | fields = ( 33 | "id", 34 | "export_status", 35 | "data_file", 36 | "progress", 37 | "export_started", 38 | "export_finished", 39 | "created", 40 | "modified", 41 | ) 42 | 43 | 44 | class CreateExportJob(serializers.Serializer): 45 | """Base Serializer to start export job. 46 | 47 | It used to set up base workflow of ExportJob creation via API. 48 | 49 | """ 50 | 51 | resource_class: type[resources.CeleryModelResource] 52 | 53 | def __init__( 54 | self, 55 | *args, 56 | ordering: collections.abc.Sequence[str] | None = None, 57 | filter_kwargs: dict[str, typing.Any] | None = None, 58 | resource_kwargs: dict[str, typing.Any] | None = None, 59 | **kwargs, 60 | ): 61 | """Set ordering, filter kwargs and current user.""" 62 | super().__init__(*args, **kwargs) 63 | self._ordering = ordering 64 | self._filter_kwargs = filter_kwargs 65 | self._resource_kwargs = resource_kwargs or {} 66 | self._request: request.Request = self.context.get("request") 67 | self._user = getattr(self._request, "user", None) 68 | 69 | def validate(self, attrs: dict[str, typing.Any]) -> dict[str, typing.Any]: 70 | """Check that ordering and filter kwargs are valid.""" 71 | self.resource_class( 72 | ordering=self._ordering, 73 | filter_kwargs=self._filter_kwargs, 74 | created_by=self._user, 75 | **self._resource_kwargs, 76 | ).get_queryset() 77 | return attrs 78 | 79 | def create( 80 | self, 81 | validated_data: dict[str, typing.Any], 82 | ) -> models.ExportJob: 83 | """Create export job.""" 84 | file_format_class = self.resource_class.get_supported_extensions_map()[ 85 | validated_data["file_format"] 86 | ] 87 | return models.ExportJob.objects.create( 88 | resource_path=self.resource_class.class_path, 89 | file_format_path=f"{file_format_class.__module__}.{file_format_class.__name__}", 90 | resource_kwargs=dict( 91 | ordering=self._ordering, 92 | filter_kwargs=self._filter_kwargs, 93 | **self._resource_kwargs, 94 | ), 95 | created_by=self._user, 96 | ) 97 | 98 | def update(self, instance, validated_data): 99 | """Empty method to pass linters checks.""" 100 | 101 | 102 | # Use it to cache already generated serializers to avoid duplication 103 | _GENERATED_EXPORT_JOB_SERIALIZERS: dict[ 104 | type[resources.CeleryModelResource], 105 | type, 106 | ] = {} 107 | 108 | 109 | def get_create_export_job_serializer( 110 | resource: type[resources.CeleryModelResource], 111 | ) -> type: 112 | """Create serializer for ExportJobs creation.""" 113 | if resource in _GENERATED_EXPORT_JOB_SERIALIZERS: 114 | return _GENERATED_EXPORT_JOB_SERIALIZERS[resource] 115 | 116 | class _CreateExportJob(CreateExportJob): 117 | """Serializer to start export job.""" 118 | 119 | resource_class: type[resources.CeleryModelResource] = resource 120 | file_format = serializers.ChoiceField( 121 | required=True, 122 | choices=[ 123 | supported_format().get_extension() 124 | for supported_format in resource.SUPPORTED_FORMATS 125 | ], 126 | ) 127 | 128 | _GENERATED_EXPORT_JOB_SERIALIZERS[resource] = type( 129 | f"{resource.__name__}CreateExportJob", 130 | (_CreateExportJob,), 131 | {}, 132 | ) 133 | return _GENERATED_EXPORT_JOB_SERIALIZERS[resource] 134 | -------------------------------------------------------------------------------- /import_export_extensions/api/serializers/import_job.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from rest_framework import request, serializers 4 | 5 | from celery import states 6 | 7 | from ... import models, resources 8 | from . import import_job_details as details 9 | from .progress import ProgressSerializer 10 | 11 | 12 | class ImportProgressSerializer(ProgressSerializer): 13 | """Serializer to show ImportJob progress.""" 14 | 15 | state = serializers.ChoiceField( 16 | choices=[ 17 | *models.ImportJob.ImportStatus.values, 18 | states.PENDING, 19 | states.STARTED, 20 | states.SUCCESS, 21 | ], 22 | ) 23 | 24 | 25 | class ImportJobSerializer(serializers.ModelSerializer): 26 | """Serializer to show information about import job.""" 27 | 28 | progress = ImportProgressSerializer() 29 | 30 | import_params = details.ImportParamsSerializer( 31 | read_only=True, 32 | source="*", 33 | ) 34 | totals = details.TotalsSerializer( 35 | read_only=True, 36 | source="*", 37 | ) 38 | parse_error = serializers.CharField( 39 | source="error_message", 40 | read_only=True, 41 | allow_blank=True, 42 | ) 43 | input_error = details.InputErrorSerializer( 44 | source="*", 45 | read_only=True, 46 | ) 47 | skipped_errors = details.SkippedErrorsSerializer( 48 | source="*", 49 | read_only=True, 50 | ) 51 | importing_data = details.ImportingDataSerializer( 52 | read_only=True, 53 | source="*", 54 | ) 55 | input_errors_file = serializers.FileField( 56 | read_only=True, 57 | allow_null=True, 58 | ) 59 | is_all_rows_shown = details.IsAllRowsShowField( 60 | source="*", 61 | read_only=True, 62 | ) 63 | 64 | class Meta: 65 | model = models.ImportJob 66 | fields = ( 67 | "id", 68 | "progress", 69 | "import_status", 70 | "import_params", 71 | "totals", 72 | "parse_error", 73 | "input_error", 74 | "skipped_errors", 75 | "is_all_rows_shown", 76 | "importing_data", 77 | "input_errors_file", 78 | "import_started", 79 | "import_finished", 80 | "force_import", 81 | "created", 82 | "modified", 83 | ) 84 | 85 | 86 | class CreateImportJob(serializers.Serializer): 87 | """Base Serializer to start import job. 88 | 89 | It used to set up base workflow of ImportJob creation via API. 90 | 91 | """ 92 | 93 | resource_class: type[resources.CeleryModelResource] 94 | 95 | file = serializers.FileField(required=True) 96 | force_import = serializers.BooleanField(default=False, required=False) 97 | skip_parse_step = serializers.BooleanField(default=False, required=False) 98 | 99 | def __init__( 100 | self, 101 | *args, 102 | resource_kwargs: dict[str, typing.Any] | None = None, 103 | **kwargs, 104 | ): 105 | """Set filter kwargs and current user.""" 106 | super().__init__(*args, **kwargs) 107 | self._request: request.Request = self.context.get("request") 108 | self._resource_kwargs = resource_kwargs or {} 109 | self._user = getattr(self._request, "user", None) 110 | 111 | def create( 112 | self, 113 | validated_data: dict[str, typing.Any], 114 | ) -> models.ImportJob: 115 | """Create import job.""" 116 | return models.ImportJob.objects.create( 117 | data_file=validated_data["file"], 118 | force_import=validated_data["force_import"], 119 | skip_parse_step=validated_data["skip_parse_step"], 120 | resource_path=self.resource_class.class_path, 121 | resource_kwargs=self._resource_kwargs, 122 | created_by=self._user, 123 | ) 124 | 125 | def update(self, instance, validated_data): 126 | """Empty method to pass linters checks.""" 127 | 128 | 129 | # Use it to cache already generated serializers to avoid duplication 130 | _GENERATED_IMPORT_JOB_SERIALIZERS: dict[ 131 | type[resources.CeleryModelResource], 132 | type, 133 | ] = {} 134 | 135 | 136 | def get_create_import_job_serializer( 137 | resource: type[resources.CeleryModelResource], 138 | ) -> type: 139 | """Create serializer for ImportJobs creation.""" 140 | if resource in _GENERATED_IMPORT_JOB_SERIALIZERS: 141 | return _GENERATED_IMPORT_JOB_SERIALIZERS[resource] 142 | 143 | class _CreateImportJob(CreateImportJob): 144 | """Serializer to start import job.""" 145 | 146 | resource_class: type[resources.CeleryModelResource] = resource 147 | 148 | _GENERATED_IMPORT_JOB_SERIALIZERS[resource] = type( 149 | f"{resource.__name__}CreateImportJob", 150 | (_CreateImportJob,), 151 | {}, 152 | ) 153 | return _GENERATED_IMPORT_JOB_SERIALIZERS[resource] 154 | -------------------------------------------------------------------------------- /import_export_extensions/api/serializers/progress.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | 3 | 4 | class ProgressInfoSerializer(serializers.Serializer): 5 | """Serializer to show progress info, like how much is done.""" 6 | 7 | current = serializers.IntegerField() 8 | total = serializers.IntegerField() 9 | 10 | 11 | class ProgressSerializer(serializers.Serializer): 12 | """Serializer to show progress of job.""" 13 | 14 | info = ProgressInfoSerializer() 15 | -------------------------------------------------------------------------------- /import_export_extensions/api/views/__init__.py: -------------------------------------------------------------------------------- 1 | from .export_job import ( 2 | BaseExportJobForUserViewSet, 3 | BaseExportJobViewSet, 4 | ExportJobForUserViewSet, 5 | ExportJobViewSet, 6 | ) 7 | from .import_job import ( 8 | BaseImportJobForUserViewSet, 9 | BaseImportJobViewSet, 10 | ImportJobForUserViewSet, 11 | ImportJobViewSet, 12 | ) 13 | -------------------------------------------------------------------------------- /import_export_extensions/api/views/export_job.py: -------------------------------------------------------------------------------- 1 | import collections.abc 2 | import contextlib 3 | 4 | from rest_framework import ( 5 | decorators, 6 | exceptions, 7 | mixins, 8 | permissions, 9 | response, 10 | status, 11 | viewsets, 12 | ) 13 | 14 | import django_filters 15 | 16 | from ... import models 17 | from .. import mixins as core_mixins 18 | 19 | 20 | class BaseExportJobViewSet( 21 | mixins.ListModelMixin, 22 | mixins.RetrieveModelMixin, 23 | viewsets.GenericViewSet, 24 | ): 25 | """Base viewset for managing export jobs.""" 26 | 27 | permission_classes = (permissions.IsAuthenticated,) 28 | serializer_class = core_mixins.ExportStartActionMixin.export_detail_serializer_class # noqa: E501 29 | queryset = models.ExportJob.objects.all() 30 | filterset_class: django_filters.rest_framework.FilterSet | None = None 31 | search_fields: collections.abc.Sequence[str] = ("id",) 32 | ordering: collections.abc.Sequence[str] = ( 33 | "id", 34 | ) 35 | ordering_fields: collections.abc.Sequence[str] = ( 36 | "id", 37 | "created", 38 | "modified", 39 | ) 40 | 41 | def __init_subclass__(cls) -> None: 42 | """Dynamically create an cancel api endpoints. 43 | 44 | Need to do this to enable action and correct open-api spec generated by 45 | drf_spectacular. 46 | 47 | """ 48 | super().__init_subclass__() 49 | decorators.action( 50 | methods=["POST"], 51 | detail=True, 52 | )(cls.cancel) 53 | # Correct specs of drf-spectacular if it is installed 54 | with contextlib.suppress(ImportError): 55 | from drf_spectacular.utils import extend_schema, extend_schema_view 56 | if hasattr(cls, "get_export_detail_serializer_class"): 57 | response_serializer = cls().get_export_detail_serializer_class() # noqa: E501 58 | else: 59 | response_serializer = cls().get_serializer_class() 60 | extend_schema_view( 61 | cancel=extend_schema( 62 | request=None, 63 | responses={ 64 | status.HTTP_200_OK: response_serializer, 65 | }, 66 | ), 67 | )(cls) 68 | 69 | def cancel(self, *args, **kwargs) -> response.Response: 70 | """Cancel export job that is in progress.""" 71 | job: models.ExportJob = self.get_object() 72 | 73 | try: 74 | job.cancel_export() 75 | except ValueError as error: 76 | raise exceptions.ValidationError(error.args[0]) from error 77 | 78 | serializer = self.get_serializer(instance=job) 79 | return response.Response( 80 | status=status.HTTP_200_OK, 81 | data=serializer.data, 82 | ) 83 | 84 | class ExportJobViewSet( 85 | core_mixins.ExportStartActionMixin, 86 | BaseExportJobViewSet, 87 | ): 88 | """Base API viewset for ExportJob model. 89 | 90 | Based on resource_class it will generate an endpoint which will allow to 91 | start an export of model which was specified in resource_class. This 92 | endpoint will support filtration based on FilterSet class specified in 93 | resource. On success this endpoint we return an instance of export, to 94 | get status of job, just use detail(retrieve) endpoint. 95 | 96 | """ 97 | 98 | export_action_name = "start" 99 | export_action_url = "start" 100 | 101 | def get_queryset(self): 102 | """Filter export jobs by resource used in viewset.""" 103 | if self.action == getattr(self, "export_action", ""): 104 | # To make it consistent and for better support of drf-spectacular 105 | return super().get_queryset() # pragma: no cover 106 | return super().get_queryset().filter( 107 | resource_path=self.resource_class.class_path, 108 | ) 109 | 110 | 111 | class ExportJobForUserViewSet( 112 | core_mixins.LimitQuerySetToCurrentUserMixin, 113 | ExportJobViewSet, 114 | ): 115 | """Viewset for providing export feature to users.""" 116 | 117 | class BaseExportJobForUserViewSet( 118 | core_mixins.LimitQuerySetToCurrentUserMixin, 119 | BaseExportJobViewSet, 120 | ): 121 | """Viewset for providing export job management to users.""" 122 | -------------------------------------------------------------------------------- /import_export_extensions/api/views/import_job.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import contextlib 3 | 4 | from rest_framework import ( 5 | decorators, 6 | exceptions, 7 | mixins, 8 | permissions, 9 | response, 10 | status, 11 | viewsets, 12 | ) 13 | 14 | from ... import models 15 | from .. import mixins as core_mixins 16 | 17 | 18 | class BaseImportJobViewSet( 19 | mixins.ListModelMixin, 20 | mixins.RetrieveModelMixin, 21 | viewsets.GenericViewSet, 22 | ): 23 | """Base viewset for managing import jobs.""" 24 | 25 | permission_classes = (permissions.IsAuthenticated,) 26 | serializer_class = core_mixins.ImportStartActionMixin.import_detail_serializer_class # noqa: E501 27 | queryset = models.ImportJob.objects.all() 28 | search_fields: collections.abc.Sequence[str] = ("id",) 29 | ordering: collections.abc.Sequence[str] = ( 30 | "id", 31 | ) 32 | ordering_fields: collections.abc.Sequence[str] = ( 33 | "id", 34 | "created", 35 | "modified", 36 | ) 37 | 38 | def __init_subclass__(cls) -> None: 39 | """Dynamically create an cancel api endpoints. 40 | 41 | Need to do this to enable action and correct open-api spec generated by 42 | drf_spectacular. 43 | 44 | """ 45 | super().__init_subclass__() 46 | decorators.action( 47 | methods=["POST"], 48 | detail=True, 49 | )(cls.cancel) 50 | decorators.action( 51 | methods=["POST"], 52 | detail=True, 53 | )(cls.confirm) 54 | # Correct specs of drf-spectacular if it is installed 55 | with contextlib.suppress(ImportError): 56 | from drf_spectacular.utils import extend_schema, extend_schema_view 57 | if hasattr(cls, "get_import_detail_serializer_class"): 58 | response_serializer = cls().get_import_detail_serializer_class() # noqa: E501 59 | else: 60 | response_serializer = cls().get_serializer_class() 61 | extend_schema_view( 62 | cancel=extend_schema( 63 | request=None, 64 | responses={ 65 | status.HTTP_200_OK: response_serializer, 66 | }, 67 | ), 68 | confirm=extend_schema( 69 | request=None, 70 | responses={ 71 | status.HTTP_200_OK: response_serializer, 72 | }, 73 | ), 74 | )(cls) 75 | 76 | def confirm(self, *args, **kwargs): 77 | """Confirm import job that has `parsed` status.""" 78 | job: models.ImportJob = self.get_object() 79 | 80 | try: 81 | job.confirm_import() 82 | except ValueError as error: 83 | raise exceptions.ValidationError(error.args[0]) from error 84 | 85 | serializer = self.get_serializer(instance=job) 86 | return response.Response( 87 | status=status.HTTP_200_OK, 88 | data=serializer.data, 89 | ) 90 | 91 | def cancel(self, *args, **kwargs): 92 | """Cancel import job that is in progress.""" 93 | job: models.ImportJob = self.get_object() 94 | 95 | try: 96 | job.cancel_import() 97 | except ValueError as error: 98 | raise exceptions.ValidationError(error.args[0]) from error 99 | 100 | serializer = self.get_serializer(instance=job) 101 | return response.Response( 102 | status=status.HTTP_200_OK, 103 | data=serializer.data, 104 | ) 105 | 106 | class ImportJobViewSet( 107 | core_mixins.ImportStartActionMixin, 108 | BaseImportJobViewSet, 109 | ): 110 | """Base API viewset for ImportJob model. 111 | 112 | Based on resource_class it will generate an endpoint which will allow to 113 | start an import to model which was specified in resource_class. On success 114 | this endpoint we return an instance of import. 115 | 116 | Endpoints: 117 | list - to get list of all import jobs 118 | details(retrieve) - to get status of import job 119 | start - create import job and start parsing data from attached file 120 | confirm - confirm import after parsing process is finished 121 | cancel - stop importing/parsing process and cancel this import job 122 | 123 | """ 124 | 125 | import_action_name = "start" 126 | import_action_url = "start" 127 | 128 | def get_queryset(self): 129 | """Filter import jobs by resource used in viewset.""" 130 | if self.action == getattr(self, "import_action", ""): 131 | # To make it consistent and for better support of drf-spectacular 132 | return super().get_queryset() # pragma: no cover 133 | return super().get_queryset().filter( 134 | resource_path=self.resource_class.class_path, 135 | ) 136 | 137 | class ImportJobForUserViewSet( 138 | core_mixins.LimitQuerySetToCurrentUserMixin, 139 | ImportJobViewSet, 140 | ): 141 | """Viewset for providing import feature to users.""" 142 | 143 | class BaseImportJobForUserViewSet( 144 | core_mixins.LimitQuerySetToCurrentUserMixin, 145 | BaseImportJobViewSet, 146 | ): 147 | """Viewset for providing export job management to users.""" 148 | -------------------------------------------------------------------------------- /import_export_extensions/apps.py: -------------------------------------------------------------------------------- 1 | import mimetypes 2 | 3 | from django.apps import AppConfig 4 | from django.conf import settings 5 | from django.utils.translation import gettext_lazy as _ 6 | 7 | # Default configuration 8 | # Maximum num of rows to be imported 9 | DEFAULT_MAX_DATASET_ROWS = 100000 10 | # After how many imported/exported rows celery task status will be updated 11 | DEFAULT_STATUS_UPDATE_ROW_COUNT = 100 12 | # Default filter class backends for export api 13 | DEFAULT_DRF_EXPORT_DJANGO_FILTERS_BACKEND = ( 14 | "django_filters.rest_framework.DjangoFilterBackend" 15 | ) 16 | DEFAULT_DRF_EXPORT_ORDERING_BACKEND = "rest_framework.filters.OrderingFilter" 17 | 18 | 19 | class CeleryImportExport(AppConfig): 20 | """Default configuration for CeleryImportExport app.""" 21 | 22 | name = "import_export_extensions" 23 | verbose_name = _("Celery Import/Export") 24 | default_auto_field = "django.db.models.BigAutoField" 25 | 26 | def ready(self): 27 | """Set up default settings.""" 28 | settings.IMPORT_EXPORT_MAX_DATASET_ROWS = getattr( 29 | settings, 30 | "IMPORT_EXPORT_MAX_DATASET_ROWS", 31 | DEFAULT_MAX_DATASET_ROWS, 32 | ) 33 | settings.MIME_TYPES_MAP = mimetypes.types_map.copy() 34 | settings.STATUS_UPDATE_ROW_COUNT = getattr( 35 | settings, 36 | "STATUS_UPDATE_ROW_COUNT", 37 | DEFAULT_STATUS_UPDATE_ROW_COUNT, 38 | ) 39 | settings.DRF_EXPORT_DJANGO_FILTERS_BACKEND = getattr( 40 | settings, 41 | "DRF_EXPORT_DJANGO_FILTERS_BACKEND", 42 | DEFAULT_DRF_EXPORT_DJANGO_FILTERS_BACKEND, 43 | ) 44 | settings.DRF_EXPORT_ORDERING_BACKEND = getattr( 45 | settings, 46 | "DRF_EXPORT_ORDERING_BACKEND", 47 | DEFAULT_DRF_EXPORT_ORDERING_BACKEND, 48 | ) 49 | -------------------------------------------------------------------------------- /import_export_extensions/fields.py: -------------------------------------------------------------------------------- 1 | from django.db.models.fields.reverse_related import ManyToManyRel 2 | 3 | from import_export.fields import Field 4 | 5 | 6 | class IntermediateManyToManyField(Field): 7 | """Resource field for M2M with custom ``through`` model. 8 | 9 | By default, ``django-import-export`` set up object attributes using 10 | ``setattr(obj, attribute_name, value)``, where ``value`` is ``QuerySet`` 11 | of related model objects. But django forbid this when `ManyToManyField`` 12 | used with custom ``through`` model. 13 | 14 | This field expects be used with custom widget that return not simple value, 15 | but dict with intermediate model attributes. 16 | 17 | For easy comments following models will be used: 18 | 19 | Artist: 20 | name 21 | bands -> 22 | 23 | Membership: 24 | artist 25 | band 26 | date_joined 27 | 28 | Band: 29 | title 30 | <- artists 31 | 32 | So this field should be used for exporting Artists with `bands` field. 33 | 34 | Save workflow is following: 35 | 1. clean data (extract dicts) 36 | 2. Remove current M2M instances of object 37 | 3. Create new M2M instances based on current object 38 | 39 | """ 40 | 41 | def _format_exception(self, exception): 42 | """Shortcut for humanizing exception.""" 43 | error = str(exception) 44 | if hasattr(exception, "messages"): 45 | error = " ".join(exception.messages) 46 | 47 | msg = f"Column '{self.column_name}': {error}" 48 | raise ValueError(msg) from exception 49 | 50 | def get_value(self, obj): 51 | """Return the value of the object's attribute. 52 | 53 | This method should return instances of intermediate model, i.e. 54 | Membership instances 55 | 56 | """ 57 | if self.attribute is None: 58 | return None 59 | 60 | m2m_rel, _, _ = self.get_relation_field_params(obj) 61 | 62 | if not obj.id: 63 | return m2m_rel.through.objects.none() 64 | 65 | through_model_accessor_name = self.get_through_model_accessor_name( 66 | obj, 67 | m2m_rel, 68 | ) 69 | return getattr(obj, through_model_accessor_name).all() 70 | 71 | def get_relation_field_params(self, obj): 72 | """Shortcut to get relation field params. 73 | 74 | Gets relation, field itself, its name and its reversed field name 75 | 76 | """ 77 | # retrieve M2M field itself (i.e. Artist.bands) 78 | field = obj._meta.get_field(self.attribute) 79 | 80 | # if field is `ManyToManyRel` - it is a reversed relation 81 | if isinstance(field, ManyToManyRel): 82 | m2m_rel = field 83 | m2m_field = m2m_rel.field 84 | field_name = m2m_field.m2m_reverse_field_name() 85 | reversed_field_name = m2m_field.m2m_field_name() 86 | return m2m_rel, field_name, reversed_field_name 87 | 88 | # otherwise it is a forward relation 89 | m2m_rel = field.remote_field 90 | m2m_field = field 91 | field_name = m2m_field.m2m_field_name() 92 | reversed_field_name = m2m_field.m2m_reverse_field_name() 93 | return m2m_rel, field_name, reversed_field_name 94 | 95 | def get_through_model_accessor_name(self, obj, m2m_rel) -> str: 96 | """Shortcut to get through model accessor name.""" 97 | for related_object in obj._meta.related_objects: 98 | if related_object.related_model is m2m_rel.through: 99 | return related_object.accessor_name 100 | 101 | raise ValueError( 102 | f"{obj._meta.model} has no relation with {m2m_rel.through}", 103 | ) 104 | 105 | def save(self, obj, data, *args, **kwargs): 106 | """Add M2M relations for obj from data. 107 | 108 | Args: 109 | obj(model instance): object being imported 110 | data(OrderedDict): all extracted data for object 111 | 112 | Example: 113 | obj - Artist instance 114 | 115 | """ 116 | if self.readonly: 117 | return 118 | 119 | ( 120 | m2m_rel, 121 | field_name, 122 | reversed_field_name, 123 | ) = self.get_relation_field_params(obj) 124 | 125 | # retrieve intermediate model 126 | # intermediate_model is the Membership model 127 | intermediate_model = m2m_rel.through 128 | 129 | # should be returned following list: 130 | # [{'band': , 'date_joined': '2016-08-18'}] 131 | instances_data = self.clean(data) 132 | 133 | # remove current related objects, 134 | # i.e. clear artists's band 135 | through_model_accessor_name = self.get_through_model_accessor_name( 136 | obj, 137 | m2m_rel, 138 | ) 139 | getattr(obj, through_model_accessor_name).all().delete() 140 | 141 | for rel_obj_data in instances_data: 142 | # add current and remote object to intermediate instance data 143 | # i.e. {'artist': , 'band': rel_obj_data['properties']} 144 | obj_data = rel_obj_data["properties"].copy() 145 | obj_data.update( 146 | { 147 | field_name: obj, 148 | reversed_field_name: rel_obj_data["object"], 149 | }, 150 | ) 151 | intermediate_obj = intermediate_model(**obj_data) 152 | try: 153 | intermediate_obj.full_clean() 154 | except Exception as e: 155 | self._format_exception(e) 156 | intermediate_obj.save() 157 | -------------------------------------------------------------------------------- /import_export_extensions/forms.py: -------------------------------------------------------------------------------- 1 | """Source code of origin application ``django-import-export``. 2 | 3 | The difference is that these forms don't required file format, it is taken from 4 | file. And in ExportForm the default export format is `xlsx`. 5 | 6 | """ 7 | 8 | from django import forms 9 | from django.utils.translation import gettext_lazy as _ 10 | 11 | from import_export.formats import base_formats 12 | 13 | 14 | class ImportForm(forms.Form): 15 | """Form for creating import.""" 16 | 17 | import_file = forms.FileField( 18 | label=_("File to import"), 19 | ) 20 | 21 | 22 | class ExportForm(forms.Form): 23 | """Form for exporting.""" 24 | 25 | file_format = forms.ChoiceField( 26 | label=_("Choose format"), 27 | choices=(), 28 | ) 29 | 30 | def __init__( 31 | self, 32 | formats: list[type[base_formats.Format]], 33 | *args, 34 | **kwargs, 35 | ): 36 | super().__init__(*args, **kwargs) 37 | choices = [] 38 | initial_choice = 0 39 | for index, export_format in enumerate(formats): 40 | extension = export_format().get_title() 41 | if extension.lower() == "xlsx": 42 | initial_choice = index 43 | choices.append((str(index), extension)) 44 | if len(formats) > 1: 45 | choices.insert(0, ("", "---")) 46 | 47 | self.fields["file_format"].choices = choices 48 | self.fields["file_format"].initial = initial_choice 49 | -------------------------------------------------------------------------------- /import_export_extensions/migrations/0002_alter_exportjob_export_status.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.4 on 2023-08-25 04:13 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("import_export_extensions", "0001_initial"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="exportjob", 14 | name="export_status", 15 | field=models.CharField( 16 | choices=[ 17 | ("CREATED", "Created"), 18 | ("EXPORTING", "Exporting"), 19 | ("EXPORT_ERROR", "Export Error"), 20 | ("EXPORTED", "Exported"), 21 | ("CANCELLED", "Cancelled"), 22 | ], 23 | default="CREATED", 24 | max_length=20, 25 | verbose_name="Job status", 26 | ), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /import_export_extensions/migrations/0003_importjob_skip_parse_step.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.4 on 2023-08-26 04:28 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("import_export_extensions", "0002_alter_exportjob_export_status"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AddField( 13 | model_name="importjob", 14 | name="skip_parse_step", 15 | field=models.BooleanField( 16 | default=False, 17 | help_text="Start importing without confirmation", 18 | verbose_name="Skip parse step", 19 | ), 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /import_export_extensions/migrations/0004_alter_exportjob_created_by_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.5 on 2023-10-06 11:54 2 | 3 | import django.db.models.deletion 4 | from django.conf import settings 5 | from django.db import migrations, models 6 | 7 | import picklefield.fields 8 | 9 | 10 | class Migration(migrations.Migration): 11 | dependencies = [ 12 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 13 | ("import_export_extensions", "0003_importjob_skip_parse_step"), 14 | ] 15 | 16 | operations = [ 17 | migrations.AlterField( 18 | model_name="exportjob", 19 | name="created_by", 20 | field=models.ForeignKey( 21 | editable=False, 22 | help_text="User which started job", 23 | null=True, 24 | on_delete=django.db.models.deletion.SET_NULL, 25 | to=settings.AUTH_USER_MODEL, 26 | verbose_name="Created by", 27 | ), 28 | ), 29 | migrations.AlterField( 30 | model_name="exportjob", 31 | name="error_message", 32 | field=models.CharField( 33 | blank=True, 34 | default=str, 35 | help_text="Python error message in case of import/export error", 36 | max_length=128, 37 | verbose_name="Error message", 38 | ), 39 | ), 40 | migrations.AlterField( 41 | model_name="exportjob", 42 | name="resource_path", 43 | field=models.CharField( 44 | help_text="Dotted path to subclass of `import_export.Resource` that should be used for import", 45 | max_length=128, 46 | verbose_name="Resource class path", 47 | ), 48 | ), 49 | migrations.AlterField( 50 | model_name="exportjob", 51 | name="result", 52 | field=picklefield.fields.PickledObjectField( 53 | default=str, 54 | editable=False, 55 | help_text="Internal job result object that contain info about job statistics. Pickled Python object", 56 | verbose_name="Job result", 57 | ), 58 | ), 59 | migrations.AlterField( 60 | model_name="exportjob", 61 | name="traceback", 62 | field=models.TextField( 63 | blank=True, 64 | default=str, 65 | help_text="Python traceback in case of import/export error", 66 | verbose_name="Traceback", 67 | ), 68 | ), 69 | migrations.AlterField( 70 | model_name="importjob", 71 | name="created_by", 72 | field=models.ForeignKey( 73 | editable=False, 74 | help_text="User which started job", 75 | null=True, 76 | on_delete=django.db.models.deletion.SET_NULL, 77 | to=settings.AUTH_USER_MODEL, 78 | verbose_name="Created by", 79 | ), 80 | ), 81 | migrations.AlterField( 82 | model_name="importjob", 83 | name="error_message", 84 | field=models.CharField( 85 | blank=True, 86 | default=str, 87 | help_text="Python error message in case of import/export error", 88 | max_length=128, 89 | verbose_name="Error message", 90 | ), 91 | ), 92 | migrations.AlterField( 93 | model_name="importjob", 94 | name="result", 95 | field=picklefield.fields.PickledObjectField( 96 | default=str, 97 | editable=False, 98 | help_text="Internal job result object that contain info about job statistics. Pickled Python object", 99 | verbose_name="Job result", 100 | ), 101 | ), 102 | migrations.AlterField( 103 | model_name="importjob", 104 | name="traceback", 105 | field=models.TextField( 106 | blank=True, 107 | default=str, 108 | help_text="Python traceback in case of import/export error", 109 | verbose_name="Traceback", 110 | ), 111 | ), 112 | ] 113 | -------------------------------------------------------------------------------- /import_export_extensions/migrations/0005_importjob_force_import.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.7 on 2023-11-16 10:49 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ( 9 | "import_export_extensions", 10 | "0004_alter_exportjob_created_by_and_more", 11 | ), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name="importjob", 17 | name="force_import", 18 | field=models.BooleanField( 19 | default=False, 20 | help_text="Import data with skip invalid rows.", 21 | verbose_name="Force import", 22 | ), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /import_export_extensions/migrations/0006_importjob_input_errors_file.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.7 on 2024-01-15 10:40 2 | 3 | import functools 4 | 5 | from django.db import migrations, models 6 | 7 | import import_export_extensions.models.tools 8 | 9 | 10 | class Migration(migrations.Migration): 11 | dependencies = [ 12 | ("import_export_extensions", "0005_importjob_force_import"), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddField( 17 | model_name="importjob", 18 | name="input_errors_file", 19 | field=models.FileField( 20 | help_text="File that contain failed rows", 21 | max_length=512, 22 | null=True, 23 | upload_to=functools.partial( 24 | import_export_extensions.models.tools.upload_file_to, 25 | *(), 26 | main_folder_name="import", 27 | ), 28 | verbose_name="Input errors file", 29 | ), 30 | ), 31 | ] 32 | -------------------------------------------------------------------------------- /import_export_extensions/migrations/0007_alter_exportjob_result_alter_importjob_result.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.7 on 2024-05-16 09:21 2 | 3 | from django.db import migrations 4 | 5 | import import_export.results 6 | import picklefield.fields 7 | 8 | 9 | class Migration(migrations.Migration): 10 | dependencies = [ 11 | ("import_export_extensions", "0006_importjob_input_errors_file"), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name="exportjob", 17 | name="result", 18 | field=picklefield.fields.PickledObjectField( 19 | default=import_export.results.Result, 20 | editable=False, 21 | help_text="Internal job result object that contain info about job statistics. Pickled Python object", 22 | verbose_name="Job result", 23 | ), 24 | ), 25 | migrations.AlterField( 26 | model_name="importjob", 27 | name="result", 28 | field=picklefield.fields.PickledObjectField( 29 | default=import_export.results.Result, 30 | editable=False, 31 | help_text="Internal job result object that contain info about job statistics. Pickled Python object", 32 | verbose_name="Job result", 33 | ), 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /import_export_extensions/migrations/0008_alter_exportjob_id_alter_importjob_id.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.1.3 on 2024-12-05 07:16 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ( 9 | "import_export_extensions", 10 | "0007_alter_exportjob_result_alter_importjob_result", 11 | ), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name="exportjob", 17 | name="id", 18 | field=models.BigAutoField( 19 | auto_created=True, 20 | primary_key=True, 21 | serialize=False, 22 | verbose_name="ID", 23 | ), 24 | ), 25 | migrations.AlterField( 26 | model_name="importjob", 27 | name="id", 28 | field=models.BigAutoField( 29 | auto_created=True, 30 | primary_key=True, 31 | serialize=False, 32 | verbose_name="ID", 33 | ), 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /import_export_extensions/migrations/0009_alter_exportjob_data_file_alter_importjob_data_file.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.1.6 on 2025-03-20 11:59 2 | 3 | import functools 4 | 5 | from django.db import migrations, models 6 | 7 | import import_export_extensions.models.tools 8 | 9 | 10 | class Migration(migrations.Migration): 11 | dependencies = [ 12 | ( 13 | "import_export_extensions", 14 | "0008_alter_exportjob_id_alter_importjob_id", 15 | ), 16 | ] 17 | 18 | operations = [ 19 | migrations.AlterField( 20 | model_name="exportjob", 21 | name="data_file", 22 | field=models.FileField( 23 | help_text="File that contain exported data", 24 | max_length=512, 25 | storage=import_export_extensions.models.tools.select_storage, 26 | upload_to=functools.partial( 27 | import_export_extensions.models.tools.upload_file_to, 28 | *(), 29 | main_folder_name="export", 30 | ), 31 | verbose_name="Data file", 32 | ), 33 | ), 34 | migrations.AlterField( 35 | model_name="importjob", 36 | name="data_file", 37 | field=models.FileField( 38 | help_text="File that contain data to be imported", 39 | max_length=512, 40 | storage=import_export_extensions.models.tools.select_storage, 41 | upload_to=functools.partial( 42 | import_export_extensions.models.tools.upload_file_to, 43 | *(), 44 | main_folder_name="import", 45 | ), 46 | verbose_name="Data file", 47 | ), 48 | ), 49 | ] 50 | -------------------------------------------------------------------------------- /import_export_extensions/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/import_export_extensions/migrations/__init__.py -------------------------------------------------------------------------------- /import_export_extensions/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .export_job import ExportJob 2 | from .import_job import ImportJob 3 | -------------------------------------------------------------------------------- /import_export_extensions/models/core.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from django.conf import settings 4 | from django.db import models 5 | from django.utils import module_loading 6 | from django.utils.translation import gettext_lazy as _ 7 | 8 | from import_export.results import Result 9 | from picklefield.fields import PickledObjectField 10 | 11 | 12 | class CreationDateTimeField(models.DateTimeField): 13 | """DateTimeField to indicate created datetime. 14 | 15 | By default, sets editable=False, blank=True, auto_now_add=True. 16 | 17 | """ 18 | 19 | def __init__(self, **kwargs): 20 | super().__init__( 21 | auto_now=True, 22 | verbose_name=_("Created"), 23 | ) 24 | 25 | 26 | class ModificationDateTimeField(models.DateTimeField): 27 | """DateTimeField to indicate modified datetime. 28 | 29 | By default, sets editable=False, blank=True, auto_now=True. 30 | 31 | Sets value to now every time the object is saved. 32 | 33 | """ 34 | 35 | def __init__(self, **kwargs): 36 | super().__init__( 37 | auto_now=True, 38 | verbose_name=_("Modified"), 39 | ) 40 | 41 | 42 | class TimeStampedModel(models.Model): 43 | """Model which tracks creation and modification time. 44 | 45 | An abstract base class model that provides self-managed "created" and 46 | "modified" fields. 47 | 48 | """ 49 | 50 | created = CreationDateTimeField() 51 | modified = ModificationDateTimeField() 52 | 53 | class Meta: 54 | abstract = True 55 | 56 | 57 | class TaskStateInfo(typing.TypedDict): 58 | """Class representing task state dict. 59 | 60 | Possible states: 61 | 1. PENDING 62 | 2. STARTED 63 | 3. SUCCESS 64 | 4. EXPORTING - custom status that also set export info 65 | 66 | https://docs.celeryproject.org/en/latest/userguide/tasks.html#states 67 | 68 | """ 69 | 70 | state: str 71 | info: dict[str, int] | None 72 | 73 | 74 | class BaseJob(TimeStampedModel): 75 | """Base model for managing celery jobs.""" 76 | 77 | resource_path = models.CharField( 78 | max_length=128, 79 | verbose_name=_("Resource class path"), 80 | help_text=_( 81 | "Dotted path to subclass of `import_export.Resource` that " 82 | "should be used for import", 83 | ), 84 | ) 85 | resource_kwargs = models.JSONField( 86 | default=dict, 87 | verbose_name=_("Resource kwargs"), 88 | help_text=_("Keyword parameters required for resource initialization"), 89 | ) 90 | traceback = models.TextField( 91 | blank=True, 92 | default=str, 93 | verbose_name=_("Traceback"), 94 | help_text=_("Python traceback in case of import/export error"), 95 | ) 96 | error_message = models.CharField( 97 | max_length=128, 98 | blank=True, 99 | default=str, 100 | verbose_name=_("Error message"), 101 | help_text=_("Python error message in case of import/export error"), 102 | ) 103 | created_by = models.ForeignKey( 104 | to=settings.AUTH_USER_MODEL, 105 | editable=False, 106 | null=True, 107 | on_delete=models.SET_NULL, 108 | verbose_name=_("Created by"), 109 | help_text=_("User which started job"), 110 | ) 111 | result = PickledObjectField( 112 | default=Result, 113 | verbose_name=_("Job result"), 114 | help_text=_( 115 | "Internal job result object that contain " 116 | "info about job statistics. Pickled Python object", 117 | ), 118 | ) 119 | 120 | class Meta: 121 | abstract = True 122 | 123 | @property 124 | def resource(self): 125 | """Get initialized resource instance.""" 126 | resource_class = module_loading.import_string(self.resource_path) 127 | resource = resource_class( 128 | created_by=self.created_by, 129 | **self.resource_kwargs, 130 | ) 131 | return resource 132 | 133 | @property 134 | def progress(self) -> TaskStateInfo | None: 135 | """Return dict with current job state.""" 136 | raise NotImplementedError 137 | -------------------------------------------------------------------------------- /import_export_extensions/models/tools.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import functools 3 | import typing 4 | import uuid 5 | 6 | from django.core.files.storage import ( 7 | InvalidStorageError, 8 | default_storage, 9 | storages, 10 | ) 11 | 12 | IMPORT_EXPORT_STORAGE_ALIAS = "django_import_export_extensions" 13 | 14 | 15 | def select_storage() -> dict[str, typing.Any]: 16 | """Pick storage for storing import/export files.""" 17 | storage = default_storage 18 | with contextlib.suppress(InvalidStorageError): 19 | storage = storages[IMPORT_EXPORT_STORAGE_ALIAS] 20 | return storage 21 | 22 | 23 | def upload_file_to( 24 | instance, 25 | filename: str, 26 | main_folder_name: str, 27 | ) -> str: 28 | """Upload instance's `file` to unique folder. 29 | 30 | Args: 31 | instance (typing.Union[ImportJob, ExportJob]): instance of job model 32 | main_folder_name(str): main folder -> import or export 33 | filename (str): file name of document's file 34 | 35 | Returns: 36 | str: Generated path for document's file. 37 | 38 | """ 39 | return ( 40 | "import_export_extensions/" 41 | f"{main_folder_name}/{uuid.uuid4()}/{filename}" 42 | ) 43 | 44 | 45 | upload_export_file_to = functools.partial( 46 | upload_file_to, 47 | main_folder_name="export", 48 | ) 49 | upload_import_file_to = functools.partial( 50 | upload_file_to, 51 | main_folder_name="import", 52 | ) 53 | upload_import_error_file_to = functools.partial( 54 | upload_file_to, 55 | main_folder_name="errors", 56 | ) 57 | -------------------------------------------------------------------------------- /import_export_extensions/results.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import typing 3 | 4 | from django.core.exceptions import ValidationError 5 | 6 | from import_export import results 7 | 8 | 9 | class Error(results.Error): 10 | """Customization of over base Error class from import export.""" 11 | 12 | def __repr__(self) -> str: 13 | """Return object representation in string format.""" 14 | return f"Error({self.error})" 15 | 16 | def __reduce__(self): 17 | """Simplify Exception object for pickling. 18 | 19 | `error` object may contain not picklable objects (for example, django's 20 | lazy text), so here it replaced with simple string. 21 | 22 | """ 23 | self.error = str(self.error) 24 | return super().__reduce__() 25 | 26 | 27 | class RowResult(results.RowResult): 28 | """Custom row result class with ability to store skipped errors in row.""" 29 | 30 | def __init__(self) -> None: 31 | """Copy of base init except creating validation error field.""" 32 | self.non_field_skipped_errors: list[results.Error] = [] 33 | self.field_skipped_errors: dict[str, list[ValidationError]] = dict() 34 | self.errors: list[Error] = [] 35 | self.diff: list[str] | None = None 36 | self.import_type = None 37 | self.row_values: dict[str, typing.Any] = {} 38 | self.object_id = None 39 | self.object_repr = None 40 | self.instance = None 41 | self.original = None 42 | # variable to store modified value of ValidationError 43 | self._validation_error: ValidationError | None = None 44 | 45 | @property 46 | def validation_error(self) -> ValidationError | None: 47 | """Return modified validation error.""" 48 | return self._validation_error 49 | 50 | @validation_error.setter 51 | def validation_error(self, value: ValidationError) -> None: 52 | """Modify passed error to reduce count of nested exception classes. 53 | 54 | Result class is saved in `ImportJob` model as pickled field 55 | and python's `pickle` can't handle 56 | `ValidationError` with high level of nested errors, 57 | therefore we reduce nesting by using string value of nested errors. 58 | 59 | If `ValidationError` has no `message_dict` attr, 60 | then it means that there're no nested exceptions 61 | and we can safely save it. 62 | 63 | Otherwise, we will go through all nested validation errors 64 | and build new `ValidationError` with only one level of 65 | nested `ValidationError` instances. 66 | 67 | """ 68 | if not hasattr(value, "message_dict"): 69 | self._validation_error = value 70 | return 71 | result = collections.defaultdict(list) 72 | for field, error_messages in value.message_dict.items(): 73 | validation_errors = [ 74 | ValidationError(message=message, code="invalid") 75 | for message in error_messages 76 | ] 77 | result[field].extend(validation_errors) 78 | self._validation_error = ValidationError(result) 79 | 80 | @property 81 | def has_skipped_errors(self) -> bool: 82 | """Return True if row contain any skipped errors.""" 83 | return bool( 84 | len(self.non_field_skipped_errors) > 0 85 | or len(self.field_skipped_errors) > 0, 86 | ) 87 | 88 | @property 89 | def skipped_errors_count(self) -> int: 90 | """Return count of skipped errors.""" 91 | return len(self.non_field_skipped_errors) + len( 92 | self.field_skipped_errors, 93 | ) 94 | 95 | @property 96 | def has_error_import_type(self) -> bool: 97 | """Return true if import type is not valid.""" 98 | return self.import_type not in self.valid_import_types 99 | 100 | 101 | class Result(results.Result): 102 | """Custom result class with ability to store info about skipped rows.""" 103 | 104 | @property 105 | def has_skipped_rows(self) -> bool: 106 | """Return True if contain any skipped rows.""" 107 | return bool(any(row.has_skipped_errors for row in self.rows)) 108 | 109 | @property 110 | def skipped_rows(self) -> list[RowResult]: 111 | """Return all rows with skipped errors.""" 112 | return list( 113 | filter(lambda row: row.has_skipped_errors, self.rows), 114 | ) 115 | -------------------------------------------------------------------------------- /import_export_extensions/signals.py: -------------------------------------------------------------------------------- 1 | from django import dispatch 2 | 3 | import_job_failed = dispatch.Signal() 4 | export_job_failed = dispatch.Signal() 5 | -------------------------------------------------------------------------------- /import_export_extensions/static/import_export_extensions/css/admin/admin.css: -------------------------------------------------------------------------------- 1 | .traceback { 2 | float: left; 3 | } 4 | 5 | .traceback p { 6 | padding-left: 0; 7 | } 8 | -------------------------------------------------------------------------------- /import_export_extensions/static/import_export_extensions/css/admin/import_result_diff.css: -------------------------------------------------------------------------------- 1 | ins { 2 | color: black; 3 | } 4 | del { 5 | color: black; 6 | } 7 | -------------------------------------------------------------------------------- /import_export_extensions/static/import_export_extensions/css/widgets/progress_bar.css: -------------------------------------------------------------------------------- 1 | progress { 2 | text-align: center; 3 | height: 2.0em; 4 | width: 100%; 5 | -webkit-appearance: none; 6 | border: none; 7 | position: relative; 8 | } 9 | progress:before { 10 | content: attr(data-label); 11 | font-size: 1.0em; 12 | font-weight: bold; 13 | color: var(--body-fg); 14 | vertical-align: center; 15 | text-align: center; 16 | 17 | /*Position text over the progress bar */ 18 | position:absolute; 19 | left:0; 20 | right:0; 21 | top: 5px; 22 | } 23 | progress::-webkit-progress-bar { 24 | background-color: var(--breadcrumbs-fg); 25 | } 26 | progress::-webkit-progress-value { 27 | background-color: var(--primary); 28 | } 29 | progress::-moz-progress-bar { 30 | background-color: var(--breadcrumbs-bg); 31 | } 32 | 33 | html[data-theme="dark"] 34 | progress::-webkit-progress-bar { 35 | background-color: var(--darkened-bg); 36 | } 37 | -------------------------------------------------------------------------------- /import_export_extensions/static/import_export_extensions/js/admin/admin.js: -------------------------------------------------------------------------------- 1 | /* 2 | This script show/hide error detail info by clicking appropriate link: 3 | - imported data (show content for each column in row) 4 | - stacktrace if debug is True 5 | */ 6 | (function($) { 7 | $(document).ready(function() { 8 | const showRowData = $('.show-error-detail'), 9 | dataStateAttr = 'state', 10 | openedState = 'opened', 11 | closedState = 'closed'; 12 | 13 | showRowData.click(function(e) { 14 | const self = $(this); 15 | e.preventDefault(); 16 | 17 | const state = self.data('state'); 18 | 19 | if (state === openedState) { 20 | self.parents('li').find('div').hide(500); 21 | self.data(dataStateAttr, closedState); 22 | } else { 23 | self.parents('li').find('div').show(500); 24 | self.data(dataStateAttr, openedState); 25 | } 26 | }); 27 | }); 28 | })(django.jQuery); 29 | -------------------------------------------------------------------------------- /import_export_extensions/static/import_export_extensions/js/widgets/progress_bar.js: -------------------------------------------------------------------------------- 1 | /* 2 | Script is used to update progress bar value using get requests to url: 3 | 4 | /admin/import_export_extensions/importjob//progress/ 5 | 6 | This script is used in two places: 7 | 8 | - ``ProgressBarWidget`` that adds progress bar to ModelAdmin. 9 | - ``import_job_status_view`` and ``export_job_status_view`` that 10 | displays current job status 11 | 12 | */ 13 | 14 | // GET requests interval in milliseconds 15 | const requestInterval = 1000; 16 | 17 | // Job statuses that mean that job is completed 18 | const jobIsCompleted = [ 19 | 'Cancelled', 20 | 'Parsed', 21 | 'Imported', 22 | 'Input_Error', 23 | 'Parse_Error', 24 | 'Exported', 25 | 'Export_Error', 26 | ]; 27 | 28 | (function($) { 29 | $(document).ready(function() { 30 | // Retrieve URL for `data-url` attribute of `progress_bar` 31 | const progressBar = $('#progress-bar'); 32 | if (!progressBar) { 33 | return; 34 | } 35 | const url = progressBar.data('url'); 36 | 37 | if (!url) { 38 | return; 39 | } 40 | 41 | function updateProgressBarValue(url) { 42 | // Retrieve URL for `data-url` attribute of `progress-bar` 43 | $.get(url, function(data) { 44 | if ($.inArray(data.status, jobIsCompleted) !== -1) { 45 | // If job is completed -- reload the page 46 | location.reload(); 47 | } else { 48 | $('#current-job-status').html(data.status); 49 | // If job isn't completed -- update progress bar and current status 50 | if (data.percent !== undefined) { 51 | progressBar.val(data.percent); 52 | progressBar.attr("data-label", data.current + '/' + data.total + ' (' +data.percent + '%)'); 53 | } 54 | // Reset function timeout 55 | setTimeout(updateProgressBarValue, requestInterval, url); 56 | } 57 | }); 58 | } 59 | updateProgressBarValue(url); 60 | }); 61 | }(django.jQuery)); 62 | -------------------------------------------------------------------------------- /import_export_extensions/tasks.py: -------------------------------------------------------------------------------- 1 | from celery import shared_task 2 | 3 | from . import models 4 | 5 | 6 | @shared_task() 7 | def parse_data_task(job_id: int): 8 | """Async task for starting data parsing.""" 9 | models.ImportJob.objects.get(pk=job_id).parse_data() 10 | 11 | 12 | @shared_task() 13 | def import_data_task(job_id: int): 14 | """Async task for starting data import.""" 15 | models.ImportJob.objects.get(pk=job_id).import_data() 16 | 17 | 18 | @shared_task() 19 | def export_data_task(job_id: int): 20 | """Async task for starting data export.""" 21 | job = models.ExportJob.objects.get(id=job_id) 22 | job.export_data() 23 | -------------------------------------------------------------------------------- /import_export_extensions/templates/admin/import_export_extensions/celery_export_results.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/import_export/base.html" %} 2 | {% load i18n %} 3 | {% load static admin_urls %} 4 | 5 | {% comment %} 6 | Template for showing export results. 7 | 8 | * Display status and allow downloading file 9 | * If data contains errors show traceback 10 | {% endcomment %} 11 | 12 | {% block extrahead %} 13 | {{ block.super }} 14 | 15 | 16 | 17 | {% endblock %} 18 | 19 | {% block breadcrumbs_last %} 20 | 21 | {% trans "Export" %} 22 | 23 | › {% trans "Export results" %} 24 | {% endblock %} 25 | 26 | {% block content %} 27 | {% block export_results %} 28 | {% if export_job.export_status == 'EXPORT_ERROR' %} 29 |

30 | {% trans "There are some errors during data exporting:" %} 31 |

32 |
33 | {{ export_job.error_message }} 34 | {% if debug %} 35 |
{{ export_job.traceback|linebreaks }}
36 | {% endif %} 37 |
38 | {% endif %} 39 | {% if export_job.export_status == 'EXPORTED' %} 40 |

41 | {% trans "You can download exported data with following link" %} 42 |

43 | {% trans "Download export data" %} 44 | {% endif %} 45 | {% endblock %} 46 | {% endblock %} 47 | -------------------------------------------------------------------------------- /import_export_extensions/templates/admin/import_export_extensions/celery_export_status.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/import_export/base.html" %} 2 | {% load i18n %} 3 | {% load static admin_urls %} 4 | 5 | {% comment %} 6 | Template to show status of export job. 7 | {% endcomment %} 8 | 9 | {% block extrastyle %} 10 | {{ block.super }} 11 | 12 | 13 | {% endblock %} 14 | 15 | {% block extrahead %} 16 | {{ block.super }} 17 | 18 | 19 | 20 | 21 | {% endblock %} 22 | 23 | 24 | {% block breadcrumbs_last %} 25 | 26 | {% trans "Export" %} 27 | 28 | › 29 | {% trans "Exporting ..." %} 30 | {% endblock %} 31 | 32 | {% block content %} 33 |
34 |
35 |
36 | 37 |

{{ export_job.export_status|title }}

38 |
39 | 40 | {% if export_job.export_status not in export_job.export_finished_statuses %} 41 |
42 | 43 | {% include "admin/import_export_extensions/progress_bar.html" with job_url=export_job_url %} 44 |
45 | {% endif %} 46 |
47 |
48 | {% endblock %} 49 | -------------------------------------------------------------------------------- /import_export_extensions/templates/admin/import_export_extensions/celery_import_status.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/import_export/base.html" %} 2 | {% load i18n %} 3 | {% load static admin_urls %} 4 | 5 | {% comment %} 6 | Template to show status of import job. 7 | {% endcomment %} 8 | 9 | {% block extrastyle %} 10 | {{ block.super }} 11 | 12 | 13 | {% endblock %} 14 | 15 | {% block extrahead %} 16 | {{ block.super }} 17 | 18 | 19 | 20 | 21 | {% endblock %} 22 | 23 | 24 | {% block breadcrumbs_last %} 25 | 26 | {% trans "Import" %} 27 | 28 | › {% trans "Importing..." %} 29 | {% endblock %} 30 | 31 | {% block content %} 32 |
33 | 34 |
35 |
36 |

{{ import_job.import_status|title }}

37 |
38 | 39 | {% if import_job.import_status not in import_job.results_statuses %} 40 |
41 | 42 | {% include "admin/import_export_extensions/progress_bar.html" with job_url=import_job_url %} 43 |
44 | {% endif %} 45 |
46 |
47 | {% endblock %} 48 | -------------------------------------------------------------------------------- /import_export_extensions/templates/admin/import_export_extensions/import_job_results.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | {% load static %} 3 | 4 | {% comment %} 5 | 6 | This template adds html block with import results to ImportJob admin page. 7 | If errors - shows lines, where errors accured. 8 | If no errors - shows imported data. 9 | 10 | {% endcomment %} 11 | 12 | 13 | 14 | 15 | 16 | {% if result.has_errors or result.has_validation_errors %} 17 | Importing data contains errors 18 |
19 | {% for error in result.base_errors %} 20 |
  • 21 | {{ error.error }} 22 | {% if debug %} 23 |
    {{ error.traceback|linebreaks }}
    24 | {% endif %} 25 |
    • 26 | {% endfor %} 27 | {% for line, errors in result.row_errors %} 28 | {% for error in errors %} 29 |
  • 30 | Row number: {{ line }} - {{ error.error }} 31 |
    32 | Show row data {% if debug %}and debug info{% endif %} 33 |
    {{ error.row.values|join:", " }}
    34 | {% if debug %} 35 | 36 | {% endif %} 37 |
    • 38 | {% endfor %} 39 | {% endfor %} 40 | {% for invalid_row in result.invalid_rows %} 41 | {% trans "Row number" %}: {{ invalid_row.number }} 42 | {% for field, errors in invalid_row.error_dict.items %} 43 |
    44 |  {{ field }}: 45 |
    46 | {% for error in errors %} 47 |  {{ error }} 48 |
    49 | {% endfor %} 50 | {% endfor %} 51 | {% endfor %} 52 | {% else %} 53 | 54 | 55 | 56 | 57 | {% for field in result.diff_headers %} 58 | 59 | {% endfor %} 60 | 61 | 62 | {% for row in result.rows %} 63 | {% if not row.errors %} 64 | 65 | 76 | {% for field in row.diff %} 77 | 80 | {% endfor %} 81 | 82 | {% endif %} 83 | {% endfor %} 84 |
    {{ field }}
    66 | {% if row.import_type == 'new' %} 67 | New 68 | {% elif row.import_type == 'skip' %} 69 | Skipped 70 | {% elif row.import_type == 'delete' %} 71 | Delete 72 | {% elif row.import_type == 'update' %} 73 | Update 74 | {% endif %} 75 | 78 | {{ field }} 79 |
    85 | {% endif %} 86 | -------------------------------------------------------------------------------- /import_export_extensions/templates/admin/import_export_extensions/progress_bar.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | 3 | 4 | -------------------------------------------------------------------------------- /invocations/__init__.py: -------------------------------------------------------------------------------- 1 | from . import ci, docs, project 2 | -------------------------------------------------------------------------------- /invocations/ci.py: -------------------------------------------------------------------------------- 1 | import saritasa_invocations 2 | from invoke import task 3 | 4 | 5 | @task 6 | def prepare(context): 7 | """Prepare ci environment for check.""" 8 | saritasa_invocations.print_success("Preparing CI") 9 | saritasa_invocations.docker.up(context) 10 | saritasa_invocations.github_actions.set_up_hosts(context) 11 | saritasa_invocations.poetry.install(context) 12 | -------------------------------------------------------------------------------- /invocations/docs.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | import invoke 4 | import saritasa_invocations 5 | 6 | LOCAL_DOCS_DIR = pathlib.Path("docs/_build") 7 | 8 | 9 | @invoke.task 10 | def build(context: invoke.Context): 11 | """Build documentation.""" 12 | saritasa_invocations.print_success("Start building of local documentation") 13 | context.run( 14 | f"sphinx-build -E -a docs {LOCAL_DOCS_DIR} --exception-on-warning", 15 | ) 16 | saritasa_invocations.print_success("Building completed") 17 | -------------------------------------------------------------------------------- /invocations/project.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import shutil 3 | 4 | import invoke 5 | import saritasa_invocations 6 | 7 | 8 | @invoke.task 9 | def init(context: invoke.Context, clean: bool = False): 10 | """Prepare env for working with project.""" 11 | saritasa_invocations.print_success("Setting up git config") 12 | saritasa_invocations.git.setup(context) 13 | saritasa_invocations.system.copy_vscode_settings(context) 14 | saritasa_invocations.print_success("Initial assembly of all dependencies") 15 | saritasa_invocations.poetry.install(context) 16 | if clean: 17 | saritasa_invocations.docker.clear(context) 18 | clear(context) 19 | saritasa_invocations.django.migrate(context) 20 | saritasa_invocations.pytest.run(context) 21 | saritasa_invocations.django.createsuperuser(context) 22 | 23 | 24 | @invoke.task 25 | def clear(context: invoke.Context): 26 | """Clear package directory from cache files.""" 27 | saritasa_invocations.print_success("Start clearing") 28 | build_dirs = ("build", "dist", ".eggs") 29 | coverage_dirs = ("htmlcov",) 30 | cache_dirs = (".mypy_cache", ".pytest_cache") 31 | 32 | saritasa_invocations.print_success("Remove cache directories") 33 | for directory in build_dirs + coverage_dirs + cache_dirs: 34 | shutil.rmtree(directory, ignore_errors=True) 35 | 36 | cwd = pathlib.Path() 37 | # remove egg paths 38 | saritasa_invocations.print_success("Remove egg directories") 39 | for path in cwd.glob("*.egg-info"): 40 | shutil.rmtree(path, ignore_errors=True) 41 | 42 | for path in cwd.glob("*.egg"): 43 | path.unlink(missing_ok=True) 44 | 45 | # remove last coverage file 46 | saritasa_invocations.print_success("Remove coverage file") 47 | pathlib.Path(".coverage").unlink(missing_ok=True) 48 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | import saritasa_invocations 2 | from invoke import Collection 3 | 4 | import invocations 5 | 6 | ns = Collection( 7 | invocations.project, 8 | invocations.docs, 9 | invocations.ci, 10 | saritasa_invocations.celery, 11 | saritasa_invocations.django, 12 | saritasa_invocations.docker, 13 | saritasa_invocations.pytest, 14 | saritasa_invocations.poetry, 15 | saritasa_invocations.git, 16 | saritasa_invocations.pre_commit, 17 | saritasa_invocations.mypy, 18 | saritasa_invocations.python, 19 | ) 20 | 21 | # Configurations for run command 22 | ns.configure( 23 | { 24 | "run": { 25 | "pty": True, 26 | "echo": True, 27 | }, 28 | "saritasa_invocations": saritasa_invocations.Config( 29 | project_name="django-import-export-extensions", 30 | celery=saritasa_invocations.CelerySettings( 31 | app="test_project.celery_app:app", 32 | ), 33 | django=saritasa_invocations.DjangoSettings( 34 | manage_file_path="test_project/manage.py", 35 | settings_path="test_project.settings", 36 | apps_path="test_project", 37 | ), 38 | github_actions=saritasa_invocations.GitHubActionsSettings( 39 | hosts=("postgres", "redis"), 40 | ), 41 | ), 42 | }, 43 | ) 44 | -------------------------------------------------------------------------------- /test_project/__init__.py: -------------------------------------------------------------------------------- 1 | from .celery_app import app as celery_app 2 | -------------------------------------------------------------------------------- /test_project/celery_app.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from django.conf import settings 4 | 5 | from celery import Celery 6 | 7 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_project.settings") 8 | 9 | app = Celery( 10 | "import_export_extensions", 11 | backend=settings.CELERY_BACKEND, 12 | broker=settings.CELERY_BROKER, 13 | ) 14 | 15 | app.config_from_object("django.conf:settings", namespace="CELERY") 16 | 17 | app.autodiscover_tasks() 18 | -------------------------------------------------------------------------------- /test_project/conftest.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | import pytest 4 | 5 | 6 | def pytest_configure() -> None: 7 | """Set up Django settings for tests. 8 | 9 | `pytest` automatically calls this function once when tests are run. 10 | 11 | """ 12 | settings.TESTING = True 13 | 14 | 15 | @pytest.fixture(scope="session", autouse=True) 16 | def django_db_setup(django_db_setup): 17 | """Set up test db for testing.""" 18 | 19 | 20 | @pytest.fixture(autouse=True) 21 | def enable_db_access_for_all_tests(django_db_setup, db): 22 | """Allow all tests to access DB.""" 23 | 24 | 25 | @pytest.fixture(scope="session", autouse=True) 26 | def _temp_directory_for_media(tmpdir_factory): 27 | """Fixture that set temp directory for all media files. 28 | 29 | This fixture changes DEFAULT_FILE_STORAGE or STORAGES variable 30 | to filesystem and provides temp dir for media. 31 | PyTest cleans up this temp dir by itself after few test runs 32 | 33 | """ 34 | if hasattr(settings, "STORAGES"): 35 | settings.STORAGES["default"]["BACKEND"] = ( 36 | "django.core.files.storage.FileSystemStorage" 37 | ) 38 | else: 39 | settings.DEFAULT_FILE_STORAGE = ( 40 | "django.core.files.storage.FileSystemStorage" 41 | ) 42 | media = tmpdir_factory.mktemp("tmp_media") 43 | settings.MEDIA_ROOT = media 44 | -------------------------------------------------------------------------------- /test_project/fake_app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/test_project/fake_app/__init__.py -------------------------------------------------------------------------------- /test_project/fake_app/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from import_export_extensions.admin import CeleryImportExportMixin 4 | 5 | from .models import Artist, Band, Instrument, Membership 6 | from .resources import ArtistResourceWithM2M, SimpleArtistResource 7 | 8 | 9 | @admin.register(Artist) 10 | class ArtistAdmin(CeleryImportExportMixin, admin.ModelAdmin): 11 | """Simple Artist admin model for tests.""" 12 | 13 | list_display = ( 14 | "name", 15 | "instrument", 16 | ) 17 | list_filter = ( 18 | "instrument__title", 19 | ) 20 | search_fields = ( 21 | "instrument__title", 22 | "name", 23 | # `@name`, `^name`, `=name` are about one model field but thanks for 24 | # first symbol they apply difference lookups in search. 25 | # These ones are used to check that type of search filter 26 | # must be only one for a model field in admin class. 27 | # They are not required, it's just for testing 28 | # https://github.com/django/django/blob/d6925f0d6beb3c08ae24bdb8fd83ddb13d1756e4/django/contrib/admin/options.py#L1138 29 | "@name", 30 | "^name", 31 | "=name", 32 | ) 33 | resource_classes = [ArtistResourceWithM2M, SimpleArtistResource] 34 | 35 | 36 | @admin.register(Instrument) 37 | class InstrumentAdmin(admin.ModelAdmin): 38 | """Simple Instrument admin model for tests.""" 39 | 40 | list_display = ("title",) 41 | 42 | 43 | @admin.register(Band) 44 | class BandAdmin(admin.ModelAdmin): 45 | """Band admin model for tests.""" 46 | 47 | list_display = ( 48 | "id", 49 | "title", 50 | ) 51 | search_fields = ( 52 | "id", 53 | "title", 54 | ) 55 | 56 | 57 | @admin.register(Membership) 58 | class MembershipAdmin(admin.ModelAdmin): 59 | """Membership admin model for tests.""" 60 | 61 | list_display = ( 62 | "id", 63 | "date_joined", 64 | ) 65 | -------------------------------------------------------------------------------- /test_project/fake_app/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/test_project/fake_app/api/__init__.py -------------------------------------------------------------------------------- /test_project/fake_app/api/views.py: -------------------------------------------------------------------------------- 1 | from rest_framework import mixins, serializers, viewsets 2 | 3 | from import_export_extensions import api 4 | 5 | from .. import models, resources 6 | 7 | 8 | class ArtistExportViewSet(api.ExportJobForUserViewSet): 9 | """Simple ViewSet for exporting Artist model.""" 10 | 11 | resource_class = resources.SimpleArtistResource 12 | export_ordering_fields = ( 13 | "id", 14 | "name", 15 | ) 16 | 17 | class ArtistImportViewSet(api.ImportJobForUserViewSet): 18 | """Simple ViewSet for importing Artist model.""" 19 | 20 | resource_class = resources.SimpleArtistResource 21 | 22 | class ArtistSerializer(serializers.ModelSerializer): 23 | """Serializer for Artist model.""" 24 | 25 | class Meta: 26 | model = models.Artist 27 | fields = ( 28 | "id", 29 | "name", 30 | "instrument", 31 | ) 32 | 33 | class ArtistViewSet( 34 | api.ExportStartActionMixin, 35 | api.ImportStartActionMixin, 36 | mixins.ListModelMixin, 37 | mixins.RetrieveModelMixin, 38 | viewsets.GenericViewSet, 39 | ): 40 | """Simple viewset for Artist model.""" 41 | 42 | resource_class = resources.SimpleArtistResource 43 | queryset = models.Artist.objects.all() 44 | serializer_class = ArtistSerializer 45 | filterset_class = resources.SimpleArtistResource.filterset_class 46 | ordering = ( 47 | "id", 48 | ) 49 | ordering_fields = ( 50 | "id", 51 | "name", 52 | ) 53 | -------------------------------------------------------------------------------- /test_project/fake_app/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class IOExtensionsAppConfig(AppConfig): 5 | """Fake app config.""" 6 | 7 | name = "test_project.fake_app" 8 | verbose_name = "Import Export Fake App" 9 | 10 | def ready(self): 11 | # Import to connect signals. 12 | from . import signals # noqa: F401 13 | -------------------------------------------------------------------------------- /test_project/fake_app/factories.py: -------------------------------------------------------------------------------- 1 | from django.core.files import base as django_files 2 | 3 | import factory 4 | from import_export.formats import base_formats as formats 5 | 6 | from import_export_extensions.models import ExportJob, ImportJob 7 | 8 | from . import models 9 | from .resources import SimpleArtistResource 10 | 11 | 12 | class InstrumentFactory(factory.django.DjangoModelFactory[models.Instrument]): 13 | """Simple factory for ``Instrument`` model.""" 14 | 15 | title = factory.Faker("name") 16 | 17 | class Meta: 18 | model = models.Instrument 19 | 20 | 21 | class ArtistFactory(factory.django.DjangoModelFactory[models.Artist]): 22 | """Simple factory for ``Artist`` model.""" 23 | 24 | name = factory.Faker("name") 25 | external_id = factory.Faker("uuid4") 26 | instrument = factory.SubFactory(InstrumentFactory) 27 | 28 | class Meta: 29 | model = models.Artist 30 | 31 | 32 | class BandFactory(factory.django.DjangoModelFactory[models.Band]): 33 | """Simple factory for ``Band`` model.""" 34 | 35 | title = factory.Faker("company") 36 | 37 | class Meta: 38 | model = models.Band 39 | 40 | 41 | class MembershipFactory(factory.django.DjangoModelFactory[models.Membership]): 42 | """Simple factory for ``Membership`` model.""" 43 | 44 | artist = factory.SubFactory(ArtistFactory) 45 | band = factory.SubFactory(BandFactory) 46 | date_joined = factory.Faker("date") 47 | 48 | class Meta: 49 | model = models.Membership 50 | 51 | 52 | class ArtistImportJobFactory(factory.django.DjangoModelFactory[ImportJob]): 53 | """Factory for creating ImportJob for Artist. 54 | 55 | Usage: 56 | ArtistImportJobFactory(artists=[artist1, artist2]) 57 | """ 58 | 59 | resource_path = "test_project.fake_app.resources.SimpleArtistResource" 60 | resource_kwargs: dict[str, str] = {} 61 | 62 | class Meta: 63 | model = ImportJob 64 | 65 | class Params: 66 | artists: list[models.Artist] = [] 67 | is_valid_file: bool = True 68 | 69 | @factory.lazy_attribute 70 | def data_file(self): 71 | """Generate `data_file` based on passed `artists`.""" 72 | resource = SimpleArtistResource() 73 | 74 | if not self.is_valid_file: 75 | # Append not existing artist with a non-existent instrument 76 | # to violate the not-null constraint 77 | self.artists.append( 78 | ArtistFactory.build(instrument=InstrumentFactory.build()), 79 | ) 80 | 81 | dataset = resource.export(self.artists) 82 | export_data = formats.CSV().export_data(dataset) 83 | content = django_files.ContentFile(export_data) 84 | return django_files.File(content.file, "data.csv") 85 | 86 | 87 | class ArtistExportJobFactory(factory.django.DjangoModelFactory[ExportJob]): 88 | """Factory for creating ExportJob for Artist.""" 89 | 90 | resource_path = "test_project.fake_app.resources.SimpleArtistResource" 91 | resource_kwargs: dict[str, str] = {} 92 | file_format_path = "import_export.formats.base_formats.CSV" 93 | 94 | class Meta: 95 | model = ExportJob 96 | -------------------------------------------------------------------------------- /test_project/fake_app/filters.py: -------------------------------------------------------------------------------- 1 | from django_filters import rest_framework as filters 2 | 3 | from .models import Artist 4 | 5 | 6 | class ArtistFilterSet(filters.FilterSet): 7 | """FilterSet for Artist resource.""" 8 | 9 | class Meta: 10 | model = Artist 11 | fields = { 12 | "id": ( 13 | "exact", 14 | "in", 15 | ), 16 | "name": ( 17 | "exact", 18 | "in", 19 | ), 20 | } 21 | -------------------------------------------------------------------------------- /test_project/fake_app/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.1.3 on 2024-12-05 07:52 2 | 3 | import django.db.models.deletion 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | initial = True 9 | 10 | dependencies: list[str] = [] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name="Band", 15 | fields=[ 16 | ( 17 | "id", 18 | models.BigAutoField( 19 | auto_created=True, 20 | primary_key=True, 21 | serialize=False, 22 | verbose_name="ID", 23 | ), 24 | ), 25 | ("title", models.CharField(max_length=100)), 26 | ], 27 | options={ 28 | "verbose_name": "Band", 29 | "verbose_name_plural": "Bands", 30 | }, 31 | ), 32 | migrations.CreateModel( 33 | name="Instrument", 34 | fields=[ 35 | ( 36 | "id", 37 | models.BigAutoField( 38 | auto_created=True, 39 | primary_key=True, 40 | serialize=False, 41 | verbose_name="ID", 42 | ), 43 | ), 44 | ("title", models.CharField(max_length=100)), 45 | ], 46 | options={ 47 | "verbose_name": "Instrument", 48 | "verbose_name_plural": "Instruments", 49 | }, 50 | ), 51 | migrations.CreateModel( 52 | name="Artist", 53 | fields=[ 54 | ( 55 | "id", 56 | models.BigAutoField( 57 | auto_created=True, 58 | primary_key=True, 59 | serialize=False, 60 | verbose_name="ID", 61 | ), 62 | ), 63 | ("name", models.CharField(max_length=100, unique=True)), 64 | ( 65 | "external_id", 66 | models.CharField( 67 | editable=False, 68 | help_text="External ID for sync import objects.", 69 | null=True, 70 | unique=True, 71 | verbose_name="External ID", 72 | ), 73 | ), 74 | ( 75 | "instrument", 76 | models.ForeignKey( 77 | on_delete=django.db.models.deletion.CASCADE, 78 | to="fake_app.instrument", 79 | ), 80 | ), 81 | ], 82 | options={ 83 | "verbose_name": "Artist", 84 | "verbose_name_plural": "Artists", 85 | }, 86 | ), 87 | migrations.CreateModel( 88 | name="Membership", 89 | fields=[ 90 | ( 91 | "id", 92 | models.BigAutoField( 93 | auto_created=True, 94 | primary_key=True, 95 | serialize=False, 96 | verbose_name="ID", 97 | ), 98 | ), 99 | ("date_joined", models.DateField()), 100 | ( 101 | "artist", 102 | models.ForeignKey( 103 | on_delete=django.db.models.deletion.CASCADE, 104 | to="fake_app.artist", 105 | ), 106 | ), 107 | ( 108 | "band", 109 | models.ForeignKey( 110 | on_delete=django.db.models.deletion.CASCADE, 111 | to="fake_app.band", 112 | ), 113 | ), 114 | ], 115 | options={ 116 | "verbose_name": "Membership", 117 | "verbose_name_plural": "Memberships", 118 | }, 119 | ), 120 | migrations.AddField( 121 | model_name="artist", 122 | name="bands", 123 | field=models.ManyToManyField( 124 | related_name="artists", 125 | through="fake_app.Membership", 126 | to="fake_app.band", 127 | ), 128 | ), 129 | ] 130 | -------------------------------------------------------------------------------- /test_project/fake_app/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/test_project/fake_app/migrations/__init__.py -------------------------------------------------------------------------------- /test_project/fake_app/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.utils.translation import gettext_lazy as _ 3 | 4 | 5 | class Instrument(models.Model): 6 | """Model representing Instrument.""" 7 | 8 | title = models.CharField(max_length=100) 9 | 10 | class Meta: 11 | verbose_name = _("Instrument") 12 | verbose_name_plural = _("Instruments") 13 | 14 | def __str__(self) -> str: 15 | """Return string representation.""" 16 | return self.title 17 | 18 | 19 | class Artist(models.Model): 20 | """Model representing artist.""" 21 | 22 | name = models.CharField(max_length=100, unique=True) 23 | bands = models.ManyToManyField( 24 | "Band", 25 | through="Membership", 26 | related_name="artists", 27 | ) 28 | 29 | instrument = models.ForeignKey( 30 | Instrument, 31 | on_delete=models.CASCADE, 32 | ) 33 | 34 | external_id = models.CharField( # noqa DJ01 35 | verbose_name=_("External ID"), 36 | help_text=_("External ID for sync import objects."), 37 | null=True, 38 | unique=True, 39 | editable=False, 40 | ) 41 | 42 | class Meta: 43 | verbose_name = _("Artist") 44 | verbose_name_plural = _("Artists") 45 | 46 | def __str__(self) -> str: 47 | """Return string representation.""" 48 | return self.name 49 | 50 | 51 | class Band(models.Model): 52 | """Model representing band.""" 53 | 54 | title = models.CharField(max_length=100) 55 | 56 | class Meta: 57 | verbose_name = _("Band") 58 | verbose_name_plural = _("Bands") 59 | 60 | def __str__(self) -> str: 61 | """Return string representation.""" 62 | return self.title 63 | 64 | 65 | class Membership(models.Model): 66 | """Model representing membership.""" 67 | 68 | artist = models.ForeignKey( 69 | Artist, 70 | on_delete=models.CASCADE, 71 | ) 72 | band = models.ForeignKey( 73 | Band, 74 | on_delete=models.CASCADE, 75 | ) 76 | date_joined = models.DateField() 77 | 78 | class Meta: 79 | verbose_name = _("Membership") 80 | verbose_name_plural = _("Memberships") 81 | 82 | def __str__(self) -> str: 83 | """Return string representation.""" 84 | return f"<{self.artist}> joined <{self.band}> on <{self.date_joined}>" 85 | -------------------------------------------------------------------------------- /test_project/fake_app/resources.py: -------------------------------------------------------------------------------- 1 | from import_export_extensions.fields import IntermediateManyToManyField 2 | from import_export_extensions.resources import CeleryModelResource 3 | from import_export_extensions.widgets import IntermediateManyToManyWidget 4 | 5 | from .filters import ArtistFilterSet 6 | from .models import Artist, Band 7 | 8 | 9 | class SimpleArtistResource(CeleryModelResource): 10 | """Artist resource with simple fields.""" 11 | 12 | filterset_class = ArtistFilterSet 13 | 14 | class Meta: 15 | model = Artist 16 | import_id_fields = ["external_id"] 17 | clean_model_instances = True 18 | fields = [ 19 | "id", 20 | "external_id", 21 | "name", 22 | "instrument", 23 | ] 24 | 25 | 26 | class ArtistResourceWithM2M(CeleryModelResource): 27 | """Artist resource with Many2Many field.""" 28 | 29 | bands = IntermediateManyToManyField( 30 | attribute="bands", 31 | column_name="Bands he played in", 32 | widget=IntermediateManyToManyWidget( 33 | rem_model=Band, 34 | rem_field="title", 35 | extra_fields=["date_joined"], 36 | instance_separator=";", 37 | ), 38 | ) 39 | 40 | class Meta: 41 | model = Artist 42 | clean_model_instances = True 43 | fields = ["id", "name", "bands", "instrument"] 44 | 45 | def get_queryset(self): 46 | """Return a queryset.""" 47 | return ( 48 | super() 49 | .get_queryset() 50 | .prefetch_related( 51 | "membership_set__band", 52 | "bands", 53 | ) 54 | ) 55 | 56 | 57 | class BandResourceWithM2M(CeleryModelResource): 58 | """Band resource with Many2Many field.""" 59 | 60 | artists = IntermediateManyToManyField( 61 | attribute="artists", 62 | column_name="Artists in band", 63 | widget=IntermediateManyToManyWidget( 64 | rem_model=Artist, 65 | rem_field="name", 66 | extra_fields=["date_joined"], 67 | instance_separator=";", 68 | ), 69 | ) 70 | 71 | class Meta: 72 | model = Band 73 | clean_model_instances = True 74 | fields = ["id", "title", "artists"] 75 | 76 | def get_queryset(self): 77 | """Return a queryset.""" 78 | return ( 79 | super() 80 | .get_queryset() 81 | .prefetch_related( 82 | "membership_set__artist", 83 | "artists", 84 | ) 85 | ) 86 | -------------------------------------------------------------------------------- /test_project/fake_app/signals.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django import dispatch 4 | 5 | from import_export_extensions.models.core import BaseJob 6 | from import_export_extensions.signals import ( 7 | export_job_failed, 8 | import_job_failed, 9 | ) 10 | 11 | 12 | @dispatch.receiver(export_job_failed) 13 | @dispatch.receiver(import_job_failed) 14 | def job_error_hook( 15 | sender, 16 | instance: BaseJob, 17 | error_message: str, 18 | traceback: str, 19 | exception: Exception | None, 20 | **kwargs, 21 | ): 22 | """Present an example of job error hook.""" 23 | logging.getLogger(__file__).warning(f"{instance}, {error_message}") 24 | -------------------------------------------------------------------------------- /test_project/manage.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python3 2 | import os 3 | import pathlib 4 | import sys 5 | 6 | sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) 7 | 8 | if __name__ == "__main__": 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") 10 | 11 | from django.core.management import execute_from_command_line 12 | 13 | execute_from_command_line(sys.argv) 14 | -------------------------------------------------------------------------------- /test_project/settings.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | import decouple 4 | 5 | # Build paths inside the project like this: BASE_DIR / "subdir" 6 | BASE_DIR = pathlib.Path(__file__).resolve().parent 7 | 8 | SECRET_KEY = "a87082n4v52u4rnvk2edv128eudfvn5" # noqa: S105 9 | 10 | ALLOWED_HOSTS = ["*"] 11 | 12 | DEBUG = True 13 | TESTING = False 14 | 15 | # Application definition 16 | 17 | INSTALLED_APPS = [ 18 | "django.contrib.admin", 19 | "django.contrib.auth", 20 | "django.contrib.contenttypes", 21 | "django.contrib.sessions", 22 | "django.contrib.messages", 23 | "django.contrib.staticfiles", 24 | "rest_framework", 25 | "drf_spectacular", 26 | "django_probes", 27 | "django_extensions", 28 | "import_export", 29 | "import_export_extensions", 30 | "test_project.fake_app", 31 | ] 32 | 33 | MIDDLEWARE = [ 34 | "django.middleware.security.SecurityMiddleware", 35 | "django.contrib.sessions.middleware.SessionMiddleware", 36 | "django.middleware.common.CommonMiddleware", 37 | "django.middleware.csrf.CsrfViewMiddleware", 38 | "django.contrib.auth.middleware.AuthenticationMiddleware", 39 | "django.contrib.messages.middleware.MessageMiddleware", 40 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 41 | ] 42 | 43 | ROOT_URLCONF = "test_project.urls" 44 | 45 | TEMPLATES = [ 46 | { 47 | "BACKEND": "django.template.backends.django.DjangoTemplates", 48 | "DIRS": [ 49 | BASE_DIR / "templates", 50 | ], 51 | "APP_DIRS": True, 52 | "OPTIONS": { 53 | "context_processors": [ 54 | "django.template.context_processors.debug", 55 | "django.template.context_processors.request", 56 | "django.contrib.auth.context_processors.auth", 57 | "django.contrib.messages.context_processors.messages", 58 | ], 59 | }, 60 | }, 61 | ] 62 | 63 | WSGI_APPLICATION = "test_project.wsgi.application" 64 | 65 | # Database 66 | # https://docs.djangoproject.com/en/dev/ref/settings/#std-setting-DATABASES 67 | 68 | DATABASES = { 69 | "default": { 70 | "ENGINE": "django.db.backends.postgresql", 71 | "ATOMIC_REQUESTS": True, 72 | "CONN_MAX_AGE": 600, 73 | "USER": decouple.config( 74 | "DB_USER", 75 | default="django-import-export-extensions-user", 76 | ), 77 | "NAME": decouple.config( 78 | "DB_NAME", 79 | default="django-import-export-extensions-dev", 80 | ), 81 | "PASSWORD": decouple.config("DB_PASSWORD", default="testpass"), 82 | "HOST": decouple.config("DB_HOST", default="postgres"), 83 | "PORT": decouple.config("DB_PORT", default=5432, cast=int), 84 | }, 85 | } 86 | 87 | AUTH_USER_MODEL = "auth.User" 88 | 89 | # Internationalization 90 | # https://docs.djangoproject.com/en/dev/topics/i18n/ 91 | 92 | LANGUAGE_CODE = "en-us" 93 | TIME_ZONE = "UTC" 94 | USE_I18N = True 95 | USE_L10N = True 96 | USE_TZ = True 97 | 98 | # Static files (CSS, JavaScript, Images) 99 | # https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/ 100 | 101 | STATIC_URL = "/static/" 102 | MEDIA_URL = "/media/" 103 | 104 | MEDIA_ROOT = BASE_DIR / "media" 105 | STATIC_ROOT = BASE_DIR / "static" 106 | 107 | # https://docs.djangoproject.com/en/dev/ref/settings/#storages 108 | STORAGES = { 109 | "default": { 110 | "BACKEND": "django.core.files.storage.FileSystemStorage", 111 | }, 112 | "staticfiles": { 113 | "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", 114 | }, 115 | "django_import_export_extensions": { 116 | "BACKEND": "django.core.files.storage.FileSystemStorage", 117 | }, 118 | } 119 | 120 | STATICFILES_FINDERS = ( 121 | "django.contrib.staticfiles.finders.FileSystemFinder", 122 | "django.contrib.staticfiles.finders.AppDirectoriesFinder", 123 | ) 124 | 125 | # Configure `drf-spectacular` to check it works for import-export API 126 | REST_FRAMEWORK = { 127 | "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", 128 | "COMPONENT_SPLIT_REQUEST": True, # Allows to upload import file from Swagger UI 129 | } 130 | 131 | # Celery settings 132 | 133 | redis_host = decouple.config("REDIS_HOST", default="redis") 134 | redis_port = decouple.config("REDIS_PORT", default=6379, cast=int) 135 | redis_db = decouple.config("REDIS_DB", default=1, cast=int) 136 | 137 | CELERY_TASK_ALWAYS_EAGER = True 138 | CELERY_TASK_STORE_EAGER_RESULT = True 139 | 140 | CELERY_TASK_SERIALIZER = "pickle" 141 | CELERY_ACCEPT_CONTENT = ["pickle", "json"] 142 | 143 | CELERY_TASK_ROUTES = {} 144 | CELERY_BROKER = f"redis://{redis_host}:{redis_port}/{redis_db}" 145 | CELERY_BACKEND = f"redis://{redis_host}:{redis_port}/{redis_db}" 146 | CELERY_TASK_DEFAULT_QUEUE = "development" 147 | 148 | # https://docs.djangoproject.com/en/dev/ref/settings/#std-setting-DEFAULT_AUTO_FIELD 149 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 150 | 151 | if DEBUG: 152 | INSTALLED_APPS += ("debug_toolbar",) 153 | MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",) 154 | 155 | def _show_toolbar_callback(request) -> bool: 156 | """Show debug toolbar exclude testing.""" 157 | from django.conf import settings 158 | 159 | return not settings.TESTING 160 | 161 | DEBUG_TOOLBAR_CONFIG = { 162 | "SHOW_TOOLBAR_CALLBACK": _show_toolbar_callback, 163 | } 164 | -------------------------------------------------------------------------------- /test_project/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/test_project/tests/__init__.py -------------------------------------------------------------------------------- /test_project/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth import get_user_model 2 | from django.contrib.auth.models import User 3 | from django.core.files.uploadedfile import SimpleUploadedFile 4 | 5 | from rest_framework import test 6 | 7 | import pytest 8 | 9 | from import_export_extensions.models import ExportJob, ImportJob 10 | 11 | from ..fake_app import factories 12 | from ..fake_app.factories import ArtistImportJobFactory 13 | from ..fake_app.models import Artist, Band, Membership 14 | 15 | 16 | @pytest.fixture 17 | def existing_artist(): 18 | """Return existing in db `Artist` instance.""" 19 | return factories.ArtistFactory() 20 | 21 | 22 | @pytest.fixture 23 | def new_artist(): 24 | """Return not existing `Artist` instance.""" 25 | return factories.ArtistFactory.build( 26 | instrument=factories.InstrumentFactory(), 27 | ) 28 | 29 | 30 | @pytest.fixture 31 | def artist_import_job( 32 | superuser: User, 33 | existing_artist: Artist, 34 | ) -> ImportJob: 35 | """Return `ImportJob` instance with specified artist.""" 36 | return factories.ArtistImportJobFactory.create( 37 | created_by=superuser, 38 | artists=[existing_artist], 39 | ) 40 | 41 | 42 | @pytest.fixture 43 | def artist_export_job( 44 | superuser: User, 45 | ) -> ExportJob: 46 | """Return `ExportJob` instance.""" 47 | return factories.ArtistExportJobFactory.create(created_by=superuser) 48 | 49 | 50 | @pytest.fixture 51 | def band() -> Band: 52 | """Return `Band` instance.""" 53 | return factories.BandFactory.create(title="Aerosmith") 54 | 55 | 56 | @pytest.fixture 57 | def membership(band: Band) -> Membership: 58 | """Return `Membership` instance with specified band.""" 59 | return factories.MembershipFactory.create(band=band) 60 | 61 | 62 | @pytest.fixture 63 | def uploaded_file(existing_artist: Artist) -> SimpleUploadedFile: 64 | """Generate valid `Artist` import file.""" 65 | import_job = ArtistImportJobFactory.build(artists=[existing_artist]) 66 | return SimpleUploadedFile( 67 | "test_file.csv", 68 | content=import_job.data_file.file.read().encode(), 69 | content_type="text/plain", 70 | ) 71 | 72 | 73 | @pytest.fixture 74 | def force_import_artist_job( 75 | superuser: User, 76 | new_artist: Artist, 77 | ) -> ImportJob: 78 | """`ImportJob` with `force_import=True` and file with invalid row.""" 79 | return ArtistImportJobFactory.create( 80 | artists=[new_artist], 81 | is_valid_file=False, 82 | force_import=True, 83 | created_by=superuser, 84 | ) 85 | 86 | 87 | @pytest.fixture 88 | def user(): 89 | """Return user instance.""" 90 | return get_user_model().objects.create( 91 | username="test_login", 92 | email="test@localhost.com", 93 | password="test_pass", 94 | is_staff=False, 95 | is_superuser=False, 96 | ) 97 | 98 | 99 | @pytest.fixture 100 | def superuser(): 101 | """Return superuser instance.""" 102 | return get_user_model().objects.create( 103 | username="admin_login", 104 | email="admin@localhost.com", 105 | password="admin_pass", 106 | is_staff=True, 107 | is_superuser=True, 108 | ) 109 | 110 | 111 | @pytest.fixture 112 | def api_client() -> test.APIClient: 113 | """Create api client.""" 114 | return test.APIClient() 115 | 116 | 117 | @pytest.fixture 118 | def admin_api_client( 119 | superuser: User, 120 | api_client: test.APIClient, 121 | ) -> test.APIClient: 122 | """Authenticate admin_user and return api client.""" 123 | api_client.force_authenticate(user=superuser) 124 | return api_client 125 | -------------------------------------------------------------------------------- /test_project/tests/integration_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/test_project/tests/integration_tests/__init__.py -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_admin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/test_project/tests/integration_tests/test_admin/__init__.py -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_admin/test_changelist_view.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.test.client import Client 3 | from django.urls import reverse 4 | 5 | from rest_framework import status 6 | 7 | 8 | def test_changelist_view_permission_context( 9 | client: Client, 10 | superuser: User, 11 | ): 12 | """Make sure changelist view has property to show import/export buttons.""" 13 | client.force_login(superuser) 14 | 15 | response = client.get( 16 | path=reverse("admin:fake_app_artist_changelist"), 17 | ) 18 | assert response.status_code == status.HTTP_200_OK 19 | context = response.context 20 | assert context["has_export_permission"] 21 | assert context["has_import_permission"] 22 | -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_admin/test_export/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/test_project/tests/integration_tests/test_admin/test_export/__init__.py -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_admin/test_export/test_admin_actions.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.test.client import Client 3 | from django.urls import reverse 4 | 5 | from rest_framework import status 6 | 7 | import pytest 8 | import pytest_mock 9 | 10 | from import_export_extensions.models import ExportJob 11 | from test_project.fake_app.factories import ArtistExportJobFactory 12 | 13 | 14 | @pytest.mark.django_db(transaction=True) 15 | def test_cancel_export_admin_action( 16 | client: Client, 17 | superuser: User, 18 | mocker: pytest_mock.MockerFixture, 19 | ): 20 | """Test `cancel_export` via admin action.""" 21 | client.force_login(superuser) 22 | 23 | revoke_mock = mocker.patch("celery.current_app.control.revoke") 24 | export_data_mock = mocker.patch( 25 | "import_export_extensions.models.ExportJob.export_data", 26 | ) 27 | job = ArtistExportJobFactory.create() 28 | 29 | response = client.post( 30 | reverse("admin:import_export_extensions_exportjob_changelist"), 31 | data={ 32 | "action": "cancel_jobs", 33 | "_selected_action": [job.pk], 34 | }, 35 | follow=True, 36 | ) 37 | job.refresh_from_db() 38 | 39 | assert response.status_code == status.HTTP_200_OK 40 | assert job.export_status == ExportJob.ExportStatus.CANCELLED 41 | assert ( 42 | response.context["messages"]._loaded_data[0].message 43 | == f"Export of {job} canceled" 44 | ) 45 | export_data_mock.assert_called_once() 46 | revoke_mock.assert_called_once_with(job.export_task_id, terminate=True) 47 | 48 | 49 | @pytest.mark.django_db(transaction=True) 50 | def test_cancel_export_admin_action_with_incorrect_export_job_status( 51 | client: Client, 52 | superuser: User, 53 | mocker: pytest_mock.MockerFixture, 54 | ): 55 | """Test `cancel_export` via admin action with wrong export job status.""" 56 | client.force_login(superuser) 57 | 58 | revoke_mock = mocker.patch("celery.current_app.control.revoke") 59 | job = ArtistExportJobFactory.create() 60 | 61 | expected_error_message = f"ExportJob with id {job.pk} has incorrect status" 62 | 63 | response = client.post( 64 | reverse("admin:import_export_extensions_exportjob_changelist"), 65 | data={ 66 | "action": "cancel_jobs", 67 | "_selected_action": [job.pk], 68 | }, 69 | follow=True, 70 | ) 71 | job.refresh_from_db() 72 | 73 | assert response.status_code == status.HTTP_200_OK 74 | assert job.export_status == ExportJob.ExportStatus.EXPORTED 75 | assert ( 76 | expected_error_message 77 | in response.context["messages"]._loaded_data[0].message 78 | ) 79 | revoke_mock.assert_not_called() 80 | -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_admin/test_export/test_admin_class.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.test.client import Client 3 | from django.urls import reverse 4 | 5 | import pytest 6 | import pytest_mock 7 | 8 | from import_export_extensions.models import ExportJob 9 | from test_project.fake_app.factories import ArtistExportJobFactory 10 | 11 | 12 | @pytest.mark.parametrize( 13 | argnames=["job_status", "expected_fieldsets"], 14 | argvalues=[ 15 | pytest.param( 16 | ExportJob.ExportStatus.CREATED, 17 | ( 18 | ( 19 | "export_status", 20 | "_model", 21 | "created", 22 | "export_started", 23 | "export_finished", 24 | ), 25 | ), 26 | id="Get fieldsets for job in status CREATED", 27 | ), 28 | pytest.param( 29 | ExportJob.ExportStatus.EXPORTED, 30 | ( 31 | ( 32 | "export_status", 33 | "_model", 34 | "created", 35 | "export_started", 36 | "export_finished", 37 | ), 38 | ("data_file",), 39 | ), 40 | id="Get fieldsets for job in status EXPORTED", 41 | ), 42 | pytest.param( 43 | ExportJob.ExportStatus.EXPORTING, 44 | ( 45 | ( 46 | "export_status", 47 | "export_progressbar", 48 | ), 49 | ), 50 | id="Get fieldsets for job in status EXPORTING", 51 | ), 52 | pytest.param( 53 | ExportJob.ExportStatus.EXPORT_ERROR, 54 | ( 55 | ( 56 | "export_status", 57 | "_model", 58 | "created", 59 | "export_started", 60 | "export_finished", 61 | ), 62 | ( 63 | "error_message", 64 | "traceback", 65 | ), 66 | ), 67 | id="Get fieldsets for job in status EXPORT_ERROR", 68 | ), 69 | ], 70 | ) 71 | def test_get_fieldsets_by_export_job_status( 72 | client: Client, 73 | superuser: User, 74 | job_status: ExportJob.ExportStatus, 75 | expected_fieldsets: tuple[tuple[str]], 76 | mocker: pytest_mock.MockerFixture, 77 | ): 78 | """Test that appropriate fieldsets returned for different job statuses.""" 79 | client.force_login(superuser) 80 | 81 | mocker.patch( 82 | "import_export_extensions.models.ExportJob.export_data", 83 | ) 84 | job = ArtistExportJobFactory.create() 85 | job.export_status = job_status 86 | job.save() 87 | 88 | response = client.get( 89 | reverse( 90 | "admin:import_export_extensions_exportjob_change", 91 | kwargs={"object_id": job.pk}, 92 | ), 93 | ) 94 | 95 | fieldsets = response.context["adminform"].fieldsets 96 | fields = [fields["fields"] for _, fields in fieldsets] 97 | 98 | assert tuple(fields) == ( 99 | *expected_fieldsets, 100 | ( 101 | "resource_path", 102 | "resource_kwargs", 103 | "file_format_path", 104 | ), 105 | ) 106 | -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_admin/test_export/test_celery_endpoints.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.test.client import Client 3 | from django.urls import reverse 4 | 5 | from rest_framework import status 6 | 7 | import pytest 8 | import pytest_mock 9 | 10 | from import_export_extensions.models import ExportJob 11 | from test_project.fake_app.factories import ArtistExportJobFactory 12 | 13 | 14 | def test_celery_export_status_view_during_export( 15 | client: Client, 16 | superuser: User, 17 | mocker: pytest_mock.MockerFixture, 18 | ): 19 | """Test export status page when export in progress.""" 20 | client.force_login(superuser) 21 | 22 | mocker.patch("import_export_extensions.tasks.export_data_task.apply_async") 23 | artist_export_job = ArtistExportJobFactory() 24 | artist_export_job.export_status = ExportJob.ExportStatus.EXPORTING 25 | artist_export_job.save() 26 | 27 | response = client.get( 28 | path=reverse( 29 | "admin:fake_app_artist_export_job_status", 30 | kwargs={"job_id": artist_export_job.pk}, 31 | ), 32 | ) 33 | 34 | expected_export_job_url = reverse( 35 | "admin:export_job_progress", 36 | kwargs={"job_id": artist_export_job.id}, 37 | ) 38 | 39 | assert response.status_code == status.HTTP_200_OK 40 | assert response.context["export_job_url"] == expected_export_job_url 41 | 42 | 43 | @pytest.mark.parametrize( 44 | argnames="incorrect_job_status", 45 | argvalues=[ 46 | ExportJob.ExportStatus.CREATED, 47 | ExportJob.ExportStatus.EXPORTING, 48 | ExportJob.ExportStatus.CANCELLED, 49 | ], 50 | ) 51 | def test_celery_export_results_view_redirect_to_status_page( 52 | client: Client, 53 | superuser: User, 54 | incorrect_job_status: ExportJob.ExportStatus, 55 | mocker: pytest_mock.MockerFixture, 56 | ): 57 | """Test redirect to export status page when job in not results statuses.""" 58 | client.force_login(superuser) 59 | 60 | mocker.patch("import_export_extensions.tasks.export_data_task.apply_async") 61 | artist_export_job = ArtistExportJobFactory() 62 | artist_export_job.export_status = incorrect_job_status 63 | artist_export_job.save() 64 | 65 | response = client.get( 66 | path=reverse( 67 | "admin:fake_app_artist_export_job_results", 68 | kwargs={"job_id": artist_export_job.pk}, 69 | ), 70 | ) 71 | 72 | expected_redirect_url = reverse( 73 | "admin:fake_app_artist_export_job_status", 74 | kwargs={"job_id": artist_export_job.pk}, 75 | ) 76 | assert response.status_code == status.HTTP_302_FOUND 77 | assert response.url == expected_redirect_url 78 | -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_admin/test_import/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/test_project/tests/integration_tests/test_admin/test_import/__init__.py -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_admin/test_import/test_admin_class.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.test.client import Client 3 | from django.urls import reverse 4 | 5 | import pytest 6 | import pytest_mock 7 | 8 | from import_export_extensions.models import ImportJob 9 | from test_project.fake_app.factories import ArtistImportJobFactory 10 | 11 | 12 | @pytest.mark.parametrize( 13 | argnames=["job_status", "expected_fieldsets"], 14 | argvalues=[ 15 | pytest.param( 16 | ImportJob.ImportStatus.CREATED, 17 | tuple(), 18 | id="Get fieldsets for job in status CREATED", 19 | ), 20 | pytest.param( 21 | ImportJob.ImportStatus.IMPORTED, 22 | ( 23 | ("_show_results",), 24 | ( 25 | "input_errors_file", 26 | "_input_errors", 27 | ), 28 | ), 29 | id="Get fieldsets for job in status IMPORTED", 30 | ), 31 | pytest.param( 32 | ImportJob.ImportStatus.IMPORTING, 33 | ( 34 | ( 35 | "import_status", 36 | "import_progressbar", 37 | ), 38 | ), 39 | id="Get fieldsets for job in status IMPORTING", 40 | ), 41 | pytest.param( 42 | ImportJob.ImportStatus.IMPORT_ERROR, 43 | (("traceback",),), 44 | id="Get fieldsets for job in status IMPORT_ERROR", 45 | ), 46 | ], 47 | ) 48 | def test_get_fieldsets_by_import_job_status( 49 | client: Client, 50 | superuser: User, 51 | job_status: ImportJob.ImportStatus, 52 | expected_fieldsets: tuple[tuple[str]], 53 | mocker: pytest_mock.MockerFixture, 54 | ): 55 | """Test that appropriate fieldsets returned for different job statuses.""" 56 | client.force_login(superuser) 57 | 58 | mocker.patch( 59 | "import_export_extensions.models.ImportJob.import_data", 60 | ) 61 | artist_import_job = ArtistImportJobFactory() 62 | artist_import_job.import_status = job_status 63 | artist_import_job.save() 64 | 65 | response = client.get( 66 | reverse( 67 | "admin:import_export_extensions_importjob_change", 68 | kwargs={"object_id": artist_import_job.pk}, 69 | ), 70 | ) 71 | 72 | fieldsets = response.context["adminform"].fieldsets 73 | fields = [fields["fields"] for _, fields in fieldsets] 74 | 75 | assert tuple(fields) == ( 76 | ( 77 | "import_status", 78 | "_model", 79 | "created_by", 80 | "created", 81 | "parse_finished", 82 | "import_started", 83 | "import_finished", 84 | ), 85 | *expected_fieldsets, 86 | ( 87 | "data_file", 88 | "resource_path", 89 | "resource_kwargs", 90 | ), 91 | ) 92 | -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_admin/test_import/test_celery_endpoints.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.test.client import Client 3 | from django.urls import reverse 4 | 5 | from rest_framework import status 6 | 7 | import pytest 8 | import pytest_mock 9 | 10 | from import_export_extensions.models import ImportJob 11 | from test_project.fake_app.factories import ArtistImportJobFactory 12 | 13 | 14 | def test_celery_import_status_view_during_import( 15 | client: Client, 16 | superuser: User, 17 | mocker: pytest_mock.MockerFixture, 18 | ): 19 | """Test import status page when import in progress.""" 20 | client.force_login(superuser) 21 | 22 | mocker.patch("import_export_extensions.tasks.parse_data_task.apply_async") 23 | artist_import_job = ArtistImportJobFactory(skip_parse_step=True) 24 | artist_import_job.import_status = ImportJob.ImportStatus.IMPORTING 25 | artist_import_job.save() 26 | 27 | response = client.get( 28 | path=reverse( 29 | "admin:fake_app_artist_import_job_status", 30 | kwargs={"job_id": artist_import_job.pk}, 31 | ), 32 | ) 33 | 34 | expected_import_job_url = reverse( 35 | "admin:import_job_progress", 36 | kwargs={"job_id": artist_import_job.id}, 37 | ) 38 | 39 | assert response.status_code == status.HTTP_200_OK 40 | assert response.context["import_job_url"] == expected_import_job_url 41 | 42 | 43 | @pytest.mark.parametrize( 44 | argnames="incorrect_job_status", 45 | argvalues=[ 46 | ImportJob.ImportStatus.CREATED, 47 | ImportJob.ImportStatus.PARSING, 48 | ImportJob.ImportStatus.PARSE_ERROR, 49 | ImportJob.ImportStatus.CONFIRMED, 50 | ImportJob.ImportStatus.IMPORTING, 51 | ImportJob.ImportStatus.CANCELLED, 52 | ], 53 | ) 54 | def test_celery_import_results_view_redirect_to_status_page( 55 | client: Client, 56 | superuser: User, 57 | incorrect_job_status: ImportJob.ImportStatus, 58 | mocker: pytest_mock.MockerFixture, 59 | ): 60 | """Test redirect to import status page when job in not result status.""" 61 | client.force_login(superuser) 62 | 63 | mocker.patch("import_export_extensions.tasks.parse_data_task.apply_async") 64 | artist_import_job = ArtistImportJobFactory() 65 | artist_import_job.import_status = incorrect_job_status 66 | artist_import_job.save() 67 | 68 | response = client.get( 69 | path=reverse( 70 | "admin:fake_app_artist_import_job_results", 71 | kwargs={"job_id": artist_import_job.pk}, 72 | ), 73 | ) 74 | 75 | expected_redirect_url = reverse( 76 | "admin:fake_app_artist_import_job_status", 77 | kwargs={"job_id": artist_import_job.pk}, 78 | ) 79 | assert response.status_code == status.HTTP_302_FOUND 80 | assert response.url == expected_redirect_url 81 | 82 | 83 | @pytest.mark.parametrize( 84 | argnames="incorrect_job_status", 85 | argvalues=[ 86 | ImportJob.ImportStatus.INPUT_ERROR, 87 | ImportJob.ImportStatus.IMPORT_ERROR, 88 | ImportJob.ImportStatus.IMPORTED, 89 | ], 90 | ) 91 | def test_celery_import_results_confirm_forbidden( 92 | client: Client, 93 | superuser: User, 94 | incorrect_job_status: ImportJob.ImportStatus, 95 | mocker: pytest_mock.MockerFixture, 96 | ): 97 | """Check that confirm from result page forbidden for not PARSED jobs.""" 98 | client.force_login(superuser) 99 | 100 | mocker.patch("import_export_extensions.tasks.parse_data_task.apply_async") 101 | artist_import_job = ArtistImportJobFactory() 102 | artist_import_job.import_status = incorrect_job_status 103 | artist_import_job.save() 104 | 105 | response = client.post( 106 | path=reverse( 107 | "admin:fake_app_artist_import_job_results", 108 | kwargs={"job_id": artist_import_job.pk}, 109 | ), 110 | data={"confirm": "Confirm import"}, 111 | ) 112 | 113 | assert response.status_code == status.HTTP_403_FORBIDDEN 114 | -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/test_project/tests/integration_tests/test_api/__init__.py -------------------------------------------------------------------------------- /test_project/tests/integration_tests/test_resources.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from import_export_extensions.resources import CeleryModelResource 4 | 5 | from ...fake_app.factories import MembershipFactory 6 | from ...fake_app.models import Artist, Membership 7 | from ...fake_app.resources import ArtistResourceWithM2M 8 | 9 | 10 | @pytest.fixture 11 | def three_bands_artist(existing_artist: Artist) -> Artist: 12 | """Return artist with three memberships.""" 13 | MembershipFactory.create_batch(3, artist=existing_artist) 14 | return existing_artist 15 | 16 | 17 | @pytest.fixture 18 | def artist_resource_with_m2m() -> ArtistResourceWithM2M: 19 | """Return Artist resource with Many2Many field instance.""" 20 | return ArtistResourceWithM2M() 21 | 22 | 23 | def _import_export_artist( 24 | resource: CeleryModelResource, 25 | artist: Artist, 26 | ) -> Artist: 27 | """Return restored Artist.""" 28 | dataset = resource.export(queryset=Artist.objects.filter(id=artist.pk)) 29 | 30 | # delete info about this artist 31 | artist.delete() 32 | 33 | # get new instance of resource 34 | resource = resource.__class__() 35 | 36 | # restore info from dataset 37 | result = resource.import_data(dataset) 38 | return Artist.objects.get(id=result.rows[0].object_id) 39 | 40 | 41 | def test_correct_restoring( 42 | artist_resource_with_m2m: CeleryModelResource, 43 | three_bands_artist: Artist, 44 | ): 45 | """Test that restoring works correct. 46 | 47 | Take a look at ``fake_app.models``. So we want to export 48 | ``Artist`` with info about his bands. And we want to store info about 49 | when artist joined some band in the same flat dataset. 50 | 51 | Test case is following: 52 | create artist's bio (with joined bands) 53 | create few other artists 54 | remove info about original artist 55 | restore his info from exported dataset 56 | 57 | """ 58 | # retrieve artist from DB 59 | restored_artist = _import_export_artist( 60 | resource=artist_resource_with_m2m, 61 | artist=three_bands_artist, 62 | ) 63 | restored_membership = Membership.objects.filter(artist=restored_artist) 64 | 65 | # check that restored artist contain same info 66 | assert restored_artist.name == three_bands_artist.name 67 | 68 | memberships = list(Membership.objects.all()) 69 | # check info about his bands 70 | expected_bands = {membership.band.id for membership in memberships} 71 | restored_bands = set(restored_artist.bands.values_list("id", flat=True)) 72 | 73 | assert expected_bands == restored_bands 74 | 75 | expected_join_dates = { 76 | membership.date_joined for membership in memberships 77 | } 78 | restored_join_dates = set( 79 | restored_membership.values_list("date_joined", flat=True), 80 | ) 81 | 82 | assert expected_join_dates == restored_join_dates 83 | -------------------------------------------------------------------------------- /test_project/tests/test_export_job.py: -------------------------------------------------------------------------------- 1 | from pytest_mock import MockerFixture 2 | 3 | from import_export_extensions.models import ExportJob 4 | 5 | from ..fake_app.factories import ArtistExportJobFactory 6 | 7 | 8 | def test_export_data_exported(artist_export_job: ExportJob): 9 | """Test that data correctly exported and data_file exists.""" 10 | artist_export_job.export_data() 11 | 12 | # ensure status updated 13 | assert artist_export_job.export_status == ExportJob.ExportStatus.EXPORTED 14 | 15 | # ensure file exists 16 | assert artist_export_job.data_file 17 | 18 | 19 | def test_export_data_error( 20 | artist_export_job: ExportJob, 21 | mocker: MockerFixture, 22 | ): 23 | """Test that exported with errors data has traceback and error_message.""" 24 | mocker.patch( 25 | target="import_export_extensions.models.ExportJob._export_data_inner", 26 | side_effect=ValueError("Unknown error"), 27 | ) 28 | 29 | artist_export_job.export_data() 30 | 31 | # ensure status updated 32 | assert ( 33 | artist_export_job.export_status == ExportJob.ExportStatus.EXPORT_ERROR 34 | ) 35 | 36 | # ensure traceback and message are collected 37 | assert artist_export_job.traceback 38 | assert artist_export_job.error_message 39 | 40 | 41 | def test_job_has_finished(artist_export_job: ExportJob): 42 | """Test that job `finished` field is set. 43 | 44 | Attribute `finished` is set then export job is completed 45 | (successfully or not). 46 | 47 | """ 48 | assert not artist_export_job.export_finished 49 | 50 | artist_export_job.export_data() 51 | 52 | assert artist_export_job.export_finished 53 | 54 | 55 | def test_export_filename_truncate(): 56 | """Test filename is truncated by ExportJob itself.""" 57 | job = ArtistExportJobFactory.build() 58 | 59 | # no error should be raised 60 | job.save() 61 | 62 | assert job.export_filename.endswith(".csv") 63 | -------------------------------------------------------------------------------- /test_project/tests/test_forms.py: -------------------------------------------------------------------------------- 1 | from import_export.formats import base_formats as formats 2 | 3 | from import_export_extensions import forms 4 | 5 | 6 | def test_export_form_set_xlsx_as_initial_choice(): 7 | """Ensure xlsx is initial choice for export form.""" 8 | available_formats = [ 9 | formats.CSV, 10 | formats.XLS, 11 | formats.XLSX, 12 | formats.JSON, 13 | ] 14 | export_form = forms.ExportForm(formats=available_formats) 15 | initial_choice = export_form.fields["file_format"].initial 16 | xlsx_choice_index = available_formats.index(formats.XLSX) 17 | 18 | assert initial_choice == xlsx_choice_index 19 | -------------------------------------------------------------------------------- /test_project/tests/test_import_job/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/saritasa-nest/django-import-export-extensions/60c8b13f1c5c3b5488b9dbc1df1bcf12a9277522/test_project/tests/test_import_job/__init__.py -------------------------------------------------------------------------------- /test_project/tests/test_import_job/test_cancel.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pytest_mock import MockerFixture 3 | 4 | from import_export_extensions.models import ImportJob 5 | 6 | from ...fake_app.factories import ArtistImportJobFactory 7 | from ...fake_app.models import Artist 8 | 9 | 10 | def test_cancel_import_error_status( 11 | existing_artist: Artist, 12 | new_artist: Artist, 13 | ): 14 | """Test `cancel_import` for import job with wrong status. 15 | 16 | There should be raised `ValueError` when status of BaseImportJob is 17 | incorrect. 18 | 19 | """ 20 | job = ArtistImportJobFactory.create( 21 | artists=[existing_artist, new_artist], 22 | ) 23 | job.import_status = ImportJob.ImportStatus.PARSED 24 | with pytest.raises( 25 | ValueError, 26 | match=f"ImportJob with id {job.id} has incorrect status", 27 | ): 28 | job.cancel_import() 29 | 30 | 31 | def test_cancel_import_success_status( 32 | existing_artist: Artist, 33 | new_artist: Artist, 34 | ): 35 | """Test `cancel_import` for import job with success status. 36 | 37 | No error should be raised and ImportJob `status` should be updated to 38 | cancelled. 39 | 40 | """ 41 | job = ArtistImportJobFactory.create( 42 | artists=[existing_artist, new_artist], 43 | ) 44 | job.cancel_import() 45 | assert job.import_status == ImportJob.ImportStatus.CANCELLED 46 | 47 | 48 | def test_cancel_import_with_celery_parse_task_id( 49 | existing_artist: Artist, 50 | new_artist: Artist, 51 | mocker: MockerFixture, 52 | ): 53 | """Test `cancel_import` for import job with celery. 54 | 55 | When celery is used -> there should be called celery task `revoke` with 56 | terminating and correct `task_id`. 57 | 58 | """ 59 | revoke = mocker.patch("celery.current_app.control.revoke") 60 | job = ArtistImportJobFactory.create( 61 | artists=[existing_artist, new_artist], 62 | ) 63 | job.cancel_import() 64 | 65 | assert job.import_status == ImportJob.ImportStatus.CANCELLED 66 | 67 | revoke.assert_called_with(job.parse_task_id, terminate=True) 68 | 69 | 70 | def test_cancel_import_with_celery_import_task_id( 71 | existing_artist: Artist, 72 | new_artist: Artist, 73 | mocker: MockerFixture, 74 | ): 75 | """Test `cancel_import` for import job with celery. 76 | 77 | When celery is used -> there should be called celery task `revoke` with 78 | terminating and correct `task_id`. 79 | 80 | """ 81 | revoke = mocker.patch("celery.current_app.control.revoke") 82 | job = ArtistImportJobFactory.create( 83 | artists=[existing_artist, new_artist], 84 | ) 85 | job.parse_data() 86 | job.confirm_import() 87 | job.cancel_import() 88 | assert job.import_status == ImportJob.ImportStatus.CANCELLED 89 | revoke.assert_called_with(job.import_task_id, terminate=True) 90 | 91 | 92 | @pytest.mark.django_db(transaction=True) 93 | def test_cancel_import_with_custom_task_id_on_parse( 94 | existing_artist: Artist, 95 | new_artist: Artist, 96 | mocker: MockerFixture, 97 | ): 98 | """Test `cancel_import` on parsing data with custom celery task_id. 99 | 100 | Check that when new ImportJob is created there would be called a 101 | `parse_data` method with auto generated `task_id`. 102 | 103 | """ 104 | parse_data = mocker.patch( 105 | "import_export_extensions.models.ImportJob.parse_data", 106 | ) 107 | job = ArtistImportJobFactory.create( 108 | artists=[existing_artist, new_artist], 109 | ) 110 | job.cancel_import() 111 | 112 | assert job.import_status == ImportJob.ImportStatus.CANCELLED 113 | 114 | parse_data.assert_called_once() 115 | 116 | 117 | @pytest.mark.django_db(transaction=True) 118 | def test_cancel_import_with_custom_task_id_on_import( 119 | existing_artist: Artist, 120 | new_artist: Artist, 121 | mocker: MockerFixture, 122 | ): 123 | """Test `cancel_import` on importing data with custom celery task_id. 124 | 125 | Check that when data for ImportJob is imported there would be called a 126 | `confirm_import` method with auto generated `task_id`. 127 | 128 | """ 129 | import_data = mocker.patch( 130 | "import_export_extensions.models.ImportJob.import_data", 131 | ) 132 | job = ArtistImportJobFactory.create( 133 | artists=[existing_artist, new_artist], 134 | ) 135 | job.parse_data() 136 | job.confirm_import() 137 | job.cancel_import() 138 | assert job.import_status == ImportJob.ImportStatus.CANCELLED 139 | 140 | import_data.assert_called_once() 141 | -------------------------------------------------------------------------------- /test_project/tests/test_import_job/test_import_data.py: -------------------------------------------------------------------------------- 1 | import django.test 2 | 3 | import pytest 4 | 5 | from import_export_extensions.models import ImportJob 6 | 7 | from ...fake_app.factories import ArtistFactory, ArtistImportJobFactory 8 | from ...fake_app.models import Artist 9 | 10 | 11 | @pytest.mark.django_db(transaction=True) 12 | def test_import_data_valid_data_file( 13 | existing_artist: Artist, 14 | new_artist: Artist, 15 | ): 16 | """Test `import_data` with valid importing data. 17 | 18 | Required logic: 19 | 20 | * run import and update instances in DB 21 | * save import result to ImportJob 22 | * update ImportJob status 23 | 24 | Case is following: 25 | 26 | * export one artist from DB and one new (not saved) artist 27 | * import data 28 | 29 | """ 30 | job = ArtistImportJobFactory.create( 31 | artists=[ 32 | existing_artist, 33 | new_artist, 34 | ], 35 | ) 36 | job.parse_data() 37 | job.confirm_import() 38 | job.refresh_from_db() 39 | 40 | assert job.import_status == ImportJob.ImportStatus.IMPORTED 41 | 42 | existing_artist_row_result = job.result.rows[0] 43 | new_artist_row_result = job.result.rows[1] 44 | 45 | # existing artist just updated 46 | assert not existing_artist_row_result.is_new() 47 | 48 | # new artist added 49 | assert new_artist_row_result.is_new() 50 | 51 | assert Artist.objects.count() == 2 52 | 53 | 54 | @pytest.mark.django_db(transaction=True) 55 | def test_job_has_finished(new_artist: Artist): 56 | """Tests that job `parse_finished` and `import_finished` is set. 57 | 58 | Attribute `parse_finished` is set then `parse` part of import job is 59 | completed (successfully or not). Same for `import_finished` attribute 60 | and `import` part of job. 61 | 62 | """ 63 | job = ArtistImportJobFactory.create(artists=[new_artist]) 64 | assert job.parse_finished is None 65 | assert job.import_finished is None 66 | 67 | job.parse_data() 68 | job.refresh_from_db() 69 | assert job.parse_finished 70 | 71 | job.confirm_import() 72 | job.refresh_from_db() 73 | assert job.import_finished 74 | 75 | 76 | def test_import_data_wrong_status(artist_import_job: ImportJob): 77 | """Test `import_data` while job in wrong status raises ValueError.""" 78 | artist_import_job.parse_data() 79 | artist_import_job.import_status = ImportJob.ImportStatus.PARSE_ERROR 80 | artist_import_job.save() 81 | 82 | with pytest.raises( 83 | ValueError, 84 | match=f"ImportJob with id {artist_import_job.id} has incorrect status", 85 | ): 86 | artist_import_job.import_data() 87 | 88 | 89 | @pytest.mark.parametrize( 90 | [ 91 | "skip_parse_step", 92 | "is_valid_file", 93 | "expected_status", 94 | "is_instance_created", 95 | ], 96 | [ 97 | [False, True, ImportJob.ImportStatus.PARSED, False], 98 | [False, False, ImportJob.ImportStatus.INPUT_ERROR, False], 99 | [True, True, ImportJob.ImportStatus.IMPORTED, True], 100 | [True, False, ImportJob.ImportStatus.IMPORT_ERROR, False], 101 | ], 102 | ) 103 | @pytest.mark.django_db(transaction=True) 104 | def test_import_data_skip_parse_step( 105 | new_artist: Artist, 106 | skip_parse_step: bool, 107 | is_valid_file: bool, 108 | expected_status: ImportJob.ImportStatus, 109 | is_instance_created: bool, 110 | ): 111 | """Test import job skip parse step logic. 112 | 113 | If `skip_parse_step=True`, 114 | then instance will import data if no errors detected. 115 | If `skip_parse_step=False`, then parse only. 116 | 117 | """ 118 | import_job: ImportJob = ArtistImportJobFactory.build( 119 | artists=[new_artist], 120 | force_import=False, 121 | skip_parse_step=skip_parse_step, 122 | is_valid_file=is_valid_file, 123 | ) 124 | import_job.save() 125 | import_job.refresh_from_db() 126 | 127 | assert import_job.import_status == expected_status 128 | assert ( 129 | Artist.objects.filter(name=new_artist.name).exists() 130 | == is_instance_created 131 | ) 132 | 133 | 134 | def test_force_import_create_correct_rows( 135 | new_artist: Artist, 136 | ): 137 | """Test import job with `force_import=True` create correct rows.""" 138 | import_job = ArtistImportJobFactory.create( 139 | artists=[new_artist], 140 | force_import=True, 141 | skip_parse_step=True, 142 | is_valid_file=False, 143 | ) 144 | import_job.import_data() 145 | import_job.refresh_from_db() 146 | assert import_job.import_status == import_job.ImportStatus.IMPORTED 147 | assert Artist.objects.filter(name=new_artist.name).exists() 148 | 149 | 150 | def test_import_data_with_validation_error(existing_artist: Artist): 151 | """Test import handles `ValidationError` instances correctly. 152 | 153 | If validation error occur, then job should end with `INPUT_ERROR` status. 154 | 155 | """ 156 | wrong_artist = ArtistFactory.build(name=existing_artist.name) 157 | job = ArtistImportJobFactory(artists=[wrong_artist]) 158 | job.parse_data() 159 | job.refresh_from_db() 160 | assert job.import_status == ImportJob.ImportStatus.INPUT_ERROR 161 | 162 | 163 | @django.test.override_settings( 164 | IMPORT_EXPORT_MAX_DATASET_ROWS=1, 165 | ) 166 | def test_import_create_with_max_rows( 167 | new_artist: Artist, 168 | ): 169 | """Test import job max dataset rows validation.""" 170 | import_job = ArtistImportJobFactory.create( 171 | artists=[new_artist], 172 | skip_parse_step=True, 173 | is_valid_file=False, 174 | ) 175 | 176 | import_job.import_data() 177 | import_job.refresh_from_db() 178 | assert import_job.import_status == import_job.ImportStatus.IMPORT_ERROR 179 | assert "Too many rows `2`" in import_job.error_message 180 | -------------------------------------------------------------------------------- /test_project/tests/test_resources.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.core.exceptions import ValidationError as DjangoValidationError 4 | 5 | from rest_framework.exceptions import ValidationError 6 | 7 | import pytest 8 | 9 | from import_export_extensions import results 10 | from test_project.fake_app.factories import ArtistFactory 11 | from test_project.fake_app.models import Artist 12 | 13 | from ..fake_app.resources import SimpleArtistResource 14 | 15 | 16 | def test_resource_get_queryset(existing_artist: Artist): 17 | """Check that `get_queryset` contains existing artist.""" 18 | assert existing_artist in SimpleArtistResource().get_queryset() 19 | 20 | 21 | def test_resource_with_filter_kwargs(existing_artist: Artist): 22 | """Check that `get_queryset` with filter kwargs exclude existing artist.""" 23 | expected_artist_name = "Expected Artist" 24 | expected_artist = ArtistFactory(name=expected_artist_name) 25 | resource_queryset = SimpleArtistResource( 26 | filter_kwargs={"name": expected_artist_name}, 27 | ).get_queryset() 28 | 29 | assert existing_artist not in resource_queryset 30 | assert expected_artist in resource_queryset 31 | 32 | 33 | def test_resource_with_invalid_filter_kwargs(): 34 | """Check that `get_queryset` raise error if filter kwargs is invalid.""" 35 | with pytest.raises( 36 | ValidationError, 37 | match=( 38 | r"{'id':.*ErrorDetail.*string='Enter a number.', code='invalid'.*" 39 | ), 40 | ): 41 | SimpleArtistResource( 42 | filter_kwargs={"id": "invalid_id"}, 43 | ).get_queryset() 44 | 45 | 46 | def test_resource_with_ordering(): 47 | """Check that `get_queryset` with ordering will apply correct ordering.""" 48 | artists = [ArtistFactory(name=str(num)) for num in range(5)] 49 | resource_queryset = SimpleArtistResource( 50 | ordering=("-name",), 51 | ).get_queryset() 52 | assert resource_queryset.last() == artists[0] 53 | assert resource_queryset.first() == artists[-1] 54 | 55 | 56 | def test_resource_with_invalid_ordering(): 57 | """Check that `get_queryset` raise error if ordering is invalid.""" 58 | with pytest.raises( 59 | DjangoValidationError, 60 | match=( 61 | re.escape( 62 | "{'ordering': [\"Cannot resolve keyword 'invalid_id' into field.\"]}", # noqa: E501 63 | ) 64 | ), 65 | ): 66 | SimpleArtistResource(ordering=("invalid_id",)).get_queryset() 67 | 68 | 69 | def test_resource_get_error_class(): 70 | """Ensure that CeleryResource overrides error class.""" 71 | error_class = SimpleArtistResource().get_error_result_class() 72 | assert error_class is results.Error 73 | -------------------------------------------------------------------------------- /test_project/tests/test_result.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | from django.core.exceptions import ValidationError 4 | from django.utils.translation import gettext_lazy as _ 5 | 6 | import pytest 7 | 8 | from import_export_extensions import results 9 | 10 | 11 | @pytest.fixture 12 | def row_result_with_skipped_errors() -> results.RowResult: 13 | """Create RowResult with skipped errors.""" 14 | row_result = results.RowResult() 15 | row_result.non_field_skipped_errors = [results.Error("Error")] 16 | row_result.field_skipped_errors = { 17 | "title": [ValidationError("Error")], 18 | } 19 | return row_result 20 | 21 | 22 | def test_reduce_error(): 23 | """Test simplify exception object for pickling.""" 24 | assert pickle.dumps(results.Error(_)) 25 | 26 | 27 | def test_result_skipped_properties( 28 | row_result_with_skipped_errors: results.RowResult, 29 | ): 30 | """Check that result properties calculate value correct.""" 31 | result = results.Result() 32 | result.rows = [row_result_with_skipped_errors] 33 | assert result.has_skipped_rows 34 | assert len(result.skipped_rows) == 1 35 | 36 | 37 | def test_row_result_properties( 38 | row_result_with_skipped_errors: results.RowResult, 39 | ): 40 | """Check that row result properties calculate value correct.""" 41 | assert row_result_with_skipped_errors.has_skipped_errors 42 | assert row_result_with_skipped_errors.skipped_errors_count == 2 43 | -------------------------------------------------------------------------------- /test_project/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import django.test 2 | from django.db.models import Q 3 | 4 | import pytest 5 | 6 | from import_export_extensions import utils 7 | 8 | AWS_STORAGE_BUCKET_NAME = "test-bucket-name" 9 | 10 | 11 | @pytest.mark.parametrize( 12 | argnames=["file_url", "expected_mime_type"], 13 | argvalues=[ 14 | pytest.param( 15 | "dir/subdir/file.invalid_extension", 16 | "application/octet-stream", 17 | id="File extension not in setting.MIME_TYPES_MAP", 18 | ), 19 | pytest.param( 20 | "dir/subdir/file.json", 21 | "application/json", 22 | id="File extension in settings.MIME_TYPES_MAP", 23 | ), 24 | pytest.param( 25 | "http://testdownload.org/file.csv?width=10", 26 | "text/csv", 27 | id="File url with GET params", 28 | ), 29 | ], 30 | ) 31 | def test_get_mime_type_by_file_url( 32 | file_url: str, 33 | expected_mime_type: str, 34 | ): 35 | """Check that correct mime type is returned.""" 36 | assert utils.get_mime_type_by_file_url(file_url) == expected_mime_type 37 | 38 | 39 | def test_clear_q_filter(): 40 | """Check that filter is cleaned correctly.""" 41 | value = "Hello world" 42 | attribute_name = "title" 43 | expected_q_filter = Q(title__iregex=r"^Hello\s+world$") 44 | 45 | assert utils.get_clear_q_filter(value, attribute_name) == expected_q_filter 46 | 47 | 48 | @django.test.override_settings( 49 | AWS_STORAGE_BUCKET_NAME=AWS_STORAGE_BUCKET_NAME, 50 | ) 51 | @pytest.mark.parametrize( 52 | argnames=["file_url", "expected_file_url"], 53 | argvalues=[ 54 | pytest.param( 55 | "http://localhost:8000/media/dir/file.csv", 56 | "dir/file.csv", 57 | id="File from media", 58 | ), 59 | pytest.param( 60 | f"http://s3.region.com/{AWS_STORAGE_BUCKET_NAME}/dir/file.csv", 61 | "dir/file.csv", 62 | id="File from s3 bucket", 63 | ), 64 | pytest.param( 65 | f"http://{AWS_STORAGE_BUCKET_NAME}.s3.region.com/dir/file.csv", 66 | "dir/file.csv", 67 | id=( 68 | # https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html 69 | "File from s3 bucket if using virtual addressing style." 70 | ), 71 | ), 72 | ], 73 | ) 74 | def test_url_to_internal_value( 75 | file_url: str, 76 | expected_file_url: str, 77 | ): 78 | """Check that file url is converted correctly.""" 79 | assert utils.url_to_internal_value(file_url) == expected_file_url 80 | -------------------------------------------------------------------------------- /test_project/tests/test_views.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from rest_framework import viewsets 4 | 5 | import pytest 6 | import pytest_mock 7 | 8 | from import_export_extensions.api.views import ( 9 | ExportJobViewSet, 10 | ImportJobViewSet, 11 | ) 12 | from test_project.fake_app.resources import SimpleArtistResource 13 | 14 | 15 | @pytest.mark.parametrize( 16 | argnames="viewset_class", 17 | argvalues=[ 18 | ExportJobViewSet, 19 | ImportJobViewSet, 20 | ], 21 | ) 22 | def test_new_viewset_class( 23 | viewset_class: type[viewsets.GenericViewSet], 24 | mocker: pytest_mock.MockerFixture, 25 | ): 26 | """Check that if drf_spectacular is not set it will not raise an error.""" 27 | mocker.patch.dict(sys.modules, {"drf_spectacular.utils": None}) 28 | 29 | class TestViewSet(viewset_class): 30 | resource_class = SimpleArtistResource 31 | 32 | assert TestViewSet is not None 33 | -------------------------------------------------------------------------------- /test_project/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.conf.urls.static import static 3 | from django.contrib import admin 4 | from django.urls import include, path, re_path 5 | 6 | from rest_framework.routers import DefaultRouter 7 | 8 | from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView 9 | 10 | from import_export_extensions import api 11 | 12 | from .fake_app.api import views 13 | 14 | ie_router = DefaultRouter() 15 | ie_router.register( 16 | "export-artist", 17 | views.ArtistExportViewSet, 18 | basename="export-artist", 19 | ) 20 | ie_router.register( 21 | "export-jobs", 22 | api.BaseExportJobForUserViewSet, 23 | basename="export-jobs", 24 | ) 25 | ie_router.register( 26 | "import-jobs", 27 | api.BaseImportJobForUserViewSet, 28 | basename="import-jobs", 29 | ) 30 | ie_router.register( 31 | "import-artist", 32 | views.ArtistImportViewSet, 33 | basename="import-artist", 34 | ) 35 | ie_router.register( 36 | "artists", 37 | views.ArtistViewSet, 38 | basename="artists", 39 | ) 40 | 41 | urlpatterns = [re_path("^admin/", admin.site.urls), *ie_router.urls] 42 | 43 | 44 | # for serving uploaded files on dev environment with django 45 | if settings.DEBUG: 46 | import debug_toolbar 47 | 48 | urlpatterns += [ 49 | path( 50 | "api/schema/", 51 | SpectacularAPIView.as_view(), 52 | name="schema", 53 | ), 54 | path( 55 | "api/schema/swagger-ui/", 56 | SpectacularSwaggerView.as_view(url_name="schema"), 57 | name="swagger-ui", 58 | ), 59 | path("__debug__/", include(debug_toolbar.urls)), 60 | ] 61 | 62 | urlpatterns += static( 63 | settings.MEDIA_URL, 64 | document_root=settings.MEDIA_ROOT, 65 | ) 66 | urlpatterns += static( 67 | settings.STATIC_URL, 68 | document_root=settings.STATIC_ROOT, 69 | ) 70 | -------------------------------------------------------------------------------- /test_project/wsgi.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from django.core.wsgi import get_wsgi_application 4 | 5 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_project.settings") 6 | 7 | application = get_wsgi_application() 8 | --------------------------------------------------------------------------------