├── tests ├── __init__.py ├── unit │ ├── __init__.py │ ├── models │ │ ├── __init__.py │ │ ├── test_importjob.py │ │ └── test_exportjob.py │ ├── test_apps.py │ ├── test_fields.py │ ├── test_command.py │ ├── test_utils.py │ ├── test_resources.py │ ├── test_admin_actions.py │ └── test_pubsub.py ├── resources │ ├── __init__.py │ ├── fake_app │ │ ├── __init__.py │ │ ├── apps.py │ │ ├── resources.py │ │ └── models.py │ └── custom-rabbitmq-conf │ │ ├── enabled_plugins │ │ └── rabbitmq.conf ├── integration │ ├── __init__.py │ ├── test_pubsub │ │ ├── __init__.py │ │ ├── test_export.py │ │ └── test_import.py │ └── test_views.py ├── utils.py └── urls.py ├── example ├── project │ ├── __init__.py │ └── settings.py ├── winners │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_admin.py │ │ ├── test_models.py │ │ └── test_utils.py │ ├── migrations │ │ ├── __init__.py │ │ └── 0001_initial.py │ ├── apps.py │ ├── admin.py │ ├── wsgi.py │ ├── models.py │ └── urls.py ├── example-data │ └── Winner-2019-06-27.csv └── manage.py ├── import_export_stomp ├── __init__.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── import_export_pubsub.py ├── migrations │ ├── __init__.py │ ├── 0005_exportjob_site_of_origin.py │ ├── 0002_auto_20190923_1132.py │ ├── 0004_exportjob_email_on_completion.py │ ├── 0007_auto_20210210_1831.py │ ├── 0008_alter_exportjob_id_alter_importjob_id.py │ ├── 0011_alter_exportjob_id_alter_importjob_id.py │ ├── 0009_alter_exportjob_options_alter_importjob_options_and_more.py │ ├── 0010_remove_exportjob_email_on_completion_and_more.py │ ├── 0006_auto_20191125_1236.py │ ├── 0003_exportjob.py │ └── 0001_initial.py ├── templatetags │ ├── __init__.py │ └── settings.py ├── widgets.py ├── locale │ ├── pt │ │ └── LC_MESSAGES │ │ │ ├── django.mo │ │ │ └── django.po │ └── eu │ │ └── LC_MESSAGES │ │ └── django.po ├── apps.py ├── urls.py ├── models │ ├── __init__.py │ ├── importjob.py │ └── exportjob.py ├── templates │ ├── email │ │ └── export_job_completion.html │ └── widgets │ │ └── signed_url_file.html ├── fields.py ├── resources.py ├── utils.py ├── pubsub.py ├── admin_actions.py ├── views.py ├── admin.py └── tasks.py ├── .github ├── CODEOWNERS ├── dependabot.yml └── pull_request_template.md ├── screenshots ├── admin.png ├── summary.png ├── import_jobs.png ├── new-winner.png ├── new_import_job.png ├── perform-import.png └── view-summary.png ├── CODE_OF_CONDUCT.md ├── scripts ├── start-formatter-lint.sh ├── start-example.sh └── start-tests.sh ├── .dockerignore ├── pytest.ini ├── MANIFEST.in ├── .editorconfig ├── .env ├── .coveragerc ├── azure-pipelines.yml ├── Dockerfile ├── pyproject.toml ├── .pre-commit-config.yaml ├── setup.cfg ├── sonar-project.properties ├── docker-compose.yml ├── .gitignore ├── settings.py ├── LICENSE └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/project/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/winners/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resources/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/winners/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /import_export_stomp/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/winners/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resources/fake_app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /import_export_stomp/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/integration/test_pubsub/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /import_export_stomp/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /import_export_stomp/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/example-data/Winner-2019-06-27.csv: -------------------------------------------------------------------------------- 1 | id,name 2 | 2,bar 3 | -------------------------------------------------------------------------------- /tests/resources/custom-rabbitmq-conf/enabled_plugins: -------------------------------------------------------------------------------- 1 | [rabbitmq_management,rabbitmq_stomp]. 2 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # .github directory and any of its subdirectories. 2 | /.github @juntossomosmais/core 3 | -------------------------------------------------------------------------------- /screenshots/admin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juntossomosmais/django-import-export-stomp/HEAD/screenshots/admin.png -------------------------------------------------------------------------------- /screenshots/summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juntossomosmais/django-import-export-stomp/HEAD/screenshots/summary.png -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | Please follow the spirit of [The PSF code of conduct](https://www.python.org/psf/conduct/) and try to be nice. 2 | -------------------------------------------------------------------------------- /screenshots/import_jobs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juntossomosmais/django-import-export-stomp/HEAD/screenshots/import_jobs.png -------------------------------------------------------------------------------- /screenshots/new-winner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juntossomosmais/django-import-export-stomp/HEAD/screenshots/new-winner.png -------------------------------------------------------------------------------- /screenshots/new_import_job.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juntossomosmais/django-import-export-stomp/HEAD/screenshots/new_import_job.png -------------------------------------------------------------------------------- /screenshots/perform-import.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juntossomosmais/django-import-export-stomp/HEAD/screenshots/perform-import.png -------------------------------------------------------------------------------- /screenshots/view-summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juntossomosmais/django-import-export-stomp/HEAD/screenshots/view-summary.png -------------------------------------------------------------------------------- /tests/resources/custom-rabbitmq-conf/rabbitmq.conf: -------------------------------------------------------------------------------- 1 | loopback_users.guest = false 2 | listeners.tcp.default = 5672 3 | management.tcp.port = 15672 4 | -------------------------------------------------------------------------------- /scripts/start-formatter-lint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | git config --global --add safe.directory /app 6 | 7 | pre-commit run --all-files 8 | -------------------------------------------------------------------------------- /import_export_stomp/widgets.py: -------------------------------------------------------------------------------- 1 | from django.forms import FileInput 2 | 3 | 4 | class SignedUrlFileInput(FileInput): 5 | template_name = "widgets/signed_url_file.html" 6 | -------------------------------------------------------------------------------- /import_export_stomp/locale/pt/LC_MESSAGES/django.mo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juntossomosmais/django-import-export-stomp/HEAD/import_export_stomp/locale/pt/LC_MESSAGES/django.mo -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # https://docs.docker.com/engine/reference/builder/#dockerignore-file 2 | * 3 | !example 4 | !import_export_stomp 5 | !scripts 6 | !manage.py 7 | !poetry* 8 | !pyproject.toml 9 | -------------------------------------------------------------------------------- /example/winners/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class WinnersConfig(AppConfig): 5 | default_auto_field = "django.db.models.BigAutoField" 6 | name = "winners" 7 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | env = 3 | PYTEST=True 4 | DJANGO_SETTINGS_MODULE=settings 5 | 6 | python_files = 7 | tests/integration/*.py 8 | tests/unit/*.py 9 | 10 | addopts = --no-migrations -vv 11 | -------------------------------------------------------------------------------- /tests/resources/fake_app/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class FakeAppConfig(AppConfig): 5 | default_auto_field = "django.db.models.BigAutoField" 6 | name = "tests.resources.fake_app" 7 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include import_export_stomp/static *.js *.css *.map *.png *.ico *.eot *.svg *.ttf *.woff *.woff2 2 | recursive-include import_export_stomp/templates *.html 3 | recursive-include import_export_stomp/locale *.mo 4 | -------------------------------------------------------------------------------- /import_export_stomp/templatetags/settings.py: -------------------------------------------------------------------------------- 1 | from django import template 2 | from django.conf import settings 3 | 4 | register = template.Library() 5 | 6 | 7 | @register.simple_tag 8 | def settings_value(name): 9 | return getattr(settings, name, "") 10 | -------------------------------------------------------------------------------- /import_export_stomp/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.utils.translation import gettext_lazy as _ 3 | 4 | 5 | class ImportExportStompConfig(AppConfig): 6 | default_auto_field = "django.db.models.BigAutoField" 7 | name = "import_export_stomp" 8 | verbose_name = _("Import Export Stomp") 9 | -------------------------------------------------------------------------------- /import_export_stomp/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from import_export_stomp.views import generate_presigned_post 4 | 5 | urlpatterns = [ 6 | path( 7 | "import-export-stomp/presigned-url/", 8 | generate_presigned_post, 9 | name="import_export_stomp_presigned_url", 10 | ), 11 | ] 12 | -------------------------------------------------------------------------------- /tests/resources/fake_app/resources.py: -------------------------------------------------------------------------------- 1 | from import_export import resources 2 | 3 | from tests.resources.fake_app.models import FakeModel 4 | 5 | 6 | class FakeResource(resources.ModelResource): 7 | class Meta: 8 | model = FakeModel 9 | fields = ("name", "value") 10 | import_id_fields = ("name",) 11 | -------------------------------------------------------------------------------- /import_export_stomp/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2019 o.s. Auto*Mat 2 | 3 | """Import all models.""" 4 | from typing import Sequence 5 | 6 | from import_export_stomp.models.exportjob import ExportJob 7 | from import_export_stomp.models.importjob import ImportJob 8 | 9 | __all__: Sequence[str] = ( 10 | "ExportJob", 11 | "ImportJob", 12 | ) 13 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | end_of_line = lf 9 | charset = utf-8 10 | trim_trailing_whitespace = true 11 | insert_final_newline = true 12 | 13 | [*.py] 14 | max_line_length = 150 15 | 16 | [*.{scss,js,html}] 17 | max_line_length = 150 18 | 19 | [*.rst] 20 | max_line_length = 150 21 | 22 | [*.yml] 23 | indent_size = 2 24 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | PYTHONPATH=. 2 | 3 | #################### 4 | #### Broker config 5 | STOMP_SERVER_HOST=rabbitmq 6 | STOMP_SERVER_PORT=61613 7 | STOMP_SERVER_USER= 8 | STOMP_SERVER_PASSWORD= 9 | STOMP_USE_SSL=False 10 | LISTENER_CLIENT_ID=import_export_stomp-listener 11 | 12 | #################### 13 | #### Django 14 | DJANGO_SUPERUSER_PASSWORD=admin 15 | DJANGO_SUPERUSER_USERNAME=admin 16 | DJANGO_SUPERUSER_EMAIL=admin@admin.com 17 | -------------------------------------------------------------------------------- /tests/unit/test_apps.py: -------------------------------------------------------------------------------- 1 | from import_export_stomp.apps import ImportExportStompConfig 2 | 3 | 4 | class TestApps: 5 | def test_import_export_stomp_app_config(self): 6 | app_name = "import_export_stomp" 7 | app_verbose_name = "Import Export Stomp" 8 | app_config = ImportExportStompConfig 9 | 10 | assert app_config.name == app_name 11 | assert app_config.verbose_name == app_verbose_name 12 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | .venv/* 4 | .tox/* 5 | .docs/* 6 | .htmlcov/* 7 | .junit/* 8 | scripts/* 9 | tests/* 10 | */tests/* 11 | *__init__.py 12 | **/migrations/*.py 13 | manage.py 14 | **/wsgi.py 15 | **/settings.py 16 | **/url.py 17 | **/seed.py 18 | **/db_routing.py 19 | import_export_stomp/admin.py 20 | example/**/* 21 | 22 | [report] 23 | show_missing = True 24 | -------------------------------------------------------------------------------- /example/winners/admin.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 o.s. Auto*Mat 2 | from django.contrib import admin 3 | from import_export.admin import ImportExportMixin 4 | 5 | from import_export_stomp.admin_actions import create_export_job_action 6 | 7 | from . import models 8 | 9 | 10 | @admin.register(models.Winner) 11 | class WinnerAdmin(ImportExportMixin, admin.ModelAdmin): 12 | list_display = ("name",) 13 | 14 | actions = (create_export_job_action,) 15 | -------------------------------------------------------------------------------- /scripts/start-example.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # https://www.gnu.org/software/bash/manual/bash.html#The-Set-Builtin 4 | # -e Exit immediately if a command exits with a non-zero status. 5 | # -x Print commands and their arguments as they are executed. 6 | set -e 7 | 8 | rm example/db.sqlite3 || true 9 | python example/manage.py migrate 10 | python example/manage.py createsuperuser --noinput 11 | python example/manage.py runserver 0.0.0.0:${DJANGO_BIND_PORT:-8080} 12 | -------------------------------------------------------------------------------- /example/winners/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for winners project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "winners.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | from typing import Tuple 3 | from unittest.mock import MagicMock 4 | 5 | from django_stomp.services.consumer import Payload 6 | 7 | 8 | def create_payload( 9 | body: Dict = None, headers: Dict = None 10 | ) -> Tuple[Payload, MagicMock, MagicMock]: 11 | ack = MagicMock() 12 | nack = MagicMock() 13 | return ( 14 | Payload(ack=ack, nack=nack, body=body or {}, headers=headers or {}), 15 | ack, 16 | nack, 17 | ) 18 | -------------------------------------------------------------------------------- /tests/resources/fake_app/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | from import_export_stomp.resources import resource_importer 4 | 5 | 6 | class FakeModel(models.Model): 7 | name = models.CharField(max_length=30) 8 | value = models.IntegerField(default=0) 9 | 10 | @classmethod 11 | def export_resource_classes(cls): 12 | return { 13 | "FakeResource": ( 14 | "FakeModel resource", 15 | resource_importer("tests.resources.fake_app.resources.FakeResource")(), 16 | ), 17 | } 18 | -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | trigger: 2 | branches: 3 | include: 4 | - main 5 | paths: 6 | include: 7 | - import_export_stomp 8 | - tests 9 | - Dockerfile 10 | - poetry.lock 11 | pr: 12 | - main 13 | - develop 14 | - release/* 15 | 16 | resources: 17 | repositories: 18 | - repository: templates 19 | type: github 20 | name: juntossomosmais/azure-pipelines-templates 21 | endpoint: github.com 22 | ref: main 23 | PoetryVersion: 1.4.2 24 | 25 | extends: 26 | template: python/library.yaml@templates 27 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "pip" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | time: "08:00" 8 | day: "sunday" 9 | reviewers: 10 | - "juntossomosmais/loyalty" 11 | - "juntossomosmais/loja-virtual" 12 | - "juntossomosmais/example" 13 | - package-ecosystem: "docker" 14 | directory: "/" 15 | schedule: 16 | interval: "monthly" 17 | reviewers: 18 | - "juntossomosmais/loyalty" 19 | - "juntossomosmais/loja-virtual" 20 | - "juntossomosmais/example" 21 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0005_exportjob_site_of_origin.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.4 on 2019-11-13 13:27 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("import_export_stomp", "0004_exportjob_email_on_completion"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="exportjob", 16 | name="site_of_origin", 17 | field=models.TextField(default="", max_length=255), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /import_export_stomp/templates/email/export_job_completion.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Document 8 | 9 | 10 | 11 |

Your export job on model {{app_label}}.{{model}} has completed. You can download the file at the following link:

12 | {{link}} 13 | 14 | 15 | -------------------------------------------------------------------------------- /import_export_stomp/fields.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.db import models 3 | 4 | from import_export_stomp.utils import get_storage_class 5 | 6 | 7 | class ImportExportFileField(models.FileField): 8 | def __init__(self, *args, **kwargs): 9 | # If the user has specified a custom storage backend, use it. 10 | if getattr(settings, "IMPORT_EXPORT_STOMP_STORAGE", None): 11 | storage_class = get_storage_class(settings.IMPORT_EXPORT_STOMP_STORAGE) 12 | kwargs["storage"] = storage_class() 13 | 14 | super().__init__(*args, **kwargs) 15 | -------------------------------------------------------------------------------- /example/winners/tests/test_admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib.messages.storage.fallback import FallbackStorage 2 | from django_admin_smoke_tests import tests 3 | 4 | 5 | class AdminSmokeTest(tests.AdminSiteSmokeTest): 6 | exclude_apps = [] 7 | fixtures = [] 8 | 9 | def post_request(self, post_data={}, params=None): # noqa 10 | request = self.factory.post("/", data=post_data) 11 | request.user = self.superuser 12 | request._dont_enforce_csrf_checks = True 13 | request.session = "session" 14 | request._messages = FallbackStorage(request) 15 | return request 16 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0002_auto_20190923_1132.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.5 on 2019-09-23 09:32 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("import_export_stomp", "0001_initial"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="importjob", 16 | name="job_status", 17 | field=models.CharField( 18 | blank=True, max_length=160, verbose_name="Status of the job" 19 | ), 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /example/winners/tests/test_models.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from django.core.files.base import ContentFile 4 | from django.test import TestCase 5 | from import_export_celery.models.importjob import ImportJob 6 | 7 | 8 | class ImportJobTestCases(TestCase): 9 | def test_delete_file_on_job_delete(self): 10 | job = ImportJob.objects.create( 11 | file=ContentFile(b"", "file.csv"), 12 | ) 13 | file_path = job.file.path 14 | assert os.path.exists(file_path) 15 | job.delete() 16 | assert not os.path.exists(file_path) 17 | assert not ImportJob.objects.filter(id=job.id).exists() 18 | -------------------------------------------------------------------------------- /tests/unit/test_fields.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.test import override_settings 3 | 4 | from import_export_stomp.fields import ImportExportFileField 5 | from import_export_stomp.utils import get_storage_class 6 | 7 | 8 | class TestFields: 9 | @override_settings( 10 | IMPORT_EXPORT_STOMP_STORAGE="django.core.files.storage.FileSystemStorage" 11 | ) 12 | def test_import_export_file_field_custom_storage(self): 13 | field = ImportExportFileField() 14 | 15 | storage_class = get_storage_class(settings.IMPORT_EXPORT_STOMP_STORAGE) 16 | 17 | assert isinstance(field.storage, storage_class) 18 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0004_exportjob_email_on_completion.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.4 on 2019-11-13 13:12 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("import_export_stomp", "0003_exportjob"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="exportjob", 16 | name="email_on_completion", 17 | field=models.BooleanField( 18 | default=True, 19 | verbose_name="Send me an email when this export job is complete", 20 | ), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /tests/unit/test_command.py: -------------------------------------------------------------------------------- 1 | import django_stomp.management.commands.pubsub 2 | 3 | from django.core.management import call_command 4 | from pytest_mock import MockerFixture 5 | 6 | from import_export_stomp.utils import IMPORT_EXPORT_STOMP_PROCESSING_QUEUE 7 | 8 | 9 | class TestCommand: 10 | def test_should_call_command_with_parameters(self, mocker: MockerFixture): 11 | mocked_start_processing = mocker.patch.object( 12 | django_stomp.management.commands.pubsub, "start_processing" 13 | ) 14 | call_command("import_export_pubsub") 15 | 16 | mocked_start_processing.assert_called_with( 17 | IMPORT_EXPORT_STOMP_PROCESSING_QUEUE, "import_export_stomp.pubsub.consumer" 18 | ) 19 | -------------------------------------------------------------------------------- /scripts/start-tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # https://www.gnu.org/software/bash/manual/bash.html#The-Set-Builtin 4 | # -e Exit immediately if a command exits with a non-zero status. 5 | # -x Print commands and their arguments as they are executed. 6 | set -e 7 | 8 | COVER_PROJECT_PATH=. 9 | TESTS_PROJECT_PATH=tests 10 | REPORTS_FOLDER_PATH=tests-reports 11 | 12 | # PYTHONPATH is needed if you use a plugin 13 | # Let's say you include addopts in pytest.ini with the following: -p tests.support.my_honest_plugin 14 | PYTHONPATH=. pytest $TESTS_PROJECT_PATH -n auto -vv --doctest-modules \ 15 | --cov=$COVER_PROJECT_PATH \ 16 | --junitxml=$REPORTS_FOLDER_PATH/junit.xml \ 17 | --cov-report=xml:$REPORTS_FOLDER_PATH/coverage.xml \ 18 | --cov-report=html:$REPORTS_FOLDER_PATH/html \ 19 | --cov-report=term 20 | -------------------------------------------------------------------------------- /example/winners/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.2 on 2019-06-28 11:47 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | initial = True 10 | 11 | dependencies = [] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name="Winner", 16 | fields=[ 17 | ( 18 | "id", 19 | models.AutoField( 20 | auto_created=True, 21 | primary_key=True, 22 | serialize=False, 23 | verbose_name="ID", 24 | ), 25 | ), 26 | ("name", models.CharField(default="", max_length=80)), 27 | ], 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /import_export_stomp/management/commands/import_export_pubsub.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | 3 | from django_stomp.management.commands.pubsub import Command as PubsubCommand 4 | 5 | from import_export_stomp.utils import IMPORT_EXPORT_STOMP_PROCESSING_QUEUE 6 | 7 | 8 | class Command(PubsubCommand): 9 | help = "Listens to queue to process messages" 10 | 11 | def add_arguments(self, parser: ArgumentParser) -> None: 12 | """ 13 | This method is empty to remove required django-stomp parameters 14 | """ 15 | 16 | def handle(self, *args, **options): 17 | super().handle( 18 | *args, 19 | **{ 20 | "source_destination": IMPORT_EXPORT_STOMP_PROCESSING_QUEUE, 21 | "callback_function": "import_export_stomp.pubsub.consumer", 22 | }, 23 | ) 24 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim 2 | 3 | # Git is required for pre-commit 4 | RUN apt update 5 | RUN apt install -y git 6 | 7 | # Creates application directory 8 | WORKDIR /app 9 | 10 | # Creates an appuser and change the ownership of the application's folder 11 | RUN useradd appuser && chown appuser ./ 12 | 13 | # Installs poetry and pip 14 | RUN pip install --upgrade pip && \ 15 | pip install poetry && \ 16 | poetry config virtualenvs.create false --local 17 | 18 | # Copy dependency definition to cache 19 | COPY --chown=appuser poetry.lock pyproject.toml ./ 20 | 21 | # Installs projects dependencies as a separate layer 22 | RUN poetry install --no-root 23 | 24 | # Temporary fix due to third party libraries having tests module 25 | RUN rm -rf ./venv/lib/python3.10/site-packages/tests 26 | 27 | # Copies and chowns for the userapp on a single layer 28 | COPY --chown=appuser . ./ 29 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0007_auto_20210210_1831.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.1.5 on 2021-02-10 18:31 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("import_export_stomp", "0006_auto_20191125_1236"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="exportjob", 16 | name="format", 17 | field=models.CharField( 18 | max_length=255, null=True, verbose_name="Format of file to be exported" 19 | ), 20 | ), 21 | migrations.AlterField( 22 | model_name="importjob", 23 | name="format", 24 | field=models.CharField( 25 | max_length=255, verbose_name="Format of file to be imported" 26 | ), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0008_alter_exportjob_id_alter_importjob_id.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.6 on 2023-05-15 16:34 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("import_export_stomp", "0007_auto_20210210_1831"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="exportjob", 16 | name="id", 17 | field=models.BigAutoField( 18 | auto_created=True, primary_key=True, serialize=False, verbose_name="ID" 19 | ), 20 | ), 21 | migrations.AlterField( 22 | model_name="importjob", 23 | name="id", 24 | field=models.BigAutoField( 25 | auto_created=True, primary_key=True, serialize=False, verbose_name="ID" 26 | ), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /example/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | sys.path.append("../") 6 | 7 | if __name__ == "__main__": 8 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.settings") 9 | try: 10 | from django.core.management import execute_from_command_line 11 | except ImportError: 12 | # The above import may fail for some other reason. Ensure that the 13 | # issue is really that Django is missing to avoid masking other 14 | # exceptions on Python 2. 15 | try: 16 | import django # noqa 17 | except ImportError: 18 | raise ImportError( 19 | "Couldn't import Django. Are you sure it's installed and " 20 | "available on your PYTHONPATH environment variable? Did you " 21 | "forget to activate a virtual environment?" 22 | ) 23 | raise 24 | execute_from_command_line(sys.argv) 25 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0011_alter_exportjob_id_alter_importjob_id.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.5 on 2023-10-18 18:24 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [ 9 | ("import_export_stomp", "0010_remove_exportjob_email_on_completion_and_more"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="exportjob", 15 | name="id", 16 | field=models.BigAutoField( 17 | auto_created=True, primary_key=True, serialize=False, verbose_name="ID" 18 | ), 19 | ), 20 | migrations.AlterField( 21 | model_name="importjob", 22 | name="id", 23 | field=models.BigAutoField( 24 | auto_created=True, primary_key=True, serialize=False, verbose_name="ID" 25 | ), 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /example/winners/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from import_export.fields import Field 3 | from import_export.resources import ModelResource 4 | 5 | 6 | class Winner(models.Model): 7 | name = models.CharField( 8 | max_length=80, 9 | null=False, 10 | blank=False, 11 | default="", 12 | ) 13 | 14 | @classmethod 15 | def export_resource_classes(cls): 16 | return { 17 | "winners": ("Winners resource", WinnersResource), 18 | "winners_all_caps": ( 19 | "Winners with all caps column resource", 20 | WinnersWithAllCapsResource, 21 | ), 22 | } 23 | 24 | 25 | class WinnersResource(ModelResource): 26 | class Meta: 27 | model = Winner 28 | 29 | def get_export_queryset(self): 30 | """To customise the queryset of the model resource with annotation override""" 31 | return self.Meta.model.objects.all() 32 | 33 | 34 | class WinnersWithAllCapsResource(WinnersResource): 35 | name_all_caps = Field() 36 | 37 | def dehydrate_name_all_caps(self, winner): 38 | return winner.name.upper() 39 | -------------------------------------------------------------------------------- /tests/urls.py: -------------------------------------------------------------------------------- 1 | """winners URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.10/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | 17 | from typing import Sequence 18 | from typing import Union 19 | 20 | from django.contrib import admin 21 | from django.urls import URLPattern 22 | from django.urls import URLResolver 23 | from django.urls import path 24 | 25 | from import_export_stomp.urls import urlpatterns as import_export_stomp_urlpatterns 26 | 27 | urlpatterns: Sequence[Union[URLResolver, URLPattern]] = [ 28 | path("admin/", admin.site.urls), 29 | ] + import_export_stomp_urlpatterns # type: ignore 30 | -------------------------------------------------------------------------------- /import_export_stomp/resources.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from importlib import import_module 4 | from typing import Callable 5 | from typing import Optional 6 | from typing import Type 7 | 8 | from django.apps import apps 9 | from import_export.resources import ModelResource 10 | from import_export.resources import Resource 11 | from import_export.resources import modelresource_factory 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | def resource_importer(resource: str) -> Callable: 17 | def main() -> Type[ModelResource]: 18 | module, obj = resource.rsplit(".", 1) 19 | imported_module = import_module(module) 20 | 21 | return getattr(imported_module, obj) 22 | 23 | return main 24 | 25 | 26 | class ModelConfig: 27 | resource: Resource 28 | 29 | def __init__( 30 | self, 31 | app_label: Optional[str] = None, 32 | model_name: Optional[str] = None, 33 | resource: Optional[Resource] = None, 34 | ): 35 | self.model = apps.get_model(app_label=app_label, model_name=model_name) 36 | logger.debug(resource) 37 | if resource: 38 | self.resource = resource() 39 | else: 40 | self.resource = modelresource_factory(self.model) 41 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Infos 2 | 3 | 4 | 5 | ## What is being delivered? 6 | 7 | 8 | 9 | ## What impacts? 10 | 11 | 12 | 13 | ## Reversal plan 14 | 15 | 16 | 17 | 18 | 19 | ## Where to monitor 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | --- 28 | Doubts about the flow? 29 | [Take a look at JS+ gitflow!](https://github.com/juntossomosmais/gitflow/blob/main/gitflow/backend/README.md) 30 | 31 | Doubts about the code standards? 32 | [Take a look at JS+ python playbook!](https://github.com/juntossomosmais/playbook/blob/master/backend/python.md) 33 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "django-import-export-stomp" 3 | version = "0.4.1" 4 | description = "Run django-import-export processes using django-stomp" 5 | authors = [ 6 | "Nilton Frederico Teixeira <9078708+niltonfrederico@users.noreply.github.com>", 7 | ] 8 | license = "MIT" 9 | readme = "README.md" 10 | packages = [{ include = "import_export_stomp" }] 11 | 12 | [tool.poetry.dependencies] 13 | python = ">=3.10,<4.0" 14 | django = ">=5.1" 15 | django-stomp = "*" 16 | django-author = "*" 17 | django-import-export = "*" 18 | tablib = {version = "*", extras = ["html", "ods", "xls", "xlsx", "yaml"]} 19 | 20 | 21 | [tool.poetry.group.dev.dependencies] 22 | ### Pytest: Essentials 23 | pytest = "*" 24 | pytest-mock = "*" 25 | pytest-django = "*" 26 | pytest-cov = "*" 27 | pytest-env = "*" 28 | pytest-xdist = "*" 29 | ### Pytest: Add-ons 30 | pytest-icdiff = "*" 31 | pytest-clarity = "*" 32 | ### Format, lint, static type checker, among others 33 | black = "*" 34 | mypy = "*" 35 | isort = "*" 36 | flake8 = "*" 37 | flake8-bugbear = "*" 38 | autoflake = "*" 39 | pre-commit = "*" 40 | #### Helpers and so on 41 | model-bakery = "*" 42 | django-storages = "^1.14.1" 43 | boto3 = "^1.28.57" 44 | 45 | [build-system] 46 | requires = ["poetry-core"] 47 | build-backend = "poetry.core.masonry.api" 48 | -------------------------------------------------------------------------------- /example/winners/urls.py: -------------------------------------------------------------------------------- 1 | """winners URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.10/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | 17 | from typing import Sequence 18 | from typing import Union 19 | 20 | from django.conf import settings 21 | from django.conf.urls.static import static 22 | from django.contrib import admin 23 | from django.urls import URLPattern 24 | from django.urls import URLResolver 25 | from django.urls import path 26 | 27 | from import_export_stomp.urls import urlpatterns as import_export_stomp_urlpatterns 28 | 29 | urlpatterns: Sequence[Union[URLResolver, URLPattern]] = ( 30 | [ 31 | path("admin/", admin.site.urls), 32 | ] 33 | + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) 34 | + import_export_stomp_urlpatterns 35 | ) # type: ignore 36 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: check-added-large-files 6 | - id: check-case-conflict 7 | - id: check-merge-conflict 8 | - id: end-of-file-fixer 9 | - id: trailing-whitespace 10 | 11 | - repo: https://github.com/asottile/pyupgrade 12 | rev: v3.19.1 13 | hooks: 14 | - id: pyupgrade 15 | args: [--py37-plus] 16 | 17 | - repo: https://github.com/pycqa/flake8 18 | rev: 7.1.2 19 | hooks: 20 | - id: flake8 21 | - repo: https://github.com/psf/black 22 | rev: 25.1.0 23 | hooks: 24 | - id: black 25 | exclude: migrations/ 26 | 27 | - repo: https://github.com/pycqa/autoflake 28 | rev: v2.3.1 29 | hooks: 30 | - id: autoflake 31 | args: 32 | [ 33 | "--in-place", 34 | "--remove-all-unused-imports", 35 | "--recursive", 36 | "--ignore-init-module-imports", 37 | ] 38 | 39 | - repo: https://github.com/pycqa/isort 40 | rev: 6.0.1 41 | hooks: 42 | - id: isort 43 | 44 | - repo: https://github.com/pre-commit/mirrors-mypy 45 | rev: v1.15.0 46 | hooks: 47 | - id: mypy 48 | additional_dependencies: 49 | - types-requests 50 | - types-python-dateutil 51 | exclude: tests/ 52 | -------------------------------------------------------------------------------- /tests/integration/test_pubsub/test_export.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import json 3 | 4 | import pytest 5 | 6 | from model_bakery import baker 7 | 8 | from import_export_stomp.models import ExportJob 9 | from import_export_stomp.pubsub import consumer 10 | from tests.resources.fake_app.models import FakeModel 11 | from tests.utils import create_payload 12 | 13 | 14 | @pytest.mark.django_db 15 | class TestExport: 16 | def test_should_export_model(self): 17 | fake_models = baker.make(FakeModel, _quantity=3) 18 | 19 | export_job = baker.make( 20 | ExportJob, 21 | format="text/csv", 22 | app_label="fake_app", 23 | model="fakemodel", 24 | resource="FakeResource", 25 | queryset=json.dumps([str(fake_model.pk) for fake_model in fake_models]), 26 | ) 27 | 28 | assert FakeModel.objects.count() == 3 29 | 30 | payload, ack, nack = create_payload( 31 | {"action": "export", "dry_run": False, "job_id": str(export_job.pk)} 32 | ) 33 | 34 | consumer(payload) 35 | 36 | nack.assert_not_called() 37 | ack.assert_called_once() 38 | 39 | export_job.refresh_from_db() 40 | export_job.job_status = "Export complete" 41 | 42 | with export_job.file.open("r") as file: 43 | csv_data = list(csv.DictReader(file)) 44 | 45 | for index, dict_row in enumerate(csv_data): 46 | dict_row["name"] = fake_models[index].name 47 | dict_row["value"] = fake_models[index].value 48 | -------------------------------------------------------------------------------- /tests/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | 3 | import import_export 4 | import pytest 5 | 6 | from pytest_mock import MockerFixture 7 | 8 | import import_export_stomp.utils 9 | 10 | from import_export_stomp.utils import IMPORT_EXPORT_STOMP_PROCESSING_QUEUE 11 | from import_export_stomp.utils import get_formats 12 | from import_export_stomp.utils import send_job_message_to_queue 13 | 14 | 15 | @pytest.mark.django_db 16 | class TestUtils: 17 | def test_get_formats(self) -> None: 18 | CSV = import_export.formats.base_formats.CSV 19 | XLSX = import_export.formats.base_formats.XLSX 20 | with contextlib.suppress(ImportError): 21 | formats = get_formats() 22 | assert CSV in formats 23 | assert XLSX in formats 24 | 25 | 26 | class TestSendMessageToQueue: 27 | @pytest.mark.parametrize( 28 | ("action", "dry_run"), 29 | (("import", False), ("import", True), ("export", False), ("export", True)), 30 | ) 31 | def test_should_send_message_to_defult_queue( 32 | self, action: str, dry_run: bool, mocker: MockerFixture 33 | ): 34 | mocked_send = mocker.MagicMock() 35 | mocker.patch.object( 36 | import_export_stomp.utils, "build_publisher", return_value=mocked_send 37 | ) 38 | job_id = 9999 39 | send_job_message_to_queue(action, job_id, dry_run) 40 | 41 | mocked_send.send.assert_called_with( 42 | queue=IMPORT_EXPORT_STOMP_PROCESSING_QUEUE, 43 | body={"action": action, "job_id": str(job_id), "dry_run": dry_run}, 44 | ) 45 | -------------------------------------------------------------------------------- /tests/unit/models/test_importjob.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from django.utils import timezone 4 | 5 | from import_export_stomp.models import ImportJob 6 | 7 | 8 | @pytest.fixture 9 | def import_job() -> ImportJob: 10 | return ImportJob(file="test.csv", model="TestModel", format="CSV") 11 | 12 | 13 | @pytest.mark.django_db 14 | class TestImportJobModel: 15 | def test_import_job_model(self, import_job: ImportJob) -> None: 16 | import_job.save() 17 | 18 | assert import_job.file == "test.csv" 19 | assert import_job.format == "CSV" 20 | assert import_job.model == "TestModel" 21 | 22 | import_job.processing_initiated = timezone.now() 23 | import_job.job_status = "Processing" 24 | import_job.save() 25 | 26 | assert import_job.processing_initiated is not None 27 | assert import_job.job_status == "Processing" 28 | 29 | format_choices = import_job.get_format_choices() 30 | assert len(format_choices) > 0 31 | assert all(isinstance(choice, tuple) for choice in format_choices) 32 | 33 | def test_importjob_post_save(self, import_job: ImportJob) -> None: 34 | import_job.save() 35 | 36 | import_job_db = ImportJob.objects.get(pk=import_job.pk) 37 | 38 | assert import_job_db.processing_initiated is not None 39 | 40 | def test_auto_delete_file_on_delete(self, import_job: ImportJob) -> None: 41 | import_job.save() 42 | import_job.delete() 43 | 44 | with pytest.raises( 45 | ValueError, match="The 'file' attribute has no file associated with it." 46 | ): 47 | import_job.file.url 48 | assert not ImportJob.objects.filter(pk=import_job.pk).exists() 49 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | select = E,F,W,B,S 3 | max-complexity = 15 4 | ignore = E203,W503,E731 5 | per-file-ignores = tests/*:S101 6 | max-line-length = 120 7 | exclude = migrations,**/migrations/* 8 | [pycodestyle] 9 | ignore = W503, E701 10 | max-line-length = 120 11 | 12 | [isort] 13 | force_single_line = true 14 | ensure_newline_before_comments = true 15 | line_length = 120 16 | skip_glob = ["**/migrations/*.py"] 17 | use_parentheses = true 18 | multi_line_output = 3 19 | include_trailing_comma = true 20 | lines_between_types = 1 21 | 22 | [mypy] 23 | files = import_export_stomp/**/*.py 24 | 25 | # flake8-mypy expects the two following for sensible formatting 26 | show_column_numbers=True 27 | show_error_context=False 28 | 29 | # do not follow imports (except for ones found in typeshed) 30 | follow_imports=skip 31 | 32 | # since we're ignoring imports, writing .mypy_cache doesn't make any sense 33 | cache_dir=/dev/null 34 | 35 | # suppress errors about unsatisfied imports 36 | ignore_missing_imports=True 37 | 38 | # allow untyped calls as a consequence of the options above 39 | disallow_untyped_calls=False 40 | 41 | # allow returning Any as a consequence of the options above 42 | warn_return_any=False 43 | 44 | # treat Optional per PEP 484 45 | strict_optional=True 46 | 47 | # ensure all execution paths are returning 48 | warn_no_return=True 49 | 50 | # lint-style cleanliness for typing needs to be disabled; returns more errors 51 | # than the full run. 52 | warn_redundant_casts=False 53 | warn_unused_ignores=False 54 | 55 | # The following are off by default since they're too noisy. 56 | # Flip them on if you feel adventurous. 57 | disallow_untyped_defs=False 58 | check_untyped_defs=False 59 | 60 | [mypy-*.migrations.*] 61 | ignore_errors = True 62 | 63 | [mypy.plugins.django-stubs] 64 | django_settings_module = "tests.settings" 65 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | # See more at: 2 | # - https://docs.sonarcloud.io/advanced-setup/analysis-parameters 3 | # - https://docs.sonarcloud.io/enriching/test-coverage-and-execution/ 4 | 5 | # Look at your Docker Compose file. You'll see `sonar` and `sonar-cli` services. 6 | sonar.host.url=http://sonar:9000 7 | 8 | # Project configuration 9 | sonar.organization=juntossomosmais 10 | sonar.projectKey=juntossomosmais_django-import-export-stomp 11 | 12 | # Language 13 | sonar.language=py 14 | sonar.python.file.suffixes=py 15 | sonar.sourceEncoding=UTF-8 16 | 17 | # Patterns used to exclude some files from coverage report. 18 | sonar.coverage.exclusions=\ 19 | **/__init__.py,\ 20 | **/settings.py,\ 21 | **/seed.py,\ 22 | **/model_admin.py,\ 23 | **/*/create_schema.py,\ 24 | **/wsgi.py,\ 25 | **/asgi.py,\ 26 | **/*/logging.py,\ 27 | **/manage.py,\ 28 | gunicorn_config.py,\ 29 | tests/**/*,\ 30 | **/tests/**/*,\ 31 | scripts/**/*,\ 32 | .tox/**/*,\ 33 | .venv/**/*,\ 34 | .mypy_cache/**/*,\ 35 | performance/**/*,\ 36 | **/migrations/*.py,\ 37 | example/**/*,\ 38 | **/*/admin.py,\ 39 | **/*/tasks.py,\ 40 | 41 | # Patterns used to exclude some source files from the duplication detection mechanism. 42 | sonar.cpd.exclusions=\ 43 | **/migrations/*.py,\ 44 | **/__init__.py,\ 45 | **/settings.py,\ 46 | **/seed_db.py,\ 47 | **/model_admin.py,\ 48 | **/*/create_schema.py,\ 49 | **/wsgi.py,\ 50 | **/asgi.py,\ 51 | **/*/logging.py,\ 52 | manage.py,\ 53 | gunicorn_config.py,\ 54 | tests/**/*,\ 55 | **/tests/**/*,\ 56 | scripts/**/*,\ 57 | .tox/**/*,\ 58 | .venv/**/*,\ 59 | .mypy_cache/**/*,\ 60 | performance/**/*, \ 61 | 62 | # Reports 63 | sonar.python.xunit.reportPath=tests-reports/junit.xml 64 | sonar.python.coverage.reportPaths=tests-reports/coverage.xml 65 | 66 | # TSHOOT 67 | sonar.verbose=false 68 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0009_alter_exportjob_options_alter_importjob_options_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1.9 on 2023-05-27 22:35 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("import_export_stomp", "0008_alter_exportjob_id_alter_importjob_id"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterModelOptions( 15 | name="exportjob", 16 | options={ 17 | "verbose_name": "Export job", 18 | "verbose_name_plural": "Export jobs", 19 | }, 20 | ), 21 | migrations.AlterModelOptions( 22 | name="importjob", 23 | options={ 24 | "verbose_name": "Import job", 25 | "verbose_name_plural": "Import jobs", 26 | }, 27 | ), 28 | migrations.AlterField( 29 | model_name="exportjob", 30 | name="id", 31 | field=models.AutoField( 32 | auto_created=True, primary_key=True, serialize=False, verbose_name="ID" 33 | ), 34 | ), 35 | migrations.AlterField( 36 | model_name="exportjob", 37 | name="site_of_origin", 38 | field=models.TextField( 39 | default="", max_length=255, verbose_name="Site of origin" 40 | ), 41 | ), 42 | migrations.AlterField( 43 | model_name="importjob", 44 | name="errors", 45 | field=models.TextField(blank=True, default="", verbose_name="Errors"), 46 | ), 47 | migrations.AlterField( 48 | model_name="importjob", 49 | name="id", 50 | field=models.AutoField( 51 | auto_created=True, primary_key=True, serialize=False, verbose_name="ID" 52 | ), 53 | ), 54 | ] 55 | -------------------------------------------------------------------------------- /example/winners/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django.test import override_settings 3 | from import_export_celery.models import ExportJob 4 | from import_export_celery.utils import DEFAULT_EXPORT_JOB_COMPLETION_MAIL_SUBJECT 5 | from import_export_celery.utils import DEFAULT_EXPORT_JOB_COMPLETION_MAIL_TEMPLATE 6 | from import_export_celery.utils import get_export_job_mail_context 7 | from import_export_celery.utils import get_export_job_mail_subject 8 | from import_export_celery.utils import get_export_job_mail_template 9 | 10 | 11 | class UtilsTestCases(TestCase): 12 | def test_get_export_job_mail_subject_by_default(self): 13 | self.assertEqual( 14 | DEFAULT_EXPORT_JOB_COMPLETION_MAIL_SUBJECT, get_export_job_mail_subject() 15 | ) 16 | 17 | @override_settings(EXPORT_JOB_COMPLETION_MAIL_SUBJECT="New subject") 18 | def test_get_export_job_mail_subject_overridden(self): 19 | self.assertEqual("New subject", get_export_job_mail_subject()) 20 | 21 | def test_get_export_job_mail_template_default(self): 22 | self.assertEqual( 23 | DEFAULT_EXPORT_JOB_COMPLETION_MAIL_TEMPLATE, get_export_job_mail_template() 24 | ) 25 | 26 | @override_settings(EXPORT_JOB_COMPLETION_MAIL_TEMPLATE="mytemplate.html") 27 | def test_get_export_job_mail_template_overridden(self): 28 | self.assertEqual("mytemplate.html", get_export_job_mail_template()) 29 | 30 | def test_get_export_job_mail_context(self): 31 | export_job = ExportJob.objects.create( 32 | app_label="winners", model="Winner", site_of_origin="http://127.0.0.1:8000" 33 | ) 34 | context = get_export_job_mail_context(export_job) 35 | expected_context = { 36 | "app_label": "winners", 37 | "model": "Winner", 38 | "link": f"http://127.0.0.1:8000/adminimport_export_celery/exportjob/{export_job.id}/change/", # noqa 39 | } 40 | self.assertEqual(context, expected_context) 41 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0010_remove_exportjob_email_on_completion_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.5 on 2023-10-02 02:33 2 | 3 | from django.core.files.storage import default_storage # Use this instead of S3 4 | from django.db import migrations 5 | 6 | import import_export_stomp.fields 7 | 8 | 9 | class Migration(migrations.Migration): 10 | dependencies = [ 11 | ( 12 | "import_export_stomp", 13 | "0009_alter_exportjob_options_alter_importjob_options_and_more", 14 | ), 15 | ] 16 | 17 | operations = [ 18 | migrations.RemoveField( 19 | model_name="exportjob", 20 | name="email_on_completion", 21 | ), 22 | migrations.AlterField( 23 | model_name="exportjob", 24 | name="file", 25 | field=import_export_stomp.fields.ImportExportFileField( 26 | max_length=255, 27 | storage=default_storage, # Use default_storage 28 | upload_to="django-import-export-stomp-export-jobs", 29 | verbose_name="exported file", 30 | ), 31 | ), 32 | migrations.AlterField( 33 | model_name="importjob", 34 | name="change_summary", 35 | field=import_export_stomp.fields.ImportExportFileField( 36 | blank=True, 37 | null=True, 38 | storage=default_storage, # Use default_storage 39 | upload_to="django-import-export-stomp-import-change-summaries", 40 | verbose_name="Summary of changes made by this import", 41 | ), 42 | ), 43 | migrations.AlterField( 44 | model_name="importjob", 45 | name="file", 46 | field=import_export_stomp.fields.ImportExportFileField( 47 | max_length=255, 48 | storage=default_storage, # Use default_storage 49 | upload_to="django-import-export-stomp-import-jobs", 50 | verbose_name="File to be imported", 51 | ), 52 | ), 53 | ] 54 | -------------------------------------------------------------------------------- /import_export_stomp/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Literal 2 | from typing import Type 3 | from typing import Union 4 | from uuid import uuid4 5 | 6 | from django import VERSION as DJANGO_VERSION 7 | from django.conf import settings 8 | from django.core.files.storage import Storage 9 | from django.utils.module_loading import import_string 10 | from django_stomp.builder import build_publisher 11 | from django_stomp.services.producer import auto_open_close_connection 12 | from django_stomp.services.producer import do_inside_transaction 13 | from import_export.formats.base_formats import DEFAULT_FORMATS 14 | 15 | USE_GET_STORAGE_CLASS = DJANGO_VERSION < (4, 2) 16 | if USE_GET_STORAGE_CLASS: 17 | from django.core.files.storage import get_storage_class as legacy_get_storage_class 18 | 19 | IMPORT_EXPORT_STOMP_PROCESSING_QUEUE = getattr( 20 | settings, 21 | "IMPORT_EXPORT_STOMP_PROCESSING_QUEUE", 22 | "/queue/django-import-export-stomp-runner", 23 | ) 24 | 25 | 26 | IMPORT_EXPORT_STOMP_EXCLUDED_FORMATS = getattr( 27 | settings, 28 | "IMPORT_EXPORT_STOMP_EXCLUDED_FORMATS", 29 | [], 30 | ) 31 | 32 | 33 | def get_storage_class(import_path: str | None = None) -> Type[Storage]: 34 | if USE_GET_STORAGE_CLASS: 35 | return legacy_get_storage_class(import_path) 36 | else: 37 | return import_string(import_path or settings.DEFAULT_FILE_STORAGE) 38 | 39 | 40 | def get_formats(): 41 | return [ 42 | format 43 | for format in DEFAULT_FORMATS 44 | if format.TABLIB_MODULE.split(".")[-1].strip("_") 45 | not in IMPORT_EXPORT_STOMP_EXCLUDED_FORMATS 46 | ] 47 | 48 | 49 | def send_job_message_to_queue( 50 | action: Union[Literal["import"], Literal["export"]], 51 | job_id: int, 52 | dry_run: bool = False, 53 | ) -> None: 54 | publisher = build_publisher(f"django-import-export-stomp-{str(uuid4())}") 55 | 56 | with auto_open_close_connection(publisher), do_inside_transaction(publisher): 57 | publisher.send( 58 | queue=IMPORT_EXPORT_STOMP_PROCESSING_QUEUE, 59 | body={"action": action, "job_id": str(job_id), "dry_run": dry_run}, 60 | ) 61 | -------------------------------------------------------------------------------- /tests/unit/test_resources.py: -------------------------------------------------------------------------------- 1 | from sys import modules 2 | from typing import Any 3 | from unittest import mock 4 | 5 | import pytest 6 | 7 | from import_export.resources import ModelResource 8 | 9 | from import_export_stomp.models import ImportJob 10 | from import_export_stomp.models.exportjob import ExportJob 11 | from import_export_stomp.resources import ModelConfig 12 | from import_export_stomp.resources import resource_importer 13 | from tests.utils import create_payload 14 | 15 | 16 | class SampleResource(ModelResource): 17 | class Meta: 18 | model = ImportJob 19 | 20 | 21 | @pytest.fixture 22 | def resource() -> SampleResource: 23 | return SampleResource 24 | 25 | 26 | @pytest.mark.django_db 27 | class TestModelConfig: 28 | def test_model_config_should_pass_when_resource_is_provided( 29 | self, resource: SampleResource 30 | ) -> None: 31 | app_label = "import_export_stomp" 32 | model_name = "ImportJob" 33 | 34 | config = ModelConfig( 35 | app_label=app_label, model_name=model_name, resource=resource 36 | ) 37 | 38 | assert config.model == ImportJob 39 | assert isinstance(config.resource, SampleResource) 40 | 41 | @mock.patch("import_export_stomp.resources.modelresource_factory") 42 | def test_model_config_should_pass_with_none_resource(self, mocker: Any) -> None: 43 | app_label = "import_export_stomp" 44 | model_name = "ExportJob" 45 | 46 | config = ModelConfig(app_label=app_label, model_name=model_name, resource=None) 47 | 48 | assert config.model == ExportJob 49 | mocker.assert_called_once_with(ExportJob) 50 | 51 | 52 | class TestResourceImporter: 53 | def test_should_import_module_from_string(self): 54 | imported = resource_importer("tests.utils.create_payload") 55 | 56 | assert "tests.utils" in modules.keys() 57 | assert imported() == create_payload 58 | 59 | def test_should_fail_to_import_inexisting_module(self): 60 | imported = resource_importer("fake.module.fake_function") 61 | 62 | with pytest.raises(ModuleNotFoundError): 63 | imported() 64 | -------------------------------------------------------------------------------- /tests/integration/test_pubsub/test_import.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from django.core.files.base import ContentFile 4 | from model_bakery import baker 5 | 6 | from import_export_stomp.models import ImportJob 7 | from import_export_stomp.pubsub import consumer 8 | from tests.resources.fake_app.models import FakeModel 9 | from tests.utils import create_payload 10 | 11 | 12 | @pytest.mark.django_db 13 | class TestImport: 14 | @pytest.fixture 15 | def csv_file(self) -> ContentFile: 16 | return ContentFile( 17 | b"name,value\nname1,1\nname2,2", 18 | name="import_csv.csv", 19 | ) 20 | 21 | def test_should_import_to_model_dry_run(self, csv_file: ContentFile): 22 | import_job = baker.make( 23 | ImportJob, file=csv_file, model="Test", format="text/csv" 24 | ) 25 | 26 | assert FakeModel.objects.count() == 0 27 | 28 | payload, ack, nack = create_payload( 29 | {"action": "import", "dry_run": True, "job_id": str(import_job.pk)} 30 | ) 31 | 32 | consumer(payload) 33 | 34 | nack.assert_not_called() 35 | ack.assert_called_once() 36 | 37 | import_job.refresh_from_db() 38 | assert import_job.job_status == "[Dry run] 5/5 Import job finished" 39 | assert import_job.imported is None 40 | 41 | assert FakeModel.objects.count() == 0 42 | 43 | def test_should_import_to_model(self, csv_file: ContentFile): 44 | import_job = baker.make( 45 | ImportJob, file=csv_file, model="Test", format="text/csv" 46 | ) 47 | 48 | assert FakeModel.objects.count() == 0 49 | 50 | payload, ack, nack = create_payload( 51 | {"action": "import", "dry_run": False, "job_id": str(import_job.pk)} 52 | ) 53 | 54 | consumer(payload) 55 | 56 | nack.assert_not_called() 57 | ack.assert_called_once() 58 | 59 | import_job.refresh_from_db() 60 | assert "Import error" not in import_job.job_status 61 | 62 | assert FakeModel.objects.count() == 2 63 | row_1, row_2 = FakeModel.objects.order_by("id").all() 64 | 65 | assert row_1.name == "name1" 66 | assert row_1.value == 1 67 | assert row_2.name == "name2" 68 | assert row_2.value == 2 69 | -------------------------------------------------------------------------------- /import_export_stomp/pubsub.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from typing import Callable 4 | from typing import Tuple 5 | from typing import Union 6 | 7 | from django_stomp.services.consumer import Payload 8 | 9 | from import_export_stomp.models import ExportJob 10 | from import_export_stomp.models import ImportJob 11 | from import_export_stomp.tasks import run_export_job 12 | from import_export_stomp.tasks import run_import_job 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | ACTIONS = ("import", "export") 17 | 18 | 19 | def validate_payload(payload: Payload): 20 | assert "action" in payload.body, "Payload needs to have 'action' key set." 21 | assert ( 22 | payload.body["action"] in ACTIONS 23 | ), "Action value needs to be 'import' or 'export'." 24 | assert "dry_run" in payload.body, "Payload needs to have 'dry_run' key set." 25 | assert isinstance(payload.body["dry_run"], bool), "'dry_run' is not a boolean." 26 | assert "job_id" in payload.body, "Payload needs to have 'job_id' key set." 27 | assert payload.body["job_id"].isnumeric(), "'job_id' is not a number." 28 | 29 | 30 | def get_job_object_and_runner( 31 | payload: Payload, 32 | ) -> Union[Tuple[ImportJob, Callable], Tuple[ExportJob, Callable]]: 33 | filters = { 34 | "pk": payload.body["job_id"], 35 | } 36 | return ( 37 | (ImportJob.objects.get(**filters | {"imported__isnull": True}), run_import_job) 38 | if payload.body["action"] == "import" 39 | else (ExportJob.objects.get(**filters), run_export_job) 40 | ) 41 | 42 | 43 | def consumer(payload: Payload): 44 | """ 45 | Consumer that processes both import/export jobs. 46 | 47 | Expected payload example: 48 | { 49 | "action": "import", 50 | "dry_run": True, 51 | "job_id": "9734b8b2-598d-4925-87da-20d453cab9d8" 52 | } 53 | """ 54 | 55 | try: 56 | validate_payload(payload) 57 | job, runner = get_job_object_and_runner(payload) 58 | except (AssertionError, ImportJob.DoesNotExist, ExportJob.DoesNotExist) as exc: 59 | logger.warning(str(exc)) 60 | # Since the error is unrecoverable we will only ack 61 | return payload.ack() 62 | 63 | runner(job, dry_run=payload.body["dry_run"]) 64 | 65 | return payload.ack() 66 | -------------------------------------------------------------------------------- /import_export_stomp/admin_actions.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from uuid import UUID 5 | 6 | from django.shortcuts import redirect 7 | from django.urls import reverse 8 | from django.utils import timezone 9 | from django.utils.translation import gettext_lazy as _ 10 | 11 | from import_export_stomp.models import ExportJob 12 | from import_export_stomp.utils import send_job_message_to_queue 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | def run_import_job_action(modeladmin, request, queryset): 18 | for instance in queryset: 19 | logger.info("Importing %s dry-run: False", instance.pk) 20 | send_job_message_to_queue( 21 | action="import", 22 | job_id=instance.pk, 23 | dry_run=False, 24 | ) 25 | 26 | 27 | run_import_job_action.short_description = _("Perform import") # type: ignore 28 | 29 | 30 | def run_import_job_action_dry(modeladmin, request, queryset): 31 | for instance in queryset: 32 | logger.info("Importing %s dry-run: True", instance.pk) 33 | send_job_message_to_queue( 34 | action="import", 35 | job_id=instance.pk, 36 | dry_run=True, 37 | ) 38 | 39 | 40 | run_import_job_action_dry.short_description = _("Perform dry import") # type: ignore 41 | 42 | 43 | def run_export_job_action(modeladmin, request, queryset): 44 | for instance in queryset: 45 | instance.processing_initiated = timezone.now() 46 | instance.save() 47 | send_job_message_to_queue(action="export", job_id=instance.pk) 48 | 49 | 50 | run_export_job_action.short_description = _("Run export job") # type: ignore 51 | 52 | 53 | def create_export_job_action(modeladmin, request, queryset): 54 | if queryset: 55 | arbitrary_obj = queryset.first() 56 | ej = ExportJob.objects.create( 57 | app_label=arbitrary_obj._meta.app_label, 58 | model=arbitrary_obj._meta.model_name, 59 | queryset=json.dumps( 60 | [ 61 | str(obj.pk) if isinstance(obj.pk, UUID) else obj.pk 62 | for obj in queryset 63 | ] 64 | ), 65 | site_of_origin=request.scheme + "://" + request.get_host(), 66 | ) 67 | 68 | rurl = reverse( 69 | "admin:%s_%s_change" 70 | % ( 71 | ej._meta.app_label, 72 | ej._meta.model_name, 73 | ), 74 | args=[ej.pk], 75 | ) 76 | return redirect(rurl) 77 | 78 | 79 | create_export_job_action.short_description = _("Export with stomp") # type: ignore 80 | -------------------------------------------------------------------------------- /tests/unit/models/test_exportjob.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from unittest.mock import Mock 4 | from unittest.mock import patch 5 | 6 | import pytest 7 | 8 | from import_export_stomp.models.exportjob import ExportJob 9 | 10 | 11 | @pytest.fixture 12 | def export_job() -> ExportJob: 13 | return ExportJob( 14 | app_label="import_export_stomp", 15 | model="ImportJob", 16 | queryset=json.dumps([1, 2, 3]), 17 | site_of_origin="http://example.com", 18 | ) 19 | 20 | 21 | @pytest.mark.django_db 22 | class TestExportJobModel: 23 | def test_exportjob_post_save_signal_handler(self) -> None: 24 | export_job = ExportJob( 25 | file="test_file.csv", format="CSV", resource="TestResource" 26 | ) 27 | export_job.save() 28 | 29 | assert export_job.processing_initiated is not None 30 | 31 | def test_get_content_type(self, export_job: ExportJob) -> None: 32 | with patch( 33 | "import_export_stomp.models.exportjob.ContentType.objects.get" 34 | ) as mock_get_content_type: 35 | content_type = export_job.get_content_type() 36 | 37 | mock_get_content_type.assert_called_once_with( 38 | app_label=export_job.app_label, 39 | model=export_job.model, 40 | ) 41 | assert content_type is mock_get_content_type.return_value 42 | 43 | def test_get_queryset_custom_resource(self, export_job: ExportJob) -> None: 44 | export_job.resource = "sample_resource" 45 | 46 | with patch.object( 47 | export_job, "get_resource_class" 48 | ) as mock_get_resource_class, patch.object(export_job, "get_content_type"): 49 | mock_resource = Mock( 50 | get_export_queryset=Mock(return_value=Mock(filter=Mock())) 51 | ) 52 | mock_get_resource_class.return_value = mock_resource 53 | 54 | export_job.get_queryset() 55 | 56 | mock_get_resource_class.assert_called_once() 57 | 58 | def test_get_queryset_default_resource(self, export_job: ExportJob) -> None: 59 | export_job.resource = "" 60 | with patch( 61 | "import_export_stomp.models.exportjob.ExportJob.get_resource_class" 62 | ) as mock_get_resource_class, patch( 63 | "import_export_stomp.models.exportjob.ExportJob.get_content_type" 64 | ) as mock_get_content_type: 65 | mock_get_resource_class.return_value = None 66 | mock_content_type = Mock(model_class=Mock(objects=Mock(filter=Mock()))) 67 | mock_get_content_type.return_value = mock_content_type 68 | 69 | export_job.get_queryset() 70 | 71 | mock_get_resource_class.assert_called_once() 72 | mock_get_content_type.assert_called_once() 73 | -------------------------------------------------------------------------------- /import_export_stomp/views.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import json 3 | 4 | from http import HTTPStatus 5 | from importlib import util 6 | 7 | from django.conf import settings 8 | from django.contrib.admin.views.decorators import staff_member_required 9 | from django.http import HttpRequest 10 | from django.http import JsonResponse 11 | from django.views.decorators.http import require_POST 12 | 13 | from import_export_stomp.utils import get_formats 14 | 15 | 16 | @require_POST 17 | @staff_member_required 18 | def generate_presigned_post(request: HttpRequest) -> JsonResponse: 19 | if not getattr(settings, "IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST"): 20 | return JsonResponse( 21 | {"error": "IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST is set to false."}, 22 | status=HTTPStatus.FAILED_DEPENDENCY, 23 | ) 24 | 25 | boto3_spec = util.find_spec("boto3") 26 | storages_spec = util.find_spec("storages") 27 | 28 | if not boto3_spec and not storages_spec: 29 | return JsonResponse( 30 | {"error": "boto3 and django-storages required for this action."}, 31 | status=HTTPStatus.FAILED_DEPENDENCY, 32 | ) 33 | 34 | # Import boto3 35 | boto3 = importlib.import_module("boto3") 36 | 37 | # Import Config from botocore.config 38 | botocore = importlib.import_module("botocore") 39 | botocore_config = botocore.config.Config 40 | 41 | data = json.loads(request.body) 42 | 43 | filename, mimetype, allowed_formats = ( 44 | data["filename"], 45 | data["mimetype"], 46 | [_format.CONTENT_TYPE for _format in get_formats()], 47 | ) 48 | 49 | if mimetype not in allowed_formats: 50 | return JsonResponse( 51 | { 52 | "error": f"File format {mimetype} is not allowed. Accepted formats: {allowed_formats}" 53 | }, 54 | status=HTTPStatus.BAD_REQUEST, 55 | ) 56 | 57 | client = boto3.client( 58 | "s3", 59 | endpoint_url=getattr(settings, "AWS_S3_ENDPOINT_URL", None), 60 | region_name=getattr(settings, "AWS_DEFAULT_REGION", None), 61 | aws_access_key_id=getattr(settings, "AWS_ACCESS_KEY_ID", None), 62 | aws_secret_access_key=getattr(settings, "AWS_SECRET_ACCESS_KEY", None), 63 | config=botocore_config(signature_version="s3v4"), 64 | ) 65 | 66 | file_path = getattr(settings, "IMPORT_EXPORT_STOMP_PRESIGNED_FOLDER", "") + filename 67 | 68 | response = client.generate_presigned_post( 69 | getattr(settings, "AWS_STORAGE_BUCKET_NAME"), 70 | file_path, 71 | ExpiresIn=getattr( 72 | settings, "IMPORT_EXPORT_STOMP_PRESIGNED_POST_EXPIRATION", 600 73 | ), 74 | ) 75 | 76 | return JsonResponse(response, status=HTTPStatus.CREATED) 77 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0006_auto_20191125_1236.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.7 on 2019-11-25 12:36 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("import_export_stomp", "0005_exportjob_site_of_origin"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="exportjob", 16 | name="job_status", 17 | field=models.CharField( 18 | blank=True, max_length=160, verbose_name="Status of the job" 19 | ), 20 | ), 21 | migrations.AlterField( 22 | model_name="exportjob", 23 | name="format", 24 | field=models.CharField( 25 | choices=[ 26 | ("text/csv", "text/csv"), 27 | ("application/vnd.ms-excel", "application/vnd.ms-excel"), 28 | ( 29 | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 30 | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 31 | ), 32 | ("text/tab-separated-values", "text/tab-separated-values"), 33 | ( 34 | "application/vnd.oasis.opendocument.spreadsheet", 35 | "application/vnd.oasis.opendocument.spreadsheet", 36 | ), 37 | ("application/json", "application/json"), 38 | ("text/yaml", "text/yaml"), 39 | ("text/html", "text/html"), 40 | ], 41 | max_length=255, 42 | null=True, 43 | verbose_name="Format of file to be exported", 44 | ), 45 | ), 46 | migrations.AlterField( 47 | model_name="importjob", 48 | name="format", 49 | field=models.CharField( 50 | choices=[ 51 | ("text/csv", "text/csv"), 52 | ("application/vnd.ms-excel", "application/vnd.ms-excel"), 53 | ( 54 | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 55 | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 56 | ), 57 | ("text/tab-separated-values", "text/tab-separated-values"), 58 | ( 59 | "application/vnd.oasis.opendocument.spreadsheet", 60 | "application/vnd.oasis.opendocument.spreadsheet", 61 | ), 62 | ("application/json", "application/json"), 63 | ("text/yaml", "text/yaml"), 64 | ("text/html", "text/html"), 65 | ], 66 | max_length=255, 67 | verbose_name="Format of file to be imported", 68 | ), 69 | ), 70 | migrations.AlterField( 71 | model_name="importjob", 72 | name="model", 73 | field=models.CharField( 74 | max_length=160, verbose_name="Name of model to import to" 75 | ), 76 | ), 77 | ] 78 | -------------------------------------------------------------------------------- /import_export_stomp/templates/widgets/signed_url_file.html: -------------------------------------------------------------------------------- 1 | {% include "django/forms/widgets/input.html" %} 2 | {% load settings %} 3 | {% csrf_token %} 4 | {% settings_value 'IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST' as IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST %} 5 | 75 | -------------------------------------------------------------------------------- /import_export_stomp/models/importjob.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from functools import partial 4 | 5 | from author.decorators import with_author 6 | from django.conf import settings 7 | from django.db import models 8 | from django.db import transaction 9 | from django.db.models.signals import post_delete 10 | from django.db.models.signals import post_save 11 | from django.dispatch import receiver 12 | from django.utils import timezone 13 | from django.utils.translation import gettext_lazy as _ 14 | from import_export.formats.base_formats import DEFAULT_FORMATS 15 | 16 | from import_export_stomp.fields import ImportExportFileField 17 | from import_export_stomp.utils import send_job_message_to_queue 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | @with_author 23 | class ImportJob(models.Model): 24 | file = ImportExportFileField( 25 | verbose_name=_("File to be imported"), 26 | upload_to="django-import-export-stomp-import-jobs", 27 | blank=False, 28 | null=False, 29 | max_length=255, 30 | ) 31 | 32 | processing_initiated = models.DateTimeField( 33 | verbose_name=_("Have we started processing the file? If so when?"), 34 | null=True, 35 | blank=True, 36 | default=None, 37 | ) 38 | 39 | imported = models.DateTimeField( 40 | verbose_name=_("Has the import been completed? If so when?"), 41 | null=True, 42 | blank=True, 43 | default=None, 44 | ) 45 | 46 | format = models.CharField( 47 | verbose_name=_("Format of file to be imported"), 48 | max_length=255, 49 | ) 50 | 51 | change_summary = ImportExportFileField( 52 | verbose_name=_("Summary of changes made by this import"), 53 | upload_to="django-import-export-stomp-import-change-summaries", 54 | blank=True, 55 | null=True, 56 | ) 57 | 58 | errors = models.TextField( 59 | verbose_name=_("Errors"), 60 | default="", 61 | blank=True, 62 | ) 63 | 64 | model = models.CharField( 65 | verbose_name=_("Name of model to import to"), 66 | max_length=160, 67 | ) 68 | 69 | job_status = models.CharField( 70 | verbose_name=_("Status of the job"), 71 | max_length=160, 72 | blank=True, 73 | ) 74 | 75 | class Meta: 76 | verbose_name = _("Import job") 77 | verbose_name_plural = _("Import jobs") 78 | app_label = "import_export_stomp" 79 | 80 | @staticmethod 81 | def get_format_choices(): 82 | """returns choices of available import formats""" 83 | return [ 84 | (f.CONTENT_TYPE, f().get_title()) 85 | for f in DEFAULT_FORMATS 86 | if f().can_import() 87 | ] 88 | 89 | 90 | @receiver(post_save, sender=ImportJob) 91 | def importjob_post_save(sender, instance, **kwargs): 92 | if not instance.processing_initiated: 93 | instance.processing_initiated = timezone.now() 94 | instance.save() 95 | transaction.on_commit( 96 | partial( 97 | send_job_message_to_queue, 98 | action="import", 99 | dry_run=getattr(settings, "IMPORT_DRY_RUN_FIRST_TIME", True), 100 | job_id=instance.pk, 101 | ) 102 | ) 103 | 104 | 105 | @receiver(post_delete, sender=ImportJob) 106 | def auto_delete_file_on_delete(sender, instance, **kwargs): 107 | """ 108 | Deletes file related to the import job 109 | """ 110 | if instance.file: 111 | try: 112 | instance.file.delete() 113 | except Exception as e: 114 | logger.error("Some error occurred while deleting ImportJob file: %s", e) 115 | ImportJob.objects.filter(id=instance.id).delete() 116 | -------------------------------------------------------------------------------- /tests/unit/test_admin_actions.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | from unittest.mock import ANY 3 | from unittest.mock import MagicMock 4 | 5 | import pytest 6 | 7 | from django.forms import model_to_dict 8 | from model_bakery import baker 9 | from pytest_mock import MockerFixture 10 | 11 | import import_export_stomp.admin_actions 12 | import import_export_stomp.utils 13 | 14 | from import_export_stomp.admin_actions import create_export_job_action 15 | from import_export_stomp.admin_actions import run_export_job_action 16 | from import_export_stomp.admin_actions import run_import_job_action 17 | from import_export_stomp.admin_actions import run_import_job_action_dry 18 | from import_export_stomp.models import ExportJob 19 | from import_export_stomp.models import ImportJob 20 | from import_export_stomp.utils import IMPORT_EXPORT_STOMP_PROCESSING_QUEUE 21 | from tests.resources.fake_app.models import FakeModel 22 | 23 | 24 | @pytest.mark.django_db 25 | class TestActions: 26 | @pytest.mark.parametrize( 27 | ("fn", "expected_dry_run"), 28 | ((run_import_job_action, False), (run_import_job_action_dry, True)), 29 | ) 30 | def test_run_import_job_action( 31 | self, mocker: MockerFixture, fn: Callable, expected_dry_run: bool 32 | ): 33 | import_job = baker.make(ImportJob) 34 | mocked_send = mocker.MagicMock() 35 | mocker.patch.object( 36 | import_export_stomp.utils, "build_publisher", return_value=mocked_send 37 | ) 38 | 39 | fn(MagicMock(), MagicMock(), ImportJob.objects.all()) 40 | 41 | mocked_send.send.assert_called_with( 42 | queue=IMPORT_EXPORT_STOMP_PROCESSING_QUEUE, 43 | body={ 44 | "action": "import", 45 | "job_id": str(import_job.pk), 46 | "dry_run": expected_dry_run, 47 | }, 48 | ) 49 | 50 | def test_run_export_job_action(self, mocker: MockerFixture): 51 | export_job = baker.make(ExportJob) 52 | mocked_send = mocker.MagicMock() 53 | mocker.patch.object( 54 | import_export_stomp.utils, "build_publisher", return_value=mocked_send 55 | ) 56 | 57 | run_export_job_action(MagicMock(), MagicMock(), ExportJob.objects.all()) 58 | 59 | mocked_send.send.assert_called_with( 60 | queue=IMPORT_EXPORT_STOMP_PROCESSING_QUEUE, 61 | body={"action": "export", "job_id": str(export_job.pk), "dry_run": False}, 62 | ) 63 | 64 | def test_run_create_export_job_action(self, mocker: MockerFixture): 65 | mocked_reverse = mocker.patch.object( 66 | import_export_stomp.admin_actions, "reverse" 67 | ) 68 | mocked_reverse.return_value = "fake" 69 | 70 | mocked_redirect = mocker.patch.object( 71 | import_export_stomp.admin_actions, "redirect" 72 | ) 73 | mocked_redirect.return_value = "fake" 74 | 75 | modeladmin_mock = MagicMock() 76 | request_mock = MagicMock() 77 | request_mock.scheme = "https://" 78 | request_mock.get_host = MagicMock(return_value="fake") 79 | 80 | assert ExportJob.objects.count() == 0 81 | 82 | fake_entry = baker.make(FakeModel) 83 | 84 | create_export_job_action( 85 | modeladmin=modeladmin_mock, 86 | request=request_mock, 87 | queryset=FakeModel.objects.all(), 88 | ) 89 | 90 | assert ExportJob.objects.count() == 1 91 | export_job = ExportJob.objects.get() 92 | 93 | assert model_to_dict(export_job) == { 94 | "id": 1, 95 | "file": ANY, 96 | "processing_initiated": None, 97 | "job_status": "", 98 | "format": None, 99 | "app_label": "fake_app", 100 | "model": "fakemodel", 101 | "resource": "", 102 | "queryset": f"[{fake_entry.pk}]", 103 | "site_of_origin": ANY, 104 | "author": None, 105 | "updated_by": None, 106 | } 107 | -------------------------------------------------------------------------------- /tests/integration/test_views.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from http import HTTPStatus 4 | from unittest.mock import ANY 5 | 6 | import pytest 7 | 8 | from django.contrib import auth 9 | from django.contrib.auth.models import User 10 | from django.test import Client 11 | from django.urls import reverse 12 | from django.utils import timezone 13 | from model_bakery import baker 14 | from pytest_django.fixtures import SettingsWrapper 15 | from pytest_mock import MockerFixture 16 | 17 | from import_export_stomp.views import util 18 | 19 | FILENAME = "csv.csv" 20 | APPLICATION_JSON = "application/json" 21 | 22 | 23 | @pytest.mark.django_db 24 | class TestGeneratePresignedPost: 25 | @pytest.fixture 26 | def client(self) -> Client: 27 | user = baker.make(User, is_staff=True, is_superuser=True) 28 | client = Client(enforce_csrf_checks=False) 29 | client.force_login(user) 30 | 31 | assert auth.get_user(client).is_authenticated 32 | 33 | return client 34 | 35 | @pytest.fixture 36 | def endpoint(self) -> str: 37 | return reverse("import_export_stomp_presigned_url") 38 | 39 | @pytest.mark.parametrize( 40 | "http_method", ("GET", "OPTIONS", "PUT", "DELETE", "PATCH") 41 | ) 42 | def test_should_fail_if_not_post( 43 | self, client: Client, endpoint: str, http_method: str 44 | ): 45 | request_fn = getattr(client, http_method.lower()) 46 | response = request_fn(endpoint) 47 | assert response.status_code != HTTPStatus.CREATED 48 | 49 | def test_should_fail_if_user_is_not_staff(self, client: Client, endpoint: str): 50 | User.objects.all().update(is_staff=False) 51 | response = client.post(endpoint) 52 | 53 | assert response.status_code == HTTPStatus.FOUND 54 | assert "/admin/login" in response.url 55 | 56 | def test_should_fail_if_content_type_is_not_a_valid_spreadsheet( 57 | self, client: Client, endpoint: str 58 | ): 59 | response = client.post( 60 | endpoint, 61 | content_type=APPLICATION_JSON, 62 | data=json.dumps({"filename": FILENAME, "mimetype": "application/fake"}), 63 | ) 64 | 65 | assert response.status_code == HTTPStatus.BAD_REQUEST 66 | assert response.json() == { 67 | "error": "File format application/fake is not allowed. Accepted formats: " 68 | "['text/csv', 'application/vnd.ms-excel', " 69 | "'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'" 70 | ", 'text/tab-separated-values', 'application/vnd.oasis.opendocument.spreadsheet', 'application/json', " 71 | "'text/yaml', 'text/html']" 72 | } 73 | 74 | def test_should_fail_if_module_is_not_present( 75 | self, 76 | client: Client, 77 | endpoint: str, 78 | mocker: MockerFixture, 79 | ): 80 | mocker.patch.object(util, "find_spec", return_value=None) 81 | 82 | response = client.post( 83 | endpoint, 84 | content_type=APPLICATION_JSON, 85 | data=json.dumps({"filename": FILENAME, "mimetype": "text/csv"}), 86 | ) 87 | 88 | assert response.status_code == HTTPStatus.FAILED_DEPENDENCY 89 | assert response.json() == { 90 | "error": "boto3 and django-storages required for this action." 91 | } 92 | 93 | def test_should_return_presigned_info( 94 | self, settings: SettingsWrapper, client: Client, endpoint: str 95 | ): 96 | response = client.post( 97 | endpoint, 98 | content_type=APPLICATION_JSON, 99 | data=json.dumps({"filename": FILENAME, "mimetype": "text/csv"}), 100 | ) 101 | 102 | assert response.status_code == HTTPStatus.CREATED 103 | assert response.json() == { 104 | "url": "http://minio:9000/example", 105 | "fields": { 106 | "key": settings.IMPORT_EXPORT_STOMP_PRESIGNED_FOLDER + FILENAME, 107 | "x-amz-algorithm": "AWS4-HMAC-SHA256", 108 | "x-amz-credential": f"minioadmin/{timezone.now().strftime('%Y%m%d')}/us-east-1/s3/aws4_request", 109 | "x-amz-date": ANY, 110 | "policy": ANY, 111 | "x-amz-signature": ANY, 112 | }, 113 | } 114 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | # https://docs.docker.com/compose/compose-file/compose-file-v3/#extension-fields 4 | x-build: &dockerfile-dev-build 5 | context: . 6 | dockerfile: Dockerfile 7 | 8 | services: 9 | db: 10 | image: postgres:16-alpine 11 | environment: 12 | - POSTGRES_PASSWORD=postgres 13 | - POSTGRES_USER=postgres 14 | ports: 15 | - "5432:5432" 16 | healthcheck: 17 | test: ["CMD-SHELL", "pg_isready"] 18 | interval: 10s 19 | timeout: 5s 20 | retries: 5 21 | networks: 22 | - example 23 | 24 | rabbitmq: 25 | image: rabbitmq:4-management 26 | ports: 27 | - 15672:15672 28 | volumes: 29 | - ./tests/resources/custom-rabbitmq-conf:/etc/rabbitmq/ 30 | healthcheck: 31 | test: rabbitmq-diagnostics -q ping 32 | interval: 10s 33 | timeout: 5s 34 | retries: 5 35 | networks: 36 | - example 37 | 38 | pubsub: 39 | build: *dockerfile-dev-build 40 | volumes: 41 | - .:/app 42 | env_file: .env 43 | environment: 44 | - AWS_S3_ENDPOINT_URL=http://minio:9000 45 | depends_on: 46 | db: 47 | condition: service_healthy 48 | rabbitmq: 49 | condition: service_healthy 50 | createbuckets: 51 | condition: service_completed_successfully 52 | command: ["python", "example/manage.py", "import_export_pubsub"] 53 | networks: 54 | - example 55 | 56 | example: 57 | build: *dockerfile-dev-build 58 | volumes: 59 | - .:/app 60 | env_file: .env 61 | ports: 62 | - "${DJANGO_BIND_PORT:-8080}:${DJANGO_BIND_PORT:-8080}" 63 | depends_on: 64 | pubsub: 65 | condition: service_started 66 | db: 67 | condition: service_healthy 68 | rabbitmq: 69 | condition: service_healthy 70 | createbuckets: 71 | condition: service_completed_successfully 72 | command: ["./scripts/start-example.sh"] 73 | networks: 74 | - example 75 | 76 | integration-tests: 77 | build: *dockerfile-dev-build 78 | volumes: 79 | - .:/app 80 | env_file: .env 81 | ports: 82 | - "${DJANGO_BIND_PORT:-8080}:${DJANGO_BIND_PORT:-8080}" 83 | depends_on: 84 | db: 85 | condition: service_healthy 86 | rabbitmq: 87 | condition: service_healthy 88 | createbuckets: 89 | condition: service_completed_successfully 90 | command: ["./scripts/start-tests.sh"] 91 | networks: 92 | - example 93 | 94 | sonar: 95 | container_name: sonar 96 | image: sonarqube:9-community 97 | environment: 98 | - SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true 99 | ports: 100 | - "9000:9000" 101 | networks: 102 | - example 103 | 104 | sonar-cli: 105 | container_name: sonar-cli 106 | image: sonarsource/sonar-scanner-cli 107 | working_dir: /api 108 | environment: 109 | - SONAR_LOGIN=admin 110 | - SONAR_PASSWORD=test 111 | volumes: 112 | - .:/api 113 | command: ["sonar-scanner", "--debug"] 114 | networks: 115 | - example 116 | 117 | minio: 118 | image: minio/minio:latest 119 | ports: 120 | - "9000:9000" 121 | - "9001:9001" 122 | command: minio server /s3-folder --console-address 0.0.0.0:9001 123 | volumes: 124 | - s3-folder:/s3-folder 125 | healthcheck: 126 | test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] 127 | interval: 20s 128 | timeout: 10s 129 | retries: 3 130 | networks: 131 | - example 132 | 133 | createbuckets: 134 | image: minio/mc 135 | depends_on: 136 | minio: 137 | condition: service_healthy 138 | volumes: 139 | - s3-folder:/s3-folder 140 | entrypoint: > 141 | /bin/sh -c " 142 | /usr/bin/mc config host add s3-folder http://minio:9000 root password; 143 | /usr/bin/mc rm -r --force s3-folder/example; 144 | /usr/bin/mc mb s3-folder/example; 145 | /usr/bin/mc policy -r set download s3-folder/example; 146 | exit 0; 147 | " 148 | networks: 149 | - example 150 | 151 | lint-formatter: 152 | build: *dockerfile-dev-build 153 | volumes: 154 | - .:/app 155 | command: ["./scripts/start-formatter-lint.sh"] 156 | networks: 157 | - example 158 | 159 | volumes: 160 | s3-folder: null 161 | 162 | networks: 163 | example: 164 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | build/ 4 | dist/ 5 | *.sqlite3 6 | *__pycache__* 7 | example/django-import-export-stomp-import-change-summaries/ 8 | example/django-import-export-stomp-import-jobs/ 9 | example/django-import-export-stomp-export-jobs/ 10 | pyenv/ 11 | django_import_export_stomp.egg-info/ 12 | db/ 13 | .idea/ 14 | 15 | # Byte-compiled / optimized / DLL files 16 | __pycache__/ 17 | *.py[cod] 18 | *$py.class 19 | 20 | # C extensions 21 | *.so 22 | 23 | # Distribution / packaging 24 | .Python 25 | build/ 26 | develop-eggs/ 27 | dist/ 28 | downloads/ 29 | eggs/ 30 | .eggs/ 31 | lib/ 32 | lib64/ 33 | parts/ 34 | sdist/ 35 | var/ 36 | wheels/ 37 | share/python-wheels/ 38 | *.egg-info/ 39 | .installed.cfg 40 | *.egg 41 | MANIFEST 42 | 43 | # PyInstaller 44 | # Usually these files are written by a python script from a template 45 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 46 | *.manifest 47 | *.spec 48 | 49 | # Installer logs 50 | pip-log.txt 51 | pip-delete-this-directory.txt 52 | 53 | # Unit test / coverage reports 54 | htmlcov/ 55 | .tox/ 56 | .nox/ 57 | .coverage 58 | .coverage.* 59 | .cache 60 | nosetests.xml 61 | coverage.xml 62 | *.cover 63 | *.py,cover 64 | .hypothesis/ 65 | .pytest_cache/ 66 | cover/ 67 | .scannerwork 68 | 69 | # Translations 70 | *.mo 71 | *.pot 72 | 73 | # Django stuff: 74 | *.log 75 | local_settings.py 76 | db.sqlite3 77 | db.sqlite3-journal 78 | 79 | # Flask stuff: 80 | instance/ 81 | .webassets-cache 82 | 83 | # Scrapy stuff: 84 | .scrapy 85 | 86 | # Sphinx documentation 87 | docs/_build/ 88 | 89 | # PyBuilder 90 | .pybuilder/ 91 | target/ 92 | 93 | # Jupyter Notebook 94 | .ipynb_checkpoints 95 | 96 | # IPython 97 | profile_default/ 98 | ipython_config.py 99 | 100 | # pyenv 101 | # For a library or package, you might want to ignore these files since the code is 102 | # intended to run in multiple environments; otherwise, check them in: 103 | # .python-version 104 | 105 | # pipenv 106 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 107 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 108 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 109 | # install all needed dependencies. 110 | #Pipfile.lock 111 | 112 | # poetry 113 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 114 | # This is especially recommended for binary packages to ensure reproducibility, and is more 115 | # commonly ignored for libraries. 116 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 117 | #poetry.lock 118 | 119 | # pdm 120 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 121 | #pdm.lock 122 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 123 | # in version control. 124 | # https://pdm.fming.dev/#use-with-ide 125 | .pdm.toml 126 | 127 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 128 | __pypackages__/ 129 | 130 | # Celery stuff 131 | celerybeat-schedule 132 | celerybeat.pid 133 | 134 | # SageMath parsed files 135 | *.sage.py 136 | 137 | # Environments 138 | .venv 139 | env/ 140 | venv/ 141 | ENV/ 142 | env.bak/ 143 | venv.bak/ 144 | 145 | # Spyder project settings 146 | .spyderproject 147 | .spyproject 148 | 149 | # Rope project settings 150 | .ropeproject 151 | 152 | # mkdocs documentation 153 | /site 154 | 155 | # mypy 156 | .mypy_cache/ 157 | .dmypy.json 158 | dmypy.json 159 | 160 | # Pyre type checker 161 | .pyre/ 162 | 163 | # pytype static type analyzer 164 | .pytype/ 165 | 166 | # Cython debug symbols 167 | cython_debug/ 168 | 169 | # PyCharm 170 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 171 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 172 | # and can be added to the global gitignore or merged into this file. For a more nuclear 173 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 174 | #.idea/ 175 | 176 | .vscode/ 177 | .vscode/* 178 | !.vscode/settings.json 179 | !.vscode/tasks.json 180 | !.vscode/launch.json 181 | !.vscode/extensions.json 182 | !.vscode/*.code-snippets 183 | 184 | # Local History for Visual Studio Code 185 | .history/ 186 | 187 | # Built Visual Studio Code Extensions 188 | *.vsix 189 | 190 | # Tests 191 | tests-reports 192 | -------------------------------------------------------------------------------- /import_export_stomp/models/exportjob.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from functools import partial 4 | 5 | from author.decorators import with_author 6 | from django.contrib.contenttypes.models import ContentType 7 | from django.db import models 8 | from django.db import transaction 9 | from django.db.models.signals import post_save 10 | from django.dispatch import receiver 11 | from django.utils import timezone 12 | from django.utils.translation import gettext_lazy as _ 13 | 14 | from import_export_stomp.fields import ImportExportFileField 15 | from import_export_stomp.utils import get_formats 16 | from import_export_stomp.utils import send_job_message_to_queue 17 | 18 | 19 | @with_author 20 | class ExportJob(models.Model): 21 | def __init__(self, *args, **kwargs): 22 | super().__init__(*args, **kwargs) 23 | self._content_type = None 24 | 25 | file = ImportExportFileField( 26 | verbose_name=_("exported file"), 27 | upload_to="django-import-export-stomp-export-jobs", 28 | blank=False, 29 | null=False, 30 | max_length=255, 31 | ) 32 | 33 | processing_initiated = models.DateTimeField( 34 | verbose_name=_("Have we started processing the file? If so when?"), 35 | null=True, 36 | blank=True, 37 | default=None, 38 | ) 39 | 40 | job_status = models.CharField( 41 | verbose_name=_("Status of the job"), 42 | max_length=160, 43 | blank=True, 44 | ) 45 | 46 | format = models.CharField( 47 | verbose_name=_("Format of file to be exported"), 48 | max_length=255, 49 | blank=False, 50 | null=True, 51 | ) 52 | 53 | app_label = models.CharField( 54 | verbose_name=_("App label of model to export from"), 55 | max_length=160, 56 | ) 57 | 58 | model = models.CharField( 59 | verbose_name=_("Name of model to export from"), 60 | max_length=160, 61 | ) 62 | 63 | resource = models.CharField( 64 | verbose_name=_("Resource to use when exporting"), 65 | max_length=255, 66 | default="", 67 | ) 68 | 69 | queryset = models.TextField( 70 | verbose_name=_("JSON list of pks to export"), 71 | null=False, 72 | ) 73 | 74 | site_of_origin = models.TextField( 75 | verbose_name=_("Site of origin"), 76 | max_length=255, 77 | default="", 78 | ) 79 | 80 | class Meta: 81 | verbose_name = _("Export job") 82 | verbose_name_plural = _("Export jobs") 83 | app_label = "import_export_stomp" 84 | 85 | def get_resource_class(self): 86 | if self.resource: 87 | return ( 88 | self.get_content_type() 89 | .model_class() 90 | .export_resource_classes()[self.resource][1] 91 | ) 92 | 93 | def get_content_type(self): 94 | if not self._content_type: 95 | self._content_type = ContentType.objects.get( 96 | app_label=self.app_label, 97 | model=self.model, 98 | ) 99 | return self._content_type 100 | 101 | def get_queryset(self): 102 | pks = json.loads(self.queryset) 103 | # If customised queryset for the model exists 104 | # then it'll apply filter on that otherwise it'll 105 | # apply filter directly on the model. 106 | resource_class = self.get_resource_class() 107 | if hasattr(resource_class, "get_export_queryset"): 108 | return resource_class().get_export_queryset().filter(pk__in=pks) 109 | return self.get_content_type().model_class().objects.filter(pk__in=pks) 110 | 111 | def get_resource_choices(self): 112 | return [ 113 | (k, v[0]) 114 | for k, v in self.get_content_type() 115 | .model_class() 116 | .export_resource_classes() 117 | .items() 118 | ] 119 | 120 | @staticmethod 121 | def get_format_choices(): 122 | """returns choices of available export formats""" 123 | return [ 124 | (f.CONTENT_TYPE, f().get_title()) for f in get_formats() if f().can_export() 125 | ] 126 | 127 | 128 | @receiver(post_save, sender=ExportJob) 129 | def exportjob_post_save(sender, instance, **kwargs): 130 | if instance.resource and not instance.processing_initiated: 131 | instance.processing_initiated = timezone.now() 132 | instance.save() 133 | transaction.on_commit( 134 | partial(send_job_message_to_queue, action="export", job_id=instance.pk) 135 | ) 136 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0003_exportjob.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.4 on 2019-11-13 11:27 2 | 3 | import django.db.models.deletion 4 | 5 | from django.conf import settings 6 | from django.db import migrations 7 | from django.db import models 8 | 9 | 10 | class Migration(migrations.Migration): 11 | dependencies = [ 12 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 13 | ("import_export_stomp", "0002_auto_20190923_1132"), 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name="ExportJob", 19 | fields=[ 20 | ( 21 | "id", 22 | models.AutoField( 23 | auto_created=True, 24 | primary_key=True, 25 | serialize=False, 26 | verbose_name="ID", 27 | ), 28 | ), 29 | ( 30 | "file", 31 | models.FileField( 32 | max_length=255, 33 | upload_to="django-import-export-stomp-export-jobs", 34 | verbose_name="exported file", 35 | ), 36 | ), 37 | ( 38 | "processing_initiated", 39 | models.DateTimeField( 40 | blank=True, 41 | default=None, 42 | null=True, 43 | verbose_name="Have we started processing the file? If so when?", 44 | ), 45 | ), 46 | ( 47 | "format", 48 | models.CharField( 49 | choices=[ 50 | ("text/csv", "text/csv"), 51 | ("application/vnd.ms-excel", "application/vnd.ms-excel"), 52 | ( 53 | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 54 | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 55 | ), 56 | ("text/tab-separated-values", "text/tab-separated-values"), 57 | ( 58 | "application/vnd.oasis.opendocument.spreadsheet", 59 | "application/vnd.oasis.opendocument.spreadsheet", 60 | ), 61 | ("application/json", "application/json"), 62 | ("text/yaml", "text/yaml"), 63 | ("text/html", "text/html"), 64 | ], 65 | max_length=40, 66 | null=True, 67 | verbose_name="Format of file to be exported", 68 | ), 69 | ), 70 | ( 71 | "app_label", 72 | models.CharField( 73 | max_length=160, verbose_name="App label of model to export from" 74 | ), 75 | ), 76 | ( 77 | "model", 78 | models.CharField( 79 | max_length=160, verbose_name="Name of model to export from" 80 | ), 81 | ), 82 | ( 83 | "resource", 84 | models.CharField( 85 | default="", 86 | max_length=255, 87 | verbose_name="Resource to use when exporting", 88 | ), 89 | ), 90 | ( 91 | "queryset", 92 | models.TextField(verbose_name="JSON list of pks to export"), 93 | ), 94 | ( 95 | "author", 96 | models.ForeignKey( 97 | blank=True, 98 | null=True, 99 | on_delete=django.db.models.deletion.SET_NULL, 100 | related_name="exportjob_create", 101 | to=settings.AUTH_USER_MODEL, 102 | verbose_name="author", 103 | ), 104 | ), 105 | ( 106 | "updated_by", 107 | models.ForeignKey( 108 | blank=True, 109 | null=True, 110 | on_delete=django.db.models.deletion.SET_NULL, 111 | related_name="exportjob_update", 112 | to=settings.AUTH_USER_MODEL, 113 | verbose_name="last updated by", 114 | ), 115 | ), 116 | ], 117 | ), 118 | ] 119 | -------------------------------------------------------------------------------- /import_export_stomp/locale/eu/LC_MESSAGES/django.po: -------------------------------------------------------------------------------- 1 | # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER 2 | # This file is distributed under the same license as the PACKAGE package. 3 | # 4 | # Urtzi Odriozola , 2023. 5 | msgid "" 6 | msgstr "" 7 | "Project-Id-Version: \n" 8 | "Report-Msgid-Bugs-To: \n" 9 | "POT-Creation-Date: 2023-05-17 20:55+0000\n" 10 | "PO-Revision-Date: 2023-06-07 21:56+0100\n" 11 | "Last-Translator: Urtzi Odriozola \n" 12 | "Language-Team: \n" 13 | "Language: eu\n" 14 | "MIME-Version: 1.0\n" 15 | "Content-Type: text/plain; charset=UTF-8\n" 16 | "Content-Transfer-Encoding: 8bit\n" 17 | "Plural-Forms: nplurals=2; plural=(n > 1);\n" 18 | 19 | #: import_export_stomp/admin.py:12 20 | msgid "Job status info" 21 | msgstr "Lanaren egoera" 22 | 23 | #: import_export_stomp/admin.py:22 import_export_stomp/models/importjob.py:62 24 | msgid "Name of model to import to" 25 | msgstr "Inportatu beharreko modeloaren izena" 26 | 27 | #: import_export_stomp/admin_actions.py:20 28 | msgid "Perform import" 29 | msgstr "Hasi inportazioa" 30 | 31 | #: import_export_stomp/admin_actions.py:29 32 | msgid "Perform dry import" 33 | msgstr "Hasi probako inportazioa" 34 | 35 | #: import_export_stomp/admin_actions.py:39 36 | msgid "Run export job" 37 | msgstr "Hasi esportazio lana" 38 | 39 | #: import_export_stomp/admin_actions.py:67 40 | msgid "Export with stomp" 41 | msgstr "Exportatu stomp bidez" 42 | 43 | #: import_export_stomp/apps.py:7 44 | msgid "Import Export Stomp" 45 | msgstr "Inportatu - Esportatu stomp bidez" 46 | 47 | #: import_export_stomp/models/exportjob.py:27 48 | msgid "exported file" 49 | msgstr "esportatutako fitxategia" 50 | 51 | #: import_export_stomp/models/exportjob.py:35 52 | #: import_export_stomp/models/importjob.py:30 53 | msgid "Have we started processing the file? If so when?" 54 | msgstr "Fitxategia prozesatzen hasi gara? Hala bada, noiz?" 55 | 56 | #: import_export_stomp/models/exportjob.py:42 57 | #: import_export_stomp/models/importjob.py:67 58 | msgid "Status of the job" 59 | msgstr "Lanaren egoera" 60 | 61 | #: import_export_stomp/models/exportjob.py:48 62 | msgid "Format of file to be exported" 63 | msgstr "Esportatu beharreko fitxategiaren formatua" 64 | 65 | #: import_export_stomp/models/exportjob.py:55 66 | msgid "App label of model to export from" 67 | msgstr "Esportatu beharreko modeloaren aplikazio izena" 68 | 69 | #: import_export_stomp/models/exportjob.py:60 70 | msgid "Name of model to export from" 71 | msgstr "Esportatu beharreko modeloaren izena" 72 | 73 | #: import_export_stomp/models/exportjob.py:65 74 | msgid "Resource to use when exporting" 75 | msgstr "Esportatzeko erabili beharreko baliabidea" 76 | 77 | #: import_export_stomp/models/exportjob.py:71 78 | msgid "JSON list of pks to export" 79 | msgstr "Esportatzeko ID-en JSON zerrenda" 80 | 81 | #: import_export_stomp/models/exportjob.py:76 82 | msgid "Send me an email when this export job is complete" 83 | msgstr "Bidali eposta bat esportazio lan hau amaitzen denean" 84 | 85 | #: import_export_stomp/models/exportjob.py:81 86 | msgid "Site of origin" 87 | msgstr "Jatorrizko gunea" 88 | 89 | #: import_export_stomp/models/exportjob.py:87 90 | msgid "Export job" 91 | msgstr "Esportazio lana" 92 | 93 | #: import_export_stomp/models/exportjob.py:88 94 | msgid "Export jobs" 95 | msgstr "Esportazio lanak" 96 | 97 | #: import_export_stomp/models/importjob.py:22 98 | msgid "File to be imported" 99 | msgstr "Inportatu beharreko fitxategia" 100 | 101 | #: import_export_stomp/models/importjob.py:37 102 | msgid "Has the import been completed? If so when?" 103 | msgstr "Inportazioa amaitu da? Hala bada, noiz?" 104 | 105 | #: import_export_stomp/models/importjob.py:44 106 | msgid "Format of file to be imported" 107 | msgstr "Inportatu beharreko fitxategiaren formatua" 108 | 109 | #: import_export_stomp/models/importjob.py:49 110 | msgid "Summary of changes made by this import" 111 | msgstr "Inportazio honek eginiko aldaketen laburpena" 112 | 113 | #: import_export_stomp/models/importjob.py:56 114 | msgid "Errors" 115 | msgstr "Erroreak" 116 | 117 | #: import_export_stomp/models/importjob.py:73 118 | msgid "Import job" 119 | msgstr "Inportazio lana" 120 | 121 | #: import_export_stomp/models/importjob.py:74 122 | msgid "Import jobs" 123 | msgstr "Inportazio lanak" 124 | 125 | #: import_export_stomp/tasks.py:61 126 | #, python-format 127 | msgid "Imported file has a wrong encoding: %s" 128 | msgstr "Inportatutako fitxategiak kodetze okerra du: %s" 129 | 130 | #: import_export_stomp/tasks.py:68 131 | #, python-format 132 | msgid "Error reading file: %s" 133 | msgstr "Errorea fitxategia irakurtzean: %s" 134 | 135 | #: import_export_stomp/tasks.py:101 136 | #, python-format 137 | msgid "" 138 | "Line: %s - %s\n" 139 | "\t%s\n" 140 | "%s" 141 | msgstr "" 142 | "Lerroa: %s - %s\n" 143 | "\t%s\n" 144 | "%s" 145 | 146 | #: import_export_stomp/tasks.py:190 147 | #, python-format 148 | msgid "Import error %s" 149 | msgstr "Inportazio errorea %s" 150 | -------------------------------------------------------------------------------- /import_export_stomp/locale/pt/LC_MESSAGES/django.po: -------------------------------------------------------------------------------- 1 | # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER 2 | # This file is distributed under the same license as the PACKAGE package. 3 | # 4 | # Dan , 2020. 5 | msgid "" 6 | msgstr "" 7 | "Project-Id-Version: \n" 8 | "Report-Msgid-Bugs-To: \n" 9 | "POT-Creation-Date: 2023-05-17 20:55+0000\n" 10 | "PO-Revision-Date: 2023-05-17 21:56+0100\n" 11 | "Last-Translator: Daniel Pluth \n" 12 | "Language-Team: \n" 13 | "Language: pt_BR\n" 14 | "MIME-Version: 1.0\n" 15 | "Content-Type: text/plain; charset=UTF-8\n" 16 | "Content-Transfer-Encoding: 8bit\n" 17 | "Plural-Forms: nplurals=2; plural=(n > 1);\n" 18 | "X-Generator: Poedit 3.3.1\n" 19 | 20 | #: import_export_stomp/admin.py:12 21 | msgid "Job status info" 22 | msgstr "Informação do estado do trabalho" 23 | 24 | #: import_export_stomp/admin.py:22 import_export_stomp/models/importjob.py:62 25 | msgid "Name of model to import to" 26 | msgstr "Nome do modelo a importar" 27 | 28 | #: import_export_stomp/admin_actions.py:20 29 | msgid "Perform import" 30 | msgstr "Confirmar importação" 31 | 32 | #: import_export_stomp/admin_actions.py:29 33 | msgid "Perform dry import" 34 | msgstr "Confirmar importação" 35 | 36 | #: import_export_stomp/admin_actions.py:39 37 | msgid "Run export job" 38 | msgstr "Correr trabalho de exportação" 39 | 40 | #: import_export_stomp/admin_actions.py:67 41 | msgid "Export with stomp" 42 | msgstr "Exportar com o stomp" 43 | 44 | #: import_export_stomp/apps.py:7 45 | msgid "Import Export Stomp" 46 | msgstr "Importar ou Exportar com o stomp" 47 | 48 | #: import_export_stomp/models/exportjob.py:27 49 | msgid "exported file" 50 | msgstr "ficheiro exportado" 51 | 52 | #: import_export_stomp/models/exportjob.py:35 53 | #: import_export_stomp/models/importjob.py:30 54 | msgid "Have we started processing the file? If so when?" 55 | msgstr "Começou a processar o ficheiro? Se sim, quando?" 56 | 57 | #: import_export_stomp/models/exportjob.py:42 58 | #: import_export_stomp/models/importjob.py:67 59 | msgid "Status of the job" 60 | msgstr "Estado do trabalho" 61 | 62 | #: import_export_stomp/models/exportjob.py:48 63 | msgid "Format of file to be exported" 64 | msgstr "Formato do ficheiro a ser exportado" 65 | 66 | #: import_export_stomp/models/exportjob.py:55 67 | msgid "App label of model to export from" 68 | msgstr "Nome da aplicação do modelo para exportar de" 69 | 70 | #: import_export_stomp/models/exportjob.py:60 71 | msgid "Name of model to export from" 72 | msgstr "Nome do modelo a exportar de" 73 | 74 | #: import_export_stomp/models/exportjob.py:65 75 | msgid "Resource to use when exporting" 76 | msgstr "Recurso a utilizar aquando da exportação" 77 | 78 | #: import_export_stomp/models/exportjob.py:71 79 | msgid "JSON list of pks to export" 80 | msgstr "Lista de ids em formato json para exportar" 81 | 82 | #: import_export_stomp/models/exportjob.py:76 83 | msgid "Send me an email when this export job is complete" 84 | msgstr "Enviar um email quando o trabalho de exportação estiver completo" 85 | 86 | #: import_export_stomp/models/exportjob.py:81 87 | msgid "Site of origin" 88 | msgstr "Site de origem" 89 | 90 | #: import_export_stomp/models/exportjob.py:87 91 | msgid "Export job" 92 | msgstr "Trabalho de exportação" 93 | 94 | #: import_export_stomp/models/exportjob.py:88 95 | msgid "Export jobs" 96 | msgstr "Trabalhos de exportação" 97 | 98 | #: import_export_stomp/models/importjob.py:22 99 | msgid "File to be imported" 100 | msgstr "Arquivo a ser importado" 101 | 102 | #: import_export_stomp/models/importjob.py:37 103 | msgid "Has the import been completed? If so when?" 104 | msgstr "A importação foi completa? Se sim, quando?" 105 | 106 | #: import_export_stomp/models/importjob.py:44 107 | msgid "Format of file to be imported" 108 | msgstr "Formato do ficheiro a ser importado" 109 | 110 | #: import_export_stomp/models/importjob.py:49 111 | msgid "Summary of changes made by this import" 112 | msgstr "Resumo das alterações feitas por esta importação" 113 | 114 | #: import_export_stomp/models/importjob.py:56 115 | msgid "Errors" 116 | msgstr "Erros" 117 | 118 | #: import_export_stomp/models/importjob.py:73 119 | msgid "Import job" 120 | msgstr "Trabalho de importação" 121 | 122 | #: import_export_stomp/models/importjob.py:74 123 | msgid "Import jobs" 124 | msgstr "Trabalhos de importação" 125 | 126 | #: import_export_stomp/tasks.py:61 127 | #, python-format 128 | msgid "Imported file has a wrong encoding: %s" 129 | msgstr "O arquivo importado tem uma codificação errada: %s" 130 | 131 | #: import_export_stomp/tasks.py:68 132 | #, python-format 133 | msgid "Error reading file: %s" 134 | msgstr "Erro a ler o ficheiro: %s" 135 | 136 | #: import_export_stomp/tasks.py:101 137 | #, python-format 138 | msgid "" 139 | "Line: %s - %s\n" 140 | "\t%s\n" 141 | "%s" 142 | msgstr "" 143 | "Linha: %s - %s\n" 144 | "\t%s\n" 145 | "%s" 146 | 147 | #: import_export_stomp/tasks.py:190 148 | #, python-format 149 | msgid "Import error %s" 150 | msgstr "Erro de importação %s" 151 | -------------------------------------------------------------------------------- /import_export_stomp/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.12 on 2019-06-28 13:55 2 | 3 | import django.db.models.deletion 4 | 5 | from django.conf import settings 6 | from django.db import migrations 7 | from django.db import models 8 | 9 | 10 | class Migration(migrations.Migration): 11 | initial = True 12 | 13 | dependencies = [ 14 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 15 | ] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name="ImportJob", 20 | fields=[ 21 | ( 22 | "id", 23 | models.AutoField( 24 | auto_created=True, 25 | primary_key=True, 26 | serialize=False, 27 | verbose_name="ID", 28 | ), 29 | ), 30 | ( 31 | "file", 32 | models.FileField( 33 | max_length=255, 34 | upload_to="django-import-export-stomp-import-jobs", 35 | verbose_name="File to be imported", 36 | ), 37 | ), 38 | ( 39 | "processing_initiated", 40 | models.DateTimeField( 41 | blank=True, 42 | default=None, 43 | null=True, 44 | verbose_name="Have we started processing the file? If so when?", 45 | ), 46 | ), 47 | ( 48 | "imported", 49 | models.DateTimeField( 50 | blank=True, 51 | default=None, 52 | null=True, 53 | verbose_name="Has the import been completed? If so when?", 54 | ), 55 | ), 56 | ( 57 | "format", 58 | models.CharField( 59 | choices=[ 60 | ("text/csv", "text/csv"), 61 | ("application/vnd.ms-excel", "application/vnd.ms-excel"), 62 | ( 63 | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 64 | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 65 | ), 66 | ("text/tab-separated-values", "text/tab-separated-values"), 67 | ( 68 | "application/vnd.oasis.opendocument.spreadsheet", 69 | "application/vnd.oasis.opendocument.spreadsheet", 70 | ), 71 | ("application/json", "application/json"), 72 | ("text/yaml", "text/yaml"), 73 | ("text/html", "text/html"), 74 | ], 75 | max_length=40, 76 | verbose_name="Format of file to be imported", 77 | ), 78 | ), 79 | ( 80 | "change_summary", 81 | models.FileField( 82 | blank=True, 83 | null=True, 84 | upload_to="django-import-export-stomp-import-change-summaries", 85 | verbose_name="Summary of changes made by this import", 86 | ), 87 | ), 88 | ("errors", models.TextField(blank=True, default="")), 89 | ( 90 | "model", 91 | models.CharField( 92 | choices=[("Winner", "Winner")], 93 | max_length=160, 94 | verbose_name="Name of model to import to", 95 | ), 96 | ), 97 | ( 98 | "author", 99 | models.ForeignKey( 100 | blank=True, 101 | null=True, 102 | on_delete=django.db.models.deletion.SET_NULL, 103 | related_name="importjob_create", 104 | to=settings.AUTH_USER_MODEL, 105 | verbose_name="author", 106 | ), 107 | ), 108 | ( 109 | "updated_by", 110 | models.ForeignKey( 111 | blank=True, 112 | null=True, 113 | on_delete=django.db.models.deletion.SET_NULL, 114 | related_name="importjob_update", 115 | to=settings.AUTH_USER_MODEL, 116 | verbose_name="last updated by", 117 | ), 118 | ), 119 | ], 120 | ), 121 | ] 122 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from logging import Formatter 4 | from pathlib import Path 5 | 6 | from import_export_stomp.apps import ImportExportStompConfig 7 | from import_export_stomp.resources import resource_importer 8 | from tests.resources.fake_app.apps import FakeAppConfig 9 | 10 | BASE_DIR = Path(__file__).resolve().parent.parent 11 | 12 | INSTALLED_APPS = [ 13 | "django.contrib.admin", 14 | "django.contrib.auth", 15 | "django.contrib.contenttypes", 16 | "django.contrib.sessions", 17 | "django.contrib.messages", 18 | "django.contrib.sites", 19 | "author", 20 | ImportExportStompConfig.name, 21 | FakeAppConfig.name, 22 | ] 23 | 24 | SITE_ID = 1 25 | 26 | ROOT_URLCONF = "tests.urls" 27 | 28 | DEBUG = True 29 | 30 | STATIC_URL = "/static/" 31 | 32 | SECRET_KEY = "2n6)=vnp8@bu0om9d05vwf7@=5vpn%)97-!d*t4zq1mku%0-@j" 33 | 34 | MIDDLEWARE = ( 35 | "django.contrib.sessions.middleware.SessionMiddleware", 36 | "django.middleware.common.CommonMiddleware", 37 | "django.middleware.csrf.CsrfViewMiddleware", 38 | "django.contrib.auth.middleware.AuthenticationMiddleware", 39 | "django.contrib.messages.middleware.MessageMiddleware", 40 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 41 | "author.middlewares.AuthorDefaultBackendMiddleware", 42 | ) 43 | 44 | TEMPLATES = [ 45 | { 46 | "BACKEND": "django.template.backends.django.DjangoTemplates", 47 | "DIRS": [], 48 | "APP_DIRS": True, 49 | "OPTIONS": { 50 | "context_processors": ( 51 | "django.contrib.auth.context_processors.auth", 52 | "django.contrib.messages.context_processors.messages", 53 | "django.template.context_processors.request", 54 | ), 55 | }, 56 | }, 57 | ] 58 | 59 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 60 | 61 | DATABASE_SSL_MODE = os.getenv("DATABASE_SSL_MODE") 62 | 63 | DATABASES = { 64 | "default": { 65 | "ENGINE": os.getenv("DATABASE_ENGINE", "django.db.backends.sqlite3"), 66 | "NAME": os.getenv("DATABASE_NAME", BASE_DIR / "db.sqlite3"), 67 | "USER": os.environ.get("DATABASE_USER"), 68 | "HOST": os.environ.get("DATABASE_HOST"), 69 | "PORT": os.environ.get("DATABASE_PORT"), 70 | "PASSWORD": os.environ.get("DATABASE_PASSWORD"), 71 | } 72 | } 73 | 74 | LOGGING = { 75 | "version": 1, 76 | "disable_existing_loggers": False, 77 | "formatters": { 78 | "standard": { 79 | "()": Formatter, 80 | "format": "%(levelname)-8s [%(asctime)s] %(name)s: %(message)s", 81 | } 82 | }, 83 | "handlers": { 84 | "console": { 85 | "class": "logging.StreamHandler", 86 | "formatter": "standard", 87 | } 88 | }, 89 | "loggers": { 90 | "": {"level": os.getenv("ROOT_LOG_LEVEL", "INFO"), "handlers": ["console"]}, 91 | "django": { 92 | "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"), 93 | "propagate": False, 94 | "handlers": ["console"], 95 | }, 96 | "django.request": { 97 | "level": os.getenv("DJANGO_REQUEST_LOG_LEVEL", "INFO"), 98 | "handlers": ["console"], 99 | "propagate": False, 100 | }, 101 | "django.db.backends": { 102 | "level": os.getenv("DJANGO_DB_BACKENDS_LOG_LEVEL", "INFO"), 103 | "propagate": False, 104 | "handlers": ["console"], 105 | }, 106 | "stomp.py": { 107 | "level": os.getenv("STOMP_LOG_LEVEL", "WARNING"), 108 | "handlers": ["console"], 109 | "propagate": False, 110 | }, 111 | }, 112 | } 113 | 114 | 115 | USE_TZ = True 116 | 117 | # STOMP SETTINGS 118 | STOMP_LISTENER_CLIENT_ID = os.getenv("STOMP_LISTENER_CLIENT_ID") 119 | STOMP_SERVER_HOST = os.getenv("STOMP_SERVER_HOST") 120 | STOMP_SERVER_PORT = os.getenv("STOMP_SERVER_PORT") 121 | STOMP_SERVER_STANDBY_HOST = os.getenv("STOMP_SERVER_STANDBY_HOST") 122 | STOMP_SERVER_STANDBY_PORT = os.getenv("STOMP_SERVER_STANDBY_PORT") 123 | STOMP_SERVER_USER = os.getenv("STOMP_SERVER_USER") 124 | STOMP_SERVER_PASSWORD = os.getenv("STOMP_SERVER_PASSWORD") 125 | STOMP_USE_SSL = bool(os.getenv("STOMP_USE_SSL", True)) 126 | STOMP_SERVER_VHOST = os.getenv("STOMP_SERVER_VHOST") 127 | STOMP_OUTGOING_HEARTBEAT = os.getenv("STOMP_OUTGOING_HEARTBEAT", 15000) 128 | STOMP_INCOMING_HEARTBEAT = os.getenv("STOMP_INCOMING_HEARTBEAT", 15000) 129 | 130 | # AWS STORAGE 131 | IMPORT_EXPORT_STOMP_STORAGE = os.getenv( 132 | "IMPORT_EXPORT_STOMP_STORAGE", "django.core.files.storage.FileSystemStorage" 133 | ) 134 | 135 | 136 | AWS_ACCESS_KEY_ID = "minioadmin" 137 | AWS_SECRET_ACCESS_KEY = "minioadmin" 138 | AWS_DEFAULT_REGION = "us-east-1" 139 | AWS_STORAGE_BUCKET_NAME = "example" 140 | AWS_S3_FILE_OVERWRITE = False 141 | AWS_S3_ENDPOINT_URL = "http://minio:9000" 142 | IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST = os.getenv( 143 | "IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST", False 144 | ) 145 | 146 | 147 | # DJANGO IMPORT EXPORT STOMP 148 | IMPORT_EXPORT_STOMP_MODELS = { 149 | "Test": { 150 | "app_label": "fake_app", 151 | "model_name": "FakeModel", 152 | "resource": resource_importer( 153 | "tests.resources.fake_app.resources.FakeResource" 154 | ), 155 | } 156 | } 157 | 158 | IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST = True 159 | IMPORT_EXPORT_STOMP_PRESIGNED_POST_EXPIRATION = 600 160 | IMPORT_EXPORT_STOMP_PRESIGNED_FOLDER = "import_export_stomp/" 161 | -------------------------------------------------------------------------------- /import_export_stomp/admin.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from typing import Optional 3 | 4 | from django import forms 5 | from django.conf import settings 6 | from django.contrib import admin 7 | from django.core.cache import cache 8 | from django.http.request import HttpRequest 9 | from django.utils.translation import gettext_lazy as _ 10 | 11 | from import_export_stomp import admin_actions 12 | from import_export_stomp.models import ExportJob 13 | from import_export_stomp.models import ImportJob 14 | from import_export_stomp.widgets import SignedUrlFileInput 15 | 16 | IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST = getattr( 17 | settings, "IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST", False 18 | ) 19 | 20 | 21 | class JobWithStatusMixin: 22 | @admin.display(description=_("Job status info")) 23 | def job_status_info(self, obj): 24 | job_status = cache.get(self.direction + "_job_status_%s" % obj.pk) 25 | if job_status: 26 | return job_status 27 | else: 28 | return obj.job_status 29 | 30 | 31 | class ImportJobForm(forms.ModelForm): 32 | model = forms.ChoiceField(label=_("Name of model to import to")) 33 | signed_url_file_key = forms.CharField( 34 | max_length=255, 35 | required=True, 36 | ) 37 | 38 | class Meta: 39 | model = ImportJob 40 | fields = [ 41 | "file", 42 | "processing_initiated", 43 | "imported", 44 | "format", 45 | "change_summary", 46 | "errors", 47 | "model", 48 | "job_status", 49 | "signed_url_file_key", 50 | ] 51 | widgets = { 52 | "file": SignedUrlFileInput(attrs={"id": "signed_url_file_input"}), 53 | } 54 | 55 | def __init__(self, *args, **kwargs): 56 | super().__init__(*args, **kwargs) 57 | 58 | if not self.instance.pk: 59 | self.fields["model"].choices = [ 60 | (x, x) 61 | for x in getattr(settings, "IMPORT_EXPORT_STOMP_MODELS", {}).keys() 62 | ] 63 | self.fields["format"].widget = forms.Select( 64 | choices=self.instance.get_format_choices() 65 | ) 66 | 67 | self.fields["signed_url_file_key"].widget.attrs["style"] = "display: none;" 68 | self.fields["signed_url_file_key"].widget.attrs["readonly"] = True 69 | 70 | if ( 71 | IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST 72 | and self.changed_data 73 | and not self.instance.pk 74 | ): 75 | del self.fields["file"] 76 | elif not IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST: 77 | self.fields["signed_url_file_key"].required = False 78 | 79 | def save(self, commit: bool = True) -> Any: 80 | if IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST: 81 | self.instance.file = self.data["signed_url_file_key"] 82 | return super().save(commit) 83 | 84 | 85 | @admin.register(ImportJob) 86 | class ImportJobAdmin(JobWithStatusMixin, admin.ModelAdmin): 87 | direction = "import" 88 | form = ImportJobForm 89 | list_display = ( 90 | "model", 91 | "job_status_info", 92 | "file", 93 | "change_summary", 94 | "imported", 95 | "author", 96 | "updated_by", 97 | ) 98 | readonly_fields = ( 99 | "job_status_info", 100 | "change_summary", 101 | "imported", 102 | "errors", 103 | "author", 104 | "updated_by", 105 | "processing_initiated", 106 | ) 107 | exclude = ("job_status",) 108 | 109 | list_filter = ("model", "imported") 110 | 111 | actions = ( 112 | admin_actions.run_import_job_action, 113 | admin_actions.run_import_job_action_dry, 114 | ) 115 | 116 | def has_change_permission( 117 | self, request: HttpRequest, obj: Optional[ImportJob] = None 118 | ) -> bool: 119 | return False 120 | 121 | 122 | class ExportJobForm(forms.ModelForm): 123 | class Meta: 124 | model = ExportJob 125 | exclude = ("site_of_origin",) 126 | 127 | def __init__(self, *args, **kwargs): 128 | super().__init__(*args, **kwargs) 129 | if not getattr(self.instance, "processing_initiated", True): 130 | self.fields["resource"].widget = forms.Select( 131 | choices=self.instance.get_resource_choices() 132 | ) 133 | self.fields["format"].widget = forms.Select( 134 | choices=self.instance.get_format_choices() 135 | ) 136 | 137 | 138 | @admin.register(ExportJob) 139 | class ExportJobAdmin(JobWithStatusMixin, admin.ModelAdmin): 140 | direction = "export" 141 | form = ExportJobForm 142 | list_display = ( 143 | "model", 144 | "app_label", 145 | "file", 146 | "job_status_info", 147 | "author", 148 | "updated_by", 149 | ) 150 | readonly_fields = ( 151 | "job_status_info", 152 | "author", 153 | "updated_by", 154 | "app_label", 155 | "model", 156 | "file", 157 | "processing_initiated", 158 | ) 159 | exclude = ("job_status",) 160 | 161 | list_filter = ("model",) 162 | 163 | def has_add_permission( 164 | self, request: HttpRequest, obj: Optional[ExportJob] = None 165 | ) -> bool: 166 | return False 167 | 168 | def has_change_permission( 169 | self, request: HttpRequest, obj: Optional[ExportJob] = None 170 | ) -> bool: 171 | if getattr(obj, "processing_initiated", True): 172 | return False 173 | 174 | return True 175 | 176 | actions = (admin_actions.run_export_job_action,) 177 | -------------------------------------------------------------------------------- /example/project/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for winners project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.10.2. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.10/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | from typing import List 16 | 17 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 18 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 19 | 20 | 21 | # Quick-start development settings - unsuitable for production 22 | # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ 23 | 24 | # SECURITY WARNING: keep the secret key used in production secret! 25 | SECRET_KEY = "_omc6hxq40u11no0uvi&g__lzj2n^4-dk#l#i+7+vgng!-bb^)" 26 | 27 | # SECURITY WARNING: don't run with debug turned on in production! 28 | DEBUG = True 29 | 30 | ALLOWED_HOSTS: List[str] = [] 31 | 32 | 33 | # Application definition 34 | 35 | INSTALLED_APPS = [ 36 | "django.contrib.admin", 37 | "django.contrib.auth", 38 | "django.contrib.contenttypes", 39 | "django.contrib.sessions", 40 | "django.contrib.messages", 41 | "django.contrib.staticfiles", 42 | "django_stomp", 43 | "import_export", 44 | "import_export_stomp", 45 | "winners", 46 | ] 47 | 48 | MIDDLEWARE = [ 49 | "django.middleware.security.SecurityMiddleware", 50 | "django.contrib.sessions.middleware.SessionMiddleware", 51 | "django.middleware.common.CommonMiddleware", 52 | "django.middleware.csrf.CsrfViewMiddleware", 53 | "django.contrib.auth.middleware.AuthenticationMiddleware", 54 | "django.contrib.messages.middleware.MessageMiddleware", 55 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 56 | "author.middlewares.AuthorDefaultBackendMiddleware", 57 | ] 58 | 59 | ROOT_URLCONF = "winners.urls" 60 | 61 | TEMPLATES = [ 62 | { 63 | "BACKEND": "django.template.backends.django.DjangoTemplates", 64 | "DIRS": ["templates"], 65 | "APP_DIRS": True, 66 | "OPTIONS": { 67 | "context_processors": [ 68 | "django.template.context_processors.debug", 69 | "django.template.context_processors.request", 70 | "django.contrib.auth.context_processors.auth", 71 | "django.contrib.messages.context_processors.messages", 72 | ], 73 | }, 74 | }, 75 | ] 76 | 77 | WSGI_APPLICATION = "winners.wsgi.application" 78 | 79 | BROKER_URL = os.environ.get("REDIS_URL", "redis://redis") 80 | REDIS_URL = os.environ.get("REDIS_URL", "redis://redis") 81 | # Database 82 | # https://docs.djangoproject.com/en/1.10/ref/settings/#databases 83 | 84 | DATABASES = { 85 | "default": { 86 | "ENGINE": os.getenv("DATABASE_ENGINE", "django.db.backends.sqlite3"), 87 | "NAME": os.getenv("DATABASE_NAME", f"{BASE_DIR}/db.sqlite3"), 88 | "USER": os.environ.get("DATABASE_USER"), 89 | "HOST": os.environ.get("DATABASE_HOST"), 90 | "PORT": os.environ.get("DATABASE_PORT"), 91 | "PASSWORD": os.environ.get("DATABASE_PASSWORD"), 92 | } 93 | } 94 | 95 | 96 | # Password validation 97 | # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators 98 | 99 | AUTH_PASSWORD_VALIDATORS = [ 100 | { 101 | "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", 102 | }, 103 | { 104 | "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", 105 | }, 106 | { 107 | "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", 108 | }, 109 | { 110 | "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", 111 | }, 112 | ] 113 | 114 | 115 | # Internationalization 116 | # https://docs.djangoproject.com/en/1.10/topics/i18n/ 117 | 118 | LANGUAGE_CODE = "en-us" 119 | 120 | TIME_ZONE = "UTC" 121 | 122 | USE_I18N = True 123 | 124 | USE_L10N = True 125 | 126 | USE_TZ = True 127 | 128 | 129 | # Static files (CSS, JavaScript, Images) 130 | # https://docs.djangoproject.com/en/1.10/howto/static-files/ 131 | 132 | STATIC_URL = "/static/" 133 | MEDIA_URL = "/media/" 134 | MEDIA_ROOT = BASE_DIR 135 | 136 | # LOGS 137 | LOGGING = { 138 | "version": 1, 139 | "disable_existing_loggers": False, 140 | "handlers": {"console": {"class": "logging.StreamHandler"}}, 141 | "loggers": { 142 | "stomp.py": { 143 | "level": "INFO", 144 | "handlers": ["console"], 145 | "propagate": False, 146 | }, 147 | }, 148 | } 149 | 150 | 151 | # STOMP SETTINGS 152 | STOMP_LISTENER_CLIENT_ID = os.getenv("STOMP_LISTENER_CLIENT_ID") 153 | STOMP_SERVER_HOST = os.getenv("STOMP_SERVER_HOST") 154 | STOMP_SERVER_PORT = os.getenv("STOMP_SERVER_PORT") 155 | STOMP_SERVER_USER = os.getenv("STOMP_SERVER_USER") 156 | STOMP_SERVER_PASSWORD = os.getenv("STOMP_SERVER_PASSWORD") 157 | STOMP_USE_SSL = bool(eval(os.getenv("STOMP_USE_SSL", "False"))) 158 | STOMP_SERVER_VHOST = os.getenv("STOMP_SERVER_VHOST") 159 | STOMP_OUTGOING_HEARTBEAT = os.getenv("STOMP_OUTGOING_HEARTBEAT", 15000) 160 | STOMP_INCOMING_HEARTBEAT = os.getenv("STOMP_INCOMING_HEARTBEAT", 15000) 161 | 162 | # AWS STORAGE 163 | IMPORT_EXPORT_STOMP_STORAGE = os.getenv( 164 | "IMPORT_EXPORT_STOMP_STORAGE", "django.core.files.storage.FileSystemStorage" 165 | ) 166 | 167 | AWS_ACCESS_KEY_ID = "minioadmin" 168 | AWS_SECRET_ACCESS_KEY = "minioadmin" 169 | AWS_DEFAULT_REGION = "us-east-1" 170 | AWS_STORAGE_BUCKET_NAME = "example" 171 | AWS_S3_FILE_OVERWRITE = False 172 | IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST = os.getenv( 173 | "IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST", False 174 | ) 175 | 176 | AWS_S3_ENDPOINT_URL = os.getenv( 177 | "AWS_S3_ENDPOINT_URL", 178 | ( 179 | "http://127.0.0.1:9000" 180 | if IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST 181 | else "http://minio:9000" 182 | ), 183 | ) 184 | 185 | # DJANGO IMPORT EXPORT STOMP 186 | IMPORT_EXPORT_STOMP_MODELS = { 187 | "Winner": {"app_label": "winners", "model_name": "Winner"} 188 | } 189 | IMPORT_EXPORT_STOMP_PRESIGNED_POST_EXPIRATION = 600 190 | IMPORT_EXPORT_STOMP_PRESIGNED_FOLDER = "import_export_stomp/" 191 | -------------------------------------------------------------------------------- /tests/unit/test_pubsub.py: -------------------------------------------------------------------------------- 1 | from contextlib import nullcontext 2 | from functools import partial 3 | from typing import Callable 4 | from typing import Dict 5 | from unittest import mock 6 | 7 | import pytest 8 | 9 | from model_bakery import baker 10 | 11 | import import_export_stomp.pubsub 12 | 13 | from import_export_stomp.models import ExportJob 14 | from import_export_stomp.models import ImportJob 15 | from import_export_stomp.pubsub import consumer 16 | from import_export_stomp.pubsub import get_job_object_and_runner 17 | from import_export_stomp.pubsub import validate_payload 18 | from import_export_stomp.tasks import run_export_job 19 | from import_export_stomp.tasks import run_import_job 20 | from tests.utils import create_payload 21 | 22 | 23 | class TestValidatePayload: 24 | @pytest.mark.parametrize( 25 | ("data", "context"), 26 | ( 27 | ( 28 | {}, 29 | partial( 30 | pytest.raises, 31 | AssertionError, 32 | match="Payload needs to have 'action' key set.", 33 | ), 34 | ), 35 | ( 36 | {"abc": True}, 37 | partial( 38 | pytest.raises, 39 | AssertionError, 40 | match="Payload needs to have 'action' key set.", 41 | ), 42 | ), 43 | ( 44 | {"action": "invalid", "job_id": "invalid"}, 45 | partial( 46 | pytest.raises, 47 | AssertionError, 48 | match="Action value needs to be 'import' or 'export'.", 49 | ), 50 | ), 51 | ( 52 | {"action": "import"}, 53 | partial( 54 | pytest.raises, 55 | AssertionError, 56 | match="Payload needs to have 'dry_run' key set.", 57 | ), 58 | ), 59 | ( 60 | {"action": "import", "job_id": "123"}, 61 | partial( 62 | pytest.raises, 63 | AssertionError, 64 | match="Payload needs to have 'dry_run' key set.", 65 | ), 66 | ), 67 | ( 68 | {"action": "import", "dry_run": "True", "job_id": "123"}, 69 | partial( 70 | pytest.raises, AssertionError, match="'dry_run' is not a boolean." 71 | ), 72 | ), 73 | ( 74 | {"action": "import", "dry_run": True, "job_id": "invalid"}, 75 | partial( 76 | pytest.raises, AssertionError, match="'job_id' is not a number." 77 | ), 78 | ), 79 | ( 80 | {"action": "export", "dry_run": True, "job_id": "invalid"}, 81 | partial( 82 | pytest.raises, AssertionError, match="'job_id' is not a number." 83 | ), 84 | ), 85 | ( 86 | {"action": "import", "dry_run": True, "job_id": "12345"}, 87 | nullcontext, 88 | ), 89 | ( 90 | {"action": "export", "dry_run": True, "job_id": "12345"}, 91 | nullcontext, 92 | ), 93 | ), 94 | ) 95 | def test_payload_should_be_validated(self, data: Dict, context: Callable): 96 | payload, _, _ = create_payload(data) 97 | with context(): 98 | validate_payload(payload) 99 | 100 | 101 | @pytest.mark.django_db 102 | class TestGetJobObject: 103 | def test_should_correct_fetch_import_job(self): 104 | import_job = baker.make(ImportJob) 105 | payload, _, _ = create_payload( 106 | {"action": "import", "dry_run": True, "job_id": str(import_job.pk)} 107 | ) 108 | 109 | job, runner = get_job_object_and_runner(payload) 110 | 111 | assert job.pk == import_job.pk 112 | assert runner == run_import_job 113 | 114 | def test_should_correct_fetch_export_job(self): 115 | export_job = baker.make(ExportJob) 116 | payload, _, _ = create_payload( 117 | {"action": "export", "dry_run": True, "job_id": str(export_job.pk)} 118 | ) 119 | 120 | job, runner = get_job_object_and_runner(payload) 121 | 122 | assert job.pk == export_job.pk 123 | assert runner == run_export_job 124 | 125 | def test_should_raise_does_not_exists_when_import_job_does_not_exists(self): 126 | payload, _, _ = create_payload( 127 | {"action": "import", "dry_run": True, "job_id": "9999"} 128 | ) 129 | with pytest.raises(ImportJob.DoesNotExist): 130 | get_job_object_and_runner(payload) 131 | 132 | def test_should_raise_does_not_exists_when_export_job_does_not_exists(self): 133 | payload, _, _ = create_payload( 134 | {"action": "export", "dry_run": True, "job_id": "9999"} 135 | ) 136 | with pytest.raises(ExportJob.DoesNotExist): 137 | get_job_object_and_runner(payload) 138 | 139 | 140 | @pytest.mark.django_db 141 | class TestConsumer: 142 | def test_consumer_should_ack_if_invalid_payload(self): 143 | payload, ack, nack = create_payload({"invalid": "payload"}) 144 | 145 | consumer(payload) 146 | 147 | ack.assert_called_once() 148 | nack.assert_not_called() 149 | 150 | @pytest.mark.parametrize( 151 | argnames="param_action, param_log_message", 152 | argvalues=(("import", "ImportJob"), ("export", "ExportJob")), 153 | ) 154 | def test_consumer_should_raise_exception_if_job_does_not_exists( 155 | self, 156 | param_action: str, 157 | param_log_message: str, 158 | caplog: pytest.LogCaptureFixture, 159 | ): 160 | payload, ack, nack = create_payload( 161 | {"action": param_action, "dry_run": True, "job_id": "9999"} 162 | ) 163 | 164 | consumer(payload) 165 | 166 | assert ( 167 | f"WARNING import_export_stomp.pubsub:pubsub.py:59 {param_log_message} matching query does not exist.\n" 168 | == caplog.text 169 | ) 170 | ack.assert_called_once() 171 | nack.assert_not_called() 172 | 173 | @mock.patch.object(import_export_stomp.pubsub, "run_import_job") 174 | def test_consumer_should_call_run_import_job(self, mock_import_job: mock.MagicMock): 175 | import_job = baker.make(ImportJob) 176 | payload, ack, nack = create_payload( 177 | {"action": "import", "dry_run": True, "job_id": str(import_job.pk)} 178 | ) 179 | 180 | consumer(payload) 181 | 182 | ack.assert_called_once() 183 | nack.assert_not_called() 184 | mock_import_job.assert_called_once() 185 | 186 | @mock.patch.object(import_export_stomp.pubsub, "run_export_job") 187 | def test_consumer_should_call_run_export_job(self, mock_export_job: mock.MagicMock): 188 | export_job = baker.make(ExportJob) 189 | payload, ack, nack = create_payload( 190 | {"action": "export", "dry_run": True, "job_id": str(export_job.pk)} 191 | ) 192 | 193 | consumer(payload) 194 | 195 | ack.assert_called_once() 196 | nack.assert_not_called() 197 | mock_export_job.assert_called_once() 198 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /import_export_stomp/tasks.py: -------------------------------------------------------------------------------- 1 | # Author: Timothy Hobbs hobbs.cz> 2 | import logging 3 | import os 4 | 5 | from typing import Any 6 | from typing import Optional 7 | 8 | from django.conf import settings 9 | from django.core.cache import cache 10 | from django.core.files.base import ContentFile 11 | from django.utils import timezone 12 | from django.utils.encoding import force_str 13 | from django.utils.translation import gettext_lazy as _ 14 | 15 | from import_export_stomp.models import ExportJob 16 | from import_export_stomp.models import ImportJob 17 | from import_export_stomp.resources import ModelConfig 18 | from import_export_stomp.utils import get_formats 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | importables = getattr(settings, "IMPORT_EXPORT_STOMP_MODELS", {}) 24 | 25 | 26 | def change_job_status(job, direction, job_status, dry_run=False): 27 | if dry_run: 28 | job_status = "[Dry run] " + job_status 29 | 30 | cache.set(direction + "_job_status_%s" % job.pk, job_status) 31 | job.job_status = job_status 32 | job.save() 33 | 34 | 35 | def get_format(job): 36 | for format in get_formats(): 37 | if job.format == format.CONTENT_TYPE: 38 | return format() 39 | 40 | 41 | def _create_dataset_or_fail(import_job: ImportJob, dry_run: bool): 42 | import_format = get_format(import_job) 43 | 44 | # Copied from https://github.com/django-import-export/django-import-export/blob/3c082f98afe7996e79f936418fced3094f141c26/import_export/admin.py#L260 sorry # noqa 45 | 46 | try: 47 | data = import_job.file.read() 48 | if not import_format.is_binary(): 49 | data = force_str(data, "utf8") 50 | return import_format.create_dataset(data) 51 | except UnicodeDecodeError as e: 52 | import_job.errors += _("Imported file has a wrong encoding: %s" % e) + "\n" 53 | change_job_status( 54 | import_job, "import", "Imported file has a wrong encoding", dry_run 55 | ) 56 | import_job.save() 57 | raise e 58 | except Exception as e: 59 | import_job.errors += _("Error reading file: %s") % e + "\n" 60 | change_job_status(import_job, "import", "Error reading file", dry_run) 61 | import_job.save() 62 | raise e 63 | 64 | 65 | def _finish_import( 66 | import_job: ImportJob, 67 | dry_run: bool, 68 | resource: Any, 69 | result: Any, 70 | skip_diff: Optional[Any], 71 | ) -> None: 72 | if dry_run: 73 | summary = "" 74 | summary += "" 75 | summary += '' 76 | summary += "" 77 | summary += "" 78 | summary += ( # TODO refactor the existing template so we can use it for this 79 | '' 80 | ) 81 | # https://github.com/django-import-export/django-import-export/blob/6575c3e1d89725701e918696fbc531aeb192a6f7/import_export/templates/admin/import_export/import.html 82 | if not result.invalid_rows and not skip_diff: 83 | cols = lambda row: "" 90 | ) 91 | summary += ( 92 | "" 100 | ) 101 | else: 102 | cols = lambda row: "" 118 | + "" 123 | ) 124 | summary += ( 125 | "" 137 | ) 138 | summary += "
".join([field for field in row.diff]) # noqa 84 | summary += ( 85 | "
change_type" 86 | + "".join( 87 | [f.column_name for f in resource.get_user_visible_fields()] 88 | ) 89 | + "
" 93 | + "
".join( 94 | [ 95 | row.import_type + "" + cols(row) 96 | for row in result.valid_rows() 97 | ] 98 | ) 99 | + "
".join( # noqa 103 | [str(field) for field in row.values] 104 | ) 105 | cols_error = lambda row: "".join( # noqa 106 | [ 107 | "" 108 | + key 109 | + "" 110 | + "
" 111 | + row.error.message_dict[key][0] 112 | + "
" 113 | for key in row.error.message_dict.keys() 114 | ] 115 | ) 116 | summary += ( 117 | "
rowerrors" 119 | + "".join( 120 | [f.column_name for f in resource.get_user_visible_fields()] 121 | ) 122 | + "
" 126 | + "
".join( 127 | [ 128 | str(row.number) 129 | + "" 130 | + cols_error(row) 131 | + "" 132 | + cols(row) 133 | for row in result.invalid_rows 134 | ] 135 | ) 136 | + "
" 139 | summary += "" 140 | summary += "" 141 | import_job.change_summary.save( 142 | os.path.split(import_job.file.name)[1] + ".html", 143 | ContentFile(summary.encode("utf-8")), 144 | ) 145 | else: 146 | import_job.imported = timezone.now() 147 | 148 | 149 | def _run_import_job(import_job: ImportJob, dry_run: bool = True) -> None: 150 | change_job_status(import_job, "import", "1/5 Import started", dry_run) 151 | if dry_run: 152 | import_job.errors = "" 153 | model_config = ModelConfig(**importables[import_job.model]) 154 | 155 | try: 156 | dataset = _create_dataset_or_fail(import_job, dry_run) 157 | except Exception: 158 | return 159 | 160 | change_job_status(import_job, "import", "2/5 Processing import data", dry_run) 161 | 162 | class Resource(model_config.resource): # type: ignore 163 | def __init__(self, import_job, *args, **kwargs): 164 | self.import_job = import_job 165 | super().__init__(*args, **kwargs) 166 | 167 | def before_import_row(self, row, **kwargs): 168 | if "row_number" in kwargs: 169 | row_number = kwargs["row_number"] 170 | if row_number % 100 == 0 or row_number == 1: 171 | change_job_status( 172 | import_job, 173 | "import", 174 | f"3/5 Importing row {row_number}/{len(dataset)}", 175 | dry_run, 176 | ) 177 | return super().before_import_row(row, **kwargs) 178 | 179 | resource = Resource(import_job=import_job) 180 | 181 | skip_diff = resource._meta.skip_diff or resource._meta.skip_html_diff 182 | 183 | result = resource.import_data(dataset, dry_run=dry_run) 184 | change_job_status(import_job, "import", "4/5 Generating import summary", dry_run) 185 | for error in result.base_errors: 186 | import_job.errors += f"\n{error.error}\n{error.traceback}\n" 187 | for line, errors in result.row_errors(): 188 | for error in errors: 189 | import_job.errors += _("Line: %s - %s\n\t%s\n%s") % ( 190 | line, 191 | error.error, 192 | ",".join(str(s) for s in error.row.values()), 193 | error.traceback, 194 | ) 195 | 196 | _finish_import(import_job, dry_run, resource, result, skip_diff) 197 | 198 | change_job_status(import_job, "import", "5/5 Import job finished", dry_run) 199 | import_job.save() 200 | 201 | 202 | def run_import_job(import_job: ImportJob, dry_run: bool = True): 203 | logger.info("Importing %s dry-run %s", import_job.pk, dry_run) 204 | try: 205 | _run_import_job(import_job, dry_run) 206 | except Exception as e: 207 | import_job.errors += _("Import error %s") % e + "\n" 208 | change_job_status(import_job, "import", "Import error", dry_run) 209 | import_job.save() 210 | return 211 | 212 | 213 | def run_export_job(export_job: ExportJob, **kwargs): 214 | logger.info("Exporting %s", export_job.pk) 215 | resource_class = export_job.get_resource_class() 216 | queryset = export_job.get_queryset() 217 | qs_len = len(queryset) 218 | 219 | class Resource(resource_class): # type:ignore 220 | def __init__(self, export_job, *args, **kwargs): 221 | self.row_number = 1 222 | self.export_job = export_job 223 | super().__init__(*args, **kwargs) 224 | 225 | def export_resource(self, *args, **kwargs): 226 | if self.row_number % 20 == 0 or self.row_number == 1: 227 | change_job_status( 228 | export_job, 229 | "export", 230 | f"Exporting row {self.row_number}/{qs_len}", 231 | ) 232 | self.row_number += 1 233 | return super().export_resource(*args, **kwargs) 234 | 235 | resource = Resource(export_job=export_job) 236 | 237 | data = resource.export(queryset) 238 | _format = get_format(export_job) 239 | serialized = _format.export_data(data) 240 | change_job_status(export_job, "export", "Export complete") 241 | filename = "{app}-{model}-{date}.{extension}".format( 242 | app=export_job.app_label, 243 | model=export_job.model, 244 | date=str(timezone.now()), 245 | extension=_format.get_extension(), 246 | ) 247 | if not _format.is_binary(): 248 | serialized = serialized.encode("utf8") 249 | export_job.file.save(filename, ContentFile(serialized)) 250 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |
3 | Juntos Somos + 4 |

Django Import Export Stomp

5 |
6 | 7 | [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=juntossomosmais_django-import-export-stomp&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=juntossomosmais_django-import-export-stomp) 8 | [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=juntossomosmais_django-import-export-stomp&metric=coverage)](https://sonarcloud.io/summary/new_code?id=juntossomosmais_django-import-export-stomp) 9 | [![Build Status](https://dev.azure.com/juntos-somos-mais-loyalty/python/_apis/build/status%2Fjuntossomosmais.django-import-export-stomp?repoName=juntossomosmais%2Fdjango-import-export-stomp&branchName=main)](https://dev.azure.com/juntos-somos-mais-loyalty/python/_build/latest?definitionId=474&repoName=juntossomosmais%2Fdjango-import-export-stomp&branchName=main) 10 | [![PyPI version](https://badge.fury.io/py/django-import-export-stomp.svg)](https://badge.fury.io/py/django-import-export-stomp) 11 | [![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg)](https://conventionalcommits.org) 12 | [![Generic badge](https://img.shields.io/badge/python-3.10_3.11-green.svg)](https://www.python.org/) 13 | [![Generic badge](https://img.shields.io/badge/django-5.1-green.svg)](https://www.djangoproject.com/) 14 | 15 | TLDR: Django plugin for file import/export on top of [django-import-export](https://github.com/django-import-export/django-import-export) using [django-stomp](https://github.com/juntossomosmais/django-stomp). 16 | 17 | ## Table of Contents 18 | 19 | - [Table of Contents](#table-of-contents) 20 | - [About](#about) 21 | - [Codebase](#codebase) 22 | - [The stack](#the-stack) 23 | - [Engineering standards](#engineering-standards) 24 | - [Commit hooks](#commit-hooks) 25 | - [Getting Started](#getting-started) 26 | - [Prerequisites](#prerequisites) 27 | - [Running example application with Docker](#running-example-application-with-docker) 28 | - [Testing the application with Docker](#testing-the-application-with-docker) 29 | - [Documentation](#documentation) 30 | - [Basic installation](#basic-installation) 31 | - [Running the consumer](#running-the-consumer) 32 | - [Setting up imports](#setting-up-imports) 33 | - [Setting up export](#setting-up-export) 34 | - [How to use upload with aws s3 presigned url](#how-to-use-upload-with-aws-s3-presigned-url) 35 | - [Performing an import](#performing-an-import) 36 | - [Performing an export](#performing-an-export) 37 | - [Settings](#settings) 38 | - [Credits](#credits) 39 | 40 | ## About 41 | 42 | Django Import Export Stomp is a django plugin that uses [django-stomp](https://github.com/juntossomosmais/django-stomp) to import/export models to spreadsheet-like files (csv, xlsx, etc.). 43 | 44 | ## Codebase 45 | 46 | ### The stack 47 | 48 | This application uses at least [Python 3.10](https://www.python.org/downloads/) with at least [Django 5.1](https://docs.djangoproject.com/en/5.1/). 49 | 50 | ### Engineering standards 51 | 52 | #### Commit hooks 53 | 54 | Currently we're using `pre-commit`. 55 | To configure it simply use the commands below. 56 | 57 | ```sh 58 | pip install pre-commit 59 | pre-commit install 60 | ``` 61 | 62 | This will automatically lint the staged files using our project standard linters. 63 | 64 | ## Getting Started 65 | 66 | This section provides a high-level requirement & quick start guide. 67 | 68 | ### Prerequisites 69 | 70 | - [docker](https://docs.docker.com/get-docker/) 71 | - [docker compose](https://docs.docker.com/compose/install/) 72 | 73 | > :warning: 74 | > 75 | > We do not recommend developing the application without docker! 76 | > 77 | > :warning: 78 | 79 | ### Running example application with Docker 80 | 81 | 1. Clone the repository via `ssh`, ie. 82 | `git clone git@github.com:juntossomosmais/django-import-export-stomp.git`. 83 | 84 | 2. Simply running `docker compose up --build example` is enough to start running `django-import-export-stomp` on a exmaple with Docker. 85 | 86 | ### Testing the application with Docker 87 | 88 | 1. If you want to run the tests run: `docker compose run integration-tests`. 89 | 90 | 2. To run the sonar analysis locally you can use `docker compose up -d sonar-client` and then `docker compose up -d sonar-cli` 91 | 92 | ## Documentation 93 | 94 | ### Basic installation 95 | 96 | 1. Install package: `pip install django-import-export-stomp` 97 | 2. Add `import_export_stomp` to your `INSTALLED_APPS` in your `settings.py` 98 | 3. Add `author.middlewares.AuthorDefaultBackendMiddleware` to your `MIDDLEWARE_CLASSES` in your `settings.py` 99 | 4. Setup [django-stomp](https://github.com/juntossomosmais/django-stomp) 100 | 101 | ### Running the consumer 102 | 103 | Run `python manage.py import_export pubsub` to start processing messages from the queues. 104 | 105 | ### Setting up imports 106 | 107 | On your settings.py add a `IMPORT_EXPORT_MODELS` variable: 108 | 109 | ```python 110 | from import_export_stomp.resources import resource_importer 111 | 112 | IMPORT_EXPORT_STOMP_MODELS = { 113 | "Name of your import": { 114 | "app_label": "fake_app", 115 | "model_name": "FakeModel", 116 | "resource": resource_importer( 117 | "tests.resources.fake_app.resources.FakeResource" 118 | ), # optional 119 | } 120 | } 121 | ``` 122 | 123 | By default a dry run of the import is initiated when the import object is created. To instead import the file immediately without a dry-run set the `IMPORT_DRY_RUN_FIRST_TIME` to `False`. 124 | 125 | `IMPORT_DRY_RUN_FIRST_TIME = False` 126 | 127 | ### Setting up export 128 | 129 | As with imports, a fully configured example project can be found in the `example` directory. 130 | 131 | 1. Add a `export_resource_classes` classmethod to the model you want to export. 132 | 133 | ```python 134 | @classmethod 135 | def export_resource_classes(cls): 136 | return { 137 | 'winners': ('Winners resource', WinnersResource), 138 | 'winners_all_caps': ('Winners with all caps column resource', WinnersWithAllCapsResource), 139 | } 140 | ``` 141 | 142 | This should return a dictionary of tuples. The keys should be unique unchanging strings, the tuples should consist of a `resource `__ and a human friendly description of that resource. 143 | 144 | 2. Add the `create_export_job_action` to the model's `ModelAdmin`. 145 | 146 | ```python 147 | from django.contrib import admin 148 | from import_export.admin_actions import create_export_job_action 149 | 150 | from . import models 151 | 152 | @admin.register(models.Winner) 153 | class WinnerAdmin(admin.ModelAdmin): 154 | list_display = ( 155 | 'name', 156 | ) 157 | 158 | actions = ( 159 | create_export_job_action, 160 | ) 161 | ``` 162 | 163 | 3. To customise export queryset you need to add `get_export_queryset` to the `ModelResource`. 164 | 165 | ```python 166 | class WinnersResource(ModelResource): 167 | class Meta: 168 | model = Winner 169 | 170 | def get_export_queryset(self): 171 | """To customise the queryset of the model resource with annotation override""" 172 | return self.Meta.model.objects.annotate(device_type=Subquery(FCMDevice.objects.filter( 173 | user=OuterRef("pk")).values("type")[:1]) 174 | ``` 175 | 176 | 4. Done! 177 | 178 | ### How to use upload with aws s3 presigned url 179 | 180 | 1. Have `boto3` and `django-storages` installed in your project: `pip install boto3 django-storages` 181 | 2. Setup [django-storages](https://github.com/jschneier/django-storages) variables - `AWS_STORAGE_BUCKET_NAME` is required. 182 | 3. Set `IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST` to `True`. 183 | 4. Add urls from `import_export_stomp.urls` to your `urls.py` 184 | 185 | ```python 186 | from import_export_stomp.urls import urlpatterns as import_export_stomp_urlpatterns 187 | 188 | urlpatterns = [...] # Your urls 189 | urlpatterns += import_export_stomp_urlpatterns 190 | ``` 191 | 192 | 5. Done! 193 | 194 | ### Performing an import 195 | 196 | You will find an example django application that uses django-import-export-stomp for importing data. Once you have it running, you can perform an import with the following steps. 197 | 198 | 1. Navigate to the example applications admin page: 199 | 200 | ![example](screenshots/admin.png) 201 | 202 | 2. Navigate to the ImportJobs table: 203 | 204 | ![example](screenshots/import_jobs.png) 205 | 206 | 3. Create a new import job. There is an example import CSV file in the example/example-data directory. Select that file. Select csv as the file format. We'll be importing to the Winner's model table. 207 | 208 | ![example](screenshots/new_import_job.png) 209 | 210 | 4. Select "Save and continue editing" to save the import job and refresh until you see that a "Summary of changes made by this import" file has been created. 211 | 212 | ![example](screenshots/summary.png) 213 | 214 | 5. You can view the summary if you want. Your import has NOT BEEN PERFORMED YET! 215 | 216 | ![example](screenshots/view-summary.png) 217 | 218 | 6. Return to the import-jobs table, select the import job we just created, and select the "Perform import" action from the actions drop down. 219 | 220 | ![example](screenshots/perform-import.png) 221 | 222 | 7. In a short time, your imported Winner object should show up in your Winners table. 223 | 224 | ![example](screenshots/new-winner.png) 225 | 226 | ### Performing an export 227 | 228 | 1. Perform the basic setup procedure described in the first section. 229 | 230 | 2. Open up the object list for your model in django admin, select the objects you wish to export, and select the `Export with stomp` admin action. 231 | 232 | 3. Select the file format and resource you want to use to export the data. 233 | 234 | 4. Save the model 235 | 236 | 5. You will receive an email when the export is done, click on the link in the email 237 | 238 | 6. Click on the link near the bottom of the page titled `Exported file`. 239 | 240 | ### Settings 241 | 242 | - `IMPORT_EXPORT_STOMP_MODELS` 243 | **Required** Dict containing all the models that will be imported. 244 | 245 | ```python 246 | { 247 | "Name of your import": { 248 | "app_label": "fake_app", 249 | "model_name": "FakeModel", 250 | "resource": resource_importer( 251 | "tests.resources.fake_app.resources.FakeResource" 252 | ), # optional -if not present will auto-create 253 | } 254 | } 255 | ``` 256 | 257 | - `IMPORT_EXPORT_STOMP_STORAGE` 258 | Storage class which import/export file field will use. Defaults to `None`. 259 | Example: `storages.backends.s3boto3.S3Boto3Storage` 260 | 261 | - `IMPORT_EXPORT_STOMP_EXCLUDED_FORMATS` 262 | List of formats to exclude from import/export. Defaults to `[]`. 263 | Example: `["csv", "xlsx"]` 264 | 265 | - `IMPORT_EXPORT_STOMP_PROCESSING_QUEUE` 266 | Name of the stomp queue that will be used to publish/consume the messages. Defaults to `/queue/django-import-export-stomp-runner` 267 | 268 | - `IMPORT_EXPORT_STOMP_USE_PRESIGNED_POST` 269 | Enables upload using presigned post url. Uses `boto3` and `django-storages`. Defaults to `False`. 270 | 271 | - `IMPORT_EXPORT_STOMP_PRESIGNED_POST_EXPIRATION` 272 | Sets signed url expiration time. Defaults to `600` 273 | 274 | - `IMPORT_EXPORT_STOMP_PRESIGNED_FOLDER` 275 | Prepends a path to the s3 key. Defaults to `""` 276 | 277 | ### Credits 278 | 279 | `django-import-export-stomp` was based on [django-import-export-celery](https://github.com/auto-mat/django-import-export-celery) developed by the Czech non-profit [auto*mat z.s.](https://auto-mat.cz). 280 | --------------------------------------------------------------------------------