├── cml ├── __init__.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── cmlpipelines.py ├── migrations │ ├── __init__.py │ └── 0001_initial.py ├── urls.py ├── admin.py ├── conf.py ├── models.py ├── templates │ └── cml │ │ └── cml-pipelines.txt ├── views.py ├── items.py ├── auth.py └── utils.py ├── tests ├── __init__.py ├── urls.py ├── test_utils.py └── settings.py ├── setup.cfg ├── requirements.txt ├── .gitignore ├── MANIFEST.in ├── .travis.yml ├── runtests.py ├── README.rst ├── setup.py └── LICENSE /cml/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cml/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cml/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | -------------------------------------------------------------------------------- /cml/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django>=2.0.0 2 | django-appconf==1.0.2 3 | six>=1.12.0 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.sqlite3 3 | .idea 4 | tmp 5 | dist 6 | django_cml.egg-info 7 | build 8 | -------------------------------------------------------------------------------- /tests/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import include, re_path 2 | 3 | urlpatterns = [ 4 | re_path(r'^cml/', include('cml.urls')), 5 | ] 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | recursive-include cml/management * 4 | recursive-include cml/templates * 5 | recursive-include cml/migrations * 6 | recursive-include docs * -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | sudo: false 4 | 5 | python: 6 | - "3.5" 7 | - "3.6" 8 | 9 | install: 10 | - pip install -r requirements.txt 11 | 12 | script: 13 | - python runtests.py 14 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | from django.core.management import execute_from_command_line 5 | 6 | 7 | def runtests(): 8 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") 9 | argv = sys.argv[:1] + ['test'] + sys.argv[1:] 10 | execute_from_command_line(argv) 11 | 12 | 13 | if __name__ == '__main__': 14 | runtests() 15 | -------------------------------------------------------------------------------- /cml/urls.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from django.conf.urls import re_path, include 3 | from . import views 4 | 5 | app_urlpatterns = [ 6 | re_path(r'^1c_exchange.php$', views.front_view, name='front_view'), 7 | re_path(r'^exchange$', views.front_view, name='front_view'), 8 | ] 9 | 10 | urlpatterns = [ 11 | re_path(r'^', include((app_urlpatterns, 'cml'), namespace='cml')), 12 | ] 13 | -------------------------------------------------------------------------------- /cml/admin.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from django.contrib import admin 3 | from .models import * 4 | 5 | 6 | @admin.register(Exchange) 7 | class ExchangeAdmin(admin.ModelAdmin): 8 | 9 | list_display = ('exchange_type', 'timestamp', 'user', 'filename') 10 | readonly_fields = ('exchange_type', 'timestamp', 'user', 'filename') 11 | 12 | def has_add_permission(self, request): 13 | return False 14 | -------------------------------------------------------------------------------- /cml/conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | from django.conf import settings 3 | from appconf import AppConf 4 | 5 | 6 | class CMLAppCong(AppConf): 7 | 8 | RESPONSE_SUCCESS = 'success' 9 | RESPONSE_PROGRESS = 'progress' 10 | RESPONSE_ERROR = 'failure' 11 | 12 | MAX_EXEC_TIME = 60 13 | USE_ZIP = False 14 | FILE_LIMIT = 0 15 | 16 | UPLOAD_ROOT = os.path.join(settings.MEDIA_ROOT, 'cml', 'tmp') 17 | 18 | DELETE_FILES_AFTER_IMPORT = True 19 | -------------------------------------------------------------------------------- /cml/models.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from django.db import models 3 | from django.conf import settings 4 | 5 | 6 | class Exchange(models.Model): 7 | 8 | class Meta: 9 | verbose_name = 'Exchange log entry' 10 | verbose_name_plural = 'Exchange logs' 11 | 12 | exchange_type_choices = { 13 | ('import', 'import'), 14 | ('export', 'export') 15 | } 16 | 17 | exchange_type = models.CharField(max_length=50, choices=exchange_type_choices) 18 | timestamp = models.DateTimeField(auto_now_add=True) 19 | user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) 20 | filename = models.CharField(max_length=200) 21 | 22 | @classmethod 23 | def log(cls, exchange_type, user, filename=u''): 24 | ex_log = Exchange(exchange_type=exchange_type, user=user, filename=filename) 25 | ex_log.save() 26 | -------------------------------------------------------------------------------- /cml/management/commands/cmlpipelines.py: -------------------------------------------------------------------------------- 1 | import os 2 | from django.core.management.base import BaseCommand, CommandError 3 | from django.template.loader import render_to_string 4 | from django.conf import settings 5 | 6 | DEFAULT_FILE_NAME = 'cml_pipelines.py' 7 | 8 | 9 | class Command(BaseCommand): 10 | help = 'Creates a template file with project cml pipelines' 11 | args = ['file'] 12 | 13 | def handle(self, file_name=None, **options): 14 | project_name = os.path.basename(os.getcwd()) 15 | dst = file_name is not None and file_name or DEFAULT_FILE_NAME 16 | if os.path.exists(dst): 17 | raise CommandError('Error: file "%s" already exists' % dst) 18 | open(dst, 'w').write(render_to_string('cml/cml-pipelines.txt', { 19 | 'project': project_name, 20 | 'file': os.path.basename(dst).split('.')[0] 21 | })) 22 | self.stdout.write('"%s" written.' % os.path.join(dst)) 23 | -------------------------------------------------------------------------------- /cml/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | from django.conf import settings 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='Exchange', 17 | fields=[ 18 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 19 | ('exchange_type', models.CharField(max_length=50, choices=[(b'export', b'export'), (b'import', b'import')])), 20 | ('timestamp', models.DateTimeField(auto_now_add=True)), 21 | ('filename', models.CharField(max_length=200)), 22 | ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), 23 | ], 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | === 2 | CML 3 | === 4 | 5 | CML is a reusable Django app for data exchange in CommerceML 2 standard. 6 | 7 | Requirements 8 | ------------ 9 | 10 | - Python 3.3, 3.4, 3.5, 3.6 11 | - Django 2.x 12 | 13 | Quick start 14 | ----------- 15 | 16 | Install using pip:: 17 | 18 | pip install django-cml 19 | 20 | Or clone the repo and add to your `PYTHONPATH`:: 21 | 22 | git clone https://github.com/ArtemiusUA/django-cml.git 23 | 24 | Add "cml" to your `INSTALLED_APPS` setting like this:: 25 | 26 | INSTALLED_APPS = [ 27 | ... 28 | 'cml', 29 | ] 30 | 31 | Include the cml URLconf in your project `urls.py` like this:: 32 | 33 | re_path(r'^cml/', include('cml.urls')), 34 | 35 | Run `python manage.py migrate` to create the cml models. 36 | 37 | Create a `cml-pipelines.py` file with `python manage.py cmlpipelines` and add it to settings file like this:: 38 | 39 | CML_PROJECT_PIPELINES = 'project.cml_pipelines' 40 | 41 | Modify pipeline objects for your needs to stack this with your models. 42 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup 3 | 4 | with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as readme: 5 | README = readme.read() 6 | 7 | os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) 8 | 9 | setup( 10 | name='django-cml', 11 | python_requires='>3.3.0', 12 | version='0.4.0', 13 | packages=['cml'], 14 | install_requires=['Django>=2.0', 'django-appconf>=1.0.1', 'six>=1.12.0'], 15 | include_package_data=True, 16 | license='BSD License', 17 | description='App for data exchange in CommerceML 2 standard..', 18 | long_description=README, 19 | url='https://github.com/ArtemiusUA/django-cml', 20 | author='Artem Merkulov', 21 | author_email='artem.merkulov@gmail.com', 22 | classifiers=[ 23 | 'Environment :: Web Environment', 24 | 'Framework :: Django', 25 | 'Framework :: Django :: 2.0', 26 | 'Intended Audience :: Developers', 27 | 'License :: OSI Approved :: BSD License', 28 | 'Operating System :: OS Independent', 29 | 'Programming Language :: Python', 30 | 'Programming Language :: Python :: 3.3', 31 | 'Programming Language :: Python :: 3.4', 32 | 'Programming Language :: Python :: 3.5', 33 | 'Topic :: Internet :: WWW/HTTP', 34 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 35 | ], 36 | ) 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Artem Merkulov. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the author nor the names of its contributors may be used 15 | to endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | from __future__ import absolute_import 3 | import os 4 | import six 5 | from six.moves import range 6 | try: 7 | from xml.etree import cElementTree as ET 8 | except ImportError: 9 | from xml.etree import ElementTree as ET 10 | from django.test import TestCase 11 | from cml.utils import ImportManager, ExportManager 12 | from cml.items import * 13 | 14 | FIXTURES_PATH = os.path.normpath(os.path.join(os.path.dirname(__file__), 'tests_fixtures')) 15 | 16 | 17 | class ImportManagerTestCase(TestCase): 18 | 19 | def setUp(self): 20 | pass 21 | 22 | def test_run(self): 23 | man = ImportManager(os.path.join(FIXTURES_PATH, 'import.xml')) 24 | man.import_all() 25 | self.assertTrue(GroupPipeline.collected_items) 26 | 27 | 28 | class ExportManagerTestCase(TestCase): 29 | 30 | def setUp(self): 31 | pass 32 | 33 | def test_run(self): 34 | man = ExportManager() 35 | man.export_all() 36 | tree = ET.fromstring(man.get_xml()) 37 | orders_elements = tree.find(u'Документ') 38 | self.assertIsNotNone(orders_elements) 39 | 40 | 41 | class GroupPipeline(object): 42 | 43 | collected_items = [] 44 | 45 | def process_item(self, item): 46 | GroupPipeline.collected_items.append(item) 47 | 48 | 49 | class OrderPipeline(object): 50 | 51 | def process_item(self, item): 52 | pass 53 | 54 | def yield_item(self): 55 | for i in range(10): 56 | item = Order() 57 | item.id = six.text_type(i) 58 | item.number = six.text_type(i) 59 | item.currency_name = six.text_type(i) 60 | item.date = datetime.now() 61 | item.currency_rate = Decimal(i) 62 | item.sum = Decimal(i) 63 | yield item 64 | 65 | -------------------------------------------------------------------------------- /tests/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 4 | 5 | SECRET_KEY = 'tests' 6 | 7 | # Application definition 8 | 9 | INSTALLED_APPS = ( 10 | 'django.contrib.auth', 11 | 'django.contrib.contenttypes', 12 | 'django.contrib.sessions', 13 | 'django.contrib.messages', 14 | 'cml' 15 | ) 16 | 17 | DATABASE_ENGINE = 'sqlite3' 18 | 19 | DATABASES = { 20 | 'default': { 21 | 'ENGINE': 'django.db.backends.sqlite3', 22 | 'NAME': ':memory:', 23 | } 24 | } 25 | 26 | MIDDLEWARE_CLASSES = ( 27 | 'django.contrib.sessions.middleware.SessionMiddleware', 28 | 'django.middleware.common.CommonMiddleware', 29 | 'django.middleware.csrf.CsrfViewMiddleware', 30 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 31 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 32 | 'django.contrib.messages.middleware.MessageMiddleware', 33 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 34 | 'django.middleware.security.SecurityMiddleware', 35 | ) 36 | 37 | ROOT_URLCONF = 'tests.urls' 38 | 39 | TEMPLATES = [ 40 | { 41 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 42 | 'DIRS': [], 43 | 'APP_DIRS': True, 44 | 'OPTIONS': { 45 | 'context_processors': [ 46 | 'django.template.context_processors.debug', 47 | 'django.template.context_processors.request', 48 | 'django.contrib.auth.context_processors.auth', 49 | 'django.contrib.messages.context_processors.messages', 50 | ], 51 | }, 52 | }, 53 | ] 54 | 55 | 56 | TIME_ZONE = 'UTC' 57 | 58 | USE_I18N = True 59 | 60 | USE_L10N = True 61 | 62 | USE_TZ = True 63 | 64 | CML_PROJECT_PIPELINES = 'tests.test_utils' 65 | -------------------------------------------------------------------------------- /cml/templates/cml/cml-pipelines.txt: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | """ 3 | This file was generated with the cmlpipelines management command. 4 | It contains the pipelines classes for that accept or send items for 5 | import/export purposes. 6 | 7 | To activate your pipelines add the following to your settings.py: 8 | CML_PROJECT_PIPELINES = '{{ project }}.{{ file }}' 9 | """ 10 | 11 | import decimal 12 | from cml.items import Order, OrderItem 13 | import orders.models as prod 14 | 15 | 16 | class GroupPipeline(object): 17 | """ 18 | Item fields: 19 | id 20 | name 21 | groups 22 | """ 23 | def process_item(self, item): 24 | pass 25 | 26 | 27 | class PropertyPipeline(object): 28 | """ 29 | Item fields: 30 | id 31 | name 32 | value_type 33 | for_products 34 | """ 35 | def process_item(self, item): 36 | pass 37 | 38 | 39 | class PropertyVariantPipeline(object): 40 | """ 41 | Item fields: 42 | id 43 | value 44 | property_id 45 | """ 46 | def process_item(self, item): 47 | pass 48 | 49 | 50 | class SkuPipeline(object): 51 | """ 52 | Item fields: 53 | id 54 | name 55 | name_full 56 | international_abbr 57 | """ 58 | def process_item(self, item): 59 | pass 60 | 61 | 62 | class TaxPipeline(object): 63 | """ 64 | Item fields: 65 | name 66 | value 67 | """ 68 | def process_item(self, item): 69 | pass 70 | 71 | 72 | class ProductPipeline(object): 73 | """ 74 | Item fields: 75 | id 76 | name 77 | sku_id 78 | group_ids 79 | properties 80 | tax_name 81 | image_path 82 | additional_fields 83 | """ 84 | def process_item(self, item): 85 | pass 86 | 87 | 88 | class PriceTypePipeline(object): 89 | """ 90 | Item fields: 91 | id 92 | name 93 | currency 94 | tax_name 95 | tax_in_sum 96 | """ 97 | def process_item(self, item): 98 | pass 99 | 100 | 101 | class OfferPipeline(object): 102 | """ 103 | Item fields: 104 | id 105 | name 106 | sku_id 107 | prices 108 | """ 109 | def process_item(self, item): 110 | pass 111 | 112 | 113 | class OrderPipeline(object): 114 | """ 115 | Item fields: 116 | id 117 | number 118 | date 119 | currency_name 120 | currency_rate 121 | operation 122 | role 123 | sum 124 | client 125 | time 126 | comment 127 | items 128 | additional_fields 129 | """ 130 | def process_item(self, item): 131 | pass 132 | 133 | def yield_item(self): 134 | pass 135 | 136 | def flush(self): 137 | pass 138 | -------------------------------------------------------------------------------- /cml/views.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from django.http import Http404 3 | from django.views.decorators.csrf import csrf_exempt 4 | from django.core.files.uploadedfile import SimpleUploadedFile 5 | from .auth import * 6 | from .utils import * 7 | from .models import * 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | @csrf_exempt 13 | @has_perm_or_basicauth('cml.add_exchange') 14 | @logged_in_or_basicauth() 15 | def front_view(request): 16 | return Dispatcher().dispatch(request) 17 | 18 | 19 | def error(request, error_text): 20 | logger.error(error_text) 21 | result = '{}\n{}'.format(settings.CML_RESPONSE_ERROR, error_text) 22 | return HttpResponse(result) 23 | 24 | 25 | def success(request, success_test=''): 26 | result = '{}\n{}'.format(settings.CML_RESPONSE_SUCCESS, success_test) 27 | return HttpResponse(result) 28 | 29 | 30 | def check_auth(request): 31 | session = request.session 32 | success_text = '{}\n{}'.format(settings.SESSION_COOKIE_NAME, session.session_key) 33 | return success(success_text) 34 | 35 | 36 | def init(request): 37 | result = 'zip={}\nfile_limit={}'.format('yes' if settings.CML_USE_ZIP else 'no', 38 | settings.CML_FILE_LIMIT) 39 | return HttpResponse(result) 40 | 41 | 42 | def upload_file(request): 43 | if request.method != 'POST': 44 | return error(request, 'Wrong HTTP method!') 45 | try: 46 | filename = request.GET['filename'] 47 | except KeyError: 48 | return error(request, 'Need a filename param!') 49 | if not os.path.exists(settings.CML_UPLOAD_ROOT): 50 | try: 51 | os.makedirs(settings.CML_UPLOAD_ROOT) 52 | except OSError: 53 | return error(request, 'Can\'t create upload directory!') 54 | filename = os.path.basename(filename) 55 | temp_file = SimpleUploadedFile(filename, request.read(), content_type='text/xml') 56 | with open(os.path.join(settings.CML_UPLOAD_ROOT, filename), 'wb') as f: 57 | for chunk in temp_file.chunks(): 58 | f.write(chunk) 59 | return success(request) 60 | 61 | 62 | def import_file(request): 63 | try: 64 | filename = request.GET['filename'] 65 | except KeyError: 66 | return error(request, 'Need a filename param!') 67 | file_path = os.path.join(settings.CML_UPLOAD_ROOT, filename) 68 | if not os.path.exists(file_path): 69 | return error(request, 'File does\'nt exists!') 70 | import_manager = ImportManager(file_path, ) 71 | try: 72 | import_manager.import_all() 73 | except Exception as e: 74 | return error(request, str(e)) 75 | if settings.CML_DELETE_FILES_AFTER_IMPORT: 76 | try: 77 | os.remove(file_path) 78 | except OSError: 79 | logger.error('Can\'t delete file after import: {}'.format(file_path)) 80 | Exchange.log('import', request.user, filename) 81 | return success(request) 82 | 83 | 84 | def export_query(request): 85 | export_manager = ExportManager() 86 | export_manager.export_all() 87 | return HttpResponse(export_manager.get_xml(), content_type='text/xml') 88 | 89 | 90 | def export_success(request): 91 | export_manager = ExportManager() 92 | Exchange.log('export', request.user) 93 | export_manager.flush() 94 | return success(request) 95 | 96 | 97 | class Dispatcher(object): 98 | 99 | def __init__(self): 100 | self.routes_map = { 101 | (u'catalog', u'checkauth'): check_auth, 102 | (u'catalog', u'init'): init, 103 | (u'catalog', u'file'): upload_file, 104 | (u'catalog', u'import'): import_file, 105 | (u'sale', u'file'): upload_file, 106 | (u'import', u'import'): import_file, 107 | (u'sale', u'checkauth'): check_auth, 108 | (u'sale', u'init'): init, 109 | (u'sale', u'query'): export_query, 110 | (u'sale', u'success'): export_success, 111 | } 112 | 113 | def dispatch(self, request): 114 | view_key = (request.GET.get('type'), request.GET.get('mode')) 115 | view = self.routes_map.get(view_key) 116 | if not view: 117 | raise Http404 118 | return view(request) 119 | -------------------------------------------------------------------------------- /cml/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | from __future__ import absolute_import 3 | from decimal import Decimal 4 | from datetime import datetime 5 | 6 | PROCESSED_ITEMS = ('Group', 'PropertyVariant', 'Property', 'PropertyVariant', 'Sku', 'Tax', 'Product', 'Offer', 'Order') 7 | 8 | 9 | class BaseItem(object): 10 | 11 | def __init__(self, xml_element=None): 12 | self.xml_element = xml_element 13 | 14 | 15 | class Group(BaseItem): 16 | 17 | def __init__(self, *args, **kwargs): 18 | super(Group, self).__init__(*args, **kwargs) 19 | self.id = u'' 20 | self.name = u'' 21 | self.groups = [] 22 | 23 | 24 | class Property(BaseItem): 25 | 26 | def __init__(self, *args, **kwargs): 27 | super(Property, self).__init__(*args, **kwargs) 28 | self.id = u'' 29 | self.name = u'' 30 | self.value_type = u'' 31 | self.for_products = False 32 | 33 | 34 | class PropertyVariant(BaseItem): 35 | 36 | def __init__(self, *args, **kwargs): 37 | super(PropertyVariant, self).__init__(*args, **kwargs) 38 | self.id = u'' 39 | self.value = u'' 40 | self.property_id = u'' 41 | 42 | 43 | class Sku(BaseItem): 44 | 45 | def __init__(self, *args, **kwargs): 46 | super(Sku, self).__init__(*args, **kwargs) 47 | self.id = u'' 48 | self.name = u'' 49 | self.name_full = u'' 50 | self.international_abbr = u'' 51 | 52 | 53 | class Tax(BaseItem): 54 | 55 | def __init__(self, *args, **kwargs): 56 | super(Tax, self).__init__(*args, **kwargs) 57 | self.name = u'' 58 | self.value = Decimal() 59 | 60 | 61 | class AdditionalField(BaseItem): 62 | 63 | def __init__(self, *args, **kwargs): 64 | super(AdditionalField, self).__init__(*args, **kwargs) 65 | self.name = u'' 66 | self.value = u'' 67 | 68 | 69 | class Product(BaseItem): 70 | 71 | def __init__(self, *args, **kwargs): 72 | super(Product, self).__init__(*args, **kwargs) 73 | self.id = u'' 74 | self.name = u'' 75 | self.sku_id = u'' 76 | self.group_ids = [] 77 | self.properties = [] 78 | self.tax_name = u'' 79 | self.image_path = u'' 80 | self.additional_fields = [] 81 | 82 | 83 | class PriceType(BaseItem): 84 | 85 | def __init__(self, *args, **kwargs): 86 | super(PriceType, self).__init__(*args, **kwargs) 87 | self.id = u'' 88 | self.name = u'' 89 | self.currency = u'' 90 | self.tax_name = u'' 91 | self.tax_in_sum = False 92 | 93 | 94 | class Price(BaseItem): 95 | 96 | def __init__(self, *args, **kwargs): 97 | super(Price, self).__init__(*args, **kwargs) 98 | self.representation = u'' 99 | self.price_type_id = u'' 100 | self.price_for_sku = Decimal() 101 | self.currency_name = u'' 102 | self.sku_name = u'' 103 | self.sku_ratio = Decimal() 104 | 105 | 106 | class Offer(BaseItem): 107 | 108 | def __init__(self, *args, **kwargs): 109 | super(Offer, self).__init__(*args, **kwargs) 110 | self.id = u'' 111 | self.name = u'' 112 | self.sku_id = u'' 113 | self.prices = [] 114 | 115 | 116 | class Client(BaseItem): 117 | 118 | def __init__(self): 119 | self.id = u'' 120 | self.name = u'' 121 | self.role = u'Покупатель' 122 | self.full_name = u'' 123 | self.first_name = u'' 124 | self.last_name = u'' 125 | self.address = u'' 126 | 127 | 128 | class OrderItem(BaseItem): 129 | 130 | def __init__(self): 131 | self.id = u'' 132 | self.name = u'' 133 | self.sku = Sku(None) 134 | self.price = Decimal() 135 | self.quant = Decimal() 136 | self.sum = Decimal() 137 | 138 | 139 | class Order(BaseItem): 140 | 141 | def __init__(self, *args, **kwargs): 142 | super(Order, self).__init__(*args, **kwargs) 143 | self.id = u'' 144 | self.number = u'' 145 | self.date = datetime.now().date() 146 | self.currency_name = u'' 147 | self.currency_rate = Decimal() 148 | self.operation = u'Заказ товара' 149 | self.role = u'Продавец' 150 | self.sum = Decimal() 151 | self.client = Client() 152 | self.time = datetime.now().time() 153 | self.comment = u'' 154 | self.items = [] 155 | self.additional_fields = [] 156 | -------------------------------------------------------------------------------- /cml/auth.py: -------------------------------------------------------------------------------- 1 | # https://www.djangosnippets.org/snippets/243/ 2 | 3 | from __future__ import absolute_import 4 | import six 5 | import base64 6 | 7 | from django.http import HttpResponse 8 | from django.contrib.auth import authenticate, login 9 | 10 | 11 | def view_or_basicauth(view, request, test_func, realm = "", *args, **kwargs): 12 | """ 13 | This is a helper function used by both 'logged_in_or_basicauth' and 14 | 'has_perm_or_basicauth' that does the nitty of determining if they 15 | are already logged in or if they have provided proper http-authorization 16 | and returning the view if all goes well, otherwise responding with a 401. 17 | """ 18 | if test_func(request.user): 19 | # Already logged in, just return the view. 20 | # 21 | return view(request, *args, **kwargs) 22 | 23 | # They are not logged in. See if they provided login credentials 24 | # 25 | if 'HTTP_AUTHORIZATION' in request.META: 26 | auth = request.META['HTTP_AUTHORIZATION'].split() 27 | if len(auth) == 2: 28 | # NOTE: We are only support basic authentication for now. 29 | # 30 | if auth[0].lower() == "basic": 31 | if six.PY2: 32 | uname, passwd = base64.b64decode(auth[1]).split(':') 33 | else: 34 | uname, passwd = base64.b64decode(auth[1]).decode('utf-8').split(':') 35 | user = authenticate(username=uname, password=passwd) 36 | if user is not None: 37 | if user.is_active: 38 | login(request, user) 39 | request.user = user 40 | if test_func(request.user): 41 | return view(request, *args, **kwargs) 42 | 43 | # Either they did not provide an authorization header or 44 | # something in the authorization attempt failed. Send a 401 45 | # back to them to ask them to authenticate. 46 | # 47 | response = HttpResponse() 48 | response.status_code = 401 49 | response['WWW-Authenticate'] = 'Basic realm="%s"' % realm 50 | return response 51 | 52 | 53 | def logged_in_or_basicauth(realm = ""): 54 | """ 55 | A simple decorator that requires a user to be logged in. If they are not 56 | logged in the request is examined for a 'authorization' header. 57 | 58 | If the header is present it is tested for basic authentication and 59 | the user is logged in with the provided credentials. 60 | 61 | If the header is not present a http 401 is sent back to the 62 | requestor to provide credentials. 63 | 64 | The purpose of this is that in several django projects I have needed 65 | several specific views that need to support basic authentication, yet the 66 | web site as a whole used django's provided authentication. 67 | 68 | The uses for this are for urls that are access programmatically such as 69 | by rss feed readers, yet the view requires a user to be logged in. Many rss 70 | readers support supplying the authentication credentials via http basic 71 | auth (and they do NOT support a redirect to a form where they post a 72 | username/password.) 73 | 74 | Use is simple: 75 | 76 | @logged_in_or_basicauth 77 | def your_view: 78 | ... 79 | 80 | You can provide the name of the realm to ask for authentication within. 81 | """ 82 | def view_decorator(func): 83 | def wrapper(request, *args, **kwargs): 84 | return view_or_basicauth(func, request, 85 | lambda u: u.is_authenticated, 86 | realm, *args, **kwargs) 87 | return wrapper 88 | return view_decorator 89 | 90 | 91 | def has_perm_or_basicauth(perm, realm = ""): 92 | """ 93 | This is similar to the above decorator 'logged_in_or_basicauth' 94 | except that it requires the logged in user to have a specific 95 | permission. 96 | 97 | Use: 98 | 99 | @logged_in_or_basicauth('asforums.view_forumcollection') 100 | def your_view: 101 | ... 102 | 103 | """ 104 | def view_decorator(func): 105 | def wrapper(request, *args, **kwargs): 106 | return view_or_basicauth(func, request, 107 | lambda u: u.has_perm(perm), 108 | realm, *args, **kwargs) 109 | return wrapper 110 | return view_decorator 111 | -------------------------------------------------------------------------------- /cml/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | from __future__ import absolute_import 3 | import os 4 | import logging 5 | import importlib 6 | from io import BytesIO 7 | import six 8 | try: 9 | from xml.etree import cElementTree as ET 10 | except ImportError: 11 | from xml.etree import ElementTree as ET 12 | from .items import * 13 | from .conf import settings 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | class ImportManager(object): 19 | 20 | def __init__(self, file_path): 21 | self.file_path = file_path 22 | self.tree = None 23 | self.item_processor = ItemProcessor() 24 | 25 | def import_all(self): 26 | try: 27 | self.tree = self._get_tree() 28 | except Exception: 29 | logger.error('Import all error!') 30 | return 31 | self.import_classifier() 32 | self.import_catalogue() 33 | self.import_offers_pack() 34 | self.import_orders() 35 | logger.info('Import success!') 36 | 37 | def _get_tree(self): 38 | if self.tree is not None: 39 | return self.tree 40 | if not os.path.exists(self.file_path): 41 | message = 'File not found {}'.format(self.file_path) 42 | logger.error(message) 43 | raise OSError(message) 44 | try: 45 | tree = ET.parse(self.file_path) 46 | except Exception as e: 47 | message = 'File parse error {}'.format(self.file_path) 48 | logger.error(message) 49 | raise e 50 | return tree 51 | 52 | def _get_cleaned_text(self, element): 53 | try: 54 | text = element.text 55 | except: 56 | text = u'' 57 | if text is None: 58 | return u'' 59 | return text.strip(u' ') 60 | 61 | def import_classifier(self): 62 | try: 63 | tree = self._get_tree() 64 | except Exception: 65 | logger.error('Import classifier error!') 66 | return 67 | classifier_element = tree.find(u'Классификатор') 68 | if classifier_element is not None: 69 | self._parse_groups(classifier_element) 70 | self._parse_properties(classifier_element) 71 | 72 | def _parse_groups(self, current_element, parent_item=None): 73 | for group_element in current_element.findall(u'Группы/Группа'): 74 | group_item = Group(group_element) 75 | group_item.id = self._get_cleaned_text(group_element.find(u'Ид')) 76 | group_item.name = self._get_cleaned_text(group_element.find(u'Наименование')) 77 | if parent_item is not None: 78 | parent_item.groups.append(group_item) 79 | self._parse_groups(group_element, group_item) 80 | # processing only top level groups 81 | if parent_item is None: 82 | self.item_processor.process_item(group_item) 83 | 84 | def _parse_properties(self, current_element): 85 | for property_element in current_element.findall(u'Свойства/Свойство'): 86 | property_item = Property(property_element) 87 | property_item.id = self._get_cleaned_text(property_element.find(u'Ид')) 88 | property_item.name = self._get_cleaned_text(property_element.find(u'Наименование')) 89 | property_item.value_type = self._get_cleaned_text(property_element.find(u'ТипЗначений')) 90 | property_item.for_products = self._get_cleaned_text(property_element.find(u'ДляТоваров')) == u'true' 91 | self.item_processor.process_item(property_item) 92 | for variant_element in property_element.findall(u'ВариантыЗначений/{}'.format(property_item.value_type)): 93 | variant = PropertyVariant(variant_element) 94 | variant.id = self._get_cleaned_text(variant_element.find(u'ИдЗначения')) 95 | variant.value = self._get_cleaned_text(variant_element.find(u'Значение')) 96 | variant.property_id = property_item.id 97 | self.item_processor.process_item(variant) 98 | 99 | def import_catalogue(self): 100 | try: 101 | tree = self._get_tree() 102 | except Exception: 103 | logger.error('Import catalogue error!') 104 | return 105 | catalogue_element = tree.find(u'Каталог') 106 | if catalogue_element is not None: 107 | self._parse_products(catalogue_element) 108 | 109 | def _parse_products(self, current_element): 110 | for product_element in current_element.findall(u'Товары/Товар'): 111 | product_item = Product(product_element) 112 | product_item.id = self._get_cleaned_text(product_element.find(u'Ид')) 113 | product_item.name = self._get_cleaned_text(product_element.find(u'Наименование')) 114 | 115 | sku_element = product_element.find(u'БазоваяЕдиница') 116 | if sku_element is not None: 117 | sku_item = Sku(sku_element) 118 | sku_item.id = sku_element.get(u'Код') 119 | sku_item.name_full = sku_element.get(u'НаименованиеПолное') 120 | sku_item.international_abbr = sku_element.get(u'МеждународноеСокращение') 121 | sku_item.name = self._get_cleaned_text(sku_element) 122 | product_item.sku_id = sku_item.id 123 | self.item_processor.process_item(sku_item) 124 | 125 | image_element = product_element.find(u'Картинка') 126 | if image_element is not None: 127 | image_text = self._get_cleaned_text(image_element) 128 | try: 129 | image_filename = os.path.basename(image_text) 130 | except Exception: 131 | image_filename = u'' 132 | if image_filename: 133 | product_item.image_path = os.path.join(settings.CML_UPLOAD_ROOT, image_filename) 134 | 135 | for group_id_element in product_element.findall(u'Группы/Ид'): 136 | product_item.group_ids.append(self._get_cleaned_text(group_id_element)) 137 | 138 | for property_element in product_element.findall(u'ЗначенияСвойств/ЗначенияСвойства'): 139 | property_id = self._get_cleaned_text(property_element.find(u'Ид')) 140 | property_variant_id = self._get_cleaned_text(property_element.find(u'Значение')) 141 | if property_variant_id: 142 | product_item.properties.append((property_id, property_variant_id)) 143 | 144 | for tax_element in product_element.findall(u'СтавкиНалогов/СтавкаНалога'): 145 | tax_item = Tax(tax_element) 146 | tax_item.name = self._get_cleaned_text(tax_element.find(u'Наименование')) 147 | try: 148 | tax_item.value = Decimal(self._get_cleaned_text(tax_element.find(u'Ставка'))) 149 | except: 150 | tax_item.value = Decimal() 151 | self.item_processor.process_item(tax_item) 152 | product_item.tax_name = tax_item.name 153 | 154 | for additional_field_element in product_element.findall(u'ЗначенияРеквизитов/ЗначениеРеквизита'): 155 | additional_field = AdditionalField(additional_field_element) 156 | additional_field.name = self._get_cleaned_text(additional_field_element.find(u'Наименование')) 157 | additional_field.value = self._get_cleaned_text(additional_field_element.find(u'Значение')) 158 | product_item.additional_fields.append(additional_field) 159 | 160 | self.item_processor.process_item(product_item) 161 | 162 | def import_offers_pack(self): 163 | try: 164 | tree = self._get_tree() 165 | except Exception: 166 | logger.error('Import offers pack error!') 167 | return 168 | offers_pack_element = tree.find(u'ПакетПредложений') 169 | if offers_pack_element is not None: 170 | self._parse_price_types(offers_pack_element) 171 | self._parse_offers(offers_pack_element) 172 | 173 | def _parse_price_types(self, current_element): 174 | for price_type_element in current_element.findall(u'ТипыЦен/ТипЦены'): 175 | price_type_item = PriceType(price_type_element) 176 | price_type_item.id = self._get_cleaned_text(price_type_element.find(u'Ид')) 177 | price_type_item.name = self._get_cleaned_text(price_type_element.find(u'Наименование')) 178 | price_type_item.currency = self._get_cleaned_text(price_type_element.find(u'Валюта')) 179 | price_type_item.tax_name = self._get_cleaned_text(price_type_element.find(u'Налог/Наименование')) 180 | if self._get_cleaned_text(price_type_element.find(u'Налог/УчтеноВСумме')) == u'true': 181 | price_type_item.tax_in_sum = True 182 | self.item_processor.process_item(price_type_item) 183 | 184 | def _parse_offers(self, current_element): 185 | for offer_element in current_element.findall(u'Предложения/Предложение'): 186 | offer_item = Offer(offer_element) 187 | offer_item.id = self._get_cleaned_text(offer_element.find(u'Ид')) 188 | offer_item.name = self._get_cleaned_text(offer_element.find(u'Наименование')) 189 | 190 | sku_element = offer_element.find(u'БазоваяЕдиница') 191 | if sku_element is not None: 192 | sku_item = Sku(sku_element) 193 | sku_item.id = sku_element.get(u'Код') 194 | sku_item.name_full = sku_element.get(u'НаименованиеПолное') 195 | sku_item.international_abbr = sku_element.get(u'МеждународноеСокращение') 196 | sku_item.name = self._get_cleaned_text(sku_element) 197 | offer_item.sku_id = sku_item.id 198 | self.item_processor.process_item(sku_item) 199 | 200 | for price_element in offer_element.findall(u'Цены/Цена'): 201 | price_item = Price(price_element) 202 | price_item.representation = self._get_cleaned_text(price_element.find(u'Представление')) 203 | price_item.price_type_id = self._get_cleaned_text(price_element.find(u'ИдТипаЦены')) 204 | price_item.price_for_sku = Decimal(self._get_cleaned_text(price_element.find(u'ЦенаЗаЕдиницу'))) 205 | price_item.currency_name = self._get_cleaned_text(price_element.find(u'Валюта')) 206 | price_item.sku_name = self._get_cleaned_text(price_element.find(u'Единица')) 207 | price_item.sku_ratio = Decimal(self._get_cleaned_text(price_element.find(u'Коэффициент'))) 208 | offer_item.prices.append(price_item) 209 | 210 | self.item_processor.process_item(offer_item) 211 | 212 | def import_orders(self): 213 | try: 214 | tree = self._get_tree() 215 | except Exception: 216 | logger.error('Import orders error!') 217 | return 218 | orders_elements = tree.find(u'Документ') 219 | if orders_elements is not None: 220 | self._parse_orders(orders_elements) 221 | 222 | def _parse_orders(self, order_elements): 223 | for order_element in order_elements: 224 | order_item = Order(order_element) 225 | order_item.id = self._get_cleaned_text(order_element.find(u'Ид')) 226 | order_item.number = self._get_cleaned_text(order_element.find(u'Номер')) 227 | order_item.date = self._get_cleaned_text(order_element.find(u'Дата')) 228 | order_item.currency_name = self._get_cleaned_text(order_element.find(u'Валюта')) 229 | order_item.currency_rate = self._get_cleaned_text(order_element.find(u'Курс')) 230 | order_item.operation = self._get_cleaned_text(order_element.find(u'ХозОперация')) 231 | order_item.role = self._get_cleaned_text(order_element.find(u'Роль')) 232 | order_item.sum = self._get_cleaned_text(order_element.find(u'Сумма')) 233 | order_item.client.id = self._get_cleaned_text(order_element.find(u'Контрагенты/Контрагент/Ид')) 234 | order_item.client.name = self._get_cleaned_text(order_element.find(u'Контрагенты/Контрагент/Наименование')) 235 | order_item.client.full_name = self._get_cleaned_text( 236 | order_element.find(u'Контрагенты/Контрагент/ПолноеНаименование')) 237 | order_item.time = self._get_cleaned_text(order_element.find(u'Время')) 238 | order_item.comment = self._get_cleaned_text(order_element.find(u'Комментарий')) 239 | item_elements = order_element.find(u'Товары/Товар') 240 | if item_elements is not None: 241 | for item_element in item_elements: 242 | order_item_item = OrderItem(item_element) 243 | order_item_item.id = self._get_cleaned_text(item_element.find(u'Ид')) 244 | order_item_item.name = self._get_cleaned_text(item_element.find(u'Наименование')) 245 | sku_element = item_element.find(u'БазоваяЕдиница') 246 | if sku_element is not None: 247 | order_item_item.sku.id = sku_element.get(u'Код') 248 | order_item_item.sku.name = self._get_cleaned_text(sku_element) 249 | order_item_item.sku.name_full = sku_element.get(u'НаименованиеПолное') 250 | order_item_item.sku.international_abbr = sku_element.get(u'МеждународноеСокращение') 251 | order_item_item.price = self._get_cleaned_text(item_element.find(u'ЦенаЗаЕдиницу')) 252 | order_item_item.quant = self._get_cleaned_text(item_element.find(u'Количество')) 253 | order_item_item.sum = self._get_cleaned_text(item_element.find(u'Сумма')) 254 | order_item.items.append(order_item_item) 255 | additional_field_elements = order_element.find(u'ЗначенияРеквизитов/ЗначениеРеквизита') 256 | if additional_field_elements is not None: 257 | for additional_field_element in additional_field_elements: 258 | additional_field_item = AdditionalField(additional_field_element) 259 | additional_field_item.name = self._get_cleaned_text(item_element.find(u'Наименование')) 260 | additional_field_item.value = self._get_cleaned_text(item_element.find(u'Значение')) 261 | self.item_processor.process_item(order_item) 262 | 263 | 264 | class ExportManager(object): 265 | 266 | def __init__(self): 267 | self.item_processor = ItemProcessor() 268 | self.root = ET.Element(u'КоммерческаяИнформация') 269 | self.root.set(u'ВерсияСхемы', '2.05') 270 | self.root.set(u'ДатаФормирования', six.text_type(datetime.now().date())) 271 | 272 | def get_xml(self): 273 | f = BytesIO() 274 | tree = ET.ElementTree(self.root) 275 | tree.write(f, encoding='windows-1251', xml_declaration=True) 276 | return f.getvalue() 277 | 278 | def export_all(self): 279 | self.export_orders() 280 | 281 | def export_orders(self): 282 | for order in self.item_processor.yield_item(Order): 283 | order_element = ET.SubElement(self.root, u'Документ') 284 | ET.SubElement(order_element, u'Ид').text = six.text_type(order.id) 285 | ET.SubElement(order_element, u'Номер').text = six.text_type(order.number) 286 | ET.SubElement(order_element, u'Дата').text = six.text_type(order.date.strftime('%Y-%m-%d')) 287 | ET.SubElement(order_element, u'Время').text = six.text_type(order.time.strftime('%H:%M:%S')) 288 | ET.SubElement(order_element, u'ХозОперация').text = six.text_type(order.operation) 289 | ET.SubElement(order_element, u'Роль').text = six.text_type(order.role) 290 | ET.SubElement(order_element, u'Валюта').text = six.text_type(order.currency_name) 291 | ET.SubElement(order_element, u'Курс').text = six.text_type(order.currency_rate) 292 | ET.SubElement(order_element, u'Сумма').text = six.text_type(order.sum) 293 | ET.SubElement(order_element, u'Комментарий').text = six.text_type(order.comment) 294 | clients_element = ET.SubElement(order_element, u'Контрагенты') 295 | client_element = ET.SubElement(clients_element, u'Контрагент') 296 | ET.SubElement(client_element, u'Ид').text = six.text_type(order.client.id) 297 | ET.SubElement(client_element, u'Наименование').text = six.text_type(order.client.name) 298 | ET.SubElement(client_element, u'Роль').text = six.text_type(order.client.role) 299 | ET.SubElement(client_element, u'ПолноеНаименование').text = six.text_type(order.client.full_name) 300 | ET.SubElement(client_element, u'Фамилия').text = six.text_type(order.client.last_name) 301 | ET.SubElement(client_element, u'Имя').text = six.text_type(order.client.first_name) 302 | address_element = ET.SubElement(clients_element, u'АдресРегистрации') 303 | ET.SubElement(clients_element, u'Представление').text = six.text_type(order.client.address) 304 | products_element = ET.SubElement(order_element, u'Товары') 305 | for order_item in order.items: 306 | product_element = ET.SubElement(products_element, u'Товар') 307 | ET.SubElement(product_element, u'Ид').text = six.text_type(order_item.id) 308 | ET.SubElement(product_element, u'Наименование').text = six.text_type(order_item.name) 309 | sku_element = ET.SubElement(product_element, u'БазоваяЕдиница ') 310 | sku_element.set(u'Код', order_item.sku.id) 311 | sku_element.set(u'НаименованиеПолное', order_item.sku.name_full) 312 | sku_element.set(u'МеждународноеСокращение', order_item.sku.international_abbr) 313 | sku_element.text = order_item.sku.name 314 | ET.SubElement(product_element, u'ЦенаЗаЕдиницу').text = six.text_type(order_item.price) 315 | ET.SubElement(product_element, u'Количество').text = six.text_type(order_item.quant) 316 | ET.SubElement(product_element, u'Сумма').text = six.text_type(order_item.sum) 317 | 318 | def flush(self): 319 | self.item_processor.flush_pipeline(Order) 320 | 321 | 322 | class ItemProcessor(object): 323 | 324 | def __init__(self): 325 | self._project_pipelines = {} 326 | self._load_project_pipelines() 327 | 328 | def _load_project_pipelines(self): 329 | try: 330 | pipelines_module_name = settings.CML_PROJECT_PIPELINES 331 | except AttributeError: 332 | logger.info('Configure CML_PROJECT_PIPELINES in settings!') 333 | return 334 | try: 335 | pipelines_module = importlib.import_module(pipelines_module_name) 336 | except ImportError: 337 | return 338 | for item_class_name in PROCESSED_ITEMS: 339 | try: 340 | pipeline_class = getattr(pipelines_module, '{}Pipeline'.format(item_class_name)) 341 | except AttributeError: 342 | continue 343 | self._project_pipelines[item_class_name] = pipeline_class() 344 | 345 | def _get_project_pipeline(self, item_class): 346 | item_class_name = item_class.__name__ 347 | return self._project_pipelines.get(item_class_name, False) 348 | 349 | def process_item(self, item): 350 | project_pipeline = self._get_project_pipeline(item.__class__) 351 | if project_pipeline: 352 | try: 353 | project_pipeline.process_item(item) 354 | except Exception as e: 355 | logger.error('Error processing of item {}: {}'.format(item.__class__.__name__, repr(e))) 356 | 357 | def yield_item(self, item_class): 358 | project_pipeline = self._get_project_pipeline(item_class) 359 | if project_pipeline: 360 | try: 361 | return project_pipeline.yield_item() 362 | except Exception as e: 363 | logger.error('Error yielding item {}: {}'.format(item_class.__name__, repr(e))) 364 | return [] 365 | return [] 366 | 367 | def flush_pipeline(self, item_class): 368 | project_pipeline = self._get_project_pipeline(item_class) 369 | if project_pipeline: 370 | try: 371 | project_pipeline.flush() 372 | except Exception as e: 373 | logger.error('Error flushing pipeline for item {}: {}'.format(item_class.__name__, repr(e))) 374 | --------------------------------------------------------------------------------