├── tests ├── __init__.py ├── migrations │ ├── __init__.py │ └── 0001_initial.py ├── test_settings.py ├── test_serializer.py ├── test_report.py ├── models.py ├── factories.py └── test_core.py ├── deep_collector ├── __init__.py ├── compat │ ├── __init__.py │ ├── fields.py │ ├── builtins.py │ ├── meta.py │ └── serializers │ │ ├── __init__.py │ │ ├── django_1_6.py │ │ ├── django_1_9.py │ │ └── django_1_7.py ├── utils.py └── core.py ├── requirements.txt ├── .gitignore ├── MANIFEST.in ├── runtests.py ├── tox.ini ├── .travis.yml ├── setup.py ├── LICENSE ├── ChangeLog.rst └── README.rst /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deep_collector/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deep_collector/compat/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | factory_boy==2.10.0 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .tox/ 3 | build/ 4 | dist/ 5 | django_deep_collector.egg-info/ 6 | -------------------------------------------------------------------------------- /deep_collector/utils.py: -------------------------------------------------------------------------------- 1 | # Kept for backward compatibility. Should use core module now. 2 | from .core import * 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include ChangeLog.rst 4 | recursive-include deep_collector/compat * 5 | -------------------------------------------------------------------------------- /deep_collector/compat/fields.py: -------------------------------------------------------------------------------- 1 | import django 2 | 3 | if django.VERSION < (1, 7): 4 | from django.contrib.contenttypes.generic import GenericForeignKey, GenericRelation 5 | else: 6 | from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation 7 | -------------------------------------------------------------------------------- /deep_collector/compat/builtins.py: -------------------------------------------------------------------------------- 1 | 2 | try: 3 | from cStringIO import StringIO 4 | except ImportError: 5 | # Python 3.x 6 | from io import StringIO 7 | 8 | 9 | try: 10 | basestring = basestring 11 | except: 12 | # Python 3.x 13 | basestring = (str, bytes) 14 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.test_settings") 7 | from django.core.management import execute_from_command_line 8 | execute_from_command_line(sys.argv[:1] + ['test'] + sys.argv[1:]) 9 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = django111,django20,django21,django22,django31 3 | 4 | [testenv] 5 | commands = python runtests.py 6 | deps = 7 | factory_boy 8 | 9 | [testenv:django111] 10 | deps = 11 | django>=1.11,<1.11.99 12 | django-discover-runner 13 | {[testenv]deps} 14 | 15 | [testenv:django20] 16 | deps = 17 | django>=2.0,<2.0.99 18 | {[testenv]deps} 19 | 20 | [testenv:django21] 21 | deps = 22 | django>=2.1,<2.1.99 23 | {[testenv]deps} 24 | 25 | [testenv:django22] 26 | deps = 27 | django>=2.2,<3 28 | {[testenv]deps} 29 | 30 | [testenv:django31] 31 | deps = 32 | django>=3.1,<3.2 33 | {[testenv]deps} 34 | -------------------------------------------------------------------------------- /tests/test_settings.py: -------------------------------------------------------------------------------- 1 | import django 2 | 3 | 4 | SECRET_KEY = 'WE DONT CARE ABOUT IT' 5 | 6 | DATABASES = { 7 | 'default': { 8 | 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 9 | 'NAME': 'keepdb.db', # Or path to database file if using sqlite3. 10 | 'USER': '', # Not used with sqlite3. 11 | 'PASSWORD': '', # Not used with sqlite3. 12 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 13 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 14 | } 15 | } 16 | 17 | INSTALLED_APPS = ('django.contrib.contenttypes', 'tests') 18 | 19 | 20 | # Using DiscoverRunner before Django 1.6 to be able to use test files with 'test*' pattern name 21 | if django.VERSION < (1, 6): 22 | INSTALLED_APPS += ('discover_runner', ) 23 | TEST_RUNNER = 'discover_runner.DiscoverRunner' -------------------------------------------------------------------------------- /deep_collector/compat/meta.py: -------------------------------------------------------------------------------- 1 | import django 2 | 3 | if django.VERSION < (1, 8): 4 | def get_all_related_objects(obj): 5 | return obj._meta.get_all_related_objects() 6 | 7 | def get_all_related_m2m_objects_with_model(obj): 8 | return obj._meta.get_all_related_m2m_objects_with_model() 9 | 10 | def get_compat_local_fields(obj): 11 | # virtual_fields are used to collect GenericForeignKey objects 12 | return obj._meta.local_fields + obj._meta.virtual_fields 13 | else: 14 | def get_all_related_objects(obj): 15 | return [ 16 | f for f in obj._meta.get_fields() 17 | if (f.one_to_many or f.one_to_one) and 18 | f.auto_created and not f.concrete 19 | ] 20 | 21 | def get_all_related_m2m_objects_with_model(obj): 22 | return [ 23 | (f, f.model if f.model != obj.__class__ else None) 24 | for f in obj._meta.get_fields(include_hidden=True) 25 | if f.many_to_many and f.auto_created 26 | ] 27 | 28 | def get_compat_local_fields(obj): 29 | return obj._meta.get_fields() 30 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - 2.7 5 | - 3.4 6 | - 3.5 7 | - 3.6 8 | - 3.9 9 | 10 | env: 11 | - TOXENV=django111 12 | - TOXENV=django20 13 | - TOXENV=django21 14 | - TOXENV=django22 15 | - TOXENV=django31 16 | 17 | matrix: 18 | exclude: 19 | - python: 2.7 20 | env: TOXENV=django20 21 | - python: 2.7 22 | env: TOXENV=django21 23 | - python: 3.4 24 | env: TOXENV=django21 25 | - python: 2.7 26 | env: TOXENV=django22 27 | - python: 3.4 28 | env: TOXENV=django22 29 | - python: 2.7 30 | env: TOXENV=django31 31 | - python: 3.4 32 | env: TOXENV=django31 33 | - python: 3.5 34 | env: TOXENV=django31 35 | 36 | script: 37 | - tox 38 | 39 | install: 40 | - pip install tox 41 | - pip install factory_boy 42 | - pip install -e . 43 | 44 | deploy: 45 | provider: pypi 46 | user: iwoca 47 | password: 48 | secure: KNgZPIZ8xv1sC5i8/NiNuUFe5Daae5iM4dz7tbAStxTvJ8W793ybxaIKL9EIFefAHwi7R9prqH+VBt2cu+td6x23OZyfbi3Ci6bpBmKjBCCPr6itMBNjuHOepvBIUWiBcewrQ6seeEdigKzKu53b5VS4zUT6mWOlwnOJV4RO4YY= 49 | on: 50 | tags: true 51 | all_branches: true 52 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | from setuptools import setup 4 | 5 | with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as readme: 6 | README = readme.read() 7 | 8 | # allow setup.py to be run from any path 9 | os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) 10 | 11 | setup( 12 | name='django-deep-collector', 13 | version='0.6.0', 14 | packages=['deep_collector'], 15 | include_package_data=True, 16 | license='BSD License', 17 | description='A simple Django app to collect related objects.', 18 | long_description=README, 19 | url='http://github.com/iwoca/django-deep-collector/', 20 | author='iwoca ltd', 21 | classifiers=[ 22 | 'Environment :: Web Environment', 23 | 'Framework :: Django', 24 | 'Intended Audience :: Developers', 25 | 'License :: OSI Approved :: BSD License', 26 | 'Operating System :: OS Independent', 27 | 'Programming Language :: Python', 28 | 'Programming Language :: Python :: 2', 29 | 'Programming Language :: Python :: 2.7', 30 | 'Programming Language :: Python :: 3', 31 | 'Programming Language :: Python :: 3.4', 32 | 'Programming Language :: Python :: 3.5', 33 | 'Programming Language :: Python :: 3.6', 34 | 'Programming Language :: Python :: 3.9', 35 | 'Topic :: Internet :: WWW/HTTP', 36 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 37 | ] 38 | ) 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Individual contributors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, 7 | this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 13 | 3. Neither the name of iwoca nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /tests/test_serializer.py: -------------------------------------------------------------------------------- 1 | from copy import copy 2 | import json 3 | from django.test import TestCase 4 | 5 | from .factories import ChildModelFactory 6 | from deep_collector.compat.serializers import MultiModelInheritanceSerializer 7 | 8 | 9 | class TestMultiModelInheritanceSerializer(TestCase): 10 | 11 | def test_that_parent_model_fields_are_in_serializated_object_if_parent_is_not_abstract(self): 12 | child_model = ChildModelFactory.create() 13 | 14 | serializer = MultiModelInheritanceSerializer() 15 | json_objects = serializer.serialize([child_model]) 16 | 17 | child_model_dict = json.loads(json_objects)[0] 18 | serialized_fields = child_model_dict['fields'].keys() 19 | self.assertIn('child_field', serialized_fields) 20 | self.assertIn('o2o', serialized_fields) 21 | self.assertIn('fkey', serialized_fields) 22 | 23 | def test_that_we_dont_alter_model_class_meta_after_serialization(self): 24 | child_model = ChildModelFactory.create() 25 | local_fields_before = copy(child_model._meta.concrete_model._meta.local_fields) 26 | local_m2m_fields_before = copy(child_model._meta.concrete_model._meta.local_many_to_many) 27 | 28 | serializer = MultiModelInheritanceSerializer() 29 | serializer.serialize([child_model]) 30 | 31 | local_fields_after = copy(child_model._meta.concrete_model._meta.local_fields) 32 | local_m2m_fields_after = copy(child_model._meta.concrete_model._meta.local_many_to_many) 33 | 34 | self.assertEqual(local_fields_before, local_fields_after) 35 | self.assertEqual(local_m2m_fields_before, local_m2m_fields_after) 36 | -------------------------------------------------------------------------------- /tests/test_report.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from deep_collector.core import RelatedObjectsCollector 4 | 5 | from .factories import BaseModelFactory, ForeignKeyToBaseModelFactory 6 | 7 | 8 | class TestLogReportGeneration(TestCase): 9 | def test_report_with_no_debug_mode(self): 10 | obj = BaseModelFactory.create() 11 | 12 | collector = RelatedObjectsCollector() 13 | collector.collect(obj) 14 | report = collector.get_report() 15 | 16 | self.assertDictEqual(report, { 17 | 'excluded_fields': [], 18 | 'log': 'Set DEBUG to True to get collector internal logs' 19 | }) 20 | 21 | def test_report_with_debug_mode(self): 22 | self.maxDiff = None 23 | obj = BaseModelFactory.create() 24 | 25 | collector = RelatedObjectsCollector() 26 | collector.DEBUG = True 27 | collector.collect(obj) 28 | report = collector.get_report() 29 | 30 | self.assertEqual(report['excluded_fields'], []) 31 | # For now, just checking that the log report is not empty. 32 | # Some work has to be done to test it more. 33 | self.assertNotEqual(report['log'], []) 34 | 35 | 36 | class TestExcludedFieldLogReportGeneration(TestCase): 37 | 38 | def test_excluded_field_report(self): 39 | obj = BaseModelFactory.create() 40 | ForeignKeyToBaseModelFactory.create_batch(fkeyto=obj, size=3) 41 | 42 | collector = RelatedObjectsCollector() 43 | collector.MAXIMUM_RELATED_INSTANCES = 3 44 | collector.collect(obj) 45 | 46 | self.assertEquals(len(collector.get_report()['excluded_fields']), 0) 47 | 48 | # If we have more related objects than expected, we are not collecting them, to avoid a too big collection 49 | collector.MAXIMUM_RELATED_INSTANCES = 2 50 | collector.collect(obj) 51 | 52 | self.assertEquals(len(collector.get_report()['excluded_fields']), 1) 53 | -------------------------------------------------------------------------------- /deep_collector/compat/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | import django 3 | 4 | 5 | if django.VERSION < (1, 7): 6 | from .django_1_6 import CustomizableLocalFieldsSerializer 7 | elif django.VERSION < (1, 9): 8 | from .django_1_7 import CustomizableLocalFieldsSerializer 9 | else: 10 | from .django_1_9 import CustomizableLocalFieldsSerializer 11 | 12 | 13 | class MultiModelInheritanceSerializer(CustomizableLocalFieldsSerializer): 14 | ''' 15 | This serializer aims to fix serialization for multi-model inheritance 16 | This functionality has been removed because considered as too much "agressive" 17 | More precisions on this commit: https://github.com/django/django/commit/12716794db 18 | 19 | ''' 20 | def get_local_fields(self, concrete_model): 21 | local_fields = super(MultiModelInheritanceSerializer, self).get_local_fields(concrete_model) 22 | return local_fields + self.parent_local_fields 23 | 24 | def get_local_m2m_fields(self, concrete_model): 25 | # We convert in list because it returns a tuple in Django 1.8+ 26 | local_m2m_fields = list(super(MultiModelInheritanceSerializer, self).get_local_m2m_fields(concrete_model)) 27 | return local_m2m_fields + self.parent_local_m2m_fields 28 | 29 | def start_object(self, obj): 30 | # Initializing local fields we want to add current object (will be parent local fields) 31 | self.parent_local_fields = [] 32 | self.parent_local_m2m_fields = [] 33 | 34 | # Recursively getting parent fields to be added to serialization 35 | # We use concrete_model to avoid problems if we have to deal with proxy models 36 | concrete_model = obj._meta.concrete_model 37 | self.collect_parent_fields(concrete_model) 38 | 39 | super(MultiModelInheritanceSerializer, self).start_object(obj) 40 | 41 | def collect_parent_fields(self, model): 42 | # Collect parent fields that are not collected by default on non-abstract models. We call it recursively 43 | # to manage parents of parents, ... 44 | parents = model._meta.parents 45 | parents_to_collect = [parent for parent, parent_ptr in parents.items() if not parent._meta.abstract] 46 | 47 | for parent in parents_to_collect: 48 | self.parent_local_fields += parent._meta.local_fields 49 | self.parent_local_m2m_fields += parent._meta.local_many_to_many 50 | 51 | self.collect_parent_fields(parent._meta.concrete_model) 52 | -------------------------------------------------------------------------------- /deep_collector/compat/serializers/django_1_6.py: -------------------------------------------------------------------------------- 1 | 2 | from django.core.serializers.json import Serializer 3 | from django.utils import six 4 | 5 | 6 | class CustomizableLocalFieldsSerializer(Serializer): 7 | """ 8 | This is a not so elegant copy/paste from django.core.serializer.base.Serializer serialize method. 9 | We wanted to add parent fields of current serialized object because they are lacking when we want to import them 10 | again. 11 | We had to redefine serialize() method to add the possibility to subclass methods that are getting local 12 | fields to serialize (get_local_fields and get_local_m2m_fields) 13 | """ 14 | def serialize(self, queryset, **options): 15 | self.options = options 16 | 17 | self.stream = options.pop("stream", six.StringIO()) 18 | self.selected_fields = options.pop("fields", None) 19 | self.use_natural_keys = options.pop("use_natural_keys", False) 20 | 21 | self.start_serialization() 22 | self.first = True 23 | for obj in queryset: 24 | self.start_object(obj) 25 | # Use the concrete parent class' _meta instead of the object's _meta 26 | # This is to avoid local_fields problems for proxy models. Refs #17717. 27 | concrete_model = obj._meta.concrete_model 28 | for field in self.get_local_fields(concrete_model): 29 | if field.serialize: 30 | if field.rel is None: 31 | if self.selected_fields is None or field.attname in self.selected_fields: 32 | self.handle_field(obj, field) 33 | else: 34 | if self.selected_fields is None or field.attname[:-3] in self.selected_fields: 35 | self.handle_fk_field(obj, field) 36 | for field in self.get_local_m2m_fields(concrete_model): 37 | if field.serialize: 38 | if self.selected_fields is None or field.attname in self.selected_fields: 39 | self.handle_m2m_field(obj, field) 40 | self.end_object(obj) 41 | if self.first: 42 | self.first = False 43 | self.end_serialization() 44 | return self.getvalue() 45 | 46 | def get_local_fields(self, concrete_model): 47 | return concrete_model._meta.local_fields 48 | 49 | def get_local_m2m_fields(self, concrete_model): 50 | return concrete_model._meta.many_to_many 51 | -------------------------------------------------------------------------------- /ChangeLog.rst: -------------------------------------------------------------------------------- 1 | ChangeLog 2 | ========= 3 | 4 | 5 | .. _v0.5.0: 6 | 7 | 0.5.0 (2019-03-02) 8 | ------------------ 9 | 10 | *New:* 11 | 12 | - Renaming the main class to be ``DeepCollector`` instead of ``RelatedObjectsCollector`` (more consistent with the package naming) 13 | - Add an event logger method to easily customise collector event logs (instead of only relying on the default python logger) 14 | - Add compatibility for Django 2.0. 15 | 16 | 17 | .. _v0.4.2: 18 | 19 | 0.4.2 (2016-07-04) 20 | ------------------ 21 | 22 | *Bugfix:* 23 | 24 | - Fixing bug introduced in 0.4.1, related to inherited model collection. The django documentation was followed 25 | to help for compatibility_, but there is an issue related to inherited models and explained in this ticket_. 26 | 27 | .. _compatibility: https://docs.djangoproject.com/en/1.9/ref/models/meta/ 28 | .. _ticket: https://code.djangoproject.com/ticket/25461 29 | 30 | 31 | .. _v0.4.1: 32 | 33 | 0.4.1 (2016-06-29) 34 | ------------------ 35 | 36 | *New:* 37 | 38 | - Adding compat function to get related and related_m2m fields (_meta API updated in Django 1.9). 39 | 40 | 41 | .. _v0.4: 42 | 43 | 0.4 (2016-06-19) 44 | ---------------- 45 | 46 | *New:* 47 | 48 | - Adding ``GenericForeignKey`` support. 49 | - Adding compat function to get local fields (_meta API updated in Django 1.9). 50 | 51 | 52 | .. _v0.3.1: 53 | 54 | 0.3.1 (2016-04-06) 55 | ------------------ 56 | 57 | *New:* 58 | 59 | - Adding Django 1.9 compatibility. 60 | 61 | 62 | .. _v0.3: 63 | 64 | 65 | 0.3 (2016-02-11) 66 | ---------------- 67 | 68 | *New:* 69 | 70 | - Adding python 3 compatibility. 71 | - Adding Django 1.8 compatibility. 72 | 73 | *bugfix:* 74 | 75 | - Fixing collector bug when collecting a ``None`` child. 76 | 77 | 78 | .. _v0.2.1: 79 | 80 | 0.2.1 (2016-02-03) 81 | ------------------ 82 | 83 | *bugfix:* 84 | 85 | - Fixing MANIFEST.in to include ``compat`` module, not added in distribution version (packaging was broken). 86 | 87 | 88 | .. _v0.2: 89 | 90 | 0.2 (2016-02-02) 91 | ---------------- 92 | 93 | *New:* 94 | 95 | - Adding new ``compat`` module. 96 | - Now compatible with Django 1.7.x. 97 | 98 | 99 | .. _v0.1.1: 100 | 101 | 0.1.1 (2015-09-23) 102 | ------------------ 103 | 104 | *New:* 105 | 106 | - Adding new ``get_report`` method to have collect detailed informations. 107 | - Various bug fixes 108 | 109 | 110 | .. _v0.1: 111 | 112 | 0.1 (2015-04-21) 113 | ---------------- 114 | 115 | *New:* 116 | 117 | - Adding new ``RelatedObjectsCollector`` to collect every object that is related to given object. 118 | - Adding new ``MultiModelInheritanceSerializer`` to properly serialize collected item, to then be imported with Django `load_data` command. 119 | -------------------------------------------------------------------------------- /deep_collector/compat/serializers/django_1_9.py: -------------------------------------------------------------------------------- 1 | from django.core.serializers.json import Serializer 2 | 3 | from ..builtins import StringIO 4 | 5 | 6 | class CustomizableLocalFieldsSerializer(Serializer): 7 | """ 8 | This is a not so elegant copy/paste from django.core.serializer.base.Serializer serialize method. 9 | We wanted to add parent fields of current serialized object because they are lacking when we want to import them 10 | again. 11 | We had to redefine serialize() method to add the possibility to subclass methods that are getting local 12 | fields to serialize (get_local_fields and get_local_m2m_fields) 13 | """ 14 | internal_use_only = False 15 | 16 | def serialize(self, queryset, **options): 17 | """ 18 | Serialize a queryset. 19 | """ 20 | self.options = options 21 | 22 | self.stream = options.pop("stream", StringIO()) 23 | self.selected_fields = options.pop("fields", None) 24 | self.use_natural_foreign_keys = options.pop('use_natural_foreign_keys', False) 25 | self.use_natural_primary_keys = options.pop('use_natural_primary_keys', False) 26 | progress_bar = self.progress_class( 27 | options.pop('progress_output', None), options.pop('object_count', 0) 28 | ) 29 | 30 | self.start_serialization() 31 | self.first = True 32 | for count, obj in enumerate(queryset, start=1): 33 | self.start_object(obj) 34 | # Use the concrete parent class' _meta instead of the object's _meta 35 | # This is to avoid local_fields problems for proxy models. Refs #17717. 36 | concrete_model = obj._meta.concrete_model 37 | for field in self.get_local_fields(concrete_model): 38 | if field.serialize: 39 | if field.remote_field is None: 40 | if self.selected_fields is None or field.attname in self.selected_fields: 41 | self.handle_field(obj, field) 42 | else: 43 | if self.selected_fields is None or field.attname[:-3] in self.selected_fields: 44 | self.handle_fk_field(obj, field) 45 | for field in self.get_local_m2m_fields(concrete_model): 46 | if field.serialize: 47 | if self.selected_fields is None or field.attname in self.selected_fields: 48 | self.handle_m2m_field(obj, field) 49 | self.end_object(obj) 50 | progress_bar.update(count) 51 | if self.first: 52 | self.first = False 53 | self.end_serialization() 54 | return self.getvalue() 55 | 56 | def get_local_fields(self, concrete_model): 57 | return concrete_model._meta.local_fields 58 | 59 | def get_local_m2m_fields(self, concrete_model): 60 | return concrete_model._meta.many_to_many 61 | -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | 2 | from django.contrib.contenttypes.models import ContentType 3 | from django.db import models 4 | 5 | from deep_collector.compat.fields import GenericForeignKey, GenericRelation 6 | 7 | 8 | class FKDummyModel(models.Model): 9 | name = models.CharField(max_length=255) 10 | 11 | 12 | class O2ODummyModel(models.Model): 13 | name = models.CharField(max_length=255) 14 | 15 | 16 | class BaseModel(models.Model): 17 | name = models.CharField(max_length=255) 18 | fkey = models.ForeignKey(FKDummyModel, on_delete=models.CASCADE) 19 | o2o = models.OneToOneField(O2ODummyModel, on_delete=models.CASCADE) 20 | 21 | 22 | class SubClassOfBaseModel(BaseModel): 23 | pass 24 | 25 | 26 | class ManyToManyToBaseModel(models.Model): 27 | name = models.CharField(max_length=255) 28 | m2m = models.ManyToManyField(BaseModel) 29 | 30 | 31 | class ManyToManyToBaseModelWithRelatedName(models.Model): 32 | name = models.CharField(max_length=255) 33 | m2m = models.ManyToManyField(BaseModel, related_name='custom_related_m2m_name') 34 | 35 | 36 | class ForeignKeyToBaseModel(models.Model): 37 | name = models.CharField(max_length=255) 38 | fkeyto = models.ForeignKey(BaseModel, on_delete=models.CASCADE) 39 | 40 | 41 | class OneToOneToBaseModel(models.Model): 42 | name = models.CharField(max_length=255) 43 | o2oto = models.OneToOneField(BaseModel, on_delete=models.CASCADE) 44 | 45 | 46 | class ClassLevel1(models.Model): 47 | name = models.CharField(max_length=255) 48 | 49 | 50 | class ClassLevel2(models.Model): 51 | name = models.CharField(max_length=255) 52 | fkey = models.ForeignKey(ClassLevel1, on_delete=models.CASCADE) 53 | 54 | 55 | class ClassLevel3(models.Model): 56 | name = models.CharField(max_length=255) 57 | fkey = models.ForeignKey(ClassLevel2, on_delete=models.CASCADE) 58 | 59 | 60 | class ChildModel(BaseModel): 61 | child_field = models.CharField(max_length=255) 62 | 63 | 64 | class InvalidFKRootModel(models.Model): 65 | valid_fk = models.ForeignKey('InvalidFKNonRootModel', related_name='valid_non_root_fk', null=True, on_delete=models.CASCADE, db_constraint=False) 66 | invalid_fk = models.ForeignKey('InvalidFKNonRootModel', related_name='invalid_non_root_fk', null=True, on_delete=models.CASCADE, db_constraint=False) 67 | 68 | 69 | class InvalidFKNonRootModel(models.Model): 70 | valid_fk = models.ForeignKey(InvalidFKRootModel, related_name='valid_root_fk', null=True, on_delete=models.CASCADE, db_constraint=False) 71 | invalid_fk = models.ForeignKey(InvalidFKRootModel, related_name='invalid_root_fk', null=True, on_delete=models.CASCADE, db_constraint=False) 72 | 73 | 74 | class GFKModel(models.Model): 75 | content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) 76 | object_id = models.PositiveIntegerField() 77 | content_object = GenericForeignKey('content_type', 'object_id') 78 | 79 | 80 | class BaseToGFKModel(models.Model): 81 | gfk_relation = GenericRelation(GFKModel) 82 | -------------------------------------------------------------------------------- /deep_collector/compat/serializers/django_1_7.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from django.core.serializers.json import Serializer 3 | from django.utils import six 4 | from django.utils.deprecation import RemovedInDjango19Warning 5 | 6 | 7 | class CustomizableLocalFieldsSerializer(Serializer): 8 | """ 9 | This is a not so elegant copy/paste from django.core.serializer.base.Serializer serialize method. 10 | We wanted to add parent fields of current serialized object because they are lacking when we want to import them 11 | again. 12 | We had to redefine serialize() method to add the possibility to subclass methods that are getting local 13 | fields to serialize (get_local_fields and get_local_m2m_fields) 14 | """ 15 | # Indicates if the implemented serializer is only available for 16 | # internal Django use. 17 | internal_use_only = False 18 | 19 | def serialize(self, queryset, **options): 20 | """ 21 | Serialize a queryset. 22 | """ 23 | self.options = options 24 | 25 | self.stream = options.pop("stream", six.StringIO()) 26 | self.selected_fields = options.pop("fields", None) 27 | self.use_natural_keys = options.pop("use_natural_keys", False) 28 | if self.use_natural_keys: 29 | warnings.warn("``use_natural_keys`` is deprecated; use ``use_natural_foreign_keys`` instead.", 30 | RemovedInDjango19Warning) 31 | self.use_natural_foreign_keys = options.pop('use_natural_foreign_keys', False) or self.use_natural_keys 32 | self.use_natural_primary_keys = options.pop('use_natural_primary_keys', False) 33 | 34 | self.start_serialization() 35 | self.first = True 36 | for obj in queryset: 37 | self.start_object(obj) 38 | # Use the concrete parent class' _meta instead of the object's _meta 39 | # This is to avoid local_fields problems for proxy models. Refs #17717. 40 | concrete_model = obj._meta.concrete_model 41 | for field in self.get_local_fields(concrete_model): 42 | if field.serialize: 43 | if field.rel is None: 44 | if self.selected_fields is None or field.attname in self.selected_fields: 45 | self.handle_field(obj, field) 46 | else: 47 | if self.selected_fields is None or field.attname[:-3] in self.selected_fields: 48 | self.handle_fk_field(obj, field) 49 | for field in self.get_local_m2m_fields(concrete_model): 50 | if field.serialize: 51 | if self.selected_fields is None or field.attname in self.selected_fields: 52 | self.handle_m2m_field(obj, field) 53 | self.end_object(obj) 54 | if self.first: 55 | self.first = False 56 | self.end_serialization() 57 | return self.getvalue() 58 | 59 | def get_local_fields(self, concrete_model): 60 | return concrete_model._meta.local_fields 61 | 62 | def get_local_m2m_fields(self, concrete_model): 63 | return concrete_model._meta.many_to_many 64 | -------------------------------------------------------------------------------- /tests/factories.py: -------------------------------------------------------------------------------- 1 | import factory 2 | 3 | 4 | from .models import (FKDummyModel, O2ODummyModel, BaseModel, ManyToManyToBaseModel, 5 | ForeignKeyToBaseModel, OneToOneToBaseModel, ClassLevel1, ClassLevel2, ClassLevel3, 6 | ManyToManyToBaseModelWithRelatedName, ChildModel, SubClassOfBaseModel) 7 | 8 | 9 | class FKDummyModelFactory(factory.django.DjangoModelFactory): 10 | name = factory.Sequence(lambda x: "FKDummyModelName#{number}".format(number=str(x))) 11 | 12 | class Meta: 13 | model = FKDummyModel 14 | 15 | class O2ODummyModelFactory(factory.django.DjangoModelFactory): 16 | name = factory.Sequence(lambda x: "O2ODummyModelName#{number}".format(number=str(x))) 17 | 18 | class Meta: 19 | model = O2ODummyModel 20 | 21 | 22 | class BaseModelFactory(factory.django.DjangoModelFactory): 23 | name = factory.Sequence(lambda x: "BaseModelName#{number}".format(number=str(x))) 24 | fkey = factory.SubFactory(FKDummyModelFactory) 25 | o2o = factory.SubFactory(O2ODummyModelFactory) 26 | 27 | class Meta: 28 | model = BaseModel 29 | 30 | 31 | class SubClassOfBaseModelFactory(BaseModelFactory): 32 | class Meta: 33 | model = SubClassOfBaseModel 34 | 35 | 36 | class ManyToManyToBaseModelFactory(factory.django.DjangoModelFactory): 37 | name = factory.Sequence(lambda x: "MaynyToManyToBaseModelName#{number}".format(number=str(x))) 38 | 39 | class Meta: 40 | model = ManyToManyToBaseModel 41 | 42 | @factory.post_generation 43 | def base_models(self, create, extracted, **kwargs): 44 | if not create: 45 | return 46 | if extracted: 47 | for base_model in extracted: 48 | self.m2m.add(base_model) 49 | 50 | 51 | class ManyToManyToBaseModelWithRelatedNameFactory(factory.django.DjangoModelFactory): 52 | name = factory.Sequence(lambda x: "MaynyToManyToBaseModelName#{number}".format(number=str(x))) 53 | 54 | class Meta: 55 | model = ManyToManyToBaseModelWithRelatedName 56 | 57 | @factory.post_generation 58 | def base_models(self, create, extracted, **kwargs): 59 | if not create: 60 | return 61 | if extracted: 62 | for base_model in extracted: 63 | self.m2m.add(base_model) 64 | 65 | class ForeignKeyToBaseModelFactory(factory.django.DjangoModelFactory): 66 | name = factory.Sequence(lambda x: "ForeignKeyToBseModelName#{number}".format(number=str(x))) 67 | fkeyto = factory.SubFactory(BaseModelFactory) 68 | 69 | class Meta: 70 | model = ForeignKeyToBaseModel 71 | 72 | 73 | class OneToOneToBaseModelFactory(factory.django.DjangoModelFactory): 74 | name = factory.Sequence(lambda x: "OneToOneToBaseModelName#{number}".format(number=str(x))) 75 | o2oto = factory.SubFactory(BaseModelFactory) 76 | 77 | class Meta: 78 | model = OneToOneToBaseModel 79 | 80 | 81 | class ClassLevel1Factory(factory.django.DjangoModelFactory): 82 | name = factory.Sequence(lambda x: "ClassLevel1#{number}".format(number=str(x))) 83 | 84 | class Meta: 85 | model = ClassLevel1 86 | 87 | 88 | class ClassLevel2Factory(factory.django.DjangoModelFactory): 89 | name = factory.Sequence(lambda x: "ClassLevel2#{number}".format(number=str(x))) 90 | fkey = factory.SubFactory(ClassLevel1Factory) 91 | 92 | class Meta: 93 | model = ClassLevel2 94 | 95 | 96 | class ClassLevel3Factory(factory.django.DjangoModelFactory): 97 | name = factory.Sequence(lambda x: "ClassLevel3#{number}".format(number=str(x))) 98 | fkey = factory.SubFactory(ClassLevel2Factory) 99 | 100 | class Meta: 101 | model = ClassLevel3 102 | 103 | 104 | class ChildModelFactory(BaseModelFactory): 105 | child_field = factory.Sequence(lambda x: "ChildField#{number}".format(number=str(x))) 106 | 107 | class Meta: 108 | model = ChildModel 109 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | Deep Collector 3 | ============== 4 | 5 | .. image:: https://travis-ci.org/iwoca/django-deep-collector.svg?branch=master 6 | :target: https://travis-ci.org/iwoca/django-deep-collector.svg 7 | 8 | Django custom collector used to get every objects that are related to given object. 9 | 10 | Install 11 | ======= 12 | 13 | :: 14 | 15 | $ pip install django-deep-collector 16 | 17 | 18 | Usage 19 | ===== 20 | 21 | Create a new instance of DeepCollector, and launch collector on one object: 22 | 23 | .. code-block:: python 24 | 25 | from deep_collector.core import DeepCollector 26 | from django.contrib.auth.models import User 27 | 28 | user = User.objects.all()[0] 29 | collector = DeepCollector() 30 | collector.collect(user) 31 | related_objects = collector.get_collected_objects() 32 | 33 | If you want to save it in a file to be 'django load_data'-like imported, you can use: 34 | 35 | .. code-block:: python 36 | 37 | string_buffer = collector.get_json_serialized_objects() 38 | 39 | 40 | How it works 41 | ============ 42 | 43 | This class is used to introspect an object, to get every other objects that depend on it, following its 44 | 'relation' fields, i.e. ForeignKey, OneToOneField, ManyToManyField, GenericForeignKey and GenericRelation. 45 | 46 | 1. We start from given object (of class classA for example), and loop over : 47 | 48 | - Its 'direct' fields, it means the relation fields that are explicitly declared in this django model. 49 | 50 | .. code-block:: python 51 | 52 | class classA(models.Model): 53 | fkey = models.ForeignKey(classB) 54 | o2o = models.OneToOneField(classC) 55 | m2m = models.ManyToManyField(classD) 56 | 57 | 58 | - Its 'related' fields, so other django model that are related to this object by relation fields. 59 | 60 | .. code-block:: python 61 | 62 | class classB(models.Model): 63 | fkeyto = models.ForeignKey(classA) 64 | 65 | class classC(models.Model): 66 | o2oto = models.OneToOneField(classA) 67 | 68 | class classD(models.Model): 69 | m2mto = models.ManyToManyField(classA) 70 | 71 | 72 | 2. For every field, we get associated object(s) of objA: 73 | 74 | - If it's a direct field, we get objects by: 75 | 76 | .. code-block:: python 77 | 78 | class classA(models.Model): 79 | fkey = models.ForeignKey(classB) # objA.fkey 80 | o2o = models.OneToOneField(classC) # objA.o2o 81 | m2m = models.ManyToManyField(classD) # objA.m2m.all() 82 | 83 | 84 | - If it's a related field, we get objects by: 85 | 86 | .. code-block:: python 87 | 88 | class classB(models.Model): 89 | fkeyto = models.ForeignKey(classA) # objA.classb_set.all() 90 | 91 | class classC(models.Model): 92 | o2oto = models.OneToOneField(classA) # objA.classC (not a manager, because OneToOneField is a unique rel) 93 | 94 | class classD(models.Model): 95 | m2mto = models.ManyToManyField(classA) # objA.classd_set.all() 96 | 97 | 98 | If we are using related_name attribute, then we access manager with its related_name: 99 | 100 | .. code-block:: python 101 | 102 | class classE(models.Model): 103 | m2mto = models.ForeignKey(classA, related_name='classE') # objA.classE.all() 104 | 105 | 3. For each associated object, we go back to step 1. and get every field, ... 106 | 107 | 108 | GenericForeignKey 109 | ================= 110 | 111 | The `GenericForeignKey` has a small exception. If you want it to be collected in the "reverse" way, you should 112 | explicitly define a `GenericRelation` in the models you want to follow this "reverse" relation. 113 | 114 | .. code-block:: python 115 | 116 | class GFKModel(models.Model): 117 | content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) 118 | object_id = models.PositiveIntegerField() 119 | content_object = GenericForeignKey('content_type', 'object_id') 120 | 121 | 122 | class BaseToGFKModel(models.Model): 123 | gfk_relation = GenericRelation(GFKModel) 124 | 125 | In above example, if you collect a `BaseToGFKModel` instance, the collector will look for all `GFKModel` instances 126 | related to your initial `BaseToGFKModel` instance. 127 | That happens because the `BaseToGFKModel` model explicitly defines a `GenericRelation`. 128 | 129 | 130 | Parameters 131 | ========== 132 | 133 | You can customize which model/field is collected. 134 | By default, every model and field is collected, but you can override some parameters to have custom behaviour: 135 | 136 | - `EXCLUDE_MODELS`: exclude models (expecting a list of '.') 137 | 138 | .. code-block:: python 139 | 140 | EXCLUDE_MODELS = ['sites.site', 'auth.permission', 'auth.group'] 141 | 142 | Every time we will try to collect an object of this model type, it won't be collected. 143 | 144 | - `EXCLUDE_DIRECT_FIELDS`: exclude direct fields from specified models 145 | 146 | .. code-block:: python 147 | 148 | EXCLUDE_DIRECT_FIELDS = { 149 | 'auth.user': ['groups'], 150 | } 151 | 152 | On User model, when we will get direct fields, we won't take into account 'groups' field. 153 | 154 | - `EXCLUDE_RELATED_FIELDS`: exclude related fields from specified models 155 | 156 | .. code-block:: python 157 | 158 | EXCLUDE_RELATED_FIELDS = { 159 | 'auth.user': ['session_set'] 160 | } 161 | 162 | On User model, we don't want to collect sessions that are associated to this user, so we put the exact accessor name we have to use to get these sessions, 'session_set', to exclude them from collection. 163 | 164 | - `ALLOWS_SAME_TYPE_AS_ROOT_COLLECT`: avoid by default to collect objects that have the same type as the root one, to prevent collecting too many data. 165 | 166 | Miscellaneous 167 | ============= 168 | 169 | To avoid some recursive collect between 2 objects (if an object has a direct field to another one, it means that other object has a related field to this first one), we detect if an object has already been collected before trying to collect it. 170 | -------------------------------------------------------------------------------- /tests/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.19 on 2021-03-31 07:46 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | initial = True 10 | 11 | dependencies = [ 12 | ('contenttypes', '0002_remove_content_type_name'), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='BaseModel', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('name', models.CharField(max_length=255)), 21 | ], 22 | ), 23 | migrations.CreateModel( 24 | name='BaseToGFKModel', 25 | fields=[ 26 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 27 | ], 28 | ), 29 | migrations.CreateModel( 30 | name='ClassLevel1', 31 | fields=[ 32 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 33 | ('name', models.CharField(max_length=255)), 34 | ], 35 | ), 36 | migrations.CreateModel( 37 | name='ClassLevel2', 38 | fields=[ 39 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 40 | ('name', models.CharField(max_length=255)), 41 | ('fkey', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tests.ClassLevel1')), 42 | ], 43 | ), 44 | migrations.CreateModel( 45 | name='FKDummyModel', 46 | fields=[ 47 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 48 | ('name', models.CharField(max_length=255)), 49 | ], 50 | ), 51 | migrations.CreateModel( 52 | name='InvalidFKNonRootModel', 53 | fields=[ 54 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 55 | ], 56 | ), 57 | migrations.CreateModel( 58 | name='O2ODummyModel', 59 | fields=[ 60 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 61 | ('name', models.CharField(max_length=255)), 62 | ], 63 | ), 64 | migrations.CreateModel( 65 | name='ChildModel', 66 | fields=[ 67 | ('basemodel_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='tests.BaseModel')), 68 | ('child_field', models.CharField(max_length=255)), 69 | ], 70 | bases=('tests.basemodel',), 71 | ), 72 | migrations.CreateModel( 73 | name='SubClassOfBaseModel', 74 | fields=[ 75 | ('basemodel_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='tests.BaseModel')), 76 | ], 77 | bases=('tests.basemodel',), 78 | ), 79 | migrations.CreateModel( 80 | name='OneToOneToBaseModel', 81 | fields=[ 82 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 83 | ('name', models.CharField(max_length=255)), 84 | ('o2oto', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='tests.BaseModel')), 85 | ], 86 | ), 87 | migrations.CreateModel( 88 | name='ManyToManyToBaseModelWithRelatedName', 89 | fields=[ 90 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 91 | ('name', models.CharField(max_length=255)), 92 | ('m2m', models.ManyToManyField(related_name='custom_related_m2m_name', to='tests.BaseModel')), 93 | ], 94 | ), 95 | migrations.CreateModel( 96 | name='ManyToManyToBaseModel', 97 | fields=[ 98 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 99 | ('name', models.CharField(max_length=255)), 100 | ('m2m', models.ManyToManyField(to='tests.BaseModel')), 101 | ], 102 | ), 103 | migrations.CreateModel( 104 | name='InvalidFKRootModel', 105 | fields=[ 106 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 107 | ('invalid_fk', models.ForeignKey(db_constraint=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='invalid_non_root_fk', to='tests.InvalidFKNonRootModel')), 108 | ('valid_fk', models.ForeignKey(db_constraint=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='valid_non_root_fk', to='tests.InvalidFKNonRootModel')), 109 | ], 110 | ), 111 | migrations.AddField( 112 | model_name='invalidfknonrootmodel', 113 | name='invalid_fk', 114 | field=models.ForeignKey(db_constraint=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='invalid_root_fk', to='tests.InvalidFKRootModel'), 115 | ), 116 | migrations.AddField( 117 | model_name='invalidfknonrootmodel', 118 | name='valid_fk', 119 | field=models.ForeignKey(db_constraint=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='valid_root_fk', to='tests.InvalidFKRootModel'), 120 | ), 121 | migrations.CreateModel( 122 | name='GFKModel', 123 | fields=[ 124 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 125 | ('object_id', models.PositiveIntegerField()), 126 | ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')), 127 | ], 128 | ), 129 | migrations.CreateModel( 130 | name='ForeignKeyToBaseModel', 131 | fields=[ 132 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 133 | ('name', models.CharField(max_length=255)), 134 | ('fkeyto', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tests.BaseModel')), 135 | ], 136 | ), 137 | migrations.CreateModel( 138 | name='ClassLevel3', 139 | fields=[ 140 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 141 | ('name', models.CharField(max_length=255)), 142 | ('fkey', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tests.ClassLevel2')), 143 | ], 144 | ), 145 | migrations.AddField( 146 | model_name='basemodel', 147 | name='fkey', 148 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tests.FKDummyModel'), 149 | ), 150 | migrations.AddField( 151 | model_name='basemodel', 152 | name='o2o', 153 | field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='tests.O2ODummyModel'), 154 | ), 155 | ] 156 | -------------------------------------------------------------------------------- /tests/test_core.py: -------------------------------------------------------------------------------- 1 | 2 | from django.test import TestCase 3 | 4 | from .factories import (BaseModelFactory, ManyToManyToBaseModelFactory, 5 | ForeignKeyToBaseModelFactory, ClassLevel3Factory, 6 | ManyToManyToBaseModelWithRelatedNameFactory, 7 | SubClassOfBaseModelFactory) 8 | from deep_collector.utils import DeepCollector, RelatedObjectsCollector 9 | from .models import (ForeignKeyToBaseModel, InvalidFKRootModel, InvalidFKNonRootModel, BaseModel, GFKModel, 10 | BaseToGFKModel) 11 | 12 | 13 | class TestDirectRelations(TestCase): 14 | 15 | def test_get_foreign_key_object(self): 16 | obj = BaseModelFactory.create() 17 | 18 | collector = DeepCollector() 19 | collector.collect(obj) 20 | self.assertIn(obj.fkey, collector.get_collected_objects()) 21 | 22 | def test_get_one_to_one_object(self): 23 | obj = BaseModelFactory.create() 24 | 25 | collector = DeepCollector() 26 | collector.collect(obj) 27 | self.assertIn(obj.o2o, collector.get_collected_objects()) 28 | 29 | def test_get_many_to_many_object(self): 30 | obj = BaseModelFactory.create() 31 | m2m_model = ManyToManyToBaseModelFactory.create(base_models=[obj]) 32 | 33 | collector = DeepCollector() 34 | collector.collect(m2m_model) 35 | self.assertIn(obj, collector.get_collected_objects()) 36 | 37 | def test_one_to_one_object_inherited(self): 38 | obj = SubClassOfBaseModelFactory.create() 39 | 40 | collector = DeepCollector() 41 | collector.collect(obj) 42 | self.assertIn(obj.o2o, collector.get_collected_objects()) 43 | 44 | 45 | class TestReverseRelations(TestCase): 46 | 47 | def test_get_reverse_foreign_key_object(self): 48 | fkey_model = ForeignKeyToBaseModelFactory.create() 49 | 50 | collector = DeepCollector() 51 | collector.collect(fkey_model.fkeyto) 52 | self.assertIn(fkey_model, collector.get_collected_objects()) 53 | 54 | def test_get_reverse_many_to_many_object(self): 55 | obj = BaseModelFactory.create() 56 | m2m_model = ManyToManyToBaseModelFactory.create(base_models=[obj]) 57 | 58 | collector = DeepCollector() 59 | collector.collect(obj) 60 | self.assertIn(m2m_model, collector.get_collected_objects()) 61 | 62 | 63 | class TestNestedObjects(TestCase): 64 | 65 | def test_recursive_foreign_keys(self): 66 | level3 = ClassLevel3Factory.create() 67 | level2 = level3.fkey 68 | level1 = level2.fkey 69 | 70 | collector = DeepCollector() 71 | 72 | # Double reverse field collection 73 | collector.collect(level1) 74 | collected_objs = collector.get_collected_objects() 75 | self.assertIn(level1, collected_objs) 76 | self.assertIn(level2, collected_objs) 77 | self.assertIn(level3, collected_objs) 78 | 79 | # Middle class collection (1 direct field and 1 reverse field) 80 | collector.collect(level2) 81 | collected_objs = collector.get_collected_objects() 82 | self.assertIn(level1, collected_objs) 83 | self.assertIn(level2, collected_objs) 84 | self.assertIn(level3, collected_objs) 85 | 86 | # Double direct field collection 87 | collector.collect(level3) 88 | collected_objs = collector.get_collected_objects() 89 | self.assertIn(level1, collected_objs) 90 | self.assertIn(level2, collected_objs) 91 | self.assertIn(level3, collected_objs) 92 | 93 | 94 | class TestCollectorParameters(TestCase): 95 | 96 | def test_model_is_excluded_when_defined_in_models_exclude_list(self): 97 | obj = BaseModelFactory.create() 98 | 99 | collector = DeepCollector() 100 | collector.EXCLUDE_MODELS = ['tests.o2odummymodel'] 101 | collector.collect(obj) 102 | 103 | self.assertNotIn(obj.o2o, collector.get_collected_objects()) 104 | 105 | def test_direct_field_is_excluded_when_defined_in_direct_field_exclude_list(self): 106 | obj = BaseModelFactory.create() 107 | 108 | collector = DeepCollector() 109 | collector.EXCLUDE_DIRECT_FIELDS = { 110 | 'tests.basemodel': ['fkey'] 111 | } 112 | collector.collect(obj) 113 | 114 | self.assertNotIn(obj.fkey, collector.get_collected_objects()) 115 | 116 | def test_related_field_is_excluded_when_defined_in_related_field_exclude_list(self): 117 | obj = BaseModelFactory.create() 118 | m2m_model = ManyToManyToBaseModelFactory.create(base_models=[obj]) 119 | 120 | collector = DeepCollector() 121 | collector.EXCLUDE_RELATED_FIELDS = { 122 | 'tests.basemodel': ['manytomanytobasemodel_set'] 123 | } 124 | collector.collect(obj) 125 | 126 | self.assertNotIn(m2m_model, collector.get_collected_objects()) 127 | 128 | def test_related_field__with_related_name_is_excluded_when_defined_in_related_field_exclude_list(self): 129 | obj = BaseModelFactory.create() 130 | m2m_model = ManyToManyToBaseModelWithRelatedNameFactory.create(base_models=[obj]) 131 | 132 | collector = DeepCollector() 133 | collector.EXCLUDE_RELATED_FIELDS = { 134 | 'tests.basemodel': ['custom_related_m2m_name'] 135 | } 136 | collector.collect(obj) 137 | 138 | self.assertNotIn(m2m_model, collector.get_collected_objects()) 139 | 140 | def test_parameter_to_avoid_collect_if_too_many_related_objects(self): 141 | obj = BaseModelFactory.create() 142 | ForeignKeyToBaseModelFactory.create_batch(fkeyto=obj, size=3) 143 | 144 | collector = DeepCollector() 145 | collector.MAXIMUM_RELATED_INSTANCES = 3 146 | collector.collect(obj) 147 | collected_objects = collector.get_collected_objects() 148 | 149 | self.assertEquals(len([x for x in collected_objects if isinstance(x, ForeignKeyToBaseModel)]), 3) 150 | 151 | # If we have more related objects than expected, we are not collecting them, to avoid a too big collection 152 | collector.MAXIMUM_RELATED_INSTANCES = 2 153 | collector.collect(obj) 154 | collected_objects = collector.get_collected_objects() 155 | self.assertEquals(len([x for x in collected_objects if isinstance(x, ForeignKeyToBaseModel)]), 0) 156 | 157 | self.assertDictEqual({ 158 | 'parent_instance': u'tests.basemodel.%s' % obj.pk, 159 | 'field_name': u'foreignkeytobasemodel_set', 160 | 'related_model': u'tests.foreignkeytobasemodel', 161 | 'count': 3, 162 | 'max_count': 2, 163 | }, collector.get_report()['excluded_fields'][0]) 164 | 165 | def test_parameter_to_avoid_collect_on_specific_model_if_too_many_related_objects(self): 166 | obj = BaseModelFactory.create() 167 | fkey1 = ForeignKeyToBaseModelFactory(fkeyto=obj) 168 | fkey2 = ForeignKeyToBaseModelFactory(fkeyto=obj) 169 | fkey3 = ForeignKeyToBaseModelFactory(fkeyto=obj) 170 | 171 | collector = DeepCollector() 172 | # If model is specified in MAXIMUM_RELATED_INSTANCES_PER_MODEL, we don't take into account 173 | # MAXIMUM_RELATED_INSTANCES parameter 174 | collector.MAXIMUM_RELATED_INSTANCES = 1 175 | collector.MAXIMUM_RELATED_INSTANCES_PER_MODEL = {'tests.foreignkeytobasemodel': 3} 176 | collector.collect(obj) 177 | collected_objects = collector.get_collected_objects() 178 | 179 | self.assertEquals(len([x for x in collected_objects if isinstance(x, ForeignKeyToBaseModel)]), 3) 180 | self.assertEquals(len(collector.get_report()['excluded_fields']), 0) 181 | 182 | # If we have more related objects than expected, we are not collecting them, to avoid a too big collection 183 | collector = DeepCollector() 184 | collector.MAXIMUM_RELATED_INSTANCES = 1 185 | collector.MAXIMUM_RELATED_INSTANCES_PER_MODEL = {'tests.foreignkeytobasemodel': 2} 186 | collector.collect(obj) 187 | collected_objects = collector.get_collected_objects() 188 | 189 | self.assertEquals(len([x for x in collected_objects if isinstance(x, ForeignKeyToBaseModel)]), 0) 190 | 191 | self.assertEquals(len(collector.get_report()['excluded_fields']), 1) 192 | self.assertDictEqual({ 193 | 'parent_instance': u'tests.basemodel.%s' % obj.pk, 194 | 'field_name': u'foreignkeytobasemodel_set', 195 | 'related_model': u'tests.foreignkeytobasemodel', 196 | 'count': 3, 197 | 'max_count': 2, 198 | }, collector.get_report()['excluded_fields'][0]) 199 | 200 | def test_parameter_to_avoid_collect_if_too_many_related_objects_through_many_to_many_field(self): 201 | obj1 = BaseModelFactory.create() 202 | obj2 = BaseModelFactory.create() 203 | obj3 = BaseModelFactory.create() 204 | root_obj = ManyToManyToBaseModelFactory.create(base_models=[obj1, obj2, obj3]) 205 | 206 | collector = DeepCollector() 207 | collector.MAXIMUM_RELATED_INSTANCES = 3 208 | collector.collect(root_obj) 209 | collected_objects = collector.get_collected_objects() 210 | 211 | self.assertEquals(len([x for x in collected_objects if isinstance(x, BaseModel)]), 3) 212 | self.assertEquals(len(collector.get_report()['excluded_fields']), 0) 213 | 214 | # If we have more related objects than expected, we are not collecting them, to avoid a too big collection 215 | collector.MAXIMUM_RELATED_INSTANCES = 2 216 | collector.collect(root_obj) 217 | collected_objects = collector.get_collected_objects() 218 | self.assertEquals(len([x for x in collected_objects if isinstance(x, BaseModel)]), 0) 219 | 220 | self.assertEquals(len(collector.get_report()['excluded_fields']), 1) 221 | self.assertDictEqual({ 222 | 'parent_instance': u'tests.manytomanytobasemodel.%s' % root_obj.pk, 223 | 'field_name': u'm2m', 224 | 'related_model': u'tests.basemodel', 225 | 'count': 3, 226 | 'max_count': 2, 227 | }, collector.get_report()['excluded_fields'][0]) 228 | 229 | def test_parameter_to_avoid_collect_on_specific_model_if_too_many_related_objects_through_many_to_many_field(self): 230 | obj1 = BaseModelFactory.create() 231 | obj2 = BaseModelFactory.create() 232 | obj3 = BaseModelFactory.create() 233 | root_obj = ManyToManyToBaseModelFactory.create(base_models=[obj1, obj2, obj3]) 234 | 235 | collector = DeepCollector() 236 | collector.MAXIMUM_RELATED_INSTANCES = 1 237 | collector.MAXIMUM_RELATED_INSTANCES_PER_MODEL = {'tests.basemodel': 3} 238 | collector.collect(root_obj) 239 | collected_objects = collector.get_collected_objects() 240 | 241 | self.assertEquals(len([x for x in collected_objects if isinstance(x, BaseModel)]), 3) 242 | self.assertEquals(len(collector.get_report()['excluded_fields']), 0) 243 | 244 | # If we have more related objects than expected, we are not collecting them, to avoid a too big collection 245 | collector.MAXIMUM_RELATED_INSTANCES = 1 246 | collector.MAXIMUM_RELATED_INSTANCES_PER_MODEL = {'tests.basemodel': 2} 247 | collector.collect(root_obj) 248 | collected_objects = collector.get_collected_objects() 249 | self.assertEquals(len([x for x in collected_objects if isinstance(x, BaseModel)]), 0) 250 | 251 | self.assertEquals(len(collector.get_report()['excluded_fields']), 1) 252 | self.assertDictEqual({ 253 | 'parent_instance': u'tests.manytomanytobasemodel.%s' % root_obj.pk, 254 | 'field_name': u'm2m', 255 | 'related_model': u'tests.basemodel', 256 | 'count': 3, 257 | 'max_count': 2, 258 | }, collector.get_report()['excluded_fields'][0]) 259 | 260 | 261 | class TestPostCollect(TestCase): 262 | @staticmethod 263 | def _generate_invalid_id(model): 264 | instance = model.objects.create() 265 | invalid_id = instance.id 266 | instance.delete() 267 | return invalid_id 268 | 269 | def test_invalid_foreign_key_doesnt_cause_matching_query_does_not_exist_exception(self): 270 | root = InvalidFKRootModel.objects.create() 271 | non_root = InvalidFKNonRootModel.objects.create() 272 | 273 | root.invalid_fk_id = self._generate_invalid_id(InvalidFKNonRootModel) 274 | root.valid_fk = non_root 275 | root.save() 276 | 277 | non_root.invalid_fk_id = self._generate_invalid_id(InvalidFKRootModel) 278 | non_root.valid_fk = root 279 | non_root.save() 280 | 281 | # when post_collect() is executed on the InvalidFKRootModel instance 282 | # this invalid foreign key shouldn't cause an error like: 283 | # "DoesNotExist: InvalidFKRootModel matching query does not exist." 284 | # OR 285 | # "DoesNotExist: InvalidFKNonRootModel matching query does not exist." 286 | 287 | collector = DeepCollector() 288 | collector.collect(root) 289 | 290 | self.assertIn(root, collector.get_collected_objects()) 291 | self.assertIn(non_root, collector.get_collected_objects()) 292 | 293 | 294 | class TestGFKRelation(TestCase): 295 | 296 | def test_get_gfk_key_object(self): 297 | obj = BaseModelFactory.create() 298 | gfkmodel = GFKModel.objects.create(content_object=obj) 299 | 300 | collector = DeepCollector() 301 | collector.collect(gfkmodel) 302 | self.assertIn(obj, collector.get_collected_objects()) 303 | 304 | def test_doesnt_automatically_get_reverse_gfk_key_object(self): 305 | obj = BaseModelFactory.create() 306 | gfkmodel = GFKModel.objects.create(content_object=obj) 307 | 308 | collector = DeepCollector() 309 | collector.collect(obj) 310 | self.assertNotIn(gfkmodel, collector.get_collected_objects()) 311 | 312 | def test_get_reverse_gfk_key_object_if_generic_relation_is_explicitly_defined(self): 313 | obj = BaseToGFKModel.objects.create() 314 | gfkmodel = GFKModel.objects.create(content_object=obj) 315 | 316 | collector = DeepCollector() 317 | collector.collect(obj) 318 | self.assertIn(gfkmodel, collector.get_collected_objects()) 319 | 320 | def test_reverse_objects_collection_limit(self): 321 | obj = BaseToGFKModel.objects.create() 322 | gfkmodel = GFKModel.objects.create(content_object=obj) 323 | gfkmodel2 = GFKModel.objects.create(content_object=obj) 324 | 325 | collector = DeepCollector() 326 | collector.MAXIMUM_RELATED_INSTANCES_PER_MODEL = {'tests.gfkmodel': 1} 327 | collector.collect(obj) 328 | self.assertNotIn(gfkmodel, collector.get_collected_objects()) 329 | self.assertNotIn(gfkmodel2, collector.get_collected_objects()) 330 | 331 | 332 | class TestBackwardCompatibility(TestCase): 333 | 334 | def test_get_foreign_key_object(self): 335 | obj = BaseModelFactory.create() 336 | 337 | collector = RelatedObjectsCollector() 338 | collector.collect(obj) 339 | self.assertIn(obj.fkey, collector.get_collected_objects()) 340 | -------------------------------------------------------------------------------- /deep_collector/core.py: -------------------------------------------------------------------------------- 1 | 2 | import logging 3 | 4 | import django 5 | from django.db.models import ForeignKey, OneToOneField 6 | 7 | from .compat.builtins import basestring, StringIO 8 | from .compat.fields import GenericForeignKey, GenericRelation 9 | from .compat.meta import (get_all_related_objects, 10 | get_all_related_m2m_objects_with_model, 11 | get_compat_local_fields) 12 | from .compat.serializers import MultiModelInheritanceSerializer 13 | 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | class DeepCollector(object): 18 | """ 19 | This class is used to introspect an object, to get every other objects that depend on it, following its 20 | 'relation' fields, i.e. ForeignKey, OneToOneField and ManyToManyField. 21 | 22 | 1. We start from given object (of class classA for example), and loop over : 23 | 24 | - Its 'direct' fields, it means the relation fields that are explicitly declared in this django model. 25 | >>> class classA(models.Model): 26 | >>> fkey = models.ForeignKey(classB) 27 | >>> o2o = models.OneToOneField(classC) 28 | >>> m2m = models.ManyToManyField(classD) 29 | 30 | - Its 'related' fields, so other django model that are related to this object by relation fields. 31 | >>> class classB(models.Model): 32 | >>> fkeyto = models.ForeignKey(classA) 33 | >>> 34 | >>> class classC(models.Model): 35 | >>> o2oto = models.OneToOneField(classA) 36 | >>> 37 | >>> class classD(models.Model): 38 | >>> m2mto = models.ManyToManyField(classA) 39 | 40 | 2. For every field, we get associated object(s) of objA: 41 | 42 | - If it's a direct field, we get objects by: 43 | >>> class classA(models.Model): 44 | >>> fkey = models.ForeignKey(classB) # objA.fkey 45 | >>> o2o = models.OneToOneField(classC) # objA.o2o 46 | >>> m2m = models.ManyToManyField(classD) # objA.m2m.all() 47 | 48 | - If it's a related field, we get objects by: 49 | >>> class classB(models.Model): 50 | >>> fkeyto = models.ForeignKey(classA) # objA.classb_set.all() 51 | >>> 52 | >>> class classC(models.Model): 53 | >>> o2oto = models.OneToOneField(classA) # objA.classC (not a manager, because OneToOneField is a unique rel) 54 | >>> 55 | >>> class classD(models.Model): 56 | >>> m2mto = models.ManyToManyField(classA) # objA.classd_set.all() 57 | 58 | If we are using related_name attribute, then we access manager with its related_name: 59 | >>> class classE(models.Model): 60 | >>> m2mto = models.ForeignKey(classA, related_name='classE') # objA.classE.all() 61 | 62 | 3. For each associated object, we go back to step 1. and get every field, ... 63 | 64 | ------------------------------------------------------------------------------------------------------------------ 65 | HOWTO use: 66 | 67 | Create a new instance of RelatedObjectsCollector, and launch collector on one object: 68 | >>> from deep_collector.core import DeepCollector 69 | >>> from django.contrib.auth.models import User 70 | >>> 71 | >>> user = User.objects.all()[0] 72 | >>> collector = DeepCollector() 73 | >>> collector.collect(user) 74 | >>> related_objects = collector.get_all_related_objects() 75 | 76 | If you want to save it in a file to be 'django load_data'-like imported, you can use: 77 | >>> string_buffer = collector.get_json_serialized_objects() 78 | 79 | ------------------------------------------------------------------------------------------------------------------ 80 | PARAMETERS: 81 | 82 | You can customize which model/field is collected. 83 | By default, every model and field is collected, but you can override some parameters to have custom behaviour: 84 | 85 | - EXCLUDE_MODELS: exclude models (expecting a list of '.') 86 | >>> EXCLUDE_MODELS = ['sites.site', 'auth.permission', 'contenttypes.contenttype', 'auth.group'] 87 | Every time we will try to collect an object of this model type, it won't be collected. 88 | 89 | - EXCLUDE_DIRECT_FIELDS: exclude direct fields from specified models 90 | >>> EXCLUDE_DIRECT_FIELDS = { 91 | 'auth.user': ['groups'], 92 | } 93 | On User model, when we will get direct fields, we won't take into account 'groups' field. 94 | 95 | - EXCLUDE_RELATED_FIELDS: Exclude related fields from specified models 96 | >>> EXCLUDE_RELATED_FIELDS = { 97 | 'auth.user': ['session_set'] 98 | } 99 | On User model, we don't want to collect sessions that are associated to this user, so we put the exact accessor 100 | name we have to use to get these session, 'session_set', to exclude it from collecting. 101 | 102 | ------------------------------------------------------------------------------------------------------------------ 103 | MISCELLANEOUS: 104 | 105 | To avoid some recursive collect between 2 objects (if an object has a direct field to another one, it means that 106 | other object has a related field to this first one), we detect if an object has already been collected before 107 | trying to collect it. 108 | 109 | We are also avoiding by default to collect objects that have the same type as the root one, to prevent collecting 110 | too many data. This behaviour can be changed with ALLOWS_SAME_TYPE_AS_ROOT_COLLECT parameter. 111 | """ 112 | 113 | # Models that won't be introspected. 114 | EXCLUDE_MODELS = [] 115 | 116 | # Direct fields that won't be introspected 117 | EXCLUDE_DIRECT_FIELDS = {} 118 | 119 | # Related fields that won't be introspected 120 | EXCLUDE_RELATED_FIELDS = {} 121 | 122 | # Allow to recursively collect other objects that have same type as root collected object. 123 | ALLOWS_SAME_TYPE_AS_ROOT_COLLECT = False 124 | 125 | MAXIMUM_RELATED_INSTANCES = 50 126 | # We are settings related instances maximum size depending on the model 127 | MAXIMUM_RELATED_INSTANCES_PER_MODEL = {} 128 | 129 | # To be used if you want a detailed report on different collector steps. 130 | DEBUG = False 131 | 132 | def clean_by_fields(self, obj, fields, get_field_fn, exclude_list): 133 | """ 134 | Function used to exclude defined fields from object collect. 135 | :param obj: the object we are collecting 136 | :param fields: every field related to this object (direct or reverse one) 137 | :param get_field_fn: function used to get accessor for each field 138 | :param exclude_list: model/fields we have defined to be excluded from collect 139 | :return: fields that are allowed to be collected 140 | """ 141 | cleaned_list = [] 142 | obj_model = get_model_from_instance(obj) 143 | 144 | for field in fields: 145 | field_accessor = get_field_fn(field) 146 | # This field is excluded if: 147 | # 1/ it's parent model key is in exclude list keys 148 | # AND 149 | # 2/ the field has been defined as excluded for this parent model 150 | is_excluded = obj_model in exclude_list and field_accessor in exclude_list[obj_model] 151 | 152 | if not is_excluded: 153 | cleaned_list.append(field) 154 | 155 | return cleaned_list 156 | 157 | def get_report(self): 158 | return { 159 | 'excluded_fields': self.excluded_fields, 160 | 'log': self.saved_log if self.DEBUG else "Set DEBUG to True to get collector internal logs", 161 | } 162 | 163 | def get_collected_objects(self): 164 | return self.collected_objs.values() 165 | 166 | def get_json_serialized_objects(self): 167 | objects = self.get_collected_objects() 168 | 169 | string_buffer = StringIO() 170 | 171 | serializer = MultiModelInheritanceSerializer() 172 | serializer.serialize( 173 | objects, 174 | stream=string_buffer, 175 | indent=2 176 | ) 177 | string_buffer.seek(0) 178 | 179 | return string_buffer 180 | 181 | def emit_event(self, **kwargs): 182 | if self.DEBUG: 183 | self.saved_log.append(kwargs) 184 | 185 | def _is_already_collected(self, parent, obj): 186 | new_key = get_key_from_instance(obj) 187 | is_already_collected = new_key in self.collected_objs 188 | 189 | if is_already_collected: 190 | self.emit_event(type='already_collected', obj=obj, parent=parent) 191 | 192 | return is_already_collected 193 | 194 | def _is_excluded_model(self, obj): 195 | obj_model = get_model_from_instance(obj) 196 | is_excluded_model = obj_model in self.EXCLUDE_MODELS 197 | 198 | if is_excluded_model: 199 | self.emit_event(type='exluded_model', obj=obj) 200 | 201 | return is_excluded_model 202 | 203 | def _is_same_type_as_root(self, obj): 204 | """ 205 | Testing if we try to collect an object of the same type as root. 206 | This is not really a good sign, because it means that we are going to collect a whole new tree, that will 207 | maybe collect a new tree, that will... 208 | """ 209 | if not self.ALLOWS_SAME_TYPE_AS_ROOT_COLLECT: 210 | obj_model = get_model_from_instance(obj) 211 | obj_key = get_key_from_instance(obj) 212 | is_same_type_as_root = obj_model == self.root_obj_model and obj_key != self.root_obj_key 213 | 214 | if is_same_type_as_root: 215 | self.emit_event(type='same_type_as_root', obj=obj) 216 | 217 | return is_same_type_as_root 218 | else: 219 | return False 220 | 221 | def is_excluded_from_collect(self, parent, obj): 222 | is_excluded_from_collect = \ 223 | self._is_already_collected(parent, obj)\ 224 | or self._is_excluded_model(obj)\ 225 | or self._is_same_type_as_root(obj) 226 | 227 | return is_excluded_from_collect 228 | 229 | def add_to_collected_object(self, parent, obj): 230 | new_key = get_key_from_instance(obj) 231 | 232 | self.collected_objs[new_key] = obj 233 | 234 | self.emit_event(type='object_collected', obj=obj, parent=parent) 235 | 236 | model = get_model_from_instance(obj) 237 | if model in self.collected_objs_history: 238 | self.collected_objs_history[model] += 1 239 | else: 240 | self.collected_objs_history[model] = 1 241 | 242 | self.emit_event(type='object_collect_history', objs=self.collected_objs_history) 243 | 244 | def collect(self, root_obj): 245 | # Resetting collected_objs if several collects are called. 246 | self.objects_to_collect = [(None, root_obj)] 247 | self.collected_objs = {} 248 | self.collected_objs_history = {} 249 | 250 | self.root_obj = root_obj 251 | self.root_obj_key = get_key_from_instance(root_obj) 252 | self.root_obj_model = get_model_from_instance(root_obj) 253 | 254 | self.excluded_fields = [] 255 | self.saved_log = [] 256 | 257 | while self.objects_to_collect: 258 | parent, obj = self.objects_to_collect.pop() 259 | children = self._collect(parent, obj) 260 | 261 | tmp_objects_to_collect = [] 262 | for child in children: 263 | if child: 264 | tmp_objects_to_collect.append((obj, child)) 265 | else: 266 | self.emit_event(type='child_none', obj=obj, parent=parent) 267 | self.objects_to_collect += tmp_objects_to_collect 268 | 269 | def filter_by_threshold(self, objects, current_instance, field_name): 270 | """ 271 | If the field we are currently working on has too many objects related to it, we want to restrict it 272 | depending on a settings-driven threshold. 273 | :param objects: The objects we want to filter 274 | :param current_obj: The current collected instance 275 | :param field_name: The current field name 276 | :return: 277 | """ 278 | objs_count = len(objects) 279 | if objs_count == 0: 280 | return [] 281 | object_example = objects[0] 282 | 283 | related_model_name = get_model_from_instance(object_example) 284 | max_count = self.get_maximum_allowed_instances_for_model(related_model_name) 285 | if objs_count > max_count: 286 | self.emit_event(type='too_many_related_objects', obj=current_instance, related_model=related_model_name) 287 | self.add_excluded_field(get_key_from_instance(current_instance), field_name, 288 | related_model_name, objs_count, max_count) 289 | return [] 290 | 291 | return objects 292 | 293 | def add_excluded_field(self, parent_instance_key, field_name, related_model_name, count, max_count): 294 | self.excluded_fields.append({ 295 | 'parent_instance': parent_instance_key, 296 | 'field_name': field_name, 297 | 'related_model': related_model_name, 298 | 'count': count, 299 | 'max_count': max_count, 300 | }) 301 | 302 | def _collect(self, parent, obj): 303 | if self.is_excluded_from_collect(parent, obj): 304 | return [] 305 | obj = self.pre_collect(obj) 306 | self.add_to_collected_object(parent, obj) 307 | 308 | # Local objects are explicit fields on current object model 309 | local_objs = self.get_local_objs(obj) 310 | 311 | # Related objects are fields defined in other models that can refer to current model 312 | related_objs = self.get_related_objs(obj) 313 | 314 | self.post_collect(obj) 315 | return local_objs + related_objs 316 | 317 | def pre_collect(self, obj): 318 | return obj 319 | 320 | def post_collect(self, obj): 321 | """ 322 | We want to manage the side-effect of not collecting other items of the same type as root model. 323 | If for example, you run the collect on a specific user that is linked to a model "A" linked (ForeignKey) 324 | to ANOTHER user. 325 | Then the collect won't collect this other user, but the collected model "A" will keep the ForeignKey value of 326 | a user we are not collecting. 327 | For now, we set the ForeignKey of ANOTHER user to the root one, to be sure that model "A" will always be linked 328 | to an existing user (of course, the meaning is changing, but only if this field is not unique. 329 | 330 | Before: 331 | user1 -> modelA -> user2843 332 | 333 | After collection: 334 | user1 -> modelA -> user1 335 | """ 336 | if not self.ALLOWS_SAME_TYPE_AS_ROOT_COLLECT: 337 | for field in self.get_local_fields(obj): 338 | if isinstance(field, ForeignKey) and not field.unique: 339 | # Relative field's API has been changed Django 2.0 340 | # See https://docs.djangoproject.com/en/2.0/releases/1.9/#field-rel-changes for details 341 | if django.VERSION[0] >= 2: 342 | remote_model = field.remote_field.model 343 | else: 344 | remote_model = field.rel.to 345 | if isinstance(self.root_obj, remote_model): 346 | setattr(obj, field.name, self.root_obj) 347 | 348 | def get_local_fields(self, obj): 349 | # Use the concrete parent class' _meta instead of the object's _meta 350 | # This is to avoid local_fields problems for proxy models. Refs #17717. 351 | concrete_model = obj._meta.concrete_model 352 | return self.clean_by_fields(obj, get_compat_local_fields(concrete_model), 353 | lambda x: x.name, self.EXCLUDE_DIRECT_FIELDS) 354 | 355 | def get_local_m2m_fields(self, obj): 356 | # Use the concrete parent class' _meta instead of the object's _meta 357 | # This is to avoid local_fields problems for proxy models. Refs #17717. 358 | concrete_model = obj._meta.concrete_model 359 | return self.clean_by_fields(obj, concrete_model._meta.local_many_to_many, 360 | lambda x: x.name, self.EXCLUDE_DIRECT_FIELDS) 361 | 362 | def get_maximum_allowed_instances_for_model(self, model): 363 | if model in self.MAXIMUM_RELATED_INSTANCES_PER_MODEL: 364 | return self.MAXIMUM_RELATED_INSTANCES_PER_MODEL[model] 365 | 366 | return self.MAXIMUM_RELATED_INSTANCES 367 | 368 | def get_local_objs(self, obj): 369 | local_objs = [] 370 | 371 | for field in self.get_local_fields(obj): 372 | if isinstance(field, ForeignKey) or isinstance(field, GenericForeignKey): 373 | self.emit_event(type='local_field', obj=obj, field=field) 374 | try: 375 | instance = getattr(obj, field.name) 376 | if instance: 377 | self.emit_event(type='local_field_w_instance', obj=obj, field=field) 378 | local_objs.append(instance) 379 | else: 380 | self.emit_event(type='local_field_wo_instance', obj=obj, field=field) 381 | except Exception as e: 382 | self.emit_event(type='local_field_no_instance', obj=obj, field=field) 383 | elif isinstance(field, GenericRelation): 384 | self.emit_event(type='local_reverse_generic_field', obj=obj, field=field) 385 | generic_manager = getattr(obj, field.name) 386 | local_objs += self.filter_by_threshold(generic_manager.all(), obj, field.name) 387 | 388 | for field in self.get_local_m2m_fields(obj): 389 | self.emit_event(type='local_m2m_field', obj=obj, field=field) 390 | m2m_manager = getattr(obj, field.name) 391 | objs_count = m2m_manager.count() 392 | 393 | if not objs_count: 394 | self.emit_event(type='local_m2m_field_wo_instance', obj=obj, field=field) 395 | else: 396 | self.emit_event(type='local_m2m_field_w_instance', obj=obj, field=field, number=objs_count) 397 | local_objs += self.filter_by_threshold(m2m_manager.all(), obj, field.name) 398 | 399 | return local_objs 400 | 401 | def get_related_fields(self, obj): 402 | return self.clean_by_fields(obj, get_all_related_objects(obj), 403 | lambda x: x.get_accessor_name(), self.EXCLUDE_RELATED_FIELDS) 404 | 405 | def get_related_m2m_fields(self, obj): 406 | return self.clean_by_fields(obj, get_all_related_m2m_objects_with_model(obj), 407 | lambda x:x[0].get_accessor_name(), self.EXCLUDE_RELATED_FIELDS) 408 | 409 | def get_related_objs(self, obj): 410 | related_objs = [] 411 | 412 | for related_field in self.get_related_fields(obj): 413 | self.emit_event(type='related_field', obj=obj, field=related_field) 414 | related_objs += self.query_related_objects(related_field, [obj]) 415 | 416 | for related_field, _ in self.get_related_m2m_fields(obj): 417 | self.emit_event(type='related_m2m_field', obj=obj, field=related_field) 418 | related_objs += self.query_related_objects(related_field, [obj]) 419 | 420 | return related_objs 421 | 422 | def query_related_objects(self, related, objs): 423 | related_objs = [] 424 | 425 | try: 426 | related_obj_or_manager = getattr(objs[0], related.get_accessor_name()) 427 | 428 | if isinstance(related.field, OneToOneField): 429 | related_objs = [related_obj_or_manager] 430 | else: 431 | related_objs = list(related_obj_or_manager.all()) 432 | # TODO: make this exception less broad 433 | except Exception: 434 | self.emit_event(type='error_related_object', obj=objs[0], field=related) 435 | 436 | if not related_objs: 437 | self.emit_event(type='no_related_object', obj=objs[0], field=related) 438 | else: 439 | self.emit_event(type='related_objects', obj=objs[0], field=related, number=len(related_objs)) 440 | 441 | related_objs = self.filter_by_threshold(related_objs, objs[0], related.get_accessor_name()) 442 | 443 | return related_objs 444 | 445 | 446 | def get_model_from_instance(obj): 447 | if obj is None: 448 | return '' 449 | 450 | try: 451 | meta = obj._meta 452 | except AttributeError: 453 | meta = obj.model._meta 454 | 455 | # in django 1.8 _meta.module_name was renamed to _meta.model_name 456 | model_name = meta.model_name if hasattr(meta, 'model_name') else meta.module_name 457 | model = meta.app_label + '.' + model_name 458 | return model 459 | 460 | 461 | def get_key_from_instance(obj): 462 | if obj is None: 463 | return '' 464 | return get_model_from_instance(obj) + '.' + str(obj.pk) 465 | 466 | 467 | # For backward compatibility 468 | RelatedObjectsCollector = DeepCollector 469 | --------------------------------------------------------------------------------