├── .coveragerc ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── CONTRIBUTING.md ├── CONTRIBUTORS ├── LICENSE ├── MANIFEST.in ├── README.md ├── entity ├── __init__.py ├── apps.py ├── config.py ├── constants.py ├── exceptions.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── sync_entities.py ├── migrations │ ├── 0001_0010_squashed.py │ ├── 0001_initial.py │ ├── 0002_entitygroup_logic_string.py │ ├── 0002_entitykind_is_active.py │ ├── 0003_auto_20150813_2234.py │ ├── 0004_auto_20150915_1747.py │ ├── 0005_remove_entitygroup_entities.py │ ├── 0006_entity_relationship_unique.py │ ├── 0007_allentityproxy.py │ ├── 0008_auto_20180329_1934.py │ ├── 0009_auto_20180402_2145.py │ ├── 0010_auto_20181213_1817.py │ └── __init__.py ├── models.py ├── signal_handlers.py ├── sync.py ├── tests │ ├── __init__.py │ ├── migrations │ │ ├── 0001_initial.py │ │ └── __init__.py │ ├── model_tests.py │ ├── models.py │ ├── registry_tests.py │ ├── sync_tests.py │ └── utils.py ├── urls.py └── version.py ├── manage.py ├── publish.py ├── release_notes.md ├── requirements ├── requirements-testing.txt └── requirements.txt ├── run_tests.py ├── settings.py ├── setup.cfg └── setup.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | omit = 4 | */migrations/* 5 | entity/version.py 6 | source = entity 7 | [report] 8 | exclude_lines = 9 | # Have to re-enable the standard pragma 10 | pragma: no cover 11 | 12 | # Don't complain if tests don't hit defensive assertion code: 13 | raise NotImplementedError 14 | fail_under = 100 15 | show_missing = 1 16 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # copied from django-cte 2 | name: entity tests 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master,develop] 8 | 9 | jobs: 10 | tests: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | python: ['3.7', '3.8', '3.9'] 16 | # Time to switch to pytest or nose2?? 17 | # nosetests is broken on 3.10 18 | # AttributeError: module 'collections' has no attribute 'Callable' 19 | # https://github.com/nose-devs/nose/issues/1099 20 | django: 21 | - 'Django~=3.2.0' 22 | - 'Django~=4.0.0' 23 | - 'Django~=4.1.0' 24 | - 'Django~=4.2.0' 25 | experimental: [false] 26 | # include: 27 | # - python: '3.9' 28 | # django: 'https://github.com/django/django/archive/refs/heads/main.zip#egg=Django' 29 | # experimental: true 30 | # # NOTE this job will appear to pass even when it fails because of 31 | # # `continue-on-error: true`. Github Actions apparently does not 32 | # # have this feature, similar to Travis' allow-failure, yet. 33 | # # https://github.com/actions/toolkit/issues/399 34 | exclude: 35 | - python: '3.7' 36 | django: 'Django~=4.0.0' 37 | - python: '3.7' 38 | django: 'Django~=4.1.0' 39 | - python: '3.7' 40 | django: 'Django~=4.2.0' 41 | services: 42 | postgres: 43 | image: postgres:latest 44 | env: 45 | POSTGRES_DB: postgres 46 | POSTGRES_PASSWORD: postgres 47 | POSTGRES_USER: postgres 48 | ports: 49 | - 5432:5432 50 | options: >- 51 | --health-cmd pg_isready 52 | --health-interval 10s 53 | --health-timeout 5s 54 | --health-retries 5 55 | steps: 56 | - uses: actions/checkout@v3 57 | - uses: actions/setup-python@v3 58 | with: 59 | python-version: ${{ matrix.python }} 60 | - name: Setup 61 | run: | 62 | python --version 63 | pip install --upgrade pip wheel 64 | pip install -r requirements/requirements.txt 65 | pip install -r requirements/requirements-testing.txt 66 | pip install "${{ matrix.django }}" 67 | pip freeze 68 | - name: Run tests 69 | env: 70 | DB_SETTINGS: >- 71 | { 72 | "ENGINE":"django.db.backends.postgresql", 73 | "NAME":"entity", 74 | "USER":"postgres", 75 | "PASSWORD":"postgres", 76 | "HOST":"localhost", 77 | "PORT":"5432" 78 | } 79 | run: | 80 | coverage run manage.py test entity 81 | coverage report --fail-under=99 82 | continue-on-error: ${{ matrix.experimental }} 83 | - name: Check style 84 | run: flake8 entity 85 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled python files 2 | *.pyc 3 | 4 | # Vim files 5 | *.swp 6 | *.swo 7 | 8 | # Virtual Environment 9 | /env/ 10 | venv/ 11 | 12 | # Coverage files 13 | .coverage 14 | 15 | # Setuptools distribution folder. 16 | /dist/ 17 | /build/ 18 | 19 | # Python egg metadata, regenerated from source files by setuptools. 20 | /*.egg-info 21 | /*.egg 22 | .eggs/ 23 | 24 | .idea/ 25 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | Contributions and issues are most welcome! All issues and pull requests are 3 | handled through GitHub on the 4 | [ambitioninc repository](https://github.com/ambitioninc/django-entity/issues). 5 | Also, please check for any existing issues before filing a new one. If you have 6 | a great idea but it involves big changes, please file a ticket before making a 7 | pull request! We want to make sure you don't spend your time coding something 8 | that might not fit the scope of the project. 9 | 10 | ## Running the tests 11 | 12 | To get the source source code and run the unit tests, run: 13 | ```bash 14 | git clone git://github.com/ambitioninc/django-entity.git 15 | cd django-entity 16 | virtualenv env 17 | . env/bin/activate 18 | python setup.py install 19 | coverage run setup.py test 20 | coverage report --fail-under=100 21 | ``` 22 | 23 | While 100% code coverage does not make a library bug-free, it significantly 24 | reduces the number of easily caught bugs! Please make sure coverage is at 100% 25 | before submitting a pull request! 26 | 27 | ## Code Quality 28 | 29 | For code quality, please run flake8: 30 | ```bash 31 | pip install flake8 32 | flake8 . 33 | ``` 34 | 35 | ## Code Styling 36 | Please arrange imports with the following style 37 | 38 | ```python 39 | # Standard library imports 40 | import os 41 | 42 | # Third party package imports 43 | from mock import patch 44 | from django.conf import settings 45 | 46 | # Local package imports 47 | from entity.version import __version__ 48 | ``` 49 | 50 | Please follow 51 | [Google's python style](http://google-styleguide.googlecode.com/svn/trunk/pyguide.html) 52 | guide wherever possible. 53 | 54 | 55 | 56 | ## Release Checklist 57 | 58 | Before a new release, please go through the following checklist: 59 | 60 | * Bump version in entity/version.py 61 | * Git tag the version 62 | * Upload to pypi: 63 | ```bash 64 | pip install wheel 65 | python setup.py sdist bdist_wheel upload 66 | ``` 67 | 68 | ## Vulnerability Reporting 69 | 70 | For any security issues, please do NOT file an issue or pull request on GitHub! 71 | Please contact [security@ambition.com](mailto:security@ambition.com) with the 72 | GPG key provided on [Ambition's website](http://ambition.com/security/). 73 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | Wes Kendall @wesleykendall wes.kendall@ambition.com (primary) 2 | Erik Swanson @Wilduck theerikswanson@gmail.com (primary) 3 | Wes Okes @wesokes wes.okes@ambition.com 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Ambition 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE 3 | recursive-include requirements * 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/ambitioninc/django-entity.svg)](https://travis-ci.org/ambitioninc/django-entity) 2 | # Django Entity - [Release Notes](release_notes.md) 3 | 4 | Django Entity is an app that provides Django projects with the ability to mirror their entities and entity relationships in a separate, well-contained and easily-accessible table. Only PostgreSQL is supported. 5 | 6 | Django Entity provides large-scale projects with the ability to better segregate their apps while minimizing the application-specific code in those apps that has to deal with entities and their relationships in the main project. 7 | 8 | What is an entity? An entity is any model in your Django project. For example, an entity could be a Django User model or a Group of Users. Similarly, an entity relationship defines a super and sub relationship among different types of entities. For example, a Group would be a super entity of a User. The Django Entity app allows you to easily express this relationship in your model definition and sync it to a centralized place that is accessible by any other app in your project. 9 | 10 | ## A Use Case 11 | Imagine that you have a Django project that defines many types of groupings of your users. For example, let's say in your enterprise project, you allow users to define their manager, their company position, and their regional branch location. Similarly, let's say that you have an app that can email groups of users based on their manager (or anyone who is under the managers of that manager), their position, or their region. This email app would likely have to know application-specific modeling of these relationships in order to be built. Similarly, doing things like querying for all users under a manager hierachy can be an expensive lookup depending on how it is modeled. 12 | 13 | Using Django Entity, the email app could be written to take an Entity model rather than having to understand the complex relationships of each group. The Entity model passed to the email app could be a CompanyPosition model, and the get_sub_entities().is_any_type(ContentType.objects.get_for_model(User)) would return all of the User models under that CompanyPosition model. This allows the email app to be completely segregated from how the main project defines its relationships. Similarly, the query to obtain all User models under a CompanyPosition could be much more efficient than querying directly from the project (depending on how the project has its models structured). 14 | 15 | ## Getting Started - Configuring Entity Syncing 16 | ### Basic Use Case 17 | 18 | Install using `pip`: 19 | 20 | ```python 21 | pip install django-entity 22 | ``` 23 | 24 | Add 'entity' to your INSTALLED_APPS setting: 25 | 26 | ```python 27 | INSTALLED_APPS = [ 28 | ... 29 | 'entity', 30 | ] 31 | ``` 32 | 33 | Similar to Django's model admin, entities are configured by registering them with the Entity registry as follows: 34 | 35 | ```python 36 | from entity.config import EntityConfig, register_entity 37 | 38 | 39 | class Account(Model): 40 | email = models.CharField(max_length=64) 41 | 42 | 43 | @register_entity() 44 | class AccountConfig(EntityConfig): 45 | queryset = Account.objects.all() 46 | ``` 47 | 48 | And just like that, the ``Account`` model is now synced to the ``Entity`` table every time an account is saved, deleted, or has any of its M2M fields updated. 49 | 50 | ### More Advanced Syncing Options 51 | Django Entity would not be much if it only synced objects to a single ``Entity`` table. In order to take advantage of the power of mirroring relationships, the user must define a configuration for the entity that inherits ``EntityConfig``. A small example of this is below and extends our account model to have a ``Group`` foreign key. 52 | 53 | ```python 54 | from entity.config import register_entity, EntityConfig 55 | 56 | 57 | class Account(Model): 58 | email = models.CharField(max_length=64) 59 | group = models.ForeignKey(Group) 60 | 61 | 62 | @register_entity() 63 | class GroupConfig(EntityConfig): 64 | queryset = Group.objects.all() 65 | 66 | 67 | @register_entity() 68 | class AccountConfig(EntityConfig): 69 | queryset = Account.objects.all() 70 | 71 | def get_super_entities(self, model_objs): 72 | return { 73 | Group: [(model_obj.id, model_obj.group_id) for model_obj in model_objs] 74 | } 75 | ``` 76 | 77 | In the above scenario, we mirrored the ``Group`` model using the default entity configuration. However, the ``Account`` model now uses a special configuration that inherits ``EntityConfig``. It overrides the ``get_super_entities`` function to return a list of all model objects that are super entities to the account. Once the account is synced, the user may then do various filtering on the relationships of accounts to groups (more on that later). 78 | 79 | Note - in the above example, we also used the ``register_entity`` decorator, which is really just short notation for doing ``entity_registry.register_entity(model_class, entity_config_class)``. 80 | 81 | Along with the ability to mirror relationships, the entity configuration can be extended to mirror metadata about an entity. For example, using the ``Account`` model in the previous example: 82 | 83 | ```python 84 | @register_entity() 85 | class AccountConfig(EntityConfig): 86 | queryset = Account.objects.all() 87 | 88 | def get_super_entities(self, model_objs): 89 | return { 90 | Group: [(model_obj.id, model_obj.group_id) for model_obj in model_objs] 91 | } 92 | 93 | def get_entity_meta(self, model_obj): 94 | return { 95 | 'email': model_obj.email 96 | } 97 | ``` 98 | 99 | With the above configuration, every account entity will have an entity_meta field (a JSON field) that has the email attribute mirrored as well. The metadata mirroring can be powerful for building generic apps on top of entities that need access to concrete fields of a concrete model (without having to prefetch all of the concrete models pointed to by the entities). 100 | 101 | Along with metadata, entities come with the ability to mirror a ``display_name`` field in order to provide a human-readable name for the entity that can also be filtered in the database. By default, the ``display_name`` field uses the result of the ``unicode()`` function applied to the concrete model instance. The user may override this behavior by overriding the ``get_display_name`` method in the entity configuration. 102 | 103 | Entities can also be configured to be active or inactive, and this is done by adding an ``get_is_active`` function to the config that returns ``True`` (the default value) if the entity is active and ``False`` otherwise. 104 | 105 | ### Advanced Syncing Continued - Entity Kinds 106 | 107 | Entities have the ability to be labeled with their "kind" for advanced filtering capabilities. The entity kind allows a user to explicitly state what type of entity is being mirrored along with providing human-readable content about the entity kind. This is done by mirroring a unique ``name`` field and a ``display_name`` field in the ``EntityKind`` object that each ``Entity`` model points to. 108 | 109 | By default, Django Entity will mirror the content type of the entity as its kind. The name field will be the ``app_label`` of the content type followed by a dot followed by the ``model`` of the content type. For cases where this name is not descriptive enough for the kind of the entity, the user has the ability to override the ``get_entity_kind`` function in the entity config. For example: 110 | 111 | ```python 112 | @register_entity() 113 | class AccountConfig(EntityConfig): 114 | queryset = Account.objects.all() 115 | 116 | def get_entity_kind(self, model_obj): 117 | return (model_obj.email_domain, 'Email domain {0}'.format(model_obj.email_domain)) 118 | ``` 119 | 120 | In the above case, the account entities are segregated into different kinds based on the domain of the email. The second value of the returned tuple provides a human-readable version of the kind that is being created. 121 | 122 | ### Even More Advanced Syncing - Watching Other Models 123 | 124 | Underneath the hood, Django Entity is syncing up the mirrored Entity table when saves, deletes, and M2M updates are happening on the mirrored models. However, some models may actually depend on objects that are not pointed to by the immediate fields of the model. For example, assume that we have the following models: 125 | 126 | ```python 127 | class Group(models.Model): 128 | group_name = models.CharField() 129 | 130 | 131 | class User(models.Model): 132 | email = models.CharField() 133 | groups = models.ManyToManyField(Group) 134 | 135 | 136 | class Account(models.Model): 137 | user = models.OneToOneField(User) 138 | ``` 139 | 140 | Now, assume that the ``Account`` model wants to add every ``Group`` model in the many to many of the ``User`` model as its super entity. This would be set up with the following config: 141 | 142 | ```python 143 | @register_entity() 144 | class GroupConfig(EntityConfig): 145 | queryset = Group.objects.all() 146 | 147 | 148 | @register_entity() 149 | class AccountConfig(EntityConfig): 150 | queryset = Account.objects.all() 151 | 152 | def get_super_entities(self, model_objs): 153 | return { 154 | Group: [ 155 | (model_obj.id, group.id) 156 | for model_obj in model_objs 157 | for group in model_obj.user.groups.all() 158 | ] 159 | } 160 | ``` 161 | 162 | Although it would be nice if this worked out of the box, Django Entity has no way of knowing that the ``Account`` model needs to be updated when the fields in its associated ``User`` model change. In order to ensure the ``Account`` model is mirrored properly, add a ``watching`` class variable to the entity config as follows: 163 | 164 | ```python 165 | @register_entity() 166 | class GroupConfig(EntityConfig): 167 | queryset = Group.objects.all() 168 | 169 | 170 | @register_entity() 171 | class AccountConfig(EntityConfig): 172 | queryset = Account.objects.all() 173 | watching = [ 174 | (User, lambda user_obj: Account.objects.filter(user=user_obj)), 175 | ] 176 | 177 | def get_super_entities(self, model_objs): 178 | return { 179 | Group: [ 180 | (model_obj.id, group.id) 181 | for model_obj in model_objs 182 | for group in model_obj.user.groups.all() 183 | ] 184 | } 185 | ``` 186 | 187 | The ``watching`` field defines a list of tuples. The first element in each tuple represents the model to watch. The second element in the tuple describes the function used to access the entity models that are related to the changed watching model. 188 | 189 | Here's another more complex example using an ``Address`` model that points to an account.: 190 | 191 | ```python 192 | class Address(models.Model): 193 | account = models.ForeignKey(Account) 194 | ``` 195 | 196 | To make the Address model sync when the ``User`` model of the ``Account`` model is changed, define an entity configuration like so: 197 | 198 | ```python 199 | @register_entity() 200 | class AddressConfig(EntityConfig): 201 | queryset = Address.objects.all() 202 | watching = [ 203 | (User, lambda user_model_obj: Address.objects.filter(account__user=user_model_obj)), 204 | ] 205 | ``` 206 | 207 | Again, all that is happening under the hood is that when a ``User`` model is changed, all entity models related to that changed user model are returned so that they can be sycned. 208 | 209 | ### Ensuring Entity Syncing Optimal Queries 210 | Since a user may need to mirror many different super entities from many different foreign keys, it is beneficial for them to provide caching hints to Django Entity. This can be done by simply providing a prefetched Django QuerySet to the ``queryset`` attribute in the entity config. For example, our previous account entity config would want to do the following: 211 | 212 | ```python 213 | @register_entity() 214 | class AccountConfig(EntityConfig): 215 | queryset = Account.objects.prefetch_related('user__groups') 216 | ``` 217 | 218 | When invididual entities or all entities are synced, the QuerySet will be used to access the ``Account`` models and passed to relevent methods of the entity config. 219 | 220 | 221 | ## Syncing Entities 222 | Models will be synced automatically when they are configured and registered with Django entity. However, the user will need to sync all entities initially after configuring the entities (and also subsequently resync all when configuration changes occur). This can be done with the sync_entities management command: 223 | 224 | ```python 225 | # Sync all entities 226 | python manage.py sync_entities 227 | ``` 228 | 229 | Similarly, you can directly call the function to sync entities in a celery processing job or in your own application code. 230 | 231 | ```python 232 | from entity.sync import sync_entities 233 | 234 | # Sync all entities 235 | sync_entities() 236 | ``` 237 | 238 | Note that the ``sync_entities()`` function takes a variable length list of model objects if the user wishes to sync individual entities: 239 | 240 | ```python 241 | from entity.sync import sync_entities 242 | 243 | # Sync three specific models 244 | sync_entities(account_model_obj, group_model_obj, another_model_obj) 245 | ``` 246 | 247 | Entity syncing can be costly depending on the amount of relationships mirrored. If the user is going to be updating many models in a row that are mirrored as entities, it is recommended to turn syncing off, explicitly sync all updated entities, and then turn syncing back on. This can be accomplished as follows: 248 | 249 | ```python 250 | from entity.signal_handlers import turn_on_syncing, turn_off_syncing 251 | from entity.sync import sync_entities 252 | 253 | 254 | # Turn off syncing since we're going to be updating many different accounts 255 | turn_off_syncing() 256 | 257 | # Update all of the accounts 258 | accounts_to_update = [list of accounts] 259 | for account in accounts_to_update: 260 | account.update(...) 261 | 262 | # Explicitly sync the entities updated to keep the mirrored entities up to date 263 | sync_entities(*accounts_to_update) 264 | 265 | # Dont forget to turn syncing back on... 266 | turn_on_syncing() 267 | ``` 268 | 269 | ## Accessing Entities 270 | After the entities have been synced, they can then be accessed in the primary entity table. The ``Entity`` model has the following fields: 271 | 272 | 1. ``entity_type``: The ``ContentType`` of the mirrored entity. 273 | 1. ``entity_id``: The object ID of the mirrored entity. 274 | 1. ``entity_meta``: A JSONField of mirrored metadata about an entity (or null or none mirrored). 275 | 1. ``entity_kind``: The EntityKind model that describes the type of mirrored entity. Defaults to parameters related to the entity content type. 276 | 1. ``is_active``: True if the entity is active, False otherwise. 277 | 278 | Along with these basic fields, all of the following functions can either be called directly on the ``Entity`` model or on the ``Entity`` model manager. 279 | 280 | ### Basic Model and Manager Functions 281 | Note that since entities are activatable (i.e. can have active and inactive states), the entity model manager only accesses active entities by default. If the user wishes to access every single entity (active or inactive), they must go through the ``all_objects`` manager, which is used in the example code below. The methods below are available on the ``objects`` and ``all_objects`` model managers, although the ``active`` and ``inactive`` methods are not useful on the ``objects`` model manager since it already filters for active entities. 282 | 283 | #### get_for_obj(model_obj) 284 | The get_for_obj function takes a model object and returns the corresponding entity. Only available in the ``Entity`` model manager. 285 | 286 | ```python 287 | test_model = TestModel.objects.create() 288 | # Get the resulting entity for the model object 289 | entity = Entity.objects.get_for_obj(test_model) 290 | ``` 291 | 292 | #### active() 293 | Returns active entities. Only applicable when using the ``all_objects`` model manager. Note that ``objects`` already filters for only active entities. 294 | 295 | #### inactive() 296 | Does the opposite of ``active()``. Only applicable when using the ``all_objects`` model manager. Note that ``objects`` already disregards inactive entities. 297 | 298 | #### is_any_kind(*entity_kinds) 299 | Returns all entities that are any of the entity kinds provided. 300 | 301 | #### is_not_any_kind(*entity_kinds) 302 | The opposite of ``is_any_kind()``. 303 | 304 | #### is_sub_to_all(*super_entities) 305 | Return entities that are sub entities of every provided super entity (or all if no super entities are provided). 306 | 307 | For example, if one wishes to filter all of the Account entities by the ones that belong to Group A and Group B, the code would look like this: 308 | 309 | ```python 310 | groupa_entity = Entity.objects.get_for_obj(Group.objects.get(name='A')) 311 | groupb_entity = Entity.objects.get_for_obj(Group.objects.get(name='B')) 312 | for e in Entity.objects.is_sub_to_all(groupa_entity, groupb_entity): 313 | # Do your thing with the results 314 | pass 315 | ``` 316 | 317 | #### is_sub_to_any(*super_entities) 318 | Return entities that are sub entities of any one of the provided super entities (or all if no super entities are provided). 319 | 320 | #### is_sub_to_all_kinds(*super_entity_kinds) 321 | Return entities for which the set of provided kinds is contained in the set of all their super-entity-kinds 322 | 323 | #### is_sub_to_any_kind(*super_entity_kinds) 324 | Return entities that have at least one super entity-kind contained in the provided set of kinds (or all if no kinds are provided) 325 | 326 | #### cache_relationships() 327 | The cache_relationships function is useful for prefetching relationship information. Accessing entities without the cache_relationships function will result in many extra database queries if filtering is performed on the entity relationships. 328 | 329 | ```python 330 | entity = Entity.objects.cache_relationships().get_for_obj(test_model) 331 | for super_entity in entity.get_super_entities(): 332 | # Perform much faster accesses on super entities... 333 | pass 334 | ``` 335 | 336 | If one wants to ignore caching sub or super entity relationships, simply pass ``cache_sub=False`` or ``cache_super=False`` as keyword arguments to the function. Note that both of these flags are turned on by default. 337 | 338 | ### Chaining Filtering Functions 339 | All of the manager functions listed can be chained, so it is possible to do the following combinations: 340 | 341 | ```python 342 | Entity.objects.is_sub_to_all(groupa_entity).is_active().is_any_kind(account_kind, team_kind) 343 | 344 | Entity.objects.inactive().is_sub_to_all(groupb_entity).cache_relationships() 345 | ``` 346 | 347 | ## Arbitrary groups of Entities 348 | 349 | Once entities and their relationships are syncing is set up, most groupings of entities will be automatically encoded with the super/sub entity relationships. However, there are occasions when the groups that are automatically encoded do not capture the full extent of groupings that are useful. 350 | 351 | In order to support arbitrary groups of entities without requiring additional syncing code, the `EntityGroup` model is provided. This model comes with convenience functions for adding and removing entities to a group, as well as methods for querying what entities are in the arbitrary group. 352 | 353 | In addition to adding individual entities to an EntityGroup, you can also add all of an entity's sub-entities with a given type to the `EntityGroup` very easily. The following does the following: 354 | 355 | 1. Creates an `EntityGroup` 356 | 2. Adds an individual entity to the group 357 | 3. Adds all the subentities of a given kind to the group 358 | 4. Queries for all the entities in the group 359 | 360 | ```python 361 | my_group = EntityGroup.objects.create() 362 | 363 | my_group.add_entity(entity=some_entity) 364 | my_group.add_entity(entity=some_super_entity, sub_entity_kind=some_entity_kind) 365 | 366 | all_entities_in_group = my_group.all_entities() 367 | ``` 368 | 369 | After the code above is run, `all_entities_in_group` will be a 370 | Queryset of `Entity`s that contains the entity `some_entity` as well 371 | as all the sub-entities of `some_super_entity` who's entity-kind is 372 | `some_entity_kind`. 373 | 374 | The following methods are available on `EntityGroup`s 375 | 376 | #### all_entitites 377 | 378 | Get a list of all individual entities in the group. This will pull out 379 | all the entities that have been added, combining all the entities that 380 | were added individually as well as all the entities that were added 381 | because they are sub-entities to a super-entity that was added the the 382 | group, with the specified entity kind. 383 | 384 | #### add_entity 385 | 386 | Add an individual entity, or all the sub-entities (with a given kind) 387 | of a super-entity to the group. There are two ways to add entities to 388 | the group with this method. The first adds an individual entity to the 389 | group. The second adds all the individuals who are a super-entity's 390 | sub-entities of a given kind to the group. 391 | 392 | This allows leveraging existing groupings as well as allowing other 393 | arbitrary additions. Both individual, and sub-entity group memberships 394 | can be added to a single `EntityGroup`. 395 | 396 | The syntax for adding an individual entity is as simple as specifying 397 | the entity to add: 398 | 399 | ```python 400 | my_group.add(some_entity) 401 | ``` 402 | 403 | And adding a sub-entity group is as simple as specifying the 404 | super-entity and the sub-entity kind: 405 | 406 | ```python 407 | my_group.add(entity=some_entity, sub_entity_kind=some_entity_kind) 408 | ``` 409 | 410 | #### bulk_add_entities 411 | 412 | Add a number of entities, or sub-entity groups to the 413 | `EntityGroup`. It takes a list of tuples, where the first item in the 414 | tuple is an `Entity` instance, and the second is either an 415 | `EntityKind` instance or `None`. 416 | 417 | ```python 418 | my_group.bulk_add_entities([ 419 | (some_entity_1, None), 420 | (some_entity_2, None), 421 | (some_super_entity_1, some_entity_kind) 422 | (some_super_entity_2, other_entity_kind) 423 | ]) 424 | ``` 425 | 426 | #### remove_entitiy 427 | 428 | Removes a given entity, or sub-entity grouping from the 429 | `EntityGroup`. This method uses the same syntax of `add_entity`. 430 | 431 | ### bulk_remove_entities 432 | 433 | Removes a number of entities or sub-entity groupings from the 434 | `EntityGroup`. This method uses the same syntax as 435 | `bulk_add_entities`. 436 | 437 | #### bulk_overwrite 438 | 439 | This method replaces all of the group members with a new set of group 440 | members. It has the same syntax as ``bulk_add_entities``. 441 | 442 | ## License 443 | MIT License (see the LICENSE file for more info). 444 | -------------------------------------------------------------------------------- /entity/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .version import __version__ 3 | -------------------------------------------------------------------------------- /entity/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class EntityConfig(AppConfig): 5 | name = 'entity' 6 | verbose_name = 'Django Entity' 7 | 8 | def ready(self): 9 | import entity.signal_handlers 10 | assert entity 11 | -------------------------------------------------------------------------------- /entity/config.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | from django.contrib.contenttypes.models import ContentType 4 | 5 | 6 | class EntityConfig(object): 7 | """ 8 | Defines the configuration for a mirrored entity. 9 | """ 10 | # The "watching" class variable is a list of tuples that specify what models this entity 11 | # config watches and the function to extract entity models from the watching model. The 12 | # function's return must be an iterable object. 13 | # 14 | # For example, assume we have an Account model that has a foreign key to a User 15 | # model. Also, the User model has a M2M to Groups. If Groups are a super entity 16 | # of an Account, the user must set up a watching variable so that the account 17 | # is synced when the M2M on the user object is changed. This is because the 18 | # M2M is not directly on the Account model and does not trigger Account syncing 19 | # by default when changed. The watching variable would look like the following: 20 | # 21 | # watching = [ 22 | # (User, lambda user_model_obj: Account.objects.filter(user=user_model_obj)) 23 | # ] 24 | # 25 | watching = [] 26 | 27 | # The queryset to fetch when syncing the entity 28 | queryset = None 29 | 30 | def get_display_name(self, model_obj): 31 | """ 32 | Returns a human-readable string for the entity. 33 | """ 34 | return u'{0}'.format(model_obj) 35 | 36 | def get_entity_kind(self, model_obj): 37 | """ 38 | Returns a tuple for a kind name and kind display name of an entity. 39 | By default, uses the app_label and model of the model object's content 40 | type as the kind. 41 | """ 42 | model_obj_ctype = ContentType.objects.get_for_model(self.queryset.model) 43 | return (u'{0}.{1}'.format(model_obj_ctype.app_label, model_obj_ctype.model), u'{0}'.format(model_obj_ctype)) 44 | 45 | def get_entity_meta(self, model_obj): 46 | """ 47 | Retrieves metadata about an entity. 48 | 49 | Returns: 50 | A dictionary of metadata about an entity or None if there is no 51 | metadata. Defaults to returning None 52 | """ 53 | return None 54 | 55 | def get_is_active(self, model_obj): 56 | """ 57 | Describes if the entity is currently active. 58 | 59 | Returns: 60 | A Boolean specifying if the entity is active. Defaults to 61 | returning True. 62 | """ 63 | return True 64 | 65 | def get_super_entities(self, model_objs, sync_all): 66 | """ 67 | Retrieves a dictionary of entity relationships. The dictionary is keyed 68 | on the model class of the super entity and each value of the dictionary 69 | is a list of tuples. The tuples specify the ID of the sub entity and 70 | the ID of the super entity. 71 | 72 | If sync_all is True, it means all models are currently being synced 73 | """ 74 | return {} 75 | 76 | 77 | class EntityRegistry(object): 78 | """ 79 | Maintains all registered entities and provides a lookup table for models to related entities. 80 | """ 81 | def __init__(self): 82 | # The registry of all models to their querysets and EntityConfigs 83 | self._entity_registry = {} 84 | 85 | # Stores a list of (model, qset_arg) tuples for each watching model 86 | self._entity_watching = defaultdict(list) 87 | 88 | @property 89 | def entity_registry(self): 90 | return self._entity_registry 91 | 92 | @property 93 | def entity_watching(self): 94 | return self._entity_watching 95 | 96 | def register_entity(self, entity_config): 97 | """ 98 | Registers an entity config 99 | """ 100 | if not issubclass(entity_config, EntityConfig): 101 | raise ValueError('Must register entity config class of subclass EntityConfig') 102 | 103 | if entity_config.queryset is None: 104 | raise ValueError('Entity config must define queryset') 105 | 106 | model = entity_config.queryset.model 107 | 108 | self._entity_registry[model] = entity_config() 109 | 110 | # Add watchers to the global look up table 111 | for watching_model, entity_model_getter in entity_config.watching: 112 | self._entity_watching[watching_model].append((model, entity_model_getter)) 113 | 114 | 115 | # Define the global registry variable 116 | entity_registry = EntityRegistry() 117 | 118 | 119 | def register_entity(): 120 | """ 121 | Registers the EntityConfig class with 122 | django entity: 123 | 124 | @register_entity() 125 | class AuthorConfig(EntityConfig): 126 | queryset = Author.objects.all() 127 | """ 128 | def _entity_config_wrapper(entity_config_class): 129 | entity_registry.register_entity(entity_config_class) 130 | return entity_config_class 131 | 132 | return _entity_config_wrapper 133 | -------------------------------------------------------------------------------- /entity/constants.py: -------------------------------------------------------------------------------- 1 | 2 | LOGIC_STRING_OPERATORS = {'&', '|', '!'} 3 | -------------------------------------------------------------------------------- /entity/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class InvalidLogicStringException(Exception): 4 | def __str__(self): 5 | return 'Invalid logic string' 6 | -------------------------------------------------------------------------------- /entity/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ambitioninc/django-entity/68be2ad83352c8b96b9117b6fcedc166a2392b88/entity/management/__init__.py -------------------------------------------------------------------------------- /entity/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ambitioninc/django-entity/68be2ad83352c8b96b9117b6fcedc166a2392b88/entity/management/commands/__init__.py -------------------------------------------------------------------------------- /entity/management/commands/sync_entities.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from entity.sync import sync_entities 4 | 5 | 6 | class Command(BaseCommand): 7 | """ 8 | A management command for syncing all entities. 9 | """ 10 | 11 | def handle(self, *args, **options): 12 | """ 13 | Runs sync entities 14 | """ 15 | sync_entities() 16 | -------------------------------------------------------------------------------- /entity/migrations/0001_0010_squashed.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.19 on 2023-05-31 16:29 2 | 3 | import django.core.serializers.json 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | 7 | # This squashed migration does a clean build of the entire entity schema. It replaces all the previous 8 | # migrations. The reason for doing the squash was that the original initial migration was using the legacy 9 | # jsonfield v0.9, which cannot be present, even for migrations, after Django4. So the steps I took were as follows: 10 | # 1. swapped out the field type from jsonfield.fields.jsonfield to django.db.models.jsonfield in 0001_initial, 11 | # essentially pretending it had been that type from the outset 12 | # 2. swapped out the old Django2 nomenclature to the Django3.1+ nomenclature (eliminated django.contrib.postgres) 13 | # 3. removed the code in migration 0006 that eliminated any non-conforming records prior to enforcing a new key. 14 | # This is safe as obviously we have no instances that have not been migrated since 2016... 15 | # 4. finally, ran the squash as: `python manage.py squashmigrations --squashed-name 0010_squashed entity 0001 0010` 16 | # to collapse the entire schema creation to CreateModel calls with no AlterModel calls required. 17 | 18 | class Migration(migrations.Migration): 19 | replaces = [('entity', '0001_initial'), ('entity', '0002_entitykind_is_active'), 20 | ('entity', '0003_auto_20150813_2234'), ('entity', '0004_auto_20150915_1747'), 21 | ('entity', '0005_remove_entitygroup_entities'), ('entity', '0006_entity_relationship_unique'), 22 | ('entity', '0007_allentityproxy'), ('entity', '0008_auto_20180329_1934'), 23 | ('entity', '0009_auto_20180402_2145'), ('entity', '0010_auto_20181213_1817')] 24 | 25 | dependencies = [ 26 | ('contenttypes', '0002_remove_content_type_name'), 27 | ('contenttypes', '0001_initial'), 28 | ] 29 | 30 | operations = [ 31 | migrations.CreateModel( 32 | name='EntityKind', 33 | fields=[ 34 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 35 | ('name', models.CharField(db_index=True, max_length=256, unique=True)), 36 | ('display_name', models.TextField(blank=True)), 37 | ('is_active', models.BooleanField(default=True)), 38 | ], 39 | ), 40 | migrations.CreateModel( 41 | name='Entity', 42 | fields=[ 43 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 44 | ('display_name', models.TextField(blank=True, db_index=True)), 45 | ('entity_id', models.IntegerField()), 46 | ('entity_meta', models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, null=True)), 47 | ('is_active', models.BooleanField(db_index=True, default=True)), 48 | ('entity_kind', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='entity.entitykind')), 49 | ('entity_type', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='contenttypes.contenttype')), 50 | ], 51 | options={ 52 | 'unique_together': {('entity_id', 'entity_type')}, 53 | }, 54 | ), 55 | migrations.CreateModel( 56 | name='EntityGroup', 57 | fields=[ 58 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 59 | ], 60 | ), 61 | migrations.CreateModel( 62 | name='EntityGroupMembership', 63 | fields=[ 64 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 65 | ('entity', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='entity.entity')), 66 | ('entity_group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='entity.entitygroup')), 67 | ('sub_entity_kind', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='entity.entitykind')), 68 | ], 69 | ), 70 | migrations.CreateModel( 71 | name='EntityRelationship', 72 | fields=[ 73 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 74 | ('sub_entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='super_relationships', to='entity.entity')), 75 | ('super_entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sub_relationships', to='entity.entity')), 76 | ], 77 | options={ 78 | 'unique_together': {('sub_entity', 'super_entity')}, 79 | }, 80 | ), 81 | migrations.CreateModel( 82 | name='AllEntityProxy', 83 | fields=[ 84 | ], 85 | options={ 86 | 'proxy': True, 87 | }, 88 | bases=('entity.entity',), 89 | ), 90 | ] 91 | -------------------------------------------------------------------------------- /entity/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from django.db import models, migrations 3 | import django.db.models.deletion 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('contenttypes', '0001_initial'), 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='Entity', 15 | fields=[ 16 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 17 | ('display_name', models.TextField(db_index=True, blank=True)), 18 | ('entity_id', models.IntegerField()), 19 | ('entity_meta', models.JSONField(null=True)), 20 | ('is_active', models.BooleanField(default=True, db_index=True)), 21 | ], 22 | options={ 23 | }, 24 | bases=(models.Model,), 25 | ), 26 | migrations.CreateModel( 27 | name='EntityKind', 28 | fields=[ 29 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 30 | ('name', models.CharField(unique=True, max_length=256, db_index=True)), 31 | ('display_name', models.TextField(blank=True)), 32 | ], 33 | options={ 34 | }, 35 | bases=(models.Model,), 36 | ), 37 | migrations.CreateModel( 38 | name='EntityRelationship', 39 | fields=[ 40 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 41 | ('sub_entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='super_relationships', to='entity.Entity')), 42 | ('super_entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sub_relationships', to='entity.Entity')), 43 | ], 44 | options={ 45 | }, 46 | bases=(models.Model,), 47 | ), 48 | migrations.AddField( 49 | model_name='entity', 50 | name='entity_kind', 51 | field=models.ForeignKey(to='entity.EntityKind', on_delete=django.db.models.deletion.PROTECT), 52 | preserve_default=True, 53 | ), 54 | migrations.AddField( 55 | model_name='entity', 56 | name='entity_type', 57 | field=models.ForeignKey(to='contenttypes.ContentType', on_delete=django.db.models.deletion.PROTECT), 58 | preserve_default=True, 59 | ), 60 | migrations.AlterUniqueTogether( 61 | name='entity', 62 | unique_together=set([('entity_id', 'entity_type', 'entity_kind')]), 63 | ), 64 | ] 65 | -------------------------------------------------------------------------------- /entity/migrations/0002_entitygroup_logic_string.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.4 on 2023-08-30 18:09 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('entity', '0001_0010_squashed'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='entitygroup', 15 | name='logic_string', 16 | field=models.TextField(blank=True, default=None, null=True), 17 | ), 18 | migrations.AddField( 19 | model_name='entitygroupmembership', 20 | name='sort_order', 21 | field=models.IntegerField(default=0), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /entity/migrations/0002_entitykind_is_active.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('entity', '0001_initial'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='entitykind', 16 | name='is_active', 17 | field=models.BooleanField(default=True), 18 | preserve_default=True, 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /entity/migrations/0003_auto_20150813_2234.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import django.db.models.deletion 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('entity', '0002_entitykind_is_active'), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='EntityGroup', 17 | fields=[ 18 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 19 | ], 20 | options={ 21 | }, 22 | bases=(models.Model,), 23 | ), 24 | migrations.CreateModel( 25 | name='EntityGroupMembership', 26 | fields=[ 27 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 28 | ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='entity.Entity')), 29 | ('entity_group', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='entity.EntityGroup')), 30 | ('sub_entity_kind', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='entity.EntityKind', null=True)), 31 | ], 32 | options={ 33 | }, 34 | bases=(models.Model,), 35 | ), 36 | migrations.AddField( 37 | model_name='entitygroup', 38 | name='entities', 39 | field=models.ManyToManyField(to='entity.Entity', through='entity.EntityGroupMembership'), 40 | preserve_default=True, 41 | ), 42 | ] 43 | -------------------------------------------------------------------------------- /entity/migrations/0004_auto_20150915_1747.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import django.db.models.deletion 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('entity', '0003_auto_20150813_2234'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='entitygroupmembership', 17 | name='entity', 18 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='entity.Entity', null=True), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /entity/migrations/0005_remove_entitygroup_entities.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('entity', '0004_auto_20150915_1747'), 11 | ] 12 | 13 | operations = [ 14 | migrations.RemoveField( 15 | model_name='entitygroup', 16 | name='entities', 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /entity/migrations/0006_entity_relationship_unique.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.8 on 2016-12-12 18:20 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, connection 6 | from django.db.models import Count, Max 7 | 8 | 9 | def disable_triggers(apps, schema_editor): 10 | """ 11 | Temporarily disable user triggers on the relationship table. We do not want things 12 | like entity history to attach onto these migrations as this is a core bug where duplicates 13 | should not exist 14 | 15 | :param apps: 16 | :param schema_editor: 17 | :return: 18 | """ 19 | with connection.cursor() as cursor: 20 | cursor.execute( 21 | """ 22 | ALTER TABLE entity_entityrelationship DISABLE TRIGGER USER; 23 | """ 24 | ) 25 | 26 | 27 | def enable_triggers(apps, schema_editor): 28 | """ 29 | Re-enable the triggers (if any) 30 | :param apps: 31 | :param schema_editor: 32 | :return: 33 | """ 34 | with connection.cursor() as cursor: 35 | cursor.execute( 36 | """ 37 | ALTER TABLE entity_entityrelationship ENABLE TRIGGER USER; 38 | """ 39 | ) 40 | 41 | 42 | def remove_duplicates(apps, schema_editor): 43 | """ 44 | Remove any duplicates from the entity relationship table 45 | :param apps: 46 | :param schema_editor: 47 | :return: 48 | """ 49 | 50 | # Get the model 51 | EntityRelationship = apps.get_model('entity', 'EntityRelationship') 52 | 53 | # Find the duplicates 54 | duplicates = EntityRelationship.objects.all().order_by( 55 | 'sub_entity_id', 56 | 'super_entity_id' 57 | ).values( 58 | 'sub_entity_id', 59 | 'super_entity_id' 60 | ).annotate( 61 | Count('sub_entity_id'), 62 | Count('super_entity_id'), 63 | max_id=Max('id') 64 | ).filter( 65 | super_entity_id__count__gt=1 66 | ) 67 | 68 | # Loop over the duplicates and delete 69 | for duplicate in duplicates: 70 | EntityRelationship.objects.filter( 71 | sub_entity_id=duplicate['sub_entity_id'], 72 | super_entity_id=duplicate['super_entity_id'] 73 | ).exclude( 74 | id=duplicate['max_id'] 75 | ).delete() 76 | 77 | 78 | class Migration(migrations.Migration): 79 | 80 | dependencies = [ 81 | ('entity', '0005_remove_entitygroup_entities'), 82 | ] 83 | 84 | operations = [ 85 | migrations.RunPython(disable_triggers), 86 | migrations.RunPython(remove_duplicates), 87 | migrations.RunPython(enable_triggers), 88 | migrations.AlterUniqueTogether( 89 | name='entityrelationship', 90 | unique_together=set([('sub_entity', 'super_entity')]), 91 | ), 92 | ] 93 | -------------------------------------------------------------------------------- /entity/migrations/0007_allentityproxy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.5 on 2017-02-15 18:23 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('entity', '0006_entity_relationship_unique'), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='AllEntityProxy', 17 | fields=[ 18 | ], 19 | options={ 20 | 'proxy': True, 21 | }, 22 | bases=('entity.entity',), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /entity/migrations/0008_auto_20180329_1934.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.11 on 2018-03-29 19:34 3 | from __future__ import unicode_literals 4 | 5 | import django.contrib.postgres.fields.jsonb 6 | from django.db import migrations 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('entity', '0007_allentityproxy'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AlterField( 17 | model_name='entity', 18 | name='entity_meta', 19 | field=django.contrib.postgres.fields.jsonb.JSONField(null=True), 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /entity/migrations/0009_auto_20180402_2145.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.11 on 2018-04-02 21:45 3 | from __future__ import unicode_literals 4 | 5 | import django.contrib.postgres.fields.jsonb 6 | import django.core.serializers.json 7 | from django.db import migrations 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | dependencies = [ 13 | ('entity', '0008_auto_20180329_1934'), 14 | ] 15 | 16 | operations = [ 17 | migrations.AlterField( 18 | model_name='entity', 19 | name='entity_meta', 20 | field=django.contrib.postgres.fields.jsonb.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, null=True), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /entity/migrations/0010_auto_20181213_1817.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.4 on 2018-12-13 18:17 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('contenttypes', '0002_remove_content_type_name'), 10 | ('entity', '0009_auto_20180402_2145'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterUniqueTogether( 15 | name='entity', 16 | unique_together={('entity_id', 'entity_type')}, 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /entity/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ambitioninc/django-entity/68be2ad83352c8b96b9117b6fcedc166a2392b88/entity/migrations/__init__.py -------------------------------------------------------------------------------- /entity/models.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from itertools import compress, chain 3 | 4 | from activatable_model.models import BaseActivatableModel, ActivatableManager, ActivatableQuerySet 5 | from django.contrib.contenttypes.fields import GenericForeignKey 6 | from django.contrib.contenttypes.models import ContentType 7 | from django.core.exceptions import ValidationError 8 | from django.core.serializers.json import DjangoJSONEncoder 9 | from django.db import models 10 | from django.db.models import Count, Q, JSONField 11 | from python3_utils import compare_on_attr 12 | from functools import reduce 13 | 14 | from entity.constants import LOGIC_STRING_OPERATORS 15 | from entity.exceptions import InvalidLogicStringException 16 | 17 | 18 | class AllEntityKindManager(ActivatableManager): 19 | """ 20 | Provides additional filtering for entity kinds. 21 | """ 22 | pass 23 | 24 | 25 | class ActiveEntityKindManager(AllEntityKindManager): 26 | """ 27 | Provides additional filtering for entity kinds. 28 | """ 29 | def get_queryset(self): 30 | return super(ActiveEntityKindManager, self).get_queryset().filter(is_active=True) 31 | 32 | 33 | class EntityKind(BaseActivatableModel): 34 | """ 35 | A kind for an Entity that is useful for filtering based on different types of entities. 36 | """ 37 | # The unique identification string for the entity kind 38 | name = models.CharField(max_length=256, unique=True, db_index=True) 39 | 40 | # A human-readable string for the entity kind 41 | display_name = models.TextField(blank=True) 42 | 43 | # True if the entity kind is active 44 | is_active = models.BooleanField(default=True) 45 | 46 | objects = ActiveEntityKindManager() 47 | all_objects = AllEntityKindManager() 48 | 49 | def __str__(self): 50 | return self.display_name 51 | 52 | 53 | class EntityQuerySet(ActivatableQuerySet): 54 | """ 55 | Provides additional queryset filtering abilities. 56 | """ 57 | def active(self): 58 | """ 59 | Returns active entities. 60 | """ 61 | return self.filter(is_active=True) 62 | 63 | def inactive(self): 64 | """ 65 | Returns inactive entities. 66 | """ 67 | return self.filter(is_active=False) 68 | 69 | def is_any_kind(self, *entity_kinds): 70 | """ 71 | Returns entities that have any of the kinds listed in entity_kinds. 72 | """ 73 | return self.filter(entity_kind__in=entity_kinds) if entity_kinds else self 74 | 75 | def is_not_any_kind(self, *entity_kinds): 76 | """ 77 | Returns entities that do not have any of the kinds listed in entity_kinds. 78 | """ 79 | return self.exclude(entity_kind__in=entity_kinds) if entity_kinds else self 80 | 81 | def is_sub_to_all(self, *super_entities): 82 | """ 83 | Given a list of super entities, return the entities that have those as a subset of their super entities. 84 | """ 85 | if super_entities: 86 | if len(super_entities) == 1: 87 | # Optimize for the case of just one super entity since this is a much less intensive query 88 | has_subset = EntityRelationship.objects.filter( 89 | super_entity=super_entities[0]).values_list('sub_entity', flat=True) 90 | else: 91 | # Get a list of entities that have super entities with all types 92 | has_subset = EntityRelationship.objects.filter( 93 | super_entity__in=super_entities).values('sub_entity').annotate(Count('super_entity')).filter( 94 | super_entity__count=len(set(super_entities))).values_list('sub_entity', flat=True) 95 | 96 | return self.filter(id__in=has_subset) 97 | else: 98 | return self 99 | 100 | def is_sub_to_any(self, *super_entities): 101 | """ 102 | Given a list of super entities, return the entities that have super entities that interset with those provided. 103 | """ 104 | if super_entities: 105 | return self.filter(id__in=EntityRelationship.objects.filter( 106 | super_entity__in=super_entities).values_list('sub_entity', flat=True)) 107 | else: 108 | return self 109 | 110 | def is_sub_to_all_kinds(self, *super_entity_kinds): 111 | """ 112 | Each returned entity will have superentites whos combined entity_kinds included *super_entity_kinds 113 | """ 114 | if super_entity_kinds: 115 | if len(super_entity_kinds) == 1: 116 | # Optimize for the case of just one 117 | has_subset = EntityRelationship.objects.filter( 118 | super_entity__entity_kind=super_entity_kinds[0]).values_list('sub_entity', flat=True) 119 | else: 120 | # Get a list of entities that have super entities with all types 121 | has_subset = EntityRelationship.objects.filter( 122 | super_entity__entity_kind__in=super_entity_kinds).values('sub_entity').annotate( 123 | Count('super_entity')).filter(super_entity__count=len(set(super_entity_kinds))).values_list( 124 | 'sub_entity', flat=True) 125 | 126 | return self.filter(pk__in=has_subset) 127 | else: 128 | return self 129 | 130 | def is_sub_to_any_kind(self, *super_entity_kinds): 131 | """ 132 | Find all entities that have super_entities of any of the specified kinds 133 | """ 134 | if super_entity_kinds: 135 | # get the pks of the desired subs from the relationships table 136 | if len(super_entity_kinds) == 1: 137 | entity_pks = EntityRelationship.objects.filter( 138 | super_entity__entity_kind=super_entity_kinds[0] 139 | ).select_related('entity_kind', 'sub_entity').values_list('sub_entity', flat=True) 140 | else: 141 | entity_pks = EntityRelationship.objects.filter( 142 | super_entity__entity_kind__in=super_entity_kinds 143 | ).select_related('entity_kind', 'sub_entity').values_list('sub_entity', flat=True) 144 | # return a queryset limited to only those pks 145 | return self.filter(pk__in=entity_pks) 146 | else: 147 | return self 148 | 149 | def cache_relationships(self, cache_super=True, cache_sub=True): 150 | """ 151 | Caches the super and sub relationships by doing a prefetch_related. 152 | """ 153 | relationships_to_cache = compress( 154 | ['super_relationships__super_entity', 'sub_relationships__sub_entity'], [cache_super, cache_sub]) 155 | return self.prefetch_related(*relationships_to_cache) 156 | 157 | 158 | class AllEntityManager(ActivatableManager): 159 | """ 160 | Provides additional entity-wide filtering abilities over all of the entity objects. 161 | """ 162 | def get_queryset(self): 163 | return EntityQuerySet(self.model) 164 | 165 | def get_for_obj(self, entity_model_obj): 166 | """ 167 | Given a saved entity model object, return the associated entity. 168 | """ 169 | return self.get(entity_type=ContentType.objects.get_for_model( 170 | entity_model_obj, for_concrete_model=False), entity_id=entity_model_obj.id) 171 | 172 | def delete_for_obj(self, entity_model_obj): 173 | """ 174 | Delete the entities associated with a model object. 175 | """ 176 | return self.filter( 177 | entity_type=ContentType.objects.get_for_model( 178 | entity_model_obj, for_concrete_model=False), entity_id=entity_model_obj.id).delete( 179 | force=True) 180 | 181 | def active(self): 182 | """ 183 | Returns active entities. 184 | """ 185 | return self.get_queryset().active() 186 | 187 | def inactive(self): 188 | """ 189 | Returns inactive entities. 190 | """ 191 | return self.get_queryset().inactive() 192 | 193 | def is_any_kind(self, *entity_kinds): 194 | """ 195 | Returns entities that have any of the kinds listed in entity_kinds. 196 | """ 197 | return self.get_queryset().is_any_kind(*entity_kinds) 198 | 199 | def is_not_any_kind(self, *entity_kinds): 200 | """ 201 | Returns entities that do not have any of the kinds listed in entity_kinds. 202 | """ 203 | return self.get_queryset().is_not_any_kind(*entity_kinds) 204 | 205 | def is_sub_to_all_kinds(self, *super_entity_kinds): 206 | """ 207 | Each returned entity will have superentites whos combined entity_kinds included *super_entity_kinds 208 | """ 209 | return self.get_queryset().is_sub_to_all_kinds(*super_entity_kinds) 210 | 211 | def is_sub_to_any_kind(self, *super_entity_kinds): 212 | """ 213 | Find all entities that have super_entities of any of the specified kinds 214 | """ 215 | return self.get_queryset().is_sub_to_any_kind(*super_entity_kinds) 216 | 217 | def is_sub_to_all(self, *super_entities): 218 | """ 219 | Given a list of super entities, return the entities that have those super entities as a subset of theirs. 220 | """ 221 | return self.get_queryset().is_sub_to_all(*super_entities) 222 | 223 | def is_sub_to_any(self, *super_entities): 224 | """ 225 | Given a list of super entities, return the entities whose super entities intersect with the provided super 226 | entities. 227 | """ 228 | return self.get_queryset().is_sub_to_any(*super_entities) 229 | 230 | def cache_relationships(self, cache_super=True, cache_sub=True): 231 | """ 232 | Caches the super and sub relationships by doing a prefetch_related. 233 | """ 234 | return self.get_queryset().cache_relationships(cache_super=cache_super, cache_sub=cache_sub) 235 | 236 | 237 | class ActiveEntityManager(AllEntityManager): 238 | """ 239 | The default 'objects' on the Entity model. This manager restricts all Entity queries to happen over active 240 | entities. 241 | """ 242 | def get_queryset(self): 243 | return EntityQuerySet(self.model).active() 244 | 245 | 246 | @compare_on_attr() 247 | class Entity(BaseActivatableModel): 248 | """ 249 | Describes an entity and its relevant metadata. Also defines if the entity is active. Filtering functions 250 | are provided that mirror the filtering functions in the Entity model manager. 251 | """ 252 | # A human-readable name for the entity 253 | display_name = models.TextField(blank=True, db_index=True) 254 | 255 | # The generic entity 256 | entity_id = models.IntegerField() 257 | entity_type = models.ForeignKey(ContentType, on_delete=models.PROTECT) 258 | entity = GenericForeignKey('entity_type', 'entity_id') 259 | 260 | # The entity kind 261 | entity_kind = models.ForeignKey(EntityKind, on_delete=models.PROTECT) 262 | 263 | # Metadata about the entity, stored as JSON 264 | entity_meta = JSONField(null=True, encoder=DjangoJSONEncoder) 265 | 266 | # True if this entity is active 267 | is_active = models.BooleanField(default=True, db_index=True) 268 | 269 | objects = ActiveEntityManager() 270 | all_objects = AllEntityManager() 271 | 272 | class Meta: 273 | unique_together = ('entity_id', 'entity_type') 274 | 275 | def get_sub_entities(self): 276 | """ 277 | Returns all of the sub entities of this entity. The returned entities may be filtered by chaining any 278 | of the functions in EntityFilter. 279 | """ 280 | return [r.sub_entity for r in self.sub_relationships.all()] 281 | 282 | def get_super_entities(self): 283 | """ 284 | Returns all of the super entities of this entity. The returned super entities may be filtered by 285 | chaining methods from EntityFilter. 286 | """ 287 | return [r.super_entity for r in self.super_relationships.all()] 288 | 289 | def __str__(self): 290 | """Return the display_name field 291 | """ 292 | return self.display_name 293 | 294 | 295 | class EntityRelationship(models.Model): 296 | """ 297 | Defines a relationship between two entities, telling which 298 | entity is a superior (or sub) to another entity. Similary, this 299 | model allows us to define if the relationship is active. 300 | """ 301 | 302 | class Meta: 303 | unique_together = ('sub_entity', 'super_entity') 304 | 305 | # The sub entity. The related name is called super_relationships since 306 | # querying this reverse relationship returns all of the relationships 307 | # super to an entity 308 | sub_entity = models.ForeignKey(Entity, related_name='super_relationships', on_delete=models.CASCADE) 309 | 310 | # The super entity. The related name is called sub_relationships since 311 | # querying this reverse relationships returns all of the relationships 312 | # sub to an entity 313 | super_entity = models.ForeignKey(Entity, related_name='sub_relationships', on_delete=models.CASCADE) 314 | 315 | 316 | class EntityGroupManager(models.Manager): 317 | 318 | def get_membership_cache(self, group_ids=None, is_active=True): 319 | """ 320 | Build a dict cache with the group membership info. Keyed off the group id and the values are 321 | a 2 element list of entity id and entity kind id (same values as the membership model). If no group ids 322 | are passed, then all groups will be fetched 323 | 324 | :param is_active: Flag indicating whether to filter on entity active status. None will not filter. 325 | :rtype: dict 326 | """ 327 | membership_queryset = EntityGroupMembership.objects.filter( 328 | # Select all memberships that are defined by a sub entity kind only 329 | Q(entity__isnull=True) | 330 | # Select memberships that define a single entity (null kind) and respect active flag 331 | (Q(entity__isnull=False) & Q(sub_entity_kind__isnull=True) & Q(entity__is_active=is_active)) | 332 | # Select memberships that are all of a kind under an entity and only query active supers 333 | (Q(entity__isnull=False) & Q(sub_entity_kind__isnull=False) & Q(entity__is_active=True)) 334 | ) 335 | 336 | if is_active is None: 337 | membership_queryset = EntityGroupMembership.objects.all() 338 | 339 | if group_ids: 340 | membership_queryset = membership_queryset.filter(entity_group_id__in=group_ids) 341 | 342 | membership_queryset = membership_queryset.order_by('sort_order', 'id') 343 | 344 | membership_queryset = membership_queryset.values_list('entity_group_id', 'entity_id', 'sub_entity_kind_id') 345 | 346 | # Iterate over the query results and build the cache dict 347 | membership_cache = {} 348 | for entity_group_id, entity_id, sub_entity_kind_id in membership_queryset: 349 | membership_cache.setdefault(entity_group_id, []) 350 | membership_cache[entity_group_id].append([entity_id, sub_entity_kind_id]) 351 | 352 | return membership_cache 353 | 354 | 355 | class EntityGroup(models.Model): 356 | """ 357 | An arbitrary group of entities and sub-entity groups. 358 | 359 | Members can be added to the group through the ``add_entity`` and 360 | ``bulk_add_entities`` methods, removed with the ``remove_entity`` 361 | and ``bulk_remove_entities`` methods, as well as completely change 362 | the members of the group with the ``bulk_overwrite`` method. 363 | 364 | Since entity groups support inclusion of individual entities, as 365 | well as groups of sub-entities of a given kind, querying for all 366 | of the individual entities in the group could be challenging. For 367 | this reason the ``all_entities`` method is included, which will 368 | return all of the individual entities in a given group. 369 | """ 370 | 371 | objects = EntityGroupManager() 372 | 373 | logic_string = models.TextField(default=None, null=True, blank=True) 374 | 375 | def all_entities(self, is_active=True): 376 | """ 377 | Return all the entities in the group. 378 | 379 | Because groups can contain both individual entities, as well 380 | as whole groups of entities, this method acts as a convenient 381 | way to get a queryset of all the entities in the group. 382 | """ 383 | return self.get_all_entities(return_models=True, is_active=is_active) 384 | 385 | def get_filter_indices(self, node): 386 | """ 387 | Makes sure that each filter referenced actually exists 388 | """ 389 | if hasattr(node, 'op'): 390 | # multi-operand operators 391 | if hasattr(node, 'values'): 392 | return list(chain(*[self.get_filter_indices(value) for value in node.values])) 393 | # unary operators 394 | elif hasattr(node, 'operand'): 395 | return list(chain(*[self.get_filter_indices(node.operand)])) 396 | elif hasattr(node, 'n'): 397 | return [node.n] 398 | return None 399 | 400 | def validate_filter_indices(self, indices, memberships): 401 | """ 402 | Raises an error if an invalid filter index is referenced or if an index is not referenced 403 | """ 404 | for index in indices: 405 | if hasattr(index, '__iter__'): 406 | return self.validate_filter_indices(index, memberships) 407 | if index < 1 or index > len(memberships): 408 | raise ValidationError('Filter logic contains an invalid filter index ({0})'.format(index)) 409 | 410 | for i in range(1, len(memberships) + 1): 411 | if i not in indices: 412 | raise ValidationError('Filter logic is missing a filter index ({0})'.format(i)) 413 | 414 | return True 415 | 416 | def _node_to_kmatch(self, node): 417 | """ 418 | Looks at an ast node and either returns the value or recursively returns the kmatch syntax. This is meant 419 | to convert the boolean logic like "1 AND 2" to kmatch syntax like ['&', [1, 2]] 420 | :return: kmatch syntax where memberships are represented by numbers 421 | :rtype: list 422 | """ 423 | if hasattr(node, 'op'): 424 | if hasattr(node, 'values'): 425 | return [node.op, [self._node_to_kmatch(value) for value in node.values]] 426 | elif hasattr(node, 'operand'): 427 | return [node.op, self._node_to_kmatch(node.operand)] 428 | elif hasattr(node, 'n'): 429 | return node.n 430 | return None 431 | 432 | def _map_kmatch_values(self, kmatch, memberships): 433 | """ 434 | Replaces index placeholders in the kmatch with the actual memberships. Any memberships that could not be matched 435 | up with a field will be replaced with None 436 | :return: the complete kmatch pattern 437 | :rtype: list 438 | """ 439 | # Check if single item 440 | if isinstance(kmatch, int): 441 | return memberships[kmatch - 1] 442 | if hasattr(kmatch, '__iter__'): 443 | return [self._map_kmatch_values(value, memberships) for value in kmatch] 444 | 445 | cls = getattr(kmatch, '__class__') 446 | if cls == ast.And: 447 | return '&' 448 | elif cls == ast.Or: 449 | return '|' 450 | elif cls == ast.Not: 451 | return '!' 452 | 453 | def _process_kmatch(self, kmatch, full_set): 454 | """ 455 | Every item is 2 elements - the operator and the value or list of values 456 | """ 457 | entity_ids = set() 458 | 459 | if isinstance(kmatch, set): 460 | return kmatch 461 | 462 | # We can always assume operator + list where the list is either sets or another operator + list 463 | if len(kmatch) != 2 or kmatch[0] not in LOGIC_STRING_OPERATORS: 464 | return kmatch 465 | 466 | # Apply the operator to the rest of the sets 467 | if kmatch[0] == '&': 468 | # Add the first element to the set 469 | entity_ids.update(self._process_kmatch(kmatch[1][0], full_set)) 470 | for next_element in kmatch[1][1:]: 471 | entity_ids &= self._process_kmatch(next_element, full_set) 472 | elif kmatch[0] == '|': 473 | # Add the first element to the set 474 | entity_ids.update(self._process_kmatch(kmatch[1][0], full_set)) 475 | for next_element in kmatch[1][1:]: 476 | entity_ids |= self._process_kmatch(next_element, full_set) 477 | elif kmatch[0] == '!': 478 | entity_ids = full_set - self._process_kmatch(kmatch[1], full_set) 479 | 480 | return entity_ids 481 | 482 | def get_all_entities(self, membership_cache=None, entities_by_kind=None, return_models=False, is_active=True): 483 | """ 484 | Returns a list of all entity ids in this group or optionally returns a queryset for all entity models. 485 | In order to reduce queries for multiple group lookups, it is expected that the membership_cache and 486 | entities_by_kind are built outside of this method and passed in as arguments. 487 | :param membership_cache: A group cache dict generated from `EntityGroup.objects.get_membership_cache()` 488 | :type membership_cache: dict 489 | :param entities_by_kind: An entities by kind dict generated from the `get_entities_by_kind` function 490 | :type entities_by_kind: dict 491 | :param return_models: If True, returns an Entity queryset, if False, returns a set of entity ids 492 | :type return_models: bool 493 | :param is_active: Flag to control entities being returned. Defaults to True for active entities only 494 | :type is_active: bool 495 | """ 496 | # If cache args were not passed, generate the cache 497 | if membership_cache is None: 498 | membership_cache = EntityGroup.objects.get_membership_cache([self.id], is_active=is_active) 499 | 500 | if entities_by_kind is None: 501 | entities_by_kind = entities_by_kind or get_entities_by_kind( 502 | membership_cache=membership_cache, 503 | is_active=is_active, 504 | ) 505 | 506 | # Build set of all entity ids for this group 507 | entity_ids = set() 508 | 509 | # This group does have entities 510 | memberships = membership_cache.get(self.id) 511 | if memberships: 512 | if self.logic_string: 513 | entity_ids = self.get_entity_ids_from_logic_string(entities_by_kind, memberships) 514 | else: 515 | # Loop over each membership in this group 516 | for entity_id, entity_kind_id in membership_cache[self.id]: 517 | if entity_id: 518 | if entity_kind_id: 519 | # All sub entities of this kind under this entity 520 | entity_ids.update(entities_by_kind[entity_kind_id][entity_id]) 521 | else: 522 | # Individual entity 523 | entity_ids.add(entity_id) 524 | else: 525 | # All entities of this kind 526 | entity_ids.update(entities_by_kind[entity_kind_id]['all']) 527 | 528 | # Check if a queryset needs to be returned 529 | if return_models: 530 | return Entity.objects.filter(id__in=entity_ids) 531 | 532 | return entity_ids 533 | 534 | def get_entity_ids_from_logic_string(self, entities_by_kind, memberships): 535 | entity_kind_id = memberships[0][1] 536 | full_set = set(entities_by_kind[entity_kind_id]['all']) 537 | 538 | try: 539 | filter_tree = ast.parse(self.logic_string.lower()) 540 | except: 541 | raise InvalidLogicStringException() 542 | 543 | expanded_memberships = [] 544 | for entity_id, entity_kind_id in memberships: 545 | if entity_id: 546 | if entity_kind_id: 547 | # All sub entities of this kind under this entity 548 | expanded_memberships.append(set(entities_by_kind[entity_kind_id][entity_id])) 549 | else: 550 | # Individual entity 551 | expanded_memberships.append({entity_id}) 552 | else: 553 | # All entities of this kind 554 | expanded_memberships.append(set(entities_by_kind[entity_kind_id]['all'])) 555 | 556 | # Make sure each index is valid 557 | indices = self.get_filter_indices(filter_tree.body[0].value) 558 | self.validate_filter_indices(indices, expanded_memberships) 559 | kmatch = self._node_to_kmatch(filter_tree.body[0].value) 560 | kmatch = self._map_kmatch_values(kmatch, expanded_memberships) 561 | entity_ids = self._process_kmatch(kmatch, full_set=full_set) 562 | 563 | return entity_ids 564 | 565 | def add_entity(self, entity, sub_entity_kind=None): 566 | """ 567 | Add an entity, or sub-entity group to this EntityGroup. 568 | 569 | :type entity: Entity 570 | :param entity: The entity to add. 571 | 572 | :type sub_entity_kind: Optional EntityKind 573 | :param sub_entity_kind: If a sub_entity_kind is given, all 574 | sub_entities of the entity will be added to this 575 | EntityGroup. 576 | """ 577 | membership = EntityGroupMembership.objects.create( 578 | entity_group=self, 579 | entity=entity, 580 | sub_entity_kind=sub_entity_kind, 581 | ) 582 | return membership 583 | 584 | def bulk_add_entities(self, entities_and_kinds): 585 | """ 586 | Add many entities and sub-entity groups to this EntityGroup. 587 | 588 | :type entities_and_kinds: List of (Entity, EntityKind) pairs. 589 | :param entities_and_kinds: A list of entity, entity-kind pairs 590 | to add to the group. In the pairs the entity-kind can be 591 | ``None``, to add a single entity, or some entity kind to 592 | add all sub-entities of that kind. 593 | """ 594 | memberships = [EntityGroupMembership( 595 | entity_group=self, 596 | entity=entity, 597 | sub_entity_kind=sub_entity_kind, 598 | ) for entity, sub_entity_kind in entities_and_kinds] 599 | created = EntityGroupMembership.objects.bulk_create(memberships) 600 | return created 601 | 602 | def remove_entity(self, entity, sub_entity_kind=None): 603 | """ 604 | Remove an entity, or sub-entity group to this EntityGroup. 605 | 606 | :type entity: Entity 607 | :param entity: The entity to remove. 608 | 609 | :type sub_entity_kind: Optional EntityKind 610 | :param sub_entity_kind: If a sub_entity_kind is given, all 611 | sub_entities of the entity will be removed from this 612 | EntityGroup. 613 | """ 614 | EntityGroupMembership.objects.get( 615 | entity_group=self, 616 | entity=entity, 617 | sub_entity_kind=sub_entity_kind, 618 | ).delete() 619 | 620 | def bulk_remove_entities(self, entities_and_kinds): 621 | """ 622 | Remove many entities and sub-entity groups to this EntityGroup. 623 | 624 | :type entities_and_kinds: List of (Entity, EntityKind) pairs. 625 | :param entities_and_kinds: A list of entity, entity-kind pairs 626 | to remove from the group. In the pairs, the entity-kind 627 | can be ``None``, to add a single entity, or some entity 628 | kind to add all sub-entities of that kind. 629 | """ 630 | criteria = [ 631 | Q(entity=entity, sub_entity_kind=entity_kind) 632 | for entity, entity_kind in entities_and_kinds 633 | ] 634 | criteria = reduce(lambda q1, q2: q1 | q2, criteria, Q()) 635 | EntityGroupMembership.objects.filter( 636 | criteria, entity_group=self).delete() 637 | 638 | def bulk_overwrite(self, entities_and_kinds): 639 | """ 640 | Update the group to the given entities and sub-entity groups. 641 | 642 | After this operation, the only members of this EntityGroup 643 | will be the given entities, and sub-entity groups. 644 | 645 | :type entities_and_kinds: List of (Entity, EntityKind) pairs. 646 | :param entities_and_kinds: A list of entity, entity-kind pairs 647 | to set to the EntityGroup. In the pairs the entity-kind 648 | can be ``None``, to add a single entity, or some entity 649 | kind to add all sub-entities of that kind. 650 | """ 651 | EntityGroupMembership.objects.filter(entity_group=self).delete() 652 | return self.bulk_add_entities(entities_and_kinds) 653 | 654 | 655 | @compare_on_attr() 656 | class AllEntityProxy(Entity): 657 | """ 658 | This is a proxy of the entity class that makes the .objects attribute 659 | access all of the entities regardless of active state. Active entities 660 | are accessed with the active_objects manager. This proxy should be used 661 | when you need foreign-key relationships to be able to access all entities 662 | regardless of active state. 663 | """ 664 | objects = AllEntityManager() 665 | active_objects = ActiveEntityManager() 666 | 667 | class Meta: 668 | proxy = True 669 | 670 | 671 | class EntityGroupMembership(models.Model): 672 | """ 673 | Membership information for entity groups. 674 | 675 | This model should usually not be queried/updated directly, but 676 | accessed through the EntityGroup api. 677 | 678 | When entity is null, it means all entities of the sub_entity_kind will be selected. 679 | When sub_entity_kind is null, only the specified entity will be selected. 680 | When entity and sub_entity_kind are both not null, it means all sub entities below 'entity' 681 | with a kind of 'sub_entity_kind' will be selected. 682 | """ 683 | entity_group = models.ForeignKey(EntityGroup, on_delete=models.CASCADE) 684 | entity = models.ForeignKey(Entity, null=True, on_delete=models.CASCADE) 685 | sub_entity_kind = models.ForeignKey(EntityKind, null=True, on_delete=models.CASCADE) 686 | sort_order = models.IntegerField(default=0) 687 | 688 | 689 | def get_entities_by_kind(membership_cache=None, is_active=True): 690 | """ 691 | Builds a dict with keys of entity kinds if and values are another dict. Each of these dicts are keyed 692 | off of a super entity id and optional have an 'all' key for any group that has a null super entity. 693 | Example structure: 694 | { 695 | entity_kind_id: { 696 | entity1_id: [1, 2, 3], 697 | entity2_id: [4, 5, 6], 698 | 'all': [1, 2, 3, 4, 5, 6] 699 | } 700 | } 701 | 702 | :rtype: dict 703 | """ 704 | # Accept an existing cache or build a new one 705 | if membership_cache is None: 706 | membership_cache = EntityGroup.objects.get_membership_cache(is_active=is_active) 707 | 708 | entities_by_kind = {} 709 | kinds_with_all = set() 710 | kinds_with_supers = set() 711 | super_ids = set() 712 | 713 | # Determine if we need to include the "universal set" aka all for a kind based on the presence of a logic_string 714 | group_ids_with_logic_string = set(EntityGroup.objects.filter( 715 | id__in=membership_cache.keys(), 716 | logic_string__isnull=False, 717 | ).values_list('id', flat=True)) 718 | 719 | # Loop over each group 720 | for group_id, memberships in membership_cache.items(): 721 | 722 | # Look at each membership 723 | for entity_id, entity_kind_id in memberships: 724 | 725 | # Only care about memberships with entity kind 726 | if entity_kind_id: 727 | 728 | # Make sure a dict exists for this kind 729 | entities_by_kind.setdefault(entity_kind_id, {}) 730 | 731 | # Always include all if there is a logic string 732 | if group_id in group_ids_with_logic_string: 733 | entities_by_kind[entity_kind_id]['all'] = [] 734 | kinds_with_all.add(entity_kind_id) 735 | 736 | # Check if this is all entities of a kind under a specific entity 737 | if entity_id: 738 | entities_by_kind[entity_kind_id][entity_id] = [] 739 | kinds_with_supers.add(entity_kind_id) 740 | super_ids.add(entity_id) 741 | else: 742 | # This is all entities of this kind 743 | entities_by_kind[entity_kind_id]['all'] = [] 744 | kinds_with_all.add(entity_kind_id) 745 | 746 | # Get entities for 'all' 747 | all_entities_for_types = Entity.all_objects.filter( 748 | entity_kind_id__in=kinds_with_all, 749 | ) 750 | if is_active is not None: 751 | all_entities_for_types = all_entities_for_types.filter(is_active=is_active) 752 | 753 | all_entities_for_types = all_entities_for_types.values_list('id', 'entity_kind_id') 754 | 755 | # Add entity ids to entity kind's all list 756 | for id, entity_kind_id in all_entities_for_types: 757 | entities_by_kind[entity_kind_id]['all'].append(id) 758 | 759 | # Get relationships for memberships defined by all of a kind under a super 760 | relationships = EntityRelationship.objects.filter( 761 | super_entity_id__in=super_ids, 762 | sub_entity__entity_kind_id__in=kinds_with_supers, 763 | ) 764 | 765 | # Make sure to respect the active flag for the sub entities under the supers 766 | if is_active is not None: 767 | relationships = relationships.filter(sub_entity__is_active=is_active) 768 | 769 | relationships = relationships.values_list( 770 | 'super_entity_id', 'sub_entity_id', 'sub_entity__entity_kind_id' 771 | ) 772 | 773 | # Add entity ids to each super entity's list 774 | for super_entity_id, sub_entity_id, sub_entity__entity_kind_id in relationships: 775 | entities_by_kind[sub_entity__entity_kind_id].setdefault(super_entity_id, []) 776 | entities_by_kind[sub_entity__entity_kind_id][super_entity_id].append(sub_entity_id) 777 | 778 | return entities_by_kind 779 | -------------------------------------------------------------------------------- /entity/signal_handlers.py: -------------------------------------------------------------------------------- 1 | from django.db.models.signals import post_save, post_delete, m2m_changed 2 | from manager_utils import post_bulk_operation 3 | 4 | from entity.config import entity_registry 5 | from entity.models import Entity 6 | from entity.sync import sync_entities, sync_entities_watching 7 | 8 | 9 | def delete_entity_signal_handler(sender, instance, **kwargs): 10 | """ 11 | Defines a signal handler for syncing an individual entity. Called when 12 | an entity is saved or deleted. 13 | """ 14 | if instance.__class__ in entity_registry.entity_registry: 15 | Entity.all_objects.delete_for_obj(instance) 16 | 17 | 18 | def save_entity_signal_handler(sender, instance, **kwargs): 19 | """ 20 | Defines a signal handler for saving an entity. Syncs the entity to 21 | the entity mirror table. 22 | """ 23 | if instance.__class__ in entity_registry.entity_registry: 24 | sync_entities(instance) 25 | 26 | if instance.__class__ in entity_registry.entity_watching: 27 | sync_entities_watching(instance) 28 | 29 | 30 | def m2m_changed_entity_signal_handler(sender, instance, action, **kwargs): 31 | """ 32 | Defines a signal handler for a manytomany changed signal. Only listens for the 33 | post actions so that entities are synced once (rather than twice for a pre and post action). 34 | """ 35 | if action == 'post_add' or action == 'post_remove' or action == 'post_clear': 36 | save_entity_signal_handler(sender, instance, **kwargs) 37 | 38 | 39 | def bulk_operation_signal_handler(sender, *args, **kwargs): 40 | """ 41 | When a bulk operation has happened on a model, sync all the entities again. 42 | NOTE - bulk syncing isn't turned on by default because of the consequences of it. 43 | For example, a user may issue a simple update to a single model, which would trigger 44 | syncing of all entities. It is up to the user to explicitly enable syncing on bulk 45 | operations with turn_on_syncing(bulk=True) 46 | """ 47 | if sender in entity_registry.entity_registry: 48 | sync_entities() 49 | 50 | 51 | def turn_off_syncing(for_post_save=True, for_post_delete=True, for_m2m_changed=True, for_post_bulk_operation=True): 52 | """ 53 | Disables all of the signals for syncing entities. By default, everything is turned off. If the user wants 54 | to turn off everything but one signal, for example the post_save signal, they would do: 55 | 56 | turn_off_sync(for_post_save=False) 57 | """ 58 | if for_post_save: 59 | post_save.disconnect(save_entity_signal_handler, dispatch_uid='save_entity_signal_handler') 60 | if for_post_delete: 61 | post_delete.disconnect(delete_entity_signal_handler, dispatch_uid='delete_entity_signal_handler') 62 | if for_m2m_changed: 63 | m2m_changed.disconnect(m2m_changed_entity_signal_handler, dispatch_uid='m2m_changed_entity_signal_handler') 64 | if for_post_bulk_operation: 65 | post_bulk_operation.disconnect(bulk_operation_signal_handler, dispatch_uid='bulk_operation_signal_handler') 66 | 67 | 68 | def turn_on_syncing(for_post_save=True, for_post_delete=True, for_m2m_changed=True, for_post_bulk_operation=False): 69 | """ 70 | Enables all of the signals for syncing entities. Everything is True by default, except for the post_bulk_operation 71 | signal. The reason for this is because when any bulk operation occurs on any mirrored entity model, it will 72 | result in every single entity being synced again. This is not a desired behavior by the majority of users, and 73 | should only be turned on explicitly. 74 | """ 75 | if for_post_save: 76 | post_save.connect(save_entity_signal_handler, dispatch_uid='save_entity_signal_handler') 77 | if for_post_delete: 78 | post_delete.connect(delete_entity_signal_handler, dispatch_uid='delete_entity_signal_handler') 79 | if for_m2m_changed: 80 | m2m_changed.connect(m2m_changed_entity_signal_handler, dispatch_uid='m2m_changed_entity_signal_handler') 81 | if for_post_bulk_operation: 82 | post_bulk_operation.connect(bulk_operation_signal_handler, dispatch_uid='bulk_operation_signal_handler') 83 | 84 | 85 | # Connect all default signal handlers 86 | turn_on_syncing() 87 | -------------------------------------------------------------------------------- /entity/sync.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provides functions for syncing entities and their relationships to the 3 | Entity and EntityRelationship tables. 4 | """ 5 | import logging 6 | from time import sleep 7 | 8 | import wrapt 9 | from collections import defaultdict 10 | 11 | from activatable_model import model_activations_changed 12 | from django import db 13 | from django.contrib.contenttypes.models import ContentType 14 | import manager_utils 15 | from django.db import transaction, connection 16 | 17 | from entity.config import entity_registry 18 | from entity.models import Entity, EntityRelationship, EntityKind 19 | 20 | 21 | LOG = logging.getLogger(__name__) 22 | 23 | 24 | def transaction_atomic_with_retry(num_retries=5, backoff=0.1): 25 | """ 26 | This is a decorator that will wrap the decorated method in an atomic transaction and 27 | retry the transaction a given number of times 28 | 29 | :param num_retries: How many times should we retry before we give up 30 | :param backoff: How long should we wait after each try 31 | """ 32 | 33 | # Create the decorator 34 | @wrapt.decorator 35 | def wrapper(wrapped, instance, args, kwargs): 36 | # Keep track of how many times we have tried 37 | num_tries = 0 38 | exception = None 39 | 40 | # Call the main sync entities method and catch any exceptions 41 | while num_tries <= num_retries: 42 | # Try running the transaction 43 | try: 44 | with transaction.atomic(): 45 | return wrapped(*args, **kwargs) 46 | # Catch any operation errors 47 | except db.utils.OperationalError as e: 48 | num_tries += 1 49 | exception = e 50 | sleep(backoff * num_tries) 51 | 52 | # If we have an exception raise it 53 | raise exception 54 | 55 | # Return the decorator 56 | return wrapper 57 | 58 | 59 | def defer_entity_syncing(*args, handler=None): 60 | """ 61 | A decorator for deferring entity syncing until after the function is complete 62 | An optional handler can be specified to handle the entity syncing. 63 | If no handler is passed the default sync_entities method will be called 64 | """ 65 | 66 | # Set a default handler 67 | handler = handler or sync_entities 68 | 69 | @wrapt.decorator 70 | def wrapper(wrapped, instance, args, kwargs): 71 | """ 72 | A decorator that can be used to defer the syncing of entities until after the method has been run 73 | This is being introduced to help avoid deadlocks in the meantime as we attempt to better understand 74 | why they are happening 75 | """ 76 | 77 | # Defer entity syncing while we run our method 78 | sync_entities.defer = True 79 | 80 | # Run the method 81 | try: 82 | return wrapped(*args, **kwargs) 83 | 84 | # After we run the method disable the deferred syncing 85 | # and sync all the entities that have been buffered to be synced 86 | finally: 87 | # Enable entity syncing again 88 | sync_entities.defer = False 89 | 90 | # Get the models that need to be synced 91 | model_objs = list(sync_entities.buffer.values()) 92 | 93 | # If none is in the model objects we need to sync all 94 | if None in sync_entities.buffer: 95 | model_objs = list() 96 | 97 | # Sync the entities that were deferred if any 98 | if len(sync_entities.buffer): 99 | handler(*model_objs) 100 | 101 | # Clear the buffer 102 | sync_entities.buffer = {} 103 | 104 | # If the decorator is called without arguments 105 | if len(args) == 1 and callable(args[0]): 106 | return wrapper(args[0]) 107 | else: 108 | return wrapper 109 | 110 | 111 | @wrapt.decorator 112 | def suppress_entity_syncing(wrapped, instance, args, kwargs): 113 | """ 114 | A decorator that can be used to completely suppress syncing of entities as a result of 115 | execution of the decorated method 116 | """ 117 | 118 | # Suppress entity syncing for the scope of the decorated method 119 | sync_entities.suppress = True 120 | 121 | # Run the method 122 | try: 123 | return wrapped(*args, **kwargs) 124 | 125 | # After we run the method return the entity sync state for future use 126 | finally: 127 | sync_entities.suppress = False 128 | 129 | 130 | def _get_super_entities_by_ctype(model_objs_by_ctype, model_ids_to_sync, sync_all): 131 | """ 132 | Given model objects organized by content type and a dictionary of all model IDs that need 133 | to be synced, organize all super entity relationships that need to be synced. 134 | 135 | Ensure that the model_ids_to_sync dict is updated with any new super entities 136 | that need to be part of the overall entity sync 137 | """ 138 | super_entities_by_ctype = defaultdict(lambda: defaultdict(list)) # pragma: no cover 139 | for ctype, model_objs_for_ctype in model_objs_by_ctype.items(): 140 | entity_config = entity_registry.entity_registry.get(ctype.model_class()) 141 | super_entities = entity_config.get_super_entities(model_objs_for_ctype, sync_all) 142 | super_entities_by_ctype[ctype] = { 143 | ContentType.objects.get_for_model(model_class, for_concrete_model=False): relationships 144 | for model_class, relationships in super_entities.items() 145 | } 146 | 147 | # Continue adding to the set of entities that need to be synced 148 | for super_entity_ctype, relationships in super_entities_by_ctype[ctype].items(): 149 | for sub_entity_id, super_entity_id in relationships: 150 | model_ids_to_sync[ctype].add(sub_entity_id) 151 | model_ids_to_sync[super_entity_ctype].add(super_entity_id) 152 | 153 | return super_entities_by_ctype 154 | 155 | 156 | def _fetch_entity_models(model_ids_to_sync, model_objs_map, model_objs_by_ctype, sync_all): 157 | """ 158 | Fetch the entity models per content type. This will also handle the 159 | case where accounts are created before _get_super_entities_by_ctype and 160 | the model_ids_to_sync do not match the model_objs_map 161 | """ 162 | for ctype, model_ids in model_ids_to_sync.items(): 163 | 164 | if sync_all: 165 | 166 | # Build a set of ids of already fetched models 167 | fetched_model_ids = { 168 | model.id 169 | for model in model_objs_by_ctype[ctype] 170 | } 171 | 172 | # Compute the set diff to see if any records are missing 173 | unfetched_model_ids = model_ids - fetched_model_ids 174 | else: 175 | unfetched_model_ids = model_ids 176 | 177 | # Check if new records 178 | if unfetched_model_ids: 179 | 180 | # Fetch the records and add them to the model_objs_map 181 | model_qset = entity_registry.entity_registry.get(ctype.model_class()).queryset 182 | model_objs_to_sync = model_qset.filter(id__in=unfetched_model_ids) 183 | for model_obj in model_objs_to_sync: 184 | model_objs_by_ctype[ctype].append(model_obj) 185 | model_objs_map[(ctype, model_obj.id)] = model_obj 186 | 187 | 188 | def _get_model_objs_to_sync(model_ids_to_sync, model_objs_map, model_objs_by_ctype, sync_all): 189 | """ 190 | Given the model IDs to sync, fetch all model objects to sync 191 | """ 192 | model_objs_to_sync = {} 193 | 194 | _fetch_entity_models(model_ids_to_sync, model_objs_map, model_objs_by_ctype, sync_all) 195 | 196 | for ctype, model_ids_to_sync_for_ctype in model_ids_to_sync.items(): 197 | model_objs_to_sync[ctype] = [ 198 | model_objs_map[ctype, model_id] 199 | for model_id in model_ids_to_sync_for_ctype 200 | ] 201 | 202 | return model_objs_to_sync 203 | 204 | 205 | def sync_entities(*model_objs): 206 | """ 207 | Syncs entities 208 | 209 | Args: 210 | model_objs (List[Model]): The model objects to sync. If empty, all entities will be synced 211 | """ 212 | 213 | if sync_entities.suppress: 214 | # Return false that we did not do anything 215 | return False 216 | 217 | # Check if we are deferring processing 218 | if sync_entities.defer: 219 | # If we dont have any model objects passed add a none to let us know that we need to sync all 220 | if not model_objs: 221 | sync_entities.buffer[None] = None 222 | else: 223 | # Add each model obj to the buffer 224 | for model_obj in model_objs: 225 | sync_entities.buffer[(model_obj.__class__, model_obj.pk)] = model_obj 226 | 227 | # Return false that we did not do anything 228 | return False 229 | 230 | # Create a syncer and sync 231 | EntitySyncer(*model_objs).sync() 232 | 233 | 234 | # Add a defer and buffer method to the sync entities method 235 | # This is used by the defer_entity_syncing decorator 236 | sync_entities.defer = False 237 | sync_entities.buffer = {} 238 | # Add a suppress attribute to the sync entities method 239 | sync_entities.suppress = False 240 | 241 | 242 | def sync_entities_watching(instance): 243 | """ 244 | Syncs entities watching changes of a model instance. 245 | """ 246 | for entity_model, entity_model_getter in entity_registry.entity_watching[instance.__class__]: 247 | model_objs = list(entity_model_getter(instance)) 248 | if model_objs: 249 | sync_entities(*model_objs) 250 | 251 | 252 | class EntitySyncer(object): 253 | """ 254 | A class that will handle the syncing of entities 255 | """ 256 | 257 | def __init__(self, *model_objs): 258 | """ 259 | Initialize the entity syncer with the models we need to sync 260 | """ 261 | 262 | # Set the model objects 263 | self.model_objs = model_objs 264 | 265 | # Are we syncing all 266 | self.sync_all = not model_objs 267 | 268 | def sync(self): 269 | # Log what we are syncing 270 | LOG.debug('sync_entities') 271 | LOG.debug(self.model_objs) 272 | 273 | # Determine if we are syncing all 274 | sync_all = not self.model_objs 275 | model_objs_map = { 276 | (ContentType.objects.get_for_model(model_obj, for_concrete_model=False), model_obj.id): model_obj 277 | for model_obj in self.model_objs 278 | } 279 | 280 | # If we are syncing all build the entire map for all entity types 281 | if self.sync_all: 282 | for model_class, entity_config in entity_registry.entity_registry.items(): 283 | model_qset = entity_config.queryset 284 | model_objs_map.update({ 285 | (ContentType.objects.get_for_model(model_class, for_concrete_model=False), model_obj.id): model_obj 286 | for model_obj in model_qset.all() 287 | }) 288 | 289 | # Organize by content type 290 | model_objs_by_ctype = defaultdict(list) 291 | for (ctype, model_id), model_obj in model_objs_map.items(): 292 | model_objs_by_ctype[ctype].append(model_obj) 293 | 294 | # Build a dict of all entities that need to be synced. These include the original models 295 | # and any super entities from super_entities_by_ctype. This dict is keyed on ctype with 296 | # a list of IDs of each model 297 | model_ids_to_sync = defaultdict(set) 298 | for (ctype, model_id), model_obj in model_objs_map.items(): 299 | model_ids_to_sync[ctype].add(model_obj.id) 300 | 301 | # For each ctype, obtain super entities. This is a dict keyed on ctype. Each value 302 | # is a dict keyed on the ctype of the super entity with a list of tuples for 303 | # IDs of sub/super entity relationships 304 | super_entities_by_ctype = _get_super_entities_by_ctype(model_objs_by_ctype, model_ids_to_sync, sync_all) 305 | 306 | # Now that we have all models we need to sync, fetch them so that we can extract 307 | # metadata and entity kinds. If we are syncing all entities, we've already fetched 308 | # everything and can fill in this data struct without doing another DB hit 309 | model_objs_to_sync = _get_model_objs_to_sync(model_ids_to_sync, model_objs_map, model_objs_by_ctype, sync_all) 310 | 311 | # Obtain all entity kind tuples associated with the models 312 | entity_kind_tuples_to_sync = set() 313 | for ctype, model_objs_to_sync_for_ctype in model_objs_to_sync.items(): 314 | entity_config = entity_registry.entity_registry.get(ctype.model_class()) 315 | for model_obj in model_objs_to_sync_for_ctype: 316 | entity_kind_tuples_to_sync.add(entity_config.get_entity_kind(model_obj)) 317 | 318 | # Build the entity kinds that we need to sync 319 | entity_kinds_to_upsert = [ 320 | EntityKind(name=name, display_name=display_name) 321 | for name, display_name in entity_kind_tuples_to_sync 322 | ] 323 | 324 | # Upsert the entity kinds 325 | upserted_entity_kinds = self.upsert_entity_kinds( 326 | entity_kinds=entity_kinds_to_upsert 327 | ) 328 | 329 | # Build a map of entity kind name to entity kind 330 | entity_kinds_map = { 331 | entity_kind.name: entity_kind 332 | for entity_kind in upserted_entity_kinds 333 | } 334 | 335 | # Now that we have all entity kinds, build all entities that need to be synced 336 | entities_to_upsert = [] 337 | for ctype, model_objs_to_sync_for_ctype in model_objs_to_sync.items(): 338 | entity_config = entity_registry.entity_registry.get(ctype.model_class()) 339 | entities_to_upsert.extend([ 340 | Entity( 341 | entity_id=model_obj.id, 342 | entity_type_id=ctype.id, 343 | entity_kind_id=entity_kinds_map[entity_config.get_entity_kind(model_obj)[0]].id, 344 | entity_meta=entity_config.get_entity_meta(model_obj), 345 | display_name=entity_config.get_display_name(model_obj), 346 | is_active=entity_config.get_is_active(model_obj) 347 | ) 348 | for model_obj in model_objs_to_sync_for_ctype 349 | ]) 350 | 351 | # Upsert the entities and get the upserted entities and the changed state 352 | upserted_entities, changed_entity_activation_state = self.upsert_entities( 353 | entities=entities_to_upsert, 354 | sync=self.sync_all 355 | ) 356 | 357 | # Call the model activations changed signal manually since we have done a bulk operation 358 | self.send_entity_activation_events(changed_entity_activation_state) 359 | 360 | # Create a map out of entities 361 | entities_map = { 362 | (entity.entity_type_id, entity.entity_id): entity 363 | for entity in upserted_entities 364 | } 365 | 366 | # Now that all entities are upserted, sync entity relationships 367 | entity_relationships_to_sync = [ 368 | EntityRelationship( 369 | sub_entity_id=entities_map[sub_ctype.id, sub_entity_id].id, 370 | super_entity_id=entities_map[super_ctype.id, super_entity_id].id, 371 | ) 372 | for sub_ctype, super_entities_by_sub_ctype in super_entities_by_ctype.items() 373 | for super_ctype, relationships in super_entities_by_sub_ctype.items() 374 | for sub_entity_id, super_entity_id in relationships 375 | if (sub_ctype.id, sub_entity_id) in entities_map and (super_ctype.id, super_entity_id) in entities_map 376 | ] 377 | 378 | # Find the entities of the original model objects we were syncing. These 379 | # are needed to properly sync entity relationships 380 | original_entity_ids = [ 381 | entities_map[ctype.id, model_obj.id].id 382 | for ctype, model_objs_for_ctype in model_objs_by_ctype.items() 383 | for model_obj in model_objs_for_ctype 384 | if (ctype.id, model_obj.id) in entities_map 385 | ] 386 | 387 | if self.sync_all: 388 | # If we're syncing everything, just sync against the entire entity relationship 389 | # table instead of doing a complex __in query 390 | sync_against = EntityRelationship.objects.all() 391 | else: 392 | sync_against = EntityRelationship.objects.filter(sub_entity_id__in=original_entity_ids) 393 | 394 | # Sync the relations 395 | self.upsert_entity_relationships( 396 | queryset=sync_against, 397 | entity_relationships=entity_relationships_to_sync 398 | ) 399 | 400 | @transaction_atomic_with_retry() 401 | def upsert_entity_kinds(self, entity_kinds): 402 | """ 403 | Given a list of entity kinds ensure they are synced properly to the database. 404 | This will ensure that only unchanged entity kinds are synced and will still return all 405 | updated entity kinds 406 | 407 | :param entity_kinds: The list of entity kinds to sync 408 | """ 409 | 410 | # Filter out unchanged entity kinds 411 | unchanged_entity_kinds = {} 412 | if entity_kinds: 413 | unchanged_entity_kinds = { 414 | (entity_kind.name, entity_kind.display_name): entity_kind 415 | for entity_kind in EntityKind.all_objects.extra( 416 | where=['(name, display_name) IN %s'], 417 | params=[tuple( 418 | (entity_kind.name, entity_kind.display_name) 419 | for entity_kind in entity_kinds 420 | )] 421 | ) 422 | } 423 | 424 | # Filter out the unchanged entity kinds 425 | changed_entity_kinds = [ 426 | entity_kind 427 | for entity_kind in entity_kinds 428 | if (entity_kind.name, entity_kind.display_name) not in unchanged_entity_kinds 429 | ] 430 | 431 | # If any of our kinds have changed upsert them 432 | upserted_enitity_kinds = [] 433 | if changed_entity_kinds: 434 | # Select all our existing entity kinds for update so we can do proper locking 435 | # We have to select all here for some odd reason, if we only select the ones 436 | # we are syncing we still run into deadlock issues 437 | list(EntityKind.all_objects.all().order_by('id').select_for_update().values_list('id', flat=True)) 438 | 439 | # Upsert the entity kinds 440 | upserted_enitity_kinds = manager_utils.bulk_upsert( 441 | queryset=EntityKind.all_objects.filter( 442 | name__in=[entity_kind.name for entity_kind in changed_entity_kinds] 443 | ), 444 | model_objs=changed_entity_kinds, 445 | unique_fields=['name'], 446 | update_fields=['display_name'], 447 | return_upserts=True 448 | ) 449 | 450 | # Return all the entity kinds 451 | return upserted_enitity_kinds + list(unchanged_entity_kinds.values()) 452 | 453 | @transaction_atomic_with_retry() 454 | def upsert_entities(self, entities, sync=False): 455 | """ 456 | Upsert a list of entities to the database 457 | :param entities: The entities to sync 458 | :param sync: Do a sync instead of an upsert 459 | """ 460 | 461 | # Select the entities we are upserting for update to reduce deadlocks 462 | if entities: 463 | # Default select for update query when syncing all 464 | select_for_update_query = ( 465 | 'SELECT FROM {table_name} ORDER BY id ASC FOR NO KEY UPDATE' 466 | ).format( 467 | table_name=Entity._meta.db_table 468 | ) 469 | select_for_update_query_params = [] 470 | 471 | # If we are not syncing all, only select those we are updating 472 | if not sync: 473 | select_for_update_query = ( 474 | 'SELECT FROM {table_name} ' 475 | 'WHERE (entity_type_id, entity_id) IN %s ' 476 | 'ORDER BY id ASC ' 477 | 'FOR NO KEY UPDATE' 478 | ).format( 479 | table_name=Entity._meta.db_table 480 | ) 481 | select_for_update_query_params = [tuple( 482 | (entity.entity_type_id, entity.entity_id) 483 | for entity in entities 484 | )] 485 | 486 | # Select the items for update 487 | with connection.cursor() as cursor: 488 | cursor.execute(select_for_update_query, select_for_update_query_params) 489 | 490 | # Compute the initial queryset and the initial state of the entities we are syncing 491 | # We need the initial state so we can compare it to the new state to determine any 492 | # entities that were activated or deactivated 493 | initial_queryset = Entity.all_objects.all() 494 | if not sync: 495 | initial_queryset = Entity.all_objects.extra( 496 | where=['(entity_type_id, entity_id) IN %s'], 497 | params=[tuple( 498 | (entity.entity_type_id, entity.entity_id) 499 | for entity in entities 500 | )] 501 | ) 502 | initial_entity_activation_state = { 503 | entity[0]: entity[1] 504 | for entity in initial_queryset.values_list('id', 'is_active') 505 | } 506 | 507 | # Sync all the entities if the sync flag is passed 508 | if sync: 509 | upserted_entities = manager_utils.sync( 510 | queryset=initial_queryset, 511 | model_objs=entities, 512 | unique_fields=['entity_type_id', 'entity_id'], 513 | update_fields=['entity_kind_id', 'entity_meta', 'display_name', 'is_active'], 514 | return_upserts=True 515 | ) 516 | # Otherwise we want to upsert our entities 517 | else: 518 | upserted_entities = manager_utils.bulk_upsert( 519 | queryset=initial_queryset, 520 | model_objs=entities, 521 | unique_fields=['entity_type_id', 'entity_id'], 522 | update_fields=['entity_kind_id', 'entity_meta', 'display_name', 'is_active'], 523 | return_upserts=True 524 | ) 525 | 526 | # Compute the current state of the entities 527 | current_entity_activation_state = { 528 | entity.id: entity.is_active 529 | for entity in upserted_entities 530 | } 531 | 532 | # Computed the changed activation state of the entities 533 | changed_entity_activation_state = {} 534 | all_entity_ids = set(initial_entity_activation_state.keys()) | set(current_entity_activation_state.keys()) 535 | for entity_id in all_entity_ids: 536 | # Get the initial activation state of the entity 537 | # Default to false so we only detect when the model has actually changed 538 | initial_activation_state = initial_entity_activation_state.get(entity_id, False) 539 | 540 | # Get the current state of the entity 541 | # Default to false here since the upserts do not return is_active=False due 542 | # to the default object manager excluding these 543 | current_activation_state = current_entity_activation_state.get(entity_id, False) 544 | 545 | # Check if the state changed and at it to the changed entity state 546 | if initial_activation_state != current_activation_state: 547 | changed_entity_activation_state[entity_id] = current_activation_state 548 | 549 | # Return the upserted entities 550 | return upserted_entities, changed_entity_activation_state 551 | 552 | @transaction_atomic_with_retry() 553 | def upsert_entity_relationships(self, queryset, entity_relationships): 554 | """ 555 | Upsert entity relationships to the database 556 | :param queryset: The base queryset to use 557 | :param entity_relationships: The entity relationships to ensure exist in the database 558 | """ 559 | 560 | # Select the relationships for update 561 | if entity_relationships: 562 | list(queryset.order_by('id').select_for_update().values_list( 563 | 'id', 564 | flat=True 565 | )) 566 | 567 | # Sync the relationships 568 | return manager_utils.sync( 569 | queryset=queryset, 570 | model_objs=entity_relationships, 571 | unique_fields=['sub_entity_id', 'super_entity_id'], 572 | update_fields=[], 573 | return_upserts=True 574 | ) 575 | 576 | def send_entity_activation_events(self, changed_entity_activation_state): 577 | """ 578 | Given a changed entity state dict, fire the appropriate signals 579 | :param changed_entity_activation_state: The changed entity activation state {entity_id: is_active} 580 | """ 581 | 582 | # Compute the activated and deactivated entities 583 | activated_entities = set() 584 | deactivated_entities = set() 585 | for entity_id, is_active in changed_entity_activation_state.items(): 586 | if is_active: 587 | activated_entities.add(entity_id) 588 | else: 589 | deactivated_entities.add(entity_id) 590 | 591 | # If any entities were activated call the activation change event with the active flag 592 | if activated_entities: 593 | model_activations_changed.send( 594 | sender=Entity, 595 | instance_ids=sorted(list(activated_entities)), 596 | is_active=True 597 | ) 598 | 599 | # If any entities were deactivated call the activation change event with the active flag as false 600 | if deactivated_entities: 601 | model_activations_changed.send( 602 | sender=Entity, 603 | instance_ids=sorted(list(deactivated_entities)), 604 | is_active=False 605 | ) 606 | -------------------------------------------------------------------------------- /entity/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ambitioninc/django-entity/68be2ad83352c8b96b9117b6fcedc166a2392b88/entity/tests/__init__.py -------------------------------------------------------------------------------- /entity/tests/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from django.db import models, migrations 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('entity', '0002_entitykind_is_active'), 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='Account', 16 | fields=[ 17 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 18 | ('email', models.CharField(max_length=256)), 19 | ('is_active', models.BooleanField(default=True)), 20 | ('is_captain', models.BooleanField(default=False)), 21 | ], 22 | options={ 23 | 'abstract': False, 24 | }, 25 | ), 26 | migrations.CreateModel( 27 | name='Competitor', 28 | fields=[ 29 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 30 | ('name', models.CharField(max_length=64)), 31 | ('is_active', models.BooleanField(default=True)), 32 | ], 33 | options={ 34 | 'abstract': False, 35 | }, 36 | ), 37 | migrations.CreateModel( 38 | name='DummyModel', 39 | fields=[ 40 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 41 | ('dummy_data', models.CharField(max_length=64)), 42 | ], 43 | options={ 44 | 'abstract': False, 45 | }, 46 | ), 47 | migrations.CreateModel( 48 | name='EntityPointer', 49 | fields=[ 50 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 51 | ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='entity.Entity')), 52 | ], 53 | options={ 54 | 'abstract': False, 55 | }, 56 | ), 57 | migrations.CreateModel( 58 | name='M2mEntity', 59 | fields=[ 60 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 61 | ], 62 | options={ 63 | 'abstract': False, 64 | }, 65 | ), 66 | migrations.CreateModel( 67 | name='MultiInheritEntity', 68 | fields=[ 69 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 70 | ('data', models.CharField(max_length=64)), 71 | ], 72 | options={ 73 | 'abstract': False, 74 | }, 75 | ), 76 | migrations.CreateModel( 77 | name='PointsToAccount', 78 | fields=[ 79 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 80 | ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tests.Account')), 81 | ], 82 | options={ 83 | 'abstract': False, 84 | }, 85 | ), 86 | migrations.CreateModel( 87 | name='PointsToM2mEntity', 88 | fields=[ 89 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 90 | ('m2m_entity', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='tests.M2mEntity')), 91 | ], 92 | options={ 93 | 'abstract': False, 94 | }, 95 | ), 96 | migrations.CreateModel( 97 | name='Team', 98 | fields=[ 99 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 100 | ('name', models.CharField(max_length=256)), 101 | ('is_active', models.BooleanField(default=True)), 102 | ], 103 | options={ 104 | 'abstract': False, 105 | }, 106 | ), 107 | migrations.CreateModel( 108 | name='TeamGroup', 109 | fields=[ 110 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 111 | ('name', models.CharField(max_length=256)), 112 | ], 113 | options={ 114 | 'abstract': False, 115 | }, 116 | ), 117 | migrations.AddField( 118 | model_name='team', 119 | name='team_group', 120 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tests.TeamGroup', null=True), 121 | ), 122 | migrations.AddField( 123 | model_name='m2mentity', 124 | name='teams', 125 | field=models.ManyToManyField(to='tests.Team'), 126 | ), 127 | migrations.AddField( 128 | model_name='account', 129 | name='competitor', 130 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tests.Competitor', null=True), 131 | ), 132 | migrations.AddField( 133 | model_name='account', 134 | name='team', 135 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tests.Team', null=True), 136 | ), 137 | migrations.AddField( 138 | model_name='account', 139 | name='team2', 140 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='tests.Team', null=True), 141 | ), 142 | migrations.AddField( 143 | model_name='account', 144 | name='team_group', 145 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tests.TeamGroup', null=True), 146 | ), 147 | ] 148 | -------------------------------------------------------------------------------- /entity/tests/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ambitioninc/django-entity/68be2ad83352c8b96b9117b6fcedc166a2392b88/entity/tests/migrations/__init__.py -------------------------------------------------------------------------------- /entity/tests/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | from entity.config import EntityConfig, register_entity 4 | from entity.models import Entity 5 | from manager_utils import ManagerUtilsManager 6 | 7 | 8 | class BaseEntityModel(models.Model): 9 | class Meta: 10 | abstract = True 11 | 12 | objects = ManagerUtilsManager() 13 | 14 | 15 | class TeamGroup(BaseEntityModel): 16 | """ 17 | A grouping of teams. 18 | """ 19 | name = models.CharField(max_length=256) 20 | 21 | def __str__(self): 22 | return self.name 23 | 24 | 25 | class Competitor(BaseEntityModel): 26 | """ 27 | An enclosing group for competitors 28 | """ 29 | name = models.CharField(max_length=64) 30 | is_active = models.BooleanField(default=True) 31 | 32 | def __str__(self): 33 | return self.name 34 | 35 | 36 | class Team(BaseEntityModel): 37 | """ 38 | A team entity model. Encapsulates accounts. 39 | """ 40 | name = models.CharField(max_length=256) 41 | # Used for testing if the entity is active 42 | is_active = models.BooleanField(default=True) 43 | # Used for additional super entity tests 44 | team_group = models.ForeignKey(TeamGroup, null=True, on_delete=models.CASCADE) 45 | 46 | 47 | class Account(BaseEntityModel): 48 | """ 49 | An account entity model 50 | """ 51 | email = models.CharField(max_length=256) 52 | # Used for testing if the entity is active 53 | is_active = models.BooleanField(default=True) 54 | # Team is a super entity for an account 55 | team = models.ForeignKey(Team, null=True, on_delete=models.CASCADE) 56 | # True if the account is a captain of its team 57 | is_captain = models.BooleanField(default=False) 58 | # The second team that the account is on 59 | team2 = models.ForeignKey(Team, null=True, related_name='+', on_delete=models.CASCADE) 60 | # The team group 61 | team_group = models.ForeignKey(TeamGroup, null=True, on_delete=models.CASCADE) 62 | # The competitor group 63 | competitor = models.ForeignKey(Competitor, null=True, on_delete=models.CASCADE) 64 | 65 | def __str__(self): 66 | return self.email 67 | 68 | 69 | class M2mEntity(BaseEntityModel): 70 | """ 71 | Used for testing syncing of a model with a M2M. 72 | """ 73 | teams = models.ManyToManyField(Team) 74 | 75 | 76 | class PointsToM2mEntity(BaseEntityModel): 77 | """ 78 | A model that points to an m2mentity. Used to recreate the scenario when an account 79 | points to a user that is included in a group. 80 | """ 81 | m2m_entity = models.OneToOneField(M2mEntity, on_delete=models.CASCADE) 82 | 83 | 84 | class PointsToAccount(BaseEntityModel): 85 | account = models.ForeignKey(Account, on_delete=models.CASCADE) 86 | 87 | 88 | class EntityPointer(BaseEntityModel): 89 | """ 90 | Describes a test model that points to an entity. Used for ensuring 91 | that syncing entities doesn't perform any Entity deletes (causing models like 92 | this to be cascade deleted) 93 | """ 94 | entity = models.ForeignKey(Entity, on_delete=models.CASCADE) 95 | 96 | 97 | class DummyModel(BaseEntityModel): 98 | """ 99 | Used to ensure that models that don't register for entity syncing aren't synced. 100 | """ 101 | dummy_data = models.CharField(max_length=64) 102 | 103 | objects = ManagerUtilsManager() 104 | 105 | 106 | class BaseEntityClass(BaseEntityModel): 107 | """ 108 | A base class that inherits EntityModelMixin. Helps ensure that mutliple-inherited 109 | entities are still synced properly. 110 | """ 111 | class Meta: 112 | abstract = True 113 | 114 | 115 | class MultiInheritEntity(BaseEntityClass): 116 | """ 117 | Verifies that entities that dont directly inherit from the EntityModelMixin are 118 | still synced properly. 119 | """ 120 | data = models.CharField(max_length=64) 121 | 122 | 123 | @register_entity() 124 | class AccountConfig(EntityConfig): 125 | """ 126 | Entity configuration for the account model 127 | """ 128 | queryset = Account.objects.select_related('team', 'team2', 'team_group', 'competitor') 129 | 130 | def get_is_active(self, model_obj): 131 | return model_obj.is_active 132 | 133 | def get_entity_meta(self, model_obj): 134 | """ 135 | Returns metadata about the account that will be serialized 136 | in the mirrored entity tables. 137 | """ 138 | return { 139 | 'email': model_obj.email, 140 | 'team': model_obj.team.name if model_obj.team else None, 141 | 'is_captain': model_obj.is_captain, 142 | 'team_is_active': model_obj.team.is_active if model_obj.team else None, 143 | } 144 | 145 | def get_super_entities(self, model_objs, sync_all): 146 | """ 147 | Gets the super entities this entity belongs to. 148 | """ 149 | if sync_all: 150 | accounts = list(self.queryset.all()) 151 | if not sync_all: 152 | accounts = model_objs 153 | 154 | return { 155 | Team: [ 156 | (a.id, a.team_id) for a in accounts if a.team_id 157 | ] + [ 158 | (a.id, a.team2_id) for a in accounts if a.team2_id 159 | ], 160 | TeamGroup: [(a.id, a.team_group_id) for a in accounts if a.team_group_id], 161 | Competitor: [(a.id, a.competitor_id) for a in accounts if a.competitor_id] 162 | } 163 | 164 | 165 | @register_entity() 166 | class TeamConfig(EntityConfig): 167 | queryset = Team.objects.select_related('team_group') 168 | 169 | def get_is_active(self, model_obj): 170 | return model_obj.is_active 171 | 172 | def get_super_entities(self, model_objs, sync_all): 173 | return { 174 | TeamGroup: [(t.id, t.team_group_id) for t in model_objs if t.team_group_id] 175 | } 176 | 177 | def get_display_name(self, model_obj): 178 | return 'team' 179 | 180 | 181 | @register_entity() 182 | class M2mEntityConfig(EntityConfig): 183 | queryset = M2mEntity.objects.prefetch_related('teams') 184 | 185 | def get_super_entities(self, model_objs, sync_all): 186 | return { 187 | Team: [ 188 | (m.id, t.id) 189 | for m in model_objs 190 | for t in m.teams.all() 191 | ] 192 | } 193 | 194 | 195 | @register_entity() 196 | class PointsToM2mEntityConfig(EntityConfig): 197 | queryset = PointsToM2mEntity.objects.prefetch_related('m2m_entity__teams') 198 | 199 | watching = [ 200 | (M2mEntity, lambda m2m_entity_obj: PointsToM2mEntity.objects.filter(m2m_entity=m2m_entity_obj)), 201 | ] 202 | 203 | def get_super_entities(self, model_objs, sync_all): 204 | return { 205 | Team: [ 206 | (p.id, t.id) 207 | for p in model_objs 208 | for t in p.m2m_entity.teams.all() 209 | ] 210 | } 211 | 212 | 213 | @register_entity() 214 | class PointsToAccountConfig(EntityConfig): 215 | queryset = PointsToAccount.objects.all() 216 | 217 | watching = [ 218 | (Competitor, lambda competitor_obj: PointsToAccount.objects.filter(account__competitor=competitor_obj)), 219 | (Team, lambda team_obj: PointsToAccount.objects.filter(account__team=team_obj)), 220 | ] 221 | 222 | def get_entity_meta(self, model_obj): 223 | return { 224 | 'competitor_name': model_obj.account.competitor.name if model_obj.account.competitor else 'None', 225 | 'team_name': model_obj.account.team.name if model_obj.account.team else 'None', 226 | } 227 | 228 | 229 | @register_entity() 230 | class TeamGroupConfig(EntityConfig): 231 | queryset = TeamGroup.objects.all() 232 | 233 | 234 | @register_entity() 235 | class CompetitorConfig(EntityConfig): 236 | queryset = Competitor.objects.all() 237 | 238 | 239 | @register_entity() 240 | class MultiInheritEntityConfig(EntityConfig): 241 | queryset = MultiInheritEntity.objects.all() 242 | -------------------------------------------------------------------------------- /entity/tests/registry_tests.py: -------------------------------------------------------------------------------- 1 | from django.db.models import Model 2 | from django.test import TestCase 3 | from unittest.mock import patch 4 | 5 | from entity.config import EntityConfig, entity_registry, EntityRegistry, register_entity 6 | 7 | 8 | class EntityRegistryTest(TestCase): 9 | """ 10 | Tests the EntityRegistry class. 11 | """ 12 | def test_entity_registry_is_instance_entity_registry(self): 13 | """ 14 | Tests the entity_registry global variable is an instance of EntityRegistry. 15 | """ 16 | self.assertTrue(isinstance(entity_registry, EntityRegistry)) 17 | 18 | def test_register_non_model_or_qset(self): 19 | """ 20 | Tests that a value error is raised when trying to register something that 21 | isnt a model or a queryset. 22 | """ 23 | class InvalidEntityObject(object): 24 | pass 25 | 26 | with self.assertRaises(ValueError): 27 | EntityRegistry().register_entity(InvalidEntityObject) 28 | 29 | def test_register_valid_entity_config(self): 30 | """ 31 | Tests registering an entity config with a model. 32 | """ 33 | class ValidRegistryModel(Model): 34 | pass 35 | 36 | class ValidEntityConfig(EntityConfig): 37 | queryset = ValidRegistryModel.objects.all() 38 | 39 | entity_registry = EntityRegistry() 40 | entity_registry.register_entity(ValidEntityConfig) 41 | entity_registry_info = entity_registry._entity_registry[ValidRegistryModel] 42 | self.assertTrue(isinstance(entity_registry_info, ValidEntityConfig)) 43 | 44 | def test_register_invalid_entity_config(self): 45 | """ 46 | Tests registering an invalid entity config that does not inherit EntityConfig 47 | """ 48 | class ValidRegistryModel(Model): 49 | pass 50 | 51 | class InvalidEntityConfig(object): 52 | pass 53 | 54 | entity_registry = EntityRegistry() 55 | with self.assertRaises(ValueError): 56 | entity_registry.register_entity(InvalidEntityConfig) 57 | 58 | def test_register_invalid_entity_config_no_qset(self): 59 | """ 60 | Tests registering an invalid entity config that does not have queryset 61 | """ 62 | class ValidRegistryModel(Model): 63 | pass 64 | 65 | class InvalidEntityConfig(EntityConfig): 66 | pass 67 | 68 | entity_registry = EntityRegistry() 69 | with self.assertRaises(ValueError): 70 | entity_registry.register_entity(InvalidEntityConfig) 71 | 72 | @patch.object(EntityRegistry, 'register_entity') 73 | def test_decorator(self, register_mock): 74 | """ 75 | Tests the decorator calls appropriate functions. 76 | """ 77 | class ValidRegistryModel(Model): 78 | pass 79 | 80 | @register_entity() 81 | class ValidEntityConfig(EntityConfig): 82 | queryset = ValidRegistryModel.objects.all() 83 | 84 | register_mock.assert_called_once_with(ValidEntityConfig) 85 | 86 | @patch.object(EntityRegistry, 'register_entity') 87 | def test_decorator_qset(self, register_mock): 88 | """ 89 | Tests the decorator calls appropriate functions. 90 | """ 91 | class ValidRegistryModel(Model): 92 | pass 93 | 94 | @register_entity() 95 | class ValidEntityConfig(EntityConfig): 96 | queryset = ValidRegistryModel.objects.all() 97 | 98 | register_mock.assert_called_once_with(ValidEntityConfig) 99 | -------------------------------------------------------------------------------- /entity/tests/sync_tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provides tests for the syncing functionalities in django entity. 3 | """ 4 | from django import db 5 | from django.contrib.contenttypes.models import ContentType 6 | from django.core.management import call_command 7 | from django_dynamic_fixture import G 8 | from entity.config import EntityRegistry 9 | from entity.models import Entity, EntityRelationship, EntityKind 10 | from entity.sync import ( 11 | sync_entities, defer_entity_syncing, transaction_atomic_with_retry, _get_super_entities_by_ctype, 12 | suppress_entity_syncing, 13 | ) 14 | from entity.signal_handlers import turn_on_syncing, turn_off_syncing 15 | from unittest.mock import patch, MagicMock, call, Mock 16 | 17 | from entity.tests.models import ( 18 | Account, Team, EntityPointer, DummyModel, MultiInheritEntity, AccountConfig, TeamConfig, TeamGroup, 19 | M2mEntity, PointsToM2mEntity, PointsToAccount, Competitor 20 | ) 21 | from entity.tests.utils import EntityTestCase 22 | 23 | 24 | class TestTurnOnOffSyncing(EntityTestCase): 25 | """ 26 | Tests turning on and off entity syncing. 27 | """ 28 | @patch('entity.signal_handlers.post_save', spec_set=True) 29 | @patch('entity.signal_handlers.post_delete', spec_set=True) 30 | @patch('entity.signal_handlers.m2m_changed', spec_set=True) 31 | @patch('entity.signal_handlers.post_bulk_operation', spec_set=True) 32 | def test_turn_on_syncing_all_handlers_true( 33 | self, post_bulk_operation_mock, m2m_changed_mock, post_delete_mock, post_save_mock): 34 | turn_on_syncing(for_post_save=True, for_post_delete=True, for_m2m_changed=True, for_post_bulk_operation=True) 35 | self.assertTrue(post_save_mock.connect.called) 36 | self.assertTrue(post_delete_mock.connect.called) 37 | self.assertTrue(m2m_changed_mock.connect.called) 38 | self.assertTrue(post_bulk_operation_mock.connect.called) 39 | 40 | @patch('entity.signal_handlers.post_save', spec_set=True) 41 | @patch('entity.signal_handlers.post_delete', spec_set=True) 42 | @patch('entity.signal_handlers.m2m_changed', spec_set=True) 43 | @patch('entity.signal_handlers.post_bulk_operation', spec_set=True) 44 | def test_turn_on_syncing_all_handlers_false( 45 | self, post_bulk_operation_mock, m2m_changed_mock, post_delete_mock, post_save_mock): 46 | turn_on_syncing( 47 | for_post_save=False, for_post_delete=False, for_m2m_changed=False, for_post_bulk_operation=False) 48 | self.assertFalse(post_save_mock.connect.called) 49 | self.assertFalse(post_delete_mock.connect.called) 50 | self.assertFalse(m2m_changed_mock.connect.called) 51 | self.assertFalse(post_bulk_operation_mock.connect.called) 52 | 53 | @patch('entity.signal_handlers.post_save', spec_set=True) 54 | @patch('entity.signal_handlers.post_delete', spec_set=True) 55 | @patch('entity.signal_handlers.m2m_changed', spec_set=True) 56 | @patch('entity.signal_handlers.post_bulk_operation', spec_set=True) 57 | def test_turn_off_syncing_all_handlers_true( 58 | self, post_bulk_operation_mock, m2m_changed_mock, post_delete_mock, post_save_mock): 59 | turn_off_syncing(for_post_save=True, for_post_delete=True, for_m2m_changed=True, for_post_bulk_operation=True) 60 | self.assertTrue(post_save_mock.disconnect.called) 61 | self.assertTrue(post_delete_mock.disconnect.called) 62 | self.assertTrue(m2m_changed_mock.disconnect.called) 63 | self.assertTrue(post_bulk_operation_mock.disconnect.called) 64 | 65 | @patch('entity.signal_handlers.post_save', spec_set=True) 66 | @patch('entity.signal_handlers.post_delete', spec_set=True) 67 | @patch('entity.signal_handlers.m2m_changed', spec_set=True) 68 | @patch('entity.signal_handlers.post_bulk_operation', spec_set=True) 69 | def test_turn_off_syncing_all_handlers_false( 70 | self, post_bulk_operation_mock, m2m_changed_mock, post_delete_mock, post_save_mock): 71 | turn_off_syncing( 72 | for_post_save=False, for_post_delete=False, for_m2m_changed=False, for_post_bulk_operation=False) 73 | self.assertFalse(post_save_mock.disconnect.called) 74 | self.assertFalse(post_delete_mock.disconnect.called) 75 | self.assertFalse(m2m_changed_mock.disconnect.called) 76 | self.assertFalse(post_bulk_operation_mock.disconnect.called) 77 | 78 | def test_post_save_turned_on_by_default(self): 79 | """ 80 | Tests that save signals are connected by default. 81 | """ 82 | with patch('entity.signal_handlers.sync_entities') as mock_handler: 83 | Account.objects.create() 84 | self.assertTrue(mock_handler.called) 85 | 86 | def test_post_delete_turned_on_by_default(self): 87 | """ 88 | Tests that delete signals are connected by default. 89 | """ 90 | a = Account.objects.create() 91 | with patch('entity.models.Entity.all_objects.delete_for_obj') as mock_handler: 92 | # Delete the object. The signal should be called 93 | a.delete() 94 | self.assertEqual(mock_handler.call_count, 1) 95 | 96 | def test_bulk_operation_turned_off_by_default(self): 97 | """ 98 | Tests that bulk operations are turned off by default. 99 | """ 100 | with patch('entity.signal_handlers.sync_entities') as mock_handler: 101 | Account.objects.bulk_create([Account() for i in range(5)]) 102 | self.assertFalse(mock_handler.called) 103 | 104 | def test_turn_off_save(self): 105 | """ 106 | Tests turning off syncing for the save signal. 107 | """ 108 | turn_off_syncing() 109 | with patch('entity.signal_handlers.sync_entities') as mock_handler: 110 | Account.objects.create() 111 | self.assertFalse(mock_handler.called) 112 | 113 | def test_turn_off_delete(self): 114 | """ 115 | Tests turning off syncing for the delete signal. 116 | """ 117 | turn_off_syncing() 118 | with patch('entity.signal_handlers.sync_entities') as mock_handler: 119 | a = Account.objects.create() 120 | self.assertFalse(mock_handler.called) 121 | a.delete() 122 | self.assertFalse(mock_handler.called) 123 | 124 | def test_turn_off_bulk(self): 125 | """ 126 | Tests turning off syncing for bulk operations. 127 | """ 128 | turn_off_syncing() 129 | with patch('entity.signal_handlers.sync_entities') as mock_handler: 130 | Account.objects.bulk_create([Account() for i in range(5)]) 131 | self.assertFalse(mock_handler.called) 132 | 133 | def test_turn_on_save(self): 134 | """ 135 | Tests turning on syncing for the save signal. 136 | """ 137 | turn_off_syncing() 138 | turn_on_syncing() 139 | with patch('entity.signal_handlers.sync_entities') as mock_handler: 140 | Account.objects.create() 141 | self.assertTrue(mock_handler.called) 142 | 143 | def test_turn_on_delete(self): 144 | """ 145 | Tests turning on syncing for the delete signal. 146 | """ 147 | turn_off_syncing() 148 | turn_on_syncing() 149 | with patch('entity.models.Entity.all_objects.delete_for_obj') as mock_handler: 150 | a = Account.objects.create() 151 | a.delete() 152 | self.assertEqual(mock_handler.call_count, 1) 153 | 154 | def test_turn_on_bulk(self): 155 | """ 156 | Tests turning on syncing for bulk operations. 157 | """ 158 | turn_off_syncing() 159 | turn_on_syncing(for_post_bulk_operation=True) 160 | with patch('entity.signal_handlers.sync_entities') as mock_handler: 161 | Account.objects.bulk_create([Account() for i in range(5)]) 162 | self.assertTrue(mock_handler.called) 163 | 164 | 165 | class SyncAllEntitiesTest(EntityTestCase): 166 | """ 167 | Tests that all entities can be synced at once and tests the management command to 168 | sync all entities. 169 | """ 170 | def test_sync_entities_management_command(self): 171 | """ 172 | Tests that the management command for syncing entities works properly. 173 | """ 174 | # Create five test accounts 175 | turn_off_syncing() 176 | for i in range(5): 177 | Account.objects.create() 178 | turn_on_syncing() 179 | 180 | # Test that the management command syncs all five entities 181 | self.assertEqual(Entity.objects.all().count(), 0) 182 | call_command('sync_entities') 183 | self.assertEqual(Entity.objects.all().count(), 5) 184 | 185 | def test_sync_dummy_data(self): 186 | """ 187 | Tests that dummy data (i.e data that does not inherit EntityModelMixin) doesn't 188 | get synced. 189 | """ 190 | # Create dummy data 191 | DummyModel.objects.create() 192 | # Sync all entities and verify that none were created 193 | sync_entities() 194 | self.assertEqual(Entity.objects.all().count(), 0) 195 | 196 | def test_sync_multi_inherited_data(self): 197 | """ 198 | Test when models are synced that don't directly inherit EntityModelMixin. 199 | """ 200 | # Create an entity that does not directly inherit EntityModelMixin 201 | MultiInheritEntity.objects.create() 202 | # Sync all entities and verify that one was created 203 | sync_entities() 204 | self.assertEqual(Entity.objects.all().count(), 1) 205 | 206 | def test_sync_all_account_no_teams(self): 207 | """ 208 | Tests syncing all accounts with no super entities. 209 | """ 210 | turn_off_syncing() 211 | # Create five test accounts 212 | accounts = [Account.objects.create() for i in range(5)] 213 | turn_on_syncing() 214 | 215 | # Sync all of the entities and verify that five Entity models were created for the Account model 216 | self.assertEqual(Entity.objects.all().count(), 0) 217 | sync_entities() 218 | self.assertEqual(Entity.objects.all().count(), 5) 219 | 220 | # Delete an account. When all entities are synced again, 221 | # there should only be four accounts 222 | turn_off_syncing() 223 | accounts[0].delete() 224 | turn_on_syncing() 225 | 226 | self.assertEqual(Entity.objects.all().count(), 5) 227 | sync_entities() 228 | self.assertEqual(Entity.objects.all().count(), 4) 229 | 230 | def test_sync_all_accounts_teams(self): 231 | """ 232 | Tests syncing of all accounts when they have super entities. 233 | """ 234 | # Create five test accounts 235 | accounts = [Account.objects.create() for i in range(5)] 236 | # Create two teams to assign to some of the accounts 237 | teams = [Team.objects.create() for i in range(2)] 238 | accounts[0].team = teams[0] 239 | accounts[0].save() 240 | accounts[1].team = teams[0] 241 | accounts[1].save() 242 | accounts[2].team = teams[1] 243 | accounts[2].save() 244 | accounts[3].team = teams[1] 245 | accounts[3].save() 246 | 247 | # Sync all the entities. There should be 7 (5 accounts 2 teams) 248 | sync_entities() 249 | self.assertEqual(Entity.objects.filter(entity_type=ContentType.objects.get_for_model(Account)).count(), 5) 250 | self.assertEqual(Entity.objects.filter(entity_type=ContentType.objects.get_for_model(Team)).count(), 2) 251 | self.assertEqual(Entity.objects.all().count(), 7) 252 | 253 | # There should be four entity relationships since four accounts have teams 254 | self.assertEqual(EntityRelationship.objects.all().count(), 4) 255 | 256 | def test_sync_all_accounts_teams_new_account_during_sync(self): 257 | """ 258 | Tests the scenario of a new account being created after account ids are fetched but before the super 259 | entities are fetched 260 | """ 261 | # Create five test accounts 262 | accounts = [Account.objects.create() for i in range(5)] 263 | # Create two teams to assign to some of the accounts 264 | teams = [Team.objects.create() for i in range(2)] 265 | accounts[0].team = teams[0] 266 | accounts[0].save() 267 | accounts[1].team = teams[0] 268 | accounts[1].save() 269 | accounts[2].team = teams[1] 270 | accounts[2].save() 271 | accounts[3].team = teams[1] 272 | accounts[3].save() 273 | 274 | def wrapped_super_entities(*args, **kwargs): 275 | if not Account.objects.filter(email='fake@fake.com').exists(): 276 | Account.objects.create( 277 | email='fake@fake.com', 278 | team=Team.objects.order_by('id')[0], 279 | team2=Team.objects.order_by('id')[1], 280 | ) 281 | 282 | return _get_super_entities_by_ctype(*args, **kwargs) 283 | 284 | # Sync all the entities. There should be 8 (6 accounts 2 teams) 285 | with patch('entity.sync._get_super_entities_by_ctype', wraps=wrapped_super_entities): 286 | sync_entities() 287 | 288 | self.assertEqual(Entity.objects.filter(entity_type=ContentType.objects.get_for_model(Account)).count(), 6) 289 | self.assertEqual(Entity.objects.filter(entity_type=ContentType.objects.get_for_model(Team)).count(), 2) 290 | self.assertEqual(Entity.objects.all().count(), 8) 291 | 292 | # There should be six entity relationships 293 | self.assertEqual(EntityRelationship.objects.all().count(), 6) 294 | 295 | def test_sync_all_accounts_teams_deleted_account_during_sync(self): 296 | """ 297 | Tests the scenario of an account being deleted after account ids are fetched but before the super 298 | entities are fetched 299 | """ 300 | # Create five test accounts 301 | accounts = [Account.objects.create() for i in range(5)] 302 | # Create two teams to assign to some of the accounts 303 | teams = [Team.objects.create() for i in range(2)] 304 | accounts[0].team = teams[0] 305 | accounts[0].email = 'fake@fake.com' 306 | accounts[0].save() 307 | accounts[1].team = teams[0] 308 | accounts[1].save() 309 | accounts[2].team = teams[1] 310 | accounts[2].save() 311 | accounts[3].team = teams[1] 312 | accounts[3].save() 313 | 314 | def wrapped_super_entities(*args, **kwargs): 315 | account = Account.objects.filter(email='fake@fake.com', is_active=True).first() 316 | if account: 317 | account.is_active = False 318 | account.save() 319 | 320 | return _get_super_entities_by_ctype(*args, **kwargs) 321 | 322 | # Sync the accounts 323 | with patch('entity.sync._get_super_entities_by_ctype', wraps=wrapped_super_entities): 324 | sync_entities(*accounts) 325 | account = Account.objects.get(email='fake@fake.com') 326 | entity = Entity.all_objects.get_for_obj(account) 327 | self.assertEqual(entity.is_active, False) 328 | 329 | # Fetch accounts and sync again - hits other block in wrapped function 330 | with patch('entity.sync._get_super_entities_by_ctype', wraps=wrapped_super_entities): 331 | accounts = Account.objects.all() 332 | sync_entities(*accounts) 333 | account = Account.objects.get(email='fake@fake.com') 334 | entity = Entity.all_objects.get_for_obj(account) 335 | self.assertEqual(entity.is_active, False) 336 | 337 | self.assertEqual(Entity.objects.filter(entity_type=ContentType.objects.get_for_model(Account)).count(), 4) 338 | self.assertEqual(Entity.objects.filter(entity_type=ContentType.objects.get_for_model(Team)).count(), 2) 339 | self.assertEqual(Entity.objects.all().count(), 6) 340 | 341 | # There should be six entity relationships 342 | self.assertEqual(EntityRelationship.objects.all().count(), 4) 343 | 344 | def test_sync_all_accounts_teams_inactive_entity_kind(self): 345 | """ 346 | Tests syncing of all accounts when they have super entities and the entity kind is inactive 347 | """ 348 | # Create five test accounts 349 | accounts = [Account.objects.create() for i in range(5)] 350 | # Create two teams to assign to some of the accounts 351 | teams = [Team.objects.create() for i in range(2)] 352 | accounts[0].team = teams[0] 353 | accounts[0].save() 354 | accounts[1].team = teams[0] 355 | accounts[1].save() 356 | accounts[2].team = teams[1] 357 | accounts[2].save() 358 | accounts[3].team = teams[1] 359 | accounts[3].save() 360 | 361 | team_ek = EntityKind.objects.get(name='tests.team') 362 | team_ek.delete() 363 | 364 | # Sync all the entities. There should be 7 (5 accounts 2 teams) 365 | sync_entities() 366 | self.assertEqual(Entity.objects.filter(entity_type=ContentType.objects.get_for_model(Account)).count(), 5) 367 | self.assertEqual(Entity.objects.filter(entity_type=ContentType.objects.get_for_model(Team)).count(), 2) 368 | self.assertEqual(Entity.objects.all().count(), 7) 369 | 370 | # There should be four entity relationships since four accounts have teams 371 | self.assertEqual(EntityRelationship.objects.all().count(), 4) 372 | 373 | 374 | class SyncSignalTests(EntityTestCase): 375 | """ 376 | Test that when we are syncing we are calling the proper signals 377 | """ 378 | 379 | @patch('entity.sync.model_activations_changed') 380 | def test_sync_all(self, mock_model_activations_changed): 381 | """ 382 | Tests that when we sync all we fire the correct signals 383 | """ 384 | 385 | # Create five test accounts 386 | turn_off_syncing() 387 | initial_accounts = [] 388 | for i in range(5): 389 | initial_accounts.append(Account.objects.create()) 390 | turn_on_syncing() 391 | 392 | # Test that the management command syncs all five entities 393 | self.assertEqual(Entity.objects.all().count(), 0) 394 | sync_entities() 395 | self.assertEqual(Entity.objects.all().count(), 5) 396 | initial_entity_ids = list(Entity.objects.all().values_list('id', flat=True)) 397 | mock_model_activations_changed.send.assert_called_once_with( 398 | sender=Entity, 399 | instance_ids=sorted(initial_entity_ids), 400 | is_active=True 401 | ) 402 | 403 | # Create five new test accounts, and deactivate our initial accounts 404 | mock_model_activations_changed.reset_mock() 405 | turn_off_syncing() 406 | new_accounts = [] 407 | for i in range(5): 408 | new_accounts.append(Account.objects.create()) 409 | for account in initial_accounts: 410 | account.delete() 411 | turn_on_syncing() 412 | 413 | # Sync entities 414 | sync_entities() 415 | 416 | # Assert that the correct signals were called 417 | self.assertEqual( 418 | mock_model_activations_changed.send.mock_calls, 419 | [ 420 | call( 421 | sender=Entity, 422 | instance_ids=sorted(list(Entity.objects.filter( 423 | entity_id__in=[account.id for account in new_accounts] 424 | ).values_list('id', flat=True))), 425 | is_active=True 426 | ), 427 | call( 428 | sender=Entity, 429 | instance_ids=sorted(initial_entity_ids), 430 | is_active=False 431 | ) 432 | ] 433 | ) 434 | 435 | # Test syncing all when nothing should have changed 436 | mock_model_activations_changed.reset_mock() 437 | sync_entities() 438 | self.assertFalse(mock_model_activations_changed.send.called) 439 | 440 | 441 | class TestEntityBulkSignalSync(EntityTestCase): 442 | """ 443 | Tests syncing when bulk operations happen. 444 | """ 445 | def setUp(self): 446 | super(TestEntityBulkSignalSync, self).setUp() 447 | turn_on_syncing(for_post_bulk_operation=True) 448 | 449 | def test_post_bulk_create(self): 450 | """ 451 | Tests that entities can have bulk creates applied to them and still be synced. 452 | """ 453 | # Bulk create five accounts 454 | accounts = [Account() for i in range(5)] 455 | Account.objects.bulk_create(accounts) 456 | # Verify that there are 5 entities 457 | self.assertEqual(Entity.objects.all().count(), 5) 458 | 459 | def test_post_bulk_update(self): 460 | """ 461 | Calls a bulk update on a list of entities. Verifies that the models are appropriately 462 | synced. 463 | """ 464 | # Create five accounts 465 | for i in range(5): 466 | Account.objects.create(email='test1@test.com') 467 | # Verify that there are five entities all with the 'test1@test.com' email 468 | for entity in Entity.objects.all(): 469 | self.assertEqual(entity.entity_meta['email'], 'test1@test.com') 470 | self.assertEqual(Entity.objects.all().count(), 5) 471 | 472 | # Bulk update the account emails to a different one 473 | Account.objects.all().update(email='test2@test.com') 474 | 475 | # Verify that the email was updated properly in all entities 476 | for entity in Entity.objects.all(): 477 | self.assertEqual(entity.entity_meta['email'], 'test2@test.com') 478 | self.assertEqual(Entity.objects.all().count(), 5) 479 | 480 | def test_invalid_entity_model(self): 481 | """ 482 | Tests that an invalid entity model is not synced on bulk update. 483 | """ 484 | DummyModel.objects.bulk_create([DummyModel()]) 485 | self.assertFalse(Entity.objects.exists()) 486 | 487 | def test_post_bulk_update_dummy(self): 488 | """ 489 | Tests that even if the dummy model is using the special model manager for bulk 490 | updates, it still does not get synced since it doesn't inherit EntityModelMixin. 491 | """ 492 | # Create five dummy models with a bulk update 493 | DummyModel.objects.bulk_create([DummyModel() for i in range(5)]) 494 | # There should be no synced entities 495 | self.assertEqual(Entity.objects.all().count(), 0) 496 | 497 | 498 | class TestWatching(EntityTestCase): 499 | """ 500 | Tests when an entity is watching another model for changes. 501 | """ 502 | def test_m2m_changed_of_another_model(self): 503 | """ 504 | Tests when an entity model is listening for a change of an m2m of another model. 505 | """ 506 | m2m_entity = G(M2mEntity) 507 | team = G(Team) 508 | points_to_m2m_entity = G(PointsToM2mEntity, m2m_entity=m2m_entity) 509 | # Three entities should be synced and there should not yet be any relationships 510 | self.assertEqual(Entity.objects.count(), 3) 511 | self.assertFalse(EntityRelationship.objects.exists()) 512 | 513 | # When a team is added to the m2m entity, it should be a super entity to the points_to_m2m_entity and 514 | # of m2m_entity 515 | m2m_entity.teams.add(team) 516 | self.assertEqual(Entity.objects.count(), 3) 517 | self.assertEqual(EntityRelationship.objects.count(), 2) 518 | 519 | points_to_m2m_entity = Entity.objects.get_for_obj(points_to_m2m_entity) 520 | team_entity = Entity.objects.get_for_obj(team) 521 | m2m_entity = Entity.objects.get_for_obj(m2m_entity) 522 | self.assertTrue(EntityRelationship.objects.filter( 523 | sub_entity=points_to_m2m_entity, super_entity=team_entity).exists()) 524 | self.assertTrue(EntityRelationship.objects.filter(sub_entity=m2m_entity, super_entity=team_entity).exists()) 525 | 526 | def test_points_to_account_config_competitor_updated(self): 527 | """ 528 | Tests that a PointsToAccount model is updated when the competitor of its account is updated. 529 | """ 530 | account = G(Account) 531 | pta = G(PointsToAccount, account=account) 532 | pta_entity = Entity.objects.get_for_obj(pta) 533 | self.assertEqual(pta_entity.entity_meta, { 534 | 'team_name': 'None', 535 | 'competitor_name': 'None', 536 | }) 537 | 538 | team = G(Team, name='team1') 539 | competitor = G(Competitor, name='competitor1') 540 | account.team = team 541 | account.competitor = competitor 542 | account.save() 543 | 544 | # Nothing should have been updated on the entity. This is because it is watching the competitor 545 | # and team models for changes. Since these models were changed before they were linked to the 546 | # account, the changes are not propagated. 547 | pta_entity = Entity.objects.get_for_obj(pta) 548 | self.assertEqual(pta_entity.entity_meta, { 549 | 'team_name': 'None', 550 | 'competitor_name': 'None', 551 | }) 552 | 553 | # Now change names of the competitors and teams. Things will be propagated. 554 | team.name = 'team2' 555 | team.save() 556 | pta_entity = Entity.objects.get_for_obj(pta) 557 | self.assertEqual(pta_entity.entity_meta, { 558 | 'team_name': 'team2', 559 | 'competitor_name': 'competitor1', 560 | }) 561 | 562 | competitor.name = 'competitor2' 563 | competitor.save() 564 | pta_entity = Entity.objects.get_for_obj(pta) 565 | self.assertEqual(pta_entity.entity_meta, { 566 | 'team_name': 'team2', 567 | 'competitor_name': 'competitor2', 568 | }) 569 | 570 | # The power of django entity compels you... 571 | 572 | 573 | class TestEntityM2mChangedSignalSync(EntityTestCase): 574 | """ 575 | Tests when an m2m changes on a synced entity. 576 | """ 577 | def test_save_model_with_m2m(self): 578 | """ 579 | Verifies that the m2m test entity is synced properly upon save. 580 | """ 581 | turn_off_syncing() 582 | m = G(M2mEntity) 583 | m.teams.add(G(Team)) 584 | turn_on_syncing() 585 | 586 | m.save() 587 | self.assertEqual(Entity.objects.count(), 2) 588 | self.assertEqual(EntityRelationship.objects.count(), 1) 589 | 590 | def test_sync_when_m2m_add(self): 591 | """ 592 | Verifies an entity is synced properly when and m2m field is added. 593 | """ 594 | m = G(M2mEntity) 595 | self.assertEqual(Entity.objects.count(), 1) 596 | self.assertEqual(EntityRelationship.objects.count(), 0) 597 | m.teams.add(G(Team)) 598 | self.assertEqual(Entity.objects.count(), 2) 599 | self.assertEqual(EntityRelationship.objects.count(), 1) 600 | 601 | def test_sync_when_m2m_delete(self): 602 | """ 603 | Verifies an entity is synced properly when and m2m field is deleted. 604 | """ 605 | m = G(M2mEntity) 606 | team = G(Team) 607 | m.teams.add(team) 608 | self.assertEqual(Entity.objects.count(), 2) 609 | self.assertEqual(EntityRelationship.objects.count(), 1) 610 | m.teams.remove(team) 611 | self.assertEqual(Entity.objects.count(), 2) 612 | self.assertEqual(EntityRelationship.objects.count(), 0) 613 | 614 | def test_sync_when_m2m_clear(self): 615 | """ 616 | Verifies an entity is synced properly when and m2m field is cleared. 617 | """ 618 | m = G(M2mEntity) 619 | team = G(Team) 620 | m.teams.add(team) 621 | self.assertEqual(Entity.objects.count(), 2) 622 | self.assertEqual(EntityRelationship.objects.count(), 1) 623 | m.teams.clear() 624 | self.assertEqual(Entity.objects.count(), 2) 625 | self.assertEqual(EntityRelationship.objects.count(), 0) 626 | 627 | 628 | class TestEntityPostSavePostDeleteSignalSync(EntityTestCase): 629 | """ 630 | Tests that entities (from the test models) are properly synced upon post_save 631 | and post_delete signals. 632 | """ 633 | def test_going_from_inactive_to_active(self): 634 | """ 635 | Tests that an inactive entity can be activated and that its active attributes 636 | are synced properly. 637 | """ 638 | a = Account.objects.create(email='test_email', is_active=False) 639 | a.is_active = True 640 | a.save() 641 | e = Entity.all_objects.get_for_obj(a) 642 | self.assertTrue(e.is_active) 643 | 644 | def test_inactive_syncing(self): 645 | """ 646 | Tests that an inactive entity's activatable properties are synced properly. 647 | """ 648 | a = Account.objects.create(email='test_email', is_active=False) 649 | e = Entity.all_objects.get_for_obj(a) 650 | self.assertFalse(e.is_active) 651 | 652 | def test_display_name_mirrored_default(self): 653 | """ 654 | Tests that the display name is mirrored to the __unicode__ of the models. This 655 | is the default behavior. 656 | """ 657 | a = Account.objects.create(email='test_email') 658 | e = Entity.objects.get_for_obj(a) 659 | self.assertEqual(e.display_name, 'test_email') 660 | 661 | def test_display_name_mirrored_custom(self): 662 | """ 663 | Tests that the display name is mirrored properly when a custom get_display_name 664 | function is defined. In this case, the function for Teams returns 'team' 665 | """ 666 | t = G(Team) 667 | e = Entity.objects.get_for_obj(t) 668 | self.assertEqual(e.display_name, 'team') 669 | 670 | def test_post_save_dummy_data(self): 671 | """ 672 | Tests that dummy data that does not inherit from EntityModelMixin is not synced 673 | when saved. 674 | """ 675 | DummyModel.objects.create() 676 | # Verify that no entities were created 677 | self.assertEqual(Entity.objects.all().count(), 0) 678 | 679 | def test_post_save_multi_inherit_model(self): 680 | """ 681 | Tests that a model that does not directly inherit EntityModelMixin is still synced. 682 | """ 683 | MultiInheritEntity.objects.create() 684 | # Verify that one entity was synced 685 | self.assertEqual(Entity.objects.all().count(), 1) 686 | 687 | def test_post_delete_inactive_entity(self): 688 | """ 689 | Tests deleting an entity that was already inactive. 690 | """ 691 | account = Account.objects.create(is_active=False) 692 | account.delete() 693 | self.assertEqual(Entity.all_objects.all().count(), 0) 694 | 695 | def test_post_delete_no_entity(self): 696 | """ 697 | Tests a post_delete on an account that has no current mirrored entity. 698 | """ 699 | # Create an account 700 | account = Account.objects.create() 701 | # Clear out the Entity table since post_save will create an entry for it 702 | Entity.objects.all().delete() 703 | 704 | # Delete the created model. No errors should occur and nothing should 705 | # be in the entity table 706 | account.delete() 707 | self.assertEqual(Entity.objects.all().count(), 0) 708 | 709 | def test_post_delete_account(self): 710 | """ 711 | Tests a post_delete on an account that has a current mirrored entity. 712 | """ 713 | # Create accounts for the test 714 | main_account = Account.objects.create() 715 | other_account = Account.objects.create() 716 | # Clear out the Entity table since post_save will create an entry for it 717 | Entity.objects.all().delete(force=True) 718 | 719 | # Create entity entries for the account object and for another account 720 | self.create_entity(main_account) 721 | self.create_entity(other_account) 722 | 723 | # Delete the created model. No errors should occur and the other account 724 | # should still be an entity in the Entity table. 725 | main_account.delete() 726 | self.assertEqual(Entity.objects.all().count(), 1) 727 | self.assertEqual(Entity.objects.filter(entity_id=other_account.id).count(), 1) 728 | 729 | def test_post_delete_account_under_team(self): 730 | """ 731 | Tests the deletion of an account that had a relationship with a team. 732 | """ 733 | # Create a team 734 | team = Team.objects.create(name='Team') 735 | # Create an account under that team 736 | account = Account.objects.create(email='test@test.com', team=team) 737 | 738 | # There should be two entities and a relationship between them. 739 | self.assertEqual(Entity.objects.all().count(), 2) 740 | self.assertEqual(EntityRelationship.objects.all().count(), 1) 741 | 742 | # Delete the account. The team entity should still exist 743 | account.delete() 744 | self.assertEqual(Entity.objects.all().count(), 1) 745 | self.assertEqual(EntityRelationship.objects.all().count(), 0) 746 | Entity.objects.get_for_obj(team) 747 | 748 | def test_post_create_account_no_relationships_active(self): 749 | """ 750 | Tests that an Entity is created when the appropriate EntityModelMixin model is 751 | created. Tests the case where the entity has no relationships. 752 | """ 753 | # Verify that there are no entities 754 | self.assertEqual(Entity.objects.all().count(), 0) 755 | 756 | # Create an account. An entity with no relationships should be created 757 | account = Account.objects.create(email='test@test.com') 758 | entity = Entity.objects.get_for_obj(account) 759 | # Check that the metadata and is_active fields were set properly 760 | self.assertEqual(entity.entity_meta, { 761 | 'email': 'test@test.com', 762 | 'is_captain': False, 763 | 'team': None, 764 | 'team_is_active': None, 765 | }) 766 | self.assertEqual(entity.is_active, True) 767 | 768 | self.assertEqual(entity.sub_relationships.all().count(), 0) 769 | self.assertEqual(entity.super_relationships.all().count(), 0) 770 | 771 | def test_post_create_account_relationships(self): 772 | """ 773 | Creates an account that has super relationships. Verifies that the entity table is updated 774 | properly. 775 | """ 776 | # Verify that there are no entities 777 | self.assertEqual(Entity.objects.all().count(), 0) 778 | 779 | # Create a team 780 | team = Team.objects.create(name='Team') 781 | # Create an account under that team 782 | account = Account.objects.create(email='test@test.com', team=team) 783 | 784 | # There should be two entities. Test their existence and values 785 | self.assertEqual(Entity.objects.all().count(), 2) 786 | account_entity = Entity.objects.get_for_obj(account) 787 | self.assertEqual(account_entity.entity_meta, { 788 | 'email': 'test@test.com', 789 | 'is_captain': False, 790 | 'team': 'Team', 791 | 'team_is_active': True, 792 | }) 793 | team_entity = Entity.objects.get_for_obj(team) 794 | self.assertEqual(team_entity.entity_meta, None) 795 | 796 | # Check that the appropriate entity relationship was created 797 | self.assertEqual(EntityRelationship.objects.all().count(), 1) 798 | relationship = EntityRelationship.objects.first() 799 | self.assertEqual(relationship.sub_entity, account_entity) 800 | self.assertEqual(relationship.super_entity, team_entity) 801 | 802 | def test_post_updated_entity_no_cascade(self): 803 | """ 804 | Verify that updating a mirrored entity does not cause the entity to be deleted (which 805 | results in a cascading delete for all pointers. 806 | """ 807 | # Create a test account 808 | account = Account.objects.create(email='test@test.com') 809 | entity = Entity.objects.get_for_obj(account) 810 | self.assertEqual(entity.entity_meta, { 811 | 'email': 'test@test.com', 812 | 'is_captain': False, 813 | 'team': None, 814 | 'team_is_active': None, 815 | }) 816 | old_entity_id = entity.id 817 | 818 | # Create an object that points to the entity. This object is created to verify that it isn't cascade 819 | # deleted when the entity is updated 820 | test_pointer = EntityPointer.objects.create(entity=entity) 821 | 822 | # Now update the account 823 | account.email = 'newemail@test.com' 824 | account.save() 825 | # Verify that the mirrored entity has the same ID 826 | entity = Entity.objects.get_for_obj(account) 827 | self.assertEqual(entity.entity_meta, { 828 | 'email': 'newemail@test.com', 829 | 'is_captain': False, 830 | 'team': None, 831 | 'team_is_active': None, 832 | }) 833 | self.assertEqual(old_entity_id, entity.id) 834 | 835 | # Verify that the pointer still exists and wasn't cascade deleted 836 | test_pointer = EntityPointer.objects.get(id=test_pointer.id) 837 | self.assertEqual(test_pointer.entity, entity) 838 | 839 | def test_post_update_account_meta(self): 840 | """ 841 | Verifies that an account's metadata is updated properly in the mirrored tables. 842 | """ 843 | # Create an account and check it's mirrored metadata 844 | account = Account.objects.create(email='test@test.com') 845 | entity = Entity.objects.get_for_obj(account) 846 | self.assertEqual(entity.entity_meta, { 847 | 'email': 'test@test.com', 848 | 'is_captain': False, 849 | 'team': None, 850 | 'team_is_active': None, 851 | }) 852 | 853 | # Update the account's metadata and check that it is mirrored 854 | account.email = 'newemail@test.com' 855 | account.save() 856 | entity = Entity.objects.get_for_obj(account) 857 | self.assertEqual(entity.entity_meta, { 858 | 'email': 'newemail@test.com', 859 | 'is_captain': False, 860 | 'team': None, 861 | 'team_is_active': None, 862 | }) 863 | 864 | def test_post_update_account_relationship_activity(self): 865 | """ 866 | Creates an account that has super relationships. Verifies that the entity table is updated 867 | properly when changing the activity of a relationship. 868 | """ 869 | # Verify that there are no entities 870 | self.assertEqual(Entity.objects.all().count(), 0) 871 | 872 | # Create a team 873 | team = Team.objects.create(name='Team') 874 | # Create an account under that team 875 | account = Account.objects.create(email='test@test.com', team=team) 876 | 877 | # There should be two entities. Test their existence and values 878 | self.assertEqual(Entity.objects.all().count(), 2) 879 | account_entity = Entity.objects.get_for_obj(account) 880 | self.assertEqual(account_entity.entity_meta, { 881 | 'email': 'test@test.com', 882 | 'is_captain': False, 883 | 'team': 'Team', 884 | 'team_is_active': True, 885 | }) 886 | team_entity = Entity.objects.get_for_obj(team) 887 | self.assertEqual(team_entity.entity_meta, None) 888 | 889 | # Check that the appropriate entity relationship was created 890 | self.assertEqual(EntityRelationship.objects.all().count(), 1) 891 | relationship = EntityRelationship.objects.first() 892 | self.assertEqual(relationship.sub_entity, account_entity) 893 | self.assertEqual(relationship.super_entity, team_entity) 894 | 895 | # Update the account to be a team captain. According to our test project, this 896 | # means it no longer has an active relationship to a team 897 | account.is_captain = True 898 | account.save() 899 | 900 | # Verify that it no longer has an active relationship 901 | self.assertEqual(EntityRelationship.objects.all().count(), 1) 902 | relationship = EntityRelationship.objects.first() 903 | self.assertEqual(relationship.sub_entity, account_entity) 904 | self.assertEqual(relationship.super_entity, team_entity) 905 | 906 | 907 | class TestSyncingMultipleEntities(EntityTestCase): 908 | """ 909 | Tests syncing multiple entities at once of different types. 910 | """ 911 | def test_sync_two_accounts(self): 912 | turn_off_syncing() 913 | team = G(Team) 914 | account1 = G(Account, team=team) 915 | account2 = G(Account, team=team) 916 | G(TeamGroup) 917 | sync_entities(account1, account2) 918 | 919 | self.assertEqual(Entity.objects.count(), 3) 920 | self.assertEqual(EntityRelationship.objects.count(), 2) 921 | 922 | def test_sync_two_accounts_one_team_group(self): 923 | turn_off_syncing() 924 | team = G(Team) 925 | account1 = G(Account, team=team) 926 | account2 = G(Account, team=team) 927 | team_group = G(TeamGroup) 928 | sync_entities(account1, account2, team_group) 929 | 930 | self.assertEqual(Entity.objects.count(), 4) 931 | self.assertEqual(EntityRelationship.objects.count(), 2) 932 | 933 | 934 | class TestCachingAndCascading(EntityTestCase): 935 | """ 936 | Tests caching, cascade syncing, and optimal queries when syncing single, multiple, or all entities. 937 | """ 938 | def test_cascade_sync_super_entities(self): 939 | """ 940 | Tests that super entities will be synced when a sub entity is synced (even if the super entities 941 | werent synced before) 942 | """ 943 | turn_off_syncing() 944 | team = G(Team) 945 | turn_on_syncing() 946 | 947 | self.assertFalse(Entity.objects.exists()) 948 | G(Account, team=team) 949 | self.assertEqual(Entity.objects.count(), 2) 950 | self.assertEqual(EntityRelationship.objects.count(), 1) 951 | 952 | def test_optimal_queries_registered_entity_with_no_qset(self): 953 | """ 954 | Tests that the optimal number of queries are performed when syncing a single entity that 955 | did not register a queryset. 956 | """ 957 | team_group = G(TeamGroup) 958 | 959 | ContentType.objects.clear_cache() 960 | with self.assertNumQueries(15): 961 | team_group.save() 962 | 963 | def test_optimal_queries_registered_entity_w_qset(self): 964 | """ 965 | Tests that the entity is refetch with its queryset when syncing an individual entity. 966 | """ 967 | account = G(Account) 968 | 969 | ContentType.objects.clear_cache() 970 | with self.assertNumQueries(18): 971 | account.save() 972 | 973 | def test_sync_all_optimal_queries(self): 974 | """ 975 | Tests optimal queries of syncing all entities. 976 | """ 977 | # Create five test accounts 978 | accounts = [Account.objects.create() for i in range(5)] 979 | # Create two teams to assign to some of the accounts 980 | teams = [Team.objects.create() for i in range(2)] 981 | accounts[0].team = teams[0] 982 | accounts[0].save() 983 | accounts[1].team = teams[0] 984 | accounts[1].save() 985 | accounts[2].team = teams[1] 986 | accounts[2].save() 987 | accounts[3].team = teams[1] 988 | accounts[3].save() 989 | 990 | # Use an entity registry that only has accounts and teams. This ensures that other registered 991 | # entity models dont pollute the test case 992 | new_registry = EntityRegistry() 993 | new_registry.register_entity(AccountConfig) 994 | new_registry.register_entity(TeamConfig) 995 | 996 | with patch('entity.sync.entity_registry') as mock_entity_registry: 997 | mock_entity_registry.entity_registry = new_registry.entity_registry 998 | ContentType.objects.clear_cache() 999 | with self.assertNumQueries(20): 1000 | sync_entities() 1001 | 1002 | self.assertEqual(Entity.objects.filter(entity_type=ContentType.objects.get_for_model(Account)).count(), 5) 1003 | self.assertEqual(Entity.objects.filter(entity_type=ContentType.objects.get_for_model(Team)).count(), 2) 1004 | self.assertEqual(Entity.objects.all().count(), 7) 1005 | 1006 | # There should be four entity relationships since four accounts have teams 1007 | self.assertEqual(EntityRelationship.objects.all().count(), 4) 1008 | 1009 | 1010 | class DeferEntitySyncingTests(EntityTestCase): 1011 | """ 1012 | Tests the defer entity syncing decorator 1013 | """ 1014 | 1015 | def test_defer(self): 1016 | @defer_entity_syncing 1017 | def test_method(test, count=5, sync_all=False): 1018 | # Create some entities 1019 | for i in range(count): 1020 | Account.objects.create() 1021 | 1022 | if sync_all: 1023 | sync_entities() 1024 | 1025 | # Assert that we do not have any entities 1026 | test.assertEqual(Entity.objects.all().count(), 0) 1027 | 1028 | # Call the test method 1029 | test_method(self, count=5) 1030 | 1031 | # Assert that after the method was run we did sync the entities 1032 | self.assertEqual(Entity.objects.all().count(), 5) 1033 | 1034 | # Delete all entities 1035 | Entity.all_objects.all()._raw_delete(Entity.objects.db) 1036 | 1037 | # Call the method again syncing all 1038 | test_method(self, count=0, sync_all=True) 1039 | 1040 | # Assert that after the method was run we did sync the entities 1041 | self.assertEqual(Entity.objects.all().count(), 5) 1042 | 1043 | # Assert that we restored the defer flag 1044 | self.assertFalse(sync_entities.defer) 1045 | 1046 | # Assert that we cleared the buffer 1047 | self.assertEqual(sync_entities.buffer, {}) 1048 | 1049 | def test_defer_nothing_synced(self): 1050 | """ 1051 | Test the defer decorator when nothing is synced 1052 | """ 1053 | 1054 | @defer_entity_syncing 1055 | def test_method(test): 1056 | # Assert that we do not have any entities 1057 | test.assertEqual(Entity.objects.all().count(), 0) 1058 | 1059 | # Call the test method 1060 | with patch('entity.sync.sync_entities') as mock_sync_entities: 1061 | # Call the method that does no syncing 1062 | test_method(self) 1063 | 1064 | # Ensure that we did not call sync entities 1065 | self.assertFalse(mock_sync_entities.called) 1066 | 1067 | def test_defer_custom_handler(self): 1068 | # Create a mock handler 1069 | mock_handler = Mock() 1070 | 1071 | # Create a test sync method to be decorated 1072 | @defer_entity_syncing(handler=mock_handler) 1073 | def test_method(count=5): 1074 | # Create some entities 1075 | for i in range(count): 1076 | Account.objects.create() 1077 | 1078 | # Call the test method 1079 | test_method(count=5) 1080 | 1081 | # Assert that we called our custom handler 1082 | self.assertEqual(Entity.objects.all().count(), 0) 1083 | mock_handler.assert_called_once_with( 1084 | *Account.objects.all() 1085 | ) 1086 | 1087 | 1088 | class SuppressEntitySyncingTests(EntityTestCase): 1089 | """ 1090 | Tests the suppress entity syncing decorator 1091 | """ 1092 | 1093 | def test_defer(self): 1094 | @suppress_entity_syncing 1095 | def test_method(test, count): 1096 | # Create some entities 1097 | for i in range(count): 1098 | Account.objects.create() 1099 | 1100 | # Assert that we do not have any entities 1101 | test.assertEqual(Entity.objects.all().count(), 0) 1102 | 1103 | # Call the test method 1104 | test_method(self, count=5) 1105 | 1106 | # Assert that after the method was run we did sync the entities 1107 | self.assertEqual(Entity.objects.all().count(), 0) 1108 | 1109 | # Assert that we restored the suppress flag 1110 | self.assertFalse(sync_entities.suppress) 1111 | 1112 | 1113 | class TransactionAtomicWithRetryTests(EntityTestCase): 1114 | """ 1115 | Test the transaction_atomic_with_retry decorator 1116 | """ 1117 | 1118 | def test_retry_operational_error(self): 1119 | exception_mock = MagicMock() 1120 | exception_mock.side_effect = db.utils.OperationalError() 1121 | 1122 | @transaction_atomic_with_retry() 1123 | def test_func(): 1124 | exception_mock() 1125 | 1126 | with self.assertRaises(db.utils.OperationalError): 1127 | test_func() 1128 | 1129 | self.assertEqual( 1130 | len(exception_mock.mock_calls), 1131 | 6 1132 | ) 1133 | 1134 | def test_retry_other_error(self): 1135 | exception_mock = MagicMock() 1136 | exception_mock.side_effect = Exception() 1137 | 1138 | @transaction_atomic_with_retry() 1139 | def test_func(): 1140 | exception_mock() 1141 | 1142 | with self.assertRaises(Exception): 1143 | test_func() 1144 | exception_mock.assert_called_once_with() 1145 | -------------------------------------------------------------------------------- /entity/tests/utils.py: -------------------------------------------------------------------------------- 1 | from django.contrib.contenttypes.models import ContentType 2 | from django.test import TestCase 3 | from entity.signal_handlers import turn_on_syncing, turn_off_syncing 4 | from entity.models import Entity, EntityKind 5 | 6 | 7 | class EntityTestCase(TestCase): 8 | """ 9 | The base class for entity tests. Provides helpers for creating entities. 10 | """ 11 | def setUp(self): 12 | """ 13 | Ensure entity syncing is in default state when tests start. 14 | """ 15 | super(EntityTestCase, self).setUp() 16 | turn_off_syncing() 17 | turn_on_syncing() 18 | 19 | def tearDown(self): 20 | """ 21 | Make sure syncing is turned back to its original state. 22 | """ 23 | super(EntityTestCase, self).tearDown() 24 | turn_off_syncing() 25 | turn_on_syncing() 26 | 27 | def create_entity(self, model_obj): 28 | """ 29 | Given a model object, create an entity. 30 | """ 31 | entity_type = ContentType.objects.get_for_model(model_obj) 32 | return Entity.objects.create( 33 | entity_type=entity_type, entity_id=model_obj.id, 34 | entity_kind=EntityKind.objects.get_or_create( 35 | name='{0}__{1}'.format(entity_type.app_label, entity_type.model), 36 | defaults={'display_name': u'{0}'.format(entity_type)} 37 | )[0]) 38 | -------------------------------------------------------------------------------- /entity/urls.py: -------------------------------------------------------------------------------- 1 | urlpatterns = [] # pragma: no cover 2 | -------------------------------------------------------------------------------- /entity/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '6.2.3' 2 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | 4 | # These lines allow nose tests to work in Python 3.10 5 | import collections.abc 6 | collections.Callable = collections.abc.Callable 7 | 8 | # Show warnings about django deprecations - uncomment for version upgrade testing 9 | import warnings 10 | from django.utils.deprecation import RemovedInNextVersionWarning 11 | warnings.filterwarnings('always', category=DeprecationWarning) 12 | warnings.filterwarnings('always', category=PendingDeprecationWarning) 13 | warnings.filterwarnings('always', category=RemovedInNextVersionWarning) 14 | 15 | from settings import configure_settings 16 | 17 | if __name__ == '__main__': 18 | configure_settings() 19 | 20 | from django.core.management import execute_from_command_line 21 | 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /publish.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | subprocess.call(['rm', '-r', 'dist/']) 4 | subprocess.call(['python', '-m', 'pip', 'install', 'build', 'twine']) 5 | subprocess.call(['python', '-m', 'build']) 6 | subprocess.call(['twine', 'check', 'dist/*']) 7 | subprocess.call(['twine', 'upload', 'dist/*']) 8 | subprocess.call(['rm', '-r', 'dist/']) 9 | -------------------------------------------------------------------------------- /release_notes.md: -------------------------------------------------------------------------------- 1 | ## Release Notes 2 | 3 | - 6.2.3: 4 | - Update the `defer_entity_syncing` decorator to support an optional handler. 5 | - 6.2.2: 6 | - Fixed entity group logic string for logic sets containing more than 2 items being operated on 7 | - 6.2.1: 8 | - Rebumped because publish messed up 9 | - 6.2.0: 10 | - Add support for boolean logic strings to apply to entity group memberships 11 | - 6.1.1: 12 | - django support for 4.2 13 | - drop django 2.2 14 | - 6.0.0: 15 | - django support for 2.2-4.1 16 | - python support for 3.7-3.9 17 | - 5.1.1: 18 | - Fix logic holes related to the active flag when generating entity group cache 19 | - 5.1.0: 20 | - Add decorator to suppress entity syncing 21 | - 5.0.2: 22 | - Only fetch new entities on a sync_all 23 | - 5.0.1: 24 | - Fix bug where an entity created or deleted during a sync would cause errors 25 | - 5.0.0: 26 | - Django 3.0, Django 3.1 27 | - Drop Django 2.0, 2.1 support 28 | - 4.2.0: 29 | - Python 3.7, Django 2.2 30 | - 4.1.1: 31 | - When calling the activation events, ignore entities that are initially inactive 32 | - 4.1.0: 33 | - Add support for properly calling activation events when entities change 34 | - 4.0.9: 35 | - Bump manager utils. 36 | - Update upserts and syncs to be non native to save wasting auto incrementing ids 37 | - 4.0.8: 38 | - BAD RELEASE DO NOT USE 39 | - 4.0.7: 40 | - When using the defer entity syncing decorator, do not call sync if the buffer is empty 41 | - 4.0.6: 42 | - Relax entity update lock to be less aggressive by using `FOR NO KEY UPDATE` lock instead the `FOR UPDATE`. This will allow the updates to not block on other parts of the system that are foreign keyed to entity 43 | - 4.0.5: 44 | - Add retry logic, and select for update calls during entity syncing 45 | - 4.0.4: 46 | - Update logging to be debug instead of info 47 | - 4.0.3: 48 | - Add locking during sync method 49 | - Add support for deferring entity syncing using a decorator 50 | - 4.0.2: 51 | - Fix a bug where the is_active flag for entity kinds was being updated during syncing 52 | - 4.0.1: 53 | - Upgrade manager utils for faster syncs 54 | - 4.0.0: 55 | - Optimize entity syncing and change syncing interface 56 | - 3.1.2: 57 | - Fix hole in deletion where a model could be added during syncing 58 | - 3.1.1: 59 | - Remove 1.10 from setup file 60 | - 3.1.0: 61 | - Add jsonfield encoder (drop support for 1.10) 62 | - 3.0.0: 63 | - Add tox to support more versions 64 | - Switch to django's JSONField 65 | - 2.0.2: 66 | - Removed celery requirement 67 | - 2.0.1: 68 | - Return only active entities by default in membership cache method 69 | - 2.0.0: 70 | - Remove python 2.7 support 71 | - Remove python 3.4 support 72 | - Remove Django 1.9 support 73 | - Remove Django 1.10 support 74 | - Add Django 2.0 support 75 | - 1.18.1: 76 | - Reduce number of queries and data selected 77 | - 1.18.0: 78 | - Optimize entity group queries by providing cache building functions 79 | - 1.17.0: 80 | - Drop Django 1.8 support 81 | - Add Django 1.10 support 82 | - Add Django 1.11 support 83 | - Add python 3.6 support 84 | - 1.15.0: 85 | - Remove SyncEntitiesTask, this task should live within the main application that entities is installed within 86 | - 1.11.0: 87 | - Added support for arbitrary groups of entities. 88 | - 1.10.0: 89 | - Added Django 1.8 support. 90 | - 1.9.0: 91 | - Updated Entity Kinds to be activatable models. 92 | - 1.8.2: 93 | - Added sorting support for Entity Models in Python 3 94 | - 1.8.0: 95 | - Added support for Django 1.7 and also backwards-compatible support for Django 1.6. 96 | - 1.7.1: 97 | - Changed the ``is_entity_active`` function in the entity configuration to be named ``get_is_active`` for consistency with other functions. 98 | - 1.6.0: 99 | - Made entities ``activatable``, i.e. they inherit the properties defined in [Django Activatable Model](https://github.com/ambitioninc/django-activatable-model) 100 | - 1.5.0: 101 | - Added entity kinds to replace inadequacies of filtering by entity content types. 102 | - Removed is_any_type and is_not_any_type and replaced those methods with is_any_kind and is_not_any_kind in the model manager. 103 | - Removed chainable entity filters. All entity filtering calls are now in the model manager. 104 | -------------------------------------------------------------------------------- /requirements/requirements-testing.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | django-nose 3 | django-dynamic-fixture 4 | psycopg2 5 | flake8 6 | -------------------------------------------------------------------------------- /requirements/requirements.txt: -------------------------------------------------------------------------------- 1 | Django>=3.2 2 | django-activatable-model>=3.1.0 3 | django-manager-utils>=3.1.0 4 | python3-utils>=0.3 5 | wrapt>=1.10.5 6 | -------------------------------------------------------------------------------- /run_tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provides the ability to run test on a standalone Django app. 3 | """ 4 | import sys 5 | from optparse import OptionParser 6 | from settings import configure_settings 7 | 8 | # Configure the default settings and setup django 9 | configure_settings() 10 | 11 | # Django nose must be imported here since it depends on the settings being configured 12 | from django_nose import NoseTestSuiteRunner 13 | 14 | 15 | def run(*test_args, **kwargs): 16 | if not test_args: 17 | test_args = ['entity'] 18 | 19 | kwargs.setdefault('interactive', False) 20 | 21 | test_runner = NoseTestSuiteRunner(**kwargs) 22 | 23 | failures = test_runner.run_tests(test_args) 24 | sys.exit(failures) 25 | 26 | 27 | if __name__ == '__main__': 28 | parser = OptionParser() 29 | parser.add_option('--verbosity', dest='verbosity', action='store', default=1, type=int) 30 | (options, args) = parser.parse_args() 31 | 32 | run(*args, **options.__dict__) 33 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | 4 | from django.conf import settings 5 | 6 | 7 | def configure_settings(): 8 | """ 9 | Configures settings for manage.py and for run_tests.py. 10 | """ 11 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') 12 | 13 | if not settings.configured: 14 | # Determine the database settings depending on if a test_db var is set in CI mode or not 15 | test_db = os.environ.get('DB', None) 16 | if test_db is None: 17 | db_config = { 18 | 'ENGINE': 'django.db.backends.postgresql', 19 | 'NAME': 'entity', 20 | 'USER': 'postgres', 21 | 'PASSWORD': '', 22 | 'HOST': 'db', 23 | } 24 | elif test_db == 'postgres': 25 | db_config = { 26 | 'ENGINE': 'django.db.backends.postgresql', 27 | 'NAME': 'entity', 28 | 'USER': 'postgres', 29 | 'PASSWORD': '', 30 | 'HOST': 'db', 31 | } 32 | else: 33 | raise RuntimeError('Unsupported test DB {0}'.format(test_db)) 34 | 35 | # Check env for db override (used for github actions) 36 | if os.environ.get('DB_SETTINGS'): 37 | db_config = json.loads(os.environ.get('DB_SETTINGS')) 38 | 39 | installed_apps = [ 40 | 'django.contrib.auth', 41 | 'django.contrib.contenttypes', 42 | 'django.contrib.sessions', 43 | 'django.contrib.messages', 44 | 'django.contrib.admin', 45 | 'django.db.utils', 46 | 'entity', 47 | 'entity.tests', 48 | ] 49 | 50 | settings.configure( 51 | TEST_RUNNER='django_nose.NoseTestSuiteRunner', 52 | SECRET_KEY='*', 53 | NOSE_ARGS=['--nocapture', '--nologcapture', '--verbosity=1'], 54 | DATABASES={ 55 | 'default': db_config, 56 | }, 57 | MIDDLEWARE=( 58 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 59 | 'django.contrib.messages.middleware.MessageMiddleware', 60 | 'django.contrib.sessions.middleware.SessionMiddleware' 61 | ), 62 | TEMPLATES=[{ 63 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 64 | 'APP_DIRS': True, 65 | 'OPTIONS': { 66 | 'context_processors': [ 67 | 'django.contrib.auth.context_processors.auth', 68 | 'django.contrib.messages.context_processors.messages', 69 | 'django.template.context_processors.request', 70 | ] 71 | } 72 | }], 73 | INSTALLED_APPS=installed_apps, 74 | ROOT_URLCONF='entity.urls', 75 | DEBUG=False, 76 | DDF_FILL_NULLABLE_FIELDS=False, 77 | DEFAULT_AUTO_FIELD='django.db.models.AutoField', 78 | ) 79 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | exclude = build,docs,venv,env,*.egg,migrations,south_migrations 4 | max-complexity = 15 5 | ignore = E402,E722,W504 6 | 7 | [bdist_wheel] 8 | universal = 1 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import re 2 | from setuptools import setup, find_packages 3 | 4 | # import multiprocessing to avoid this bug (http://bugs.python.org/issue15881#msg170215_ 5 | import multiprocessing 6 | assert multiprocessing 7 | 8 | 9 | def get_version(): 10 | """ 11 | Extracts the version number from the version.py file. 12 | """ 13 | VERSION_FILE = 'entity/version.py' 14 | mo = re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', open(VERSION_FILE, 'rt').read(), re.M) 15 | if mo: 16 | return mo.group(1) 17 | else: 18 | raise RuntimeError('Unable to find version string in {0}.'.format(VERSION_FILE)) 19 | 20 | 21 | def get_lines(file_path): 22 | return open(file_path, 'r').read().split('\n') 23 | 24 | 25 | install_requires = get_lines('requirements/requirements.txt') 26 | tests_require = get_lines('requirements/requirements-testing.txt') 27 | 28 | 29 | setup( 30 | name='django-entity', 31 | version=get_version(), 32 | description='Entity relationship management for Django', 33 | long_description=open('README.md').read(), 34 | long_description_content_type='text/markdown', 35 | url='http://github.com/ambitioninc/django-entity/', 36 | author='Wes Kendall', 37 | author_email='wesleykendall@gmail.com', 38 | packages=find_packages(), 39 | classifiers=[ 40 | 'Programming Language :: Python', 41 | 'Programming Language :: Python :: 3.7', 42 | 'Programming Language :: Python :: 3.8', 43 | 'Programming Language :: Python :: 3.9', 44 | 'Intended Audience :: Developers', 45 | 'License :: OSI Approved :: MIT License', 46 | 'Operating System :: OS Independent', 47 | 'Framework :: Django', 48 | 'Framework :: Django :: 3.2', 49 | 'Framework :: Django :: 4.0', 50 | 'Framework :: Django :: 4.1', 51 | 'Framework :: Django :: 4.2', 52 | ], 53 | install_requires=install_requires, 54 | tests_require=tests_require, 55 | extras_require={'dev': tests_require}, 56 | test_suite='run_tests.run', 57 | include_package_data=True, 58 | zip_safe=False, 59 | ) 60 | --------------------------------------------------------------------------------