├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.rst ├── idmapper ├── __init__.py ├── base.py ├── manager.py ├── models.py └── tests.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | /build 3 | /dist 4 | /django_idmapper.egg-info -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009, David Cramer 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include setup.py README.rst LICENSE MANIFEST.in 2 | global-exclude *~ -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | This fork of django-idmapper fixes some bugs that prevented the idmapper from 2 | being used in many instances. In particular, the caching manager is now inherited 3 | by SharedMemoryManager subclasses, and it is used when Django uses an automatic 4 | manager (see http://docs.djangoproject.com/en/dev/topics/db/managers/#controlling-automatic-manager-types). This means access through foreign keys now uses 5 | identity mapping. 6 | 7 | Tested with Django version 1.2 alpha 1 SVN-12375. 8 | 9 | My modifications are usually accompanied by comments marked with "CL:". 10 | 11 | Django Identity Mapper 12 | ====================== 13 | 14 | A pluggable Django application which allows you to explicitally mark your models to use an identity mapping pattern. This will share instances of the same model in memory throughout your interpreter. 15 | 16 | Please note, that deserialization (such as from the cache) will *not* use the identity mapper. 17 | 18 | Usage 19 | ----- 20 | To use the shared memory model you simply need to inherit from it (instead of models.Model). This enable all queries (and relational queries) to this model to use the shared memory instance cache, effectively creating a single instance for each unique row (based on primary key) in the queryset. 21 | 22 | For example, if you want to simply mark all of your models as a SharedMemoryModel, you might as well just import it as models. 23 | :: 24 | 25 | from idmapper import models 26 | 27 | class MyModel(models.SharedMemoryModel): 28 | name = models.CharField(...) 29 | 30 | Because the system is isolated, you may mix and match SharedMemoryModels with regular Models. The module idmapper.models imports everything from django.db.models and only adds SharedMemoryModel, so you can simply replace your import of models from django.db. 31 | :: 32 | 33 | from idmapper import models 34 | 35 | class MyModel(models.SharedMemoryModel): 36 | name = models.CharField(...) 37 | fkey = models.ForeignKey('Other') 38 | 39 | class Other(models.Model): 40 | name = models.CharField(...) 41 | 42 | References 43 | ---------- 44 | 45 | Original code and concept: http://code.djangoproject.com/ticket/17 -------------------------------------------------------------------------------- /idmapper/__init__.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import warnings 3 | 4 | __version__ = (0, 2) 5 | 6 | def _get_git_revision(path): 7 | revision_file = os.path.join(path, 'refs', 'heads', 'master') 8 | if not os.path.exists(revision_file): 9 | return None 10 | fh = open(revision_file, 'r') 11 | try: 12 | return fh.read() 13 | finally: 14 | fh.close() 15 | 16 | def get_revision(): 17 | """ 18 | :returns: Revision number of this branch/checkout, if available. None if 19 | no revision number can be determined. 20 | """ 21 | package_dir = os.path.dirname(__file__) 22 | checkout_dir = os.path.normpath(os.path.join(package_dir, '..')) 23 | path = os.path.join(checkout_dir, '.git') 24 | if os.path.exists(path): 25 | return _get_git_revision(path) 26 | return None 27 | 28 | __build__ = get_revision() 29 | 30 | def lazy_object(location): 31 | def inner(*args, **kwargs): 32 | parts = location.rsplit('.', 1) 33 | warnings.warn('`idmapper.%s` is deprecated. Please use `%s` instead.' % (parts[1], location), DeprecationWarning) 34 | imp = __import__(parts[0], globals(), locals(), [parts[1]], -1) 35 | func = getattr(imp, parts[1]) 36 | if callable(func): 37 | return func(*args, **kwargs) 38 | return func 39 | return inner 40 | 41 | SharedMemoryModel = lazy_object('idmapper.models.SharedMemoryModel') -------------------------------------------------------------------------------- /idmapper/base.py: -------------------------------------------------------------------------------- 1 | from weakref import WeakValueDictionary 2 | 3 | from django.core.signals import request_finished 4 | from django.db.models.base import Model, ModelBase 5 | from django.db.models.signals import post_save, pre_delete, \ 6 | post_syncdb 7 | 8 | from manager import SharedMemoryManager 9 | 10 | 11 | class SharedMemoryModelBase(ModelBase): 12 | # CL: upstream had a __new__ method that skipped ModelBase's __new__ if 13 | # SharedMemoryModelBase was not in the model class's ancestors. It's not 14 | # clear what was the intended purpose, but skipping ModelBase.__new__ 15 | # broke things; in particular, default manager inheritance. 16 | 17 | def __call__(cls, *args, **kwargs): 18 | """ 19 | this method will either create an instance (by calling the default implementation) 20 | or try to retrieve one from the class-wide cache by infering the pk value from 21 | args and kwargs. If instance caching is enabled for this class, the cache is 22 | populated whenever possible (ie when it is possible to infer the pk value). 23 | """ 24 | def new_instance(): 25 | return super(SharedMemoryModelBase, cls).__call__(*args, **kwargs) 26 | 27 | instance_key = cls._get_cache_key(args, kwargs) 28 | # depending on the arguments, we might not be able to infer the PK, so in that case we create a new instance 29 | if instance_key is None: 30 | return new_instance() 31 | 32 | # cached_instance = cls.get_cached_instance(instance_key) 33 | # if cached_instance is None: 34 | cached_instance = new_instance() 35 | cls.cache_instance(cached_instance) 36 | 37 | return cached_instance 38 | 39 | def _prepare(cls): 40 | cls.__instance_cache__ = WeakValueDictionary() 41 | super(SharedMemoryModelBase, cls)._prepare() 42 | 43 | 44 | class SharedMemoryModel(Model): 45 | # CL: setting abstract correctly to allow subclasses to inherit the default 46 | # manager. 47 | __metaclass__ = SharedMemoryModelBase 48 | 49 | objects = SharedMemoryManager() 50 | 51 | class Meta: 52 | abstract = True 53 | 54 | def _get_cache_key(cls, args, kwargs): 55 | """ 56 | This method is used by the caching subsystem to infer the PK value from the constructor arguments. 57 | It is used to decide if an instance has to be built or is already in the cache. 58 | """ 59 | result = None 60 | # Quick hack for my composites work for now. 61 | if hasattr(cls._meta, 'pks'): 62 | pk = cls._meta.pks[0] 63 | else: 64 | pk = cls._meta.pk 65 | # get the index of the pk in the class fields. this should be calculated *once*, but isn't atm 66 | pk_position = cls._meta.fields.index(pk) 67 | if len(args) > pk_position: 68 | # if it's in the args, we can get it easily by index 69 | result = args[pk_position] 70 | elif pk.attname in kwargs: 71 | # retrieve the pk value. Note that we use attname instead of name, to handle the case where the pk is a 72 | # a ForeignKey. 73 | result = kwargs[pk.attname] 74 | elif pk.name != pk.attname and pk.name in kwargs: 75 | # ok we couldn't find the value, but maybe it's a FK and we can find the corresponding object instead 76 | result = kwargs[pk.name] 77 | 78 | if result is not None and isinstance(result, Model): 79 | # if the pk value happens to be a model instance (which can happen wich a FK), we'd rather use its own pk as the key 80 | result = result._get_pk_val() 81 | return result 82 | _get_cache_key = classmethod(_get_cache_key) 83 | 84 | def get_cached_instance(cls, id): 85 | """ 86 | Method to retrieve a cached instance by pk value. Returns None when not found 87 | (which will always be the case when caching is disabled for this class). Please 88 | note that the lookup will be done even when instance caching is disabled. 89 | """ 90 | return cls.__instance_cache__.get(id) 91 | get_cached_instance = classmethod(get_cached_instance) 92 | 93 | def cache_instance(cls, instance): 94 | """ 95 | Method to store an instance in the cache. 96 | """ 97 | if instance._get_pk_val() is not None: 98 | cls.__instance_cache__[instance._get_pk_val()] = instance 99 | cache_instance = classmethod(cache_instance) 100 | 101 | def _flush_cached_by_key(cls, key): 102 | try: 103 | del cls.__instance_cache__[key] 104 | except KeyError: 105 | pass 106 | _flush_cached_by_key = classmethod(_flush_cached_by_key) 107 | 108 | def flush_cached_instance(cls, instance): 109 | """ 110 | Method to flush an instance from the cache. The instance will always be flushed from the cache, 111 | since this is most likely called from delete(), and we want to make sure we don't cache dead objects. 112 | """ 113 | cls._flush_cached_by_key(instance._get_pk_val()) 114 | flush_cached_instance = classmethod(flush_cached_instance) 115 | 116 | def flush_instance_cache(cls): 117 | cls.__instance_cache__ = WeakValueDictionary() 118 | flush_instance_cache = classmethod(flush_instance_cache) 119 | 120 | 121 | # Use a signal so we make sure to catch cascades. 122 | def flush_cache(**kwargs): 123 | for model in SharedMemoryModel.__subclasses__(): 124 | model.flush_instance_cache() 125 | request_finished.connect(flush_cache) 126 | post_syncdb.connect(flush_cache) 127 | 128 | 129 | def flush_cached_instance(sender, instance, **kwargs): 130 | # XXX: Is this the best way to make sure we can flush? 131 | if not hasattr(instance, 'flush_cached_instance'): 132 | return 133 | sender.flush_cached_instance(instance) 134 | pre_delete.connect(flush_cached_instance) 135 | 136 | 137 | def update_cached_instance(sender, instance, **kwargs): 138 | if not hasattr(instance, 'cache_instance'): 139 | return 140 | sender.cache_instance(instance) 141 | post_save.connect(update_cached_instance) 142 | -------------------------------------------------------------------------------- /idmapper/manager.py: -------------------------------------------------------------------------------- 1 | from django.db.models.manager import Manager 2 | try: 3 | from django.db import router 4 | except: 5 | pass 6 | 7 | 8 | class SharedMemoryManager(Manager): 9 | # CL: this ensures our manager is used when accessing instances via 10 | # ForeignKey etc. (see docs) 11 | use_for_related_fields = True 12 | 13 | # CL: in the dev version of django, ReverseSingleRelatedObjectDescriptor 14 | # will call us as: 15 | # rel_obj = rel_mgr.using(db).get(**params) 16 | # We need to handle using, or the get method will be called on a vanilla 17 | # queryset, and we won't get a change to use the cache. 18 | def using(self, alias): 19 | if alias == router.db_for_read(self.model): 20 | return self 21 | else: 22 | return super(SharedMemoryManager, self).using(alias) 23 | 24 | # TODO: improve on this implementation 25 | # We need a way to handle reverse lookups so that this model can 26 | # still use the singleton cache, but the active model isn't required 27 | # to be a SharedMemoryModel. 28 | def get(self, **kwargs): 29 | items = kwargs.keys() 30 | inst = None 31 | if len(items) == 1: 32 | # CL: support __exact 33 | key = items[0] 34 | if key.endswith('__exact'): 35 | key = key[:-len('__exact')] 36 | if key in ('pk', self.model._meta.pk.attname): 37 | inst = self.model.get_cached_instance(kwargs[items[0]]) 38 | if inst is None: 39 | inst = super(SharedMemoryManager, self).get(**kwargs) 40 | return inst 41 | -------------------------------------------------------------------------------- /idmapper/models.py: -------------------------------------------------------------------------------- 1 | from django.db.models import * 2 | from base import SharedMemoryModel -------------------------------------------------------------------------------- /idmapper/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from base import SharedMemoryModel 4 | from django.db import models 5 | 6 | class Category(SharedMemoryModel): 7 | name = models.CharField(max_length=32) 8 | 9 | class RegularCategory(models.Model): 10 | name = models.CharField(max_length=32) 11 | 12 | class Article(SharedMemoryModel): 13 | name = models.CharField(max_length=32) 14 | category = models.ForeignKey(Category) 15 | category2 = models.ForeignKey(RegularCategory) 16 | 17 | class RegularArticle(models.Model): 18 | name = models.CharField(max_length=32) 19 | category = models.ForeignKey(Category) 20 | category2 = models.ForeignKey(RegularCategory) 21 | 22 | class SharedMemorysTest(TestCase): 23 | # TODO: test for cross model relation (singleton to regular) 24 | 25 | def setUp(self): 26 | n = 0 27 | category = Category.objects.create(name="Category %d" % (n,)) 28 | regcategory = RegularCategory.objects.create(name="Category %d" % (n,)) 29 | 30 | for n in xrange(0, 10): 31 | Article.objects.create(name="Article %d" % (n,), category=category, category2=regcategory) 32 | RegularArticle.objects.create(name="Article %d" % (n,), category=category, category2=regcategory) 33 | 34 | def testSharedMemoryReferences(self): 35 | article_list = Article.objects.all().select_related('category') 36 | last_article = article_list[0] 37 | for article in article_list[1:]: 38 | self.assertEquals(article.category is last_article.category, True) 39 | last_article = article 40 | 41 | def testRegularReferences(self): 42 | article_list = RegularArticle.objects.all().select_related('category') 43 | last_article = article_list[0] 44 | for article in article_list[1:]: 45 | self.assertEquals(article.category2 is last_article.category2, False) 46 | last_article = article 47 | 48 | def testMixedReferences(self): 49 | article_list = RegularArticle.objects.all().select_related('category') 50 | last_article = article_list[0] 51 | for article in article_list[1:]: 52 | self.assertEquals(article.category is last_article.category, True) 53 | last_article = article 54 | 55 | article_list = Article.objects.all().select_related('category') 56 | last_article = article_list[0] 57 | for article in article_list[1:]: 58 | self.assertEquals(article.category2 is last_article.category2, False) 59 | last_article = article 60 | 61 | def testObjectDeletion(self): 62 | # This must execute first so its guaranteed to be in memory. 63 | article_list = list(Article.objects.all().select_related('category')) 64 | 65 | article = Article.objects.all()[0:1].get() 66 | pk = article.pk 67 | article.delete() 68 | self.assertEquals(pk not in Article.__instance_cache__, True) 69 | 70 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup, find_packages 4 | 5 | import idmapper 6 | 7 | setup( 8 | name='django-idmapper', 9 | version=".".join(map(str, idmapper.__version__)), 10 | author='David Cramer', 11 | author_email='dcramer@gmail.com', 12 | url='http://github.com/dcramer/django-idmapper', 13 | description = 'An identify mapper for the Django ORM', 14 | packages=find_packages(), 15 | include_package_data=True, 16 | classifiers=[ 17 | "Framework :: Django", 18 | "Intended Audience :: Developers", 19 | "Intended Audience :: System Administrators", 20 | "Operating System :: OS Independent", 21 | "Topic :: Software Development" 22 | ], 23 | ) 24 | --------------------------------------------------------------------------------