├── .arcconfig ├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── modeldict ├── __init__.py ├── base.py ├── models.py └── redis.py ├── runtests.py ├── setup.py └── tests ├── __init__.py └── modeldict ├── __init__.py ├── models.py ├── tests.py └── urls.py /.arcconfig: -------------------------------------------------------------------------------- 1 | { 2 | "project_id": "django-modeldict", 3 | "conduit_uri" : "http://phabricator.local.disqus.net/", 4 | "arcanist_configuration": "DisqusConfiguration", 5 | "copyright_holder": "Disqus, Inc.", 6 | "immutable_history": false, 7 | "differential.field-selector": "DisqusDifferentialFieldSelector", 8 | "phutil_libraries": { 9 | "disqus": "/usr/local/include/php/libdisqus/src" 10 | }, 11 | "lint_engine": "ComprehensiveLintEngine", 12 | "lint.pep8.options": "--ignore=W391,W292,W293,E501,E225", 13 | "lint.jshint.prefix": "node_modules/jshint/bin", 14 | "lint.jshint.bin": "hint" 15 | } 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .coverage 3 | .tox 4 | *.pyc 5 | *.log 6 | *.egg 7 | *.db 8 | *.pid 9 | pip-log.txt 10 | /cover 11 | /build 12 | /dist 13 | /*.egg-info 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.6" 4 | - "2.7" 5 | env: 6 | - DJANGO=1.2.7 7 | - DJANGO=1.3.1 8 | - DJANGO=1.4 9 | install: 10 | - pip install -q Django==$DJANGO --use-mirrors 11 | - pip install pep8 --use-mirrors 12 | - pip install https://github.com/dcramer/pyflakes/tarball/master 13 | - pip install -q -e . --use-mirrors 14 | before_script: 15 | - "pep8 --exclude=migrations --ignore=E501,E225 modeldict" 16 | - pyflakes -x W modeldict 17 | script: 18 | - python setup.py test 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2010 DISQUS 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include setup.py README.rst MANIFEST.in LICENSE 2 | global-exclude *~ -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: 2 | flake8 --exclude=migrations --ignore=E501,E225,E121,E123,E124,E125,E127,E128 --exit-zero modeldict || exit 1 3 | python setup.py test 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ------------------ 2 | NOTICE: Deprecated 3 | ------------------ 4 | 5 | This project is deprecated and no longer actively maintained by `Disqus `_. However there is a fork being maintained by YPlan at `github.com/YPlan/django-modeldict `_ and a similar project by Disqus at `github.com/disqus/durabledict `_. 6 | 7 | ---------------- 8 | django-modeldict 9 | ---------------- 10 | 11 | ModelDict is a very efficient way to store things like settings in your database. The entire model is transformed into a dictionary (lazily) as well as stored in your cache. It's invalidated only when it needs to be (both in process and based on ``CACHE_BACKEND``). 12 | 13 | Quick example usage. More docs to come (maybe?):: 14 | 15 | 16 | class Setting(models.Model): 17 | key = models.CharField(max_length=32) 18 | value = models.CharField(max_length=200) 19 | settings = ModelDict(Setting, key='key', value='value', instances=False) 20 | 21 | # access missing value 22 | settings['foo'] 23 | >>> KeyError 24 | 25 | # set the value 26 | settings['foo'] = 'hello' 27 | 28 | # fetch the current value using either method 29 | Setting.objects.get(key='foo').value 30 | >>> 'hello' 31 | 32 | settings['foo'] 33 | >>> 'hello' 34 | -------------------------------------------------------------------------------- /modeldict/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ('VERSION', 'ModelDict') 2 | 3 | try: 4 | VERSION = __import__('pkg_resources') \ 5 | .get_distribution('django-modeldict').version 6 | except Exception, e: 7 | VERSION = 'unknown' 8 | 9 | from modeldict.models import ModelDict 10 | -------------------------------------------------------------------------------- /modeldict/base.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.core.cache import cache 4 | 5 | NoValue = object() 6 | 7 | 8 | class CachedDict(object): 9 | def __init__(self, cache=cache, timeout=30): 10 | cls_name = type(self).__name__ 11 | 12 | self._local_cache = None 13 | self._local_last_updated = None 14 | 15 | self._last_checked_for_remote_changes = None 16 | self.timeout = timeout 17 | 18 | self.remote_cache = cache 19 | self.remote_cache_key = cls_name 20 | self.remote_cache_last_updated_key = '%s.last_updated' % (cls_name,) 21 | 22 | def __getitem__(self, key): 23 | self._populate() 24 | 25 | try: 26 | return self._local_cache[key] 27 | except KeyError: 28 | value = self.get_default(key) 29 | 30 | if value is NoValue: 31 | raise 32 | 33 | return value 34 | 35 | def __setitem__(self, key, value): 36 | raise NotImplementedError 37 | 38 | def __delitem__(self, key): 39 | raise NotImplementedError 40 | 41 | def __len__(self): 42 | if self._local_cache is None: 43 | self._populate() 44 | 45 | return len(self._local_cache) 46 | 47 | def __contains__(self, key): 48 | self._populate() 49 | return key in self._local_cache 50 | 51 | def __iter__(self): 52 | self._populate() 53 | return iter(self._local_cache) 54 | 55 | def __repr__(self): 56 | return "<%s: %s>" % (self.__class__.__name__, self.model.__name__) 57 | 58 | def iteritems(self): 59 | self._populate() 60 | return self._local_cache.iteritems() 61 | 62 | def itervalues(self): 63 | self._populate() 64 | return self._local_cache.itervalues() 65 | 66 | def iterkeys(self): 67 | self._populate() 68 | return self._local_cache.iterkeys() 69 | 70 | def keys(self): 71 | return list(self.iterkeys()) 72 | 73 | def values(self): 74 | return list(self.itervalues()) 75 | 76 | def items(self): 77 | self._populate() 78 | return self._local_cache.items() 79 | 80 | def get(self, key, default=None): 81 | self._populate() 82 | return self._local_cache.get(key, default) 83 | 84 | def pop(self, key, default=NoValue): 85 | value = self.get(key, default) 86 | 87 | try: 88 | del self[key] 89 | except KeyError: 90 | pass 91 | 92 | return value 93 | 94 | def setdefault(self, key, value): 95 | if key not in self: 96 | self[key] = value 97 | 98 | def get_default(self, key): 99 | return NoValue 100 | 101 | def local_cache_has_expired(self): 102 | """ 103 | Returns ``True`` if the in-memory cache has expired. 104 | """ 105 | if not self._last_checked_for_remote_changes: 106 | return True # Never checked before 107 | 108 | recheck_at = self._last_checked_for_remote_changes + self.timeout 109 | return time.time() > recheck_at 110 | 111 | def local_cache_is_invalid(self): 112 | """ 113 | Returns ``True`` if the local cache is invalid and needs to be 114 | refreshed with data from the remote cache. 115 | 116 | A return value of ``None`` signifies that no data was available. 117 | """ 118 | # If the local_cache is empty, avoid hitting memcache entirely 119 | if self._local_cache is None: 120 | return True 121 | 122 | remote_last_updated = self.remote_cache.get( 123 | self.remote_cache_last_updated_key 124 | ) 125 | 126 | if not remote_last_updated: 127 | # TODO: I don't like how we're overloading the return value here for 128 | # this method. It would be better to have a separate method or 129 | # @property that is the remote last_updated value. 130 | return None # Never been updated 131 | 132 | return int(remote_last_updated) > self._local_last_updated 133 | 134 | def get_cache_data(self): 135 | """ 136 | Pulls data from the cache backend. 137 | """ 138 | return self._get_cache_data() 139 | 140 | def clear_cache(self): 141 | """ 142 | Clears the in-process cache. 143 | """ 144 | self._local_cache = None 145 | self._local_last_updated = None 146 | self._last_checked_for_remote_changes = None 147 | 148 | def _populate(self, reset=False): 149 | """ 150 | Ensures the cache is populated and still valid. 151 | 152 | The cache is checked when: 153 | 154 | - The local timeout has been reached 155 | - The local cache is not set 156 | 157 | The cache is invalid when: 158 | 159 | - The global cache has expired (via remote_cache_last_updated_key) 160 | """ 161 | now = int(time.time()) 162 | 163 | # If asked to reset, then simply set local cache to None 164 | if reset: 165 | self._local_cache = None 166 | # Otherwise, if the local cache has expired, we need to go check with 167 | # our remote last_updated value to see if the dict values have changed. 168 | elif self.local_cache_has_expired(): 169 | 170 | local_cache_is_invalid = self.local_cache_is_invalid() 171 | 172 | # If local_cache_is_invalid is None, that means that there was no 173 | # data present, so we assume we need to add the key to cache. 174 | if local_cache_is_invalid is None: 175 | self.remote_cache.add(self.remote_cache_last_updated_key, now) 176 | 177 | # Now, if the remote has changed OR it was None in the first place, 178 | # pull in the values from the remote cache and set it to the 179 | # local_cache 180 | if local_cache_is_invalid or local_cache_is_invalid is None: 181 | self._local_cache = self.remote_cache.get(self.remote_cache_key) 182 | 183 | # No matter what, we've updated from remote, so mark ourselves as 184 | # such so that we won't expire until the next timeout 185 | self._local_last_updated = now 186 | 187 | # Update from cache if local_cache is still empty 188 | if self._local_cache is None: 189 | self._update_cache_data() 190 | 191 | # No matter what happened, we last checked for remote changes just now 192 | self._last_checked_for_remote_changes = now 193 | 194 | return self._local_cache 195 | 196 | def _update_cache_data(self): 197 | self._local_cache = self.get_cache_data() 198 | 199 | now = int(time.time()) 200 | self._local_last_updated = now 201 | self._last_checked_for_remote_changes = now 202 | 203 | # We only set remote_cache_last_updated_key when we know the cache is 204 | # current because setting this will force all clients to invalidate 205 | # their cached data if it's newer 206 | self.remote_cache.set(self.remote_cache_key, self._local_cache) 207 | self.remote_cache.set( 208 | self.remote_cache_last_updated_key, 209 | self._last_checked_for_remote_changes 210 | ) 211 | 212 | def _get_cache_data(self): 213 | raise NotImplementedError 214 | 215 | def _cleanup(self, *args, **kwargs): 216 | # We set _last_updated to a false value to ensure we hit the 217 | # last_updated cache on the next request 218 | self._last_checked_for_remote_changes = None 219 | -------------------------------------------------------------------------------- /modeldict/models.py: -------------------------------------------------------------------------------- 1 | from django.db.models.signals import post_save, post_delete 2 | from django.core.signals import request_finished 3 | 4 | from modeldict.base import CachedDict, NoValue 5 | 6 | 7 | try: 8 | from celery.signals import task_postrun 9 | except ImportError: # celery must not be installed 10 | has_celery = False 11 | else: 12 | has_celery = True 13 | 14 | 15 | class ModelDict(CachedDict): 16 | """ 17 | Dictionary-style access to a model. Populates a cache and a local in-memory 18 | store to avoid multiple hits to the database. 19 | 20 | Specifying ``instances=True`` will cause the cache to store instances rather 21 | than simple values. 22 | 23 | If ``auto_create=True`` accessing modeldict[key] when key does not exist will 24 | attempt to create it in the database. 25 | 26 | Functions in two different ways, depending on the constructor: 27 | 28 | # Given ``Model`` that has a column named ``foo`` where the value is "bar": 29 | 30 | mydict = ModelDict(Model, value='foo') 31 | mydict['test'] 32 | >>> 'bar' #doctest: +SKIP 33 | 34 | If you want to use another key besides ``pk``, you may specify that in the 35 | constructor. However, this will be used as part of the cache key, so it's recommended 36 | to access it in the same way throughout your code. 37 | 38 | mydict = ModelDict(Model, key='foo', value='id') 39 | mydict['bar'] 40 | >>> 'test' #doctest: +SKIP 41 | 42 | """ 43 | def __init__(self, model, key='pk', value=None, instances=False, auto_create=False, *args, **kwargs): 44 | assert value is not None 45 | 46 | super(ModelDict, self).__init__(*args, **kwargs) 47 | 48 | cls_name = type(self).__name__ 49 | model_name = model.__name__ 50 | 51 | self.key = key 52 | self.value = value 53 | 54 | self.model = model 55 | self.instances = instances 56 | self.auto_create = auto_create 57 | 58 | self.remote_cache_key = '%s:%s:%s' % (cls_name, model_name, self.key) 59 | self.remote_cache_last_updated_key = '%s.last_updated:%s:%s' % (cls_name, model_name, self.key) 60 | 61 | request_finished.connect(self._cleanup) 62 | post_save.connect(self._post_save, sender=model) 63 | post_delete.connect(self._post_delete, sender=model) 64 | 65 | if has_celery: 66 | task_postrun.connect(self._cleanup) 67 | 68 | def __setitem__(self, key, value): 69 | if isinstance(value, self.model): 70 | value = getattr(value, self.value) 71 | 72 | manager = self.model._default_manager 73 | instance, created = manager.get_or_create( 74 | defaults={self.value: value}, 75 | **{self.key: key} 76 | ) 77 | 78 | # Ensure we're updating the value in the database if it changes 79 | if getattr(instance, self.value) != value: 80 | setattr(instance, self.value, value) 81 | manager.filter(**{self.key: key}).update(**{self.value: value}) 82 | self._post_save(sender=self.model, instance=instance, created=False) 83 | 84 | def __delitem__(self, key): 85 | self.model._default_manager.filter(**{self.key: key}).delete() 86 | # self._populate(reset=True) 87 | 88 | def setdefault(self, key, value): 89 | if isinstance(value, self.model): 90 | value = getattr(value, self.value) 91 | 92 | instance, created = self.model._default_manager.get_or_create( 93 | defaults={self.value: value}, 94 | **{self.key: key} 95 | ) 96 | 97 | def get_default(self, key): 98 | if not self.auto_create: 99 | return NoValue 100 | result = self.model.objects.get_or_create(**{self.key: key})[0] 101 | if self.instances: 102 | return result 103 | return getattr(result, self.value) 104 | 105 | def _get_cache_data(self): 106 | qs = self.model._default_manager 107 | if self.instances: 108 | return dict((getattr(i, self.key), i) for i in qs.all()) 109 | return dict(qs.values_list(self.key, self.value)) 110 | 111 | # Signals 112 | 113 | def _post_save(self, sender, instance, created, **kwargs): 114 | self._populate(reset=True) 115 | 116 | def _post_delete(self, sender, instance, **kwargs): 117 | self._populate(reset=True) 118 | -------------------------------------------------------------------------------- /modeldict/redis.py: -------------------------------------------------------------------------------- 1 | from django.core.signals import request_finished 2 | 3 | from modeldict.base import CachedDict 4 | 5 | 6 | class RedisDict(CachedDict): 7 | """ 8 | Dictionary-style access to a redis hash table. Populates a cache and a local 9 | in-memory to avoid multiple hits to the database. 10 | 11 | Functions just like you'd expect it:: 12 | 13 | mydict = RedisDict('my_redis_key', Redis()) 14 | mydict['test'] 15 | >>> 'bar' #doctest: +SKIP 16 | 17 | """ 18 | def __init__(self, keyspace, connection, *args, **kwargs): 19 | super(CachedDict, self).__init__(*args, **kwargs) 20 | 21 | self.keyspace = keyspace 22 | self.conn = connection 23 | 24 | self.remote_cache_key = 'RedisDict:%s' % (keyspace,) 25 | self.remote_cache_last_updated_key = 'RedisDict.last_updated:%s' % (keyspace,) 26 | 27 | request_finished.connect(self._cleanup) 28 | 29 | def __setitem__(self, key, value): 30 | self.conn.hset(self.keyspace, key, value) 31 | if value != self._local_cache.get(key): 32 | self._local_cache[key] = value 33 | self._populate(reset=True) 34 | 35 | def __delitem__(self, key): 36 | self.conn.hdel(self.keyspace, key) 37 | self._local_cache.pop(key) 38 | self._populate(reset=True) 39 | 40 | def _get_cache_data(self): 41 | return self.conn.hgetall(self.keyspace) 42 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import absolute_import 3 | 4 | import sys 5 | from os.path import dirname, abspath 6 | 7 | sys.path.insert(0, dirname(abspath(__file__))) 8 | 9 | from django.conf import settings 10 | 11 | if not settings.configured: 12 | settings.configure( 13 | DATABASES={ 14 | 'default': { 15 | 'ENGINE': 'django.db.backends.sqlite3', 16 | } 17 | }, 18 | INSTALLED_APPS=[ 19 | 'django.contrib.contenttypes', 20 | 21 | 'modeldict', 22 | 'tests.modeldict', 23 | ], 24 | ROOT_URLCONF='', 25 | DEBUG=False, 26 | ) 27 | 28 | from django_nose import NoseTestSuiteRunner 29 | 30 | 31 | def runtests(*test_args, **kwargs): 32 | if 'south' in settings.INSTALLED_APPS: 33 | from south.management.commands import patch_for_test_db_setup 34 | patch_for_test_db_setup() 35 | 36 | if not test_args: 37 | test_args = ['tests'] 38 | 39 | kwargs.setdefault('interactive', False) 40 | 41 | test_runner = NoseTestSuiteRunner(**kwargs) 42 | 43 | failures = test_runner.run_tests(test_args) 44 | sys.exit(failures) 45 | 46 | if __name__ == '__main__': 47 | from optparse import OptionParser 48 | parser = OptionParser() 49 | parser.add_option('--verbosity', dest='verbosity', action='store', default=1, type=int) 50 | parser.add_options(NoseTestSuiteRunner.options) 51 | (options, args) = parser.parse_args() 52 | 53 | runtests(*args, **options.__dict__) 54 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup, find_packages 4 | 5 | try: 6 | import multiprocessing # NOQA 7 | except: 8 | pass 9 | 10 | tests_require = [ 11 | 'Django', 12 | 'celery', 13 | 'django-nose>=1.0', 14 | 'mock>=0.8.0', 15 | ] 16 | 17 | setup( 18 | name='django-modeldict', 19 | version='1.4.1', 20 | author='DISQUS', 21 | author_email='opensource@disqus.com', 22 | url='http://github.com/disqus/django-modeldict/', 23 | description='Stores a model as a dictionary', 24 | packages=find_packages(), 25 | zip_safe=False, 26 | tests_require=tests_require, 27 | test_suite='runtests.runtests', 28 | include_package_data=True, 29 | classifiers=[ 30 | 'Framework :: Django', 31 | 'Intended Audience :: Developers', 32 | 'Intended Audience :: System Administrators', 33 | 'Operating System :: OS Independent', 34 | 'Topic :: Software Development' 35 | ], 36 | ) 37 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | from tests import * -------------------------------------------------------------------------------- /tests/modeldict/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disqus/django-modeldict/00350ad19b125c60c29bec989757420db5b4926b/tests/modeldict/__init__.py -------------------------------------------------------------------------------- /tests/modeldict/models.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from django.db import models 4 | 5 | 6 | class ModelDictModel(models.Model): 7 | key = models.CharField(max_length=32, unique=True) 8 | value = models.CharField(max_length=32, default='') 9 | -------------------------------------------------------------------------------- /tests/modeldict/tests.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import mock 4 | import time 5 | 6 | from django.core.cache import cache 7 | from django.core.signals import request_finished 8 | from django.test import TestCase, TransactionTestCase 9 | 10 | from modeldict import ModelDict 11 | from modeldict.base import CachedDict 12 | from .models import ModelDictModel 13 | 14 | 15 | class ModelDictTest(TransactionTestCase): 16 | # XXX: uses transaction test due to request_finished signal causing a rollback 17 | urls = 'tests.modeldict.urls' 18 | 19 | def setUp(self): 20 | cache.clear() 21 | 22 | def assertHasReceiver(self, signal, function): 23 | for ident, reciever in signal.receivers: 24 | if reciever() is function: 25 | return True 26 | return False 27 | 28 | def test_api(self): 29 | base_count = ModelDictModel.objects.count() 30 | 31 | mydict = ModelDict(ModelDictModel, key='key', value='value') 32 | mydict['foo'] = 'bar' 33 | self.assertEquals(mydict['foo'], 'bar') 34 | self.assertEquals(ModelDictModel.objects.values_list('value', flat=True).get(key='foo'), 'bar') 35 | self.assertEquals(ModelDictModel.objects.count(), base_count + 1) 36 | mydict['foo'] = 'bar2' 37 | self.assertEquals(mydict['foo'], 'bar2') 38 | self.assertEquals(ModelDictModel.objects.values_list('value', flat=True).get(key='foo'), 'bar2') 39 | self.assertEquals(ModelDictModel.objects.count(), base_count + 1) 40 | mydict['foo2'] = 'bar' 41 | self.assertEquals(mydict['foo2'], 'bar') 42 | self.assertEquals(ModelDictModel.objects.values_list('value', flat=True).get(key='foo2'), 'bar') 43 | self.assertEquals(ModelDictModel.objects.count(), base_count + 2) 44 | del mydict['foo2'] 45 | self.assertRaises(KeyError, mydict.__getitem__, 'foo2') 46 | self.assertFalse(ModelDictModel.objects.filter(key='foo2').exists()) 47 | self.assertEquals(ModelDictModel.objects.count(), base_count + 1) 48 | 49 | ModelDictModel.objects.create(key='foo3', value='hello') 50 | 51 | self.assertEquals(mydict['foo3'], 'hello') 52 | self.assertTrue(ModelDictModel.objects.filter(key='foo3').exists(), True) 53 | self.assertEquals(ModelDictModel.objects.count(), base_count + 2) 54 | 55 | request_finished.send(sender=self) 56 | 57 | self.assertEquals(mydict._last_checked_for_remote_changes, None) 58 | 59 | # These should still error because even though the cache repopulates (local cache) 60 | # the remote cache pool does not 61 | # self.assertRaises(KeyError, mydict.__getitem__, 'foo3') 62 | # self.assertTrue(ModelDictModel.objects.filter(key='foo3').exists()) 63 | # self.assertEquals(ModelDictModel.objects.count(), base_count + 2) 64 | 65 | self.assertEquals(mydict['foo'], 'bar2') 66 | self.assertEquals(ModelDictModel.objects.values_list('value', flat=True).get(key='foo'), 'bar2') 67 | self.assertEquals(ModelDictModel.objects.count(), base_count + 2) 68 | 69 | self.assertEquals(mydict.pop('foo'), 'bar2') 70 | self.assertEquals(mydict.pop('foo', None), None) 71 | self.assertFalse(ModelDictModel.objects.filter(key='foo').exists()) 72 | self.assertEquals(ModelDictModel.objects.count(), base_count + 1) 73 | 74 | def test_modeldict_instances(self): 75 | base_count = ModelDictModel.objects.count() 76 | 77 | mydict = ModelDict(ModelDictModel, key='key', value='value', instances=True) 78 | mydict['foo'] = ModelDictModel(key='foo', value='bar') 79 | self.assertTrue(isinstance(mydict['foo'], ModelDictModel)) 80 | self.assertTrue(mydict['foo'].pk) 81 | self.assertEquals(mydict['foo'].value, 'bar') 82 | self.assertEquals(ModelDictModel.objects.values_list('value', flat=True).get(key='foo'), 'bar') 83 | self.assertEquals(ModelDictModel.objects.count(), base_count + 1) 84 | old_pk = mydict['foo'].pk 85 | mydict['foo'] = ModelDictModel(key='foo', value='bar2') 86 | self.assertTrue(isinstance(mydict['foo'], ModelDictModel)) 87 | self.assertEquals(mydict['foo'].pk, old_pk) 88 | self.assertEquals(mydict['foo'].value, 'bar2') 89 | self.assertEquals(ModelDictModel.objects.values_list('value', flat=True).get(key='foo'), 'bar2') 90 | self.assertEquals(ModelDictModel.objects.count(), base_count + 1) 91 | 92 | # test deletion 93 | mydict['foo'].delete() 94 | self.assertTrue('foo' not in mydict) 95 | 96 | def test_modeldict_expirey(self): 97 | base_count = ModelDictModel.objects.count() 98 | 99 | mydict = ModelDict(ModelDictModel, key='key', value='value') 100 | 101 | self.assertEquals(mydict._local_cache, None) 102 | 103 | mydict['test_modeldict_expirey'] = 'hello' 104 | 105 | self.assertEquals(len(mydict._local_cache), base_count + 1) 106 | self.assertEquals(mydict['test_modeldict_expirey'], 'hello') 107 | 108 | self.client.get('/') 109 | 110 | self.assertEquals(mydict._last_checked_for_remote_changes, None) 111 | self.assertEquals(mydict['test_modeldict_expirey'], 'hello') 112 | self.assertEquals(len(mydict._local_cache), base_count + 1) 113 | 114 | request_finished.send(sender=self) 115 | 116 | self.assertEquals(mydict._last_checked_for_remote_changes, None) 117 | self.assertEquals(mydict['test_modeldict_expirey'], 'hello') 118 | self.assertEquals(len(mydict._local_cache), base_count + 1) 119 | 120 | def test_modeldict_no_auto_create(self): 121 | # without auto_create 122 | mydict = ModelDict(ModelDictModel, key='key', value='value') 123 | self.assertRaises(KeyError, lambda x: x['hello'], mydict) 124 | self.assertEquals(ModelDictModel.objects.count(), 0) 125 | 126 | def test_modeldict_auto_create_no_value(self): 127 | # with auto_create and no value 128 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 129 | repr(mydict['hello']) 130 | self.assertEquals(ModelDictModel.objects.count(), 1) 131 | self.assertEquals(ModelDictModel.objects.get(key='hello').value, '') 132 | 133 | def test_modeldict_auto_create(self): 134 | # with auto_create and value 135 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 136 | mydict['hello'] = 'foo' 137 | self.assertEquals(ModelDictModel.objects.count(), 1) 138 | self.assertEquals(ModelDictModel.objects.get(key='hello').value, 'foo') 139 | 140 | def test_save_behavior(self): 141 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 142 | mydict['hello'] = 'foo' 143 | for n in xrange(10): 144 | mydict[str(n)] = 'foo' 145 | self.assertEquals(len(mydict), 11) 146 | self.assertEquals(ModelDictModel.objects.count(), 11) 147 | 148 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 149 | m = ModelDictModel.objects.get(key='hello') 150 | m.value = 'bar' 151 | m.save() 152 | 153 | self.assertEquals(ModelDictModel.objects.count(), 11) 154 | self.assertEquals(len(mydict), 11) 155 | self.assertEquals(mydict['hello'], 'bar') 156 | 157 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 158 | m = ModelDictModel.objects.get(key='hello') 159 | m.value = 'bar2' 160 | m.save() 161 | 162 | self.assertEquals(ModelDictModel.objects.count(), 11) 163 | self.assertEquals(len(mydict), 11) 164 | self.assertEquals(mydict['hello'], 'bar2') 165 | 166 | def test_django_signals_are_connected(self): 167 | from django.db.models.signals import post_save, post_delete 168 | from django.core.signals import request_finished 169 | 170 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 171 | self.assertHasReceiver(post_save, mydict._post_save) 172 | self.assertHasReceiver(post_delete, mydict._post_delete) 173 | self.assertHasReceiver(request_finished, mydict._cleanup) 174 | 175 | def test_celery_signals_are_connected(self): 176 | from celery.signals import task_postrun 177 | 178 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 179 | self.assertHasReceiver(task_postrun, mydict._cleanup) 180 | 181 | 182 | class CacheIntegrationTest(TestCase): 183 | def setUp(self): 184 | self.cache = mock.Mock() 185 | self.cache.get.return_value = {} 186 | self.mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True, cache=self.cache) 187 | 188 | def test_switch_creation(self): 189 | self.mydict['hello'] = 'foo' 190 | self.assertEquals(self.cache.get.call_count, 0) 191 | self.assertEquals(self.cache.set.call_count, 2) 192 | self.cache.set.assert_any_call(self.mydict.remote_cache_key, {u'hello': u'foo'}) 193 | self.cache.set.assert_any_call(self.mydict.remote_cache_last_updated_key, self.mydict._last_checked_for_remote_changes) 194 | 195 | def test_switch_change(self): 196 | self.mydict['hello'] = 'foo' 197 | self.cache.reset_mock() 198 | self.mydict['hello'] = 'bar' 199 | self.assertEquals(self.cache.get.call_count, 0) 200 | self.assertEquals(self.cache.set.call_count, 2) 201 | self.cache.set.assert_any_call(self.mydict.remote_cache_key, {u'hello': u'bar'}) 202 | self.cache.set.assert_any_call(self.mydict.remote_cache_last_updated_key, self.mydict._last_checked_for_remote_changes) 203 | 204 | def test_switch_delete(self): 205 | self.mydict['hello'] = 'foo' 206 | self.cache.reset_mock() 207 | del self.mydict['hello'] 208 | self.assertEquals(self.cache.get.call_count, 0) 209 | self.assertEquals(self.cache.set.call_count, 2) 210 | self.cache.set.assert_any_call(self.mydict.remote_cache_key, {}) 211 | self.cache.set.assert_any_call(self.mydict.remote_cache_last_updated_key, self.mydict._last_checked_for_remote_changes) 212 | 213 | def test_switch_access(self): 214 | self.mydict['hello'] = 'foo' 215 | self.cache.reset_mock() 216 | foo = self.mydict['hello'] 217 | foo = self.mydict['hello'] 218 | foo = self.mydict['hello'] 219 | foo = self.mydict['hello'] 220 | self.assertEquals(foo, 'foo') 221 | self.assertEquals(self.cache.get.call_count, 0) 222 | self.assertEquals(self.cache.set.call_count, 0) 223 | 224 | def test_switch_access_without_local_cache(self): 225 | self.mydict['hello'] = 'foo' 226 | self.mydict._local_cache = None 227 | self.mydict._last_checked_for_remote_changes = None 228 | self.cache.reset_mock() 229 | foo = self.mydict['hello'] 230 | self.assertEquals(foo, 'foo') 231 | # "1" here signifies that we didn't ask the remote cache for its last 232 | # updated value 233 | self.assertEquals(self.cache.get.call_count, 1) 234 | self.assertEquals(self.cache.set.call_count, 0) 235 | self.cache.get.assert_any_call(self.mydict.remote_cache_key) 236 | self.cache.reset_mock() 237 | foo = self.mydict['hello'] 238 | foo = self.mydict['hello'] 239 | foo = self.mydict['hello'] 240 | self.assertEquals(self.cache.get.call_count, 0) 241 | self.assertEquals(self.cache.set.call_count, 0) 242 | 243 | def test_switch_access_with_expired_local_cache(self): 244 | self.mydict['hello'] = 'foo' 245 | self.mydict._last_checked_for_remote_changes = None 246 | self.cache.reset_mock() 247 | foo = self.mydict['hello'] 248 | self.assertEquals(foo, 'foo') 249 | self.assertEquals(self.cache.get.call_count, 2) 250 | self.assertEquals(self.cache.set.call_count, 0) 251 | self.cache.get.assert_any_call(self.mydict.remote_cache_last_updated_key) 252 | self.cache.reset_mock() 253 | foo = self.mydict['hello'] 254 | foo = self.mydict['hello'] 255 | self.assertEquals(self.cache.get.call_count, 0) 256 | self.assertEquals(self.cache.set.call_count, 0) 257 | 258 | def test_does_not_pull_down_all_data(self): 259 | self.mydict['hello'] = 'foo' 260 | self.cache.get.return_value = self.mydict._local_last_updated - 100 261 | self.cache.reset_mock() 262 | 263 | self.mydict._cleanup() 264 | 265 | self.assertEquals(self.mydict['hello'], 'foo') 266 | self.cache.get.assert_called_once_with( 267 | self.mydict.remote_cache_last_updated_key 268 | ) 269 | 270 | 271 | class CachedDictTest(TestCase): 272 | def setUp(self): 273 | self.cache = mock.Mock() 274 | self.mydict = CachedDict(timeout=100, cache=self.cache) 275 | 276 | @mock.patch('modeldict.base.CachedDict._update_cache_data') 277 | @mock.patch('modeldict.base.CachedDict.local_cache_has_expired', mock.Mock(return_value=True)) 278 | @mock.patch('modeldict.base.CachedDict.local_cache_is_invalid', mock.Mock(return_value=False)) 279 | def test_expired_does_update_data(self, _update_cache_data): 280 | self.mydict._local_cache = {} 281 | self.mydict._last_checked_for_remote_changes = time.time() 282 | self.mydict._populate() 283 | 284 | self.assertFalse(_update_cache_data.called) 285 | 286 | @mock.patch('modeldict.base.CachedDict._update_cache_data') 287 | @mock.patch('modeldict.base.CachedDict.local_cache_has_expired', mock.Mock(return_value=False)) 288 | @mock.patch('modeldict.base.CachedDict.local_cache_is_invalid', mock.Mock(return_value=True)) 289 | def test_reset_does_expire(self, _update_cache_data): 290 | self.mydict._local_cache = {} 291 | self.mydict._last_checked_for_remote_changes = time.time() 292 | self.mydict._populate(reset=True) 293 | 294 | _update_cache_data.assert_called_once_with() 295 | 296 | @mock.patch('modeldict.base.CachedDict._update_cache_data') 297 | @mock.patch('modeldict.base.CachedDict.local_cache_has_expired', mock.Mock(return_value=False)) 298 | @mock.patch('modeldict.base.CachedDict.local_cache_is_invalid', mock.Mock(return_value=True)) 299 | def test_does_not_expire_by_default(self, _update_cache_data): 300 | self.mydict._local_cache = {} 301 | self.mydict._last_checked_for_remote_changes = time.time() 302 | self.mydict._populate() 303 | 304 | self.assertFalse(_update_cache_data.called) 305 | 306 | def test_is_expired_missing_last_checked_for_remote_changes(self): 307 | self.mydict._last_checked_for_remote_changes = None 308 | self.assertTrue(self.mydict.local_cache_has_expired()) 309 | self.assertFalse(self.cache.get.called) 310 | 311 | def test_is_expired_last_updated_beyond_timeout(self): 312 | self.mydict._local_last_updated = time.time() - 101 313 | self.assertTrue(self.mydict.local_cache_has_expired()) 314 | 315 | def test_is_expired_within_bounds(self): 316 | self.mydict._last_checked_for_remote_changes = time.time() 317 | 318 | def test_is_not_expired_if_remote_cache_is_old(self): 319 | # set it to an expired time 320 | self.mydict._local_cache = dict(a=1) 321 | self.mydict._local_last_updated = time.time() - 101 322 | self.cache.get.return_value = self.mydict._local_last_updated 323 | 324 | result = self.mydict.local_cache_is_invalid() 325 | 326 | self.cache.get.assert_called_once_with(self.mydict.remote_cache_last_updated_key) 327 | self.assertFalse(result) 328 | 329 | def test_is_expired_if_remote_cache_is_new(self): 330 | # set it to an expired time, but with a local cache 331 | self.mydict._local_cache = dict(a=1) 332 | self.mydict._last_checked_for_remote_changes = time.time() - 101 333 | self.cache.get.return_value = time.time() 334 | 335 | result = self.mydict.local_cache_is_invalid() 336 | 337 | self.cache.get.assert_called_once_with( 338 | self.mydict.remote_cache_last_updated_key 339 | ) 340 | self.assertEquals(result, True) 341 | -------------------------------------------------------------------------------- /tests/modeldict/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import * 2 | 3 | def dummy_view(request): 4 | from django.http import HttpResponse 5 | return HttpResponse() 6 | 7 | urlpatterns = patterns('', 8 | url(r'^$', dummy_view, name='modeldict-home'), 9 | ) --------------------------------------------------------------------------------