├── .editorconfig ├── .gitignore ├── .travis.yml ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── modeldict ├── __init__.py ├── base.py ├── models.py └── redis.py ├── pytest.ini ├── requirements.in ├── requirements.txt ├── runtests.py ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── settings.py ├── testapp │ ├── __init__.py │ ├── models.py │ ├── test_base.py │ └── test_modeldict.py └── urls.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .eggs 18 | .installed.cfg 19 | lib 20 | lib64 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .cache 27 | .coverage 28 | .coverage* 29 | .pytest_cache 30 | .tox 31 | nosetests.xml 32 | htmlcov 33 | 34 | # Translations 35 | *.mo 36 | 37 | # Mr Developer 38 | .mr.developer.cfg 39 | .project 40 | .pydevproject 41 | 42 | # Complexity 43 | output/*.html 44 | output/*/index.html 45 | 46 | # Sphinx 47 | docs/_build 48 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | dist: xenial 3 | 4 | notifications: 5 | email: false 6 | 7 | language: python 8 | python: '3.6' 9 | cache: pip 10 | 11 | install: 12 | - pip install tox 13 | 14 | script: 15 | - tox 16 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | ======= 4 | History 5 | ======= 6 | 7 | Pending release 8 | --------------- 9 | 10 | .. Add new release notes below here 11 | 12 | 2.0.3 (2019-05-17) 13 | ------------------ 14 | 15 | * **Retired: this project is no longer maintained.** I (Adam Johnson) no longer 16 | have time to continue maintaining this. I was doing so to support 17 | `gargoyle-yplan `__, a fork for my 18 | ex-employer YPlan. If you'd like to sponsor ongoing maintenance or take it 19 | over yourself, please contact me@adamj.eu. 20 | 21 | 2.0.2 (2019-04-28) 22 | ------------------ 23 | 24 | * Tested with Django 2.2. No changes were needed for compatibility. 25 | 26 | 2.0.1 (2019-02-15) 27 | ------------------ 28 | 29 | * No functional changes. This is a re-release of version 2.0.0 to fix immutable 30 | metadata on PyPI so that Pip on Python 2 doesn't pick up the Python 3 only 31 | 2.X series. **Version 2.0.0 will be pulled from PyPI on 2019-03-01.** 32 | 33 | 2.0.0 (2019-01-29) 34 | ------------------ 35 | 36 | **This version is due to be pulled from PyPI, please use version 2.0.1 as per 37 | its above release note.** 38 | 39 | * Drop Python 2 support, only Python 3.4+ is supported now. 40 | * Drop Django 1.8, 1.9, and 1.10 support. Only Django 1.11+ is supported now. 41 | * Remove ``iteritems()``, ``iterkeys()``, and ``itervalues()`` methods from 42 | ``ModelDict``, and move ``items()``, ``keys()``, and ``values()`` to Python 3 43 | semantics, returning iterators rather than lists. 44 | * Include LICENSE file in wheel. 45 | * Tested with Django 2.1. No changes were needed for compatibility. 46 | 47 | 1.5.4 (2016-10-28) 48 | ------------------ 49 | 50 | * Fixed a race condition in threaded code. See https://github.com/adamchainz/django-modeldict/pull/40 for a detailed 51 | explanation. Thanks @Jaidan. 52 | 53 | 1.5.3 (2016-09-20) 54 | ------------------ 55 | 56 | * Stop rounding ``time.time()`` down to the nearest integer, so we are more fine grained around expiration. It might 57 | also fix a subtle timing bug around re-fetching the remote cache unnecessarily. 58 | 59 | 1.5.2 (2016-07-31) 60 | ------------------ 61 | 62 | * Fixed update missing when ``_local_last_updated`` could be set even when it 63 | wasn't updated 64 | * Fixed update missing from integer rounding in time comparison 65 | * Fixed ``CachedDict.__repr__`` so it works for other subclasses of 66 | ``CachedDict`` than ``ModelDict`` (don't assume ``self.model`` exists) 67 | 68 | 1.5.1 (2016-06-13) 69 | ------------------ 70 | 71 | * Fixed local cache never expiring if value was checked too often. 72 | * Use Django's ``cache.set_many`` for more efficient storage. 73 | 74 | 1.5.0 (2016-01-11) 75 | ------------------ 76 | 77 | * Forked by YPlan 78 | * Fixed concurrency TOCTTOU bug for threaded Django servers. 79 | * Stopped including the 'tests' directory in package 80 | * Django 1.8 and 1.9 supported. 81 | * Python 3 support added. 82 | * Fixed ``setdefault()`` to return the value that was set/found, as per normal dict semantics. Thanks @olevinsky. 83 | 84 | 1.4.1 (2012-12-04) 85 | ------------------ 86 | 87 | * Last release by Disqus 88 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2010 DISQUS 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include setup.py README.rst HISTORY.rst MANIFEST.in LICENSE 2 | global-exclude *~ 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | django-modeldict 3 | ================ 4 | 5 | .. image:: https://img.shields.io/pypi/v/django-modeldict-yplan.svg 6 | :target: https://pypi.python.org/pypi/django-modeldict-yplan 7 | 8 | .. image:: https://travis-ci.org/adamchainz/django-modeldict.svg?branch=master 9 | :target: https://travis-ci.org/adamchainz/django-modeldict 10 | 11 | **Retired: this project is no longer maintained.** I (Adam Johnson) no longer 12 | have time to continue maintaining this. I was doing so to support 13 | `gargoyle-yplan `__, a fork for my 14 | ex-employer YPlan. If you'd like to sponsor ongoing maintenance or take it over 15 | yourself, please contact me@adamj.eu. 16 | 17 | ``ModelDict`` is a very efficient way to store things like settings in your database. The entire model is transformed into a dictionary (lazily) as well as stored in your cache. It's invalidated only when it needs to be (both in process and based on ``CACHE_BACKEND``). 18 | 19 | It was originally created by `Disqus `_, but due to the inactivity we at YPlan have taken over maintenance on this fork. 20 | 21 | Requirements 22 | ------------ 23 | 24 | Tested with all combinations of: 25 | 26 | * Python: 3.6 27 | * Django: 1.11, 2.0, 2.1, 2.2 28 | 29 | Python 3.4+ supported. 30 | 31 | Install 32 | ------- 33 | 34 | Install it with **pip**: 35 | 36 | .. code-block:: bash 37 | 38 | pip install django-modeldict-yplan 39 | 40 | Make sure you ``pip uninstall django-modeldict`` first if you're upgrading from the original to this fork - the packages clash. 41 | 42 | Example Usage 43 | ------------- 44 | 45 | .. code-block:: python 46 | 47 | # You'll need a model with fields to use as key and value in the dict 48 | class Setting(models.Model): 49 | key = models.CharField(max_length=32) 50 | value = models.CharField(max_length=200) 51 | 52 | # Create the ModelDict... 53 | settings = ModelDict(Setting, key='key', value='value', instances=False) 54 | 55 | # And you can treat it like a normal dict: 56 | 57 | # Missing values = KeyError 58 | settings['foo'] 59 | >>> KeyError 60 | 61 | # Sets supported 62 | settings['foo'] = 'hello' 63 | 64 | # Fetch the current value using normal dictionary access 65 | settings['foo'] 66 | >>> 'hello' 67 | 68 | # ...or by normal model queries 69 | Setting.objects.get(key='foo').value 70 | >>> 'hello' 71 | -------------------------------------------------------------------------------- /modeldict/__init__.py: -------------------------------------------------------------------------------- 1 | from .models import ModelDict 2 | 3 | __version__ = '2.0.3' 4 | VERSION = __version__ # legacy 5 | 6 | __all__ = ('__version__', 'VERSION', 'ModelDict') 7 | -------------------------------------------------------------------------------- /modeldict/base.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.core.cache import cache 4 | 5 | NoValue = object() 6 | 7 | 8 | class CachedDict(object): 9 | def __init__(self, cache=cache, timeout=30): 10 | cls_name = type(self).__name__ 11 | 12 | self._local_cache = {} 13 | self._local_last_updated = None 14 | 15 | self._last_checked_for_remote_changes = 0.0 16 | self.timeout = timeout 17 | 18 | self.remote_cache = cache 19 | self.remote_cache_key = cls_name 20 | self.remote_cache_last_updated_key = '%s.last_updated' % (cls_name,) 21 | 22 | def __getitem__(self, key): 23 | self._populate() 24 | 25 | try: 26 | return self._local_cache[key] 27 | except KeyError: 28 | if self._local_last_updated is None: 29 | # Another thread reset the cache 30 | return self[key] 31 | value = self.get_default(key) 32 | 33 | if value is NoValue: 34 | raise 35 | 36 | return value 37 | 38 | def __setitem__(self, key, value): 39 | raise NotImplementedError 40 | 41 | def __delitem__(self, key): 42 | raise NotImplementedError 43 | 44 | def __len__(self): 45 | self._populate() 46 | return len(self._local_cache) 47 | 48 | def __contains__(self, key): 49 | self._populate() 50 | return key in self._local_cache 51 | 52 | def __iter__(self): 53 | self._populate() 54 | return iter(self._local_cache) 55 | 56 | def __repr__(self): 57 | return "<%s>" % (self.__class__.__name__,) 58 | 59 | def items(self): 60 | self._populate() 61 | return self._local_cache.items() 62 | 63 | def values(self): 64 | self._populate() 65 | return self._local_cache.values() 66 | 67 | def keys(self): 68 | self._populate() 69 | return self._local_cache.keys() 70 | 71 | def get(self, key, default=None): 72 | self._populate() 73 | return self._local_cache.get(key, default) 74 | 75 | def pop(self, key, default=NoValue): 76 | value = self.get(key, default) 77 | 78 | try: 79 | del self[key] 80 | except KeyError: # pragma: no cover 81 | # Concurrent edit 82 | pass # pragma: no cover 83 | 84 | return value 85 | 86 | def setdefault(self, key, value): 87 | if key not in self: 88 | self[key] = value 89 | return self[key] 90 | 91 | def get_default(self, key): 92 | return NoValue 93 | 94 | def local_cache_has_expired(self): 95 | """ 96 | Returns ``True`` if the in-memory cache has expired. 97 | """ 98 | recheck_at = self._last_checked_for_remote_changes + self.timeout 99 | return time.time() > recheck_at 100 | 101 | def local_cache_is_invalid(self): 102 | """ 103 | Returns ``True`` if the local cache is invalid and needs to be 104 | refreshed with data from the remote cache. 105 | 106 | A return value of ``None`` signifies that no data was available. 107 | """ 108 | # If the local_cache hasn't been set up, avoid hitting memcache entirely 109 | if self._local_last_updated is None: 110 | return True 111 | 112 | remote_last_updated = self.remote_cache.get( 113 | self.remote_cache_last_updated_key 114 | ) 115 | 116 | if not remote_last_updated: 117 | # TODO: I don't like how we're overloading the return value here for 118 | # this method. It would be better to have a separate method or 119 | # @property that is the remote last_updated value. 120 | return None # Never been updated 121 | 122 | return ( 123 | self._local_last_updated is None or 124 | remote_last_updated > self._local_last_updated 125 | ) 126 | 127 | def get_cache_data(self): 128 | """ 129 | Pulls data from the cache backend. 130 | """ 131 | return self._get_cache_data() 132 | 133 | def clear_cache(self): 134 | """ 135 | Clears the in-process cache. 136 | """ 137 | self._local_last_updated = None 138 | self._last_checked_for_remote_changes = 0.0 139 | self._local_cache.clear() 140 | 141 | def _populate(self, reset=False): 142 | """ 143 | Ensures the cache is populated and still valid. 144 | 145 | The cache is checked when: 146 | 147 | - The local timeout has been reached 148 | - The local cache is not set 149 | 150 | The cache is invalid when: 151 | 152 | - The global cache has expired (via remote_cache_last_updated_key) 153 | """ 154 | now = time.time() 155 | 156 | if reset: 157 | self.clear_cache() 158 | # Otherwise, if the local cache has expired, we need to go check with 159 | # our remote last_updated value to see if the dict values have changed. 160 | elif self.local_cache_has_expired(): 161 | 162 | local_cache_is_invalid = self.local_cache_is_invalid() 163 | 164 | # If local_cache_is_invalid is None, that means that there was no 165 | # data present, so we assume we need to add the key to cache. 166 | if local_cache_is_invalid is None: 167 | self.remote_cache.add(self.remote_cache_last_updated_key, now) 168 | 169 | # Now, if the remote has changed OR it was None in the first place, 170 | # pull in the values from the remote cache and set it to the 171 | # local_cache 172 | if local_cache_is_invalid or local_cache_is_invalid is None: 173 | remote_value = self.remote_cache.get(self.remote_cache_key) 174 | if remote_value is not None: 175 | self._local_cache = remote_value 176 | # We've updated from remote, so mark ourselves as 177 | # such so that we won't expire until the next timeout 178 | self._local_last_updated = now 179 | 180 | # We last checked for remote changes just now 181 | self._last_checked_for_remote_changes = now 182 | 183 | # Update from cache if local_cache is still empty 184 | if self._local_last_updated is None: 185 | self._update_cache_data() 186 | 187 | return self._local_cache 188 | 189 | def _update_cache_data(self): 190 | self._local_cache = self.get_cache_data() 191 | 192 | now = time.time() 193 | self._local_last_updated = now 194 | self._last_checked_for_remote_changes = now 195 | 196 | # We only set remote_cache_last_updated_key when we know the cache is 197 | # current because setting this will force all clients to invalidate 198 | # their cached data if it's newer 199 | self.remote_cache.set_many({ 200 | self.remote_cache_key: self._local_cache, 201 | self.remote_cache_last_updated_key: self._last_checked_for_remote_changes, 202 | }) 203 | 204 | def _get_cache_data(self): 205 | raise NotImplementedError 206 | 207 | def _cleanup(self, *args, **kwargs): 208 | # We set _last_updated to a false value to ensure we hit the 209 | # last_updated cache on the next request 210 | self._last_checked_for_remote_changes = 0.0 211 | -------------------------------------------------------------------------------- /modeldict/models.py: -------------------------------------------------------------------------------- 1 | from django.core.signals import request_finished 2 | from django.db.models.signals import post_delete, post_save 3 | 4 | from .base import CachedDict, NoValue 5 | 6 | try: 7 | from celery.signals import task_postrun 8 | except ImportError: # pragma: no cover 9 | # celery must not be installed 10 | has_celery = False # pragma: no cover 11 | else: 12 | has_celery = True 13 | 14 | 15 | class ModelDict(CachedDict): 16 | """ 17 | Dictionary-style access to a model. Populates a cache and a local in-memory 18 | store to avoid multiple hits to the database. 19 | 20 | Specifying ``instances=True`` will cause the cache to store instances rather 21 | than simple values. 22 | 23 | If ``auto_create=True`` accessing modeldict[key] when key does not exist will 24 | attempt to create it in the database. 25 | 26 | Functions in two different ways, depending on the constructor: 27 | 28 | # Given ``Model`` that has a column named ``foo`` where the value is "bar": 29 | 30 | mydict = ModelDict(Model, value='foo') 31 | mydict['test'] 32 | >>> 'bar' #doctest: +SKIP 33 | 34 | If you want to use another key besides ``pk``, you may specify that in the 35 | constructor. However, this will be used as part of the cache key, so it's recommended 36 | to access it in the same way throughout your code. 37 | 38 | mydict = ModelDict(Model, key='foo', value='id') 39 | mydict['bar'] 40 | >>> 'test' #doctest: +SKIP 41 | 42 | """ 43 | def __init__(self, model, key='pk', value=None, instances=False, auto_create=False, *args, **kwargs): 44 | assert value is not None 45 | 46 | super(ModelDict, self).__init__(*args, **kwargs) 47 | 48 | cls_name = type(self).__name__ 49 | model_name = model.__name__ 50 | 51 | self.key = key 52 | self.value = value 53 | 54 | self.model = model 55 | self.instances = instances 56 | self.auto_create = auto_create 57 | 58 | self.remote_cache_key = '%s:%s:%s' % (cls_name, model_name, self.key) 59 | self.remote_cache_last_updated_key = '%s.last_updated:%s:%s' % (cls_name, model_name, self.key) 60 | 61 | request_finished.connect(self._cleanup) 62 | post_save.connect(self._post_save, sender=model) 63 | post_delete.connect(self._post_delete, sender=model) 64 | 65 | if has_celery: 66 | task_postrun.connect(self._cleanup) 67 | 68 | def __setitem__(self, key, value): 69 | if isinstance(value, self.model): 70 | value = getattr(value, self.value) 71 | 72 | manager = self.model._default_manager 73 | instance, created = manager.get_or_create( 74 | defaults={self.value: value}, 75 | **{self.key: key} 76 | ) 77 | 78 | # Ensure we're updating the value in the database if it changes 79 | if getattr(instance, self.value) != value: 80 | setattr(instance, self.value, value) 81 | manager.filter(**{self.key: key}).update(**{self.value: value}) 82 | self._post_save(sender=self.model, instance=instance, created=False) 83 | 84 | def __delitem__(self, key): 85 | self.model._default_manager.filter(**{self.key: key}).delete() 86 | # self._populate(reset=True) 87 | 88 | def __repr__(self): 89 | return "<%s: %s>" % (self.__class__.__name__, self.model.__name__) 90 | 91 | def setdefault(self, key, value): 92 | if isinstance(value, self.model): 93 | value = getattr(value, self.value) 94 | 95 | instance, created = self.model._default_manager.get_or_create( 96 | defaults={self.value: value}, 97 | **{self.key: key} 98 | ) 99 | value = getattr(instance, self.value) 100 | self[key] = value 101 | return value 102 | 103 | def get_default(self, key): 104 | if not self.auto_create: 105 | return NoValue 106 | result = self.model.objects.get_or_create(**{self.key: key})[0] 107 | if self.instances: 108 | return result 109 | return getattr(result, self.value) 110 | 111 | def _get_cache_data(self): 112 | qs = self.model._default_manager 113 | if self.instances: 114 | return dict((getattr(i, self.key), i) for i in qs.all()) 115 | return dict(qs.values_list(self.key, self.value)) 116 | 117 | # Signals 118 | 119 | def _post_save(self, sender, instance, created, **kwargs): 120 | self._populate(reset=True) 121 | 122 | def _post_delete(self, sender, instance, **kwargs): 123 | self._populate(reset=True) 124 | -------------------------------------------------------------------------------- /modeldict/redis.py: -------------------------------------------------------------------------------- 1 | from django.core.signals import request_finished 2 | 3 | from .base import CachedDict 4 | 5 | 6 | class RedisDict(CachedDict): 7 | """ 8 | Dictionary-style access to a redis hash table. Populates a cache and a local 9 | in-memory to avoid multiple hits to the database. 10 | 11 | Functions just like you'd expect it:: 12 | 13 | mydict = RedisDict('my_redis_key', Redis()) 14 | mydict['test'] 15 | >>> 'bar' #doctest: +SKIP 16 | 17 | """ 18 | def __init__(self, keyspace, connection, *args, **kwargs): 19 | super(CachedDict, self).__init__(*args, **kwargs) 20 | 21 | self.keyspace = keyspace 22 | self.conn = connection 23 | 24 | self.remote_cache_key = 'RedisDict:%s' % (keyspace,) 25 | self.remote_cache_last_updated_key = 'RedisDict.last_updated:%s' % (keyspace,) 26 | 27 | request_finished.connect(self._cleanup) 28 | 29 | def __setitem__(self, key, value): 30 | self.conn.hset(self.keyspace, key, value) 31 | if value != self._local_cache.get(key): 32 | self._local_cache[key] = value 33 | self._populate(reset=True) 34 | 35 | def __delitem__(self, key): 36 | self.conn.hdel(self.keyspace, key) 37 | self._local_cache.pop(key) 38 | self._populate(reset=True) 39 | 40 | def _get_cache_data(self): 41 | return self.conn.hgetall(self.keyspace) 42 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = --cov modeldict 3 | --cov-report term-missing 4 | --cov-fail-under 80 5 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | celery 2 | docutils 3 | flake8 4 | isort 5 | multilint 6 | Pygments 7 | pytest 8 | pytest-cov 9 | pytest-django 10 | pytz 11 | sqlparse 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # __main__.py compile -r -U 6 | # 7 | amqp==2.4.2 # via kombu 8 | atomicwrites==1.3.0 # via pytest 9 | attrs==19.1.0 # via pytest 10 | billiard==3.6.0.0 # via celery 11 | celery==4.3.0 12 | coverage==4.5.3 # via pytest-cov 13 | docutils==0.14 14 | entrypoints==0.3 # via flake8 15 | flake8==3.7.7 16 | isort==4.3.18 17 | kombu==4.5.0 # via celery 18 | mccabe==0.6.1 # via flake8 19 | more-itertools==7.0.0 # via pytest 20 | multilint==2.4.0 21 | pluggy==0.11.0 # via pytest 22 | py==1.8.0 # via pytest 23 | pycodestyle==2.5.0 # via flake8 24 | pyflakes==2.1.1 # via flake8 25 | pygments==2.4.0 26 | pytest-cov==2.7.1 27 | pytest-django==3.4.8 28 | pytest==4.4.1 29 | pytz==2019.1 30 | six==1.12.0 # via multilint, pytest 31 | sqlparse==0.3.0 32 | vine==1.3.0 # via amqp, celery 33 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | import pytest 6 | 7 | 8 | def main(): 9 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') 10 | sys.path.insert(0, 'tests') 11 | return pytest.main() 12 | 13 | 14 | if __name__ == '__main__': 15 | sys.exit(main()) 16 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | 4 | [isort] 5 | known_first_party = modeldict,nexus,testapp 6 | known_third_party = django 7 | line_length = 120 8 | multi_line_output = 5 9 | 10 | [metadata] 11 | license_file = LICENSE 12 | 13 | [tool:multilint] 14 | paths = modeldict 15 | runtests.py 16 | setup.py 17 | tests 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import re 4 | 5 | from setuptools import find_packages, setup 6 | 7 | 8 | def get_version(filename): 9 | with open(filename, 'r') as fp: 10 | contents = fp.read() 11 | return re.search(r"__version__ = ['\"]([^'\"]+)['\"]", contents).group(1) 12 | 13 | 14 | version = get_version(os.path.join('modeldict', '__init__.py')) 15 | 16 | with open('README.rst', 'r') as readme_file: 17 | readme = readme_file.read() 18 | 19 | with open('HISTORY.rst', 'r') as history_file: 20 | history = history_file.read().replace('.. :changelog:', '') 21 | 22 | setup( 23 | name='django-modeldict-yplan', 24 | version=version, 25 | author='DISQUS', 26 | author_email='opensource@disqus.com', 27 | maintainer='Adam Johnson', 28 | maintainer_email='me@adamj.eu', 29 | url='https://github.com/adamchainz/django-modeldict', 30 | description='Stores a model as a dictionary', 31 | long_description=readme + '\n\n' + history, 32 | packages=find_packages(exclude=['tests']), 33 | zip_safe=False, 34 | license='Apache License 2.0', 35 | include_package_data=True, 36 | python_requires='>=3.4', 37 | install_requires=[ 38 | 'Django>=1.11', 39 | ], 40 | classifiers=[ 41 | 'Development Status :: 7 - Inactive', 42 | 'Framework :: Django', 43 | 'Framework :: Django :: 1.11', 44 | 'Framework :: Django :: 2.0', 45 | 'Framework :: Django :: 2.1', 46 | 'Framework :: Django :: 2.2', 47 | 'Intended Audience :: Developers', 48 | 'Intended Audience :: System Administrators', 49 | 'License :: OSI Approved :: Apache Software License', 50 | 'Operating System :: OS Independent', 51 | 'Programming Language :: Python :: 3 :: Only', 52 | 'Programming Language :: Python :: 3', 53 | 'Programming Language :: Python :: 3.4', 54 | 'Programming Language :: Python :: 3.5', 55 | 'Programming Language :: Python :: 3.6', 56 | 'Programming Language :: Python', 57 | 'Topic :: Software Development' 58 | ], 59 | ) 60 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adamchainz/django-modeldict/bfec69e751d5d42c01e5a5365819e121ac54a828/tests/__init__.py -------------------------------------------------------------------------------- /tests/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import django 4 | 5 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 6 | 7 | DEBUG = True 8 | TEMPLATE_DEBUG = DEBUG 9 | 10 | SECRET_KEY = 'NOTASECRET' 11 | 12 | DATABASES = { 13 | 'default': { 14 | 'ENGINE': 'django.db.backends.sqlite3', 15 | 'NAME': '', 16 | } 17 | } 18 | 19 | CACHES = { 20 | 'default': { 21 | 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 22 | }, 23 | } 24 | 25 | ALLOWED_HOSTS = [] 26 | 27 | INSTALLED_APPS = ( 28 | 'django.contrib.auth', 29 | 'django.contrib.contenttypes', 30 | 'django.contrib.sessions', 31 | 'modeldict', 32 | 'testapp', 33 | ) 34 | 35 | _middleware = [ 36 | 'django.middleware.common.CommonMiddleware', 37 | 'django.middleware.csrf.CsrfViewMiddleware', 38 | 'django.contrib.sessions.middleware.SessionMiddleware', 39 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 40 | 'django.contrib.messages.middleware.MessageMiddleware', 41 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 42 | ] 43 | if django.VERSION >= (1, 10): 44 | MIDDLEWARE = _middleware 45 | else: 46 | MIDDLEWARE_CLASSES = _middleware 47 | 48 | ROOT_URLCONF = 'urls' 49 | LANGUAGE_CODE = 'en-us' 50 | TIME_ZONE = 'UTC' 51 | USE_I18N = True 52 | USE_L10N = True 53 | 54 | TEMPLATES = [ 55 | { 56 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 57 | 'DIRS': [], 58 | 'APP_DIRS': True, 59 | 'OPTIONS': { 60 | 'context_processors': [ 61 | 'django.template.context_processors.debug', 62 | 'django.template.context_processors.request', 63 | 'django.contrib.auth.context_processors.auth', 64 | 'django.contrib.messages.context_processors.messages', 65 | ], 66 | }, 67 | }, 68 | ] 69 | -------------------------------------------------------------------------------- /tests/testapp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adamchainz/django-modeldict/bfec69e751d5d42c01e5a5365819e121ac54a828/tests/testapp/__init__.py -------------------------------------------------------------------------------- /tests/testapp/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class ModelDictModel(models.Model): 5 | key = models.CharField(max_length=32, unique=True) 6 | value = models.CharField(max_length=32, default='') 7 | -------------------------------------------------------------------------------- /tests/testapp/test_base.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from django.test import SimpleTestCase 3 | 4 | from modeldict.base import CachedDict 5 | 6 | 7 | class CachedDictTest(SimpleTestCase): 8 | 9 | def test_setitem_not_implemented(self): 10 | d = CachedDict() 11 | 12 | with pytest.raises(NotImplementedError): 13 | d['x'] = 'foo' 14 | 15 | def test_delitem_not_implemented(self): 16 | d = CachedDict() 17 | 18 | with pytest.raises(NotImplementedError): 19 | del d['x'] 20 | -------------------------------------------------------------------------------- /tests/testapp/test_modeldict.py: -------------------------------------------------------------------------------- 1 | import time 2 | from unittest import mock 3 | 4 | import pytest 5 | from django.core.cache import cache 6 | from django.core.signals import request_finished 7 | from django.test import TestCase, TransactionTestCase 8 | 9 | from modeldict import ModelDict 10 | from modeldict.base import CachedDict 11 | from testapp.models import ModelDictModel 12 | 13 | 14 | class ModelDictTest(TransactionTestCase): 15 | # XXX: uses transaction test due to request_finished signal causing a rollback 16 | 17 | def setUp(self): 18 | cache.clear() 19 | 20 | def assertHasReceiver(self, signal, function): 21 | for ident, reciever in signal.receivers: 22 | if reciever() is function: 23 | return True 24 | return False 25 | 26 | def test_api(self): 27 | base_count = ModelDictModel.objects.count() 28 | 29 | mydict = ModelDict(ModelDictModel, key='key', value='value') 30 | mydict['foo'] = 'bar' 31 | assert mydict['foo'] == 'bar' 32 | assert ModelDictModel.objects.values_list('value', flat=True).get(key='foo') == 'bar' 33 | assert ModelDictModel.objects.count() == base_count + 1 34 | mydict['foo'] = 'bar2' 35 | assert mydict['foo'] == 'bar2' 36 | assert ModelDictModel.objects.values_list('value', flat=True).get(key='foo') == 'bar2' 37 | assert ModelDictModel.objects.count() == base_count + 1 38 | mydict['foo2'] = 'bar' 39 | assert mydict['foo2'] == 'bar' 40 | assert ModelDictModel.objects.values_list('value', flat=True).get(key='foo2') == 'bar' 41 | assert ModelDictModel.objects.count() == base_count + 2 42 | del mydict['foo2'] 43 | with pytest.raises(KeyError): 44 | mydict.__getitem__('foo2') 45 | assert not ModelDictModel.objects.filter(key='foo2').exists() 46 | assert ModelDictModel.objects.count() == base_count + 1 47 | 48 | ModelDictModel.objects.create(key='foo3', value='hello') 49 | 50 | assert mydict['foo3'] == 'hello' 51 | assert ModelDictModel.objects.filter(key='foo3').exists(), True 52 | assert ModelDictModel.objects.count() == base_count + 2 53 | 54 | request_finished.send(sender=self) 55 | 56 | assert mydict._last_checked_for_remote_changes == 0.0 57 | 58 | # These should still error because even though the cache repopulates (local cache) 59 | # the remote cache pool does not 60 | # self.assertRaises(KeyError, mydict.__getitem__, 'foo3') 61 | # self.assertTrue(ModelDictModel.objects.filter(key='foo3').exists()) 62 | # self.assertEquals(ModelDictModel.objects.count(), base_count + 2) 63 | 64 | assert mydict['foo'] == 'bar2' 65 | assert ModelDictModel.objects.values_list('value', flat=True).get(key='foo') == 'bar2' 66 | assert ModelDictModel.objects.count() == base_count + 2 67 | 68 | assert mydict.pop('foo') == 'bar2' 69 | assert mydict.pop('foo', None) is None 70 | assert not ModelDictModel.objects.filter(key='foo').exists() 71 | assert ModelDictModel.objects.count() == base_count + 1 72 | 73 | def test_modeldict_instances(self): 74 | base_count = ModelDictModel.objects.count() 75 | 76 | mydict = ModelDict(ModelDictModel, key='key', value='value', instances=True) 77 | mydict['foo'] = ModelDictModel(key='foo', value='bar') 78 | assert isinstance(mydict['foo'], ModelDictModel) 79 | assert mydict['foo'].pk 80 | assert mydict['foo'].value == 'bar' 81 | assert ModelDictModel.objects.values_list('value', flat=True).get(key='foo') == 'bar' 82 | assert ModelDictModel.objects.count() == base_count + 1 83 | old_pk = mydict['foo'].pk 84 | mydict['foo'] = ModelDictModel(key='foo', value='bar2') 85 | assert isinstance(mydict['foo'], ModelDictModel) 86 | assert mydict['foo'].pk == old_pk 87 | assert mydict['foo'].value == 'bar2' 88 | assert ModelDictModel.objects.values_list('value', flat=True).get(key='foo') == 'bar2' 89 | assert ModelDictModel.objects.count() == base_count + 1 90 | 91 | # test deletion 92 | mydict['foo'].delete() 93 | assert 'foo' not in mydict 94 | 95 | def test_modeldict_instances_auto_create(self): 96 | mydict = ModelDict(ModelDictModel, key='key', value='value', instances=True, auto_create=True) 97 | 98 | obj = mydict['foo'] 99 | assert isinstance(obj, ModelDictModel) 100 | assert obj.value == '' 101 | 102 | def test_modeldict_len_empty(self): 103 | mydict = ModelDict(ModelDictModel, key='key', value='value') 104 | assert len(mydict) == 0 105 | 106 | def test_modeldict_len_one(self): 107 | mydict = ModelDict(ModelDictModel, key='key', value='value') 108 | mydict['hello'] = 'world' 109 | assert len(mydict) == 1 110 | 111 | def test_modeldict_len_two(self): 112 | mydict = ModelDict(ModelDictModel, key='key', value='value') 113 | mydict['hello'] = 'world' 114 | mydict['hi'] = 'world' 115 | assert len(mydict) == 2 116 | 117 | def test_modeldict_iter(self): 118 | mydict = ModelDict(ModelDictModel, key='key', value='value') 119 | mydict['hello'] = 'world' 120 | assert next(iter(mydict)) == 'hello' 121 | 122 | def test_modeldict_items(self): 123 | mydict = ModelDict(ModelDictModel, key='key', value='value') 124 | mydict['hello'] = 'world' 125 | assert list(mydict.items()) == [('hello', 'world')] 126 | 127 | def test_modeldict_values(self): 128 | mydict = ModelDict(ModelDictModel, key='key', value='value') 129 | mydict['hello'] = 'world' 130 | assert list(mydict.values()) == ['world'] 131 | 132 | def test_modeldict_keys(self): 133 | mydict = ModelDict(ModelDictModel, key='key', value='value') 134 | mydict['hello'] = 'world' 135 | assert list(mydict.keys()) == ['hello'] 136 | 137 | def test_modeldict_expirey(self): 138 | base_count = ModelDictModel.objects.count() 139 | 140 | mydict = ModelDict(ModelDictModel, key='key', value='value') 141 | 142 | assert mydict._local_cache == {} 143 | 144 | mydict['test_modeldict_expirey'] = 'hello' 145 | 146 | assert len(mydict._local_cache) == base_count + 1 147 | assert mydict['test_modeldict_expirey'] == 'hello' 148 | 149 | self.client.get('/') 150 | 151 | assert mydict._last_checked_for_remote_changes == 0.0 152 | assert mydict['test_modeldict_expirey'] == 'hello' 153 | assert len(mydict._local_cache) == base_count + 1 154 | 155 | request_finished.send(sender=self) 156 | 157 | assert mydict._last_checked_for_remote_changes == 0.0 158 | assert mydict['test_modeldict_expirey'] == 'hello' 159 | assert len(mydict._local_cache) == base_count + 1 160 | 161 | def test_modeldict_no_auto_create(self): 162 | # without auto_create 163 | mydict = ModelDict(ModelDictModel, key='key', value='value') 164 | with pytest.raises(KeyError): 165 | mydict['hello'] 166 | assert ModelDictModel.objects.count() == 0 167 | 168 | def test_modeldict_auto_create_no_value(self): 169 | # with auto_create and no value 170 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 171 | repr(mydict['hello']) 172 | assert ModelDictModel.objects.count() == 1 173 | assert ModelDictModel.objects.get(key='hello').value == '' 174 | 175 | def test_modeldict_auto_create(self): 176 | # with auto_create and value 177 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 178 | mydict['hello'] = 'foo' 179 | assert ModelDictModel.objects.count() == 1 180 | assert ModelDictModel.objects.get(key='hello').value == 'foo' 181 | 182 | def test_save_behavior(self): 183 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 184 | mydict['hello'] = 'foo' 185 | for n in range(10): 186 | mydict[str(n)] = 'foo' 187 | assert len(mydict) == 11 188 | assert ModelDictModel.objects.count() == 11 189 | 190 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 191 | m = ModelDictModel.objects.get(key='hello') 192 | m.value = 'bar' 193 | m.save() 194 | 195 | assert ModelDictModel.objects.count() == 11 196 | assert len(mydict) == 11 197 | assert mydict['hello'] == 'bar' 198 | 199 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 200 | m = ModelDictModel.objects.get(key='hello') 201 | m.value = 'bar2' 202 | m.save() 203 | 204 | assert ModelDictModel.objects.count() == 11 205 | assert len(mydict) == 11 206 | assert mydict['hello'] == 'bar2' 207 | 208 | def test_setdefault(self): 209 | mydict = ModelDict(ModelDictModel, key='key', value='value') 210 | 211 | with pytest.raises(KeyError): 212 | mydict['hello'] 213 | 214 | ret = mydict.setdefault('hello', 'world') 215 | assert ret == 'world' 216 | assert mydict['hello'] == 'world' 217 | 218 | ret = mydict.setdefault('hello', 'world2') 219 | assert ret == 'world' 220 | assert mydict['hello'] == 'world' 221 | 222 | def test_setdefault_instances(self): 223 | mydict = ModelDict(ModelDictModel, key='key', value='value') 224 | 225 | with pytest.raises(KeyError): 226 | mydict['hello'] 227 | 228 | instance = ModelDictModel(key='hello', value='world') 229 | ret = mydict.setdefault('hello', instance) 230 | assert ret == 'world' 231 | assert mydict['hello'] == 'world' 232 | 233 | instance2 = ModelDictModel(key='hello', value='world2') 234 | ret = mydict.setdefault('hello', instance2) 235 | assert ret == 'world' 236 | assert mydict['hello'] == 'world' 237 | 238 | def test_django_signals_are_connected(self): 239 | from django.db.models.signals import post_save, post_delete 240 | from django.core.signals import request_finished 241 | 242 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 243 | self.assertHasReceiver(post_save, mydict._post_save) 244 | self.assertHasReceiver(post_delete, mydict._post_delete) 245 | self.assertHasReceiver(request_finished, mydict._cleanup) 246 | 247 | def test_celery_signals_are_connected(self): 248 | from celery.signals import task_postrun 249 | 250 | mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True) 251 | self.assertHasReceiver(task_postrun, mydict._cleanup) 252 | 253 | 254 | class CacheIntegrationTest(TestCase): 255 | def setUp(self): 256 | self.cache = mock.Mock() 257 | self.cache.get.return_value = {} 258 | self.mydict = ModelDict(ModelDictModel, key='key', value='value', auto_create=True, cache=self.cache) 259 | 260 | def test_switch_creation(self): 261 | self.mydict['hello'] = 'foo' 262 | assert self.cache.get.call_count == 0 263 | assert self.cache.set_many.call_count == 1 264 | self.cache.set_many.assert_any_call({ 265 | self.mydict.remote_cache_key: {u'hello': u'foo'}, 266 | self.mydict.remote_cache_last_updated_key: self.mydict._last_checked_for_remote_changes, 267 | }) 268 | 269 | def test_switch_change(self): 270 | self.mydict['hello'] = 'foo' 271 | self.cache.reset_mock() 272 | self.mydict['hello'] = 'bar' 273 | assert self.cache.get.call_count == 0 274 | assert self.cache.set_many.call_count == 1 275 | self.cache.set_many.assert_any_call({ 276 | self.mydict.remote_cache_key: {u'hello': u'bar'}, 277 | self.mydict.remote_cache_last_updated_key: self.mydict._last_checked_for_remote_changes 278 | }) 279 | 280 | def test_switch_delete(self): 281 | self.mydict['hello'] = 'foo' 282 | self.cache.reset_mock() 283 | del self.mydict['hello'] 284 | assert self.cache.get.call_count == 0 285 | assert self.cache.set_many.call_count == 1 286 | self.cache.set_many.assert_any_call({ 287 | self.mydict.remote_cache_key: {}, 288 | self.mydict.remote_cache_last_updated_key: self.mydict._last_checked_for_remote_changes 289 | }) 290 | 291 | def test_switch_access(self): 292 | self.mydict['hello'] = 'foo' 293 | self.cache.reset_mock() 294 | foo = self.mydict['hello'] 295 | foo = self.mydict['hello'] 296 | foo = self.mydict['hello'] 297 | foo = self.mydict['hello'] 298 | assert foo == 'foo' 299 | assert self.cache.get.call_count == 0 300 | assert self.cache.set_many.call_count == 0 301 | 302 | def test_switch_access_without_local_cache(self): 303 | self.mydict['hello'] = 'foo' 304 | self.mydict._local_cache = {} 305 | self.mydict._local_last_updated = None 306 | self.mydict._last_checked_for_remote_changes = 0.0 307 | self.cache.reset_mock() 308 | foo = self.mydict['hello'] 309 | assert foo == 'foo' 310 | # "1" here signifies that we didn't ask the remote cache for its last 311 | # updated value 312 | assert self.cache.get.call_count == 1 313 | assert self.cache.set_many.call_count == 0 314 | self.cache.get.assert_any_call(self.mydict.remote_cache_key) 315 | self.cache.reset_mock() 316 | foo = self.mydict['hello'] 317 | foo = self.mydict['hello'] 318 | foo = self.mydict['hello'] 319 | assert self.cache.get.call_count == 0 320 | assert self.cache.set_many.call_count == 0 321 | 322 | def test_switch_access_with_expired_local_cache(self): 323 | self.mydict['hello'] = 'foo' 324 | self.mydict._last_checked_for_remote_changes = 0.0 325 | self.cache.reset_mock() 326 | foo = self.mydict['hello'] 327 | assert foo == 'foo' 328 | assert self.cache.get.call_count == 2 329 | assert self.cache.set_many.call_count == 0 330 | self.cache.get.assert_any_call(self.mydict.remote_cache_last_updated_key) 331 | self.cache.reset_mock() 332 | foo = self.mydict['hello'] 333 | foo = self.mydict['hello'] 334 | assert self.cache.get.call_count == 0 335 | assert self.cache.set_many.call_count == 0 336 | 337 | def test_does_not_pull_down_all_data(self): 338 | self.mydict['hello'] = 'foo' 339 | self.cache.get.return_value = self.mydict._local_last_updated - 100 340 | self.cache.reset_mock() 341 | 342 | self.mydict._cleanup() 343 | 344 | assert self.mydict['hello'] == 'foo' 345 | self.cache.get.assert_called_once_with( 346 | self.mydict.remote_cache_last_updated_key 347 | ) 348 | 349 | 350 | class CachedDictTest(TestCase): 351 | def setUp(self): 352 | self.cache = mock.Mock() 353 | self.mydict = CachedDict(timeout=100, cache=self.cache) 354 | 355 | @mock.patch('modeldict.base.CachedDict._update_cache_data') 356 | @mock.patch('modeldict.base.CachedDict.local_cache_has_expired', mock.Mock(return_value=True)) 357 | @mock.patch('modeldict.base.CachedDict.local_cache_is_invalid', mock.Mock(return_value=False)) 358 | def test_expired_does_update_data(self, _update_cache_data): 359 | self.mydict._local_cache = {} 360 | self.mydict._local_last_updated = time.time() 361 | self.mydict._last_checked_for_remote_changes = time.time() 362 | self.mydict._populate() 363 | 364 | assert not _update_cache_data.called 365 | 366 | @mock.patch('modeldict.base.CachedDict._update_cache_data') 367 | @mock.patch('modeldict.base.CachedDict.local_cache_has_expired', mock.Mock(return_value=False)) 368 | @mock.patch('modeldict.base.CachedDict.local_cache_is_invalid', mock.Mock(return_value=True)) 369 | def test_reset_does_expire(self, _update_cache_data): 370 | self.mydict._local_cache = {} 371 | self.mydict._local_last_updated = time.time() 372 | self.mydict._last_checked_for_remote_changes = time.time() 373 | self.mydict._populate(reset=True) 374 | 375 | _update_cache_data.assert_called_once_with() 376 | 377 | @mock.patch('modeldict.base.CachedDict._update_cache_data') 378 | @mock.patch('modeldict.base.CachedDict.local_cache_has_expired', mock.Mock(return_value=False)) 379 | @mock.patch('modeldict.base.CachedDict.local_cache_is_invalid', mock.Mock(return_value=True)) 380 | def test_does_not_expire_by_default(self, _update_cache_data): 381 | self.mydict._local_cache = {} 382 | self.mydict._local_last_updated = time.time() 383 | self.mydict._last_checked_for_remote_changes = time.time() 384 | self.mydict._populate() 385 | 386 | assert not _update_cache_data.called 387 | 388 | def test_is_expired_missing_last_checked_for_remote_changes(self): 389 | self.mydict._last_checked_for_remote_changes = 0.0 390 | assert self.mydict.local_cache_has_expired() 391 | assert not self.cache.get.called 392 | 393 | def test_is_expired_last_updated_beyond_timeout(self): 394 | self.mydict._local_last_updated = time.time() - 101 395 | assert self.mydict.local_cache_has_expired() 396 | 397 | def test_is_expired_within_bounds(self): 398 | self.mydict._last_checked_for_remote_changes = time.time() 399 | 400 | def test_is_not_expired_if_remote_cache_is_old(self): 401 | # set it to an expired time 402 | self.mydict._local_cache = {'a': 1} 403 | self.mydict._local_last_updated = time.time() - 100 404 | self.cache.get.return_value = self.mydict._local_last_updated - 1 405 | 406 | result = self.mydict.local_cache_is_invalid() 407 | 408 | self.cache.get.assert_called_once_with(self.mydict.remote_cache_last_updated_key) 409 | assert not result 410 | 411 | def test_is_expired_if_remote_cache_is_new(self): 412 | # set it to an expired time, but with a local cache 413 | self.mydict._local_cache = dict(a=1) 414 | last_update = time.time() - 101 415 | self.mydict._local_last_updated = last_update 416 | self.mydict._last_checked_for_remote_changes = last_update 417 | self.cache.get.return_value = time.time() 418 | 419 | result = self.mydict.local_cache_is_invalid() 420 | 421 | assert result 422 | self.cache.get.assert_called_once_with( 423 | self.mydict.remote_cache_last_updated_key 424 | ) 425 | 426 | def test_is_invalid_if_local_cache_is_none(self): 427 | self.mydict._local_cache = None 428 | assert self.mydict.local_cache_is_invalid() 429 | 430 | def test_is_invalid_if_remote_cache_updated_right_after_local_last_updated(self): 431 | cache.clear() 432 | mydict = CachedDict(timeout=100) 433 | 434 | mydict.remote_cache.set_many({ 435 | mydict.remote_cache_key: {'MYFLAG': 'value1'}, 436 | mydict.remote_cache_last_updated_key: 12345 437 | }) 438 | 439 | # load the local cache from remote cache 440 | # this sets: mydict._local_last_updated = time.time() 441 | mydict._populate() 442 | 443 | # simulate remote cache updated by external process 444 | # remote_cache[remote_cache_last_updated_key] = time.time() 445 | mydict.remote_cache.set_many({ 446 | mydict.remote_cache_key: {'MYFLAG': 'value2'}, 447 | mydict.remote_cache_last_updated_key: time.time() 448 | }) 449 | 450 | assert mydict.local_cache_is_invalid() 451 | 452 | def test_populate_timeout(self): 453 | cache.clear() 454 | mydict = CachedDict(timeout=100) 455 | 456 | now = time.time() 457 | mydict.remote_cache.set_many({ 458 | mydict.remote_cache_key: {'MYFLAG': 'value1'}, 459 | mydict.remote_cache_last_updated_key: now 460 | }) 461 | 462 | # load the local cache from remote cache 463 | mydict._populate() 464 | 465 | mydict.remote_cache.set_many({ 466 | mydict.remote_cache_key: {'MYFLAG': 'value2'}, 467 | mydict.remote_cache_last_updated_key: now + 1 468 | }) 469 | 470 | # before timeout: local cache should not be updated 471 | with mock.patch('time.time', mock.Mock(return_value=now + mydict.timeout - 1)): 472 | mydict._populate() 473 | mydict._populate() 474 | mydict._populate() 475 | assert mydict._local_cache == {'MYFLAG': 'value1'} 476 | 477 | # after timeout: local cache should be updated 478 | with mock.patch('time.time', mock.Mock(return_value=now + mydict.timeout + 1)): 479 | mydict._populate() 480 | assert mydict._local_cache == {'MYFLAG': 'value2'} 481 | 482 | def test_local_last_updated(self): 483 | cache.clear() 484 | mydict = CachedDict(timeout=100) 485 | mydict.remote_cache.set_many({ 486 | mydict.remote_cache_key: {'MYFLAG': 'value1'}, 487 | mydict.remote_cache_last_updated_key: 12345 488 | }) 489 | # load the local cache from remote cache 490 | # this sets: mydict._local_last_updated = time.time() 491 | mydict._populate() 492 | local_last_updated = mydict._local_last_updated 493 | assert mydict._local_cache == {'MYFLAG': 'value1'} 494 | 495 | with mock.patch('time.time', mock.Mock(return_value=time.time() + 101)): 496 | mydict.remote_cache.set_many({ 497 | mydict.remote_cache_key: {'MYFLAG': 'value2'}, 498 | mydict.remote_cache_last_updated_key: time.time() 499 | }) 500 | assert mydict.local_cache_has_expired() 501 | assert mydict.local_cache_is_invalid() 502 | 503 | mydict._populate() 504 | 505 | assert mydict._local_cache == {'MYFLAG': 'value2'} 506 | assert mydict._local_last_updated != local_last_updated 507 | 508 | def test_local_last_updated_not_updated_if_not_needed(self): 509 | cache.clear() 510 | mydict = CachedDict(timeout=100) 511 | mydict.remote_cache.set_many({ 512 | mydict.remote_cache_key: {'MYFLAG': 'value1'}, 513 | mydict.remote_cache_last_updated_key: 12345 514 | }) 515 | # load the local cache from remote cache 516 | # this sets: mydict._local_last_updated = time.time() 517 | mydict._populate() 518 | local_last_updated = mydict._local_last_updated 519 | 520 | with mock.patch('time.time', mock.Mock(return_value=time.time() + 101)): 521 | assert mydict.local_cache_has_expired() 522 | assert not mydict.local_cache_is_invalid() 523 | 524 | mydict._populate() 525 | 526 | assert mydict._local_last_updated == local_last_updated 527 | -------------------------------------------------------------------------------- /tests/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | 4 | def dummy_view(request): 5 | from django.http import HttpResponse 6 | return HttpResponse() 7 | 8 | 9 | urlpatterns = [ 10 | url(r'^$', dummy_view, name='modeldict-home'), 11 | ] 12 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py3-django{111,20,21,22}, 4 | py3-codestyle 5 | 6 | [testenv] 7 | setenv = 8 | PYTHONDONTWRITEBYTECODE=1 9 | deps = 10 | django111: Django>=1.11a1,<2.0 11 | django20: Django>=2.0,<2.1 12 | django21: Django>=2.1,<2.2 13 | django22: Django>=2.2,<2.3 14 | -rrequirements.txt 15 | commands = ./runtests.py {posargs} 16 | 17 | [testenv:py3-codestyle] 18 | deps = -rrequirements.txt 19 | skip_install = true 20 | commands = multilint 21 | --------------------------------------------------------------------------------