├── .coveragerc ├── .gitignore ├── .gitmodules ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── README.rst ├── VERSION ├── django_cereal ├── __init__.py ├── pickle.py └── tests │ ├── __init__.py │ ├── pickle.py │ └── testapp │ ├── __init__.py │ └── models.py ├── requirements.txt ├── requirements ├── README.rst ├── default.txt ├── pkgutils.txt └── test.txt ├── runtests.py ├── scripts ├── git-tools-hooks │ └── git-on-release.sh └── removepyc.sh ├── setup.cfg ├── setup.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | cover_pylib = False 4 | 5 | [report] 6 | exclude_lines = 7 | # Have to re-enable the standard pragma 8 | pragma: no cover 9 | 10 | # Don't complain about missing debug-only code: 11 | def __repr__ 12 | if self\.debug 13 | 14 | # Don't complain if tests don't hit defensive assertion code: 15 | raise AssertionError 16 | raise NotImplementedError 17 | 18 | # Don't complain if non-runnable code isn't run: 19 | if 0: 20 | if __name__ == .__main__.: 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | cover 46 | django_cereal/tests/cover 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | # Project Specific 62 | .idea 63 | .venv 64 | .project 65 | .pydevproject 66 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "scripts/git-tools"] 2 | path = scripts/git-tools 3 | url = https://github.com/alexhayes/git-tools.git 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Release 0.2.1 - Mon Dec 14 12:26:56 AEDT 2015 2 | 3 | - Fixed issue in docs. 4 | 5 | # Release 0.2.0 - Mon Dec 14 12:25:12 AEDT 2015 6 | 7 | - Support for Django 1.9 and Python 3.5 - updated docs accordingly. 8 | - Fixed issue with post release hook. 9 | - Added hook to update version in __init__.py 10 | 11 | # Release 0.1.2 - Wed Jul 1 10:01:42 AEST 2015 12 | 13 | - Fixes #1 - 'MySQL server has gone away' with --maxtasksperchild 14 | - Added VERSION file. 15 | - Added git-tools 16 | - Tox now tests Django 1.7 17 | 18 | # Release 0.1.1 - Fri Jun 19 14:50:00 AEST 2015 19 | 20 | - Fixed issue with inherited models. 21 | 22 | # Release 0.1.0 - Fri Jun 19 11:30:00 AEST 2015 23 | 24 | - Initial release 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Alex Hayes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGELOG.md 2 | include LICENSE 3 | include README.rst 4 | include MANIFEST.in 5 | include setup.cfg 6 | include setup.py 7 | recursive-include django-cereal *.py 8 | recursive-include docs * 9 | recursive-include requirements *.txt 10 | prune *.pyc 11 | prune *.sw* 12 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | django-cereal 3 | ============= 4 | 5 | Efficient serialization of `Django`_ `Models`_ for use in `Celery`_ that ensure 6 | the state of the world. 7 | 8 | It supports Django 1.7, 1.8 and 1.9 for Python versions 2.7, 3.3, 3.4, 3.5 and 9 | pypy (where Django supports the Python version). 10 | 11 | .. _`Django`: https://www.djangoproject.com/ 12 | .. _`Models`: https://docs.djangoproject.com/en/stable/topics/db/models/ 13 | .. _`Celery`: http://www.celeryproject.org/ 14 | 15 | Scenario 16 | ======== 17 | 18 | If you're using `Django`_ and `Celery`_ you're most likely passing instances 19 | of `models`_ back and forth between tasks or, as the Celery `docs suggest`_, 20 | you're passing just the primary key to a task and then retrieving the the model 21 | instance with the primary key. 22 | 23 | If you're doing the former, it's potentially inefficient and certainly dangerous 24 | as by the time the task executes the models data could be changed! 25 | 26 | If you're using the later, you're probably wondering to yourself, surely there 27 | is a better way?! While it's efficient and certainly readable it's not exactly 28 | much fun continually fetching the model at the start of each task... 29 | 30 | You may also be using model methods as tasks, but unless you're using something 31 | similar to `this refresh decorator`_, you'll potentially have stale model data. 32 | 33 | django-cereal to the rescue... 34 | 35 | .. _`Django`: https://www.djangoproject.com/ 36 | .. _`Celery`: http://www.celeryproject.org/ 37 | .. _`models`: https://docs.djangoproject.com/en/stable/topics/db/models/ 38 | .. _`docs suggest`: http://docs.celeryproject.org/en/latest/userguide/tasks.html?highlight=model#state 39 | .. _`this refresh decorator`: https://bitbucket.org/alexhayes/django-toolkit/src/93d23b254bb1edcf31ff5b0f91673fc439f26438/django_toolkit/models/decorators.py?at=master#cl-3 40 | 41 | 42 | How It Works 43 | ============ 44 | 45 | django-cereal works by using an alternative serializer before the task is sent 46 | to the message bus and then retrieves a fresh instance of the model during 47 | deserialization. Currently only `pickle`_ is supported (feel free to fork and 48 | implement for JSON or YAML). 49 | 50 | Essentially when the model is serialized only the primary key and the model's 51 | class are pickled. This is obviously not quite as efficient as pickling just the 52 | models primary key, but it's certainly better than serializing the entire model! 53 | 54 | When the task is picked up by a Celery worker and deserialized an instance of 55 | the model is retrieved using :code:`YourModel.objects.get(pk=xxx)` and thus this 56 | approach is also safe as you're not using stale model data in your task. 57 | 58 | The serializer is `registered with kombu`_ and safely patches 59 | :code:`django.db.Model.__reduce__` - it only operates inside the scope of kombu 60 | and thus doesn't mess with a model's pickling outside of kombu. 61 | 62 | .. _`pickle`: https://docs.python.org/2/library/pickle.html 63 | .. _`registered with kombu`: http://kombu.readthedocs.org/en/latest/userguide/serialization.html#creating-extensions-using-setuptools-entry-points 64 | 65 | 66 | Installation 67 | ============ 68 | 69 | You can install django-cereal either via the Python Package Index (PyPI) 70 | or from github. 71 | 72 | To install using pip; 73 | 74 | .. code-block:: bash 75 | 76 | $ pip install django-cereal 77 | 78 | From github; 79 | 80 | .. code-block:: bash 81 | 82 | $ pip install git+https://github.com/alexhayes/django-cereal.git 83 | 84 | 85 | Usage 86 | ===== 87 | 88 | All that is required is that you specify the kwarg :code:`serializer` when 89 | defining a task. 90 | 91 | .. code-block:: python 92 | 93 | from django_cereal.pickle import DJANGO_CEREAL_PICKLE 94 | 95 | @app.task(serializer=DJANGO_CEREAL_PICKLE) 96 | def my_task(my_model): 97 | ... 98 | 99 | There is also a helper task that you can use which defines the serializer if 100 | it's not set. 101 | 102 | .. code-block:: python 103 | 104 | from django_cereal.pickle import task 105 | 106 | @task 107 | def my_task(my_model): 108 | ... 109 | 110 | Another approach is to set :code:`CELERY_TASK_SERIALIZER` to 111 | :code:`django-cereal-pickle`. 112 | 113 | 114 | Model Task Methods 115 | ================== 116 | 117 | You can also use task methods on your Django models, so you don't have to define 118 | them in a tasks.py. For example; 119 | 120 | .. code-block:: python 121 | 122 | from celery.contrib.methods import task_method 123 | from django_cereal.pickle import DJANGO_CEREAL_PICKLE 124 | from yourproject.celery import app 125 | 126 | 127 | task_method_kwargs = dict(filter=task_method, 128 | serializer=DJANGO_CEREAL_PICKLE) 129 | 130 | 131 | class MyModel(models.Model): 132 | 133 | @app.task(name='MyModel.foo', **task_method_kwargs) 134 | def foo(self): 135 | # self is an instance of MyModel 136 | 137 | 138 | Then, you can call your task as follows; 139 | 140 | .. code-block:: python 141 | 142 | bar = MyModel.objects.get(...) 143 | bar.foo.delay() 144 | 145 | 146 | Just like your would a normal task but you can stop defining tasks that simply 147 | orchestrate calls on a model and just call the model directly. 148 | 149 | 150 | Chaining Task Methods 151 | ===================== 152 | 153 | While not directly related to serialization of Django models, if you are using 154 | Django Model methods as tasks, or any class methods as tasks for that matter, 155 | and you are chaining these tasks you may be interested in the 156 | `@ensure_self decorator`_ (see `Celery issue #2137`_ for more details). 157 | 158 | .. _`@ensure_self decorator`: https://github.com/alexhayes/django-toolkit/blob/master/django_toolkit/celery/decorators.py#L3 159 | .. _`Celery issue #2137`: https://github.com/celery/celery/issues/2137 160 | 161 | 162 | Database Connections 163 | ==================== 164 | 165 | Note that if you use the :code:`--maxtasksperworker` flag in Celery, or under 166 | other similar situations, the connection to a database in Django could become 167 | unusable, with errors such as the following thrown; 168 | 169 | .. code-block:: python 170 | 171 | OperationalError(2006, 'MySQL server has gone away') 172 | 173 | This is now handled by the unpickling by closing down the database connection 174 | which forces a new connection to be created. 175 | 176 | Perhaps in the future there may be a nicer way of handling this, for instance, 177 | a new connection is created each time a worker is created, but for now the fix 178 | in place works, even if it's not ideal. 179 | 180 | 181 | License 182 | ======= 183 | 184 | This software is licensed under the `MIT License`. See the ``LICENSE`` 185 | file in the top distribution directory for the full license text. 186 | 187 | 188 | Author 189 | ====== 190 | 191 | Alex Hayes 192 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.2.1 2 | -------------------------------------------------------------------------------- /django_cereal/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Efficient serialization of Django Models for use in Celery that ensure the state of the world.""" 3 | # :copyright: (c) 2015 Alex Hayes and individual contributors, 4 | # All rights reserved. 5 | # :license: MIT License, see LICENSE for more details. 6 | 7 | 8 | from collections import namedtuple 9 | 10 | version_info_t = namedtuple( 11 | 'version_info_t', ('major', 'minor', 'micro', 'releaselevel', 'serial'), 12 | ) 13 | 14 | VERSION = version_info_t(0, 2, 1, '', '') 15 | __version__ = '{0.major}.{0.minor}.{0.micro}{0.releaselevel}'.format(VERSION) 16 | __author__ = 'Alex Hayes' 17 | __contact__ = 'alex@alution.com' 18 | __homepage__ = 'http://github.com/alexhayes/django-cereal' 19 | __docformat__ = 'restructuredtext' 20 | 21 | # -eof meta- 22 | -------------------------------------------------------------------------------- /django_cereal/pickle.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | django_cereal/pickle.py 4 | ~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | Efficient serializing of Django Models for use in Celery using pickle. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | import pickle 11 | import logging 12 | from contextlib import contextmanager 13 | 14 | from django.db import models 15 | from django.db.utils import OperationalError 16 | 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | __all__ = ['DJANGO_CEREAL_PICKLE', 'patched_model', 'model_encode', 22 | 'model_decode', 'task', 'register_args'] 23 | 24 | DJANGO_CEREAL_PICKLE = 'django_cereal_pickle' 25 | 26 | 27 | def _model_unpickle(cls, data): 28 | """Unpickle a model by retrieving it from the database.""" 29 | auto_field_value = data['pk'] 30 | try: 31 | obj = cls.objects.get(pk=auto_field_value) 32 | except Exception as e: 33 | if isinstance(e, OperationalError): 34 | # Attempt reconnect, we've probably hit; 35 | # OperationalError(2006, 'MySQL server has gone away') 36 | logger.debug("Caught OperationalError, closing database connection.", exc_info=e) 37 | from django.db import connection 38 | connection.close() 39 | obj = cls.objects.get(pk=auto_field_value) 40 | else: 41 | raise 42 | return obj 43 | _model_unpickle.__safe_for_unpickle__ = True 44 | 45 | 46 | def _reduce(self): 47 | cls = self.__class__ 48 | data = {'pk': self.pk} 49 | return (_model_unpickle, (cls, data), data) 50 | 51 | 52 | @contextmanager 53 | def patched_model(): 54 | """Context Manager that safely patches django.db.Model.__reduce__().""" 55 | 56 | patched = ('__reduce__', '__getstate__', '__setstate__') 57 | originals = {} 58 | for patch in patched: 59 | try: 60 | originals[patch] = getattr(models.Model, patch) 61 | except: 62 | pass 63 | 64 | try: 65 | # Patch various parts of the model 66 | models.Model.__reduce__ = _reduce 67 | try: 68 | del models.Model.__getstate__ 69 | except: 70 | pass 71 | try: 72 | del models.Model.__setstate__ 73 | except: # pragma: no cover 74 | pass 75 | 76 | yield 77 | 78 | finally: 79 | # Restore the model 80 | for patch in patched: 81 | try: 82 | setattr(models.Model, patch, originals[patch]) 83 | except KeyError: 84 | try: 85 | delattr(models.Model, patch) 86 | except AttributeError: 87 | pass 88 | 89 | 90 | def model_encode(data): 91 | """Pickle data utilising the patched_model context manager.""" 92 | with patched_model(): 93 | return pickle.dumps(data) 94 | 95 | 96 | def model_decode(data): 97 | """Unpickle data utilising the patched_model context manager.""" 98 | 99 | # Note we must import here to avoid recursion issue with kombu entry points registration 100 | from kombu.serialization import unpickle 101 | 102 | with patched_model(): 103 | return unpickle(data) 104 | 105 | 106 | def task(func, *args, **kwargs): 107 | """ 108 | A task decorator that uses the django-cereal pickler as the default serializer. 109 | """ 110 | 111 | # Note we must import here to avoid recursion issue with kombu entry points registration 112 | from celery import shared_task 113 | 114 | if 'serializer' not in kwargs: 115 | kwargs['serializer'] = DJANGO_CEREAL_PICKLE 116 | return shared_task(func, *args, **kwargs) 117 | 118 | 119 | register_args = (model_encode, model_decode, 120 | 'application/x-python-serialize', 'binary') 121 | -------------------------------------------------------------------------------- /django_cereal/tests/__init__.py: -------------------------------------------------------------------------------- 1 | from .pickle import * 2 | -------------------------------------------------------------------------------- /django_cereal/tests/pickle.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django.db import models 3 | 4 | 5 | 6 | class PickleTestCase(TestCase): 7 | 8 | def test_basic_patch(self): 9 | """Test that patching works as expected.""" 10 | from django_cereal.pickle import model_encode, model_decode 11 | from django_cereal.tests.testapp.models import ModelWithBasicField 12 | 13 | expected = ModelWithBasicField.objects.create(name='foo') 14 | actual = model_decode(model_encode(expected)) 15 | 16 | self.assertGreater(expected.pk, 0) 17 | self.assertEqual(actual.pk, expected.pk) 18 | self.assertEqual(actual.name, 'foo') 19 | 20 | def test_deep_patch(self): 21 | """Test that models contained deep inside a dict are serialized correctly.""" 22 | from django_cereal.pickle import model_encode, model_decode 23 | from django_cereal.tests.testapp.models import ModelWithBasicField 24 | 25 | bar = ModelWithBasicField.objects.create(name='bar') 26 | eggs = ModelWithBasicField.objects.create(name='sausage') 27 | expected = {'foo': {'bar': bar, 28 | 'eggs': eggs}} 29 | actual = model_decode(model_encode(expected)) 30 | 31 | self.assertEqual(actual, expected) 32 | 33 | def test_raises_on_doesnotexist(self): 34 | """Tests that decoding raises DoesNotExist if the item can't be found in the database.""" 35 | from django_cereal.pickle import model_encode, model_decode 36 | from django_cereal.tests.testapp.models import ModelWithBasicField 37 | 38 | dne = ModelWithBasicField(id=1) 39 | encoded = model_encode(dne) 40 | self.assertRaises(ModelWithBasicField.DoesNotExist, model_decode, encoded) 41 | 42 | def test_cleanup(self): 43 | """Test that the serialization cleans up after itself.""" 44 | from django_cereal.pickle import model_encode, model_decode 45 | from django_cereal.tests.testapp.models import ModelWithBasicField 46 | 47 | patched = ('__reduce__', '__setstate__', '__getstate__') 48 | expected = {} 49 | actual = {} 50 | for patch in patched: 51 | try: 52 | expected[patch] = getattr(models.Model, patch) 53 | except: 54 | expected[patch] = None 55 | 56 | m = ModelWithBasicField.objects.create(name='foo') 57 | model_decode(model_encode(m)) 58 | 59 | for patch in patched: 60 | try: 61 | actual[patch] = getattr(models.Model, patch) 62 | except: 63 | actual[patch] = None 64 | 65 | for key, value in expected.items(): 66 | self.assertIn(key, actual, 'Expected key %s to exist in actual' % key) 67 | self.assertEqual(actual[key], value, "Expected %s in actual to be equal to %s not %s." % (key, value, actual[key])) 68 | 69 | def test_inherited_model(self): 70 | """Test that patching an inherited models works as expected.""" 71 | from django_cereal.pickle import model_encode, model_decode 72 | from django_cereal.tests.testapp.models import ModelWithParentModel 73 | 74 | expected = ModelWithParentModel.objects.create(name='foo') 75 | actual = model_decode(model_encode(expected)) 76 | 77 | self.assertGreater(expected.pk, 0) 78 | self.assertEqual(actual.pk, expected.pk) 79 | self.assertEqual(actual.name, 'foo') 80 | -------------------------------------------------------------------------------- /django_cereal/tests/testapp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexhayes/django-cereal/ab5b7f0283c6604c4df658542f7381262e600e5d/django_cereal/tests/testapp/__init__.py -------------------------------------------------------------------------------- /django_cereal/tests/testapp/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class ModelWithBasicField(models.Model): 5 | name = models.CharField(max_length=32) 6 | 7 | def __unicode__(self): 8 | return u'%s' % self.pk 9 | 10 | 11 | class ModelWithParentModel(ModelWithBasicField): 12 | foo = models.BooleanField(default=False) 13 | 14 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==1.8.2 2 | amqp==1.4.6 3 | anyjson==0.3.3 4 | argparse==1.2.1 5 | billiard==3.3.0.20 6 | blessings==1.6 7 | celery==3.1.18 8 | coverage==3.7.1 9 | django-nose==1.4 10 | kombu==3.0.26 11 | nose==1.3.7 12 | nose-progressive==1.5.1 13 | pytz==2015.4 14 | wsgiref==0.1.2 15 | -------------------------------------------------------------------------------- /requirements/README.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | pip requirements files 3 | ======================== 4 | 5 | 6 | Index 7 | ===== 8 | 9 | * :file:`requirements/default.txt` 10 | 11 | Default requirements for Python 2.7+. 12 | 13 | * :file:`requirements/test.txt` 14 | 15 | Requirements needed to run the full unittest suite via ./runtests.py 16 | 17 | * :file:`requirements/pkgutils.txt` 18 | 19 | Extra requirements required to perform package distribution maintenance. 20 | 21 | Examples 22 | ======== 23 | 24 | Installing requirements 25 | ----------------------- 26 | 27 | :: 28 | 29 | $ pip install -U -r requirements/default.txt 30 | 31 | 32 | Running the tests 33 | ----------------- 34 | 35 | :: 36 | 37 | $ pip install -U -r requirements/default.txt 38 | $ pip install -U -r requirements/test.txt 39 | -------------------------------------------------------------------------------- /requirements/default.txt: -------------------------------------------------------------------------------- 1 | celery>=3.1,<3.2 2 | kombu>=3.0.25,<3.1 3 | -------------------------------------------------------------------------------- /requirements/pkgutils.txt: -------------------------------------------------------------------------------- 1 | setuptools>=1.3.2 2 | wheel 3 | tox 4 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | 4 | import django 5 | from django.conf import settings 6 | from django.core.management import execute_from_command_line 7 | 8 | 9 | if not settings.configured: 10 | settings.configure( 11 | INSTALLED_APPS=[ 12 | 'django.contrib.auth', 13 | 'django.contrib.contenttypes', 14 | 'django.contrib.admin', 15 | 'django.contrib.sessions', 16 | 'django_nose', 17 | 'django_cereal.tests.testapp', 18 | ], 19 | # Django replaces this, but it still wants it. *shrugs* 20 | DATABASE_ENGINE='django.db.backends.sqlite3', 21 | DATABASES={ 22 | 'default': { 23 | 'ENGINE': 'django.db.backends.sqlite3', 24 | 'NAME': ':memory:', 25 | } 26 | }, 27 | MIDDLEWARE_CLASSES={}, 28 | ) 29 | 30 | 31 | def runtests(): 32 | argv = sys.argv[:1] + ['test'] + sys.argv[1:] 33 | execute_from_command_line(argv) 34 | 35 | 36 | if __name__ == '__main__': 37 | runtests() 38 | -------------------------------------------------------------------------------- /scripts/git-tools-hooks/git-on-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VERSION=$1 4 | DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 5 | INIT_PATH="$DIR/../../django_cereal/__init__.py" 6 | 7 | echo "### MUNGING VERSION $VERSION INTO $INIT_PATH" 8 | 9 | if [ -z "$VERSION" ]; then 10 | echo "VERSION must be specified!" 11 | exit 1 12 | fi 13 | 14 | PY_VERSION=`echo $VERSION | sed -e 's/\./, /g'` 15 | 16 | # Replace the version number 17 | sed --in-place -e "s/VERSION = version_info_t(.*)/VERSION = version_info_t($PY_VERSION, '', '')/g" $INIT_PATH 18 | 19 | # Ensure we haven't caused some kind of error 20 | python $INIT_PATH 21 | 22 | # Check the last commands return code 23 | rc=$?; if [[ $rc != 0 ]]; then exit $rc; fi 24 | 25 | # We're all good, add it to git for the commit 26 | git add $INIT_PATH 27 | -------------------------------------------------------------------------------- /scripts/removepyc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | (cd "${1:-.}"; 3 | find . -name "*.pyc" | xargs rm -- 2>/dev/null) || echo "ok" 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst 3 | 4 | [wheel] 5 | universal = 1 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | try: 5 | from setuptools import setup, find_packages 6 | from setuptools.command.test import test 7 | is_setuptools = True 8 | except ImportError: 9 | raise 10 | from ez_setup import use_setuptools 11 | use_setuptools() 12 | from setuptools import setup, find_packages # noqa 13 | from setuptools.command.test import test # noqa 14 | is_setuptools = False 15 | 16 | import os 17 | import sys 18 | import codecs 19 | 20 | NAME = 'django-cereal' 21 | entrypoints = {} 22 | extra = {} 23 | 24 | # -*- Classifiers -*- 25 | 26 | classes = """ 27 | Development Status :: 4 - Beta 28 | Framework :: Django 29 | Framework :: Django :: 1.7 30 | Framework :: Django :: 1.8 31 | License :: OSI Approved :: MIT License 32 | Topic :: System :: Distributed Computing 33 | Topic :: Software Development :: Object Brokering 34 | Intended Audience :: Developers 35 | Programming Language :: Python 36 | Programming Language :: Python :: 2 37 | Programming Language :: Python :: 2.6 38 | Programming Language :: Python :: 2.7 39 | Programming Language :: Python :: 3 40 | Programming Language :: Python :: 3.3 41 | Programming Language :: Python :: 3.4 42 | Programming Language :: Python :: Implementation :: CPython 43 | Programming Language :: Python :: Implementation :: PyPy 44 | Operating System :: OS Independent 45 | Operating System :: POSIX 46 | Operating System :: Microsoft :: Windows 47 | Operating System :: MacOS :: MacOS X 48 | """ 49 | classifiers = [s.strip() for s in classes.split('\n') if s] 50 | 51 | # -*- Distribution Meta -*- 52 | 53 | import re 54 | re_meta = re.compile(r'__(\w+?)__\s*=\s*(.*)') 55 | re_vers = re.compile(r'VERSION\s*=.*?\((.*?)\)') 56 | re_doc = re.compile(r'^"""(.+?)"""') 57 | rq = lambda s: s.strip("\"'") 58 | 59 | 60 | def add_default(m): 61 | attr_name, attr_value = m.groups() 62 | return ((attr_name, rq(attr_value)), ) 63 | 64 | 65 | def add_version(m): 66 | v = list(map(rq, m.groups()[0].split(', '))) 67 | return (('VERSION', '.'.join(v[0:3]) + ''.join(v[3:])), ) 68 | 69 | 70 | def add_doc(m): 71 | return (('doc', m.groups()[0]), ) 72 | 73 | pats = {re_meta: add_default, 74 | re_vers: add_version, 75 | re_doc: add_doc} 76 | here = os.path.abspath(os.path.dirname(__file__)) 77 | with open(os.path.join(here, 'django_cereal/__init__.py')) as meta_fh: 78 | meta = {} 79 | for line in meta_fh: 80 | if line.strip() == '# -eof meta-': 81 | break 82 | for pattern, handler in pats.items(): 83 | m = pattern.match(line.strip()) 84 | if m: 85 | meta.update(handler(m)) 86 | 87 | # -*- Installation Requires -*- 88 | 89 | py_version = sys.version_info 90 | 91 | 92 | def strip_comments(l): 93 | return l.split('#', 1)[0].strip() 94 | 95 | 96 | def reqs(*f): 97 | return [ 98 | r for r in ( 99 | strip_comments(l) for l in open( 100 | os.path.join(os.getcwd(), 'requirements', *f)).readlines() 101 | ) if r] 102 | 103 | install_requires = reqs('default.txt') 104 | 105 | # -*- Tests Requires -*- 106 | 107 | tests_require = reqs('test.txt') 108 | 109 | # -*- Long Description -*- 110 | 111 | if os.path.exists('README.rst'): 112 | long_description = codecs.open('README.rst', 'r', 'utf-8').read() 113 | else: 114 | long_description = 'See http://pypi.python.org/pypi/django-cereal' 115 | 116 | # -*- Entry Points -*- # 117 | 118 | entrypoints['kombu.serializers'] = ['django_cereal_pickle = django_cereal.pickle:register_args'] 119 | 120 | # -*- %%% -*- 121 | 122 | setup( 123 | name=NAME, 124 | version=meta['VERSION'], 125 | description=meta['doc'], 126 | author=meta['author'], 127 | author_email=meta['contact'], 128 | url=meta['homepage'], 129 | platforms=['any'], 130 | license='MIT', 131 | packages=find_packages(exclude=['tests', 'tests.*']), 132 | zip_safe=False, 133 | install_requires=install_requires, 134 | tests_require=tests_require, 135 | test_suite='nose.collector', 136 | classifiers=classifiers, 137 | entry_points=entrypoints, 138 | long_description=long_description) 139 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{27,33,34,py}-django17 4 | py{27,33,34,35,py}-django18 5 | py{27,34,35,py}-django19 6 | 7 | [testenv] 8 | sitepackages = False 9 | commands = {toxinidir}/scripts/removepyc.sh {toxinidir} 10 | {toxinidir}/runtests.py 11 | setenv = C_DEBUG_TEST = 1 12 | PIP_DOWNLOAD_CACHE=~/.pip-cache 13 | deps = 14 | -r{toxinidir}/requirements/default.txt 15 | django19: Django==1.9 16 | django18: Django==1.8.5 17 | django17: Django==1.7.9 18 | py{27,33,34,35,py}: -r{toxinidir}/requirements/test.txt 19 | --------------------------------------------------------------------------------