├── .gitignore ├── .travis.yml ├── CHANGELOG ├── LICENSE ├── MANIFEST.in ├── README.rst ├── bench.ini ├── django_pickling.py ├── pytest.ini ├── setup.py ├── test_requirements.txt ├── tests ├── __init__.py ├── bench.py ├── django_settings.py ├── manage.py ├── migrations │ ├── 0001_initial.py │ └── __init__.py ├── models.py ├── settings.py └── test_db.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | dist 3 | *.egg-info 4 | build 5 | .cache 6 | .tox 7 | .benchmarks 8 | sqlite.db 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | cache: pip 4 | python: 5 | - "2.7" 6 | - "3.3" 7 | - "3.4" 8 | - "3.5" 9 | - "3.6" 10 | install: 11 | - pip install tox-travis 12 | script: 13 | - tox 14 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | 1.0 2 | - support Django 1.10 and 1.11 3 | - support Python 3, really 4 | - more optimizations on size and time 5 | - fixed pickling dynamic models like m2m through 6 | - do not break on fields changing positions 7 | - no django version check warning 8 | 9 | 0.2 10 | - support Python 3 11 | 12 | 0.1 13 | - initial version 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2017, Alexander Schepanovski. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of django-pickling nor the names of its contributors may 15 | be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include CHANGELOG 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Django pickling 2 | =============== 3 | 4 | Makes django models pickling 2-3 times faster and compact. 5 | 6 | 7 | Requirements 8 | ------------ 9 | 10 | | Python 2.7 or 3.3+, Django 1.8+ 11 | 12 | 13 | Installation and setup 14 | ---------------------- 15 | 16 | $ pip install django-pickling 17 | 18 | Then add ``django_pickling`` to your ``INSTALLED_APPS``. 19 | 20 | 21 | CAVEATS 22 | ------- 23 | 24 | 1. No Django version checks are performed. 25 | 2. If fields list changes you will see TypeErrors instead of AttributeErrors. 26 | 27 | In both cases you should wipe your cache or change keys. 28 | Note that you will need to deal with this anyway, 29 | with django-pickling you'll just get weirder errors. 30 | 31 | Another thing is that objects with deferred fields are not optimized. 32 | -------------------------------------------------------------------------------- /bench.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | python_paths = . 3 | DJANGO_SETTINGS_MODULE=tests.settings 4 | python_files = bench.py 5 | python_classes = Bench 6 | python_functions = bench_ 7 | addopts = --benchmark-group-by=func 8 | -------------------------------------------------------------------------------- /django_pickling.py: -------------------------------------------------------------------------------- 1 | VERSION = (1, 0) 2 | __version__ = '.'.join(map(str, VERSION)) 3 | 4 | 5 | from django.apps import apps 6 | from django.conf import settings 7 | from django.db.models import Model 8 | from django.db.models.base import ModelState 9 | try: 10 | from itertools import izip 11 | except ImportError: 12 | izip = zip 13 | 14 | 15 | def attnames(cls, _cache={}): 16 | try: 17 | return _cache[cls] 18 | except KeyError: 19 | _cache[cls] = tuple(sorted(f.attname for f in cls._meta.fields)) 20 | return _cache[cls] 21 | 22 | 23 | def model_unpickle(model, vector, db, adding, _cache={}): 24 | try: 25 | cls = _cache[model] 26 | except KeyError: 27 | # Only needed in Django 1.8 and 1.9 28 | if not apps.ready: 29 | apps.populate(settings.INSTALLED_APPS) 30 | cls = _cache[model] = apps.get_model(*model.split('.')) 31 | obj = cls.__new__(cls) 32 | obj.__dict__.update(izip(attnames(cls), vector)) 33 | 34 | # Restore state. This is the fastest way to create object I know. 35 | obj._state = ModelState.__new__(ModelState) 36 | obj._state.__dict__ = {'db': db, 'adding': adding} 37 | 38 | return obj 39 | model_unpickle.__safe_for_unpickle__ = True 40 | 41 | 42 | def Model__reduce__(self): 43 | cls = self.__class__ 44 | opts = cls._meta 45 | # We do not pickle class but its identifier to work with dynamic models like m2m through ones 46 | # We use concat instead of tuple to spead up loads at an expense of dumps 47 | # This is the fastest way to concat here, formats are slower 48 | model = opts.app_label + '.' + opts.object_name 49 | data = self.__dict__.copy() 50 | state = data.pop('_state') 51 | try: 52 | # Popping all known attributes into vector, leaving the rest in data 53 | vector = tuple(data.pop(name) for name in attnames(cls)) 54 | return (model_unpickle, (model, vector, state.db, state.adding), data) 55 | except KeyError: 56 | # data.pop() raises when some attnames are deferred 57 | return original_Model__reduce__(self) 58 | 59 | 60 | if Model.__reduce__ != Model__reduce__: 61 | original_Model__reduce__ = Model.__reduce__ 62 | Model.__reduce__ = Model__reduce__ 63 | del Model.__setstate__ # Drop django version check 64 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | python_paths = . 3 | DJANGO_SETTINGS_MODULE=tests.settings 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='django-pickling', 5 | version='1.0', 6 | author='Alexander Schepanovski', 7 | author_email='suor.web@gmail.com', 8 | 9 | description='Efficient pickling for django models.', 10 | long_description=open('README.rst').read(), 11 | url='http://github.com/Suor/django-pickling', 12 | license='BSD', 13 | 14 | py_modules=['django_pickling'], 15 | install_requires=['django'], 16 | 17 | classifiers=[ 18 | 'Development Status :: 5 - Production/Stable', 19 | 'License :: OSI Approved :: BSD License', 20 | 'Operating System :: OS Independent', 21 | 22 | 'Programming Language :: Python', 23 | 'Programming Language :: Python :: 2', 24 | 'Programming Language :: Python :: 2.7', 25 | 'Programming Language :: Python :: 3', 26 | 'Programming Language :: Python :: 3.3', 27 | 'Programming Language :: Python :: 3.4', 28 | 'Programming Language :: Python :: 3.5', 29 | 'Programming Language :: Python :: 3.6', 30 | 'Framework :: Django', 31 | 'Framework :: Django :: 1.8', 32 | 'Framework :: Django :: 1.9', 33 | 'Framework :: Django :: 1.10', 34 | 'Framework :: Django :: 1.11', 35 | 36 | 'Framework :: Django', 37 | 'Environment :: Web Environment', 38 | 'Intended Audience :: Developers', 39 | 'Topic :: Software Development :: Libraries :: Python Modules', 40 | ] 41 | ) 42 | -------------------------------------------------------------------------------- /test_requirements.txt: -------------------------------------------------------------------------------- 1 | Django>=1.8 2 | pytest>=3.0.7 3 | pytest-django>=3.1.2 4 | pytest-pythonpath>=0.7.1 5 | pytest-warnings 6 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Suor/django-pickling/8ff02731246396f7d3ba043b8a1bd6d98c002b96/tests/__init__.py -------------------------------------------------------------------------------- /tests/bench.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | pytestmark = pytest.mark.django_db 3 | 4 | from .test_db import pickle, post 5 | 6 | 7 | def bench_dumps(benchmark, post): 8 | benchmark(pickle.dumps, post, -1) 9 | 10 | 11 | def bench_loads(benchmark, post): 12 | stored = pickle.dumps(post, -1) 13 | benchmark(pickle.loads, stored) 14 | -------------------------------------------------------------------------------- /tests/django_settings.py: -------------------------------------------------------------------------------- 1 | # This is a settings file to run Django tests with django-pickling in effect 2 | print('Installing django-pickling') 3 | import django_pickling 4 | 5 | DATABASES = { 6 | 'default': { 7 | 'ENGINE': 'django.db.backends.sqlite3', 8 | }, 9 | 'other': { 10 | 'ENGINE': 'django.db.backends.sqlite3', 11 | } 12 | } 13 | 14 | SECRET_KEY = "django_tests_secret_key" 15 | 16 | # Use a fast hasher to speed up tests. 17 | PASSWORD_HASHERS = [ 18 | 'django.contrib.auth.hashers.MD5PasswordHasher', 19 | ] 20 | -------------------------------------------------------------------------------- /tests/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | sys.path.insert(0, '.') 6 | 7 | 8 | if __name__ == "__main__": 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") 10 | 11 | from django.core.management import execute_from_command_line 12 | 13 | execute_from_command_line(sys.argv) 14 | -------------------------------------------------------------------------------- /tests/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.1 on 2017-05-21 04:38 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='Post', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('title', models.CharField(max_length=127)), 21 | ], 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /tests/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Suor/django-pickling/8ff02731246396f7d3ba043b8a1bd6d98c002b96/tests/migrations/__init__.py -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class Post(models.Model): 5 | title = models.CharField(max_length=127) 6 | 7 | def __str__(self): 8 | return 'id=%s title=%s' % (self.pk, self.title) 9 | -------------------------------------------------------------------------------- /tests/settings.py: -------------------------------------------------------------------------------- 1 | INSTALLED_APPS = [ 2 | 'django_pickling', 3 | 'tests', 4 | ] 5 | 6 | DATABASES = { 7 | 'default': { 8 | 'ENGINE': 'django.db.backends.sqlite3', 9 | 'NAME': 'sqlite.db', 10 | 'USER': '', 11 | 'PASSWORD': '', 12 | 'HOST': '', 13 | 'PORT': '', 14 | } 15 | } 16 | 17 | SECRET_KEY = 'abc' 18 | -------------------------------------------------------------------------------- /tests/test_db.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | try: 3 | import cPickle as pickle 4 | except ImportError: 5 | import pickle 6 | 7 | from .models import Post 8 | 9 | 10 | pytestmark = pytest.mark.django_db 11 | 12 | 13 | @pytest.mark.django_db 14 | @pytest.fixture 15 | def post(): 16 | return Post.objects.create(title='Pickling') 17 | 18 | 19 | def test_equal(post): 20 | restored = pickle.loads(pickle.dumps(post, -1)) 21 | assert restored == post 22 | 23 | 24 | def test_packed(post): 25 | stored = pickle.dumps(post) 26 | assert b'model_unpickle' in stored # Our unpickling function is used 27 | assert b'title' not in stored # Attributes are packed 28 | 29 | 30 | def test_state_packed(post): 31 | stored = pickle.dumps(post, -1) 32 | assert b'_state' not in stored 33 | assert b'db' not in stored 34 | assert b'adding' not in stored 35 | 36 | 37 | def test_deferred(post): 38 | p = Post.objects.defer('title').get(pk=post.pk) 39 | restored = pickle.loads(pickle.dumps(p, -1)) 40 | assert restored == p 41 | 42 | 43 | def test_field_swap(post): 44 | stored = pickle.dumps(post, -1) 45 | Post._meta.fields = Post._meta.fields[::-1] 46 | # Drop attnames cache 47 | from django_pickling import attnames 48 | attnames.__defaults__[0].pop(Post) 49 | 50 | assert pickle.loads(stored) == post 51 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | minversion = 2.7 3 | envlist = 4 | py27-dj{18,19,110,111}, 5 | py33-dj18, 6 | py34-dj{18,19,110,111}, 7 | py35-dj{18,19,110,111}, 8 | py36-dj111 9 | 10 | [testenv] 11 | deps = 12 | dj18: Django>=1.8,<1.9 13 | dj19: Django>=1.9,<1.10 14 | dj110: Django>=1.10,<1.11 15 | dj111: Django>=1.11,<1.12 16 | pytest 17 | pytest-django 18 | pytest-pythonpath 19 | pytest-warnings 20 | commands = py.test -W error 21 | --------------------------------------------------------------------------------