├── VERSION ├── django_migration_fixture ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── create_initial_data_fixtures.py └── __init__.py ├── requirements ├── default.txt ├── pkgutils.txt ├── test.txt ├── test3.txt └── README.rst ├── setup.cfg ├── MANIFEST.in ├── CHANGELOG.rst ├── .gitignore ├── LICENSE ├── setup.py └── README.rst /VERSION: -------------------------------------------------------------------------------- 1 | 0.5.1 -------------------------------------------------------------------------------- /django_migration_fixture/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements/default.txt: -------------------------------------------------------------------------------- 1 | Django>=1.7,<1.9 2 | -------------------------------------------------------------------------------- /django_migration_fixture/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements/pkgutils.txt: -------------------------------------------------------------------------------- 1 | setuptools>=1.3.2 2 | wheel 3 | tox 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst 3 | 4 | [wheel] 5 | universal = 1 6 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | nose==1.3.7 2 | nose-progressive==1.5.1 3 | git+https://github.com/alexhayes/django-nose.git 4 | coverage==3.7.1 -------------------------------------------------------------------------------- /requirements/test3.txt: -------------------------------------------------------------------------------- 1 | nose==1.3.7 2 | nose-progressive==1.5.1 3 | git+https://github.com/alexhayes/django-nose.git 4 | coverage==3.7.1 -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGELOG.md 2 | include LICENSE 3 | include README.rst 4 | include MANIFEST.in 5 | include setup.cfg 6 | include setup.py 7 | recursive-include django_migration_fixture *.py 8 | recursive-include docs * 9 | recursive-include requirements *.txt 10 | prune *.pyc 11 | prune *.sw* 12 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | CHANGELOG 2 | ========= 3 | 4 | 0.5.0 5 | ----- 6 | 7 | - Ensured that the app registry used for deserialization is that provided by the 8 | migration, not the current app registry which may not match the current state 9 | of the database or the fixture itself. 10 | - Changed README to rst format. 11 | - Refactor setup.py. 12 | 13 | 0.4.0 14 | ----- 15 | 16 | - Python 3 support 17 | - Support non reversible migrations 18 | 19 | -------------------------------------------------------------------------------- /requirements/README.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | pip requirements files 3 | ======================== 4 | 5 | 6 | Index 7 | ===== 8 | 9 | * :file:`requirements/default.txt` 10 | 11 | Default requirements for Python 2.7+. 12 | 13 | * :file:`requirements/test.txt` 14 | 15 | Requirements needed to run the full unittest suite via ./runtests.py 16 | 17 | * :file:`requirements/pkgutils.txt` 18 | 19 | Extra requirements required to perform package distribution maintenance. 20 | 21 | Examples 22 | ======== 23 | 24 | Installing requirements 25 | ----------------------- 26 | 27 | :: 28 | 29 | $ pip install -U -r requirements/default.txt 30 | 31 | 32 | Running the tests 33 | ----------------- 34 | 35 | :: 36 | 37 | $ pip install -U -r requirements/default.txt 38 | $ pip install -U -r requirements/test.txt 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | 56 | # Eclipse 57 | .project 58 | .pydevproject 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Alex Hayes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | try: 5 | from setuptools import setup, find_packages 6 | from setuptools.command.test import test 7 | is_setuptools = True 8 | except ImportError: 9 | raise 10 | from ez_setup import use_setuptools 11 | use_setuptools() 12 | from setuptools import setup, find_packages # noqa 13 | from setuptools.command.test import test # noqa 14 | is_setuptools = False 15 | 16 | import os 17 | import sys 18 | import codecs 19 | 20 | NAME = 'django-migration-fixture' 21 | extra = {} 22 | 23 | # -*- Classifiers -*- 24 | 25 | classes = """ 26 | Development Status :: 4 - Beta 27 | Framework :: Django 28 | Framework :: Django :: 1.7 29 | Framework :: Django :: 1.8 30 | License :: OSI Approved :: MIT License 31 | Topic :: Database 32 | Topic :: Software Development :: Code Generators 33 | Intended Audience :: Developers 34 | Programming Language :: Python 35 | Programming Language :: Python :: 2 36 | Programming Language :: Python :: 2.7 37 | Programming Language :: Python :: 3 38 | Programming Language :: Python :: 3.3 39 | Programming Language :: Python :: 3.4 40 | Programming Language :: Python :: Implementation :: CPython 41 | Operating System :: OS Independent 42 | Operating System :: POSIX 43 | Operating System :: Microsoft :: Windows 44 | Operating System :: MacOS :: MacOS X 45 | """ 46 | classifiers = [s.strip() for s in classes.split('\n') if s] 47 | 48 | # -*- Distribution Meta -*- 49 | 50 | import re 51 | re_meta = re.compile(r'__(\w+?)__\s*=\s*(.*)') 52 | re_vers = re.compile(r'VERSION\s*=.*?\((.*?)\)') 53 | re_doc = re.compile(r'^"""(.+?)"""') 54 | rq = lambda s: s.strip("\"'") 55 | 56 | def add_default(m): 57 | attr_name, attr_value = m.groups() 58 | return ((attr_name, rq(attr_value)), ) 59 | 60 | 61 | def add_version(m): 62 | v = list(map(rq, m.groups()[0].split(', '))) 63 | return (('VERSION', '.'.join(v[0:3]) + ''.join(v[3:])), ) 64 | 65 | 66 | def add_doc(m): 67 | return (('doc', m.groups()[0]), ) 68 | 69 | pats = {re_meta: add_default, 70 | re_vers: add_version, 71 | re_doc: add_doc} 72 | here = os.path.abspath(os.path.dirname(__file__)) 73 | with open(os.path.join(here, 'django_migration_fixture/__init__.py')) as meta_fh: 74 | meta = {} 75 | for line in meta_fh: 76 | if line.strip() == '# -eof meta-': 77 | break 78 | for pattern, handler in pats.items(): 79 | m = pattern.match(line.strip()) 80 | if m: 81 | meta.update(handler(m)) 82 | 83 | # -*- Installation Requires -*- 84 | 85 | py_version = sys.version_info 86 | 87 | 88 | def strip_comments(l): 89 | return l.split('#', 1)[0].strip() 90 | 91 | 92 | def reqs(*f): 93 | return [ 94 | r for r in ( 95 | strip_comments(l) for l in open( 96 | os.path.join(os.getcwd(), 'requirements', *f)).readlines() 97 | ) if r] 98 | 99 | install_requires = reqs('default.txt') 100 | 101 | # -*- Tests Requires -*- 102 | 103 | tests_require = reqs('test.txt') 104 | 105 | # -*- Long Description -*- 106 | 107 | if os.path.exists('README.rst'): 108 | long_description = codecs.open('README.rst', 'r', 'utf-8').read() 109 | else: 110 | long_description = 'See http://pypi.python.org/pypi/django-migration-fixture' 111 | 112 | # -*- %%% -*- 113 | 114 | setup( 115 | name=NAME, 116 | version=meta['VERSION'], 117 | description=meta['doc'], 118 | author=meta['author'], 119 | author_email=meta['contact'], 120 | url=meta['homepage'], 121 | platforms=['any'], 122 | license='MIT', 123 | packages=find_packages(exclude=['tests', 'tests.*', 'scripts']), 124 | zip_safe=False, 125 | install_requires=install_requires, 126 | keywords=['django', 'migrations', 'initial data', 'fixtures'], 127 | classifiers=classifiers, 128 | long_description=long_description) 129 | 130 | -------------------------------------------------------------------------------- /django_migration_fixture/management/commands/create_initial_data_fixtures.py: -------------------------------------------------------------------------------- 1 | try: 2 | from StringIO import StringIO 3 | except ImportError: 4 | from io import StringIO 5 | import glob 6 | import os 7 | import re 8 | 9 | from django.apps import apps 10 | from django.core import management 11 | from django.core.management import BaseCommand 12 | from django.core.management.base import CommandError 13 | from django.db.migrations import writer 14 | 15 | 16 | class Command(BaseCommand): 17 | help = "Locate initial_data.* files and create data migrations for them." 18 | 19 | def handle(self, *args, **options): 20 | # Loop through all apps 21 | for app in apps.get_app_configs(): 22 | # Look for initial_data.* files 23 | for fixture_path in glob.glob(os.path.join(app.path, 'fixtures', 'initial_data.*')): 24 | # Ensure the app has at least an initial migration 25 | if not glob.glob(os.path.join(app.path, 'migrations', '0001*')): 26 | self.stdout.write(self.style.MIGRATE_HEADING("Migrations for '%s':" % app.label) + "\n") 27 | self.stdout.write(" %s\n" % (self.style.WARNING("Ignoring '%s' - not migrated." % os.path.basename(fixture_path)),)) 28 | continue 29 | 30 | if self.migration_exists(app, fixture_path): 31 | self.stdout.write(self.style.MIGRATE_HEADING("Migrations for '%s':" % app.label) + "\n") 32 | self.stdout.write(" %s\n" % (self.style.NOTICE("Ignoring '%s' - migration already exists." % os.path.basename(fixture_path)),)) 33 | continue 34 | 35 | # Finally create our data migration 36 | self.create_migration(app, fixture_path) 37 | 38 | def monkey_patch_migration_template(self, app, fixture_path): 39 | """ 40 | Monkey patch the django.db.migrations.writer.MIGRATION_TEMPLATE 41 | 42 | Monkey patching django.db.migrations.writer.MIGRATION_TEMPLATE means that we 43 | don't have to do any complex regex or reflection. 44 | 45 | It's hacky... but works atm. 46 | """ 47 | self._MIGRATION_TEMPLATE = writer.MIGRATION_TEMPLATE 48 | module_split = app.module.__name__.split('.') 49 | 50 | if len(module_split) == 1: 51 | module_import = "import %s\n" % module_split[0] 52 | else: 53 | module_import = "from %s import %s\n" % ( 54 | '.'.join(module_split[:-1]), 55 | module_split[-1:][0], 56 | ) 57 | 58 | writer.MIGRATION_TEMPLATE = writer.MIGRATION_TEMPLATE\ 59 | .replace( 60 | '%(imports)s', 61 | "%(imports)s" + "\nfrom django_migration_fixture import fixture\n%s" % module_import 62 | )\ 63 | .replace( 64 | '%(operations)s', 65 | " migrations.RunPython(**fixture(%s, ['%s'])),\n" % ( 66 | app.label, 67 | os.path.basename(fixture_path) 68 | ) + "%(operations)s\n" 69 | ) 70 | 71 | def restore_migration_template(self): 72 | """ 73 | Restore the migration template. 74 | """ 75 | writer.MIGRATION_TEMPLATE = self._MIGRATION_TEMPLATE 76 | 77 | def migration_exists(self, app, fixture_path): 78 | """ 79 | Return true if it looks like a migration already exists. 80 | """ 81 | base_name = os.path.basename(fixture_path) 82 | # Loop through all migrations 83 | for migration_path in glob.glob(os.path.join(app.path, 'migrations', '*.py')): 84 | if base_name in open(migration_path).read(): 85 | return True 86 | return False 87 | 88 | def create_migration(self, app, fixture_path): 89 | """ 90 | Create a data migration for app that uses fixture_path. 91 | """ 92 | self.monkey_patch_migration_template(app, fixture_path) 93 | 94 | out = StringIO() 95 | management.call_command('makemigrations', app.label, empty=True, stdout=out) 96 | 97 | self.restore_migration_template() 98 | 99 | self.stdout.write(out.getvalue()) 100 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | django-migration-fixture 2 | ======================== 3 | 4 | Easily use fixtures in Django 1.7+ data migrations. 5 | 6 | The app also contains a management command to automatically convert 7 | :code:`initial_data.*` into migrations. 8 | 9 | Prior to Django 1.7 :code:`initial_data.*` files where automatically run when the 10 | management command :code:`sync_db` was run, however this behaviour was 11 | discontinued in Django 1.7. Thus, this app is useful if you rely on this 12 | behaviour. 13 | 14 | Essentially it leaves the :code:`initial_data.*` file in place and generates a 15 | data migration - as outlined 16 | `in the docs `_. 17 | 18 | Install 19 | ------- 20 | 21 | .. code-block:: python 22 | 23 | pip install django-migration-fixture 24 | 25 | Then add :code:`django_migration_fixture` to your :code:`INSTALLED_APPS`. 26 | 27 | .. code-block:: python 28 | 29 | INSTALLED_APPS += ( 30 | 'django_migration_fixture', 31 | ) 32 | 33 | Usage 34 | ----- 35 | 36 | To automatically change your old-skool :code:`initial_data.*` files the simplest 37 | method is to simply call the :code:`create_initial_data_fixtures` management 38 | command. 39 | 40 | .. code-block:: bash 41 | 42 | ./manage.py create_initial_data_fixtures 43 | 44 | The management command will automatically look for :code:`initial_data.*` files 45 | in your list of :code:`INSTALLED_APPS` and for each file found creates a data 46 | migration, similar to the following; 47 | 48 | .. code-block:: python 49 | 50 | # -*- coding: utf-8 -*- 51 | from __future__ import unicode_literals 52 | from django.db import models, migrations 53 | from django_migration_fixture import fixture 54 | import myapp 55 | 56 | class Migration(migrations.Migration): 57 | 58 | operations = [ 59 | migrations.RunPython(**fixture(myapp, 'initial_data.yaml')) 60 | ] 61 | 62 | From this point it's just a matter of running `migrate` to apply your data 63 | migrations. 64 | 65 | Note that this solution also supports rolling back your migration by deleting 66 | using primary key. If your migration should not be reversible then you can pass 67 | `reversible=False` to `fixture()`. 68 | 69 | You can use this app for any fixtures, they don't have to be your initial_data 70 | files. Simply create a empty migration and add a call to 71 | :code:`migrations.RunPython(**fixture(myapp, 'foo.yaml'))`. 72 | 73 | API Documentation 74 | ----------------- 75 | 76 | :code:`fixture(app, fixtures, fixtures_dir='fixtures', raise_does_not_exist=False, reversible=True)` 77 | 78 | - *app* is a Django app that contains your fixtures 79 | - *fixtures* can take either a string or a list of fixture files. The extension 80 | is used as the format supplied to :code:`django.core.serializers.deserialize` 81 | - *fixtures_dir* is the directory inside your app that contains the fixtures 82 | - *ignore_does_not_exist* - if set to True then 83 | :code:`django_migration_fixture.FixtureObjectDoesNotExist` is raised if when 84 | attempting a rollback the object in the fixture does not exist. 85 | - *reversible* - if set to :code:`False` then any attempt to reverse the 86 | migration will raise :code:`django.db.migrations.migration.IrreversibleError`. 87 | 88 | Essentially :code:`fixture()` returns a dict containing the keys :code:`code` 89 | and :code:`reverse_code` which attempt to apply your fixture and rollback your 90 | fixture, respectively. 91 | 92 | Inspiration 93 | ----------- 94 | 95 | While attempting to migrate a large Django project to 1.7 I came across an issue 96 | which caused me to create Django `ticket 24023 `_. 97 | 98 | The project has a lot of fixtures that ensure a baseline state and converting 99 | them to code isn't really ideal, thus this project. 100 | 101 | That issue has since been closed as a duplicate of 102 | `ticket 23699 `_ which itself has 103 | been closed and released in Django 1.7.2. 104 | 105 | Needless to say, you may still need to create data migrations for fixtures 106 | regardless of the issue I came across. 107 | 108 | Author 109 | ------ 110 | 111 | Alex Hayes 112 | -------------------------------------------------------------------------------- /django_migration_fixture/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Easily use fixtures in Django 1.7+ data migrations.""" 3 | # :copyright: (c) 2015 Alex Hayes and individual contributors, 4 | # All rights reserved. 5 | # :license: MIT License, see LICENSE for more details. 6 | 7 | from collections import namedtuple 8 | 9 | version_info_t = namedtuple( 10 | 'version_info_t', ('major', 'minor', 'micro', 'releaselevel', 'serial'), 11 | ) 12 | 13 | VERSION = version_info_t(0, 5, 1, '', '') 14 | __version__ = '{0.major}.{0.minor}.{0.micro}{0.releaselevel}'.format(VERSION) 15 | __author__ = 'Alex Hayes' 16 | __contact__ = 'alex@alution.com' 17 | __homepage__ = 'http://github.com/alexhayes/django-migration-fixture' 18 | __docformat__ = 'restructuredtext' 19 | 20 | # -eof meta- 21 | 22 | from contextlib import contextmanager 23 | import os 24 | import django 25 | from django.core import serializers 26 | from six import string_types 27 | from functools import wraps 28 | 29 | 30 | class FixtureObjectDoesNotExist(Exception): 31 | """ 32 | Raised if when attempting to roll back a fixture the instance can't be found 33 | """ 34 | pass 35 | 36 | 37 | def fixture(app, fixtures, fixtures_dir='fixtures', raise_does_not_exist=False, 38 | reversible=True, models=[]): 39 | """ 40 | Load fixtures using a data migration. 41 | 42 | The migration will by default provide a rollback, deleting items by primary 43 | key. This is not always what you want ; you may set reversible=False to 44 | prevent rolling back. 45 | 46 | Usage: 47 | 48 | import myapp 49 | import anotherapp 50 | 51 | operations = [ 52 | migrations.RunPython(**fixture(myapp, 'eggs.yaml')), 53 | migrations.RunPython(**fixture(anotherapp, ['sausage.json', 'walks.yaml'])) 54 | migrations.RunPython(**fixture(yap, ['foo.json'], reversible=False)) 55 | ] 56 | """ 57 | fixture_path = os.path.join(app.__path__[0], fixtures_dir) 58 | if isinstance(fixtures, string_types): 59 | fixtures = [fixtures] 60 | 61 | def get_format(fixture): 62 | return os.path.splitext(fixture)[1][1:] 63 | 64 | def get_objects(): 65 | for fixture in fixtures: 66 | with open(os.path.join(fixture_path, fixture), 'rb') as f: 67 | objects = serializers.deserialize(get_format(fixture), 68 | f, 69 | ignorenonexistent=True) 70 | for obj in objects: 71 | yield obj 72 | 73 | def patch_apps(func): 74 | """ 75 | Patch the app registry. 76 | 77 | Note that this is necessary so that the Deserializer does not use the 78 | current version of the model, which may not necessarily be representative 79 | of the model the fixture was created for. 80 | """ 81 | @wraps(func) 82 | def inner(apps, schema_editor): 83 | try: 84 | # Firstly patch the serializers registry 85 | original_apps = django.core.serializers.python.apps 86 | django.core.serializers.python.apps = apps 87 | return func(apps, schema_editor) 88 | 89 | finally: 90 | # Ensure we always unpatch the serializers registry 91 | django.core.serializers.python.apps = original_apps 92 | 93 | return inner 94 | 95 | @patch_apps 96 | def load_fixture(apps, schema_editor): 97 | for obj in get_objects(): 98 | obj.save() 99 | 100 | @patch_apps 101 | def unload_fixture(apps, schema_editor): 102 | for obj in get_objects(): 103 | model = apps.get_model(app.__name__, obj.object.__class__.__name__) 104 | kwargs = dict() 105 | if 'id' in obj.object.__dict__: 106 | kwargs.update(id=obj.object.__dict__.get('id')) 107 | elif 'slug' in obj.object.__dict__: 108 | kwargs.update(slug=obj.object.__dict__.get('slug')) 109 | else: 110 | kwargs.update(**obj.object.__dict__) 111 | try: 112 | model.objects.get(**kwargs).delete() 113 | except model.DoesNotExist: 114 | if not raise_does_not_exist: 115 | raise FixtureObjectDoesNotExist(("Model %s instance with " 116 | "kwargs %s does not exist." 117 | % (model, kwargs))) 118 | 119 | kwargs = dict(code=load_fixture) 120 | 121 | if reversible: 122 | kwargs['reverse_code'] = unload_fixture 123 | 124 | return kwargs 125 | --------------------------------------------------------------------------------