├── .coveragerc ├── .github └── workflows │ └── django.yml ├── .gitignore ├── CHANGELOG ├── README.rst ├── dbdiff ├── __init__.py ├── apps.py ├── exceptions.py ├── fixture.py ├── fixtures │ ├── dbdiff_test_group.json │ └── empty.json ├── plugin.py ├── sequence.py ├── serializers │ ├── __init__.py │ ├── base.py │ └── json.py ├── test.py ├── tests │ ├── __init__.py │ ├── decimal_test │ │ ├── __init__.py │ │ ├── expected.json │ │ ├── expected_no_reminder.json │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ ├── 0002_auto_20160102_0914.py │ │ │ └── __init__.py │ │ └── models.py │ ├── inheritance │ │ ├── __init__.py │ │ └── models.py │ ├── nonintpk │ │ ├── __init__.py │ │ └── models.py │ ├── project │ │ ├── __init__.py │ │ ├── settings.py │ │ ├── settings_mysql.py │ │ ├── settings_postgresql.py │ │ ├── settings_sqlite.py │ │ └── urls.py │ ├── test_compare.py │ ├── test_decimal.py │ ├── test_fixture.py │ ├── test_mixin.json │ ├── test_mixin.py │ ├── test_plugin.py │ └── test_utils.py └── utils.py ├── setup.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | omit = 3 | dbdiff/tests/project/* 4 | -------------------------------------------------------------------------------- /.github/workflows/django.yml: -------------------------------------------------------------------------------- 1 | name: Python package 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | python-version: 17 | - '3.8' 18 | - '3.9' 19 | - '3.10' 20 | - '3.11' 21 | - '3.12' 22 | services: 23 | postgres: 24 | image: postgres:14-alpine 25 | env: 26 | POSTGRES_USER: postgres 27 | POSTGRES_DB: postgres 28 | POSTGRES_PASSWORD: dbdiff 29 | ports: 30 | - 5432:5432 31 | options: >- 32 | --health-cmd pg_isready 33 | --health-interval 10s 34 | --health-timeout 5s 35 | --health-retries 5 36 | mysql: 37 | image: mysql:latest 38 | ports: 39 | - 3306:3306 40 | env: 41 | MYSQL_ROOT_PASSWORD: dbdiff 42 | steps: 43 | - uses: actions/checkout@v1 44 | - name: Set up Python ${{ matrix.python-version }} 45 | uses: actions/setup-python@v4 46 | with: 47 | python-version: ${{ matrix.python-version }} 48 | - name: Install dependencies 49 | run: | 50 | python -m pip install --upgrade pip 51 | pip install tox tox-gh-actions codecov 52 | - name: Test with tox 53 | run: tox -v 54 | env: 55 | DB_HOST: 127.0.0.1 56 | DB_PASSWORD: dbdiff 57 | PGPASSWORD: dbdiff 58 | - name: Codecov 59 | run: codecov 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .venv 3 | .Python 4 | *.pyc 5 | *.orig 6 | .cache/ 7 | *,cover 8 | .coverage 9 | .tox/ 10 | .idea/ 11 | *.sqlite 12 | 13 | *.egg-info 14 | build/ 15 | dist/ 16 | venv/ 17 | *.egg/ 18 | src/cities_light/version.py 19 | 20 | # Ignore the downloaded data. 21 | src/cities_light/data/ 22 | 23 | # vscode & codespaces 24 | .vscode/* 25 | !.vscode/settings.json 26 | !.vscode/tasks.json 27 | !.vscode/launch.json 28 | !.vscode/extensions.json 29 | *.code-workspace 30 | pythonenv3.8/ 31 | 32 | # Local History for Visual Studio Code 33 | .history/ 34 | 35 | dbdiff/tests/db.sqlite3 36 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | Unreleased Refresh supported dependencies and support 2 | 3 | 0.9.6 Add support for Python 3.12 4 | 5 | 0.9.5 Add Fixture.exclude global 6 | 7 | 0.9.4 Fixes missing upstream commits in 0.9.3 8 | 9 | 0.9.3 Add FIXTURE_REWRITE env var 10 | 11 | 0.9.0 Pytest-django plugin update 12 | 13 | 0.8.1 Model inheritance support 14 | 15 | 0.8.0 Pytest plugin 16 | 17 | 0.7.3 Bugfix with no reminder in decimals 18 | 19 | 0.7.2 Django 2.0 support, added support for relative fixture paths 20 | 21 | 0.7.1 Allow passing models by string names 22 | 23 | 0.7.0 Added DbdiffTestMixin 24 | 25 | 0.6.0 Added exclude 26 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://travis-ci.org/yourlabs/django-dbdiff.svg 2 | :target: https://travis-ci.org/yourlabs/django-dbdiff 3 | .. image:: https://codecov.io/github/yourlabs/django-dbdiff/coverage.svg?branch=master 4 | :target: https://codecov.io/github/yourlabs/django-dbdiff?branch=master 5 | .. image:: https://badge.fury.io/py/django-dbdiff.png 6 | :target: http://badge.fury.io/py/django-dbdiff 7 | 8 | django-dbdiff 9 | ~~~~~~~~~~~~~ 10 | 11 | I'm pretty lazy when it comes to writing tests for existing code, however, I'm 12 | even lazier when it comes to repetitive manual testing action. 13 | 14 | This package aims at de-duplicating the data import tests from 15 | django-representatives and django-representatives-votes which is re-used in 16 | django-cities-light. 17 | 18 | Database state assertion 19 | ======================== 20 | 21 | A nice way to test a data import script is to create a source data fixture with 22 | a subset of data, ie. with only 10 cities instead of 28K or only 3 european 23 | parliament representatives instead of 3600, feed the import function with that 24 | and then compare the database state with a django fixture. This looks like what 25 | I was used to do: 26 | 27 | - use such a command to create a small data extract 28 | `shuf -n3 cities15000.txt > cities_light/tests/cities_test_fixture.txt`, 29 | - use it against the import script on a clean database, 30 | - verify the database manually, and run 31 | `django-admin dumpdata --indent=4 cities_light > cities_light/tests/cities_test_expected.txt` 32 | - then, make a test case that calls the import script against the fixture, 33 | - write and maintain some funny (fuzzy ?) repetitive test code to ensure that 34 | the database is in the expected state. 35 | 36 | When a bug is fixed, just add the case to the fixture and repeat the process to 37 | create new expected data dumps, use coverage to ensure no case is missed. 38 | 39 | With django-dbdiff, I just need to maintain to initial data extract, and test 40 | it with ``Fixture('appname/path/to/fixture', 41 | models=[YourModelToTest]).assertNoDiff()`` in a 42 | ``django.test.TransactionTestCase`` which has ``reset_sequences=True``: 43 | 44 | - if the fixture in question doesn't exist, it'll be automatically created on 45 | with dumpdata for the concerned models on the first run, raising 46 | "FixtureCreated" exception to fail the test and inform of the path of the 47 | created fixture, so that it doesn't mislead the user in thinking the test 48 | passed with an existing fixture, 49 | - if the fixture exists, it'll run dumpdata on the models concerned and GNU 50 | diff it against the fixture, if there's any output it'll be raised in the 51 | "DiffFound" exception, failing the test and printing the diff. 52 | 53 | Usage 54 | ===== 55 | 56 | Example: 57 | 58 | .. code-block:: python 59 | 60 | from django import TransactionTestCase 61 | from dbdiff.fixture import Fixture 62 | 63 | 64 | class YourImportTest(test.TransactionTestCase): 65 | reset_sequences = True 66 | 67 | def test_your_import(self): 68 | your_import() 69 | 70 | Fixture('yourapp/tests/yourtest.json', 71 | models=[YourModel]).assertNoDiff() 72 | 73 | The first time, it will raise a ``FixtureCreated`` exception, and the test will 74 | fail. This is to inform the user that the test didn't really run. On the next 75 | run though, it will pass. 76 | 77 | If any difference is found between the database and the test fixture, then 78 | ``diff()`` will return the diff as outputed by GNU diff. 79 | 80 | If you need to ignore fields globally, set the class-level variable exclude as such: 81 | 82 | .. code-block:: python 83 | 84 | Fixture.exclude = {'mrsrequest.mrsrequest': ['token']} 85 | 86 | Instead of deleting the fixtures manually before running the tests to 87 | regenerate them, just run your tests with FIXTURE_REWRITE=1 environment 88 | variable. This will overwrite the fixtures and make the tests look like it 89 | passed. 90 | 91 | See tests and docstrings for crunchy details. 92 | 93 | Requirements 94 | ============ 95 | 96 | MySQL, SQLite and PostgreSQL, Python 3.8 to 3.12 are supported along with 97 | Django 3.2 to 5.0 - it's always better to support django's master so that we 98 | can **upgrade easily when it is released**, which is one of the selling points 99 | for having 100% coverage. 100 | 101 | Install 102 | ======= 103 | 104 | Install ``django-dbdiff`` with pip and add ``dbdiff`` to ``INSTALLED_APPS``. 105 | 106 | Django model observer 107 | ===================== 108 | 109 | It is interresting to note that a related, perhaps sort-of similar app exists: 110 | https://github.com/Griffosx/djmo 111 | -------------------------------------------------------------------------------- /dbdiff/__init__.py: -------------------------------------------------------------------------------- 1 | """dbdiff module enable diffing a fixture against the database.""" 2 | 3 | default_app_config = 'dbdiff.apps.DefaultConfig' 4 | -------------------------------------------------------------------------------- /dbdiff/apps.py: -------------------------------------------------------------------------------- 1 | """AppConfig for dbdiff.""" 2 | 3 | import os 4 | 5 | from django.apps import AppConfig 6 | from django.core.serializers import register_serializer 7 | 8 | from .utils import patch_transaction_test_case 9 | 10 | 11 | class DefaultConfig(AppConfig): 12 | """ 13 | Register patched serializers and patch TransactionTestCase for sqlite. 14 | 15 | .. py:attribute:: debug 16 | 17 | If True, then diff commands will be printed to stdout and temporary 18 | files will not be deleted. 19 | """ 20 | 21 | name = 'dbdiff' 22 | debug = False 23 | default_indent = 4 24 | 25 | def ready(self): 26 | """ 27 | Register dbdiff.serializers.json and set debug. 28 | 29 | Enables debug if a DBDIFF_DEBUG environment variable is found. 30 | 31 | It is important to use serializers which dump data in a predictible way 32 | because this app relies on diff between an expected - user-generated 33 | and versioned - fixture and dumped database data. This method also 34 | overrides the default json serializer with dbdiff's. 35 | 36 | When dbdiff is installed, ``dumpdata`` will use its serializers which 37 | have predictible output and cross-database support, so fixtures dumped 38 | without dbdiff installed will have to be regenerated after dbdiff is 39 | installed to be usable with dbdiff. 40 | 41 | """ 42 | self.debug = os.environ.get('DBDIFF_DEBUG', False) 43 | register_serializer('json', 'dbdiff.serializers.json') 44 | patch_transaction_test_case() 45 | -------------------------------------------------------------------------------- /dbdiff/exceptions.py: -------------------------------------------------------------------------------- 1 | """Exceptions for dbdiff module.""" 2 | import pprint 3 | 4 | 5 | class DbDiffException(Exception): 6 | """Base exception for this app.""" 7 | 8 | 9 | class DiffFound(DbDiffException): 10 | """Raised when a diff is found by the context manager.""" 11 | 12 | def _add_messages(self, msg, title, tree): 13 | if tree: 14 | for model, instances in tree.items(): 15 | msg.append( 16 | title % ( 17 | len(instances), 18 | model 19 | ) 20 | ) 21 | 22 | for pk, fields in instances.items(): 23 | msg.append('#%s:\n%s' % (pk, pprint.pformat(fields))) 24 | 25 | def __init__(self, fixture, unexpected, missing, diff): 26 | """Exception for when a diff was found.""" 27 | msg = ['Diff found with dump at %s' % fixture.path] 28 | 29 | self._add_messages( 30 | msg, 31 | '%s unexpected instance(s) of %s found in the dump:', 32 | unexpected 33 | ) 34 | 35 | self._add_messages( 36 | msg, 37 | '%s expected instance(s) of %s missing from dump:', 38 | missing 39 | ) 40 | 41 | if diff: 42 | for model, instances in diff.items(): 43 | msg.append( 44 | '%s instance(s) of %s have not expected fields' % ( 45 | len(instances), 46 | model 47 | ) 48 | ) 49 | 50 | for pk, fields in instances.items(): 51 | msg.append('#%s:' % pk) 52 | 53 | for field, values in fields.items(): 54 | msg.append(' %s:' % field) 55 | msg.append('- %s' % pprint.pformat(values[0])) 56 | msg.append('+ %s' % pprint.pformat(values[1])) 57 | 58 | super(DiffFound, self).__init__('\n'.join(msg)) 59 | 60 | 61 | class FixtureCreated(DbDiffException): 62 | """ 63 | Raised when a fixture was created. 64 | 65 | This purposely fails a test, to avoid misleading the user into thinking 66 | that the test was properly executed against a versioned fixture. Imagine 67 | one pushes a test without the fixture, it will break because of this 68 | exception in CI. 69 | 70 | However, this should only happen once per fixture - unless the user in 71 | question forgets to commit the generated fixture ! 72 | """ 73 | -------------------------------------------------------------------------------- /dbdiff/fixture.py: -------------------------------------------------------------------------------- 1 | """Public fixture API.""" 2 | 3 | import copy 4 | import json 5 | import os 6 | import tempfile 7 | 8 | from django.apps import apps 9 | from django.core.management import call_command 10 | 11 | import ijson 12 | 13 | from .exceptions import DiffFound, FixtureCreated 14 | from .utils import ( 15 | diff, 16 | get_absolute_path, 17 | get_model_names, 18 | get_tree, 19 | ) 20 | 21 | 22 | REWRITE = os.getenv('FIXTURE_REWRITE') 23 | 24 | 25 | class Fixture(object): 26 | """ 27 | Is able to print out diffs between database and a fixture. 28 | 29 | .. py:attribute:: path 30 | 31 | Absolute path to the fixture. 32 | 33 | .. py:attribute:: models 34 | 35 | List of models concerned by the fixture. 36 | 37 | .. py:attribute:: database 38 | 39 | Database name to use, 'default' by default. 40 | 41 | .. py:attribute:: exclude 42 | 43 | Class attribute, dict of model fields to ignore in the form of: 44 | {'app.model': ['fieldN']} 45 | """ 46 | 47 | exclude = dict() 48 | 49 | def __init__(self, relative_path, models=None, database=None): 50 | """ 51 | Instanciate a FixtureDiff on a database. 52 | 53 | relative_path is used to calculate :py:attr:`path`, with 54 | :py:func:`~utils.get_absolute_path`. 55 | 56 | If models is None, then it will be generated from reading the 57 | fixture file, but generated fixtures will include all models. 58 | 59 | database should be the name of the database to use, `default` by 60 | default. 61 | """ 62 | self.path = get_absolute_path(relative_path) 63 | self.models = models if models else self.parse_models() 64 | self.database = database or 'default' 65 | 66 | def parse_models(self): 67 | """Return the list of models inside the fixture file.""" 68 | with open(self.path, 'r') as f: 69 | return [apps.get_model(i.lower()) 70 | for i in ijson.items(f, 'item.model')] 71 | 72 | @property 73 | def exists(self): 74 | """Return True if :py:attr:`path` exists.""" 75 | return os.path.exists(self.path) 76 | 77 | @property 78 | def indent(self): 79 | """Return the indentation of the fixture file or the default indent.""" 80 | if not os.path.exists(self.path): 81 | return apps.get_app_config('dbdiff').default_indent 82 | 83 | with open(self.path, 'r') as f: 84 | line = f.readline() 85 | 86 | while line and ':' not in line: 87 | line = f.readline() 88 | 89 | if not line: 90 | return apps.get_app_config('dbdiff').default_indent 91 | 92 | return len(line) - len(line.lstrip(' ')) 93 | 94 | def diff(self, exclude=None): 95 | """ 96 | Diff the fixture against a datadump of fixture models. 97 | 98 | If passed, exclude should be a list of field names to exclude from 99 | being diff'ed. 100 | """ 101 | fh, dump_path = tempfile.mkstemp('_dbdiff') 102 | 103 | exclude_final = copy.copy(self.exclude) 104 | exclude_final.update(exclude or {}) 105 | 106 | with os.fdopen(fh, 'w') as f: 107 | self.dump(f) 108 | 109 | with open(self.path, 'r') as e, open(dump_path, 'r') as r: 110 | expected, result = json.load(e), json.load(r) 111 | 112 | unexpected, missing, different = diff( 113 | get_tree(expected, exclude_final), 114 | get_tree(result, exclude_final), 115 | ) 116 | 117 | if not unexpected and not missing and not diff: 118 | os.unlink(dump_path) 119 | return None 120 | 121 | return unexpected, missing, different 122 | 123 | def load(self): 124 | """Load fixture into the database.""" 125 | call_command('loaddata', self.path) 126 | 127 | def dump(self, out): 128 | """Write fixture with dumpdata for fixture models.""" 129 | call_command( 130 | 'dumpdata', 131 | *get_model_names(self.models), 132 | format='json', 133 | traceback=True, 134 | indent=self.indent, 135 | stdout=out, 136 | use_natural_foreign_keys=True 137 | ) 138 | 139 | def assertNoDiff(self, exclude=None): # noqa 140 | """Assert that the fixture doesn't have any diff with the database. 141 | 142 | If the fixture doesn't exist then it's written but 143 | :py:class:`~exceptions.FixtureCreated` is raised. 144 | 145 | If a diff was found it will raise :py:class:`~exceptions.DiffFound`. 146 | """ 147 | if REWRITE or not self.exists: 148 | with open(self.path, 'w+') as f: 149 | self.dump(f) 150 | if not REWRITE: 151 | raise FixtureCreated(self) 152 | 153 | unexpected, missing, different = self.diff(exclude=exclude) 154 | 155 | if unexpected or missing or different: 156 | raise DiffFound(self, unexpected, missing, different) 157 | 158 | def __str__(self): 159 | """Return :py:attr:`path` for string representation.""" 160 | return self.path 161 | -------------------------------------------------------------------------------- /dbdiff/fixtures/dbdiff_test_group.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "fields": { 4 | "name": "initial_name", 5 | "permissions": [] 6 | }, 7 | "model": "auth.group", 8 | "pk": 1 9 | } 10 | ] 11 | -------------------------------------------------------------------------------- /dbdiff/fixtures/empty.json: -------------------------------------------------------------------------------- 1 | [ 2 | ] 3 | -------------------------------------------------------------------------------- /dbdiff/plugin.py: -------------------------------------------------------------------------------- 1 | """Pytest plugin for django-dbdiff. 2 | 3 | The marker enables the smarter sequence reset feature previously available in 4 | the DbdiffTestMixin in pytest, example usage:: 5 | 6 | @dbdiff(models=[YourModel]) 7 | def your_test(): 8 | assert YourModel.objects.create().pk == 1 9 | """ 10 | import pytest 11 | 12 | from .sequence import sequence_reset 13 | 14 | 15 | @pytest.fixture(autouse=True) 16 | def _dbdiff_marker(request): 17 | marker = request.node.get_closest_marker('dbdiff') 18 | if not marker: 19 | return 20 | 21 | # Enable transactional db 22 | request.getfixturevalue('transactional_db') 23 | 24 | for model in marker.kwargs['models']: 25 | sequence_reset(model) 26 | 27 | 28 | def pytest_load_initial_conftests(early_config, parser, args): 29 | """Register the dbdiff mark.""" 30 | early_config.addinivalue_line( 31 | 'markers', 32 | 'dbdiff(models, reset_sequences=True): Mark the test as using ' 33 | 'the django test database. The *transaction* argument marks will ' 34 | "allow you to use real transactions in the test like Django's " 35 | 'TransactionTestCase.') 36 | -------------------------------------------------------------------------------- /dbdiff/sequence.py: -------------------------------------------------------------------------------- 1 | """Smarter model pk sequence reset.""" 2 | from django.db import connection, models 3 | 4 | 5 | def pk_sequence_get(model): 6 | """Return a list of table, column tuples which should have sequences.""" 7 | for field in model._meta.get_fields(): 8 | if not getattr(field, 'primary_key', False): 9 | continue 10 | if not isinstance(field, models.AutoField): 11 | continue 12 | return (field.db_column or field.column, field.model._meta.db_table) 13 | return (None, None) 14 | 15 | 16 | def sequence_reset(model): 17 | """ 18 | Better sequence reset than TransactionTestCase. 19 | 20 | The difference with using TransactionTestCase with reset_sequences=True is 21 | that this will reset sequences for the given models to their higher value, 22 | supporting pre-existing models which could have been created by a 23 | migration. 24 | """ 25 | pk_field, table = pk_sequence_get(model) 26 | if not pk_field: 27 | return 28 | 29 | if connection.vendor == 'postgresql': 30 | reset = """ 31 | SELECT 32 | setval( 33 | pg_get_serial_sequence('{table}', '{column}'), 34 | coalesce(max({column}),0) + 1, 35 | false 36 | ) 37 | FROM {table} 38 | """ 39 | elif connection.vendor == 'sqlite': 40 | reset = """ 41 | UPDATE sqlite_sequence 42 | SET seq=(SELECT max({column}) from {table}) 43 | WHERE name='{table}' 44 | """ 45 | elif connection.vendor == 'mysql': 46 | cursor = connection.cursor() 47 | cursor.execute( 48 | 'SELECT MAX({column}) + 1 FROM {table}'.format( 49 | column=pk_field, table=table 50 | ) 51 | ) 52 | result = cursor.fetchone()[0] or 0 53 | reset = 'ALTER TABLE {table} AUTO_INCREMENT = %s' % result 54 | 55 | connection.cursor().execute( 56 | reset.format(column=pk_field, table=table) 57 | ) 58 | -------------------------------------------------------------------------------- /dbdiff/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | """Serializers with predictible (ordered) output.""" 2 | -------------------------------------------------------------------------------- /dbdiff/serializers/base.py: -------------------------------------------------------------------------------- 1 | """Shared code for serializers.""" 2 | 3 | import collections 4 | import datetime 5 | import decimal 6 | 7 | 8 | class BaseSerializerMixin(object): 9 | """Serializer mixin for predictible and cross-db dumps.""" 10 | 11 | @classmethod 12 | def recursive_dict_sort(cls, data): 13 | """ 14 | Return a recursive OrderedDict for a dict. 15 | 16 | Django's default model-to-dict logic - implemented in 17 | django.core.serializers.python.Serializer.get_dump_object() - returns a 18 | dict, this app registers a slightly modified version of the default 19 | json serializer which returns OrderedDicts instead. 20 | """ 21 | ordered_data = collections.OrderedDict(sorted(data.items())) 22 | 23 | for key, value in ordered_data.items(): 24 | if isinstance(value, dict): 25 | ordered_data[key] = cls.recursive_dict_sort(value) 26 | 27 | return ordered_data 28 | 29 | @classmethod 30 | def remove_microseconds(cls, data): 31 | """ 32 | Strip microseconds from datetimes for mysql. 33 | 34 | MySQL doesn't have microseconds in datetimes, so dbdiff's serializer 35 | removes microseconds from datetimes so that fixtures are cross-database 36 | compatible which make them usable for cross-database testing. 37 | """ 38 | for key, value in data['fields'].items(): 39 | if not isinstance(value, datetime.datetime): 40 | continue 41 | 42 | data['fields'][key] = datetime.datetime( 43 | year=value.year, 44 | month=value.month, 45 | day=value.day, 46 | hour=value.hour, 47 | minute=value.minute, 48 | second=value.second, 49 | tzinfo=value.tzinfo 50 | ) 51 | 52 | @classmethod 53 | def normalize_decimals(cls, data): 54 | """ 55 | Strip trailing zeros for constitency. 56 | 57 | In addition, dbdiff serialization forces Decimal normalization, because 58 | trailing zeros could happen in inconsistent ways. 59 | """ 60 | for key, value in data['fields'].items(): 61 | if not isinstance(value, decimal.Decimal): 62 | continue 63 | 64 | if value % 1 == 0: 65 | data['fields'][key] = int(value) 66 | else: 67 | data['fields'][key] = value.normalize() 68 | 69 | def get_dump_object(self, obj): 70 | """ 71 | Actual method used by Django serializers to dump dicts. 72 | 73 | By overridding this method, we're able to run our various 74 | data dump predictability methods. 75 | """ 76 | data = super(BaseSerializerMixin, self).get_dump_object(obj) 77 | self.remove_microseconds(data) 78 | self.normalize_decimals(data) 79 | data = self.recursive_dict_sort(data) 80 | return data 81 | -------------------------------------------------------------------------------- /dbdiff/serializers/json.py: -------------------------------------------------------------------------------- 1 | """Django JSON Serializer override.""" 2 | 3 | from django.core.serializers import json as upstream 4 | 5 | from .base import BaseSerializerMixin 6 | 7 | 8 | __all__ = ('Serializer', 'Deserializer') 9 | 10 | 11 | class Serializer(BaseSerializerMixin, upstream.Serializer): 12 | """Sorted dict JSON serializer.""" 13 | 14 | 15 | Deserializer = upstream.Deserializer 16 | -------------------------------------------------------------------------------- /dbdiff/test.py: -------------------------------------------------------------------------------- 1 | """Convenience test mixin.""" 2 | from django.core.management import call_command 3 | 4 | from .fixture import Fixture 5 | from .sequence import sequence_reset 6 | 7 | 8 | class DbdiffTestMixin(object): 9 | """ 10 | Convenience mixin with better sequence resetting than TransactionTestCase. 11 | 12 | The difference with using TransactionTestCase with reset_sequences=True is 13 | that this will reset sequences for the given models to their higher value, 14 | supporting pre-existing models which could have been created by a 15 | migration. 16 | 17 | The test case subclass requires some attributes and an implementation of a 18 | ``dbdiff_test()`` method that does the actual import call that this 19 | should test. Example usage:: 20 | 21 | class FrancedataImportTest(DbdiffTestMixin, test.TestCase): 22 | dbdiff_models = [YourModel] 23 | dbdiff_exclude = {'*': ['created']} 24 | dbdiff_reset_sequences = True 25 | dbdiff_expected = 'yourapp/tests/yourexpectedfixture.json' 26 | dbdiff_fixtures = ['your-fixtures.json'] 27 | 28 | def dbdiff_test(self): 29 | fixture = os.path.join( 30 | os.path.dirname(__file__), 31 | 'representatives_fixture.json' 32 | ) 33 | 34 | with open(fixture, 'r') as f: 35 | do_your_import.main(f) 36 | 37 | Supports postgresql. 38 | """ 39 | 40 | def test_db_import(self): 41 | """Actual test method, ran by the test suite.""" 42 | call_command('flush', interactive=False) 43 | 44 | for fixture in getattr(self, 'dbdiff_fixtures', []): 45 | call_command('loaddata', fixture) 46 | 47 | for model in self.dbdiff_models: 48 | sequence_reset(model) 49 | 50 | self.dbdiff_test() 51 | 52 | Fixture( 53 | self.dbdiff_expected, 54 | models=self.dbdiff_models, 55 | ).assertNoDiff( 56 | exclude=self.dbdiff_exclude, 57 | ) 58 | -------------------------------------------------------------------------------- /dbdiff/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Tests for the dbdiff module.""" 2 | -------------------------------------------------------------------------------- /dbdiff/tests/decimal_test/__init__.py: -------------------------------------------------------------------------------- 1 | """App to test DecimalField dump predictibility.""" 2 | -------------------------------------------------------------------------------- /dbdiff/tests/decimal_test/expected.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "fields": { 4 | "test_field": "1.1" 5 | }, 6 | "model": "decimal_test.testmodel", 7 | "pk": 1 8 | } 9 | ] 10 | -------------------------------------------------------------------------------- /dbdiff/tests/decimal_test/expected_no_reminder.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "fields": { 4 | "test_field": 10 5 | }, 6 | "model": "decimal_test.testmodel", 7 | "pk": 1 8 | } 9 | ] 10 | -------------------------------------------------------------------------------- /dbdiff/tests/decimal_test/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | from django.db import models, migrations 2 | 3 | 4 | class Migration(migrations.Migration): 5 | 6 | dependencies = [ 7 | ] 8 | 9 | operations = [ 10 | migrations.CreateModel( 11 | name='TestModel', 12 | fields=[ 13 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 14 | ('test_field', models.DecimalField(max_digits=3, decimal_places=3)), 15 | ], 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /dbdiff/tests/decimal_test/migrations/0002_auto_20160102_0914.py: -------------------------------------------------------------------------------- 1 | from django.db import models, migrations 2 | 3 | 4 | class Migration(migrations.Migration): 5 | 6 | dependencies = [ 7 | ('decimal_test', '0001_initial'), 8 | ] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name='testmodel', 13 | name='test_field', 14 | field=models.DecimalField(max_digits=5, decimal_places=2), 15 | ), 16 | ] 17 | -------------------------------------------------------------------------------- /dbdiff/tests/decimal_test/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yourlabs/django-dbdiff/892a8631c1945d405bb36dbbd2d247189017391d/dbdiff/tests/decimal_test/migrations/__init__.py -------------------------------------------------------------------------------- /dbdiff/tests/decimal_test/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class TestModel(models.Model): 5 | test_field = models.DecimalField(max_digits=5, decimal_places=2) 6 | -------------------------------------------------------------------------------- /dbdiff/tests/inheritance/__init__.py: -------------------------------------------------------------------------------- 1 | """Test app for model inheritance.""" 2 | -------------------------------------------------------------------------------- /dbdiff/tests/inheritance/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class Parent(models.Model): 5 | pass 6 | 7 | 8 | class Child(Parent): 9 | name = models.CharField(max_length=50) 10 | -------------------------------------------------------------------------------- /dbdiff/tests/nonintpk/__init__.py: -------------------------------------------------------------------------------- 1 | """Test that we don't crash with non sequence pks.""" 2 | -------------------------------------------------------------------------------- /dbdiff/tests/nonintpk/models.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from django.db import models 4 | 5 | 6 | class Nonintpk(models.Model): 7 | # dbdiff should not try to reset this sequence 8 | id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) 9 | -------------------------------------------------------------------------------- /dbdiff/tests/project/__init__.py: -------------------------------------------------------------------------------- 1 | """Test project settings.""" 2 | -------------------------------------------------------------------------------- /dbdiff/tests/project/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for project project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.8.3.dev20150604012123. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/dev/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/dev/ref/settings/ 11 | """ 12 | 13 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 14 | import os 15 | 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/dev/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = '9kq$zffboob0)(sn_mr+^cw*3hair=vp=@616u#gk!31dymqgo' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = [] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = ( 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | 'dbdiff', 41 | 42 | 'dbdiff.tests.decimal_test', 43 | 'dbdiff.tests.nonintpk', 44 | 'dbdiff.tests.inheritance', 45 | ) 46 | 47 | MIDDLEWARE_CLASSES = ( 48 | 'django.contrib.sessions.middleware.SessionMiddleware', 49 | 'django.middleware.common.CommonMiddleware', 50 | 'django.middleware.csrf.CsrfViewMiddleware', 51 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 52 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 53 | 'django.contrib.messages.middleware.MessageMiddleware', 54 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 55 | 'django.middleware.security.SecurityMiddleware', 56 | ) 57 | 58 | ROOT_URLCONF = 'dbdiff.tests.project.urls' 59 | 60 | TEMPLATES = [ 61 | { 62 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 63 | 'DIRS': [], 64 | 'APP_DIRS': True, 65 | 'OPTIONS': { 66 | 'context_processors': [ 67 | 'django.template.context_processors.debug', 68 | 'django.template.context_processors.request', 69 | 'django.contrib.auth.context_processors.auth', 70 | 'django.contrib.messages.context_processors.messages', 71 | ], 72 | }, 73 | }, 74 | ] 75 | 76 | WSGI_APPLICATION = 'dbdiff.tests.project.wsgi.application' 77 | 78 | 79 | # Database 80 | # https://docs.djangoproject.com/en/dev/ref/settings/#databases 81 | 82 | DATABASES = { 83 | 'default': { 84 | 'ENGINE': 'django.db.backends.sqlite3', 85 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 86 | } 87 | } 88 | 89 | 90 | # Internationalization 91 | # https://docs.djangoproject.com/en/dev/topics/i18n/ 92 | 93 | LANGUAGE_CODE = 'en-us' 94 | 95 | TIME_ZONE = 'UTC' 96 | 97 | USE_I18N = True 98 | 99 | USE_L10N = True 100 | 101 | USE_TZ = True 102 | 103 | 104 | # Static files (CSS, JavaScript, Images) 105 | # https://docs.djangoproject.com/en/dev/howto/static-files/ 106 | 107 | STATIC_URL = '/static/' 108 | -------------------------------------------------------------------------------- /dbdiff/tests/project/settings_mysql.py: -------------------------------------------------------------------------------- 1 | from .settings import * # noqa 2 | 3 | import os 4 | 5 | DATABASES = { 6 | 'default': { 7 | 'ENGINE': 'django.db.backends.mysql', 8 | 'HOST': os.environ.get('DB_HOST', ''), 9 | 'NAME': os.environ.get('DB_NAME', 'dbdiff_test'), 10 | 'USER': os.environ.get('DB_USER', 'root'), 11 | 'PASSWORD': os.environ.get('DB_PASSWORD', ''), 12 | 'PORT': os.environ.get('DB_PORT', ''), 13 | 'OPTIONS': { 14 | 'charset': 'utf8mb4', 15 | }, 16 | 'TEST':{ 17 | 'CHARSET': 'utf8mb4', 18 | 'COLLATION': 'utf8mb4_unicode_ci', 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /dbdiff/tests/project/settings_postgresql.py: -------------------------------------------------------------------------------- 1 | from .settings import * # noqa 2 | 3 | import os 4 | 5 | DATABASES = { 6 | 'default': { 7 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', 8 | 'HOST': os.environ.get('DB_HOST', ''), 9 | 'NAME': os.environ.get('DB_NAME', 'dbdiff_test'), 10 | 'USER': os.environ.get('DB_USER', 'postgres'), 11 | 'PASSWORD': os.environ.get('DB_PASSWORD', ''), 12 | 'PORT': os.environ.get('DB_PORT', '5432'), 13 | 'OPTIONS': {}, 14 | 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /dbdiff/tests/project/settings_sqlite.py: -------------------------------------------------------------------------------- 1 | from .settings import * # noqa 2 | 3 | DATABASES = { 4 | 'default': { 5 | 'ENGINE': 'django.db.backends.sqlite3', 6 | 'NAME': ':memory:', 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /dbdiff/tests/project/urls.py: -------------------------------------------------------------------------------- 1 | urlpatterns = [] 2 | -------------------------------------------------------------------------------- /dbdiff/tests/test_compare.py: -------------------------------------------------------------------------------- 1 | """Public API tests.""" 2 | 3 | import os 4 | import tempfile 5 | 6 | from django import test 7 | from django.contrib.auth.models import Group 8 | 9 | from ..exceptions import DiffFound, FixtureCreated 10 | from ..fixture import Fixture 11 | 12 | 13 | class SmokeTest(test.TransactionTestCase): 14 | def setUp(self): 15 | fd, self.fixture_path = tempfile.mkstemp(suffix='_dbdiff') 16 | 17 | def test_000_fixture_auto_create(self): 18 | fixture = Fixture(self.fixture_path, models=[Group]) 19 | 20 | # Should auto-create the diff 21 | if fixture.exists: 22 | os.unlink(fixture.path) 23 | 24 | Group.objects.create(name='testgroup') 25 | 26 | with self.assertRaises(FixtureCreated): 27 | fixture.assertNoDiff() 28 | 29 | assert fixture.exists 30 | 31 | with open(self.fixture_path, 'r') as f: 32 | result = f.read() 33 | 34 | expected = '''[ 35 | { 36 | "fields": { 37 | "name": "testgroup", 38 | "permissions": [] 39 | }, 40 | "model": "auth.group", 41 | "pk": 1 42 | } 43 | ]''' 44 | 45 | assert expected.strip() == result.strip() 46 | 47 | # It should pass now 48 | fixture.assertNoDiff() 49 | 50 | # It should break now ! 51 | Group.objects.all().update(name='BOOM') 52 | expected = ''' 53 | 1 instance(s) of auth.group have not expected fields 54 | #1: 55 | name: 56 | - 'testgroup' 57 | + 'BOOM' 58 | ''' 59 | 60 | with self.assertRaises(DiffFound) as result: 61 | fixture.assertNoDiff() 62 | self.assert_message_is(expected, result) 63 | 64 | # Excluding the name parameter, there should be no diff 65 | fixture.assertNoDiff(exclude={'auth.group': ['name']}) 66 | 67 | # Assert it finds unexpected model 68 | Group.objects.create(name='unexpected') 69 | expected = ''' 70 | 1 unexpected instance(s) of auth.group found in the dump: 71 | #2: 72 | {'name': 'unexpected', 'permissions': []} 73 | 1 instance(s) of auth.group have not expected fields 74 | #1: 75 | name: 76 | - 'testgroup' 77 | + 'BOOM' 78 | ''' 79 | 80 | with self.assertRaises(DiffFound) as result: 81 | fixture.assertNoDiff() 82 | self.assert_message_is(expected, result) 83 | 84 | # Assert it finds missing model 85 | Group.objects.get(pk=1).delete() 86 | expected = ''' 87 | 1 unexpected instance(s) of auth.group found in the dump: 88 | #2: 89 | {'name': 'unexpected', 'permissions': []} 90 | 1 expected instance(s) of auth.group missing from dump: 91 | #1: 92 | {'name': 'testgroup', 'permissions': []} 93 | ''' 94 | 95 | with self.assertRaises(DiffFound) as result: 96 | fixture.assertNoDiff() 97 | self.assert_message_is(expected, result) 98 | 99 | def assert_message_is(self, expected, result): 100 | msg = result.exception.args[0] 101 | out = '\n'.join(msg.split('\n')[1:]) 102 | assert out.strip() == expected.strip() 103 | -------------------------------------------------------------------------------- /dbdiff/tests/test_decimal.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from django import test 4 | 5 | from .decimal_test.models import TestModel as DecimalTestModel 6 | from ..fixture import Fixture 7 | 8 | 9 | class DecimalDiffTest(test.TransactionTestCase): 10 | reset_sequences = True 11 | 12 | expected = os.path.join( 13 | 'dbdiff', 14 | 'tests', 15 | 'decimal_test', 16 | 'expected.json' 17 | ) 18 | model = DecimalTestModel 19 | 20 | def test_data_diff_is_empty_with_two_decimal_float(self): 21 | self.model.objects.create(test_field=1.10) 22 | Fixture(self.expected).assertNoDiff() 23 | 24 | def test_data_diff_is_empty_with_one_decimal_float(self): 25 | self.model.objects.create(test_field=1.1) 26 | Fixture(self.expected).assertNoDiff() 27 | 28 | def test_data_diff_is_empty_with_two_decimal_string(self): 29 | self.model.objects.create(test_field='1.10') 30 | Fixture(self.expected).assertNoDiff() 31 | 32 | def test_data_diff_is_empty_with_one_decimal_string(self): 33 | self.model.objects.create(test_field='1.1') 34 | Fixture(self.expected).assertNoDiff() 35 | 36 | 37 | class NoReminderDecimalDiffTest(test.TransactionTestCase): 38 | reset_sequences = True 39 | 40 | expected = os.path.join( 41 | 'dbdiff', 42 | 'tests', 43 | 'decimal_test', 44 | 'expected_no_reminder.json' 45 | ) 46 | model = DecimalTestModel 47 | 48 | def test_data_diff_is_empty_with_no_decimal_string(self): 49 | self.model.objects.create(test_field=10) 50 | Fixture(self.expected).assertNoDiff() 51 | 52 | def test_data_diff_is_empty_with_no_decimal_int(self): 53 | self.model.objects.create(test_field='10') 54 | Fixture(self.expected).assertNoDiff() 55 | -------------------------------------------------------------------------------- /dbdiff/tests/test_fixture.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from django import test 4 | from django.contrib.auth.models import Group 5 | 6 | from ..fixture import Fixture 7 | 8 | 9 | class FixtureTest(test.TransactionTestCase): 10 | def setUp(self): 11 | self.fixture = Fixture('dbdiff/fixtures/dbdiff_test_group.json') 12 | 13 | def test_fixture_path(self): 14 | assert self.fixture.path == os.path.abspath(os.path.join( 15 | os.path.dirname(__file__), 16 | '..', 17 | 'fixtures', 18 | 'dbdiff_test_group.json' 19 | )) 20 | 21 | def test_indent(self): 22 | assert self.fixture.indent == 4 23 | 24 | def test_models(self): 25 | assert self.fixture.models == [Group] 26 | -------------------------------------------------------------------------------- /dbdiff/tests/test_mixin.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "fields": { 4 | "app_label": "admin", 5 | "model": "logentry" 6 | }, 7 | "model": "contenttypes.contenttype", 8 | "pk": 1 9 | }, 10 | { 11 | "fields": { 12 | "app_label": "auth", 13 | "model": "permission" 14 | }, 15 | "model": "contenttypes.contenttype", 16 | "pk": 2 17 | }, 18 | { 19 | "fields": { 20 | "app_label": "auth", 21 | "model": "group" 22 | }, 23 | "model": "contenttypes.contenttype", 24 | "pk": 3 25 | }, 26 | { 27 | "fields": { 28 | "app_label": "auth", 29 | "model": "user" 30 | }, 31 | "model": "contenttypes.contenttype", 32 | "pk": 4 33 | }, 34 | { 35 | "fields": { 36 | "app_label": "contenttypes", 37 | "model": "contenttype" 38 | }, 39 | "model": "contenttypes.contenttype", 40 | "pk": 5 41 | }, 42 | { 43 | "fields": { 44 | "app_label": "sessions", 45 | "model": "session" 46 | }, 47 | "model": "contenttypes.contenttype", 48 | "pk": 6 49 | }, 50 | { 51 | "fields": { 52 | "app_label": "decimal_test", 53 | "model": "testmodel" 54 | }, 55 | "model": "contenttypes.contenttype", 56 | "pk": 7 57 | }, 58 | { 59 | "fields": { 60 | "app_label": "nonintpk", 61 | "model": "nonintpk" 62 | }, 63 | "model": "contenttypes.contenttype", 64 | "pk": 8 65 | }, 66 | { 67 | "fields": { 68 | "app_label": "inheritance", 69 | "model": "parent" 70 | }, 71 | "model": "contenttypes.contenttype", 72 | "pk": 9 73 | }, 74 | { 75 | "fields": { 76 | "app_label": "inheritance", 77 | "model": "child" 78 | }, 79 | "model": "contenttypes.contenttype", 80 | "pk": 10 81 | }, 82 | { 83 | "fields": { 84 | "app_label": "", 85 | "model": "" 86 | }, 87 | "model": "contenttypes.contenttype", 88 | "pk": 11 89 | } 90 | ] 91 | -------------------------------------------------------------------------------- /dbdiff/tests/test_mixin.py: -------------------------------------------------------------------------------- 1 | from dbdiff.test import DbdiffTestMixin 2 | 3 | from django import test 4 | from django.contrib.contenttypes.models import ContentType 5 | from django.db import connection 6 | 7 | 8 | class ContentTypeTestCase(DbdiffTestMixin, test.TestCase): 9 | dbdiff_models = [ContentType] 10 | dbdiff_exclude = {'*': ['created']} 11 | dbdiff_reset_sequences = True 12 | dbdiff_expected = 'dbdiff/tests/test_mixin.json' 13 | 14 | def test_db_import(self): 15 | if connection.vendor != 'postgresql': 16 | return # not supported for now 17 | super(ContentTypeTestCase, self).test_db_import() 18 | self.assertTrue(self.dbdiff_test_executed) 19 | 20 | def dbdiff_test(self): 21 | ContentType.objects.create() 22 | self.dbdiff_test_executed = True 23 | -------------------------------------------------------------------------------- /dbdiff/tests/test_plugin.py: -------------------------------------------------------------------------------- 1 | from dbdiff.tests.decimal_test.models import TestModel as DecimalModel 2 | from dbdiff.tests.inheritance.models import Child, Parent 3 | from dbdiff.tests.nonintpk.models import Nonintpk 4 | 5 | import pytest 6 | 7 | 8 | @pytest.mark.dbdiff(models=[DecimalModel]) 9 | def test_insert_first(): 10 | assert DecimalModel.objects.count() == 0 11 | assert DecimalModel.objects.create(test_field=1).pk == 1 12 | 13 | 14 | @pytest.mark.dbdiff(models=[DecimalModel]) 15 | def test_still_first_pk(): 16 | assert DecimalModel.objects.count() == 0 17 | assert DecimalModel.objects.create(test_field=1).pk == 1 18 | 19 | 20 | @pytest.mark.dbdiff(models=[DecimalModel, Nonintpk]) 21 | def test_doesnt_reset_nonintpk_which_would_fail(): 22 | assert DecimalModel.objects.count() == 0 23 | 24 | 25 | @pytest.mark.dbdiff(models=[Child]) 26 | def test_inheritance_sequence_reset(): 27 | assert Child.objects.count() == 0 28 | assert Child.objects.create(name='1').pk == 1 29 | 30 | 31 | @pytest.mark.dbdiff(models=[Child]) 32 | def test_inheritance_sequence_reset_again(): 33 | assert Parent.objects.count() == 0 34 | assert Child.objects.create(name='1').pk == 1 35 | -------------------------------------------------------------------------------- /dbdiff/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dbdiff.utils import get_absolute_path, get_model_names 4 | 5 | from django.contrib.auth.models import Group 6 | 7 | 8 | def test_get_model_names(): 9 | assert get_model_names([Group, 'auth.user']) == ['auth.group', 'auth.user'] 10 | 11 | 12 | def test_get_absolute_path_starting_with_dot(): 13 | assert get_absolute_path('./foo') == os.path.join( 14 | os.path.abspath(os.path.dirname('__file__')), 15 | 'foo', 16 | ) 17 | -------------------------------------------------------------------------------- /dbdiff/utils.py: -------------------------------------------------------------------------------- 1 | """Utils for dbdiff.""" 2 | 3 | import os 4 | 5 | from django.apps import apps 6 | from django.db import connections 7 | 8 | from importlib.util import find_spec 9 | 10 | def get_tree(dump, exclude=None): 11 | """Return a tree of model -> pk -> fields.""" 12 | exclude = exclude or {} 13 | tree = {} 14 | 15 | for instance in dump: 16 | if instance['model'] not in tree: 17 | tree[instance['model']] = {} 18 | 19 | exclude_fields = exclude.get(instance['model'], []) 20 | exclude_fields += exclude.get('*', []) 21 | 22 | tree[instance['model']][instance['pk']] = { 23 | name: value for name, value in instance['fields'].items() 24 | if name not in exclude_fields 25 | } 26 | 27 | return tree 28 | 29 | 30 | def _get_unexpected(expected, result): 31 | unexpected = {} 32 | 33 | for model, result_instances in result.items(): 34 | expected_pks = expected.get(model, {}).keys() 35 | 36 | for pk, result_fields in result_instances.items(): 37 | if pk in expected_pks: 38 | continue 39 | 40 | unexpected.setdefault(model, {}) 41 | unexpected[model][pk] = result_fields 42 | 43 | return unexpected 44 | 45 | 46 | def diff(expected, result): 47 | """Return unexpected, missing and diff between expected and result.""" 48 | missing, diff = {}, {} 49 | 50 | unexpected = _get_unexpected(expected, result) 51 | 52 | for model, expected_instances in expected.items(): 53 | for pk, expected_fields in expected_instances.items(): 54 | if pk not in result.get(model, {}): 55 | missing.setdefault(model, {}) 56 | missing[model][pk] = expected_fields 57 | continue 58 | 59 | result_fields = result[model][pk] 60 | if expected_fields == result_fields: 61 | continue 62 | 63 | diff.setdefault(model, {}) 64 | diff[model].setdefault(pk, {}) 65 | 66 | for expected_field, expected_value in expected_fields.items(): 67 | result_value = result_fields[expected_field] 68 | 69 | if expected_value == result_value: 70 | continue 71 | 72 | diff[model][pk][expected_field] = ( 73 | expected_value, 74 | result_value 75 | ) 76 | return unexpected, missing, diff 77 | 78 | 79 | def get_absolute_path(path): 80 | """Return the absolute path to an app-relative path.""" 81 | if path.startswith('/'): 82 | return path 83 | 84 | if path.startswith('.'): 85 | module_path = '.' 86 | else: 87 | module_path = find_spec(path.split('/')[0]).submodule_search_locations[0] 88 | 89 | return os.path.abspath(os.path.join( 90 | module_path, 91 | *path.split('/')[1:] 92 | )) 93 | 94 | 95 | def get_model_names(model_classes): 96 | """Return model names for model classes.""" 97 | return [ 98 | m if isinstance(m, str) 99 | else '%s.%s' % (m._meta.app_label, m._meta.model_name) 100 | for m in model_classes 101 | ] 102 | 103 | 104 | def get_models_tables(models): 105 | """Return the list of tables for the given models.""" 106 | tables = set() 107 | 108 | for model in models: 109 | tables.add(model._meta.db_table) 110 | tables.update(f.m2m_db_table() for f in 111 | model._meta.local_many_to_many) 112 | 113 | return list(tables) 114 | 115 | 116 | def patch_transaction_test_case(): 117 | """Monkeypatch TransactionTestCase._reset_sequences to support SQLite.""" 118 | from django.test.testcases import TransactionTestCase 119 | TransactionTestCase.old_reset_sequences = \ 120 | TransactionTestCase._reset_sequences 121 | 122 | def new_reset_sequences(self, db_name): 123 | self.old_reset_sequences(db_name) 124 | connection = connections[db_name] 125 | 126 | if connection.vendor != 'sqlite': 127 | return 128 | 129 | tables = get_models_tables(apps.get_models()) 130 | statements = [ 131 | "UPDATE SQLITE_SEQUENCE SET SEQ=0 WHERE NAME='%s';" % t 132 | for t in tables 133 | ] 134 | 135 | cursor = connection.cursor() 136 | 137 | for statement in statements: 138 | cursor.execute(statement) 139 | 140 | TransactionTestCase._reset_sequences = new_reset_sequences 141 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import os 3 | 4 | 5 | # Utility function to read the README file. 6 | # Used for the long_description. It's nice, because now 1) we have a top level 7 | # README file and 2) it's easier to type in the README file than to put a raw 8 | # string in below ... 9 | def read(fname): 10 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 11 | 12 | 13 | setup( 14 | name='django-dbdiff', 15 | version='0.9.6', 16 | description='Database data diffing against fixtures for testing', 17 | author='James Pic', 18 | author_email='jamespic@gmail.com', 19 | url='https://github.com/yourlabs/django-dbdiff', 20 | packages=find_packages(), 21 | include_package_data=True, 22 | long_description=read('README.rst'), 23 | license='MIT', 24 | keywords='django test database fixture diff', 25 | install_requires=['ijson', 'json_delta'], 26 | entry_points={'pytest11': ['dbdiff = dbdiff.plugin']}, 27 | classifiers=[ 28 | 'Development Status :: 4 - Beta', 29 | 'Environment :: Web Environment', 30 | 'Intended Audience :: Developers', 31 | 'License :: OSI Approved :: MIT License', 32 | 'Operating System :: OS Independent', 33 | 'Framework :: Django', 34 | 'Framework :: Django :: 3.2', 35 | 'Framework :: Django :: 4.0', 36 | 'Framework :: Django :: 4.1', 37 | 'Framework :: Django :: 4.2', 38 | 'Framework :: Django :: 5.0', 39 | 'Programming Language :: Python', 40 | 'Programming Language :: Python :: 3', 41 | 'Programming Language :: Python :: 3.8', 42 | 'Programming Language :: Python :: 3.9', 43 | 'Programming Language :: Python :: 3.10', 44 | 'Programming Language :: Python :: 3.11', 45 | 'Programming Language :: Python :: 3.12', 46 | 'Topic :: Internet :: WWW/HTTP', 47 | 'Topic :: Software Development :: Libraries :: Python Modules', 48 | ], 49 | ) 50 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{310,311,312}-django50-{sqlite,mysql,postgresql} 4 | py{38,39,310,311}-django42-{sqlite,mysql,postgresql} 5 | py{38,39,310,311}-django41-{sqlite,mysql,postgresql} 6 | py{38,39,310}-django40-{sqlite,mysql,postgresql} 7 | py{38,39,310}-django32-{sqlite,mysql,postgresql} 8 | qa 9 | skip_missing_interpreters = True 10 | sitepackages = False 11 | 12 | [gh-actions] 13 | python = 14 | 3.8: py38, docs, checkqa, pylint, mypy 15 | 3.9: py39 16 | 3.10: py310 17 | 3.11: py311 18 | 3.12: py312 19 | 20 | [testenv] 21 | usedevelop = true 22 | commands = 23 | mysql: mysql -u root -h {env:DB_HOST} --password={env:DB_PASSWORD} --protocol tcp -e 'drop database if exists test_dbdiff_test;' 24 | postgresql: psql -U postgres -h {env:DB_HOST} -c 'drop database if exists test_dbdiff_test;' 25 | pytest -vv --cov dbdiff --create-db --strict -r fEsxXw {posargs:dbdiff} 26 | allowlist_externals = 27 | mysql 28 | psql 29 | deps = 30 | pytest 31 | pytest-django 32 | pytest-cov 33 | mock 34 | coverage 35 | django50: Django>=5.0rc1,<5.1 36 | django42: Django>=4.2,<5.0 37 | django41: Django>=4.1,<4.2 38 | django40: Django>=4.0,<4.1 39 | django32: Django>=3.2,<4.0 40 | postgresql: psycopg2-binary==2.9.9 41 | mysql: mysqlclient 42 | setenv = 43 | PIP_ALLOW_EXTERNAL=true 44 | DJANGO_SETTINGS_MODULE=dbdiff.tests.project.settings 45 | sqlite: DJANGO_SETTINGS_MODULE=dbdiff.tests.project.settings_sqlite 46 | postgresql: DJANGO_SETTINGS_MODULE=dbdiff.tests.project.settings_postgresql 47 | postgresql: DB_NAME=dbdiff_test 48 | postgresql: DB_ENGINE=postgresql_psycopg2 49 | postgresql: DB_USER=postgres 50 | mysql: DJANGO_SETTINGS_MODULE=dbdiff.tests.project.settings_mysql 51 | mysql: DB_NAME=dbdiff_test 52 | mysql: DB_ENGINE=mysql 53 | mysql: DB_USER=root 54 | passenv = 55 | TEST_* 56 | DBDIFF_* 57 | DB_* 58 | PGPASSWORD 59 | 60 | [testenv:qa] 61 | basepython = python3.8 62 | commands = 63 | flake8 --show-source --exclude tests --max-complexity=7 --ignore=D203 dbdiff 64 | flake8 --show-source --exclude migrations --max-complexity=3 --ignore=D100,D101,D102,D103 dbdiff/tests 65 | 66 | deps = 67 | flake8 68 | mccabe 69 | flake8-debugger 70 | flake8-import-order 71 | flake8-docstrings 72 | pep8-naming 73 | --------------------------------------------------------------------------------