├── .github └── workflows │ ├── release.yml │ └── test.yml ├── .gitignore ├── CHANGELOG.rst ├── LICENSE ├── README.rst ├── setup.cfg ├── setup.py ├── syzygy ├── __init__.py ├── apps.py ├── autodetector.py ├── checks.py ├── compat.py ├── conf.py ├── constants.py ├── exceptions.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── makemigrations.py │ │ └── migrate.py ├── operations.py ├── plan.py └── quorum │ ├── __init__.py │ ├── backends │ ├── __init__.py │ ├── base.py │ └── cache.py │ └── exceptions.py ├── tests ├── __init__.py ├── models.py ├── settings │ ├── __init__.py │ ├── mysql.py │ └── postgresql.py ├── test_autodetector.py ├── test_checks.py ├── test_commands.py ├── test_migrations │ ├── __init__.py │ ├── ambiguous │ │ ├── 0001_initial.py │ │ └── __init__.py │ ├── crash │ │ ├── 0001_pre_deploy.py │ │ └── __init__.py │ ├── empty │ │ └── .gitkeep │ ├── functional │ │ ├── 0001_pre_deploy.py │ │ ├── 0002_post_deploy.py │ │ └── __init__.py │ ├── merge_conflict │ │ ├── 0001_initial.py │ │ ├── 0002_first.py │ │ ├── 0002_second.py │ │ └── __init__.py │ └── null_field_removal │ │ ├── 0001_initial.py │ │ └── __init__.py ├── test_operations.py ├── test_plan.py └── test_quorum.py └── tox.ini /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | build: 10 | if: github.repository == 'charettes/django-syzygy' 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | with: 16 | fetch-depth: 0 17 | 18 | - name: Set up Python 19 | uses: actions/setup-python@v2 20 | with: 21 | python-version: 3.9 22 | 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install -U pip 26 | python -m pip install -U setuptools twine wheel 27 | - name: Build package 28 | run: | 29 | python setup.py --version 30 | python setup.py sdist --format=gztar bdist_wheel 31 | twine check dist/* 32 | - name: Upload packages to Pypi 33 | if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') 34 | uses: pypa/gh-action-pypi-publish@release/v1 35 | with: 36 | user: __token__ 37 | password: ${{ secrets.PYPI_API_TOKEN }} 38 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | 9 | strategy: 10 | fail-fast: false 11 | max-parallel: 5 12 | matrix: 13 | python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] 14 | 15 | services: 16 | postgres: 17 | image: postgres:17 18 | env: 19 | POSTGRES_USER: postgres 20 | POSTGRES_PASSWORD: postgres 21 | POSTGRES_DB: postgres 22 | ports: 23 | - 5432:5432 24 | options: >- 25 | --health-cmd pg_isready 26 | --health-interval 10s 27 | --health-timeout 5s 28 | --health-retries 5 29 | mysql: 30 | image: mysql:8.0 31 | env: 32 | MYSQL_USER: mysql 33 | MYSQL_PASSWORD: mysql 34 | MYSQL_DATABASE: syzygy 35 | MYSQL_ROOT_PASSWORD: password 36 | ports: 37 | - 3306:3306 38 | options: >- 39 | --health-cmd "mysqladmin ping" 40 | --health-interval 10s 41 | --health-timeout 5s 42 | --health-retries 5 43 | 44 | steps: 45 | - uses: actions/checkout@v4 46 | 47 | - name: Set up Python ${{ matrix.python-version }} 48 | uses: actions/setup-python@v5 49 | with: 50 | python-version: ${{ matrix.python-version }} 51 | 52 | - name: Get pip cache dir 53 | id: pip-cache 54 | run: | 55 | echo "::set-output name=dir::$(pip cache dir)" 56 | 57 | - name: Cache 58 | uses: actions/cache@v4 59 | with: 60 | path: ${{ steps.pip-cache.outputs.dir }} 61 | key: 62 | ${{ matrix.python-version }}-v1-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/tox.ini') }} 63 | restore-keys: | 64 | ${{ matrix.python-version }}-v1- 65 | 66 | - name: Install dependencies 67 | run: | 68 | python -m pip install --upgrade pip 69 | python -m pip install --upgrade tox tox-gh-actions 70 | 71 | - name: Tox tests 72 | run: | 73 | tox 74 | env: 75 | DB_POSTGRES_USER: postgres 76 | DB_POSTGRES_PASSWORD: postgres 77 | DB_POSTGRES_HOST: localhost 78 | DB_POSTGRES_PORT: 5432 79 | DB_MYSQL_USER: root 80 | DB_MYSQL_PASSWORD: password 81 | DB_MYSQL_HOST: "127.0.0.1" 82 | DB_MYSQL_PORT: 3306 83 | 84 | - name: Coveralls 85 | uses: AndreMiras/coveralls-python-action@develop 86 | with: 87 | parallel: true 88 | flag-name: Unit Test 89 | 90 | coveralls_finish: 91 | needs: test 92 | runs-on: ubuntu-latest 93 | steps: 94 | - name: Coveralls Finished 95 | uses: AndreMiras/coveralls-python-action@develop 96 | with: 97 | parallel-finished: true 98 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | syntax:glob 2 | *.py[co] 3 | dist/ 4 | *.egg-info/* 5 | .coverage 6 | .tox 7 | .mypy_cache/ 8 | htmlcov/ -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | 1.2.2 2 | ===== 3 | 4 | :release-date: not-released 5 | 6 | - Address crash of system checks when an installed app has an empty 7 | `migrations` directory. (#62) 8 | 9 | 1.2.1 10 | ===== 11 | 12 | :release-date: 2025-04-23 13 | 14 | - Avoid unnecessary prompting for a default value on `ManyToManyField` 15 | removals. (#59) 16 | - Address a ``makemigration`` crash when adding a ``ForeignKey`` with a 17 | callable ``default``. (#60) 18 | 19 | 1.2.0 20 | ===== 21 | 22 | :release-date: 2025-02-03 23 | 24 | - Add support for MySQL. 25 | - Adjust `makemigrations` command to take advantage of auto-detector class. (#49) 26 | - Add support for Django 5.2 and Python 3.13. 27 | - Drop support for Python 3.8. 28 | - Ensure staged renames and alters are properly serialized. (#52) 29 | - Address improper handling of rename operation questioning. (#53) 30 | - Address improper monkey-patching of `AlterField.migration_name_fragment`. (#56) 31 | 32 | 1.1.0 33 | ===== 34 | :release-date: 2024-05-24 35 | 36 | - Address typos in `AmbiguousPlan` error messages. 37 | - Mention `MIGRATION_STAGES_OVERRIDE` on ambiguous plan involving third party apps. 38 | 39 | 1.0.1 40 | ===== 41 | :release-date: 2024-04-13 42 | 43 | - Avoid unnecessary two-step migration for nullable without default additions. 44 | - Avoid splitting many-to-many field additions in stages. (#42) 45 | - Adjust ambiguous stage auto-detection interactive questioning. (#44) 46 | 47 | 1.0.0 48 | ===== 49 | :release-date: 2023-10-10 50 | 51 | - Initial release 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, Simon Charette 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | django-syzygy 2 | ============= 3 | 4 | .. image:: https://github.com/charettes/django-syzygy/actions/workflows/test.yml/badge.svg?branch=master 5 | :target: https://github.com/charettes/django-syzygy/actions?query=branch%3Amaster 6 | :alt: Build Status 7 | 8 | .. image:: https://coveralls.io/repos/github/charettes/django-syzygy/badge.svg?branch=master 9 | :target: https://coveralls.io/github/charettes/django-syzygy?branch=master 10 | :alt: Coverage status 11 | 12 | 13 | Django application providing database migration tooling to automate their deployment. 14 | 15 | Inspired by a `2015 post from Ludwig Hähne`_ and experience dealing with migration at Zapier_. 16 | 17 | .. _`2015 post from Ludwig Hähne`: https://pankrat.github.io/2015/django-migrations-without-downtimes/#django-wishlist 18 | .. _Zapier: https://zapier.com 19 | 20 | Currently only tested against PostgreSQL, SQLite, and MySQL. 21 | 22 | Note that while MySQL is supported it doesn't support transactional DDL meaning 23 | it will `require manual intervention if a migration fails to apply`_ which makes 24 | problematic to use in an automated CI/CD setup. 25 | 26 | .. _`require manual intervention if a migration fails to apply`: https://docs.djangoproject.com/en/5.1/topics/migrations/#mysql 27 | 28 | Installation 29 | ------------ 30 | 31 | .. code:: sh 32 | 33 | pip install django-syzygy 34 | 35 | Usage 36 | ----- 37 | 38 | Add ``'syzygy'`` to your ``INSTALLED_APPS`` 39 | 40 | .. code:: python 41 | 42 | # settings.py 43 | INSTALLED_APPS = [ 44 | ... 45 | 'syzygy', 46 | ... 47 | ] 48 | 49 | Setup you deployment pipeline to run ``migrate --pre-deploy`` before rolling 50 | out your code changes and ``migrate`` afterwards to apply the postponed 51 | migrations. 52 | 53 | .. note:: 54 | 55 | Adding ``'syzygy'`` to your ``INSTALLED_APPS`` will override the default 56 | ``makemigrations`` and ``migrate`` management commands due to the lack 57 | of other injection point into the migration framework. If you plan on 58 | using this package with other migration related ones you might have to 59 | define your own command overrides that inherit from all involved packages 60 | ``makemigrations.Command`` and ``migrate.Command`` definitions. 61 | 62 | Concept 63 | ------- 64 | 65 | When dealing with database migrations in the context of an highly available 66 | application managed through continuous deployment the Django migration 67 | leaves a lot to be desired in terms of the sequencing of operations it 68 | generates. 69 | 70 | The automatically generated schema alterations for field additions, removals, 71 | renames, and others do not account for deployments where versions of the old 72 | and the new code must co-exist for a short period of time. 73 | 74 | For example, adding a field with a ``default`` does not persist a database 75 | level default which prevents ``INSERT`` from the pre-existing code which 76 | ignores the existence of tentatively added field from succeeding. 77 | 78 | Figuring out the proper sequencing of operations is doable but non-trivial and 79 | error prone. Syzygy ought to provide a solution to this problem by introducing 80 | a notion of *prerequisite* and *postponed* migrations with regards to 81 | deployment and generating migrations that are aware of this sequencing. 82 | 83 | A migration is assumed to be a *prerequisite* to deployment unless it contains 84 | a destructive operation or the migration has its ``stage`` class attribute set 85 | to ``Stage.POST_DEPLOY``. When this attribute is defined it will bypass 86 | ``operations`` based heuristics. 87 | 88 | e.g. this migration would be considered a *prerequisite* 89 | 90 | .. code:: python 91 | 92 | class Migration(migrations.Migration): 93 | operations = [ 94 | AddField('model', 'field', models.IntegerField(null=True)) 95 | ] 96 | 97 | while the following migrations would be *postponed* 98 | 99 | .. code:: python 100 | 101 | class Migration(migrations.Migration): 102 | operations = [ 103 | RemoveField('model', 'field'), 104 | ] 105 | 106 | .. code:: python 107 | 108 | from syzygy import Stage 109 | 110 | class Migration(migrations.Migration): 111 | stage = Stage.POST_DEPLOY 112 | 113 | operations = [ 114 | RunSQL(...), 115 | ] 116 | 117 | To take advantage of this new notion of migration stage the `migrate` command 118 | allows migrations meant to be run before a deployment to be targeted using 119 | `--pre-deploy` flag. 120 | 121 | What it does and doesn't do 122 | --------------------------- 123 | 124 | It does 125 | ^^^^^^^ 126 | - Introduce a notion of pre and post-deployment migrations and support their 127 | creation, management, and deployment sequencing through adjustments made to 128 | the ``makemigrations`` and ``migrate`` command. 129 | - Automatically split operations known to cause deployment sequencing issues 130 | in pre and post deployment stages. 131 | - Refuse the temptation to guess in the face of ambiguity and force developers 132 | to reflect about the sequencing of their operations when dealing with 133 | non-trival changes. It is meant to provide guardrails with safe quality of 134 | life defaults. 135 | 136 | It doesn't 137 | ^^^^^^^^^^ 138 | - Generate operations that are guaranteed to minimize contention on your 139 | database. You should investigate the usage of `database specific solutions`_ 140 | for that. 141 | - Allow developers to completely abstract the notion of sequencing of 142 | of operations. There are changes that are inherently unsafe or not deployable 143 | in an atomic manner and you should be prepared to deal with them. 144 | 145 | .. _`database specific solutions`: https://pypi.org/project/django-pg-zero-downtime-migrations/ 146 | 147 | Specialized operations 148 | ---------------------- 149 | 150 | Syzygy overrides the ``makemigrations`` command to automatically split 151 | and organize operations in a way that allows them to safely be applied 152 | in pre and post-deployment stages. 153 | 154 | Field addition 155 | ^^^^^^^^^^^^^^ 156 | 157 | When adding a field to an existing model Django will generate an 158 | ``AddField`` operation that roughly translates to the following SQL 159 | 160 | .. code:: sql 161 | 162 | ALTER TABLE "author" ADD COLUMN "dob" int NOT NULL DEFAULT 1988; 163 | ALTER TABLE "author" ALTER COLUMN "dob" DROP DEFAULT; 164 | 165 | Which isn't safe as the immediate removal of the database level ``DEFAULT`` 166 | prevents the code deployed at the time of migration application from inserting 167 | new records. 168 | 169 | In order to make this change safe syzygy splits the operation in two, a 170 | specialized ``AddField`` operation that performs the column addition without 171 | the ``DROP DEFAULT`` and follow up ``PostAddField`` operation that drops the 172 | database level default. The first is marked as ``Stage.PRE_DEPLOY`` and the 173 | second as ``Stage.POST_DEPLOY``. 174 | 175 | .. note:: 176 | 177 | On Django 5.0+ the specialized operations are respectively replaced by 178 | vanilla ``AddField`` and ``AlterField`` ones that make use of the newly 179 | introduced support for ``db_default`` feature. 180 | 181 | Field removal 182 | ^^^^^^^^^^^^^ 183 | 184 | When removing a field from an existing model Django will generate a 185 | ``RemoveField`` operation that roughly translates to the following SQL 186 | 187 | .. code:: sql 188 | 189 | ALTER TABLE "author" DROP COLUMN "dob"; 190 | 191 | Such operation cannot be run before deployment because it would cause 192 | any ``SELECT``, ``INSERT``, and ``UPDATE`` initiated by the pre-existing code 193 | to crash while doing it after deployment would cause ``INSERT`` crashes in the 194 | newly-deployed code that _forgot_ the existence of the field. 195 | 196 | In order to make this change safe syzygy splits the operation in two, a 197 | specialized ``PreRemoveField`` operation adds a database level ``DEFAULT`` to 198 | the column if a ``Field.default`` is present or make the field nullable 199 | otherwise and a second vanilla ``RemoveField`` operation. The first is marked as 200 | ``Stage.PRE_DEPLOY`` and the second as ``Stage.POST_DEPLOY`` just like any 201 | ``RemoveField``. 202 | 203 | The presence of a database level ``DEFAULT`` or the removal of the ``NOT NULL`` 204 | constraint ensures a smooth rollout sequence. 205 | 206 | .. note:: 207 | 208 | On Django 5.0+ the specialized ``PreRemoveField`` operation is replaced by 209 | a vanilla ``AlterField`` that make use of the newly introduced support for 210 | ``db_default`` feature. 211 | 212 | Checks 213 | ------ 214 | 215 | In order to prevent the creation of migrations mixing operations of different 216 | *stages* this package registers `system checks`_. These checks will generate an error 217 | for every migration with an ambiguous ``stage``. 218 | 219 | e.g. a migration mixing inferred stages would result in a check error 220 | 221 | .. code:: python 222 | 223 | class Migration(migrations.Migration): 224 | operations = [ 225 | AddField('model', 'other_field', models.IntegerField(null=True)), 226 | RemoveField('model', 'field'), 227 | ] 228 | 229 | By default, syzygy should *not* generate automatically migrations and you should 230 | only run into check failures when manually creating migrations or adding syzygy 231 | to an historical project. 232 | 233 | For migrations that are part of your project and trigger a failure of this check 234 | it is recommended to manually annotate them with proper ``stage: syzygy.stageStage`` 235 | annotations. For third party migrations you should refer to the following section. 236 | 237 | .. _`system checks`: https://docs.djangoproject.com/en/stable/topics/checks/ 238 | 239 | Third-party migrations 240 | ---------------------- 241 | 242 | As long as the adoption of migration stages concept is not generalized your 243 | project might depend on third-party apps containing migrations with an 244 | ambiguous sequence of operations. 245 | 246 | Since an explicit ``stage`` cannot be explicitly assigned by editing these 247 | migrations a fallback or an override stage can be specified through the 248 | respective ``MIGRATION_STAGES_FALLBACK`` and ``MIGRATION_STAGES_OVERRIDE`` 249 | settings. 250 | 251 | By default third-party app migrations with an ambiguous sequence of operations 252 | will fallback to ``Stage.PRE_DEPLOY`` but this behavior can be changed by 253 | setting ``MIGRATION_THIRD_PARTY_STAGES_FALLBACK`` to ``Stage.POST_DEPLOY`` or 254 | disabled by setting it to ``None``. 255 | 256 | .. note:: 257 | 258 | The third-party app detection logic relies on the ``site`` `Python module`_ 259 | and is known to not properly detect all kind of third-party Django 260 | applications. You should rely on ``MIGRATION_STAGES_FALLBACK`` and 261 | ``MIGRATION_STAGES_OVERRIDE`` to configure stages if it doesn't work for your 262 | setup. 263 | 264 | .. _`Python module`: https://docs.python.org/3/library/site.html 265 | 266 | Reverts 267 | ------- 268 | 269 | Migration revert are also supported and result in inverting the nature of 270 | migrations. A migration that is normally considered a *prerequisite* would then 271 | be *postponed* when reverted. 272 | 273 | CI Integration 274 | -------------- 275 | 276 | In order to ensure that no feature branch includes an ambiguous sequence of 277 | operations users are encouraged to include a job that attempts to run the 278 | ``migrate --pre-deploy`` command against a database that only includes the 279 | changes from the target branch. 280 | 281 | For example, given a feature branch ``add-shiny-feature`` and a target branch 282 | of ``main`` a script would look like 283 | 284 | .. code:: sh 285 | 286 | git checkout main 287 | python manage.py migrate 288 | git checkout add-shiny-feature 289 | python manage.py migrate --pre-deploy 290 | 291 | Assuming the feature branch contains a sequence of operations that cannot be 292 | applied in a single atomic deployment consisting of pre-deployment, deployment, 293 | and post-deployment stages the ``migrate --pre-deploy`` command will fail with 294 | an ``AmbiguousPlan`` exception detailing the ambiguity and resolution paths. 295 | 296 | Migration quorum 297 | ---------------- 298 | 299 | When deploying migrations to multiple clusters sharing the same database it's 300 | important that: 301 | 302 | 1. Migrations are applied only once 303 | 2. Pre-deployment migrations are applied before deployment in any clusters is 304 | takes place 305 | 3. Post-deployment migrations are only applied once all clusters are done 306 | deploying 307 | 308 | The built-in ``migrate`` command doesn't offer any guarantees with regards to 309 | serializability of invocations, in other words naively calling ``migrate`` from 310 | multiple clusters before or after a deployment could cause some migrations to 311 | be attempted to be applied twice. 312 | 313 | To circumvent this limitation Syzygy introduces a ``--quorum `` flag to the 314 | ``migrate`` command that allow clusters coordination to take place. 315 | 316 | When specified the ``migrate --quorum `` command will wait for at least 317 | ``N`` number invocations of ``migrate`` for the planned migrations before proceeding 318 | with applying them once and blocking on all callers until the operation completes. 319 | 320 | In order to use the ``--quorum`` feature you must configure the ``MIGRATION_QUORUM_BACKEND`` 321 | setting to point to a quorum backend such as cache based one provided by Sygyzy 322 | 323 | .. code:: python 324 | 325 | MIGRATION_QUORUM_BACKEND = 'syzygy.quorum.backends.cache.CacheQuorum' 326 | 327 | or 328 | 329 | .. code:: python 330 | 331 | CACHES = { 332 | ..., 333 | 'quorum': { 334 | ... 335 | }, 336 | } 337 | MIGRATION_QUORUM_BACKEND = { 338 | 'backend': 'syzygy.quorum.backends.cache.CacheQuorum', 339 | 'alias': 'quorum', 340 | } 341 | 342 | .. note:: 343 | 344 | In order for ``CacheQuorum`` to work properly in a distributed environment it 345 | must be pointed at a backend that supports atomic ``incr`` operations such as 346 | Memcached or Redis. 347 | 348 | 349 | Development 350 | ----------- 351 | 352 | Make your changes, and then run tests via tox: 353 | 354 | .. code:: sh 355 | 356 | tox 357 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [isort] 2 | combine_as_imports=true 3 | include_trailing_comma=true 4 | multi_line_output=3 5 | 6 | [coverage:run] 7 | source = syzygy 8 | branch = 1 9 | relative_files = 1 10 | 11 | [flake8] 12 | max-line-length = 119 13 | 14 | [mypy] 15 | plugins = mypy_django_plugin.main 16 | 17 | [mypy.plugins.django-stubs] 18 | django_settings_module = "tests.settings" 19 | 20 | [wheel] 21 | universal = 1 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import find_packages, setup 3 | 4 | with open("README.rst") as file_: 5 | long_description = file_.read() 6 | 7 | setup( 8 | name="django-syzygy", 9 | version="1.2.1", 10 | description="Deployment aware tooling for Django migrations.", 11 | long_description=long_description, 12 | long_description_content_type="text/x-rst", 13 | url="https://github.com/charettes/django-syzygy", 14 | author="Simon Charette", 15 | author_email="charette.s@gmail.com", 16 | install_requires=["Django>=4.2"], 17 | packages=find_packages(exclude=["tests", "tests.*"]), 18 | license="MIT License", 19 | classifiers=[ 20 | "Environment :: Web Environment", 21 | "Framework :: Django", 22 | "Framework :: Django :: 4.2", 23 | "Framework :: Django :: 5.0", 24 | "Framework :: Django :: 5.1", 25 | "Framework :: Django :: 5.2", 26 | "Intended Audience :: Developers", 27 | "License :: OSI Approved :: MIT License", 28 | "Operating System :: OS Independent", 29 | "Programming Language :: Python", 30 | "Programming Language :: Python :: 3.9", 31 | "Programming Language :: Python :: 3.10", 32 | "Programming Language :: Python :: 3.11", 33 | "Programming Language :: Python :: 3.12", 34 | "Programming Language :: Python :: 3.13", 35 | "Topic :: Software Development :: Libraries :: Python Modules", 36 | ], 37 | ) 38 | -------------------------------------------------------------------------------- /syzygy/__init__.py: -------------------------------------------------------------------------------- 1 | from .constants import Stage 2 | from .exceptions import AmbiguousPlan, AmbiguousStage 3 | 4 | __all__ = ("AmbiguousPlan", "AmbiguousStage", "Stage") 5 | 6 | 7 | default_app_config = "syzygy.apps.SyzygyConfig" 8 | -------------------------------------------------------------------------------- /syzygy/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.core import checks 3 | from django.core.signals import setting_changed 4 | 5 | from .checks import check_migrations 6 | from .conf import _configure, _watch_settings 7 | 8 | 9 | class SyzygyConfig(AppConfig): 10 | name = __package__ 11 | 12 | def ready(self): 13 | _configure() 14 | checks.register(check_migrations, "migrations") 15 | setting_changed.connect(_watch_settings) 16 | -------------------------------------------------------------------------------- /syzygy/autodetector.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import sys 3 | from typing import NamedTuple 4 | 5 | from django.db.migrations import operations 6 | from django.db.migrations.autodetector import ( 7 | MigrationAutodetector as _MigrationAutodetector, 8 | ) 9 | from django.db.migrations.operations.base import Operation 10 | from django.db.migrations.questioner import InteractiveMigrationQuestioner 11 | from django.db.models.fields import NOT_PROVIDED 12 | from django.utils.functional import cached_property 13 | 14 | from .compat import OperationDependency 15 | from .constants import Stage 16 | from .exceptions import AmbiguousStage 17 | from .operations import ( 18 | AlterField, 19 | RenameField, 20 | RenameModel, 21 | get_post_add_field_operation, 22 | get_pre_add_field_operation, 23 | get_pre_remove_field_operation, 24 | ) 25 | from .plan import partition_operations 26 | 27 | STAGE_SPLIT = "__stage__" 28 | 29 | 30 | class OperationStage(Operation): 31 | """ 32 | Fake operation that serves as a placeholder to break operations into 33 | multiple migrations. 34 | """ 35 | 36 | 37 | class StageDependency(NamedTuple): 38 | app_label: str 39 | operation: OperationStage 40 | 41 | 42 | class StagedOperationDependency(NamedTuple): 43 | app_label: str 44 | model_name: str 45 | operation: Operation 46 | 47 | 48 | class MigrationAutodetector(_MigrationAutodetector): 49 | """ 50 | Migration auto-detector that splits migrations containing sequence of 51 | operations incompatible with staged deployments. 52 | 53 | It works by inserting fake `Stage` operations into a fake __stage__ 54 | application since `_build_migration_list` will only split operations of a 55 | single application into multiple migrations if it has external 56 | dependencies. 57 | 58 | By creating a chain of external application dependencies between operations:: 59 | 60 | app.FirstOperation -> __stage__.OperationStage -> app.SecondOperation 61 | 62 | The auto-detector will generate a sequence of migrations of the form:: 63 | 64 | app.Migration1(operations=[FirstOperation]) 65 | __stage__.Migration1(operations=[OperationStage]) 66 | app.Migration2(operations=[FirstOperation]) 67 | 68 | And automatically remove the __stage__ migrations since it's a not 69 | an existing application. 70 | """ 71 | 72 | def __init__(self, *args, **kwargs): 73 | self.style = kwargs.pop("style", None) 74 | super().__init__(*args, **kwargs) 75 | 76 | @cached_property 77 | def has_interactive_questionner(self) -> bool: 78 | return not self.questioner.dry_run and isinstance( 79 | self.questioner, InteractiveMigrationQuestioner 80 | ) 81 | 82 | def add_operation(self, app_label, operation, dependencies=None, beginning=False): 83 | if isinstance(operation, operations.RenameField): 84 | print( 85 | self.style.WARNING( 86 | "Renaming a column from a database table actively relied upon might cause downtime " 87 | "during deployment.", 88 | ), 89 | file=sys.stderr, 90 | ) 91 | choice = self.questioner.defaults.get("ask_rename_field_stage", 1) 92 | if self.has_interactive_questionner: 93 | choice = self.questioner._choice_input( 94 | "Please choose an appropriate action to take:", 95 | [ 96 | ( 97 | f"Quit, and let me add a new {operation.model_name}.{operation.new_name} field meant " 98 | f"to be backfilled with {operation.model_name}.{operation.old_name} values" 99 | ), 100 | ( 101 | f"Assume the currently deployed code doesn't reference {app_label}.{operation.model_name} " 102 | f"on reachable code paths and mark the operation to be applied before deployment. " 103 | + self.style.MIGRATE_LABEL( 104 | "This might cause downtime if your assumption is wrong", 105 | ) 106 | ), 107 | ( 108 | f"Assume the newly deployed code doesn't reference {app_label}.{operation.model_name} on " 109 | "reachable code paths and mark the operation to be applied after deployment. " 110 | + self.style.MIGRATE_LABEL( 111 | "This might cause downtime if your assumption is wrong", 112 | ) 113 | ), 114 | ], 115 | ) 116 | if choice == 1: 117 | sys.exit(3) 118 | else: 119 | stage = Stage.PRE_DEPLOY if choice == 2 else Stage.POST_DEPLOY 120 | operation = RenameField.for_stage(operation, stage) 121 | if isinstance(operation, operations.RenameModel): 122 | from_db_table = ( 123 | self.from_state.models[app_label, operation.old_name_lower].options.get( 124 | "db_table" 125 | ) 126 | or f"{app_label}_{operation.old_name_lower}" 127 | ) 128 | to_db_table = self.to_state.models[ 129 | app_label, operation.new_name_lower 130 | ].options.get("db_table") 131 | if from_db_table != to_db_table: 132 | print( 133 | self.style.WARNING( 134 | "Renaming an actively relied on database table might cause downtime during deployment." 135 | ), 136 | file=sys.stderr, 137 | ) 138 | choice = self.questioner.defaults.get("ask_rename_model_stage", 1) 139 | if self.has_interactive_questionner: 140 | choice = self.questioner._choice_input( 141 | "Please choose an appropriate action to take:", 142 | [ 143 | ( 144 | f"Quit, and let me manually set {app_label}.{operation.new_name}.Meta.db_table to " 145 | f'"{from_db_table}" to avoid renaming its underlying table' 146 | ), 147 | ( 148 | f"Assume the currently deployed code doesn't reference " 149 | f"{app_label}.{operation.old_name} on reachable code paths and mark the operation to " 150 | "be applied before the deployment. " 151 | + self.style.MIGRATE_LABEL( 152 | "This might cause downtime if your assumption is wrong", 153 | ) 154 | ), 155 | ( 156 | f"Assume the newly deployed code doesn't reference {app_label}.{operation.new_name} " 157 | "on reachable code paths and mark the operation to be applied after the deployment. " 158 | + self.style.MIGRATE_LABEL( 159 | "This might cause downtime if your assumption is wrong", 160 | ) 161 | ), 162 | ], 163 | ) 164 | if choice == 1: 165 | sys.exit(3) 166 | else: 167 | stage = Stage.PRE_DEPLOY if choice == 2 else Stage.POST_DEPLOY 168 | operation = RenameModel.for_stage(operation, stage) 169 | elif isinstance(operation, operations.AlterField) and not operation.field.null: 170 | # Addition of not-NULL constraints must be performed post-deployment. 171 | from_field = self.from_state.models[ 172 | app_label, operation.model_name_lower 173 | ].fields[operation.name] 174 | if from_field.null: 175 | operation = AlterField.for_stage(operation, Stage.POST_DEPLOY) 176 | super().add_operation(app_label, operation, dependencies, beginning) 177 | 178 | def _generate_added_field(self, app_label, model_name, field_name): 179 | # Delegate most of the logic to super() ... 180 | super()._generate_added_field(app_label, model_name, field_name) 181 | old_add_field = self.generated_operations[app_label][-1] 182 | field = old_add_field.field 183 | if ( 184 | field.many_to_many 185 | or (field.null and not field.has_default()) 186 | or getattr(field, "db_default", NOT_PROVIDED) is not NOT_PROVIDED 187 | ): 188 | return 189 | # ... otherwise swap the added operation by an adjusted one. 190 | add_field = get_pre_add_field_operation( 191 | old_add_field.model_name, 192 | old_add_field.name, 193 | old_add_field.field, 194 | preserve_default=old_add_field.preserve_default, 195 | ) 196 | add_field._auto_deps = old_add_field._auto_deps 197 | self.generated_operations[app_label][-1] = add_field 198 | stage = OperationStage() 199 | self.add_operation( 200 | STAGE_SPLIT, 201 | stage, 202 | dependencies=[StagedOperationDependency(app_label, STAGE_SPLIT, add_field)], 203 | ) 204 | post_add_field = get_post_add_field_operation( 205 | model_name=model_name, 206 | name=field_name, 207 | field=field, 208 | preserve_default=add_field.preserve_default, 209 | ) 210 | super().add_operation( 211 | app_label, 212 | post_add_field, 213 | dependencies=[ 214 | StageDependency(STAGE_SPLIT, stage), 215 | ], 216 | ) 217 | 218 | def _generate_removed_field(self, app_label, model_name, field_name): 219 | field = self.from_state.models[app_label, model_name].fields[field_name] 220 | remove_default = field.default 221 | if ( 222 | # Nullable fields will use null if not specified. 223 | (remove_default is NOT_PROVIDED and field.null) 224 | # Fields with a db_default will use the value if not specified. 225 | or getattr(field, "db_default", NOT_PROVIDED) is not NOT_PROVIDED 226 | # Many-to-many fields are not backend by concrete columns. 227 | or field.many_to_many 228 | ): 229 | return super()._generate_removed_field(app_label, model_name, field_name) 230 | 231 | if remove_default is NOT_PROVIDED: 232 | if self.has_interactive_questionner: 233 | choice = self.questioner._choice_input( 234 | "You are trying to remove a non-nullable field '%s' from %s without a default; " 235 | "we can't do that (the database needs a default for inserts before the removal).\n" 236 | "Please select a fix:" % (field_name, model_name), 237 | [ 238 | ( 239 | "Provide a one-off default now (will be set at the " 240 | "database level in pre-deployment stage)" 241 | ), 242 | ( 243 | "Make the field temporarily nullable (attempts at reverting the " 244 | "field removal might fail)" 245 | ), 246 | ], 247 | ) 248 | if choice == 1: 249 | remove_default = self.questioner._ask_default() 250 | elif choice == 2: 251 | remove_default = None 252 | else: 253 | sys.exit(3) 254 | else: 255 | remove_default = self.questioner.defaults.get("ask_remove_default") 256 | if remove_default is not NOT_PROVIDED: 257 | field = field.clone() 258 | if remove_default is None: 259 | field.null = True 260 | else: 261 | field.default = remove_default 262 | pre_remove_field = get_pre_remove_field_operation( 263 | model_name=model_name, name=field_name, field=field 264 | ) 265 | self.add_operation(app_label, pre_remove_field) 266 | stage = OperationStage() 267 | self.add_operation( 268 | STAGE_SPLIT, 269 | stage, 270 | dependencies=[ 271 | StagedOperationDependency( 272 | app_label, STAGE_SPLIT, self.generated_operations[app_label][-1] 273 | ), 274 | ], 275 | ) 276 | self.add_operation( 277 | app_label, 278 | operations.RemoveField(model_name=model_name, name=field_name), 279 | dependencies=[ 280 | OperationDependency( 281 | app_label, 282 | model_name, 283 | field_name, 284 | OperationDependency.Type.REMOVE_ORDER_WRT, 285 | ), 286 | OperationDependency( 287 | app_label, 288 | model_name, 289 | field_name, 290 | OperationDependency.Type.ALTER_FOO_TOGETHER, 291 | ), 292 | StageDependency(STAGE_SPLIT, stage), 293 | ], 294 | ) 295 | 296 | def check_dependency(self, operation, dependency): 297 | if isinstance(dependency, (StageDependency, StagedOperationDependency)): 298 | return dependency.operation is operation 299 | return super().check_dependency(operation, dependency) 300 | 301 | def _build_migration_list(self, *args, **kwargs): 302 | # Ensure generated operations sequence for each apps are partitioned 303 | # by stage. 304 | for app_label, app_operations in list(self.generated_operations.items()): 305 | if app_label == STAGE_SPLIT: 306 | continue 307 | try: 308 | pre_operations, post_operations = partition_operations( 309 | app_operations, app_label 310 | ) 311 | except AmbiguousStage: 312 | operations_description = "".join( 313 | f"- {operation.describe()} \n" for operation in app_operations 314 | ) 315 | print( 316 | f'The auto-detected operations for the "{app_label}" ' 317 | "app cannot be partitioned into deployment stages:\n" 318 | f"{operations_description}", 319 | file=sys.stderr, 320 | ) 321 | if self.has_interactive_questionner: 322 | abort = ( 323 | self.questioner._choice_input( 324 | "", 325 | [ 326 | ( 327 | "Let `makemigrations` complete. You'll have to " 328 | "manually break you operations in migrations " 329 | "with non-ambiguous stages." 330 | ), 331 | ( 332 | "Abort `makemigrations`. You'll have to reduce " 333 | "the number of model changes before running " 334 | "`makemigrations` again." 335 | ), 336 | ], 337 | ) 338 | == 2 339 | ) 340 | else: 341 | abort = self.questioner.defaults.get("ask_ambiguous_abort", False) 342 | if abort: 343 | sys.exit(3) 344 | continue 345 | if pre_operations and post_operations: 346 | stage = OperationStage() 347 | self.add_operation( 348 | STAGE_SPLIT, 349 | stage, 350 | dependencies=[ 351 | StagedOperationDependency( 352 | app_label, STAGE_SPLIT, pre_operations[-1] 353 | ) 354 | ], 355 | ) 356 | post_operations[0]._auto_deps.append( 357 | StageDependency(STAGE_SPLIT, stage) 358 | ) 359 | # Assign updated operations as they might have be re-ordered by 360 | # `partition_operations`. 361 | self.generated_operations[app_label] = pre_operations + post_operations 362 | super()._build_migration_list(*args, **kwargs) 363 | # Remove all dangling references to stage migrations. 364 | if self.migrations.pop(STAGE_SPLIT, None): 365 | for migration in itertools.chain.from_iterable(self.migrations.values()): 366 | migration.dependencies = [ 367 | dependency 368 | for dependency in migration.dependencies 369 | if dependency[0] != STAGE_SPLIT 370 | ] 371 | -------------------------------------------------------------------------------- /syzygy/checks.py: -------------------------------------------------------------------------------- 1 | import os 2 | from importlib import import_module 3 | 4 | from django.apps import apps 5 | from django.core.checks import Error 6 | from django.db.migrations.loader import MigrationLoader 7 | from django.utils.module_loading import import_string 8 | 9 | from .plan import must_post_deploy_migration 10 | 11 | 12 | def check_migrations(app_configs, **kwargs): 13 | if app_configs is None: 14 | app_configs = apps.get_app_configs() 15 | errors = [] 16 | hint = ( 17 | "Assign an explicit stage to it, break its operation into multiple " 18 | "migrations if it's not already applied or define an explicit stage for " 19 | "it using `MIGRATION_STAGE_OVERRIDE` or `MIGRATION_STAGE_FALLBACK` if the " 20 | "migration is not under your control." 21 | ) 22 | for app_config in app_configs: 23 | # Most of the following code is taken from MigrationLoader.load_disk 24 | # while allowing non-global app_configs to be used. 25 | module_name, _explicit = MigrationLoader.migrations_module(app_config.label) 26 | if module_name is None: # pragma: no cover 27 | continue 28 | try: 29 | module = import_module(module_name) 30 | except ImportError: 31 | # This is not the place to deal with migration issues. 32 | continue 33 | if module.__file__ is None: 34 | # Skip empty migration folders, which can happen 35 | # for example when switching between branches. 36 | continue 37 | directory = os.path.dirname(module.__file__) 38 | migration_names = set() 39 | for name in os.listdir(directory): 40 | if name.endswith(".py"): 41 | import_name = name.rsplit(".", 1)[0] 42 | migration_names.add(import_name) 43 | for migration_name in migration_names: 44 | try: 45 | migration_class = import_string( 46 | f"{module_name}.{migration_name}.Migration" 47 | ) 48 | except ImportError: 49 | # This is not the place to deal with migration issues. 50 | continue 51 | migration = migration_class(migration_name, app_config.label) 52 | try: 53 | must_post_deploy_migration(migration) 54 | except ValueError as e: 55 | errors.append( 56 | Error( 57 | str(e), 58 | hint=hint, 59 | obj=(migration.app_label, migration.name), 60 | id="migrations.0001", 61 | ) 62 | ) 63 | return errors 64 | -------------------------------------------------------------------------------- /syzygy/compat.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | import django 4 | from django.utils.functional import cached_property 5 | 6 | field_db_default_supported = django.VERSION >= (5,) 7 | 8 | try: 9 | from django.db.migrations.autodetector import ( # type: ignore[attr-defined] 10 | OperationDependency, 11 | ) 12 | 13 | except ImportError: 14 | 15 | class OperationDependency( # type: ignore[no-redef] 16 | namedtuple("OperationDependency", "app_label model_name field_name type") 17 | ): 18 | class Type: 19 | CREATE = True 20 | REMOVE = False 21 | ALTER = "alter" 22 | REMOVE_ORDER_WRT = "order_wrt_unset" 23 | ALTER_FOO_TOGETHER = "foo_together_change" 24 | 25 | @cached_property 26 | def model_name_lower(self): 27 | return self.model_name.lower() 28 | 29 | @cached_property 30 | def field_name_lower(self): 31 | return self.field_name.lower() 32 | -------------------------------------------------------------------------------- /syzygy/conf.py: -------------------------------------------------------------------------------- 1 | import site 2 | from typing import Dict, Optional 3 | 4 | from django.apps import AppConfig, apps 5 | from django.conf import settings 6 | 7 | from .constants import Stage 8 | 9 | __all__ = ( 10 | "MIGRATION_STAGES_OVERRIDE", 11 | "MIGRATION_STAGES_FALLBACK", 12 | "is_third_party_app", 13 | ) 14 | 15 | MigrationStagesSetting = Dict[str, Stage] 16 | 17 | MIGRATION_STAGES_OVERRIDE: MigrationStagesSetting 18 | MIGRATION_STAGES_FALLBACK: MigrationStagesSetting 19 | 20 | 21 | def is_third_party_app(app: AppConfig) -> bool: 22 | """ 23 | Return whether or not the app config originates from a third-party 24 | package. 25 | """ 26 | for prefix in site.PREFIXES: 27 | if app.path.startswith(prefix): 28 | return True 29 | return False 30 | 31 | 32 | def _configure() -> None: 33 | global MIGRATION_STAGES_OVERRIDE 34 | global MIGRATION_STAGES_FALLBACK 35 | MIGRATION_STAGES_OVERRIDE = getattr(settings, "MIGRATION_STAGES_OVERRIDE", {}) 36 | MIGRATION_STAGES_FALLBACK = getattr(settings, "MIGRATION_STAGES_FALLBACK", {}) 37 | third_party_stages_fallback: Optional[Stage] = getattr( 38 | settings, "MIGRATION_THIRD_PARTY_STAGES_FALLBACK", Stage.PRE_DEPLOY 39 | ) 40 | if third_party_stages_fallback: 41 | for app in apps.get_app_configs(): 42 | if is_third_party_app(app): 43 | MIGRATION_STAGES_FALLBACK.setdefault( 44 | app.label, third_party_stages_fallback 45 | ) 46 | 47 | 48 | watched_settings = { 49 | "MIGRATION_STAGES_OVERRIDE", 50 | "MIGRATION_STAGES_FALLBACK", 51 | "MIGRATION_THIRD_PARTY_STAGES_FALLBACK", 52 | } 53 | 54 | 55 | def _watch_settings(setting, **kwargs): 56 | if setting in watched_settings: 57 | _configure() 58 | -------------------------------------------------------------------------------- /syzygy/constants.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class Stage(Enum): 5 | PRE_DEPLOY = "pre_deploy" 6 | POST_DEPLOY = "post_deployment" 7 | -------------------------------------------------------------------------------- /syzygy/exceptions.py: -------------------------------------------------------------------------------- 1 | class AmbiguousStage(ValueError): 2 | pass 3 | 4 | 5 | class AmbiguousPlan(ValueError): 6 | pass 7 | -------------------------------------------------------------------------------- /syzygy/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charettes/django-syzygy/6614c7d42c6eb6cb9c9c927630ccb2bbdb13f38b/syzygy/management/__init__.py -------------------------------------------------------------------------------- /syzygy/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charettes/django-syzygy/6614c7d42c6eb6cb9c9c927630ccb2bbdb13f38b/syzygy/management/commands/__init__.py -------------------------------------------------------------------------------- /syzygy/management/commands/makemigrations.py: -------------------------------------------------------------------------------- 1 | from django.core.management.commands import makemigrations 2 | 3 | from syzygy.autodetector import MigrationAutodetector 4 | 5 | 6 | class Command(makemigrations.Command): 7 | def add_arguments(self, parser): 8 | super().add_arguments(parser) 9 | parser.add_argument( 10 | "--disable-syzygy", 11 | action="store_true", 12 | help=( 13 | "Disable syzygy operation injection and stage splitting. " 14 | "Please report issues requiring usage of this flag upstream." 15 | ), 16 | ) 17 | 18 | def handle(self, *args, disable_syzygy, **options): 19 | if disable_syzygy: 20 | return super().handle(*args, **options) 21 | # Monkey-patch makemigrations.MigrationAutodetector since the command 22 | # doesn't allow it to be overridden in any other way. 23 | MigrationAutodetector_ = makemigrations.MigrationAutodetector 24 | style = self.style 25 | 26 | class StyledMigrationAutodetector(MigrationAutodetector): 27 | def __init__(self, *args, **kwargs): 28 | super().__init__(*args, **kwargs, style=style) 29 | 30 | if hasattr(self, "autodetector"): 31 | self.autodetector = StyledMigrationAutodetector 32 | else: 33 | makemigrations.MigrationAutodetector = StyledMigrationAutodetector 34 | try: 35 | super().handle(*args, **options) 36 | finally: 37 | if not hasattr(self, "autodetector"): 38 | makemigrations.MigrationAutodetector = MigrationAutodetector_ 39 | -------------------------------------------------------------------------------- /syzygy/management/commands/migrate.py: -------------------------------------------------------------------------------- 1 | import time 2 | from contextlib import contextmanager 3 | from datetime import timedelta 4 | from typing import Iterator 5 | 6 | from django.apps import apps 7 | from django.core.management import CommandError 8 | from django.core.management.commands import migrate 9 | from django.db import connections 10 | from django.db.migrations.exceptions import AmbiguityError 11 | from django.db.migrations.executor import MigrationExecutor 12 | 13 | from syzygy.constants import Stage 14 | from syzygy.plan import Plan, get_pre_deploy_plan, hash_plan 15 | from syzygy.quorum import ( 16 | QuorumDisolved, 17 | join_quorum, 18 | poll_quorum, 19 | sever_quorum, 20 | ) 21 | 22 | 23 | class PreDeployMigrationExecutor(MigrationExecutor): 24 | def migration_plan(self, targets, clean_start=False) -> Plan: 25 | plan = super().migration_plan(targets, clean_start=clean_start) 26 | if not clean_start: 27 | try: 28 | plan = get_pre_deploy_plan(plan) 29 | except ValueError as exc: 30 | raise CommandError(str(exc)) from exc 31 | return plan 32 | 33 | 34 | @contextmanager 35 | def _patch_executor(stage: Stage): 36 | """ 37 | Monkey-patch migrate.MigrationExecutor if necessary since the command 38 | doesn't allow it to be overridden in any other way. 39 | """ 40 | if stage is Stage.PRE_DEPLOY: 41 | migrate.MigrationExecutor = PreDeployMigrationExecutor # type: ignore 42 | try: 43 | yield 44 | finally: 45 | migrate.MigrationExecutor = MigrationExecutor # type: ignore 46 | 47 | 48 | class Command(migrate.Command): 49 | def add_arguments(self, parser): 50 | super().add_arguments(parser) 51 | parser.add_argument( 52 | "--pre-deploy", 53 | action="store_const", 54 | const=Stage.PRE_DEPLOY, 55 | default=Stage.POST_DEPLOY, 56 | dest="stage", 57 | help="Only run migrations staged for pre-deployment.", 58 | ) 59 | parser.add_argument( 60 | "--quorum", 61 | type=int, 62 | default=1, 63 | dest="quorum", 64 | help="Number of parties required to proceed with the migration plan.", 65 | ) 66 | parser.add_argument( 67 | "--quorum-timeout", 68 | type=int, 69 | default=int(timedelta(minutes=30).total_seconds()), 70 | help="Number of seconds to wait before giving up waiting for quorum.", 71 | ) 72 | 73 | def _get_plan(self, **options): # pragma: no cover 74 | # XXX: Unfortunate copy-pasta from migrate.Command.handle. 75 | 76 | # Get the database we're operating from 77 | db = options["database"] 78 | connection = connections[db] 79 | 80 | # Hook for backends needing any database preparation 81 | connection.prepare_database() 82 | # Work out which apps have migrations and which do not 83 | executor = migrate.MigrationExecutor( 84 | connection, self.migration_progress_callback 85 | ) 86 | 87 | # Raise an error if any migrations are applied before their dependencies. 88 | executor.loader.check_consistent_history(connection) 89 | 90 | # Before anything else, see if there's conflicting apps and drop out 91 | # hard if there are any 92 | conflicts = executor.loader.detect_conflicts() 93 | if conflicts: 94 | name_str = "; ".join( 95 | "%s in %s" % (", ".join(names), app) for app, names in conflicts.items() 96 | ) 97 | raise CommandError( 98 | "Conflicting migrations detected; multiple leaf nodes in the " 99 | "migration graph: (%s).\nTo fix them run " 100 | "'python manage.py makemigrations --merge'" % name_str 101 | ) 102 | 103 | # If they supplied command line arguments, work out what they mean. 104 | run_syncdb = options["run_syncdb"] 105 | if options["app_label"]: 106 | # Validate app_label. 107 | app_label = options["app_label"] 108 | try: 109 | apps.get_app_config(app_label) 110 | except LookupError as err: 111 | raise CommandError(str(err)) 112 | if run_syncdb: 113 | if app_label in executor.loader.migrated_apps: 114 | raise CommandError( 115 | "Can't use run_syncdb with app '%s' as it has migrations." 116 | % app_label 117 | ) 118 | elif app_label not in executor.loader.migrated_apps: 119 | raise CommandError("App '%s' does not have migrations." % app_label) 120 | 121 | if options["app_label"] and options["migration_name"]: 122 | migration_name = options["migration_name"] 123 | if migration_name == "zero": 124 | targets = [(app_label, None)] 125 | else: 126 | try: 127 | migration = executor.loader.get_migration_by_prefix( 128 | app_label, migration_name 129 | ) 130 | except AmbiguityError: 131 | raise CommandError( 132 | "More than one migration matches '%s' in app '%s'. " 133 | "Please be more specific." % (migration_name, app_label) 134 | ) 135 | except KeyError: 136 | raise CommandError( 137 | "Cannot find a migration matching '%s' from app '%s'." 138 | % (migration_name, app_label) 139 | ) 140 | targets = [(app_label, migration.name)] 141 | elif options["app_label"]: 142 | targets = [ 143 | key for key in executor.loader.graph.leaf_nodes() if key[0] == app_label 144 | ] 145 | else: 146 | targets = executor.loader.graph.leaf_nodes() 147 | 148 | return executor.migration_plan(targets) 149 | 150 | def _poll_until_quorum( 151 | self, namespace: str, quorum: int, quorum_timeout: int 152 | ) -> float: 153 | started_at = time.monotonic() 154 | while not poll_quorum(namespace, quorum): 155 | if (time.monotonic() - started_at) > quorum_timeout: 156 | raise RuntimeError("Migration plan quorum timeout") 157 | time.sleep(1) 158 | return time.monotonic() - started_at 159 | 160 | def _join_or_poll_until_quorum( 161 | self, namespace: str, quorum: int, quorum_timeout: int 162 | ) -> float: 163 | if join_quorum(namespace, quorum): 164 | return 0 165 | return self._poll_until_quorum(namespace, quorum, quorum_timeout) 166 | 167 | @contextmanager 168 | def _handle_quorum( 169 | self, quorum: int, quorum_timeout: int, options: dict 170 | ) -> Iterator[bool]: 171 | """ 172 | Context manager that handles migration application quorum by only 173 | allowing a single caller to proceed with application and preventing 174 | exit attempts until the application is completes. 175 | 176 | This ensures only a single invocation is allowed to proceed once 177 | quorum is reached and that context can only be exited once the 178 | invocation application succeeds. 179 | """ 180 | if quorum < 2: 181 | yield True 182 | return 183 | verbosity = options["verbosity"] 184 | plan = self._get_plan(**options) 185 | if not plan: 186 | yield True 187 | return 188 | database = options["database"] 189 | plan_hash = hash_plan(plan) 190 | pre_namespace = f"pre:{database}:{plan_hash}" 191 | post_namespace = f"post:{database}:{plan_hash}" 192 | if join_quorum(pre_namespace, quorum): 193 | if verbosity: 194 | self.stdout.write( 195 | "Reached pre-migrate quorum, proceeding with planned migrations..." 196 | ) 197 | try: 198 | yield True 199 | except: # noqa: E722 200 | # Bare except clause to capture any form of termination. 201 | self.stderr.write( 202 | "Encountered exception while applying migrations, disovling quorum." 203 | ) 204 | sever_quorum(post_namespace, quorum) 205 | raise 206 | if verbosity: 207 | self.stdout.write("Waiting for post-migrate quorum...") 208 | duration = self._join_or_poll_until_quorum( 209 | post_namespace, quorum, quorum_timeout 210 | ) 211 | if verbosity: 212 | self.stdout.write( 213 | f"Reached post-migrate quorum after {duration:.2f}s..." 214 | ) 215 | return 216 | yield False 217 | if verbosity: 218 | self.stdout.write("Waiting for pre-migrate quorum...") 219 | duration = self._poll_until_quorum(pre_namespace, quorum, quorum_timeout) 220 | if verbosity: 221 | self.stdout.write(f"Reached pre-migrate quorum after {duration:.2f}s...") 222 | self.stdout.write("Waiting for migrations to be applied by remote party...") 223 | try: 224 | duration = self._join_or_poll_until_quorum( 225 | post_namespace, quorum, quorum_timeout 226 | ) 227 | except QuorumDisolved as exc: 228 | raise CommandError( 229 | "Error encountered by remote party while applying migration, aborting." 230 | ) from exc 231 | if verbosity: 232 | self.stdout.write(f"Reached post-migrate quorum after {duration:.2f}s...") 233 | self.stdout.write("Migrations applied by remote party") 234 | return 235 | 236 | def handle( 237 | self, 238 | *args, 239 | stage: Stage, 240 | quorum: int, 241 | quorum_timeout: int, 242 | **options, 243 | ): 244 | with _patch_executor(stage), self._handle_quorum( 245 | quorum, quorum_timeout, options 246 | ) as proceed: 247 | if proceed: 248 | super().handle(*args, **options) 249 | -------------------------------------------------------------------------------- /syzygy/operations.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | 3 | from django.db import models 4 | from django.db.migrations import operations 5 | from django.db.models.fields import NOT_PROVIDED 6 | from django.utils.functional import cached_property 7 | 8 | from .compat import field_db_default_supported 9 | from .constants import Stage 10 | 11 | 12 | def _alter_field_db_default_sql_params(schema_editor, model, name, drop=False): 13 | field = model._meta.get_field(name) 14 | changes_sql, params = schema_editor._alter_column_default_sql( 15 | model, None, field, drop=drop 16 | ) 17 | sql = schema_editor.sql_alter_column % { 18 | "table": schema_editor.quote_name(model._meta.db_table), 19 | "changes": changes_sql, 20 | } 21 | return sql, params 22 | 23 | 24 | def _alter_field_db_default(schema_editor, model, name, drop=False): 25 | sql, params = _alter_field_db_default_sql_params( 26 | schema_editor, model, name, drop=drop 27 | ) 28 | schema_editor.execute(sql, params) 29 | 30 | 31 | @contextmanager 32 | def _force_field_alteration(schema_editor): 33 | # Django implements an optimization to prevent SQLite table rebuilds 34 | # when unnecessary. Until proper db_default alteration support lands this 35 | # optimization has to be disabled under some circumstances. 36 | _field_should_be_altered = schema_editor._field_should_be_altered 37 | schema_editor._field_should_be_altered = lambda old_field, new_field: True 38 | try: 39 | yield 40 | finally: 41 | schema_editor._field_should_be_altered = _field_should_be_altered 42 | 43 | 44 | @contextmanager 45 | def _include_column_default(schema_editor, field_name): 46 | column_sql_ = schema_editor.column_sql 47 | 48 | def column_sql(model, field, include_default=False): 49 | include_default |= field.name == field_name 50 | # XXX: SQLite doesn't support parameterized DDL but this isn't an 51 | # issue upstream since this method is never called with 52 | # `include_default=True` due to table rebuild. 53 | sql, params = column_sql_(model, field, include_default) 54 | return sql % tuple(params), () 55 | 56 | schema_editor.column_sql = column_sql 57 | try: 58 | with _force_field_alteration(schema_editor): 59 | yield 60 | finally: 61 | schema_editor.column_sql = column_sql_ 62 | 63 | 64 | class PreRemoveField(operations.AlterField): 65 | """ 66 | Perform database operations required to make sure an application with a 67 | rolling deployment won't crash prior to a field removal. 68 | 69 | If the field has a `default` value defined its corresponding column is 70 | altered to use it until the field is removed otherwise the field is made 71 | NULL'able if it's not already. 72 | """ 73 | 74 | def state_forwards(self, app_label, state): 75 | pass 76 | 77 | def database_forwards(self, app_label, schema_editor, from_state, to_state): 78 | to_state = to_state.clone() 79 | super().state_forwards(app_label, to_state) 80 | model = to_state.apps.get_model(app_label, self.model_name) 81 | if not self.allow_migrate_model(schema_editor.connection.alias, model): 82 | return 83 | # Ensure column meant to be removed have database level default defined 84 | # or is made NULL'able prior to removal to allow INSERT during the 85 | # deployment stage. 86 | field = model._meta.get_field(self.name) 87 | if field.default is not NOT_PROVIDED: 88 | if schema_editor.connection.vendor == "sqlite": 89 | with _include_column_default(schema_editor, self.name): 90 | super().database_forwards( 91 | app_label, schema_editor, from_state, to_state 92 | ) 93 | else: 94 | _alter_field_db_default(schema_editor, model, self.name) 95 | else: 96 | nullable_field = field.clone() 97 | nullable_field.null = True 98 | operation = operations.AlterField( 99 | self.model_name, self.name, nullable_field 100 | ) 101 | operation.state_forwards(app_label, to_state) 102 | operation.database_forwards(app_label, schema_editor, from_state, to_state) 103 | 104 | @property 105 | def migration_name_fragment(self): 106 | if self.field.default is not NOT_PROVIDED: 107 | return "set_db_default_%s_%s" % ( 108 | self.model_name_lower, 109 | self.name, 110 | ) 111 | return "set_nullable_%s_%s" % ( 112 | self.model_name_lower, 113 | self.name, 114 | ) 115 | 116 | def describe(self): 117 | if self.field.default is not NOT_PROVIDED: 118 | return "Set database DEFAULT of field %s on %s" % ( 119 | self.name, 120 | self.model_name, 121 | ) 122 | return "Set field %s of %s NULLable" % (self.name, self.model_name) 123 | 124 | 125 | if field_db_default_supported: 126 | 127 | def get_pre_remove_field_operation(model_name, name, field): 128 | if field.db_default is not NOT_PROVIDED: 129 | raise ValueError( 130 | "Fields with a db_default don't require a pre-deployment operation." 131 | ) 132 | field = field.clone() 133 | if field.has_default(): 134 | field.db_default = field.get_default() 135 | fragment = f"set_db_default_{model_name.lower()}_{name}" 136 | description = f"Set database DEFAULT of field {name} on {model_name}" 137 | else: 138 | field.null = True 139 | fragment = f"set_nullable_{model_name.lower()}_{name}" 140 | description = f"Set field {name} of {model_name} NULLable" 141 | operation = AlterField(model_name, name, field, stage=Stage.PRE_DEPLOY) 142 | operation.migration_name_fragment = fragment 143 | operation.describe = lambda: description 144 | return operation 145 | 146 | # XXX: Shim kept for historical migrations generated before Django 5. 147 | PreRemoveField = get_pre_remove_field_operation # type: ignore[assignment,misc] # noqa: F811 148 | else: 149 | get_pre_remove_field_operation = PreRemoveField 150 | 151 | 152 | class AddField(operations.AddField): 153 | """ 154 | Subclass of `AddField` that preserves the database default on database 155 | application. 156 | """ 157 | 158 | @contextmanager 159 | def _prevent_drop_default(self, schema_editor, model): 160 | # On other backends the most straightforward way 161 | drop_default_sql_params = _alter_field_db_default_sql_params( 162 | schema_editor, model, self.name, drop=True 163 | ) 164 | execute_ = schema_editor.execute 165 | 166 | def execute(sql, params=()): 167 | if (sql, params) == drop_default_sql_params: 168 | return 169 | return execute_(sql, params) 170 | 171 | schema_editor.execute = execute 172 | try: 173 | yield 174 | finally: 175 | schema_editor.execute = execute_ 176 | 177 | def _preserve_column_default(self, schema_editor, model): 178 | # XXX: Hopefully future support for `Field.db_default` will add better 179 | # injection points to `BaseSchemaEditor.add_field`. 180 | if schema_editor.connection.vendor == "sqlite": 181 | # On the SQLite backend the strategy is different since it emulates 182 | # ALTER support by rebuilding tables. 183 | return _include_column_default(schema_editor, self.name) 184 | return self._prevent_drop_default(schema_editor, model) 185 | 186 | def database_forwards(self, app_label, schema_editor, from_state, to_state): 187 | model = to_state.apps.get_model(app_label, self.model_name) 188 | if not self.allow_migrate_model(schema_editor.connection.alias, model): 189 | return 190 | # Defer the removal of DEFAUT to `PostAddField` 191 | with self._preserve_column_default(schema_editor, model): 192 | return super().database_forwards( 193 | app_label, schema_editor, from_state, to_state 194 | ) 195 | 196 | 197 | if field_db_default_supported: 198 | 199 | def get_pre_add_field_operation(model_name, name, field, preserve_default=True): 200 | if field.db_default is not NOT_PROVIDED: 201 | raise ValueError( 202 | "Fields with a db_default don't require a pre-deployment operation." 203 | ) 204 | field = field.clone() 205 | if isinstance(field, models.ForeignKey): 206 | # XXX: Replicate ForeignKey.get_default() logic in a way that 207 | # doesn't require the field references to be pre-emptively 208 | # resolved. This will need to be fixed upstream if we ever 209 | # implement model state schema alterations. 210 | # See https://code.djangoproject.com/ticket/29898 211 | field_default = super(models.ForeignKey, field).get_default() 212 | if ( 213 | isinstance(field_default, models.Model) 214 | and field_default._meta.label_lower == field.related_model.lower() 215 | ): 216 | target_field = field.to_fields[0] or "pk" 217 | field_default = getattr(field_default, target_field) 218 | else: 219 | field_default = field.get_default() 220 | field.db_default = field_default 221 | operation = operations.AddField(model_name, name, field, preserve_default) 222 | return operation 223 | 224 | # XXX: Shim kept for historical migrations generated before Django 5. 225 | AddField = get_pre_add_field_operation # type: ignore[assignment,misc] # noqa: F811 226 | else: 227 | get_pre_add_field_operation = AddField 228 | 229 | 230 | class PostAddField(operations.AlterField): 231 | """ 232 | Elidable operation that drops a previously preserved database default. 233 | """ 234 | 235 | stage = Stage.POST_DEPLOY 236 | 237 | def state_forwards(self, app_label, state): 238 | pass 239 | 240 | def database_forwards(self, app_label, schema_editor, from_state, to_state): 241 | model = to_state.apps.get_model(app_label, self.model_name) 242 | if not self.allow_migrate_model(schema_editor.connection.alias, model): 243 | return 244 | if schema_editor.connection.vendor == "sqlite": 245 | # Trigger a table rebuild to DROP the database level DEFAULT 246 | with _force_field_alteration(schema_editor): 247 | super().database_forwards( 248 | app_label, schema_editor, from_state, to_state 249 | ) 250 | else: 251 | _alter_field_db_default(schema_editor, model, self.name, drop=True) 252 | 253 | def database_backwards(self, app_label, schema_editor, from_state, to_state): 254 | to_state = to_state.clone() 255 | super().state_forwards(app_label, to_state) 256 | model = to_state.apps.get_model(app_label, self.model_name) 257 | if not self.allow_migrate_model(schema_editor.connection.alias, model): 258 | return 259 | if schema_editor.connection.vendor == "sqlite": 260 | with _include_column_default(schema_editor, self.name): 261 | super().database_forwards( 262 | app_label, schema_editor, from_state, to_state 263 | ) 264 | else: 265 | _alter_field_db_default(schema_editor, model, self.name) 266 | 267 | @property 268 | def migration_name_fragment(self): 269 | return "drop_db_default_%s_%s" % ( 270 | self.model_name_lower, 271 | self.name, 272 | ) 273 | 274 | def describe(self): 275 | return "Drop database DEFAULT of field %s on %s" % ( 276 | self.name, 277 | self.model_name, 278 | ) 279 | 280 | 281 | if field_db_default_supported: 282 | 283 | def get_post_add_field_operation(model_name, name, field, preserve_default=True): 284 | if field.db_default is not NOT_PROVIDED: 285 | raise ValueError( 286 | "Fields with a db_default don't require a post-deployment operation." 287 | ) 288 | field = field.clone() 289 | field.db_default = NOT_PROVIDED 290 | if not preserve_default: 291 | field.default = NOT_PROVIDED 292 | operation = AlterField( 293 | model_name, 294 | name, 295 | field, 296 | stage=Stage.POST_DEPLOY, 297 | ) 298 | operation.migration_name_fragment = ( 299 | f"drop_db_default_{model_name.lower()}_{name}" 300 | ) 301 | operation.describe = ( 302 | lambda: f"Drop database DEFAULT of field {name} on {model_name}" 303 | ) 304 | return operation 305 | 306 | # XXX: Shim kept for historical migrations generated before Django 5. 307 | PostAddField = get_post_add_field_operation # type: ignore[assignment,misc] # noqa: F811 308 | else: 309 | get_post_add_field_operation = PostAddField 310 | 311 | 312 | class StagedOperation(operations.base.Operation): 313 | stage: Stage 314 | 315 | def __init__(self, *args, **kwargs): 316 | self.stage = kwargs.pop("stage") 317 | super().__init__(*args, **kwargs) 318 | 319 | @classmethod 320 | def for_stage(cls, operation: operations.base.Operation, stage: Stage): 321 | _, args, kwargs = operation.deconstruct() 322 | kwargs["stage"] = stage 323 | return cls(*args, **kwargs) 324 | 325 | def deconstruct(self): 326 | name, args, kwargs = super().deconstruct() 327 | kwargs["stage"] = self.stage 328 | return name, args, kwargs 329 | 330 | 331 | class RenameField(StagedOperation, operations.RenameField): 332 | """ 333 | Subclass of ``RenameField`` that explicitly defines a stage for the rare 334 | instances where a rename operation is safe to perform. 335 | """ 336 | 337 | # XXX: Explicitly define the signature as migration serializer rely on 338 | # __init__ introspection to assign the kwargs. 339 | def __init__(self, model_name, old_name, new_name, stage): 340 | super().__init__( 341 | model_name=model_name, old_name=old_name, new_name=new_name, stage=stage 342 | ) 343 | 344 | 345 | class RenameModel(StagedOperation, operations.RenameModel): 346 | """ 347 | Subclass of ``RenameModel`` that explicitly defines a stage for the rare 348 | instances where a rename operation is safe to perform. 349 | """ 350 | 351 | # XXX: Explicitly define the signature as migration serializer rely on 352 | # __init__ introspection to assign the kwargs. 353 | def __init__(self, old_name, new_name, stage): 354 | super().__init__(old_name=old_name, new_name=new_name, stage=stage) 355 | 356 | 357 | class AlterField(StagedOperation, operations.AlterField): 358 | """ 359 | Subclass of ``AlterField`` that allows explicitly defining a stage. 360 | """ 361 | 362 | # XXX: Explicitly define the signature as migration serializer rely on 363 | # __init__ introspection to assign the kwargs. 364 | def __init__(self, model_name, name, field, stage, preserve_default=True): 365 | super().__init__( 366 | model_name=model_name, 367 | name=name, 368 | field=field, 369 | stage=stage, 370 | preserve_default=preserve_default, 371 | ) 372 | 373 | @cached_property 374 | def migration_name_fragment(self): 375 | # Redefine as a `cached_property` to allow `get_pre_remove_field_operation` 376 | # to assign a more appropriate value. 377 | return super().migration_name_fragment 378 | -------------------------------------------------------------------------------- /syzygy/plan.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from typing import Dict, List, Optional, Tuple 3 | 4 | from django.apps import apps 5 | from django.db.migrations import DeleteModel, Migration, RemoveField 6 | from django.db.migrations.operations.base import Operation 7 | 8 | from . import conf 9 | from .constants import Stage 10 | from .exceptions import AmbiguousPlan, AmbiguousStage 11 | 12 | Plan = List[Tuple[Migration, bool]] 13 | 14 | 15 | def hash_plan(plan: Plan) -> str: 16 | """Return a stable hash from a migration plan.""" 17 | return hashlib.sha1( 18 | ";".join(f"{migration}:{backward}" for migration, backward in plan).encode() 19 | ).hexdigest() 20 | 21 | 22 | def get_operation_stage(operation: Operation) -> Stage: 23 | """Return the heuristically determined `Stage` of the operation.""" 24 | try: 25 | stage = operation.stage # type: ignore 26 | except AttributeError: 27 | pass 28 | else: 29 | return Stage(stage) 30 | if isinstance(operation, (DeleteModel, RemoveField)): 31 | return Stage.POST_DEPLOY 32 | return Stage.PRE_DEPLOY 33 | 34 | 35 | def partition_operations( 36 | operations: List[Operation], 37 | app_label: str, 38 | ) -> Tuple[List[Operation], List[Operation]]: 39 | """ 40 | Partition an ordered list of operations by :class:`syzygy.constants.Stage.PRE_DEPLOY`. 41 | 42 | If `operations` contains 43 | :attr:`syzygy.constants.Stage.POST_DEPLOY` stage members followed 44 | :attr:`syzygy.constants.Stage.PRE_DEPLOY` stage members and they cannot be 45 | reordered a :class:`syzygy.exceptions.AmbiguousStage` exception will be 46 | raised. 47 | """ 48 | stage_operations: Dict[Stage, List[Operation]] = { 49 | Stage.PRE_DEPLOY: [], 50 | Stage.POST_DEPLOY: [], 51 | } 52 | post_deploy_operations = stage_operations[Stage.POST_DEPLOY] 53 | for operation in operations: 54 | operation_stage = get_operation_stage(operation) 55 | if operation_stage is Stage.PRE_DEPLOY and post_deploy_operations: 56 | # Attempt to re-order `operation` if a pre-deploy stage one is 57 | # encountered after a post-deployment one if allowed. 58 | if all( 59 | op.reduce(operation, app_label) is True for op in post_deploy_operations 60 | ): 61 | stage_operations[Stage.PRE_DEPLOY].append(operation) 62 | continue 63 | raise AmbiguousStage( 64 | "Post-deployment operations cannot be followed by " 65 | "pre-deployments operations" 66 | ) 67 | stage_operations[operation_stage].append(operation) 68 | return stage_operations[Stage.PRE_DEPLOY], post_deploy_operations 69 | 70 | 71 | def _get_migration_stage_override(migration: Migration) -> Optional[Stage]: 72 | """ 73 | Return the `Stage` override configured through setting:`MIGRATION_STAGES_OVERRIDE` 74 | of the migration. 75 | """ 76 | override = conf.MIGRATION_STAGES_OVERRIDE 77 | return override.get(f"{migration.app_label}.{migration.name}") or override.get( 78 | migration.app_label 79 | ) 80 | 81 | 82 | def _get_migration_stage_fallback(migration: Migration) -> Optional[Stage]: 83 | """ 84 | Return the `Stage` fallback configured through setting:`MIGRATION_STAGES_FALLBACK` 85 | of the migration. 86 | """ 87 | fallback = conf.MIGRATION_STAGES_FALLBACK 88 | return fallback.get(f"{migration.app_label}.{migration.name}") or fallback.get( 89 | migration.app_label 90 | ) 91 | 92 | 93 | def _get_defined_stage(migration: Migration) -> Optional[Stage]: 94 | """ 95 | Return the explicitly defined `Stage` of a migration or 96 | `None` if not defined. 97 | """ 98 | return getattr(migration, "stage", None) or _get_migration_stage_override(migration) 99 | 100 | 101 | def get_migration_stage(migration: Migration) -> Optional[Stage]: 102 | """ 103 | Return the `Stage` of the migration. 104 | 105 | If not specified through setting:`MIGRATION_STAGES` or a `stage` 106 | :class:`django.db.migrations.Migration` class attribute it will be 107 | tentatively deduced from its list of 108 | attr:`django.db.migrations.Migration.operations`. 109 | 110 | If the migration doesn't have any `operations` then `None` will be returned 111 | and a :class:`syzygy.exceptions.AmbiguousStage` exception will be raised 112 | if it contains operations of mixed stages. 113 | """ 114 | stage = _get_defined_stage(migration) 115 | if stage is not None: 116 | return stage 117 | for operation in migration.operations: 118 | operation_stage = get_operation_stage(operation) 119 | if stage is None: 120 | stage = operation_stage 121 | elif operation_stage != stage: 122 | fallback_stage = _get_migration_stage_fallback(migration) 123 | if fallback_stage: 124 | stage = fallback_stage 125 | break 126 | raise AmbiguousStage( 127 | f"Cannot automatically determine stage of {migration}." 128 | ) 129 | return stage 130 | 131 | 132 | def must_post_deploy_migration( 133 | migration: Migration, backward: bool = False 134 | ) -> Optional[bool]: 135 | """ 136 | Return whether or not migration must be run after deployment. 137 | 138 | If not specified through a `stage` :class:`django.db.migrations.Migration` 139 | class attribute it will be tentatively deduced from its list of 140 | attr:`django.db.migrations.Migration.operations`. 141 | 142 | In cases of ambiguity a :class:`syzygy.exceptions.AmbiguousStage` exception 143 | will be raised. 144 | """ 145 | migration_stage = get_migration_stage(migration) 146 | if migration_stage is None: 147 | return None 148 | if migration_stage is Stage.PRE_DEPLOY: 149 | return backward 150 | return not backward 151 | 152 | 153 | def get_pre_deploy_plan(plan: Plan) -> Plan: 154 | """ 155 | Trim provided plan to its leading contiguous pre-deployment sequence. 156 | 157 | If the plan contains non-contiguous sequence of pre-deployment migrations 158 | or migrations with ambiguous deploy stage a :class:`syzygy.exceptions.AmbiguousPlan` 159 | exception is raised. 160 | """ 161 | pre_deploy_plan: Plan = [] 162 | post_deploy_plan = {} 163 | for migration, backward in plan: 164 | if must_post_deploy_migration(migration, backward): 165 | post_deploy_plan[migration.app_label, migration.name] = migration 166 | else: 167 | post_deploy_dep = None 168 | if post_deploy_plan: 169 | post_deploy_dep = next( 170 | ( 171 | post_deploy_plan[dependency] 172 | for dependency in migration.dependencies 173 | if dependency in post_deploy_plan 174 | ), 175 | None, 176 | ) 177 | if post_deploy_dep: 178 | inferred = [] 179 | stage_defined = _get_defined_stage(migration) is not None 180 | post_stage_defined = _get_defined_stage(post_deploy_dep) is not None 181 | if stage_defined: 182 | stage_origin = "defined" 183 | else: 184 | stage_origin = "inferred" 185 | inferred.append(migration) 186 | if post_stage_defined: 187 | post_stage_origin = "defined" 188 | else: 189 | post_stage_origin = "inferred" 190 | inferred.append(post_deploy_dep) 191 | msg = ( 192 | f"Plan contains a non-contiguous sequence of pre-deployment " 193 | f"migrations. Migration {migration} is {stage_origin} to be applied " 194 | f"pre-deployment but it depends on {post_deploy_dep} which is " 195 | f"{post_stage_origin} to be applied post-deployment." 196 | ) 197 | if inferred: 198 | first_party_inferred = [] 199 | third_party_inferred = [] 200 | for migration in inferred: 201 | try: 202 | app = apps.get_app_config(migration.app_label) 203 | except LookupError: 204 | pass 205 | else: 206 | if conf.is_third_party_app(app): 207 | third_party_inferred.append(str(migration)) 208 | continue 209 | first_party_inferred.append(str(migration)) 210 | if first_party_inferred: 211 | first_party_names = " or ".join(first_party_inferred) 212 | msg += ( 213 | f" Defining an explicit `Migration.stage: syzygy.Stage` " 214 | f"for {first_party_names} " 215 | ) 216 | if third_party_inferred: 217 | if first_party_inferred: 218 | msg += "or setting " 219 | else: 220 | msg += " Setting " 221 | msg += " or ".join( 222 | f"`MIGRATION_STAGES_OVERRIDE[{migration_name!r}]`" 223 | for migration_name in third_party_inferred 224 | ) 225 | msg += " to an explicit `syzygy.Stage` " 226 | msg += "to bypass inference might help." 227 | for migration in inferred: 228 | try: 229 | app = apps.get_app_config(migration.app_label) 230 | except LookupError: 231 | continue 232 | if not conf.is_third_party_app(app): 233 | continue 234 | break 235 | raise AmbiguousPlan(msg) 236 | pre_deploy_plan.append((migration, backward)) 237 | return pre_deploy_plan 238 | -------------------------------------------------------------------------------- /syzygy/quorum/__init__.py: -------------------------------------------------------------------------------- 1 | from functools import lru_cache 2 | from typing import Union 3 | 4 | from django.conf import settings 5 | from django.core.exceptions import ImproperlyConfigured 6 | from django.utils.module_loading import import_string 7 | 8 | from .backends.base import QuorumBase 9 | from .exceptions import QuorumDisolved 10 | 11 | __all__ = ("join_quorum", "sever_quorum", "poll_quorum", "QuorumDisolved") 12 | 13 | 14 | @lru_cache(maxsize=1) 15 | def _get_quorum(backend_path, **backend_options) -> QuorumBase: 16 | backend_cls = import_string(backend_path) 17 | return backend_cls(**backend_options) 18 | 19 | 20 | def _get_configured_quorum() -> QuorumBase: 21 | try: 22 | config: Union[dict, str] = settings.MIGRATION_QUORUM_BACKEND # type: ignore 23 | except AttributeError: 24 | raise ImproperlyConfigured( 25 | "The `MIGRATION_QUORUM_BACKEND` setting must be configured " 26 | "for syzygy.quorum to be used" 27 | ) 28 | backend_path: str 29 | backend_options: dict 30 | if isinstance(config, str): 31 | backend_path = config 32 | backend_options = {} 33 | elif isinstance(config, dict) and config.get("backend"): 34 | backend_options = config.copy() 35 | backend_path = backend_options.pop("backend") 36 | else: 37 | raise ImproperlyConfigured( 38 | "The `MIGRATION_QUORUM_BACKEND` setting must either be an import " 39 | "path string or a dict with a 'backend' path key string" 40 | ) 41 | try: 42 | return _get_quorum(backend_path, **backend_options) 43 | except ImportError as exc: 44 | raise ImproperlyConfigured( 45 | f"Cannot import `MIGRATION_QUORUM_BACKEND` backend '{backend_path}'" 46 | ) from exc 47 | except TypeError as exc: 48 | raise ImproperlyConfigured( 49 | f"Cannot initialize `MIGRATION_QUORUM_BACKEND` backend '{backend_path}' " 50 | f"with {backend_options!r}" 51 | ) from exc 52 | 53 | 54 | def join_quorum(namespace: str, quorum: int) -> bool: 55 | return _get_configured_quorum().join(namespace=namespace, quorum=quorum) 56 | 57 | 58 | def sever_quorum(namespace: str, quorum: int) -> bool: 59 | return _get_configured_quorum().sever(namespace=namespace, quorum=quorum) 60 | 61 | 62 | def poll_quorum(namespace: str, quorum: int) -> bool: 63 | return _get_configured_quorum().poll(namespace=namespace, quorum=quorum) 64 | -------------------------------------------------------------------------------- /syzygy/quorum/backends/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charettes/django-syzygy/6614c7d42c6eb6cb9c9c927630ccb2bbdb13f38b/syzygy/quorum/backends/__init__.py -------------------------------------------------------------------------------- /syzygy/quorum/backends/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class QuorumBase(ABC): 5 | """ 6 | An abstraction to allow multiple parties to obtain quorum before 7 | proceeding with a coordinated action. 8 | 9 | For a particular `namespace` the `join` method must be called 10 | `quorum` times and `poll` must be called ``quorum - 1`` times or 11 | for each party that got returned `False` when calling `join`. 12 | 13 | The `sever` method must be called instead of `join` by parties 14 | that do not intend to participate in attaining `namespace`'s quorum. 15 | It must result in other parties raising `QuorumDisolved` on their next 16 | `poll` for that `namespace`. 17 | """ 18 | 19 | @abstractmethod 20 | def join(self, namespace: str, quorum: int) -> bool: 21 | """Join the `namespace` and return whether or not `quorum` was attained.""" 22 | 23 | @abstractmethod 24 | def sever(self, namespace: str, quorum: int): 25 | """Sever the `namespace`'s quorum attainment process.""" 26 | 27 | @abstractmethod 28 | def poll(self, namespace: str, quorum: int) -> bool: 29 | """ 30 | Return whether or not `namespace`'s `quorum` was reached. 31 | 32 | Raise `QuorumDisolved` if the quorom attainment process was 33 | severed. 34 | """ 35 | -------------------------------------------------------------------------------- /syzygy/quorum/backends/cache.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple 2 | 3 | from django.core.cache import DEFAULT_CACHE_ALIAS, caches 4 | 5 | from ..exceptions import QuorumDisolved 6 | from .base import QuorumBase 7 | 8 | 9 | class CacheQuorum(QuorumBase): 10 | namespace_key_format = "syzygy-quorum:{namespace}" 11 | 12 | def __init__( 13 | self, 14 | alias: str = DEFAULT_CACHE_ALIAS, 15 | timeout: int = 3600, 16 | version: Optional[int] = None, 17 | ): 18 | self.cache = caches[alias] 19 | self.timeout = timeout 20 | self.version = version 21 | 22 | @classmethod 23 | def _get_namespace_keys(cls, namespace: str) -> Tuple[str, str]: 24 | namespace_key = cls.namespace_key_format.format(namespace=namespace) 25 | clear_namespace_key = f"{namespace_key}:clear" 26 | return namespace_key, clear_namespace_key 27 | 28 | def _clear(self, namespace: str): 29 | self.cache.delete_many( 30 | self._get_namespace_keys(namespace), version=self.version 31 | ) 32 | 33 | def join(self, namespace: str, quorum: int) -> bool: 34 | namespace_key, clear_namespace_key = self._get_namespace_keys(namespace) 35 | self.cache.add(namespace_key, 0, timeout=self.timeout, version=self.version) 36 | self.cache.add( 37 | clear_namespace_key, 38 | quorum - 1, 39 | timeout=self.timeout, 40 | version=self.version, 41 | ) 42 | current = self.cache.incr(namespace_key, version=self.version) 43 | if current == quorum: 44 | return True 45 | return False 46 | 47 | def sever(self, namespace: str, quorum: int): 48 | namespace_key, _ = self._get_namespace_keys(namespace) 49 | self.cache.add(namespace_key, 0, timeout=self.timeout, version=self.version) 50 | self.cache.decr(namespace_key, quorum, version=self.version) 51 | 52 | def poll(self, namespace: str, quorum: int) -> bool: 53 | namespace_key, clear_namespace_key = self._get_namespace_keys(namespace) 54 | current = self.cache.get(namespace_key, version=self.version) 55 | if current == quorum: 56 | if self.cache.decr(clear_namespace_key, version=self.version) == 0: 57 | self._clear(namespace) 58 | return True 59 | elif current <= 0: 60 | if self.cache.decr(clear_namespace_key, version=self.version) == 0: 61 | self._clear(namespace) 62 | raise QuorumDisolved 63 | return False 64 | -------------------------------------------------------------------------------- /syzygy/quorum/exceptions.py: -------------------------------------------------------------------------------- 1 | class QuorumDisolved(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charettes/django-syzygy/6614c7d42c6eb6cb9c9c927630ccb2bbdb13f38b/tests/__init__.py -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class Foo(models.Model): 5 | pass 6 | 7 | 8 | class Bar(models.Model): 9 | name = models.CharField(max_length=100, unique=True) 10 | 11 | class Meta: 12 | managed = False 13 | -------------------------------------------------------------------------------- /tests/settings/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Tuple 2 | 3 | SECRET_KEY = "not-secret-anymore" 4 | 5 | TIME_ZONE = "America/Montreal" 6 | 7 | DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3"}} 8 | 9 | INSTALLED_APPS = ["syzygy", "tests"] 10 | 11 | SYZYGY_POSTPONE: Dict[Tuple[str, str], bool] = {} 12 | 13 | DEFAULT_AUTO_FIELD = "django.db.models.AutoField" 14 | -------------------------------------------------------------------------------- /tests/settings/mysql.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from . import * # noqa 4 | 5 | DB_HOST = os.environ.get("DB_MYSQL_HOST", "127.0.0.1") 6 | DB_PORT = os.environ.get("DB_MYSQL_PORT", "3306") 7 | DB_USER = os.environ.get("DB_MYSQL_USER", "mysql") 8 | DB_PASSWORD = os.environ.get("DB_MYSQL_PASSWORD", "mysql") 9 | 10 | 11 | DATABASES = { 12 | "default": { 13 | "ENGINE": "django.db.backends.mysql", 14 | "HOST": DB_HOST, 15 | "PORT": DB_PORT, 16 | "NAME": "syzygy", 17 | "USER": DB_USER, 18 | "PASSWORD": DB_PASSWORD, 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tests/settings/postgresql.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from . import * # noqa 4 | 5 | DB_HOST = os.environ.get("DB_POSTGRES_HOST", "localhost") 6 | DB_PORT = os.environ.get("DB_POSTGRES_PORT", "5432") 7 | DB_USER = os.environ.get("DB_POSTGRES_USER", "postgres") 8 | DB_PASSWORD = os.environ.get("DB_POSTGRES_PASSWORD", "postgres") 9 | 10 | DATABASES = { 11 | "default": { 12 | "ENGINE": "django.db.backends.postgresql", 13 | "HOST": DB_HOST, 14 | "PORT": DB_PORT, 15 | "NAME": "syzygy", 16 | "USER": DB_USER, 17 | "PASSWORD": DB_PASSWORD, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /tests/test_autodetector.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | from unittest import mock, skipUnless 3 | 4 | from django.core.management.color import color_style 5 | from django.db import migrations, models 6 | from django.db.migrations.questioner import ( 7 | InteractiveMigrationQuestioner, 8 | MigrationQuestioner, 9 | ) 10 | from django.db.migrations.state import ModelState, ProjectState 11 | from django.test import TestCase 12 | from django.test.utils import captured_stderr, captured_stdin, captured_stdout 13 | 14 | from syzygy.autodetector import STAGE_SPLIT, MigrationAutodetector 15 | from syzygy.compat import field_db_default_supported 16 | from syzygy.constants import Stage 17 | from syzygy.exceptions import AmbiguousStage 18 | from syzygy.operations import ( 19 | AddField, 20 | AlterField, 21 | PostAddField, 22 | PreRemoveField, 23 | RenameField, 24 | RenameModel, 25 | ) 26 | from syzygy.plan import get_migration_stage 27 | 28 | from .models import Bar 29 | 30 | 31 | class AutodetectorTestCase(TestCase): 32 | style = color_style() 33 | 34 | @staticmethod 35 | def make_project_state(model_states: List[ModelState]) -> ProjectState: 36 | project_state = ProjectState() 37 | for model_state in model_states: 38 | project_state.add_model(model_state.clone()) 39 | return project_state 40 | 41 | def get_changes( 42 | self, 43 | before_states: List[ModelState], 44 | after_states: List[ModelState], 45 | questioner: Optional[MigrationQuestioner] = None, 46 | ) -> List[migrations.Migration]: 47 | changes = MigrationAutodetector( 48 | self.make_project_state(before_states), 49 | self.make_project_state(after_states), 50 | questioner=questioner, 51 | style=self.style, 52 | )._detect_changes() 53 | self.assertNotIn(STAGE_SPLIT, changes) 54 | return changes 55 | 56 | 57 | class AutodetectorTests(AutodetectorTestCase): 58 | def _test_field_addition(self, field, expected_db_default=None): 59 | from_model = ModelState("tests", "Model", []) 60 | to_model = ModelState("tests", "Model", [("field", field)]) 61 | changes = self.get_changes([from_model], [to_model])["tests"] 62 | self.assertEqual(len(changes), 2) 63 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 64 | self.assertEqual(changes[0].dependencies, []) 65 | self.assertEqual(len(changes[0].operations), 1) 66 | pre_operation = changes[0].operations[0] 67 | if field_db_default_supported: 68 | self.assertIsInstance(pre_operation, migrations.AddField) 69 | self.assertEqual( 70 | pre_operation.field.db_default, 71 | expected_db_default or field.get_default(), 72 | ) 73 | else: 74 | self.assertIsInstance(pre_operation, AddField) 75 | self.assertEqual(get_migration_stage(changes[1]), Stage.POST_DEPLOY) 76 | self.assertEqual(changes[1].dependencies, [("tests", "auto_1")]) 77 | self.assertEqual(len(changes[1].operations), 1) 78 | post_operation = changes[1].operations[0] 79 | if field_db_default_supported: 80 | self.assertIsInstance(post_operation, AlterField) 81 | self.assertIs(post_operation.field.db_default, models.NOT_PROVIDED) 82 | else: 83 | self.assertIsInstance(post_operation, PostAddField) 84 | 85 | def test_field_addition(self): 86 | fields = [ 87 | models.IntegerField(default=42), 88 | models.IntegerField(null=True, default=42), 89 | models.IntegerField(default=lambda: 42), 90 | # Foreign keys with callable defaults should have their associated 91 | # db_default generated with care. 92 | (models.ForeignKey("tests.Model", models.CASCADE, default=42), 42), 93 | ( 94 | models.ForeignKey( 95 | "tests.Bar", models.CASCADE, default=lambda: Bar(id=42) 96 | ), 97 | 42, 98 | ), 99 | ( 100 | models.ForeignKey( 101 | "tests.bar", 102 | models.CASCADE, 103 | to_field="name", 104 | default=lambda: Bar(id=123, name="bar"), 105 | ), 106 | "bar", 107 | ), 108 | ] 109 | for field in fields: 110 | if isinstance(field, tuple): 111 | field, expected_db_default = field 112 | else: 113 | expected_db_default = None 114 | with self.subTest(field=field): 115 | self._test_field_addition(field, expected_db_default) 116 | 117 | def test_many_to_many_addition(self): 118 | from_model = ModelState("tests", "Model", []) 119 | to_model = ModelState( 120 | "tests", "Model", [("field", models.ManyToManyField("self"))] 121 | ) 122 | changes = self.get_changes([from_model], [to_model])["tests"] 123 | self.assertEqual(len(changes), 1) 124 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 125 | self.assertEqual(changes[0].dependencies, []) 126 | self.assertEqual(len(changes[0].operations), 1) 127 | operation = changes[0].operations[0] 128 | self.assertIsInstance(operation, migrations.AddField) 129 | 130 | def test_nullable_field_addition(self): 131 | """ 132 | No action required if the field is already NULL'able and doesn't have 133 | a `default`. 134 | """ 135 | from_model = ModelState("tests", "Model", []) 136 | to_model = ModelState( 137 | "tests", "Model", [("field", models.IntegerField(null=True))] 138 | ) 139 | changes = self.get_changes([from_model], [to_model])["tests"] 140 | self.assertEqual(len(changes), 1) 141 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 142 | 143 | @skipUnless(field_db_default_supported, "Field.db_default is not supported") 144 | def test_db_default_field_addition(self): 145 | """ 146 | No action required if the field already has a `db_default` 147 | """ 148 | from_model = ModelState("tests", "Model", []) 149 | to_model = ModelState( 150 | "tests", "Model", [("field", models.IntegerField(db_default=42))] 151 | ) 152 | changes = self.get_changes([from_model], [to_model])["tests"] 153 | self.assertEqual(len(changes), 1) 154 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 155 | 156 | def _test_field_removal(self, field): 157 | from_model = ModelState("tests", "Model", [("field", field)]) 158 | to_model = ModelState("tests", "Model", []) 159 | changes = self.get_changes([from_model], [to_model])["tests"] 160 | self.assertEqual(len(changes), 2) 161 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 162 | self.assertEqual(changes[0].dependencies, []) 163 | self.assertEqual(len(changes[0].operations), 1) 164 | pre_operation = changes[0].operations[0] 165 | if field_db_default_supported: 166 | self.assertIsInstance(pre_operation, migrations.AlterField) 167 | if field.has_default(): 168 | self.assertEqual(pre_operation.field.db_default, 42) 169 | else: 170 | self.assertIs(pre_operation.field.null, True) 171 | else: 172 | self.assertIsInstance(pre_operation, PreRemoveField) 173 | if not field.has_default(): 174 | self.assertIs(pre_operation.field.null, True) 175 | self.assertEqual(get_migration_stage(changes[1]), Stage.POST_DEPLOY) 176 | self.assertEqual(changes[1].dependencies, [("tests", "auto_1")]) 177 | self.assertEqual(len(changes[1].operations), 1) 178 | self.assertIsInstance(changes[1].operations[0], migrations.RemoveField) 179 | 180 | def test_field_removal(self): 181 | fields = [ 182 | models.IntegerField(), 183 | models.IntegerField(default=42), 184 | models.IntegerField(null=True, default=42), 185 | ] 186 | for field in fields: 187 | with self.subTest(field=field): 188 | self._test_field_removal(field) 189 | 190 | def test_many_to_many_removal(self): 191 | from_model = ModelState( 192 | "tests", "Model", [("field", models.ManyToManyField("self"))] 193 | ) 194 | to_model = ModelState("tests", "Model", []) 195 | changes = self.get_changes([from_model], [to_model])["tests"] 196 | self.assertEqual(len(changes), 1) 197 | self.assertEqual(get_migration_stage(changes[0]), Stage.POST_DEPLOY) 198 | self.assertEqual(changes[0].dependencies, []) 199 | self.assertEqual(len(changes[0].operations), 1) 200 | operation = changes[0].operations[0] 201 | self.assertIsInstance(operation, migrations.RemoveField) 202 | 203 | def test_nullable_field_removal(self): 204 | """ 205 | No action required if the field is already NULL'able and doesn't have 206 | a `default`. 207 | """ 208 | from_model = ModelState( 209 | "tests", "Model", [("field", models.IntegerField(null=True))] 210 | ) 211 | to_model = ModelState("tests", "Model", []) 212 | changes = self.get_changes([from_model], [to_model])["tests"] 213 | self.assertEqual(len(changes), 1) 214 | self.assertEqual(get_migration_stage(changes[0]), Stage.POST_DEPLOY) 215 | 216 | @skipUnless(field_db_default_supported, "Field.db_default is not supported") 217 | def test_db_default_field_removal(self): 218 | """ 219 | No action required if the field already has a `db_default` 220 | """ 221 | from_model = ModelState( 222 | "tests", "Model", [("field", models.IntegerField(db_default=42))] 223 | ) 224 | to_model = ModelState("tests", "Model", []) 225 | changes = self.get_changes([from_model], [to_model])["tests"] 226 | self.assertEqual(len(changes), 1) 227 | self.assertEqual(get_migration_stage(changes[0]), Stage.POST_DEPLOY) 228 | 229 | def test_non_nullable_field_removal_default(self): 230 | from_model = ModelState("tests", "Model", [("field", models.IntegerField())]) 231 | to_model = ModelState("tests", "Model", []) 232 | changes = self.get_changes( 233 | [from_model], [to_model], MigrationQuestioner({"ask_remove_default": 42}) 234 | )["tests"] 235 | self.assertEqual(len(changes), 2) 236 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 237 | self.assertEqual(changes[0].operations[0].field.default, 42) 238 | self.assertEqual(get_migration_stage(changes[1]), Stage.POST_DEPLOY) 239 | 240 | def test_alter_field_null_to_not_null(self): 241 | from_model = ModelState( 242 | "tests", "Model", [("field", models.IntegerField(null=True))] 243 | ) 244 | to_model = ModelState("tests", "Model", [("field", models.IntegerField())]) 245 | changes = self.get_changes([from_model], [to_model])["tests"] 246 | self.assertEqual(len(changes), 1) 247 | self.assertEqual(get_migration_stage(changes[0]), Stage.POST_DEPLOY) 248 | 249 | def test_field_rename(self): 250 | from_models = [ 251 | ModelState( 252 | "tests", 253 | "Foo", 254 | [ 255 | ("id", models.IntegerField(primary_key=True)), 256 | ("foo", models.BooleanField(default=False)), 257 | ], 258 | ), 259 | ] 260 | to_models = [ 261 | ModelState( 262 | "tests", 263 | "Foo", 264 | [ 265 | ("id", models.IntegerField(primary_key=True)), 266 | ("bar", models.BooleanField(default=False)), 267 | ], 268 | ), 269 | ] 270 | questioner = MigrationQuestioner({"ask_rename": True}) 271 | with captured_stderr(), self.assertRaisesMessage(SystemExit, "3"): 272 | self.get_changes(from_models, to_models, questioner)["tests"] 273 | # Pre-deploy rename. 274 | questioner.defaults["ask_rename_field_stage"] = 2 275 | with captured_stderr(): 276 | changes = self.get_changes(from_models, to_models, questioner)["tests"] 277 | self.assertEqual(len(changes), 1) 278 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 279 | self.assertIsInstance(changes[0].operations[0], RenameField) 280 | # Post-deploy rename. 281 | questioner.defaults["ask_rename_field_stage"] = 3 282 | with captured_stderr(): 283 | changes = self.get_changes(from_models, to_models, questioner)["tests"] 284 | self.assertEqual(len(changes), 1) 285 | self.assertEqual(get_migration_stage(changes[0]), Stage.POST_DEPLOY) 286 | self.assertIsInstance(changes[0].operations[0], RenameField) 287 | 288 | def test_model_rename(self): 289 | from_models = [ 290 | ModelState( 291 | "tests", 292 | "Foo", 293 | [ 294 | ("id", models.IntegerField(primary_key=True)), 295 | ], 296 | ), 297 | ] 298 | to_models = [ 299 | ModelState( 300 | "tests", 301 | "Bar", 302 | [ 303 | ("id", models.IntegerField(primary_key=True)), 304 | ], 305 | ), 306 | ] 307 | questioner = MigrationQuestioner( 308 | { 309 | "ask_rename_model": True, 310 | } 311 | ) 312 | with captured_stderr(), self.assertRaisesMessage(SystemExit, "3"): 313 | self.get_changes(from_models, to_models, questioner)["tests"] 314 | # Pre-deploy rename. 315 | questioner.defaults["ask_rename_model_stage"] = 2 316 | with captured_stderr(): 317 | changes = self.get_changes(from_models, to_models, questioner)["tests"] 318 | self.assertEqual(len(changes), 1) 319 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 320 | self.assertIsInstance(changes[0].operations[0], RenameModel) 321 | # Post-deploy rename. 322 | questioner.defaults["ask_rename_model_stage"] = 3 323 | with captured_stderr(): 324 | changes = self.get_changes(from_models, to_models, questioner)["tests"] 325 | self.assertEqual(len(changes), 1) 326 | self.assertEqual(get_migration_stage(changes[0]), Stage.POST_DEPLOY) 327 | self.assertIsInstance(changes[0].operations[0], RenameModel) 328 | # db_table override 329 | to_models[0].options["db_table"] = "tests_foo" 330 | changes = self.get_changes(from_models, to_models, questioner)["tests"] 331 | self.assertEqual(len(changes), 1) 332 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 333 | self.assertIsInstance(changes[0].operations[0], migrations.RenameModel) 334 | 335 | 336 | class AutodetectorStageTests(AutodetectorTestCase): 337 | def test_mixed_stage_same_app(self): 338 | from_models = [ 339 | ModelState( 340 | "tests", "Model", [("field", models.IntegerField(primary_key=True))] 341 | ) 342 | ] 343 | to_models = [ 344 | ModelState( 345 | "tests", 346 | "OtherModel", 347 | [("field", models.IntegerField(primary_key=True))], 348 | ) 349 | ] 350 | changes = self.get_changes(from_models, to_models)["tests"] 351 | self.assertEqual(len(changes), 2) 352 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 353 | self.assertEqual(changes[0].dependencies, []) 354 | self.assertEqual(len(changes[0].operations), 1) 355 | self.assertIsInstance(changes[0].operations[0], migrations.CreateModel) 356 | self.assertEqual(get_migration_stage(changes[1]), Stage.POST_DEPLOY) 357 | self.assertEqual(changes[1].dependencies, [("tests", "auto_1")]) 358 | self.assertEqual(len(changes[1].operations), 1) 359 | self.assertIsInstance(changes[1].operations[0], migrations.DeleteModel) 360 | 361 | def test_mixed_stage_reorder(self): 362 | from_models = [ 363 | ModelState("tests", "Foo", [("id", models.IntegerField(primary_key=True))]), 364 | ModelState( 365 | "tests", 366 | "Bar", 367 | [ 368 | ("id", models.IntegerField(primary_key=True)), 369 | ("foo", models.ForeignKey("Foo", models.CASCADE)), 370 | ], 371 | ), 372 | ] 373 | to_models = [ 374 | ModelState( 375 | "tests", 376 | "Foo", 377 | [ 378 | ("id", models.IntegerField(primary_key=True)), 379 | ("bar", models.BooleanField(default=False)), 380 | ], 381 | ), 382 | ] 383 | changes = self.get_changes(from_models, to_models)["tests"] 384 | self.assertEqual(len(changes), 2) 385 | self.assertEqual(get_migration_stage(changes[0]), Stage.PRE_DEPLOY) 386 | self.assertEqual(changes[0].dependencies, []) 387 | self.assertEqual(len(changes[0].operations), 1) 388 | self.assertIsInstance(changes[0].operations[0], migrations.AddField) 389 | self.assertEqual(get_migration_stage(changes[1]), Stage.POST_DEPLOY) 390 | self.assertEqual(changes[1].dependencies, [("tests", "auto_1")]) 391 | self.assertEqual(len(changes[1].operations), 2) 392 | if field_db_default_supported: 393 | self.assertIsInstance(changes[1].operations[0], AlterField) 394 | else: 395 | self.assertIsInstance(changes[1].operations[0], PostAddField) 396 | self.assertIsInstance(changes[1].operations[1], migrations.DeleteModel) 397 | 398 | def test_mixed_stage_failure(self): 399 | from_models = [ 400 | ModelState("tests", "Foo", [("id", models.IntegerField(primary_key=True))]), 401 | ModelState( 402 | "tests", 403 | "Bar", 404 | [ 405 | ("id", models.IntegerField(primary_key=True)), 406 | ("foo", models.ForeignKey("Foo", models.CASCADE)), 407 | ], 408 | ), 409 | ] 410 | to_models = [ 411 | ModelState( 412 | "tests", 413 | "Foo", 414 | [ 415 | ("id", models.IntegerField(primary_key=True)), 416 | ("bar", models.BooleanField(default=False)), 417 | ], 418 | ), 419 | ] 420 | with mock.patch( 421 | "syzygy.autodetector.partition_operations", side_effect=AmbiguousStage 422 | ), captured_stderr() as stderr: 423 | self.get_changes(from_models, to_models)["tests"] 424 | self.assertIn( 425 | 'The auto-detected operations for the "tests" app cannot be partitioned into deployment stages:', 426 | stderr.getvalue(), 427 | ) 428 | self.assertIn( 429 | "- Remove field foo from bar", 430 | stderr.getvalue(), 431 | ) 432 | questioner = MigrationQuestioner({"ask_ambiguous_abort": True}) 433 | with self.assertRaisesMessage(SystemExit, "3"), mock.patch( 434 | "syzygy.autodetector.partition_operations", side_effect=AmbiguousStage 435 | ), captured_stderr() as stderr: 436 | self.get_changes(from_models, to_models, questioner)["tests"] 437 | 438 | 439 | class InteractiveAutodetectorTests(AutodetectorTestCase): 440 | def test_field_rename(self): 441 | from_models = [ 442 | ModelState( 443 | "tests", 444 | "Foo", 445 | [ 446 | ("id", models.IntegerField(primary_key=True)), 447 | ("foo", models.BooleanField(default=False)), 448 | ], 449 | ), 450 | ] 451 | to_models = [ 452 | ModelState( 453 | "tests", 454 | "Foo", 455 | [ 456 | ("id", models.IntegerField(primary_key=True)), 457 | ("bar", models.BooleanField(default=False)), 458 | ], 459 | ), 460 | ] 461 | with captured_stdin() as stdin, captured_stdout() as stdout, captured_stderr() as stderr: 462 | questioner = InteractiveMigrationQuestioner() 463 | stdin.write("y\n2\n") 464 | stdin.seek(0) 465 | self.get_changes(from_models, to_models, questioner) 466 | self.assertIn( 467 | self.style.WARNING( 468 | "Renaming a column from a database table actively relied upon might cause downtime during deployment." 469 | ), 470 | stderr.getvalue(), 471 | ) 472 | self.assertIn( 473 | "1) Quit, and let me add a new foo.bar field meant to be backfilled with foo.foo values", 474 | stdout.getvalue(), 475 | ) 476 | self.assertIn( 477 | self.style.MIGRATE_LABEL( 478 | "This might cause downtime if your assumption is wrong" 479 | ), 480 | stdout.getvalue(), 481 | ) 482 | 483 | def test_model_rename(self): 484 | from_models = [ 485 | ModelState( 486 | "tests", 487 | "Foo", 488 | [ 489 | ("id", models.IntegerField(primary_key=True)), 490 | ], 491 | ), 492 | ] 493 | to_models = [ 494 | ModelState( 495 | "tests", 496 | "Bar", 497 | [ 498 | ("id", models.IntegerField(primary_key=True)), 499 | ], 500 | ), 501 | ] 502 | with captured_stdin() as stdin, captured_stdout() as stdout, captured_stderr() as stderr: 503 | questioner = InteractiveMigrationQuestioner() 504 | stdin.write("y\n2\n") 505 | stdin.seek(0) 506 | self.get_changes(from_models, to_models, questioner) 507 | self.assertIn( 508 | self.style.WARNING( 509 | "Renaming an actively relied on database table might cause downtime during deployment." 510 | ), 511 | stderr.getvalue(), 512 | ) 513 | self.assertIn( 514 | '1) Quit, and let me manually set tests.Bar.Meta.db_table to "tests_foo" to avoid ' 515 | "renaming its underlying table", 516 | stdout.getvalue(), 517 | ) 518 | self.assertIn( 519 | self.style.MIGRATE_LABEL( 520 | "This might cause downtime if your assumption is wrong" 521 | ), 522 | stdout.getvalue(), 523 | ) 524 | 525 | def test_mixed_stage_failure(self): 526 | from_models = [ 527 | ModelState("tests", "Foo", [("id", models.IntegerField(primary_key=True))]), 528 | ModelState( 529 | "tests", 530 | "Bar", 531 | [ 532 | ("id", models.IntegerField(primary_key=True)), 533 | ("foo", models.ForeignKey("Foo", models.CASCADE)), 534 | ], 535 | ), 536 | ] 537 | to_models = [ 538 | ModelState( 539 | "tests", 540 | "Foo", 541 | [ 542 | ("id", models.IntegerField(primary_key=True)), 543 | ("bar", models.BooleanField(default=False)), 544 | ], 545 | ), 546 | ] 547 | with mock.patch( 548 | "syzygy.autodetector.partition_operations", side_effect=AmbiguousStage 549 | ), captured_stdin() as stdin, captured_stdout() as stdout, captured_stderr(): 550 | questioner = InteractiveMigrationQuestioner() 551 | stdin.write("1\n") 552 | stdin.seek(0) 553 | self.get_changes(from_models, to_models, questioner)["tests"] 554 | self.assertEqual( 555 | stdout.getvalue(), 556 | "\n 1) Let `makemigrations` complete. You'll have to manually break you operations in migrations " 557 | "with non-ambiguous stages.\n 2) Abort `makemigrations`. You'll have to reduce the number of model " 558 | "changes before running `makemigrations` again.\nSelect an option: ", 559 | ) 560 | -------------------------------------------------------------------------------- /tests/test_checks.py: -------------------------------------------------------------------------------- 1 | from django.apps import apps 2 | from django.core.checks import run_checks 3 | from django.core.checks.messages import Error 4 | from django.test import SimpleTestCase 5 | 6 | 7 | class ChecksTests(SimpleTestCase): 8 | hint = ( 9 | "Assign an explicit stage to it, break its operation into multiple " 10 | "migrations if it's not already applied or define an explicit stage for " 11 | "it using `MIGRATION_STAGE_OVERRIDE` or `MIGRATION_STAGE_FALLBACK` if the " 12 | "migration is not under your control." 13 | ) 14 | 15 | def test_ambiguous_stage(self): 16 | with self.settings( 17 | MIGRATION_MODULES={"tests": "tests.test_migrations.ambiguous"} 18 | ): 19 | checks = run_checks( 20 | app_configs=[apps.get_app_config("tests")], tags={"migrations"} 21 | ) 22 | self.assertEqual(len(checks), 1) 23 | self.assertEqual( 24 | checks[0], 25 | Error( 26 | msg="Cannot automatically determine stage of tests.0001_initial.", 27 | hint=self.hint, 28 | obj=("tests", "0001_initial"), 29 | id="migrations.0001", 30 | ), 31 | ) 32 | 33 | def test_empty_migration_folder(self): 34 | with self.settings(MIGRATION_MODULES={"tests": "tests.test_migrations.empty"}): 35 | checks = run_checks( 36 | app_configs=[apps.get_app_config("tests")], tags={"migrations"} 37 | ) 38 | self.assertEqual(len(checks), 0) 39 | -------------------------------------------------------------------------------- /tests/test_commands.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | from multiprocessing.pool import ThreadPool 3 | from unittest import mock 4 | 5 | from django.core.cache import cache 6 | from django.core.management import CommandError, call_command 7 | from django.db import connection, connections 8 | from django.db.migrations.recorder import MigrationRecorder 9 | from django.test import TestCase, TransactionTestCase, override_settings 10 | 11 | from syzygy.constants import Stage 12 | 13 | 14 | class BaseMigrateTests(TransactionTestCase): 15 | def setUp(self) -> None: 16 | super().setUp() 17 | recorder = MigrationRecorder(connection) 18 | recorder.ensure_schema() 19 | self.addCleanup(recorder.flush) 20 | 21 | def call_command(self, *args, **options): 22 | stdout = StringIO() 23 | call_command("migrate", "tests", *args, no_color=True, stdout=stdout, **options) 24 | return stdout.getvalue() 25 | 26 | def get_applied_migrations(self): 27 | return { 28 | name 29 | for (app_label, name) in MigrationRecorder(connection).applied_migrations() 30 | if app_label == "tests" 31 | } 32 | 33 | 34 | class MigrateTests(BaseMigrateTests): 35 | @override_settings(MIGRATION_MODULES={"tests": "tests.test_migrations.functional"}) 36 | def test_pre_deploy_forward(self): 37 | stdout = self.call_command(plan=True, stage=Stage.PRE_DEPLOY) 38 | self.assertIn("tests.0001_pre_deploy", stdout) 39 | self.assertNotIn("tests.0002_post_deploy", stdout) 40 | call_command("migrate", "tests", stage=Stage.PRE_DEPLOY, verbosity=0) 41 | self.assertEqual(self.get_applied_migrations(), {"0001_pre_deploy"}) 42 | 43 | @override_settings(MIGRATION_MODULES={"tests": "tests.test_migrations.functional"}) 44 | def test_pre_deploy_backward(self): 45 | self.call_command(verbosity=0) 46 | self.assertEqual( 47 | self.get_applied_migrations(), {"0001_pre_deploy", "0002_post_deploy"} 48 | ) 49 | stdout = self.call_command("zero", plan=True, stage=Stage.PRE_DEPLOY) 50 | self.assertIn("tests.0002_post_deploy", stdout) 51 | self.assertNotIn("tests.0001_pre_deploy", stdout) 52 | self.call_command("zero", stage=Stage.PRE_DEPLOY) 53 | self.assertEqual(self.get_applied_migrations(), {"0001_pre_deploy"}) 54 | 55 | @override_settings(MIGRATION_MODULES={"tests": "tests.test_migrations.ambiguous"}) 56 | def test_ambiguous(self): 57 | with self.assertRaisesMessage( 58 | CommandError, "Cannot automatically determine stage of tests.0001_initial." 59 | ): 60 | self.call_command(plan=True, stage=Stage.PRE_DEPLOY, verbosity=0) 61 | 62 | 63 | @override_settings(MIGRATION_QUORUM_BACKEND="syzygy.quorum.backends.cache.CacheQuorum") 64 | class MigrateQuorumTests(BaseMigrateTests): 65 | def setUp(self): 66 | super().setUp() 67 | self.addCleanup(cache.clear) 68 | 69 | @override_settings(MIGRATION_MODULES={"tests": "tests.test_migrations.functional"}) 70 | def test_empty_plan(self): 71 | """Quorum is entirely skipped when no migrations are planed""" 72 | stdout = self.call_command("zero", quorum=2) 73 | self.assertNotIn("quorum", stdout) 74 | 75 | def _call_command_thread(self, options): 76 | stdout = self.call_command(**options) 77 | connections.close_all() 78 | return stdout 79 | 80 | def _call_failing_command_thread(self, options): 81 | stdout = StringIO() 82 | stderr = StringIO() 83 | with self.assertRaises(Exception) as exc: 84 | call_command( 85 | "migrate", 86 | "tests", 87 | no_color=True, 88 | stdout=stdout, 89 | stderr=stderr, 90 | **options, 91 | ) 92 | connections.close_all() 93 | return exc.exception, stdout.getvalue(), stderr.getvalue() 94 | 95 | def call_command_with_quorum(self, stage, quorum=3): 96 | with mock.patch("time.sleep"), ThreadPool(processes=quorum) as pool: 97 | stdouts = pool.map_async( 98 | self._call_command_thread, 99 | [{"stage": stage, "quorum": quorum}] * quorum, 100 | ).get() 101 | apply_stdouts = [] 102 | for stdout in stdouts: 103 | if stdout.startswith( 104 | "Reached pre-migrate quorum, proceeding with planned migrations..." 105 | ): 106 | apply_stdouts.append(stdout) 107 | self.assertIn("Reached post-migrate quorum after", stdout) 108 | else: 109 | self.assertTrue(stdout.startswith("Waiting for pre-migrate quorum...")) 110 | self.assertIn("Reached pre-migrate quorum after", stdout) 111 | self.assertIn( 112 | "Waiting for migrations to be applied by remote party...", stdout 113 | ) 114 | self.assertIn("Reached post-migrate quorum after", stdout) 115 | self.assertIn("Migrations applied by remote party", stdout) 116 | if not apply_stdouts: 117 | self.fail("Migrations were not applied") 118 | if len(apply_stdouts) > 1: 119 | self.fail("Migrations were applied more than once") 120 | return apply_stdouts[0] 121 | 122 | @override_settings(MIGRATION_MODULES={"tests": "tests.test_migrations.functional"}) 123 | def test_pre_deploy(self): 124 | stdout = self.call_command_with_quorum(stage=Stage.PRE_DEPLOY) 125 | self.assertIn("tests.0001_pre_deploy", stdout) 126 | self.assertNotIn("tests.0002_post_deploy", stdout) 127 | self.assertEqual(self.get_applied_migrations(), {"0001_pre_deploy"}) 128 | 129 | @override_settings(MIGRATION_MODULES={"tests": "tests.test_migrations.functional"}) 130 | def test_post_deploy(self): 131 | stdout = self.call_command_with_quorum(stage=Stage.POST_DEPLOY) 132 | self.assertIn("tests.0001_pre_deploy", stdout) 133 | self.assertIn("tests.0002_post_deploy", stdout) 134 | self.assertEqual( 135 | self.get_applied_migrations(), {"0001_pre_deploy", "0002_post_deploy"} 136 | ) 137 | 138 | @override_settings(MIGRATION_MODULES={"tests": "tests.test_migrations.functional"}) 139 | def test_quorum_timeout(self): 140 | msg = "Migration plan quorum timeout" 141 | with self.assertRaisesMessage(RuntimeError, msg): 142 | self.call_command(quorum=2, quorum_timeout=1) 143 | 144 | @override_settings(MIGRATION_MODULES={"tests": "tests.test_migrations.crash"}) 145 | def test_quorum_severed(self): 146 | quorum = 3 147 | stage = Stage.PRE_DEPLOY 148 | with mock.patch("time.sleep"), ThreadPool(processes=quorum) as pool: 149 | results = pool.map_async( 150 | self._call_failing_command_thread, 151 | [{"stage": stage, "quorum": quorum}] * quorum, 152 | ).get() 153 | severed = 0 154 | severer = 0 155 | for exc, stdout, stderr in results: 156 | if ( 157 | isinstance(exc, CommandError) 158 | and str(exc) 159 | == "Error encountered by remote party while applying migration, aborting." 160 | ): 161 | self.assertIn( 162 | "Waiting for migrations to be applied by remote party...", stdout 163 | ) 164 | self.assertEqual(stderr, "") 165 | severed += 1 166 | elif str(exc) == "Test crash": 167 | self.assertIn( 168 | "Reached pre-migrate quorum, proceeding with planned migrations...", 169 | stdout, 170 | ) 171 | self.assertIn( 172 | "Encountered exception while applying migrations, disovling quorum", 173 | stderr, 174 | ) 175 | severer += 1 176 | else: 177 | self.fail(f"Unexpected exception: {exc}") 178 | self.assertEqual(severed, quorum - 1) 179 | self.assertEqual(severer, 1) 180 | 181 | 182 | class MakeMigrationsTests(TestCase): 183 | def test_disabled(self): 184 | failure = AssertionError("syzygy should be disabled") 185 | with mock.patch( 186 | "syzygy.management.commands.makemigrations.MigrationAutodetector", 187 | side_effect=failure, 188 | ): 189 | call_command( 190 | "makemigrations", 191 | "tests", 192 | verbosity=0, 193 | dry_run=True, 194 | disable_syzygy=True, 195 | ) 196 | 197 | @override_settings( 198 | MIGRATION_MODULES={"tests": "tests.test_migrations.null_field_removal"} 199 | ) 200 | def test_null_field_removal(self): 201 | stdout = StringIO() 202 | call_command( 203 | "makemigrations", "tests", no_color=True, dry_run=True, stdout=stdout 204 | ) 205 | output = stdout.getvalue() 206 | self.assertIn("null_field_removal/0002_set_nullable_foo_bar.py", output) 207 | self.assertIn("Set field bar of foo NULLable", output) 208 | self.assertIn("null_field_removal/0003_remove_foo_bar.py", output) 209 | self.assertIn("Remove field bar from foo", output) 210 | 211 | @override_settings( 212 | MIGRATION_MODULES={"tests": "tests.test_migrations.merge_conflict"} 213 | ) 214 | def test_merge_conflict(self): 215 | stdout = StringIO() 216 | call_command( 217 | "makemigrations", 218 | "tests", 219 | merge=True, 220 | interactive=False, 221 | no_color=True, 222 | dry_run=True, 223 | stdout=stdout, 224 | ) 225 | self.assertIn("0002_first", stdout.getvalue()) 226 | self.assertIn("0002_second", stdout.getvalue()) 227 | -------------------------------------------------------------------------------- /tests/test_migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charettes/django-syzygy/6614c7d42c6eb6cb9c9c927630ccb2bbdb13f38b/tests/test_migrations/__init__.py -------------------------------------------------------------------------------- /tests/test_migrations/ambiguous/0001_initial.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | 4 | class Migration(migrations.Migration): 5 | operations = [migrations.CreateModel("Foo", []), migrations.DeleteModel("Bar")] 6 | -------------------------------------------------------------------------------- /tests/test_migrations/ambiguous/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charettes/django-syzygy/6614c7d42c6eb6cb9c9c927630ccb2bbdb13f38b/tests/test_migrations/ambiguous/__init__.py -------------------------------------------------------------------------------- /tests/test_migrations/crash/0001_pre_deploy.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | from syzygy import Stage 4 | 5 | 6 | def crash(*args, **kwargs): 7 | raise Exception("Test crash") 8 | 9 | 10 | class Migration(migrations.Migration): 11 | initial = True 12 | stage = Stage.PRE_DEPLOY 13 | atomic = False 14 | 15 | operations = [migrations.RunPython(crash)] 16 | -------------------------------------------------------------------------------- /tests/test_migrations/crash/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charettes/django-syzygy/6614c7d42c6eb6cb9c9c927630ccb2bbdb13f38b/tests/test_migrations/crash/__init__.py -------------------------------------------------------------------------------- /tests/test_migrations/empty/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_migrations/functional/0001_pre_deploy.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | from syzygy import Stage 4 | 5 | 6 | class Migration(migrations.Migration): 7 | initial = True 8 | stage = Stage.PRE_DEPLOY 9 | atomic = False 10 | -------------------------------------------------------------------------------- /tests/test_migrations/functional/0002_post_deploy.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | from syzygy import Stage 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [("tests", "0001_pre_deploy")] 8 | stage = Stage.POST_DEPLOY 9 | atomic = False 10 | -------------------------------------------------------------------------------- /tests/test_migrations/functional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charettes/django-syzygy/6614c7d42c6eb6cb9c9c927630ccb2bbdb13f38b/tests/test_migrations/functional/__init__.py -------------------------------------------------------------------------------- /tests/test_migrations/merge_conflict/0001_initial.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | 4 | class Migration(migrations.Migration): 5 | pass 6 | -------------------------------------------------------------------------------- /tests/test_migrations/merge_conflict/0002_first.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | 4 | class Migration(migrations.Migration): 5 | dependencies = [("tests", "0001_initial")] 6 | -------------------------------------------------------------------------------- /tests/test_migrations/merge_conflict/0002_second.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | 4 | class Migration(migrations.Migration): 5 | dependencies = [("tests", "0001_initial")] 6 | -------------------------------------------------------------------------------- /tests/test_migrations/merge_conflict/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charettes/django-syzygy/6614c7d42c6eb6cb9c9c927630ccb2bbdb13f38b/tests/test_migrations/merge_conflict/__init__.py -------------------------------------------------------------------------------- /tests/test_migrations/null_field_removal/0001_initial.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations, models 2 | 3 | 4 | class Migration(migrations.Migration): 5 | initial = True 6 | operations = [ 7 | migrations.CreateModel( 8 | "Foo", 9 | fields=[ 10 | ( 11 | "id", 12 | models.AutoField( 13 | auto_created=True, 14 | primary_key=True, 15 | serialize=False, 16 | verbose_name="ID", 17 | ), 18 | ), 19 | ("bar", models.IntegerField()), 20 | ], 21 | ), 22 | migrations.CreateModel( 23 | "Bar", 24 | fields=[ 25 | ( 26 | "id", 27 | models.AutoField( 28 | auto_created=True, 29 | primary_key=True, 30 | serialize=False, 31 | verbose_name="ID", 32 | ), 33 | ), 34 | ("name", models.CharField(unique=True)), 35 | ], 36 | options={"managed": False}, 37 | ), 38 | ] 39 | -------------------------------------------------------------------------------- /tests/test_migrations/null_field_removal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charettes/django-syzygy/6614c7d42c6eb6cb9c9c927630ccb2bbdb13f38b/tests/test_migrations/null_field_removal/__init__.py -------------------------------------------------------------------------------- /tests/test_operations.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional, Tuple, TypeVar 2 | from unittest import mock, skipUnless 3 | 4 | from django.db import connection, migrations, models 5 | from django.db.migrations.operations.base import Operation 6 | from django.db.migrations.optimizer import MigrationOptimizer 7 | from django.db.migrations.serializer import OperationSerializer 8 | from django.db.migrations.state import ProjectState 9 | from django.db.migrations.writer import OperationWriter 10 | from django.db.models.fields import NOT_PROVIDED 11 | from django.test import TestCase, TransactionTestCase 12 | 13 | from syzygy.autodetector import MigrationAutodetector 14 | from syzygy.compat import field_db_default_supported 15 | from syzygy.constants import Stage 16 | from syzygy.operations import ( 17 | AlterField, 18 | RenameField, 19 | RenameModel, 20 | get_post_add_field_operation, 21 | get_pre_add_field_operation, 22 | get_pre_remove_field_operation, 23 | ) 24 | from syzygy.plan import get_operation_stage 25 | 26 | if connection.features.can_rollback_ddl: 27 | SchemaTestCase = TestCase 28 | else: 29 | 30 | class SchemaTestCase(TransactionTestCase): 31 | @classmethod 32 | def setUpClass(cls): 33 | super().setUpClass() 34 | with connection.cursor() as cursor: 35 | cls.tables = { 36 | table.name 37 | for table in connection.introspection.get_table_list(cursor) 38 | } 39 | 40 | def tearDown(self): 41 | super().tearDown() 42 | with connection.cursor() as cursor: 43 | created_tables = { 44 | table.name 45 | for table in connection.introspection.get_table_list(cursor) 46 | } - self.tables 47 | if created_tables: 48 | with connection.schema_editor() as schema_editor: 49 | sql_delete_table = schema_editor.sql_delete_table 50 | for table in created_tables: 51 | with connection.cursor() as cursor: 52 | cursor.execute(sql_delete_table % {"table": table}) 53 | 54 | 55 | O = TypeVar("O", bound=Operation) 56 | 57 | 58 | class OperationTestCase(SchemaTestCase): 59 | @classmethod 60 | def setUpClass(cls): 61 | connection.disable_constraint_checking() 62 | super().setUpClass() 63 | 64 | @classmethod 65 | def tearDownClass(cls): 66 | super().tearDownClass() 67 | connection.enable_constraint_checking() 68 | 69 | @staticmethod 70 | def apply_operation( 71 | operation: Operation, state: Optional[ProjectState] = None 72 | ) -> ProjectState: 73 | if state is None: 74 | from_state = ProjectState() 75 | else: 76 | from_state = state.clone() 77 | to_state = from_state.clone() 78 | operation.state_forwards("tests", to_state) 79 | with connection.schema_editor() as schema_editor: 80 | operation.database_forwards("tests", schema_editor, from_state, to_state) 81 | return to_state 82 | 83 | @classmethod 84 | def apply_operations( 85 | cls, operations: List[Operation], state: Optional[ProjectState] = None 86 | ) -> Optional[ProjectState]: 87 | for operation in operations: 88 | state = cls.apply_operation(operation, state) 89 | return state 90 | 91 | def assert_optimizes_to( 92 | self, operations: List[Operation], expected: List[Operation] 93 | ): 94 | optimized = MigrationOptimizer().optimize(operations, "tests") 95 | deep_deconstruct = MigrationAutodetector( 96 | ProjectState(), ProjectState() 97 | ).deep_deconstruct 98 | self.assertEqual(deep_deconstruct(optimized), deep_deconstruct(expected)) 99 | 100 | def serde_roundtrip(self, operation: O) -> O: 101 | serialized, imports = OperationWriter(operation, indentation=0).serialize() 102 | locals = {} 103 | exec( 104 | "\n".join(list(imports) + [f"operation = {serialized[:-1]}"]), 105 | {}, 106 | locals, 107 | ) 108 | return locals["operation"] 109 | 110 | def assert_serde_roundtrip_equal(self, operation: Operation): 111 | self.assertEqual( 112 | self.serde_roundtrip(operation).deconstruct(), operation.deconstruct() 113 | ) 114 | 115 | 116 | class PreAddFieldTests(OperationTestCase): 117 | def test_database_forwards(self, preserve_default=True): 118 | model_name = "TestModel" 119 | field_name = "foo" 120 | field = models.IntegerField(default=42) 121 | state = self.apply_operation( 122 | migrations.CreateModel(model_name, [("id", models.AutoField())]), 123 | ) 124 | pre_model = state.apps.get_model("tests", model_name) 125 | state = self.apply_operation( 126 | get_pre_add_field_operation( 127 | model_name, field_name, field, preserve_default=preserve_default 128 | ), 129 | state, 130 | ) 131 | post_model = state.apps.get_model("tests", model_name) 132 | pre_model.objects.create() 133 | self.assertEqual(post_model.objects.get().foo, 42) 134 | 135 | def test_database_forwards_discard_default(self): 136 | self.test_database_forwards(preserve_default=False) 137 | 138 | def test_deconstruct(self): 139 | model_name = "TestModel" 140 | field_name = "foo" 141 | field = models.IntegerField(default=42) 142 | operation = get_pre_add_field_operation(model_name, field_name, field) 143 | deconstructed = operation.deconstruct() 144 | if field_db_default_supported: 145 | self.assertEqual( 146 | operation.deconstruct(), 147 | ( 148 | "AddField", 149 | [], 150 | {"model_name": model_name, "name": field_name, "field": mock.ANY}, 151 | ), 152 | ) 153 | self.assertEqual( 154 | deconstructed[2]["field"].deconstruct(), 155 | ( 156 | None, 157 | "django.db.models.IntegerField", 158 | [], 159 | {"default": 42, "db_default": 42}, 160 | ), 161 | ) 162 | else: 163 | self.assertEqual( 164 | deconstructed, 165 | ( 166 | "AddField", 167 | [], 168 | {"model_name": model_name, "name": field_name, "field": field}, 169 | ), 170 | ) 171 | serializer = OperationSerializer(operation) 172 | serialized, imports = serializer.serialize() 173 | self.assertTrue(serialized.startswith("syzygy.operations.AddField")) 174 | self.assertIn("import syzygy.operations", imports) 175 | 176 | 177 | class PostAddFieldTests(OperationTestCase): 178 | def test_database_forwards( 179 | self, preserve_default=True 180 | ) -> Tuple[ProjectState, ProjectState]: 181 | model_name = "TestModel" 182 | field_name = "foo" 183 | field = models.IntegerField(default=42) 184 | from_state = self.apply_operations( 185 | [ 186 | migrations.CreateModel(model_name, [("id", models.AutoField())]), 187 | get_pre_add_field_operation( 188 | model_name, field_name, field, preserve_default=preserve_default 189 | ), 190 | ] 191 | ) 192 | to_state = self.apply_operation( 193 | get_post_add_field_operation( 194 | model_name, field_name, field, preserve_default=preserve_default 195 | ), 196 | from_state.clone(), 197 | ) 198 | if not preserve_default: 199 | self.assertIs( 200 | NOT_PROVIDED, 201 | to_state.models["tests", model_name.lower()].fields[field_name].default, 202 | ) 203 | with connection.cursor() as cursor: 204 | fields = connection.introspection.get_table_description( 205 | cursor, "tests_testmodel" 206 | ) 207 | self.assertIsNone(fields[-1].default) 208 | return from_state, to_state 209 | 210 | def test_database_forwards_discard_default(self): 211 | self.test_database_forwards(preserve_default=False) 212 | 213 | def test_database_backwards(self, preserve_default=True): 214 | from_state, to_state = self.test_database_forwards(preserve_default) 215 | model_name = "TestModel" 216 | field_name = "foo" 217 | field = models.IntegerField(default=42) 218 | with connection.schema_editor() as schema_editor: 219 | get_post_add_field_operation( 220 | model_name, field_name, field 221 | ).database_backwards("tests", schema_editor, to_state, from_state) 222 | if not preserve_default: 223 | self.assertIs( 224 | NOT_PROVIDED, 225 | from_state.models["tests", model_name.lower()] 226 | .fields[field_name] 227 | .default, 228 | ) 229 | with connection.cursor() as cursor: 230 | fields = connection.introspection.get_table_description( 231 | cursor, "tests_testmodel" 232 | ) 233 | for field in fields: 234 | if field.name == "foo": 235 | break 236 | else: 237 | self.fail('Could not find field "foo"') 238 | self.assertEqual(int(field.default), 42) 239 | 240 | def test_database_backwards_discard_default(self): 241 | self.test_database_backwards(preserve_default=False) 242 | 243 | def test_stage(self): 244 | model_name = "TestModel" 245 | field_name = "foo" 246 | field = models.IntegerField(default=42) 247 | self.assertEqual( 248 | get_operation_stage( 249 | get_post_add_field_operation(model_name, field_name, field) 250 | ), 251 | Stage.POST_DEPLOY, 252 | ) 253 | 254 | def test_migration_name_fragment(self): 255 | self.assertEqual( 256 | get_post_add_field_operation( 257 | "TestModel", "foo", models.IntegerField(default=42) 258 | ).migration_name_fragment, 259 | "drop_db_default_testmodel_foo", 260 | ) 261 | 262 | def test_describe(self): 263 | self.assertEqual( 264 | get_post_add_field_operation( 265 | "TestModel", "foo", models.IntegerField(default=42) 266 | ).describe(), 267 | "Drop database DEFAULT of field foo on TestModel", 268 | ) 269 | 270 | def test_deconstruct(self): 271 | model_name = "TestModel" 272 | field_name = "foo" 273 | field = models.IntegerField(default=42) 274 | operation = get_post_add_field_operation(model_name, field_name, field) 275 | deconstructed = operation.deconstruct() 276 | if field_db_default_supported: 277 | self.assertEqual( 278 | deconstructed, 279 | ( 280 | "AlterField", 281 | [], 282 | { 283 | "model_name": model_name, 284 | "name": field_name, 285 | "field": mock.ANY, 286 | "stage": Stage.POST_DEPLOY, 287 | }, 288 | ), 289 | ) 290 | self.assertEqual( 291 | deconstructed[2]["field"].deconstruct(), 292 | ( 293 | None, 294 | "django.db.models.IntegerField", 295 | [], 296 | {"default": 42}, 297 | ), 298 | ) 299 | else: 300 | self.assertEqual( 301 | deconstructed, 302 | ( 303 | "PostAddField", 304 | [], 305 | {"model_name": model_name, "name": field_name, "field": field}, 306 | ), 307 | ) 308 | serializer = OperationSerializer(operation) 309 | serialized, imports = serializer.serialize() 310 | self.assertTrue(serialized.startswith("syzygy.operations.PostAddField")) 311 | self.assertIn("import syzygy.operations", imports) 312 | 313 | def test_reduce(self): 314 | model_name = "TestModel" 315 | field_name = "foo" 316 | field = models.IntegerField(default=42) 317 | operations = [ 318 | get_pre_add_field_operation(model_name, field_name, field), 319 | get_post_add_field_operation(model_name, field_name, field), 320 | ] 321 | self.assert_optimizes_to( 322 | operations, 323 | [ 324 | migrations.AddField(model_name, field_name, field), 325 | ], 326 | ) 327 | 328 | 329 | class PreRemoveFieldTests(OperationTestCase): 330 | def test_database_forwards_null(self): 331 | model_name = "TestModel" 332 | field = models.IntegerField() 333 | operations = [ 334 | migrations.CreateModel(model_name, [("foo", field)]), 335 | get_pre_remove_field_operation( 336 | model_name, 337 | "foo", 338 | field, 339 | ), 340 | ] 341 | state = self.apply_operations(operations) 342 | pre_model = state.apps.get_model("tests", model_name) 343 | remove_field = migrations.RemoveField(model_name, "foo") 344 | remove_field.state_forwards("tests", state) 345 | post_model = state.apps.get_model("tests", model_name) 346 | post_model.objects.create() 347 | self.assertIsNone(pre_model.objects.get().foo) 348 | 349 | def test_database_forwards_default(self): 350 | model_name = "TestModel" 351 | field = models.IntegerField(default=42) 352 | operations = [ 353 | migrations.CreateModel(model_name, [("foo", field)]), 354 | get_pre_remove_field_operation( 355 | model_name, 356 | "foo", 357 | field, 358 | ), 359 | ] 360 | state = self.apply_operations(operations) 361 | pre_model = state.apps.get_model("tests", model_name) 362 | remove_field = migrations.RemoveField(model_name, "foo") 363 | remove_field.state_forwards("tests", state) 364 | post_model = state.apps.get_model("tests", model_name) 365 | post_model.objects.create() 366 | self.assertEqual(pre_model.objects.get().foo, 42) 367 | 368 | def test_migration_name_fragment(self): 369 | self.assertEqual( 370 | get_pre_remove_field_operation( 371 | "TestModel", "foo", models.IntegerField(default=42) 372 | ).migration_name_fragment, 373 | "set_db_default_testmodel_foo", 374 | ) 375 | self.assertEqual( 376 | get_pre_remove_field_operation( 377 | "TestModel", "foo", models.IntegerField() 378 | ).migration_name_fragment, 379 | "set_nullable_testmodel_foo", 380 | ) 381 | 382 | def test_describe(self): 383 | self.assertEqual( 384 | get_pre_remove_field_operation( 385 | "TestModel", "foo", models.IntegerField(default=42) 386 | ).describe(), 387 | "Set database DEFAULT of field foo on TestModel", 388 | ) 389 | self.assertEqual( 390 | get_pre_remove_field_operation( 391 | "TestModel", "foo", models.IntegerField() 392 | ).describe(), 393 | "Set field foo of TestModel NULLable", 394 | ) 395 | 396 | def test_deconstruct(self): 397 | model_name = "TestModel" 398 | field_name = "foo" 399 | field = models.IntegerField(default=42) 400 | operation = get_pre_remove_field_operation( 401 | model_name, 402 | field_name, 403 | field, 404 | ) 405 | deconstructed = operation.deconstruct() 406 | if field_db_default_supported: 407 | self.assertEqual( 408 | deconstructed, 409 | ( 410 | "AlterField", 411 | [], 412 | { 413 | "model_name": model_name, 414 | "name": field_name, 415 | "field": mock.ANY, 416 | "stage": Stage.PRE_DEPLOY, 417 | }, 418 | ), 419 | ) 420 | self.assertEqual( 421 | deconstructed[2]["field"].deconstruct(), 422 | ( 423 | None, 424 | "django.db.models.IntegerField", 425 | [], 426 | {"default": 42, "db_default": 42}, 427 | ), 428 | ) 429 | else: 430 | self.assertEqual( 431 | deconstructed, 432 | ( 433 | "PreRemoveField", 434 | [], 435 | {"model_name": model_name, "name": field_name, "field": field}, 436 | ), 437 | ) 438 | serializer = OperationSerializer(operation) 439 | serialized, imports = serializer.serialize() 440 | self.assertTrue(serialized.startswith("syzygy.operations.PreRemoveField")) 441 | self.assertIn("import syzygy.operations", imports) 442 | 443 | def test_elidable(self): 444 | model_name = "TestModel" 445 | field_name = "foo" 446 | field = models.IntegerField(default=42) 447 | operations = [ 448 | get_pre_remove_field_operation( 449 | model_name, 450 | field_name, 451 | field, 452 | ), 453 | migrations.RemoveField(model_name, field_name, field), 454 | ] 455 | self.assert_optimizes_to(operations, [operations[-1]]) 456 | 457 | @skipUnless(field_db_default_supported, "Field.db_default not supported") 458 | def test_defined_db_default(self): 459 | with self.assertRaisesMessage( 460 | ValueError, 461 | "Fields with a db_default don't require a pre-deployment operation.", 462 | ): 463 | get_pre_remove_field_operation( 464 | "model", "field", models.IntegerField(db_default=42) 465 | ) 466 | 467 | 468 | class RenameFieldTests(OperationTestCase): 469 | def test_serialization(self): 470 | self.assert_serde_roundtrip_equal( 471 | RenameField("model", "old_field", "new_field", stage=Stage.POST_DEPLOY) 472 | ) 473 | 474 | 475 | class RenameModelTests(OperationTestCase): 476 | def test_serialization(self): 477 | self.assert_serde_roundtrip_equal( 478 | RenameModel("old_name", "new_name", stage=Stage.POST_DEPLOY) 479 | ) 480 | 481 | 482 | class AlterFieldTests(OperationTestCase): 483 | def test_serialization(self): 484 | operation = self.serde_roundtrip( 485 | AlterField( 486 | "model", 487 | "field", 488 | models.IntegerField(), 489 | Stage.PRE_DEPLOY, 490 | ) 491 | ) 492 | self.assertEqual(operation.model_name, "model") 493 | self.assertEqual(operation.name, "field") 494 | self.assertEqual( 495 | operation.field.deconstruct(), models.IntegerField().deconstruct() 496 | ) 497 | self.assertEqual(operation.stage, Stage.PRE_DEPLOY) 498 | 499 | def test_migration_name_fragment(self): 500 | operation = AlterField( 501 | "model", 502 | "field", 503 | models.IntegerField(), 504 | Stage.PRE_DEPLOY, 505 | ) 506 | self.assertEqual(operation.migration_name_fragment, "alter_model_field") 507 | operation.migration_name_fragment = "alter_what_ever" 508 | self.assertEqual(operation.migration_name_fragment, "alter_what_ever") 509 | -------------------------------------------------------------------------------- /tests/test_plan.py: -------------------------------------------------------------------------------- 1 | import os 2 | import site 3 | from itertools import product 4 | from types import ModuleType 5 | 6 | from django.apps import AppConfig 7 | from django.conf import settings 8 | from django.db.migrations import CreateModel, DeleteModel, Migration 9 | from django.db.migrations.operations.base import Operation 10 | from django.db.migrations.operations.fields import RemoveField 11 | from django.test import SimpleTestCase 12 | 13 | from syzygy.constants import Stage 14 | from syzygy.exceptions import AmbiguousPlan, AmbiguousStage 15 | from syzygy.plan import ( 16 | get_migration_stage, 17 | get_operation_stage, 18 | get_pre_deploy_plan, 19 | hash_plan, 20 | must_post_deploy_migration, 21 | partition_operations, 22 | ) 23 | 24 | 25 | class HashPlanTests(SimpleTestCase): 26 | def test_stable(self): 27 | plan = [(Migration("0001_initial", "tests"), True)] 28 | self.assertEqual(hash_plan(plan), "a4a35230c7d1942265f1bc8f9ce53e05a50848be") 29 | 30 | def test_order(self): 31 | first = (Migration("0001_initial", "tests"), True) 32 | second = (Migration("0002_second", "tests"), True) 33 | self.assertNotEqual(hash_plan([first, second]), hash_plan([second, first])) 34 | 35 | def test_backward(self): 36 | forward = (Migration("0001_initial", "tests"), True) 37 | backward = (Migration("0001_initial", "tests"), False) 38 | self.assertNotEqual(hash_plan([forward]), hash_plan([backward])) 39 | 40 | def test_migration_name(self): 41 | first = (Migration("0001_initial", "tests"), True) 42 | second = (Migration("0002_second", "tests"), True) 43 | self.assertNotEqual(hash_plan([first]), hash_plan([second])) 44 | 45 | def test_app_label(self): 46 | test_app = (Migration("0001_initial", "tests"), True) 47 | other_app = (Migration("0001_initial", "other"), True) 48 | self.assertNotEqual(hash_plan([test_app]), hash_plan([other_app])) 49 | 50 | 51 | class GetOperationStageTests(SimpleTestCase): 52 | def test_pre_deploy_operations(self): 53 | pre_deploy_operation = Operation() 54 | pre_deploy_operation.stage = Stage.PRE_DEPLOY 55 | operations = [CreateModel("model", []), pre_deploy_operation] 56 | for operation in operations: 57 | with self.subTest(operation=operation): 58 | self.assertIs(get_operation_stage(operation), Stage.PRE_DEPLOY) 59 | 60 | def test_post_deploy_operations(self): 61 | post_deploy_operation = Operation() 62 | post_deploy_operation.stage = Stage.POST_DEPLOY 63 | operations = [ 64 | DeleteModel("model"), 65 | RemoveField("model", "field"), 66 | post_deploy_operation, 67 | ] 68 | for operation in operations: 69 | with self.subTest(operation=operation): 70 | self.assertIs(get_operation_stage(operation), Stage.POST_DEPLOY) 71 | 72 | 73 | class PartitionOperationsTests(SimpleTestCase): 74 | pre_deploy_operations = [ 75 | CreateModel("model", []), 76 | ] 77 | post_deploy_operations = [ 78 | DeleteModel("model"), 79 | ] 80 | 81 | def test_empty(self): 82 | self.assertEqual(partition_operations([], "migrations"), ([], [])) 83 | 84 | def test_pre_deploy_only(self): 85 | self.assertEqual( 86 | partition_operations(self.pre_deploy_operations, "migrations"), 87 | (self.pre_deploy_operations, []), 88 | ) 89 | 90 | def test_post_deploy_only(self): 91 | self.assertEqual( 92 | partition_operations(self.post_deploy_operations, "migrations"), 93 | ([], self.post_deploy_operations), 94 | ) 95 | 96 | def test_mixed(self): 97 | self.assertEqual( 98 | partition_operations( 99 | self.pre_deploy_operations + self.post_deploy_operations, "migrations" 100 | ), 101 | (self.pre_deploy_operations, self.post_deploy_operations), 102 | ) 103 | 104 | def test_mixed_reorder(self): 105 | post_deploy_operations = [DeleteModel("other")] 106 | self.assertEqual( 107 | partition_operations( 108 | post_deploy_operations + self.pre_deploy_operations, "migrations" 109 | ), 110 | (self.pre_deploy_operations, post_deploy_operations), 111 | ) 112 | 113 | def test_ambiguous(self): 114 | with self.assertRaises(AmbiguousStage): 115 | partition_operations( 116 | self.post_deploy_operations + self.pre_deploy_operations, "migrations" 117 | ) 118 | 119 | 120 | class GetMigrationStageTests(SimpleTestCase): 121 | def setUp(self): 122 | self.migration = Migration(app_label="tests", name="migration") 123 | 124 | def test_stage_override_setting(self): 125 | with self.settings(): 126 | del settings.MIGRATION_STAGES_OVERRIDE 127 | self.assertIsNone(get_migration_stage(self.migration)) 128 | 129 | overrides = ["tests.migration", "tests"] 130 | for stage, override in product(Stage, overrides): 131 | with self.subTest(stage=stage, override=override), self.settings( 132 | MIGRATION_STAGES_OVERRIDE={override: stage} 133 | ): 134 | self.assertIs(get_migration_stage(self.migration), stage) 135 | 136 | def test_stage_attribute(self): 137 | for stage in Stage: 138 | with self.subTest(stage=stage): 139 | self.migration.stage = stage 140 | self.assertIs(get_migration_stage(self.migration), stage) 141 | 142 | def test_operations_stages(self): 143 | self.assertIsNone(get_migration_stage(self.migration)) 144 | 145 | self.migration.operations = [CreateModel("model", [])] 146 | self.assertEqual(get_migration_stage(self.migration), Stage.PRE_DEPLOY) 147 | 148 | self.migration.operations = [ 149 | DeleteModel("model"), 150 | RemoveField("model", "field"), 151 | ] 152 | self.assertEqual(get_migration_stage(self.migration), Stage.POST_DEPLOY) 153 | 154 | def test_ambiguous_operations(self): 155 | self.migration.operations = [CreateModel("model", []), DeleteModel("model")] 156 | with self.assertRaises(AmbiguousStage): 157 | get_migration_stage(self.migration) 158 | 159 | def test_stage_fallback_setting(self): 160 | self.migration.operations = [CreateModel("model", []), DeleteModel("model")] 161 | with self.assertRaises(AmbiguousStage): 162 | get_migration_stage(self.migration) 163 | 164 | overrides = ["tests.migration", "tests"] 165 | for stage, override in product(Stage, overrides): 166 | with self.subTest(stage=stage, override=override), self.settings( 167 | MIGRATION_STAGES_FALLBACK={override: stage} 168 | ): 169 | self.assertIs(get_migration_stage(self.migration), stage) 170 | 171 | 172 | class MustPostDeployMigrationTests(SimpleTestCase): 173 | def setUp(self): 174 | self.migration = Migration(app_label="tests", name="migration") 175 | 176 | def test_forward(self): 177 | self.assertIsNone(must_post_deploy_migration(self.migration)) 178 | self.migration.stage = Stage.PRE_DEPLOY 179 | self.assertIs(must_post_deploy_migration(self.migration), False) 180 | self.migration.stage = Stage.POST_DEPLOY 181 | self.assertIs(must_post_deploy_migration(self.migration), True) 182 | 183 | def test_backward(self): 184 | self.migration.stage = Stage.PRE_DEPLOY 185 | self.assertIs(must_post_deploy_migration(self.migration, True), True) 186 | self.migration.stage = Stage.POST_DEPLOY 187 | self.assertIs(must_post_deploy_migration(self.migration, True), False) 188 | 189 | def test_ambiguous_operations(self): 190 | self.migration.operations = [CreateModel("model", []), DeleteModel("model")] 191 | with self.assertRaises(AmbiguousStage): 192 | must_post_deploy_migration(self.migration) 193 | 194 | 195 | class GetPreDeployPlanTests(SimpleTestCase): 196 | def setUp(self): 197 | self.pre_deploy = Migration(app_label="tests", name="0001") 198 | self.pre_deploy.stage = Stage.PRE_DEPLOY 199 | self.post_deploy = Migration(app_label="tests", name="0002") 200 | self.post_deploy.dependencies = [("tests", "0001")] 201 | self.post_deploy.stage = Stage.POST_DEPLOY 202 | 203 | def test_forward(self): 204 | plan = [(self.pre_deploy, False), (self.post_deploy, False)] 205 | self.assertEqual(get_pre_deploy_plan(plan), [(self.pre_deploy, False)]) 206 | 207 | def test_backward(self): 208 | plan = [(self.post_deploy, True), (self.pre_deploy, True)] 209 | self.assertEqual(get_pre_deploy_plan(plan), [(self.post_deploy, True)]) 210 | 211 | def test_non_contiguous_free(self): 212 | post_deploy_free = Migration(app_label="other", name="0001") 213 | post_deploy_free.stage = Stage.POST_DEPLOY 214 | plan = [ 215 | (post_deploy_free, False), 216 | (self.pre_deploy, False), 217 | (self.post_deploy, False), 218 | ] 219 | self.assertEqual(get_pre_deploy_plan(plan), [(self.pre_deploy, False)]) 220 | 221 | def test_non_contiguous_free_backward(self): 222 | pre_deploy_free = Migration(app_label="other", name="0001") 223 | pre_deploy_free.stage = Stage.PRE_DEPLOY 224 | plan = [ 225 | (pre_deploy_free, True), 226 | (self.post_deploy, True), 227 | (self.pre_deploy, True), 228 | ] 229 | self.assertEqual(get_pre_deploy_plan(plan), [(self.post_deploy, True)]) 230 | 231 | def test_non_contiguous_deps(self): 232 | pre_deploy_dep = Migration(app_label="other", name="0001") 233 | pre_deploy_dep.stage = Stage.PRE_DEPLOY 234 | pre_deploy_dep.dependencies = [("tests", "0002")] 235 | plan = [ 236 | (self.pre_deploy, False), 237 | (self.post_deploy, False), 238 | (pre_deploy_dep, False), 239 | ] 240 | msg = ( 241 | "Plan contains a non-contiguous sequence of pre-deployment migrations. " 242 | "Migration other.0001 is defined to be applied pre-deployment but it " 243 | "depends on tests.0002 which is defined to be applied post-deployment." 244 | ) 245 | with self.assertRaisesMessage(AmbiguousPlan, msg): 246 | get_pre_deploy_plan(plan) 247 | del pre_deploy_dep.stage 248 | pre_deploy_dep.operations = [ 249 | CreateModel("model", []), 250 | ] 251 | msg = ( 252 | "Plan contains a non-contiguous sequence of pre-deployment migrations. " 253 | "Migration other.0001 is inferred to be applied pre-deployment but it " 254 | "depends on tests.0002 which is defined to be applied post-deployment. " 255 | "Defining an explicit `Migration.stage: syzygy.Stage` for other.0001 " 256 | "to bypass inference might help." 257 | ) 258 | with self.assertRaisesMessage(AmbiguousPlan, msg): 259 | get_pre_deploy_plan(plan) 260 | del self.post_deploy.stage 261 | self.post_deploy.operations = [ 262 | DeleteModel("model"), 263 | ] 264 | msg = ( 265 | "Plan contains a non-contiguous sequence of pre-deployment migrations. " 266 | "Migration other.0001 is inferred to be applied pre-deployment but it " 267 | "depends on tests.0002 which is inferred to be applied post-deployment. " 268 | "Defining an explicit `Migration.stage: syzygy.Stage` for other.0001 " 269 | "or tests.0002 to bypass inference might help." 270 | ) 271 | with self.assertRaisesMessage(AmbiguousPlan, msg): 272 | get_pre_deploy_plan(plan) 273 | 274 | def test_non_contiguous_deps_third_party(self): 275 | pre_deploy_dep = Migration(app_label="third_party", name="0001") 276 | pre_deploy_dep.operations = [ 277 | CreateModel("model", []), 278 | ] 279 | pre_deploy_dep.dependencies = [("tests", "0002")] 280 | plan = [ 281 | (self.pre_deploy, False), 282 | (self.post_deploy, False), 283 | (pre_deploy_dep, False), 284 | ] 285 | msg = ( 286 | "Plan contains a non-contiguous sequence of pre-deployment migrations. " 287 | "Migration third_party.0001 is inferred to be applied pre-deployment but it " 288 | "depends on tests.0002 which is defined to be applied post-deployment. " 289 | "Setting `MIGRATION_STAGES_OVERRIDE['third_party.0001']` to an explicit " 290 | "`syzygy.Stage` to bypass inference might help." 291 | ) 292 | third_party_module = ModuleType("third_party") 293 | third_party_module.__file__ = os.path.join(site.PREFIXES[0], "third_party.py") 294 | third_party_app_config = AppConfig("third_party", third_party_module) 295 | with self.modify_settings( 296 | INSTALLED_APPS={"append": [third_party_app_config]} 297 | ), self.assertRaisesMessage(AmbiguousPlan, msg): 298 | get_pre_deploy_plan(plan) 299 | del self.post_deploy.stage 300 | self.post_deploy.operations = [ 301 | DeleteModel("model"), 302 | ] 303 | msg = ( 304 | "Plan contains a non-contiguous sequence of pre-deployment migrations. " 305 | "Migration third_party.0001 is inferred to be applied pre-deployment but it " 306 | "depends on tests.0002 which is inferred to be applied post-deployment. " 307 | "Defining an explicit `Migration.stage: syzygy.Stage` for tests.0002 or " 308 | "setting `MIGRATION_STAGES_OVERRIDE['third_party.0001']` to an explicit " 309 | "`syzygy.Stage` to bypass inference might help." 310 | ) 311 | with self.modify_settings( 312 | INSTALLED_APPS={"append": [third_party_app_config]} 313 | ), self.assertRaisesMessage(AmbiguousPlan, msg): 314 | get_pre_deploy_plan(plan) 315 | -------------------------------------------------------------------------------- /tests/test_quorum.py: -------------------------------------------------------------------------------- 1 | import time 2 | import uuid 3 | from multiprocessing.pool import ThreadPool 4 | from random import shuffle 5 | 6 | from django.core.exceptions import ImproperlyConfigured 7 | from django.test.testcases import SimpleTestCase 8 | from django.test.utils import override_settings 9 | 10 | from syzygy.quorum import ( 11 | QuorumDisolved, 12 | join_quorum, 13 | poll_quorum, 14 | sever_quorum, 15 | ) 16 | 17 | 18 | class QuorumConfigurationTests(SimpleTestCase): 19 | def test_missing_setting(self): 20 | msg = ( 21 | "The `MIGRATION_QUORUM_BACKEND` setting must be configured " 22 | "for syzygy.quorum to be used" 23 | ) 24 | with self.assertRaisesMessage(ImproperlyConfigured, msg): 25 | join_quorum("foo", 1) 26 | 27 | @override_settings(MIGRATION_QUORUM_BACKEND={}) 28 | def test_misconfigured_setting(self): 29 | msg = ( 30 | "The `MIGRATION_QUORUM_BACKEND` setting must either be an import " 31 | "path string or a dict with a 'backend' path key string" 32 | ) 33 | with self.assertRaisesMessage(ImproperlyConfigured, msg): 34 | join_quorum("foo", 1) 35 | 36 | @override_settings(MIGRATION_QUORUM_BACKEND="syzygy.void") 37 | def test_cannot_import_backend(self): 38 | msg = "Cannot import `MIGRATION_QUORUM_BACKEND` backend 'syzygy.void'" 39 | with self.assertRaisesMessage(ImproperlyConfigured, msg): 40 | join_quorum("foo", 1) 41 | 42 | @override_settings( 43 | MIGRATION_QUORUM_BACKEND={ 44 | "backend": "syzygy.quorum.backends.cache.CacheQuorum", 45 | "unsupported": True, 46 | } 47 | ) 48 | def test_cannot_initialize_backend(self): 49 | msg = ( 50 | "Cannot initialize `MIGRATION_QUORUM_BACKEND` backend " 51 | "'syzygy.quorum.backends.cache.CacheQuorum' with {'unsupported': True}" 52 | ) 53 | with self.assertRaisesMessage(ImproperlyConfigured, msg): 54 | join_quorum("foo", 1) 55 | 56 | 57 | class BaseQuorumTestMixin: 58 | quorum = 5 59 | 60 | def test_multiple(self): 61 | namespace = str(uuid.uuid4()) 62 | self.assertFalse(join_quorum(namespace, 2)) 63 | self.assertFalse(poll_quorum(namespace, 2)) 64 | self.assertTrue(join_quorum(namespace, 2)) 65 | self.assertTrue(poll_quorum(namespace, 2)) 66 | 67 | @classmethod 68 | def attain_quorum(cls, namespace): 69 | if join_quorum(namespace, cls.quorum): 70 | return True 71 | while not poll_quorum(namespace, cls.quorum): 72 | time.sleep(0.01) 73 | return False 74 | 75 | def test_attainment(self): 76 | namespace = str(uuid.uuid4()) 77 | 78 | with ThreadPool(processes=self.quorum) as pool: 79 | results = pool.map_async( 80 | self.attain_quorum, [namespace] * self.quorum 81 | ).get() 82 | 83 | self.assertEqual(sum(1 for result in results if result is True), 1) 84 | self.assertEqual(sum(1 for result in results if result is False), 4) 85 | 86 | def test_attainment_namespace_reuse(self): 87 | namespace = str(uuid.uuid4()) 88 | self.assertFalse(join_quorum(namespace, 2)) 89 | self.assertTrue(join_quorum(namespace, 2)) 90 | self.assertTrue(poll_quorum(namespace, 2)) 91 | # Once quorum is reached its associated namespace is immediately 92 | # cleared to make it reusable. 93 | self.assertFalse(join_quorum(namespace, 2)) 94 | self.assertTrue(join_quorum(namespace, 2)) 95 | self.assertTrue(poll_quorum(namespace, 2)) 96 | 97 | def test_disolution(self): 98 | namespace = str(uuid.uuid4()) 99 | 100 | calls = [(self.attain_quorum, (namespace,))] * (self.quorum - 1) 101 | calls.append((sever_quorum, (namespace, self.quorum))) 102 | shuffle(calls) 103 | 104 | with ThreadPool(processes=self.quorum) as pool: 105 | results = [pool.apply_async(func, args) for func, args in calls] 106 | pool.close() 107 | pool.join() 108 | 109 | disolved = 0 110 | for result in results: 111 | try: 112 | attained = result.get() 113 | except QuorumDisolved: 114 | disolved += 1 115 | else: 116 | if attained is not None: 117 | self.fail(f"Unexpected quorum attainment: {attained}") 118 | self.assertEqual(disolved, self.quorum - 1) 119 | 120 | def test_disolution_namespace_reuse(self): 121 | namespace = str(uuid.uuid4()) 122 | self.assertFalse(join_quorum(namespace, 2)) 123 | sever_quorum(namespace, 2) 124 | with self.assertRaises(QuorumDisolved): 125 | poll_quorum(namespace, 2) 126 | # Once quorum is disolved its associated namespace is immediately 127 | # cleared to make it reusable. 128 | self.assertFalse(join_quorum(namespace, 2)) 129 | self.assertTrue(join_quorum(namespace, 2)) 130 | self.assertTrue(poll_quorum(namespace, 2)) 131 | 132 | 133 | @override_settings(MIGRATION_QUORUM_BACKEND="syzygy.quorum.backends.cache.CacheQuorum") 134 | class CacheQuorumTests(BaseQuorumTestMixin, SimpleTestCase): 135 | pass 136 | 137 | 138 | @override_settings( 139 | CACHES={ 140 | "quorum": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, 141 | }, 142 | MIGRATION_QUORUM_BACKEND={ 143 | "backend": "syzygy.quorum.backends.cache.CacheQuorum", 144 | "alias": "quorum", 145 | "version": 46, 146 | }, 147 | ) 148 | class CacheQuorumConfigsTests(BaseQuorumTestMixin, SimpleTestCase): 149 | pass 150 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = true 3 | args_are_paths = false 4 | envlist = 5 | black 6 | flake8 7 | isort 8 | mypy 9 | pypi 10 | py39-4.2-{sqlite,postgresql,mysql} 11 | py310-{4.2,5.0,5.1,5.2,main}-{sqlite,postgresql,mysql} 12 | py{311,312}-{4.2,5.0,5.1,5.2,main}-{sqlite,postgresql,mysql} 13 | py313-{5.1,5.2,main}-{sqlite,postgresql,mysql} 14 | 15 | [gh-actions] 16 | python = 17 | 3.9: py39, black, flake8, isort 18 | 3.10: py310, mypy 19 | 3.11: py311 20 | 3.12: py312 21 | 3.13: py313 22 | 23 | [testenv] 24 | basepython = 25 | py39: python3.9 26 | py310: python3.10 27 | py311: python3.11 28 | py312: python3.12 29 | py313: python3.13 30 | usedevelop = true 31 | setenv = 32 | DJANGO_SETTINGS_MODULE=tests.settings 33 | postgresql: DJANGO_SETTINGS_MODULE=tests.settings.postgresql 34 | mysql: DJANGO_SETTINGS_MODULE=tests.settings.mysql 35 | passenv = 36 | GITHUB_* 37 | DB_* 38 | commands = 39 | {envpython} -R -Wonce {envbindir}/coverage run -a -m django test -v2 {posargs} 40 | coverage report 41 | deps = 42 | coverage 43 | 4.2: Django>=4.2,<5 44 | 5.0: Django>=5,<5.1 45 | 5.1: Django>=5.1,<5.2 46 | 5.2: Django>=5.2a1,<6.0 47 | main: https://github.com/django/django/archive/main.tar.gz 48 | postgresql: psycopg2-binary 49 | mysql: mysqlclient 50 | ignore_outcome = 51 | main: true 52 | 53 | [testenv:black] 54 | usedevelop = false 55 | basepython = python3.9 56 | commands = black --check syzygy tests 57 | deps = black 58 | 59 | [testenv:flake8] 60 | usedevelop = false 61 | basepython = python3.9 62 | commands = flake8 63 | deps = flake8 64 | 65 | [testenv:isort] 66 | usedevelop = false 67 | basepython = python3.9 68 | commands = isort --check-only --diff syzygy tests 69 | deps = 70 | isort 71 | Django>=3.2,<4 72 | 73 | [testenv:mypy] 74 | usedevelop = false 75 | basepython = python3.10 76 | commands = mypy -p syzygy --warn-redundant-casts --warn-unused-ignores 77 | deps = 78 | django>=5.2,<6 79 | mypy>=1.13 80 | django-stubs>=5.2 81 | 82 | [testenv:pypi] 83 | usedevelop = false 84 | basepython = python3.9 85 | commands = 86 | python setup.py sdist --format=gztar bdist_wheel 87 | twine check dist/* 88 | deps = 89 | pip 90 | setuptools 91 | twine 92 | wheel 93 | --------------------------------------------------------------------------------