├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.rst ├── django_sqlibrist ├── __init__.py ├── helpers.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── sqlibrist.py └── settings.py ├── requirements.txt ├── setup.cfg ├── setup.py └── sqlibrist ├── __init__.py ├── commands ├── __init__.py ├── diff.py ├── info.py ├── init.py ├── initdb.py ├── makemigration.py ├── migrate.py ├── status.py └── test_connection.py ├── engines.py └── helpers.py /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | *.egg-info 3 | build/ 4 | dist/ 5 | upload.sh -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) [year] [fullname] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | global-exclude *.pyc 2 | 3 | include LICENSE 4 | include README.rst 5 | 6 | recursive-include sqlibrist *.py 7 | recursive-include django_sqlibrist *.py -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | sqlibrist 2 | ========= 3 | 4 | Sqlibrist is command-line tool, made for developers, who do not use ORM to manage their database 5 | structure. Programming database objects and deploying them to production 6 | is not easy. Naive approach is to manually write patches with SQL statements and then replay 7 | them on others DB instances. This, being simple and straightforward, may get tricky 8 | when your database structure grows in size and have numerous inter-dependent 9 | objects. 10 | 11 | Sqlibrist in essense is tool to make the process of creating SQL patches much more 12 | easy, and as side-effect it proposes a way to organize your SQL code. It does not 13 | dictate design desisions, or stands on your way when you do something wrong 14 | (notorious shooting in foot). All database objects are described declaratively 15 | in separate files in the form of ``CREATE TABLE`` or ``CREATE FUNCTION``, having 16 | dependency instructions. 17 | 18 | You may think of sqlibrist as Version Control System for database. The whole thing 19 | is inspired by Sqitch (see Alternatives below) and Django Migrations (may be you 20 | remember Django South). Every time you invoke ``makemigration`` command, snapshot 21 | of current scheme is made and compared with previous snapshot. Then, SQL patch 22 | is created with instructions to recreate all changed objects cascadely with their 23 | dependencies, create new or remove deleted. In the latter case, sqlibrist will not 24 | let you delete object that has left dependants. 25 | 26 | Currently PostgreSQL is supported. MySQL support is experimental, and not well-tested 27 | yet. 28 | 29 | 30 | Platform compatibility 31 | ====================== 32 | 33 | Linux, Mac OS, Windows. See installation instructions for each below. 34 | 35 | 36 | Requirements 37 | ============ 38 | 39 | Python dependencies: 40 | 41 | - PyYAML 42 | - psycopg2 (optional) 43 | - mysql-python (optional) 44 | 45 | Installation 46 | ============ 47 | 48 | Linux 49 | ----- 50 | 51 | **Ubuntu/Debian** 52 | 53 | First install required libraries:: 54 | 55 | $ sudo apt-get install python-pip python-dev libyaml-dev 56 | $ sudo apt-get install libmysqlclient-dev # for MySQL 57 | $ sudo apt-get install libpq-dev # PostgreSQL 58 | 59 | Sqlibrist can be installed into virtualenv:: 60 | 61 | $ pip install sqlibrist 62 | 63 | or system-wide:: 64 | 65 | $ sudo pip install sqlibrist 66 | 67 | **Fedora/CentOS/RHEL** 68 | 69 | First install required libraries (replace ``dnf`` to ``yum`` if you are using 70 | pre-dnf package manager):: 71 | 72 | $ sudo dnf install python-devel python-pip libyaml-devel 73 | $ sudo dnf install postgresql-devel # PostgreSQL 74 | 75 | $ sudo dnf install mariadb-devel # for MariaDB 76 | or 77 | $ sudo dnf install mysql++-devel # for MySQL 78 | 79 | Sqlibrist can be installed into virtualenv:: 80 | 81 | $ pip install sqlibrist 82 | 83 | or system-wide:: 84 | 85 | $ sudo pip install sqlibrist 86 | 87 | 88 | MacOS 89 | ----- 90 | 91 | First install required libraries:: 92 | 93 | $ sudo easy_install pip 94 | $ brew install mysql # for MySQL 95 | $ brew install postgres # PostgreSQL 96 | 97 | 98 | Sqlibrist can be installed into virtualenv:: 99 | 100 | $ pip install sqlibrist 101 | 102 | or system-wide:: 103 | 104 | $ sudo pip install sqlibrist 105 | 106 | Also you need to install database dependencies 107 | MySQL:: 108 | 109 | $ pip install mysql-python 110 | 111 | PostgreSQL:: 112 | 113 | $ pip install psycopg2 114 | 115 | Windows 116 | ------- 117 | TODO 118 | 119 | Tutorial 120 | ======== 121 | 122 | Let's create simple project and go through typical steps of DB schema manageent. 123 | This will be small webshop. 124 | 125 | Create empty directory:: 126 | 127 | $ mkdir shop_schema 128 | $ cd shop_schema 129 | 130 | Then we need to create sqlibrist database structure, where we will keep 131 | schema and migrations:: 132 | 133 | $ sqlibrist init 134 | Creating directories... 135 | Done. 136 | 137 | You will get the following DB structure:: 138 | 139 | shop_schema 140 | sqlibrist.yaml 141 | migrations 142 | schema 143 | constraints 144 | functions 145 | indexes 146 | tables 147 | triggers 148 | types 149 | views 150 | 151 | In ``sqlibrist.yaml`` you will configure DB connections:: 152 | 153 | --- 154 | default: 155 | engine: pg 156 | user: 157 | name: 158 | password: 159 | # host: 127.0.0.1 160 | # port: 5432 161 | 162 | ``host`` and ``port`` are optional. 163 | 164 | Once you configured DB connection, test if is correct:: 165 | 166 | $ sqlibrist test_connection 167 | Connection OK 168 | 169 | Next we need to create sqlibrist migrations table:: 170 | 171 | $ sqlibrist initdb 172 | Creating db... 173 | Creating schema and migrations log table... 174 | 175 | Done. 176 | 177 | Now we are ready to build our DB schema. 178 | 179 | Create file ``shop_schema/schema/tables/user.sql``:: 180 | 181 | --UP 182 | CREATE TABLE "user" ( 183 | id SERIAL PRIMARY KEY, 184 | name TEXT, 185 | password TEXT); 186 | 187 | The first line ``--UP`` means that the following are SQL statements for 'forward' 188 | migration. The opposite is optional ``--DOWN``, which contains instructions for reverting. 189 | To be safe, and not accidentally drop any table with your data, we will not include 190 | anything like DROP TABLE. Working with table upgrades and ``--DOWN`` is on the way 191 | below. 192 | 193 | ``shop_schema/schema/tables/product.sql``:: 194 | 195 | --UP 196 | CREATE TABLE product ( 197 | id SERIAL PRIMARY KEY, 198 | name TEXT, 199 | price MONEY); 200 | 201 | ``shop_schema/schema/tables/order.sql``:: 202 | 203 | --REQ tables/user 204 | --UP 205 | CREATE TABLE "order" ( 206 | id SERIAL PRIMARY KEY, 207 | user_id INTEGER REFERENCES "user"(id), 208 | date DATE); 209 | 210 | Important here is the ``--REQ tables/user`` statement. It tells sqlibrist, that 211 | ``order`` table depends on ``user`` table. This will guarantee, that ``user`` will 212 | be created before ``order``. 213 | 214 | ``shop_schema/schema/tables/order_product.sql``:: 215 | 216 | --REQ tables/order 217 | --UP 218 | CREATE TABLE order_product ( 219 | id SERIAL PRIMARY KEY, 220 | order_id INTEGER REFERENCES "order"(id), 221 | product_id INTEGER REFERENCES product(id), 222 | quantity INTEGER); 223 | 224 | Ok, now let's create our first migration:: 225 | 226 | $ sqlibrist makemigration -n 'initial' 227 | Creating: 228 | tables/user 229 | tables/product 230 | tables/order 231 | tables/order_product 232 | Creating new migration 0001-initial 233 | 234 | New files were created in ``shop_schema/migrations/0001-initial``:: 235 | 236 | up.sql 237 | down.sql 238 | schema.json 239 | 240 | ``up.sql`` contains SQL to apply your changes (create tables), ``down.sql`` has nothing 241 | notable, since our .sql files have no ``--DOWN`` section, and the ``schema.json`` 242 | has snapshot of current schema. 243 | 244 | If you want to make more changes to the schema files prior to applying newly created 245 | migration, delete the directory with those 3 files, in our case ``0001-initial``. 246 | 247 | You are free to review and edit ``up.sql`` and ``down.sql``, of course if you know what 248 | you are doing. **DO NOT edit schema.json**. 249 | 250 | Now go ahead and apply our migration:: 251 | 252 | $ sqlibrist migrate 253 | Applying migration 0001-initial... done 254 | 255 | Well done! Tables are created, but let's do something more interesting. 256 | 257 | We will create view that shows all user orders with order total: 258 | 259 | ``shop_schema/schema/views/user_orders.sql``:: 260 | 261 | --REQ tables/user 262 | --REQ tables/order 263 | --REQ tables/product 264 | --REQ tables/order_product 265 | 266 | --UP 267 | CREATE VIEW user_orders AS SELECT 268 | u.id as user_id, 269 | o.id as order_id, 270 | o.date, 271 | SUM(p.price*op.quantity) AS total 272 | 273 | FROM "user" u 274 | INNER JOIN "order" o ON u.id=o.user_id 275 | INNER JOIN order_product op ON o.id=op.order_id 276 | INNER JOIN product p ON p.id=op.product_id 277 | 278 | GROUP BY o.id, u.id; 279 | 280 | --DOWN 281 | DROP VIEW user_orders; 282 | 283 | ... and function to return only given user's orders: 284 | 285 | ``shop_schema/schema/functions/get_user_orders.sql``:: 286 | 287 | --REQ views/user_orders 288 | 289 | --UP 290 | CREATE FUNCTION get_user_orders(_user_id INTEGER) 291 | RETURNS SETOF user_orders 292 | LANGUAGE SQL AS $$ 293 | 294 | SELECT * FROM user_orders 295 | WHERE user_id=_user_id; 296 | 297 | $$; 298 | 299 | --DOWN 300 | DROP FUNCTION get_user_orders(INTEGER); 301 | 302 | Next create new migration and apply it:: 303 | 304 | $ sqlibrist makemigration -n 'user_orders view and function' 305 | Creating: 306 | views/user_orders 307 | functions/get_user_orders 308 | Creating new migration 0002-user_orders view and function 309 | 310 | $ sqlibrist migrate 311 | Applying migration 0002-user_orders view and function... done 312 | 313 | We have four tables, one view and one function. 314 | 315 | Now you want to add one more field in the ``user_orders`` view. There can be couple 316 | of issues here: 317 | 318 | * we could try to drop and create updated view, but the database server will 319 | complain, that *get_user_orders* function depends on droppable view; 320 | 321 | * we could be smart and create view with ``CREATE OR REPLACE VIEW user_orders...``, 322 | however single view's fields and their types make separate type, and the 323 | function ``get_user_orders`` returns that type. We can't simply change view type 324 | without recreating the function. 325 | 326 | This is where sqlibrist comes to help. Add one more field ``SUM(op.quantity) as order_total`` 327 | to the ``user_orders`` view:: 328 | 329 | --REQ tables/user 330 | --REQ tables/order 331 | --REQ tables/product 332 | --REQ tables/order_product 333 | 334 | --UP 335 | CREATE VIEW user_orders AS SELECT 336 | u.id as user_id, 337 | o.id as order_id, 338 | o.date, 339 | SUM(p.price*op.quantity) AS total, 340 | SUM(op.quantity) as order_total 341 | 342 | FROM "user" u 343 | INNER JOIN "order" o ON u.id=o.user_id 344 | INNER JOIN order_product op ON o.id=op.order_id 345 | INNER JOIN product p ON p.id=op.product_id 346 | 347 | GROUP BY o.id, u.id; 348 | 349 | --DOWN 350 | DROP VIEW user_orders; 351 | 352 | We can see, what was changed from the latest schema snapshot:: 353 | 354 | $ sqlibrist -V diff 355 | Changed items: 356 | views/user_orders 357 | --- 358 | 359 | +++ 360 | 361 | @@ -2,7 +2,8 @@ 362 | 363 | u.id as user_id, 364 | o.id as order_id, 365 | o.date, 366 | - SUM(p.price*op.quantity) AS total 367 | + SUM(p.price*op.quantity) AS total, 368 | + SUM(op.quantity) as total_quantity 369 | 370 | FROM "user" u 371 | INNER JOIN "order" o ON u.id=o.user_id 372 | 373 | Now let's make migration:: 374 | 375 | $ sqlibrist makemigration 376 | Updating: 377 | dropping: 378 | functions/get_user_orders 379 | views/user_orders 380 | creating: 381 | views/user_orders 382 | functions/get_user_orders 383 | Creating new migration 0003-auto 384 | 385 | You can see, that sqlibrist first drops ``get_user_orders`` function, after that 386 | ``user_orders`` view does not have dependent objects and can be dropped too. 387 | Then view and function are created in order, opposite to dropping. 388 | Apply our changes:: 389 | 390 | $ sqlibrist migrate 391 | Applying migration 0003-auto... done 392 | 393 | Last topic is to make change to table structure. Since we did not add ``--DROP`` section 394 | to our tables, any change has to be made manually. This is done in several steps: 395 | 396 | 1. Edit CREATE TABLE definition to reflect new structure; 397 | 2. Generate new migration with ``makemigration`` command; 398 | 3. Manually edit new migration's ``up.sql`` with ALTER TABLE instructions. 399 | 400 | To demonstrate this, let's add field ``type text`` to the ``product`` table. It will 401 | look like this: 402 | 403 | ``shop_schema/schema/tables/product.sql``:: 404 | 405 | --UP 406 | CREATE TABLE product ( 407 | id SERIAL PRIMARY KEY, 408 | name TEXT, 409 | "type" TEXT, 410 | price MONEY); 411 | 412 | This was #1. Next create new migration:: 413 | 414 | $ sqlibrist makemigration -n 'new product field' 415 | Updating: 416 | dropping: 417 | functions/get_user_orders 418 | views/user_orders 419 | creating: 420 | views/user_orders 421 | functions/get_user_orders 422 | Creating new migration 0004-new product field 423 | 424 | Please, pay attention here, that even though we changed product table definition, 425 | ``tables/product`` is not in migration process, but ALL dependent objects are recreated. 426 | This behavior is intended. This was #2. 427 | 428 | Now #3: open ``shop_schema/migrations/0004-new product field/up.sql`` with your editor 429 | and look for line 12 with text ``-- ==== Add your instruction here ====``. This is 430 | the point in migration when all dependent objects are dropped and you can issue 431 | ALTER TABLE instructions. 432 | 433 | Just below this line paste following:: 434 | 435 | ALTER TABLE product 436 | ADD COLUMN "type" TEXT; 437 | 438 | Your ``up.sql`` will look like this:: 439 | 440 | -- begin -- 441 | DROP FUNCTION get_user_orders(INTEGER); 442 | -- end -- 443 | 444 | 445 | -- begin -- 446 | DROP VIEW user_orders; 447 | -- end -- 448 | 449 | 450 | -- begin -- 451 | -- ==== Add your instruction here ==== 452 | ALTER TABLE product 453 | ADD COLUMN "type" TEXT; 454 | -- end -- 455 | 456 | 457 | -- begin -- 458 | CREATE VIEW user_orders AS SELECT 459 | u.id as user_id, 460 | o.id as order_id, 461 | o.date, 462 | SUM(p.price*op.quantity) AS total, 463 | SUM(op.quantity) as total_quantity 464 | 465 | FROM "user" u 466 | INNER JOIN "order" o ON u.id=o.user_id 467 | INNER JOIN order_product op ON o.id=op.order_id 468 | INNER JOIN product p ON p.id=op.product_id 469 | 470 | GROUP BY o.id, u.id; 471 | -- end -- 472 | 473 | 474 | -- begin -- 475 | CREATE FUNCTION get_user_orders(_user_id INTEGER) 476 | RETURNS SETOF user_orders 477 | LANGUAGE SQL AS $$ 478 | 479 | SELECT * FROM user_orders 480 | WHERE user_id=_user_id; 481 | 482 | $$; 483 | -- end -- 484 | 485 | Migration text is self-explanatory: drop function and view, alter table and then 486 | create view and function, with respect to their dependencies. 487 | 488 | Finally, apply your changes:: 489 | 490 | $ sqlibrist migrate 491 | Applying migration 0004-new product field... done 492 | 493 | 494 | Rules of thumb 495 | ============== 496 | 497 | * **do not add CASCADE to DROP statements, even when dropping views/functions/indexes**. 498 | You may and will implicitly drop table(s) with your data; 499 | 500 | * **avoid circular dependencies**. If you create objects that depend on each other 501 | in circle, sqlibrist will not know, how to update them. I bet, you will try to 502 | do so, but migration will not be created and sqlibrist will show you warning and 503 | dependency path; 504 | 505 | * **do not create --DOWN sections for tables**. Manually write ALTER TABLE instructions 506 | as described in the Tutorial; 507 | 508 | * **always test migrations on your test database before applying them to production**. 509 | 510 | 511 | Django integration 512 | ================== 513 | 514 | Sqlibrist has a very small application to integrate itself into your Django 515 | project and access DB configuration. 516 | 517 | Installation 518 | ------------ 519 | 520 | Add ``'django_sqlibrist'`` to INSTALLED_APPS 521 | 522 | Settings 523 | -------- 524 | 525 | ``SQLIBRIST_DIRECTORY`` - Path to the directory with schema and migrations files. 526 | Defaults to project's BASE_DIR/sql 527 | 528 | Usage 529 | ----- 530 | :: 531 | 532 | $ python manage.py sqlibrist [options] 533 | 534 | If you want your tables to be accessible from Django ORM and/or for using 535 | Django Admin for these tables, add following attributes to the model's ``Meta`` class: 536 | :: 537 | 538 | class SomeTable(models.Model): 539 | field1 = models.CharField() 540 | ... 541 | class Meta: 542 | managed = False # will tell Django to not create migrations for that table 543 | table_name = 'sometable' # name of your table 544 | 545 | If primary key has other name than ``id`` and type not Integer, add that field to 546 | model class with ``primary_key=True`` argument, for example:: 547 | 548 | my_key = models.IntegerField(primary_key=True) 549 | 550 | Migrating existing models 551 | ------------------------- 552 | TODO: 553 | 554 | 555 | Alternatives 556 | ============ 557 | 558 | Sqlibrist is not new concept, it has a lot of alternatives, most notable, I think, 559 | is [sqitch](http://sqitch.org/). It is great tool, with rich development history and 560 | community around it. I started using it at first, however it did not make me completely 561 | happy. My problem with sqitch was pretty hard installation progress 562 | (shame on me, first of all). It is written in Perl and has huge number of dependencies. 563 | For man, unfamiliar with Perl pachage systems, it was quite a challenge to 564 | install sqitch on 3 different Linux distributions: Fedora, Ubuntu and Arch. 565 | In addition, I found sqitch's dependency tracking being complicated and unobvious 566 | to perform relatively simple schema changes. Don't get me wrong - I am not 567 | advocating you against using sqitch, you should try it yourself. 568 | 569 | 570 | TODO 571 | ==== 572 | 573 | - documentation 574 | * django_sqlibrist: Migrating existing models 575 | * detailed info on all commands 576 | 577 | Changelog 578 | ========= 579 | 580 | 0.1.10 string encoding fix 581 | 582 | 0.1.9 config file parsing fixed, MAC installation added (thanks to https://github.com/tolyadouble); django integration improved 583 | 584 | 0.1.8 fixes 585 | 586 | 0.1.7 fixes 587 | 588 | 0.1.6 fixes 589 | 590 | 0.1.5 django_sqlibrist takes engine and connection from django project settings 591 | 592 | 0.1.4 django_sqlibrist configurator fixed 593 | 594 | 0.1.3 django_sqlibrist configurator fixed 595 | 596 | 0.1.2 LazyConfig fixed 597 | 598 | 0.1.1 fixed loading config file 599 | 600 | 0.1.0 django_sqlibrist gets DB connection settings from Django project's settings instead of config file 601 | 602 | 0.0.7 django_sqlibrist moved to separate package and is importable in settings.py as "django_sqlibrist" 603 | -------------------------------------------------------------------------------- /django_sqlibrist/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from django_sqlibrist.helpers import patch_test_db_creation 3 | 4 | patch_test_db_creation() -------------------------------------------------------------------------------- /django_sqlibrist/helpers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from django.conf import settings 3 | 4 | from sqlibrist.helpers import ENGINE_POSTGRESQL, ENGINE_MYSQL, BadConfig 5 | 6 | DEFAULT_PORTS = { 7 | ENGINE_POSTGRESQL: '5432' 8 | } 9 | 10 | 11 | def get_config(): 12 | """ 13 | Gets engine type from Django settings 14 | """ 15 | DB = settings.DATABASES['default'] 16 | ENGINE = DB.get('ENGINE', '') 17 | config = {} 18 | 19 | if 'postgresql' in ENGINE \ 20 | or 'psycopg' in ENGINE: 21 | config['engine'] = ENGINE_POSTGRESQL 22 | elif 'mysql' in ENGINE: 23 | config['engine'] = ENGINE_MYSQL 24 | 25 | else: 26 | raise BadConfig('Django configured with unsupported database engine: ' 27 | '%s' % DB.get('ENGINE', '')) 28 | 29 | return config 30 | 31 | 32 | class Args(object): 33 | """ 34 | Wrapper for "options" argument for Django command. Translates attribute 35 | access to dict item access 36 | """ 37 | def __init__(self, options): 38 | self.options = options 39 | 40 | def __getattr__(self, item): 41 | return self.options[item] 42 | 43 | 44 | def patch_test_db_creation(): 45 | from django.db.backends.base.creation import BaseDatabaseCreation 46 | 47 | create_test_db_original = BaseDatabaseCreation.create_test_db 48 | 49 | def create_test_db_patched(*args, **kwargs): 50 | test_database_name = create_test_db_original(*args, **kwargs) 51 | 52 | from django.core.management import call_command 53 | call_command('sqlibrist', 'initdb') 54 | call_command('sqlibrist', 'migrate') 55 | 56 | return test_database_name 57 | 58 | BaseDatabaseCreation.create_test_db = create_test_db_patched 59 | -------------------------------------------------------------------------------- /django_sqlibrist/management/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | -------------------------------------------------------------------------------- /django_sqlibrist/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | -------------------------------------------------------------------------------- /django_sqlibrist/management/commands/sqlibrist.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import absolute_import 3 | 4 | import os 5 | from contextlib import contextmanager 6 | 7 | from django.core.management import BaseCommand 8 | from django.db import connection 9 | 10 | from django_sqlibrist.helpers import get_config, Args 11 | from django_sqlibrist.settings import SQLIBRIST_DIRECTORY 12 | from sqlibrist.helpers import get_command_parser, SqlibristException, \ 13 | handle_exception 14 | 15 | 16 | @contextmanager 17 | def chdir(target): 18 | current_dir = os.curdir 19 | os.chdir(target) 20 | yield 21 | os.chdir(current_dir) 22 | 23 | 24 | class Command(BaseCommand): 25 | def add_arguments(self, parser): 26 | get_command_parser(parser) 27 | 28 | def handle(self, *args, **options): 29 | config = get_config() 30 | 31 | with chdir(SQLIBRIST_DIRECTORY): 32 | try: 33 | options['func'](Args(options), config, connection) 34 | except SqlibristException as e: 35 | handle_exception(e) 36 | -------------------------------------------------------------------------------- /django_sqlibrist/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | import os 3 | 4 | from django.conf import settings 5 | 6 | SQLIBRIST_DIRECTORY = getattr(settings, 7 | 'SQLIBRIST_DIRECTORY', 8 | os.path.join(settings.BASE_DIR, 'sql')) 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | PyYAML>=3.11 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | import sqlibrist 3 | import os 4 | from setuptools import setup 5 | 6 | with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as readme: 7 | README = readme.read() 8 | 9 | os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) 10 | 11 | setup( 12 | name='sqlibrist', 13 | version=sqlibrist.VERSION, 14 | packages=['sqlibrist', 'django_sqlibrist'], 15 | include_package_data=True, 16 | license='MIT License', 17 | description='Simple tool for managing DB structure, automating patch ' 18 | 'creation for DB structure migration.', 19 | long_description=README, 20 | url='https://github.com/condograde/sqlibrist', 21 | author='Serj Zavadsky', 22 | author_email='fevral13@gmail.com', 23 | install_requires=['PyYAML'], 24 | classifiers=[ 25 | 'Environment :: Web Environment', 26 | 'Environment :: Console', 27 | 'Development Status :: 4 - Beta', 28 | 'Intended Audience :: Developers', 29 | 'Intended Audience :: System Administrators', 30 | 'License :: OSI Approved :: MIT License', 31 | 'Operating System :: OS Independent', 32 | 'Programming Language :: Python', 33 | 'Programming Language :: Python :: 2.7', 34 | 'Programming Language :: Python :: 3.3', 35 | 'Programming Language :: Python :: 3.4', 36 | 'Programming Language :: Python :: 3.5', 37 | 'Topic :: Database', 38 | 'Topic :: Database :: Database Engines/Servers', 39 | 'Topic :: Utilities', 40 | ], 41 | keywords='sqlibrist, db structure, sql, schema migration', 42 | entry_points={ 43 | 'console_scripts': [ 44 | 'sqlibrist=sqlibrist:main' 45 | ] 46 | } 47 | ) 48 | -------------------------------------------------------------------------------- /sqlibrist/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | VERSION = '0.1.10' 3 | 4 | from sqlibrist.helpers import SqlibristException, handle_exception, \ 5 | get_command_parser, LazyConfig 6 | 7 | 8 | def main(): 9 | parser = get_command_parser() 10 | args = parser.parse_args() 11 | config = LazyConfig(args) 12 | try: 13 | args.func(args, config) 14 | except SqlibristException as e: 15 | handle_exception(e) 16 | 17 | 18 | if __name__ == '__main__': 19 | main() 20 | -------------------------------------------------------------------------------- /sqlibrist/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | -------------------------------------------------------------------------------- /sqlibrist/commands/diff.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import print_function 3 | 4 | import difflib 5 | 6 | from sqlibrist.helpers import get_last_schema, get_current_schema, \ 7 | compare_schemas 8 | 9 | 10 | def diff(args, config, connection=None): 11 | verbose = args.verbose 12 | last_schema = get_last_schema() 13 | 14 | current_schema = get_current_schema() 15 | 16 | added, removed, changed = compare_schemas(last_schema, current_schema) 17 | 18 | if any((added, removed, changed)): 19 | if added: 20 | print('New items:') 21 | for item in added: 22 | print(' %s' % item) 23 | 24 | if removed: 25 | print('Removed items:') 26 | for item in removed: 27 | print(' %s' % item) 28 | 29 | if changed: 30 | print('Changed items:') 31 | for item in changed: 32 | print(' %s' % item) 33 | if verbose: 34 | _diff = difflib.unified_diff(last_schema[item]['up'], 35 | current_schema[item]['up']) 36 | print('\n'.join(_diff)) 37 | 38 | else: 39 | print('No changes') 40 | -------------------------------------------------------------------------------- /sqlibrist/commands/info.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import print_function 3 | 4 | 5 | def info(args, config, connection=None): 6 | from sqlibrist import VERSION 7 | print('Version: %s' % VERSION) 8 | print(config) 9 | -------------------------------------------------------------------------------- /sqlibrist/commands/init.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import print_function 3 | 4 | import os 5 | 6 | DEFAULT_CONFIG_FILE = 'sqlibrist.yaml' 7 | DEFAULT_CONFIG = """--- 8 | default: 9 | engine: pg 10 | # engine: mysql 11 | user: 12 | name: 13 | password: 14 | # host: 127.0.0.1 15 | # port: 5432 16 | """ 17 | 18 | 19 | def init(args, config, connection=None): 20 | print('Creating directories...') 21 | dirlist = ( 22 | 'schema', 23 | 'schema/tables', 24 | 'schema/functions', 25 | 'schema/views', 26 | 'schema/triggers', 27 | 'schema/indexes', 28 | 'schema/types', 29 | 'schema/constraints', 30 | 'migrations' 31 | ) 32 | 33 | for dirname in dirlist: 34 | if not os.path.isdir(dirname): 35 | os.mkdir(dirname) 36 | 37 | if not os.path.isfile(DEFAULT_CONFIG_FILE): 38 | with open(DEFAULT_CONFIG_FILE, 'w') as f: 39 | f.write(DEFAULT_CONFIG) 40 | print('Done.') 41 | -------------------------------------------------------------------------------- /sqlibrist/commands/initdb.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import print_function 3 | 4 | from sqlibrist.helpers import get_engine 5 | 6 | 7 | def initdb(args, config, connection=None): 8 | engine = get_engine(config, connection) 9 | 10 | print('Creating db...') 11 | engine.create_migrations_table() 12 | print('Done.') 13 | 14 | 15 | -------------------------------------------------------------------------------- /sqlibrist/commands/makemigration.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import print_function 3 | 4 | from sqlibrist.helpers import get_last_schema, save_migration, \ 5 | get_current_schema, compare_schemas, mark_affected_items 6 | 7 | 8 | def makemigration(args, config, connection=None): 9 | empty = args.empty 10 | dry_run = args.dry_run 11 | migration_name = args.name 12 | 13 | current_schema = get_current_schema() 14 | execution_plan_up = [] 15 | execution_plan_down = [] 16 | 17 | if not empty: 18 | last_schema = get_last_schema() or {} 19 | 20 | added, removed, changed = compare_schemas(last_schema, current_schema) 21 | 22 | added_items = sorted([current_schema[name] for name in added], 23 | key=lambda i: i['degree']) 24 | 25 | if added_items: 26 | print('Creating:') 27 | for item in added_items: 28 | print(' %s' % item['name']) 29 | 30 | execution_plan_up.append(item['up']) 31 | execution_plan_down.append(item['down']) 32 | 33 | for name in changed: 34 | current_schema[name]['status'] = 'changed' 35 | for name in changed: 36 | mark_affected_items(current_schema, name) 37 | 38 | changed_items = sorted([item 39 | for item in current_schema.values() 40 | if item.get('status') == 'changed'], 41 | key=lambda i: i['degree']) 42 | 43 | if changed_items: 44 | print('Updating:') 45 | print(' dropping:') 46 | for item in reversed(changed_items): 47 | if item['down']: 48 | print(' %s' % item['name']) 49 | 50 | if item['name'] in last_schema \ 51 | and last_schema[item['name']]['down']: 52 | execution_plan_up.append(last_schema[item['name']]['down']) 53 | execution_plan_down.append(last_schema[item['name']]['up']) 54 | elif item['name'] not in last_schema and item['down']: 55 | execution_plan_up.append(item['down']) 56 | 57 | execution_plan_up.append( 58 | ['-- ==== Add your instruction here ====']) 59 | 60 | print(' creating:') 61 | for item in changed_items: 62 | if item['down']: 63 | print(' %s' % item['name']) 64 | execution_plan_up.append(item['up']) 65 | execution_plan_down.append(item['down']) 66 | 67 | removed_items = sorted( 68 | [last_schema[name] for name in removed], 69 | key=lambda i: i['degree'], 70 | reverse=True) 71 | if removed_items: 72 | print('Deleting:') 73 | for item in removed_items: 74 | print(' %s' % item['name']) 75 | 76 | execution_plan_up.append(item['down']) 77 | execution_plan_down.append(last_schema[item['name']]['up']) 78 | 79 | default_suffix = 'auto' 80 | else: 81 | default_suffix = 'manual' 82 | 83 | suffix = ('-%s' % (migration_name or default_suffix)) 84 | 85 | if not dry_run: 86 | save_migration(current_schema, 87 | execution_plan_up, 88 | reversed(execution_plan_down), 89 | suffix) 90 | -------------------------------------------------------------------------------- /sqlibrist/commands/migrate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import print_function 3 | 4 | import glob 5 | import os 6 | 7 | from sqlibrist.helpers import get_engine, ApplyMigrationFailed, \ 8 | MigrationIrreversible 9 | 10 | 11 | def unapplied_migrations(migration_list, applied_migrations): 12 | ml = [m.split('/')[-1] for m in migration_list] 13 | for migration in applied_migrations: 14 | try: 15 | ml.remove(migration[0]) 16 | except ValueError: 17 | print('Miration "%s" is not in created ' 18 | 'migration list, probably, this DB ' 19 | 'is from another branch' % migration[0]) 20 | return ml 21 | 22 | 23 | def migrate(args, config, connection=None): 24 | fake = args.fake 25 | revert = args.revert 26 | till_migration_name = args.migration 27 | engine = get_engine(config, connection) 28 | 29 | applied_migrations = engine.get_applied_migrations() 30 | 31 | if applied_migrations and revert: 32 | last_applied_migration = applied_migrations[-1][0] 33 | try: 34 | with open(os.path.join('migrations', 35 | last_applied_migration, 36 | 'down.sql')) as f: 37 | down = f.read() 38 | except IOError: 39 | raise MigrationIrreversible('Migration %s does not ' 40 | 'have down.sql - reverting ' 41 | 'impossible' % last_applied_migration) 42 | 43 | print('Un-Applying migration %s... ' % last_applied_migration, end='') 44 | if fake: 45 | print('(fake run) ', end='') 46 | try: 47 | engine.unapply_migration(last_applied_migration, down, fake) 48 | except ApplyMigrationFailed: 49 | print('Error, rolled back') 50 | else: 51 | print('done') 52 | return 53 | 54 | elif not revert: 55 | migration_list = unapplied_migrations( 56 | sorted(glob.glob('migrations/*')), 57 | applied_migrations) 58 | else: 59 | # no migrations at all 60 | migration_list = sorted(glob.glob('migrations/*')) 61 | 62 | for migration in migration_list: 63 | with open(os.path.join('migrations', migration, 'up.sql')) as f: 64 | up = f.read() 65 | 66 | migration_name = migration.split('/')[-1] 67 | print('Applying migration %s... ' % migration_name, end='') 68 | if fake: 69 | print('(fake run) ', end='') 70 | try: 71 | engine.apply_migration(migration_name, up, fake) 72 | except ApplyMigrationFailed: 73 | print('Error, rolled back') 74 | break 75 | else: 76 | print('done') 77 | if till_migration_name \ 78 | and migration_name == till_migration_name: 79 | break 80 | -------------------------------------------------------------------------------- /sqlibrist/commands/status.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import print_function 3 | 4 | import os 5 | 6 | from sqlibrist.helpers import get_engine 7 | 8 | 9 | def status(args, config, connection=None): 10 | """ 11 | 1. get applied migrations 12 | 2. get all migrations 13 | 3. check unapplied migrations 14 | """ 15 | 16 | engine = get_engine(config, connection) 17 | 18 | applied_migrations = {m[0] for m in engine.get_applied_migrations()} 19 | all_migrations = sorted(os.listdir('migrations/')) 20 | for i, migration in enumerate(all_migrations): 21 | if migration in applied_migrations: 22 | print('Migration %s - applied' % migration) 23 | else: 24 | print('Migration %s - NOT applied' % migration) 25 | -------------------------------------------------------------------------------- /sqlibrist/commands/test_connection.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import print_function 3 | 4 | from sqlibrist.helpers import get_engine 5 | 6 | 7 | def test_connection(args, config, connection=None): 8 | engine = get_engine(config, connection) 9 | 10 | db_connection = engine.get_connection() 11 | 12 | print('Connection OK') 13 | db_connection.close() 14 | -------------------------------------------------------------------------------- /sqlibrist/engines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import absolute_import, print_function 3 | 4 | 5 | class BaseEngine(object): 6 | def __init__(self, config, connection=None): 7 | self.config = config 8 | self.connection = connection 9 | 10 | def get_connection(self): 11 | raise NotImplementedError 12 | 13 | def create_migrations_table(self): 14 | raise NotImplementedError 15 | 16 | def get_applied_migrations(self): 17 | raise NotImplementedError 18 | 19 | def apply_migration(self, name, statements, fake=False): 20 | raise NotImplementedError 21 | 22 | def unapply_migration(self, name, statements, fake=False): 23 | raise NotImplementedError 24 | 25 | 26 | class Postgresql(BaseEngine): 27 | def get_connection(self): 28 | if self.connection is None: 29 | import psycopg2 30 | self.connection = psycopg2.connect( 31 | database=self.config.get('name'), 32 | user=self.config.get('user'), 33 | host=self.config.get('host'), 34 | password=self.config.get('password'), 35 | port=self.config.get('port'), 36 | ) 37 | return self.connection 38 | 39 | def create_migrations_table(self): 40 | connection = self.get_connection() 41 | print('Creating schema and migrations log table...\n') 42 | with connection.cursor() as cursor: 43 | cursor.execute('CREATE SCHEMA IF NOT EXISTS sqlibrist;') 44 | 45 | cursor.execute(''' 46 | CREATE TABLE IF NOT EXISTS sqlibrist.migrations ( 47 | id SERIAL PRIMARY KEY, 48 | migration TEXT, 49 | datetime TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP 50 | ); 51 | ''') 52 | 53 | def get_applied_migrations(self): 54 | connection = self.get_connection() 55 | with connection.cursor() as cursor: 56 | cursor.execute(''' 57 | SELECT migration FROM sqlibrist.migrations 58 | ORDER BY datetime; ''') 59 | return cursor.fetchall() 60 | 61 | def get_last_applied_migration(self): 62 | connection = self.get_connection() 63 | with connection.cursor() as cursor: 64 | cursor.execute(''' 65 | SELECT migration FROM sqlibrist.migrations 66 | ORDER BY datetime DESC 67 | LIMIT 1; ''') 68 | result = cursor.fetchone() 69 | return result and result[0] or None 70 | 71 | def apply_migration(self, name, statements, fake=False): 72 | import psycopg2 73 | connection = self.get_connection() 74 | with connection.cursor() as cursor: 75 | try: 76 | if not fake and statements.strip(): 77 | cursor.execute(statements) 78 | except ( 79 | psycopg2.OperationalError, 80 | psycopg2.ProgrammingError) as e: 81 | connection.rollback() 82 | print(e.message) 83 | from sqlibrist.helpers import ApplyMigrationFailed 84 | 85 | raise ApplyMigrationFailed 86 | else: 87 | cursor.execute('INSERT INTO sqlibrist.migrations ' 88 | '(migration) VALUES (%s);', 89 | [name.split('/')[-1]]) 90 | connection.commit() 91 | 92 | def unapply_migration(self, name, statements, fake=False): 93 | import psycopg2 94 | connection = self.get_connection() 95 | with connection.cursor() as cursor: 96 | try: 97 | if not fake: 98 | cursor.execute(statements) 99 | except ( 100 | psycopg2.OperationalError, 101 | psycopg2.ProgrammingError) as e: 102 | connection.rollback() 103 | print(e.message) 104 | from sqlibrist.helpers import ApplyMigrationFailed 105 | 106 | raise ApplyMigrationFailed 107 | else: 108 | cursor.execute('DELETE FROM sqlibrist.migrations ' 109 | 'WHERE migration = (%s); ', [name]) 110 | connection.commit() 111 | 112 | 113 | class MySQL(BaseEngine): 114 | def get_connection(self): 115 | if self.connection is None: 116 | import MySQLdb 117 | self.connection = MySQLdb.connect( 118 | db=self.config.get('name'), 119 | user=self.config.get('user'), 120 | host=self.config.get('host', '127.0.0.1'), 121 | passwd=self.config.get('password'), 122 | port=self.config.get('port'), 123 | ) 124 | return self.connection 125 | 126 | def create_migrations_table(self): 127 | connection = self.get_connection() 128 | cursor = connection.cursor() 129 | print('Creating migrations log table...\n') 130 | cursor.execute(''' 131 | CREATE TABLE IF NOT EXISTS sqlibrist_migrations ( 132 | id SERIAL PRIMARY KEY, 133 | migration TEXT, 134 | `datetime` TIMESTAMP 135 | ); 136 | ''') 137 | 138 | def get_applied_migrations(self): 139 | connection = self.get_connection() 140 | cursor = connection.cursor() 141 | cursor.execute(''' 142 | SELECT migration FROM sqlibrist_migrations 143 | ORDER BY `datetime`; ''') 144 | return cursor.fetchall() 145 | 146 | def get_last_applied_migration(self): 147 | connection = self.get_connection() 148 | cursor = connection.cursor() 149 | 150 | cursor.execute(''' 151 | SELECT migration FROM sqlibrist_migrations 152 | ORDER BY `datetime` DESC 153 | LIMIT 1; ''') 154 | result = cursor.fetchone() 155 | return result and result[0] or None 156 | 157 | def apply_migration(self, name, statements, fake=False): 158 | import MySQLdb 159 | connection = self.get_connection() 160 | cursor = connection.cursor() 161 | 162 | try: 163 | if not fake and statements.strip(): 164 | cursor.execute(statements) 165 | except (MySQLdb.OperationalError, MySQLdb.ProgrammingError) as e: 166 | print('\n'.join(map(str, e.args))) 167 | from sqlibrist.helpers import ApplyMigrationFailed 168 | 169 | raise ApplyMigrationFailed 170 | else: 171 | cursor.execute('INSERT INTO sqlibrist_migrations ' 172 | '(migration) VALUES (%s);', 173 | [name.split('/')[-1]]) 174 | 175 | def unapply_migration(self, name, statements, fake=False): 176 | import MySQLdb 177 | connection = self.get_connection() 178 | cursor = connection.cursor() 179 | 180 | try: 181 | if not fake: 182 | cursor.execute(statements) 183 | except (MySQLdb.OperationalError, MySQLdb.ProgrammingError) as e: 184 | print('\n'.join(map(str, e.args))) 185 | from sqlibrist.helpers import ApplyMigrationFailed 186 | 187 | raise ApplyMigrationFailed 188 | else: 189 | cursor.execute('DELETE FROM sqlibrist_migrations ' 190 | 'WHERE migration = (%s); ', [name]) 191 | -------------------------------------------------------------------------------- /sqlibrist/helpers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | from __future__ import print_function 3 | 4 | import argparse 5 | import glob 6 | import hashlib 7 | import os 8 | import re 9 | from json import loads, dumps 10 | 11 | from sqlibrist.engines import Postgresql, MySQL 12 | 13 | ENGINE_POSTGRESQL = 'pg' 14 | ENGINE_MYSQL = 'mysql' 15 | 16 | ENGINES = { 17 | ENGINE_POSTGRESQL: Postgresql, 18 | ENGINE_MYSQL: MySQL 19 | } 20 | 21 | 22 | class SqlibristException(Exception): 23 | pass 24 | 25 | 26 | class CircularDependencyException(SqlibristException): 27 | pass 28 | 29 | 30 | class UnknownDependencyException(SqlibristException): 31 | pass 32 | 33 | 34 | class BadConfig(SqlibristException): 35 | pass 36 | 37 | 38 | class ApplyMigrationFailed(SqlibristException): 39 | pass 40 | 41 | 42 | class MigrationIrreversible(SqlibristException): 43 | pass 44 | 45 | 46 | class LazyConfig(object): 47 | def __init__(self, args): 48 | self.args = args 49 | 50 | def load_config(self): 51 | import yaml 52 | from yaml.scanner import ScannerError 53 | 54 | try: 55 | with open(self.args.config_file) as config_file: 56 | configs = yaml.load(config_file.read()) 57 | except IOError: 58 | raise BadConfig('No config file %s found!' % self.args.config_file) 59 | except ScannerError: 60 | raise BadConfig('Bad config file syntax') 61 | else: 62 | try: 63 | self._dict = configs[self.args.config] 64 | except KeyError: 65 | raise BadConfig('No config named %s found!' % self.args.config) 66 | 67 | def get(self, key, default=None): 68 | try: 69 | return self[key] 70 | except KeyError: 71 | return default 72 | 73 | def __getitem__(self, key): 74 | try: 75 | if key == 'port': 76 | return int(self._dict[key]) 77 | else: 78 | return str(self._dict[key]) 79 | except AttributeError: 80 | self.load_config() 81 | return self[key] 82 | 83 | def __repr__(self): 84 | try: 85 | return self._dict 86 | except AttributeError: 87 | self.load_config() 88 | return repr(self) 89 | 90 | 91 | def get_engine(config, connection=None): 92 | try: 93 | return ENGINES[config['engine']](config, connection) 94 | except KeyError: 95 | raise BadConfig('DB engine not selected in config or wrong engine ' 96 | 'name (must be one of %s)' % ','.join(ENGINES.keys())) 97 | 98 | 99 | def get_last_schema(): 100 | schemas = sorted(glob.glob('migrations/*')) 101 | if schemas: 102 | with open(os.path.join(schemas[-1], 'schema.json'), 'r') as f: 103 | schema = loads(f.read()) 104 | else: 105 | schema = {} 106 | return schema 107 | 108 | 109 | def extract_reqs(lines): 110 | for line in lines: 111 | if line.strip().startswith('--REQ'): 112 | _, requirement = line.split() 113 | yield requirement 114 | 115 | 116 | def extract_up(lines): 117 | on = False 118 | for line in lines: 119 | if line.strip().startswith('--UP'): 120 | on = True 121 | elif line.strip().startswith('--DOWN'): 122 | raise StopIteration 123 | elif on: 124 | yield line.rstrip() 125 | 126 | 127 | def extract_down(lines): 128 | on = False 129 | for line in lines: 130 | if line.strip().startswith('--DOWN'): 131 | on = True 132 | elif on: 133 | yield line.rstrip() 134 | 135 | 136 | def init_item(directory, filename): 137 | with open(os.path.join(directory, filename), 'r') as f: 138 | lines = f.readlines() 139 | 140 | filename = '/'.join(directory.split('/')[1:] + [filename[:-4]]) 141 | requires = list(extract_reqs(lines)) 142 | up = list(extract_up(lines)) 143 | down = list(extract_down(lines)) 144 | _hash = hashlib.md5(re.sub(r'\s{2,}', '', ''.join(up)).encode()).hexdigest() 145 | 146 | return (filename, 147 | {'hash': _hash, 148 | 'name': filename, 149 | 'requires': requires, 150 | 'required': [], 151 | 'up': up, 152 | 'down': down}) 153 | 154 | 155 | def schema_collector(): 156 | files_generator = os.walk('schema') 157 | for directory, subdirectories, files in files_generator: 158 | for filename in files: 159 | if filename.endswith('.sql'): 160 | yield init_item(directory, filename) 161 | 162 | 163 | def check_for_circular_dependencies(schema, name, metadata, stack=()): 164 | if name in stack: 165 | raise CircularDependencyException(stack + (name,)) 166 | for requires in metadata['requires']: 167 | check_for_circular_dependencies(schema, 168 | requires, 169 | schema[requires], 170 | stack + (name,)) 171 | 172 | 173 | def calculate_cumulative_degree(schema, name, metadata, degree=0): 174 | return len(metadata['requires']) \ 175 | + sum([calculate_cumulative_degree(schema, 176 | requirement, 177 | schema[requirement]) 178 | for requirement in metadata['requires']]) 179 | 180 | 181 | def get_current_schema(): 182 | schema = dict(schema_collector()) 183 | 184 | item_names = schema.keys() 185 | 186 | for name, metadata in schema.items(): 187 | for requirement in metadata['requires']: 188 | if requirement not in item_names: 189 | raise UnknownDependencyException((requirement, name)) 190 | 191 | schema[requirement]['required'].append(name) 192 | for name, metadata in schema.items(): 193 | check_for_circular_dependencies(schema, name, metadata) 194 | metadata['degree'] = calculate_cumulative_degree(schema, name, metadata) 195 | return schema 196 | 197 | 198 | def compare_schemas(last_schema, current_schema): 199 | last_set = set(last_schema.keys()) 200 | current_set = set(current_schema.keys()) 201 | 202 | added = current_set - last_set 203 | removed = last_set - current_set 204 | changed = [item 205 | for item in last_set.intersection(current_set) 206 | if last_schema[item]['hash'] != current_schema[item]['hash']] 207 | 208 | return added, removed, changed 209 | 210 | 211 | def save_migration(schema, plan_up, plan_down, suffix=''): 212 | migration_name = '%04.f%s' % (len(glob.glob('migrations/*')) + 1, suffix) 213 | dirname = os.path.join('migrations', migration_name) 214 | print('Creating new migration %s' % migration_name) 215 | os.mkdir(dirname) 216 | schema_filename = os.path.join(dirname, 'schema.json') 217 | with open(schema_filename, 'w') as f: 218 | f.write(dumps(schema, indent=2)) 219 | 220 | plans = ( 221 | ('up.sql', plan_up), 222 | ('down.sql', plan_down), 223 | ) 224 | for plan_name, instructions in plans: 225 | with open(os.path.join(dirname, plan_name), 'w') as f: 226 | for item in instructions: 227 | f.write('-- begin --\n') 228 | f.write('\n'.join(item)) 229 | f.write('\n') 230 | f.write('-- end --\n') 231 | f.write('\n\n') 232 | 233 | 234 | def mark_affected_items(schema, name): 235 | schema[name]['status'] = 'changed' 236 | for required in schema[name]['required']: 237 | mark_affected_items(schema, required) 238 | 239 | 240 | def handle_exception(e): 241 | if isinstance(e, CircularDependencyException): 242 | print('Circular dependency:') 243 | print(' %s' % ' >\n '.join(e.message)) 244 | elif isinstance(e, UnknownDependencyException): 245 | print('Unknown dependency %s at %s' % e.message) 246 | elif isinstance(e, (BadConfig, MigrationIrreversible)): 247 | print(e.message) 248 | 249 | 250 | def get_command_parser(parser=None): 251 | from sqlibrist.commands.diff import diff 252 | from sqlibrist.commands.init import init 253 | from sqlibrist.commands.initdb import initdb 254 | from sqlibrist.commands.makemigration import makemigration 255 | from sqlibrist.commands.status import status 256 | from sqlibrist.commands.test_connection import test_connection 257 | from sqlibrist.commands.migrate import migrate 258 | from sqlibrist.commands.info import info 259 | 260 | _parser = parser or argparse.ArgumentParser() 261 | _parser.add_argument('--config-file', '-f', 262 | help='Config file, default is sqlibrist.yaml', 263 | type=str, 264 | default=os.environ.get('SQLIBRIST_CONFIG_FILE', 265 | 'sqlibrist.yaml')) 266 | _parser.add_argument('--config', '-c', 267 | help='Config name in config file, ' 268 | 'default is "default"', 269 | type=str, 270 | default=os.environ.get('SQLIBRIST_CONFIG', 'default')) 271 | 272 | subparsers = _parser.add_subparsers(parser_class=argparse.ArgumentParser) 273 | 274 | # print info 275 | print_info_parser = subparsers.add_parser('info', 276 | help='Print sqlibrist info') 277 | print_info_parser.add_argument('--verbose', '-v', 278 | action='store_true', default=False) 279 | print_info_parser.set_defaults(func=info) 280 | 281 | # test_connection 282 | test_connection_parser = subparsers.add_parser('test_connection', 283 | help='Test DB connection') 284 | test_connection_parser.add_argument('--verbose', '-v', 285 | action='store_true', default=False) 286 | test_connection_parser.set_defaults(func=test_connection) 287 | 288 | # init 289 | init_parser = subparsers.add_parser('init', 290 | help='Init directory structure') 291 | init_parser.add_argument('--verbose', '-v', 292 | action='store_true', default=False) 293 | init_parser.set_defaults(func=init) 294 | 295 | # initdb 296 | initdb_parser = subparsers.add_parser('initdb', 297 | help='Create DB table for ' 298 | 'migrations tracking') 299 | initdb_parser.add_argument('--verbose', '-v', 300 | action='store_true', default=False) 301 | initdb_parser.set_defaults(func=initdb) 302 | 303 | # makemigrations 304 | makemigration_parser = subparsers.add_parser('makemigration', 305 | help='Create new migration') 306 | makemigration_parser.set_defaults(func=makemigration) 307 | makemigration_parser.add_argument('--verbose', '-v', 308 | action='store_true', default=False) 309 | 310 | makemigration_parser.add_argument('--empty', 311 | help='Create migration with empty up.sql ' 312 | 'for manual instructions', 313 | action='store_true', 314 | default=False) 315 | makemigration_parser.add_argument('--name', '-n', 316 | help='Optional migration name', 317 | type=str, 318 | default='') 319 | makemigration_parser.add_argument('--dry-run', 320 | help='Do not save migration', 321 | action='store_true', 322 | default=False) 323 | 324 | # migrate 325 | migrate_parser = subparsers.add_parser('migrate', 326 | help='Apply pending migrations') 327 | migrate_parser.set_defaults(func=migrate) 328 | migrate_parser.add_argument('--verbose', '-v', 329 | action='store_true', default=False) 330 | migrate_parser.add_argument('--fake', 331 | help='Mark pending migrations as applied', 332 | action='store_true', 333 | default=False) 334 | migrate_parser.add_argument('--dry-run', 335 | help='Do not make actual changes to the DB', 336 | action='store_true', 337 | default=False) 338 | migrate_parser.add_argument('--migration', '-m', 339 | help='Apply up to given migration number', 340 | type=str) 341 | migrate_parser.add_argument('--revert', '-r', 342 | help='Unapply last migration', 343 | action='store_true') 344 | 345 | # diff 346 | diff_parser = subparsers.add_parser('diff', help='Show changes to schema') 347 | diff_parser.set_defaults(func=diff) 348 | diff_parser.add_argument('--verbose', '-v', 349 | action='store_true', default=False) 350 | 351 | # status 352 | status_parser = subparsers.add_parser('status', 353 | help='Show unapplied migrations') 354 | status_parser.add_argument('--verbose', '-v', 355 | action='store_true', default=False) 356 | status_parser.set_defaults(func=status) 357 | return _parser 358 | --------------------------------------------------------------------------------