├── requirements-dev.txt ├── test.sh ├── tox.ini ├── .circleci └── config.yml ├── setup.cfg ├── bin └── cassandra-migrate ├── LICENSE ├── setup.py ├── cassandra_migrate ├── test │ └── test_cql.py ├── __init__.py ├── cql.py ├── migration.py ├── config.py ├── cli.py └── migrator.py ├── .gitignore └── README.rst /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | coverage 3 | flake8 4 | tox 5 | bumpversion 6 | twine 7 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | flake8 cassandra_migrate 6 | coverage erase 7 | coverage run --source cassandra_migrate -m py.test 8 | coverage report --include='cassandra_migrate/**' --omit='cassandra_migrate/test/**' 9 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py27 4 | py35 5 | py36 6 | toxworkdir = {env:TOXWORKDIR:} 7 | 8 | [testenv] 9 | whitelist_externals = true 10 | deps = 11 | -rrequirements-dev.txt 12 | commands = 13 | ./test.sh 14 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | jobs: 4 | test: 5 | docker: 6 | - image: themattrix/tox 7 | steps: 8 | - checkout 9 | - run: pip install --upgrade pip 10 | - run: pip install tox 11 | - run: 12 | command: tox 13 | name: Test 14 | 15 | workflows: 16 | main: 17 | jobs: 18 | - test 19 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.3.5 3 | commit = True 4 | tag = True 5 | message = Release {new_version} 6 | tag_name = {new_version} 7 | 8 | [bumpversion:file:setup.py] 9 | 10 | [bumpversion:file:setup.cfg] 11 | 12 | [bdist_wheel] 13 | universal = 1 14 | 15 | [flake8] 16 | exclude = .git,__pycache__,venv,build,dist 17 | max-line-length = 80 18 | 19 | [tool:pytest] 20 | testpaths = cassandra_migrate/test 21 | python_classes = 22 | 23 | -------------------------------------------------------------------------------- /bin/cassandra-migrate: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | # encoding: utf-8 3 | 4 | from __future__ import unicode_literals 5 | 6 | from yaml import Loader, SafeLoader 7 | from cassandra_migrate.cli import main 8 | 9 | # Force YAML to load strings as unicode 10 | def construct_yaml_str(self, node): 11 | # Override the default string handling function 12 | # to always return unicode objects 13 | return self.construct_scalar(node) 14 | 15 | Loader.add_constructor('tag:yaml.org,2002:str', construct_yaml_str) 16 | SafeLoader.add_constructor('tag:yaml.org,2002:str', construct_yaml_str) 17 | 18 | main() 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2017 Cobli 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | VERSION = '0.3.5' 4 | 5 | install_requires = [ 6 | "arrow==0.17.0", 7 | "cassandra-driver==3.24.0", 8 | "cassandra-migrate==0.3.5", 9 | "click==7.1.2", 10 | "future==0.18.2", 11 | "geomet==0.2.1.post1", 12 | "python-dateutil==2.8.1", 13 | "PyYAML==5.*", 14 | "six==1.15.0", 15 | "tabulate==0.8.9", 16 | "typing-extensions==3.7.4.3"] 17 | 18 | setup(name='cassandra-migrate', 19 | packages=['cassandra_migrate'], 20 | version=VERSION, 21 | description='Simple Cassandra database migration program.', 22 | long_description=open('README.rst').read(), 23 | url='https://github.com/Cobliteam/cassandra-migrate', 24 | download_url='https://github.com/Cobliteam/cassandra-migrate/archive/{}.tar.gz'.format(VERSION), 25 | author='Daniel Miranda', 26 | author_email='daniel@cobli.co', 27 | license='MIT', 28 | install_requires=install_requires, 29 | scripts=['bin/cassandra-migrate'], 30 | keywords='cassandra schema migration') 31 | -------------------------------------------------------------------------------- /cassandra_migrate/test/test_cql.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import pytest 4 | 5 | from cassandra_migrate.cql import CqlSplitter 6 | 7 | 8 | @pytest.mark.parametrize('cql,statements', [ 9 | # Two statements, with whitespace 10 | (''' 11 | CREATE TABLE hello; 12 | CREATE TABLE world; 13 | ''', 14 | ['CREATE TABLE hello', 'CREATE TABLE world']), 15 | # Two statements, no whitespace 16 | ('''CREATE TABLE hello;CREATE TABLE world;''', 17 | ['CREATE TABLE hello', 'CREATE TABLE world']), 18 | # Two statements, with line and block comments 19 | (''' 20 | // comment 21 | -- comment 22 | CREATE TABLE hello; 23 | /* comment; comment 24 | */ 25 | CREATE TABLE world; 26 | ''', 27 | ['CREATE TABLE hello', 'CREATE TABLE world']), 28 | # Statements with semicolons inside strings 29 | (''' 30 | CREATE TABLE 'hello;'; 31 | CREATE TABLE "world;" 32 | ''', 33 | ["CREATE TABLE 'hello;'", 'CREATE TABLE "world;"']), 34 | # Double-dollar-sign quoted strings, as reported in PR #24 35 | ('INSERT INTO test (test) VALUES ' 36 | '($$Pesky semicolon here ;Hello$$);', 37 | ["INSERT INTO test (test) VALUES ($$Pesky semicolon here ;Hello$$)"]) 38 | ]) 39 | def test_cql_split(cql, statements): 40 | result = CqlSplitter.split(cql.strip()) 41 | assert result == statements 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *.cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # Jupyter Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # SageMath parsed files 79 | *.sage.py 80 | 81 | # Environments 82 | .env 83 | .venv 84 | env/ 85 | venv/ 86 | ENV/ 87 | 88 | # Spyder project settings 89 | .spyderproject 90 | .spyproject 91 | 92 | # Rope project settings 93 | .ropeproject 94 | 95 | # mkdocs documentation 96 | /site 97 | 98 | # mypy 99 | .mypy_cache/ 100 | 101 | # Visual Studio Code 102 | .vscode/ 103 | -------------------------------------------------------------------------------- /cassandra_migrate/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # flake8: noqa: E402,F401 3 | 4 | from __future__ import print_function, unicode_literals 5 | 6 | 7 | class MigrationError(RuntimeError): 8 | """Base class for migration errors""" 9 | pass 10 | 11 | 12 | class FailedMigration(MigrationError): 13 | """Database state contains failed migrations""" 14 | 15 | def __init__(self, version, name): 16 | self.version = version 17 | self.migration_name = name 18 | 19 | super(FailedMigration, self).__init__( 20 | 'Migration failed, cannot continue ' 21 | '(version {}): {}'.format(version, name)) 22 | 23 | 24 | class ConcurrentMigration(MigrationError): 25 | """Database state contains failed migrations""" 26 | 27 | def __init__(self, version, name): 28 | self.version = version 29 | self.migration_name = name 30 | 31 | super(ConcurrentMigration, self).__init__( 32 | 'Migration already in progress ' 33 | '(version {}): {}'.format(version, name)) 34 | 35 | 36 | class InconsistentState(MigrationError): 37 | """Database state differs from specified migrations""" 38 | 39 | def __init__(self, migration, version): 40 | self.migration = migration 41 | self.version = version 42 | 43 | super(InconsistentState, self).__init__( 44 | 'Found inconsistency between specified migration and stored ' 45 | 'version: {} != {}'.format(migration, version)) 46 | 47 | 48 | class UnknownMigration(MigrationError): 49 | """Database contains migrations that have not been specified""" 50 | def __init__(self, version, name): 51 | self.version = version 52 | self.migration_name = name 53 | 54 | super(UnknownMigration, self).__init__( 55 | 'Found version in database without corresponding ' 56 | 'migration (version {}): {} '.format(version, name)) 57 | 58 | 59 | 60 | from .migration import Migration 61 | from .config import MigrationConfig 62 | from .migrator import Migrator 63 | -------------------------------------------------------------------------------- /cassandra_migrate/cql.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | from __future__ import print_function, unicode_literals 4 | 5 | import re 6 | from collections import namedtuple 7 | 8 | 9 | class CqlSplitter(object): 10 | """ 11 | Makeshift CQL parser that can only split up multiple statements. 12 | 13 | C* does not accept multiple DDL queries as a single string, as it can with 14 | DML queries using batches. Hence, we must split up CQL files to run each 15 | statement individually. Do that by using a simple Regex scanner, that just 16 | recognizes strings, comments and delimiters, which is enough to split up 17 | statements without tripping when semicolons are commented or escaped. 18 | """ 19 | 20 | Token = namedtuple('Token', 'tpe token') 21 | 22 | LINE_COMMENT = 1 23 | BLOCK_COMMENT = 2 24 | STRING = 3 25 | SEMICOLON = 4 26 | OTHER = 5 27 | WHITESPACE = 6 28 | 29 | @classmethod 30 | def scanner(cls): 31 | if not getattr(cls, '_scanner', None): 32 | def h(tpe): 33 | return lambda sc, tk: cls.Token(tpe, tk) 34 | 35 | cls._scanner = re.Scanner([ 36 | (r"(--|//).*?$", h(cls.LINE_COMMENT)), 37 | (r"\/\*.+?\*\/", h(cls.BLOCK_COMMENT)), 38 | (r'"(?:[^"\\]|\\.)*"', h(cls.STRING)), 39 | (r"'(?:[^'\\]|\\.)*'", h(cls.STRING)), 40 | (r"\$\$(?:[^\$\\]|\\.)*\$\$", h(cls.STRING)), 41 | (r";", h(cls.SEMICOLON)), 42 | (r"\s+", h(cls.WHITESPACE)), 43 | (r".", h(cls.OTHER)) 44 | ], re.MULTILINE | re.DOTALL) 45 | return cls._scanner 46 | 47 | @classmethod 48 | def split(cls, query): 49 | """Split up content, and return individual statements uncommented""" 50 | tokens, match = cls.scanner().scan(query) 51 | cur_statement = '' 52 | statements = [] 53 | 54 | for i, tk in enumerate(tokens): 55 | if tk.tpe == cls.LINE_COMMENT: 56 | pass 57 | elif tk.tpe == cls.SEMICOLON: 58 | stm = cur_statement.strip() 59 | if stm: 60 | statements.append(stm) 61 | cur_statement = '' 62 | elif tk.tpe in (cls.WHITESPACE, cls.BLOCK_COMMENT): 63 | cur_statement += ' ' 64 | elif tk.tpe in (cls.STRING, cls.OTHER): 65 | cur_statement += tk.token 66 | 67 | stm = cur_statement.strip() 68 | if stm: 69 | statements.append(stm) 70 | 71 | return statements 72 | -------------------------------------------------------------------------------- /cassandra_migrate/migration.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | from __future__ import (absolute_import, division, 4 | print_function, unicode_literals) 5 | from builtins import open, bytes 6 | 7 | import re 8 | import os 9 | import glob 10 | import hashlib 11 | import io 12 | from collections import namedtuple 13 | 14 | import arrow 15 | 16 | 17 | class Migration(namedtuple('Migration', 18 | 'path name is_python content checksum')): 19 | """ 20 | Data class representing the specification of a migration 21 | 22 | Migrations can take the form of CQL files or Python scripts, and usually 23 | have names starting with a version string that can be ordered. 24 | A checksum is kept to allow detecting changes to previously applied 25 | migrations""" 26 | 27 | __slots__ = () 28 | 29 | class State(object): 30 | """Possible states of a migration, as saved in C*""" 31 | 32 | SUCCEEDED = 'SUCCEEDED' 33 | FAILED = 'FAILED' 34 | SKIPPED = 'SKIPPED' 35 | IN_PROGRESS = 'IN_PROGRESS' 36 | 37 | @staticmethod 38 | def _natural_sort_key(s): 39 | """Generate a sort key for natural sorting""" 40 | k = tuple(int(text) if text.isdigit() else text 41 | for text in re.split(r'([0-9]+)', s)) 42 | return k 43 | 44 | @classmethod 45 | def load(cls, path): 46 | """Load a migration from a given file""" 47 | with open(path, 'r', encoding='utf-8') as fp: 48 | content = fp.read() 49 | 50 | checksum = bytes(hashlib.sha256(content.encode('utf-8')).digest()) 51 | # Should use enum but python3 requires importing an extra library 52 | # Reconsidering the use of enums. This is a binary decision. 53 | # Boolean will work just fine. 54 | is_python = bool(re.findall(r"\.(py)$", os.path.abspath(path))) 55 | 56 | return cls(os.path.abspath(path), os.path.basename(path), 57 | is_python, content, checksum) 58 | 59 | @classmethod 60 | def sort_paths(cls, paths): 61 | """Sort paths naturally by basename, to order by migration version""" 62 | return sorted(paths, 63 | key=lambda p: cls._natural_sort_key(os.path.basename(p))) 64 | 65 | @classmethod 66 | def glob_all(cls, base_path, *patterns): 67 | """Load all paths matching a glob as migrations in sorted order""" 68 | 69 | paths = [] 70 | for pattern in patterns: 71 | paths.extend(glob.iglob(os.path.join(base_path, pattern))) 72 | 73 | return list(map(cls.load, cls.sort_paths(paths))) 74 | 75 | @classmethod 76 | def generate(cls, config, description, output): 77 | fname_fmt = config.new_migration_name 78 | text_cql_fmt = config.new_cql_migration_text 79 | text_py_fmt = config.new_python_migration_text 80 | 81 | clean_desc = re.sub(r'[\W\s]+', '_', description) 82 | next_version = len(config.migrations) + 1 83 | date = arrow.utcnow() 84 | 85 | format_args = { 86 | 'desc': clean_desc, 87 | 'full_desc': description, 88 | 'next_version': next_version, 89 | 'date': date, 90 | 'keyspace': config.keyspace 91 | } 92 | 93 | if output == "python": 94 | file_extension = ".py" 95 | file_content = text_py_fmt.format(**format_args) 96 | else: 97 | file_extension = ".cql" 98 | file_content = text_cql_fmt.format(**format_args) 99 | 100 | fname = fname_fmt.format(**format_args) + file_extension 101 | new_path = os.path.join(config.migrations_path, fname) 102 | 103 | cls._create_file(new_path, file_content) 104 | 105 | return new_path 106 | 107 | @classmethod 108 | def _create_file(cls, path, content): 109 | """Creates physical file""" 110 | with io.open(path, 'w', encoding='utf-8') as f: 111 | f.write(content + '\n') 112 | 113 | def __str__(self): 114 | return 'Migration("{}")'.format(self.name) 115 | -------------------------------------------------------------------------------- /cassandra_migrate/config.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | from __future__ import (absolute_import, division, 4 | print_function, unicode_literals) 5 | from builtins import str, open 6 | 7 | import os 8 | import yaml 9 | 10 | from .migration import Migration 11 | 12 | 13 | DEFAULT_NEW_MIGRATION_TEXT = """ 14 | /* Cassandra migration for keyspace {keyspace}. 15 | Version {next_version} - {date} 16 | 17 | {full_desc} */ 18 | """.lstrip() 19 | 20 | DEFAULT_NEW_CQL_MIGRATION_TEXT = """ 21 | /* Cassandra migration for keyspace {keyspace}. 22 | Version {next_version} - {date} 23 | 24 | {full_desc} */ 25 | """.lstrip() 26 | 27 | DEFAULT_NEW_PYTHON_MIGRATION_TEXT = """ 28 | # Cassandra migration for keyspace {keyspace}. 29 | # Version {next_version} - {date} 30 | # {full_desc} 31 | 32 | def execute(session): 33 | "Main method for your migration. Do not rename this method." 34 | 35 | print("Cassandra session: ", session) 36 | 37 | """.lstrip() 38 | 39 | 40 | def _assert_type(data, key, tpe, default=None): 41 | """Extract and verify if a key in a dictionary has a given type""" 42 | value = data.get(key, default) 43 | if not isinstance(value, tpe): 44 | raise ValueError("Config error: {}: expected {}, found {}".format( 45 | key, tpe, type(value))) 46 | return value 47 | 48 | 49 | class MigrationConfig(object): 50 | """ 51 | Data class containing all configuration for migration operations 52 | 53 | Configuration includes: 54 | - Keyspace to be managed 55 | - Possible keyspace profiles, to configure replication in different 56 | environments 57 | - Path to load migration files from 58 | - Table to store migrations state in 59 | - The loaded migrations themselves (instances of Migration) 60 | """ 61 | 62 | DEFAULT_PROFILES = { 63 | 'dev': { 64 | 'replication': {'class': 'SimpleStrategy', 'replication_factor': 1}, 65 | 'durable_writes': True 66 | } 67 | } 68 | 69 | def __init__(self, data, base_path): 70 | """ 71 | Initialize a migration configuration from a data dict and base path. 72 | 73 | The data will usually be loaded from a YAML file, and must contain 74 | at least `keyspace`, `migrations_path` and `migrations_table` 75 | """ 76 | 77 | self.keyspace = _assert_type(data, 'keyspace', str) 78 | 79 | self.profiles = self.DEFAULT_PROFILES.copy() 80 | profiles = _assert_type(data, 'profiles', dict, default={}) 81 | for name, profile in profiles.items(): 82 | self.profiles[name] = { 83 | 'replication': _assert_type(profile, 'replication', dict), 84 | 'durable_writes': _assert_type(profile, 'durable_writes', 85 | bool, default=True) 86 | } 87 | 88 | migrations_path = _assert_type(data, 'migrations_path', str) 89 | self.migrations_path = os.path.join(base_path, migrations_path) 90 | 91 | self.migrations = Migration.glob_all( 92 | self.migrations_path, '*.cql', '*.py') 93 | 94 | self.migrations_table = _assert_type(data, 'migrations_table', str, 95 | default='database_migrations') 96 | 97 | self.new_migration_name = _assert_type( 98 | data, 'new_migration_name', str, 99 | default='v{next_version}_{desc}') 100 | 101 | self.new_migration_text = _assert_type( 102 | data, 'new_migration_text', str, 103 | default=DEFAULT_NEW_MIGRATION_TEXT) 104 | 105 | self.new_cql_migration_text = _assert_type( 106 | data, 'new_cql_migration_text', str, 107 | default=DEFAULT_NEW_CQL_MIGRATION_TEXT) 108 | 109 | self.new_python_migration_text = _assert_type( 110 | data, 'new_python_migration_text', str, 111 | default=DEFAULT_NEW_PYTHON_MIGRATION_TEXT) 112 | 113 | @classmethod 114 | def load(cls, path): 115 | """Load a migration config from a file, using it's dir. as base path""" 116 | with open(path, 'r', encoding='utf-8') as f: 117 | config = yaml.load(f, Loader=yaml.SafeLoader) 118 | 119 | return cls(config, os.path.dirname(path)) 120 | -------------------------------------------------------------------------------- /cassandra_migrate/cli.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | from __future__ import (absolute_import, division, 4 | print_function, unicode_literals) 5 | 6 | import sys 7 | import os 8 | import logging 9 | import argparse 10 | import subprocess 11 | 12 | from cassandra_migrate import (Migrator, Migration, MigrationConfig, 13 | MigrationError) 14 | 15 | 16 | def open_file(filename): 17 | if sys.platform == 'win32': 18 | os.startfile(filename) 19 | else: 20 | if 'XDG_CURRENT_DESKTOP' in os.environ: 21 | opener = ['xdg-open'] 22 | elif 'EDITOR' in os.environ: 23 | opener = [os.environ['EDITOR']] 24 | else: 25 | opener = ['vi'] 26 | 27 | opener.append(filename) 28 | subprocess.call(opener) 29 | 30 | 31 | def main(): 32 | logging.basicConfig(level=logging.INFO) 33 | logging.getLogger("cassandra.policies").setLevel(logging.ERROR) 34 | 35 | parser = argparse.ArgumentParser( 36 | description='Simple Cassandra migration tool') 37 | parser.add_argument('-H', '--hosts', default='127.0.0.1', 38 | help='Comma-separated list of contact points') 39 | parser.add_argument('-p', '--port', type=int, default=9042, 40 | help='Connection port') 41 | parser.add_argument('-u', '--user', 42 | help='Connection username') 43 | parser.add_argument('-P', '--password', 44 | help='Connection password') 45 | parser.add_argument('-c', '--config-file', default='cassandra-migrate.yml', 46 | help='Path to configuration file') 47 | parser.add_argument('-m', '--profile', default='dev', 48 | help='Name of keyspace profile to use') 49 | parser.add_argument('-s', '--ssl-cert', default=None, 50 | help=""" 51 | File path of .pem or .crt containing certificate of the 52 | cassandra host you are connecting to (or the 53 | certificate of the CA that signed the host certificate). 54 | If this option is provided, cassandra-migrate will use 55 | ssl to connect to the cluster. If this option is not 56 | provided, the -k and -t options will be ignored. """) 57 | parser.add_argument('-k', '--ssl-client-private-key', default=None, 58 | help=""" 59 | File path of the .key file containing the private key 60 | of the host on which the cassandra-migrate command is 61 | run. This option must be used in conjuction with the 62 | -t option. This option is ignored unless the -s 63 | option is provided.""") 64 | parser.add_argument('-t', '--ssl-client-cert', default=None, 65 | help=""" 66 | File path of the .crt file containing the public 67 | certificate of the host on which the cassandra-migrate 68 | command is run. This certificate (or the CA that signed 69 | it) must be trusted by the cassandra host that 70 | migrations are run against. This option must be used in 71 | conjuction with the -k option. This option is ignored 72 | unless the -s option is provided.""") 73 | parser.add_argument('-y', '--assume-yes', action='store_true', 74 | help='Automatically answer "yes" for all questions') 75 | 76 | cmds = parser.add_subparsers(help='sub-command help') 77 | 78 | bline = cmds.add_parser( 79 | 'baseline', 80 | help='Baseline database state, advancing migration information without ' 81 | 'making changes') 82 | bline.set_defaults(action='baseline') 83 | 84 | reset = cmds.add_parser( 85 | 'reset', 86 | help='Reset database state, by dropping the keyspace (if it exists) ' 87 | 'and recreating it from scratch') 88 | reset.set_defaults(action='reset') 89 | 90 | mgrat = cmds.add_parser( 91 | 'migrate', 92 | help='Migrate database up to the most recent (or specified) version ' 93 | 'by applying any new migration scripts in sequence') 94 | mgrat.add_argument('-f', '--force', action='store_true', 95 | help='Force migration even if last attempt failed') 96 | mgrat.set_defaults(action='migrate') 97 | 98 | stats = cmds.add_parser( 99 | 'status', 100 | help='Print current state of keyspace') 101 | stats.set_defaults(action='status') 102 | 103 | genrt = cmds.add_parser( 104 | 'generate', 105 | help='Generate a new migration file') 106 | genrt.add_argument( 107 | 'description', 108 | help='Brief description of the new migration') 109 | genrt.add_argument( 110 | '--python', 111 | dest='migration_type', 112 | action='store_const', 113 | const='python', 114 | default='cql', 115 | help='Generates a Python script instead of CQL.') 116 | 117 | genrt.set_defaults(action='generate') 118 | 119 | for sub in (bline, reset, mgrat): 120 | sub.add_argument('db_version', metavar='VERSION', nargs='?', 121 | help='Database version to baseline/reset/migrate to') 122 | 123 | opts = parser.parse_args() 124 | # enable user confirmation if we're running the script from a TTY 125 | opts.cli_mode = sys.stdin.isatty() 126 | config = MigrationConfig.load(opts.config_file) 127 | 128 | if opts.action == 'generate': 129 | new_path = Migration.generate(config=config, 130 | description=opts.description, 131 | output=opts.migration_type) 132 | if sys.stdin.isatty(): 133 | open_file(new_path) 134 | 135 | print(os.path.basename(new_path)) 136 | else: 137 | with Migrator(config=config, profile=opts.profile, 138 | hosts=opts.hosts.split(','), port=opts.port, 139 | user=opts.user, password=opts.password, 140 | host_cert_path=opts.ssl_cert, 141 | client_key_path=opts.ssl_client_private_key, 142 | client_cert_path=opts.ssl_client_cert) as migrator: 143 | cmd_method = getattr(migrator, opts.action) 144 | if not callable(cmd_method): 145 | print('Error: invalid command', file=sys.stderr) 146 | sys.exit(1) 147 | 148 | try: 149 | cmd_method(opts) 150 | except MigrationError as e: 151 | print('Error: {}'.format(e), file=sys.stderr) 152 | sys.exit(1) 153 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | +---------------+---------------------+ 2 | | BRANCH CI | STATUS | 3 | +===============+=====================+ 4 | | master | |Master Badge| | 5 | +---------------+---------------------+ 6 | 7 | .. |Master Badge| image:: https://circleci.com/gh/Cobliteam/cassandra-migrate/tree/master.svg?style=svg&circle-token=cd6c01feb75b2abc6d4123426170114452dbb3c0 8 | :target:https://app.circleci.com/pipelines/github/Cobliteam/cassandra-migrate 9 | 10 | 11 | Cassandra-migrate 12 | ================= 13 | 14 | Simple Cassandra schema migration tool. 15 | 16 | Installation 17 | ------------ 18 | 19 | Run ``pip install cassandra-migrate``, or ``python ./setup.py install`` 20 | 21 | Reasoning 22 | --------- 23 | 24 | Unlike other available tools, this one: 25 | 26 | - Written in Python for easy installation 27 | - Does not require ``cqlsh``, just the Python driver 28 | - Supports baselining existing database to given versions 29 | - Supports partial advancement 30 | - Supports locking for concurrent instances using Lightweight Transactions 31 | - Verifies stored migrations against configured migrations 32 | - Stores content, checksum, date and state of every migration 33 | - Supports deploying with different keyspace configurations for different environments 34 | - Supports cql and python scripts migrations 35 | 36 | Configuration 37 | ------------- 38 | 39 | Databases are configured through YAML files. For example: 40 | 41 | .. code:: yaml 42 | 43 | keyspace: herbie 44 | profiles: 45 | prod: 46 | replication: 47 | class: SimpleStrategy 48 | replication_factor: 3 49 | migrations_path: ./migrations 50 | 51 | Where the ``migrations`` folder (relative to the config file). contains 52 | ``.cql`` or ``.py`` files. The files are loaded in lexical order. 53 | 54 | The default convention is to name them in the form: ``v001_my_migration.{cql | py}``. 55 | A custom naming scheme can be specified with the ``new_migration_name`` option. 56 | 57 | Note: new_migration_text is deprecated. The specific file type option should be used instead. 58 | 59 | For example 60 | 61 | .. code:: yaml 62 | 63 | # Date-based migration names 64 | new_migration_name: "v{date:YYYYMMDDHHmmss}_{desc}" 65 | 66 | # Default migration names 67 | new_migration_name: "v{next_version:03d}_{desc}" 68 | 69 | # Custom initial migration content 70 | new_migration_text: | 71 | /* Cassandra migration for keyspace {keyspace}. 72 | Version {next_version} - {date} 73 | 74 | {full_desc} */ 75 | 76 | # Custom initial migration content for cql scripts 77 | new_cql_migration_text: | 78 | /* Cassandra migration for keyspace {keyspace}. 79 | Version {next_version} - {date} 80 | 81 | {full_desc} */ 82 | 83 | # Custom initial migration content for python scripts 84 | new_python_migration_text: | 85 | # Cassandra migration for keyspace {keyspace}. 86 | # Version {next_version} - {date} 87 | # {full_desc} */ 88 | 89 | def execute(session, **kwargs): 90 | """ 91 | Main method for your migration. Do not rename this method. 92 | 93 | Raise an exception of any kind to abort the migration. 94 | """ 95 | 96 | print("Cassandra session: ", session) 97 | 98 | 99 | ``new_migration_name`` is a new-style Python format string, which can use the 100 | following parameters: 101 | 102 | - ``next_version``: Number of the newly generated migration (as an ``int``). 103 | - ``desc``: filename-clean description of the migration, as specified 104 | by the user. 105 | - ``full_desc``: unmodified description, possibly containing special characters. 106 | - ``date``: current date in UTC. Pay attention to the choice of formatting, 107 | otherwise you might include spaces in the file name. The above example should 108 | be a good starting point. 109 | - ``keyspace``: name of the configured keyspace. 110 | 111 | The format string should *not* contain the .cql or .py extensions, as it they 112 | added automatically. 113 | 114 | ``new_migraton_text`` is handled with the same rules outline above, but defines 115 | the initial content of the migration file, if the type-specific options below 116 | ared not set. 117 | 118 | ``new_cql_migraton_text`` defines the initial content of CQL migration files. 119 | 120 | ``new_python_migraton_text`` defines the initial content of Python migration 121 | files. 122 | 123 | 124 | Profiles 125 | -------- 126 | 127 | Profiles can be defined in the configuration file. They can configure 128 | the ``replication`` and ``durable_writes`` parameters for 129 | ``CREATE KEYSPACE``. A default ``dev`` profile is implicitly defined 130 | using a replication factor of 1. 131 | 132 | Usage 133 | ----- 134 | 135 | Common parameters: 136 | 137 | :: 138 | 139 | -H HOSTS, --hosts HOSTS 140 | Comma-separated list of contact points 141 | -p PORT, --port PORT Connection port 142 | -u USER, --user USER Connection username 143 | -P PASSWORD, --password PASSWORD 144 | Connection password 145 | -c CONFIG_FILE, --config-file CONFIG_FILE 146 | Path to configuration file 147 | -m PROFILE, --profile PROFILE 148 | Name of keyspace profile to use 149 | -s SSL_CERT, --ssl-cert SSL_CERT 150 | File path of .pem or .crt containing certificate of 151 | the cassandra host you are connecting to (or the 152 | certificate of the CA that signed the host 153 | certificate). If this option is provided, cassandra- 154 | migrate will use ssl to connect to the cluster. If 155 | this option is not provided, the -k and -t options 156 | will be ignored. 157 | -k SSL_CLIENT_PRIVATE_KEY, --ssl-client-private-key SSL_CLIENT_PRIVATE_KEY 158 | File path of the .key file containing the private key 159 | of the host on which the cassandra-migrate command is 160 | run. This option must be used in conjuction with the 161 | -t option. This option is ignored unless the -s option 162 | is provided. 163 | -t SSL_CLIENT_CERT, --ssl-client-cert SSL_CLIENT_CERT 164 | File path of the .crt file containing the public 165 | certificate of the host on which the cassandra-migrate 166 | command is run. This certificate (or the CA that 167 | signed it) must be trusted by the cassandra host that 168 | migrations are run against. This option must be used 169 | in conjuction with the -k option. This option is 170 | ignored unless the -s option is provided. 171 | -y, --assume-yes Automatically answer "yes" for all questions 172 | 173 | migrate 174 | ~~~~~~~ 175 | 176 | Advances a database to the latest (or chosen) version of migrations. 177 | Creates the keyspace and migrations table if necessary. 178 | 179 | Migrate will refuse to run if a previous attempt failed. To override 180 | that after cleaning up any leftovers (as Cassandra has no DDL 181 | transactions), use the ``--force`` option. 182 | 183 | Examples: 184 | 185 | .. code:: bash 186 | 187 | # Migrate to the latest database version using the default configuration file, 188 | # connecting to Cassandra in the local machine. 189 | cassandra-migrate -H 127.0.0.1 migrate 190 | 191 | # Migrate to version 2 using a specific config file. 192 | cassandra-migrate -c mydb.yml migrate 2 193 | 194 | # Migrate to a version by name. 195 | cassandra-migrate migrate v005_my_changes.cql 196 | 197 | # Force migration after a failure 198 | cassandra-migrate migrate 2 --force 199 | 200 | reset 201 | ~~~~~ 202 | 203 | Reset the database by dropping an existing keyspace, then running a 204 | migration. 205 | 206 | Examples: 207 | 208 | .. code:: bash 209 | 210 | # Reset the database to the latest version 211 | cassandra-migrate reset 212 | 213 | # Reset the database to a specifis version 214 | cassandra-migrate reset 3 215 | 216 | baseline 217 | ~~~~~~~~ 218 | 219 | Advance an existing database version without actually running the 220 | migrations. 221 | 222 | Useful for starting to manage a pre-existing database without recreating 223 | it from scratch. 224 | 225 | Examples: 226 | 227 | .. code:: bash 228 | 229 | # Baseline the existing database to the latest version 230 | cassandra-migrate baseline 231 | 232 | # Baseline the existing database to a specific version 233 | cassandra-migrate baseline 5 234 | 235 | status 236 | ~~~~~~ 237 | 238 | Print the current status of the database. 239 | 240 | Example: 241 | 242 | .. code:: bash 243 | 244 | cassandra-migrate status 245 | 246 | generate 247 | ~~~~~~~~ 248 | 249 | Generate a new migration file with the appropriate name and a basic header 250 | template, in the configured ``migrations_path``. 251 | 252 | When running the command interactively, the file will be opened by the default 253 | editor. The newly-generated file name will be printed to stdout. 254 | 255 | To generate a Python script, specify the ``--python`` option. 256 | 257 | See the configuration section for details on migration naming. 258 | 259 | Example: 260 | 261 | .. code:: bash 262 | 263 | cassandra-migrate generate "My migration description" 264 | 265 | cassandra-migrate generate "My migration description" --python 266 | 267 | 268 | License (MIT) 269 | ------------- 270 | 271 | :: 272 | 273 | Copyright (C) 2017 Cobli 274 | 275 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 276 | 277 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 278 | 279 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 280 | -------------------------------------------------------------------------------- /cassandra_migrate/migrator.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | from __future__ import (absolute_import, division, 4 | print_function, unicode_literals) 5 | from builtins import input, str 6 | 7 | import re 8 | import logging 9 | import uuid 10 | import codecs 11 | import sys 12 | import os 13 | import importlib 14 | from functools import wraps 15 | from future.moves.itertools import zip_longest 16 | 17 | import arrow 18 | from tabulate import tabulate 19 | from cassandra import ConsistencyLevel 20 | from cassandra.cluster import Cluster 21 | from cassandra.auth import PlainTextAuthProvider 22 | from cassandra_migrate import (Migration, FailedMigration, InconsistentState, 23 | UnknownMigration, ConcurrentMigration) 24 | from cassandra_migrate.cql import CqlSplitter 25 | 26 | 27 | CREATE_MIGRATIONS_TABLE = """ 28 | CREATE TABLE {keyspace}.{table} ( 29 | id uuid, 30 | version int, 31 | name text, 32 | content text, 33 | checksum blob, 34 | state text, 35 | applied_at timestamp, 36 | PRIMARY KEY (id) 37 | ) WITH caching = {{'keys': 'NONE', 'rows_per_partition': 'NONE'}}; 38 | """ 39 | 40 | CREATE_KEYSPACE = """ 41 | CREATE KEYSPACE {keyspace} 42 | WITH REPLICATION = {replication} 43 | AND DURABLE_WRITES = {durable_writes}; 44 | """ 45 | 46 | DROP_KEYSPACE = """ 47 | DROP KEYSPACE IF EXISTS "{keyspace}"; 48 | """ 49 | 50 | CREATE_DB_VERSION = """ 51 | INSERT INTO "{keyspace}"."{table}" 52 | (id, version, name, content, checksum, state, applied_at) 53 | VALUES (%s, %s, %s, %s, %s, %s, toTimestamp(now())) IF NOT EXISTS 54 | """ 55 | 56 | FINALIZE_DB_VERSION = """ 57 | UPDATE "{keyspace}"."{table}" SET state = %s WHERE id = %s IF state = %s 58 | """ 59 | 60 | DELETE_DB_VERSION = """ 61 | DELETE FROM "{keyspace}"."{table}" WHERE id = %s IF state = %s 62 | """ 63 | 64 | 65 | def cassandra_ddl_repr(data): 66 | """Generate a string representation of a map suitable for use in C* DDL""" 67 | if isinstance(data, str): 68 | return "'" + re.sub(r"(? 0 ' 219 | 'or the name of an existing migration') 220 | return num 221 | 222 | def _q(self, query, **kwargs): 223 | """ 224 | Format a query with the configured keyspace and migration table 225 | 226 | `keyspace` and `table` are interpolated as named arguments 227 | """ 228 | return query.format( 229 | keyspace=self.config.keyspace, table=self.config.migrations_table, 230 | **kwargs) 231 | 232 | def _execute(self, query, *args, **kwargs): 233 | """Execute a query with the current session""" 234 | 235 | self.logger.debug('Executing query: {}'.format(query)) 236 | return self.session.execute(query, *args, **kwargs) 237 | 238 | def _keyspace_exists(self): 239 | self._init_session() 240 | 241 | return self.config.keyspace in self.cluster.metadata.keyspaces 242 | 243 | def _ensure_keyspace(self): 244 | """Create the keyspace if it does not exist""" 245 | 246 | if self._keyspace_exists(): 247 | return 248 | 249 | self.logger.info("Creating keyspace '{}'".format(self.config.keyspace)) 250 | 251 | profile = self.current_profile 252 | self._execute(self._q( 253 | CREATE_KEYSPACE, 254 | replication=cassandra_ddl_repr(profile['replication']), 255 | durable_writes=cassandra_ddl_repr(profile['durable_writes']))) 256 | 257 | self.cluster.refresh_keyspace_metadata(self.config.keyspace) 258 | 259 | def _table_exists(self): 260 | self._init_session() 261 | 262 | ks_metadata = self.cluster.metadata.keyspaces.get(self.config.keyspace, 263 | None) 264 | # Fail if the keyspace is missing. If it should be created 265 | # automatically _ensure_keyspace() must be called first. 266 | if not ks_metadata: 267 | raise ValueError("Keyspace '{}' does not exist, " 268 | "stopping".format(self.config.keyspace)) 269 | 270 | return self.config.migrations_table in ks_metadata.tables 271 | 272 | def _ensure_table(self): 273 | """Create the migration table if it does not exist""" 274 | 275 | if self._table_exists(): 276 | return 277 | 278 | self.logger.info( 279 | "Creating table '{table}' in keyspace '{keyspace}'".format( 280 | keyspace=self.config.keyspace, 281 | table=self.config.migrations_table)) 282 | 283 | self._execute(self._q(CREATE_MIGRATIONS_TABLE)) 284 | self.cluster.refresh_table_metadata(self.config.keyspace, 285 | self.config.migrations_table) 286 | 287 | def _verify_migrations(self, migrations, ignore_failed=False, 288 | ignore_concurrent=False): 289 | """Verify if the version history persisted in C* matches the migrations 290 | 291 | Migrations with corresponding DB versions must have the same content 292 | and name. 293 | Every DB version must have a corresponding migration. 294 | Migrations without corresponding DB versions are considered pending, 295 | and returned as a result. 296 | 297 | Returns a list of tuples of (version, migration), with version starting 298 | from 1 and incrementing linearly. 299 | """ 300 | 301 | # Load all the currently existing versions and sort them by version 302 | # number, as Cassandra can only sort it for us by partition. 303 | cur_versions = self._execute(self._q( 304 | 'SELECT * FROM "{keyspace}"."{table}"')) 305 | cur_versions = sorted(cur_versions, key=lambda v: v.version) 306 | 307 | last_version = None 308 | version_pairs = zip_longest(cur_versions, migrations) 309 | 310 | # Work through ordered pairs of (existing version, migration), so that 311 | # stored versions and expected migrations can be compared for any 312 | # differences. 313 | for i, (version, migration) in enumerate(version_pairs, 1): 314 | # If version is empty, the migration has not yet been applied. 315 | # Keep track of the first such version, and append it to the 316 | # pending migrations list. 317 | if not version: 318 | break 319 | 320 | # If migration is empty, we have a version in the database with 321 | # no corresponding file. That might mean we're running the wrong 322 | # migration or have an out-of-date state, so we must fail. 323 | if not migration: 324 | raise UnknownMigration(version.version, version.name) 325 | 326 | # A migration was previously run and failed. 327 | if version.state == Migration.State.FAILED: 328 | if ignore_failed: 329 | break 330 | 331 | raise FailedMigration(version.version, version.name) 332 | 333 | last_version = version.version 334 | 335 | # A migration is in progress. 336 | if version.state == Migration.State.IN_PROGRESS: 337 | if ignore_concurrent: 338 | break 339 | raise ConcurrentMigration(version.version, version.name) 340 | 341 | # A stored version's migrations differs from the one in the FS. 342 | if version.content != migration.content or \ 343 | version.name != migration.name or \ 344 | bytearray(version.checksum) != bytearray(migration.checksum): 345 | raise InconsistentState(migration, version) 346 | 347 | if not last_version: 348 | pending_migrations = list(migrations) 349 | else: 350 | pending_migrations = list(migrations)[last_version:] 351 | 352 | if not pending_migrations: 353 | self.logger.info('Database is already up-to-date') 354 | else: 355 | self.logger.info( 356 | 'Pending migrations found. Current version: {}, ' 357 | 'Latest version: {}'.format(last_version, len(migrations))) 358 | 359 | pending_migrations = enumerate( 360 | pending_migrations, (last_version or 0) + 1) 361 | return last_version, cur_versions, list(pending_migrations) 362 | 363 | def _create_version(self, version, migration): 364 | """ 365 | Write an in-progress version entry to C* 366 | 367 | The migration is inserted with the given `version` number if and only 368 | if it does not exist already (using a CompareAndSet operation). 369 | 370 | If the insert suceeds (with the migration marked as in-progress), we 371 | can continue and actually execute it. Otherwise, there was a concurrent 372 | write and we must fail to allow the other write to continue. 373 | 374 | """ 375 | 376 | self.logger.info('Writing in-progress migration version {}: {}'.format( 377 | version, migration)) 378 | 379 | version_id = uuid.uuid4() 380 | result = self._execute( 381 | self._q(CREATE_DB_VERSION), 382 | (version_id, version, migration.name, migration.content, 383 | bytearray(migration.checksum), Migration.State.IN_PROGRESS)) 384 | 385 | if not result or not result[0].applied: 386 | raise ConcurrentMigration(version, migration.name) 387 | 388 | return version_id 389 | 390 | def _apply_cql_migration(self, version, migration): 391 | """ 392 | Persist and apply a cql migration 393 | 394 | First create an in-progress version entry, apply the script, then 395 | finalize the entry as succeeded, failed or skipped. 396 | """ 397 | 398 | self.logger.info('Applying cql migration') 399 | 400 | statements = CqlSplitter.split(migration.content) 401 | 402 | try: 403 | if statements: 404 | self.logger.info('Executing migration with ' 405 | '{} CQL statements'.format(len(statements))) 406 | 407 | for statement in statements: 408 | self.session.execute(statement) 409 | except Exception: 410 | self.logger.exception('Failed to execute migration') 411 | raise FailedMigration(version, migration.name) 412 | 413 | def _apply_python_migration(self, version, migration): 414 | """ 415 | Persist and apply a python migration 416 | 417 | First create an in-progress version entry, apply the script, then 418 | finalize the entry as succeeded, failed or skipped. 419 | """ 420 | self.logger.info('Applying python script') 421 | 422 | try: 423 | mod, _ = os.path.splitext(os.path.basename(migration.path)) 424 | migration_script = importlib.import_module(mod) 425 | migration_script.execute(self._session) 426 | except Exception: 427 | self.logger.exception('Failed to execute script') 428 | raise FailedMigration(version, migration.name) 429 | 430 | def _apply_migration(self, version, migration, skip=False): 431 | """ 432 | Persist and apply a migration 433 | 434 | When `skip` is True, do everything but actually run the script, for 435 | example, when baselining instead of migrating. 436 | """ 437 | 438 | self.logger.info('Advancing to version {}'.format(version)) 439 | 440 | version_uuid = self._create_version(version, migration) 441 | new_state = Migration.State.FAILED 442 | sys.path.append(self.config.migrations_path) 443 | 444 | result = None 445 | 446 | try: 447 | if skip: 448 | self.logger.info('Migration is marked for skipping, ' 449 | 'not actually running script') 450 | else: 451 | if migration.is_python: 452 | self._apply_python_migration(version, migration) 453 | else: 454 | self._apply_cql_migration(version, migration) 455 | except Exception: 456 | self.logger.exception('Failed to execute migration') 457 | raise FailedMigration(version, migration.name) 458 | else: 459 | new_state = (Migration.State.SUCCEEDED if not skip 460 | else Migration.State.SKIPPED) 461 | finally: 462 | self.logger.info('Finalizing migration version with ' 463 | 'state {}'.format(new_state)) 464 | result = self._execute( 465 | self._q(FINALIZE_DB_VERSION), 466 | (new_state, version_uuid, Migration.State.IN_PROGRESS)) 467 | 468 | if not result or not result[0].applied: 469 | raise ConcurrentMigration(version, migration.name) 470 | 471 | def _cleanup_previous_versions(self, cur_versions): 472 | if not cur_versions: 473 | return 474 | 475 | last_version = cur_versions[-1] 476 | if last_version.state != Migration.State.FAILED: 477 | return 478 | 479 | self.logger.warn( 480 | 'Cleaning up previous failed migration ' 481 | '(version {}): {}'.format(last_version.version, last_version.name)) 482 | 483 | result = self._execute( 484 | self._q(DELETE_DB_VERSION), 485 | (last_version.id, Migration.State.FAILED)) 486 | 487 | if not result[0].applied: 488 | raise ConcurrentMigration(last_version.version, 489 | last_version.name) 490 | 491 | def _advance(self, migrations, target, cur_versions, skip=False, 492 | force=False): 493 | """Apply all necessary migrations to reach a target version""" 494 | if force: 495 | self._cleanup_previous_versions(cur_versions) 496 | 497 | target_version = self._get_target_version(target) 498 | 499 | if migrations: 500 | # Set default keyspace so migrations don't need to refer to it 501 | # manually 502 | # Fixes https://github.com/Cobliteam/cassandra-migrate/issues/5 503 | self.session.execute('USE {};'.format(self.config.keyspace)) 504 | 505 | for version, migration in migrations: 506 | if version > target_version: 507 | break 508 | 509 | self._apply_migration(version, migration, skip=skip) 510 | 511 | self.cluster.refresh_schema_metadata() 512 | 513 | def baseline(self, opts): 514 | """Baseline a database, by advancing migration state without changes""" 515 | 516 | self._check_cluster() 517 | self._ensure_table() 518 | 519 | last_version, cur_versions, pending_migrations = \ 520 | self._verify_migrations(self.config.migrations, 521 | ignore_failed=False) 522 | 523 | self._advance(pending_migrations, opts.db_version, cur_versions, 524 | skip=True) 525 | 526 | @confirmation_required 527 | def migrate(self, opts): 528 | """ 529 | Migrate a database to a given version, applying any needed migrations 530 | """ 531 | 532 | self._check_cluster() 533 | 534 | self._ensure_keyspace() 535 | self._ensure_table() 536 | 537 | last_version, cur_versions, pending_migrations = \ 538 | self._verify_migrations(self.config.migrations, 539 | ignore_failed=opts.force) 540 | 541 | self._advance(pending_migrations, opts.db_version, cur_versions, 542 | force=opts.force) 543 | 544 | @confirmation_required 545 | def reset(self, opts): 546 | """Reset a database, by dropping the keyspace then migrating""" 547 | self._check_cluster() 548 | 549 | self.logger.info("Dropping existing keyspace '{}'".format( 550 | self.config.keyspace)) 551 | 552 | self._execute(self._q(DROP_KEYSPACE)) 553 | self.cluster.refresh_schema_metadata() 554 | 555 | opts.force = False 556 | self.migrate(opts) 557 | 558 | @staticmethod 559 | def _bytes_to_hex(bs): 560 | return codecs.getencoder('hex')(bs)[0] 561 | 562 | def status(self, opts): 563 | self._check_cluster() 564 | 565 | if not self._keyspace_exists(): 566 | print("Keyspace '{}' does not exist".format(self.config.keyspace)) 567 | return 568 | 569 | if not self._table_exists(): 570 | print( 571 | "Migration table '{table}' does not exist in " 572 | "keyspace '{keyspace}'".format( 573 | keyspace=self.config.keyspace, 574 | table=self.config.migrations_table)) 575 | return 576 | 577 | last_version, cur_versions, pending_migrations = \ 578 | self._verify_migrations(self.config.migrations, 579 | ignore_failed=True, 580 | ignore_concurrent=True) 581 | latest_version = len(self.config.migrations) 582 | 583 | print(tabulate(( 584 | ('Keyspace:', self.config.keyspace), 585 | ('Migrations table:', self.config.migrations_table), 586 | ('Current DB version:', last_version), 587 | ('Latest DB version:', latest_version)), 588 | tablefmt='plain')) 589 | 590 | if cur_versions: 591 | print('\n## Applied migrations\n') 592 | 593 | data = [] 594 | for version in cur_versions: 595 | checksum = self._bytes_to_hex(version.checksum) 596 | date = arrow.get(version.applied_at).format() 597 | data.append(( 598 | str(version.version), 599 | version.name, 600 | version.state, 601 | date, 602 | checksum)) 603 | print(tabulate(data, headers=['#', 'Name', 'State', 604 | 'Date applied', 'Checksum'])) 605 | 606 | if pending_migrations: 607 | print('\n## Pending migrations\n') 608 | 609 | data = [] 610 | for version, migration in pending_migrations: 611 | checksum = self._bytes_to_hex(migration.checksum) 612 | data.append(( 613 | str(version), 614 | migration.name, 615 | checksum)) 616 | print(tabulate(data, headers=['#', 'Name', 'Checksum'])) 617 | --------------------------------------------------------------------------------