├── .gitignore ├── stellar ├── __main__.py ├── __init__.py ├── config.py ├── models.py ├── operations.py ├── app.py └── command.py ├── MANIFEST.in ├── requirements.txt ├── .travis.yml ├── tox.ini ├── tests ├── test_models.py ├── test_operations.py └── test_starts.py ├── LICENSE ├── setup.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | .tox 4 | dist -------------------------------------------------------------------------------- /stellar/__main__.py: -------------------------------------------------------------------------------- 1 | from .command import main 2 | 3 | main() 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include stellar *.md *.txt *.py 2 | recursive-include tests *.md *.txt *.py 3 | include README.md 4 | include LICENSE 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | PyYAML>=3.11 2 | SQLAlchemy>=0.9.6 3 | humanize>=0.5.1 4 | schema>=0.3.1 5 | psutil>=2.1.1 6 | click>=3.1 7 | SQLAlchemy-Utils>=0.26.13 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - 2.7 4 | - 3.3 5 | - 3.4 6 | - pypy 7 | install: 8 | - pip install -q -e . 9 | script: 10 | - py.test 11 | -------------------------------------------------------------------------------- /stellar/__init__.py: -------------------------------------------------------------------------------- 1 | from . import app 2 | from . import command 3 | from . import config 4 | from . import models 5 | from . import operations 6 | 7 | __version__ = app.__version__ 8 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py27, py33, py34, pypy 8 | 9 | [testenv] 10 | deps = 11 | pytest 12 | commands = 13 | pip install -e . 14 | py.test 15 | -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | from stellar.models import get_unique_hash, Table, Snapshot 2 | 3 | 4 | def test_get_unique_hash(): 5 | assert get_unique_hash() 6 | assert get_unique_hash() != get_unique_hash() 7 | assert len(get_unique_hash()) == 32 8 | 9 | 10 | def test_table(): 11 | table = Table( 12 | table_name='hapsu', 13 | snapshot=Snapshot( 14 | snapshot_name='snapshot', 15 | project_name='myproject', 16 | hash='3330484d0a70eecab84554b5576b4553' 17 | ) 18 | ) 19 | assert len(table.get_table_name('master')) == 24 20 | -------------------------------------------------------------------------------- /tests/test_operations.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from stellar.operations import _get_pid_column 4 | 5 | 6 | class ConnectionMock(object): 7 | def __init__(self, version): 8 | self.version = version 9 | 10 | def execute(self, query): 11 | return self 12 | 13 | def first(self): 14 | return [self.version] 15 | 16 | 17 | class TestGetPidColumn(object): 18 | @pytest.mark.parametrize('version', ['9.1', '8.9', '9.1.9', '8.9.9']) 19 | def test_returns_procpid_for_version_older_than_9_2(self, version): 20 | raw_conn = ConnectionMock(version=version) 21 | assert _get_pid_column(raw_conn) == 'procpid' 22 | 23 | @pytest.mark.parametrize('version', [ 24 | '9.2', '9.3', '10.0', '9.2.1', '9.6beta1', '10.1.1', 25 | '10.3 (Ubuntu 10.3-1.pgdg16.04+1)' 26 | ]) 27 | def test_returns_pid_for_version_equal_or_newer_than_9_2(self, version): 28 | raw_conn = ConnectionMock(version=version) 29 | assert _get_pid_column(raw_conn) == 'pid' 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Fast Monkeys Oy 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /tests/test_starts.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import stellar 3 | import tempfile 4 | 5 | 6 | class TestCase(object): 7 | @pytest.yield_fixture(autouse=True) 8 | def config(self, monkeypatch): 9 | with tempfile.NamedTemporaryFile() as tmp: 10 | def load_test_config(self): 11 | self.config = { 12 | 'stellar_url': 'sqlite:///%s' % tmp.name, 13 | 'url': 'sqlite:///%s' % tmp.name, 14 | 'project_name': 'test_project', 15 | 'tracked_databases': ['test_database'], 16 | 'TEST': True 17 | } 18 | return None 19 | monkeypatch.setattr(stellar.app.Stellar, 'load_config', load_test_config) 20 | yield 21 | 22 | 23 | class Test(TestCase): 24 | def test_setup_method_works(self, monkeypatch): 25 | monkeypatch.setattr( 26 | stellar.app.Stellar, 27 | 'create_stellar_database', 28 | lambda x: None 29 | ) 30 | app = stellar.app.Stellar() 31 | for key in ( 32 | 'TEST', 33 | 'stellar_url', 34 | 'url', 35 | 'project_name', 36 | 'tracked_databases', 37 | ): 38 | assert app.config[key] 39 | 40 | def test_shows_not_enough_arguments(self): 41 | with pytest.raises(SystemExit) as e: 42 | stellar.command.main() 43 | 44 | def test_app_context(self, monkeypatch): 45 | monkeypatch.setattr( 46 | stellar.app.Stellar, 47 | 'create_stellar_database', 48 | lambda x: None 49 | ) 50 | app = stellar.app.Stellar() 51 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import os 3 | import re 4 | 5 | from setuptools import setup, find_packages 6 | 7 | 8 | # https://bitbucket.org/zzzeek/alembic/raw/f38eaad4a80d7e3d893c3044162971971ae0 9 | # 09bf/setup.py 10 | with open( 11 | os.path.join(os.path.dirname(__file__), 'stellar', 'app.py') 12 | ) as app_file: 13 | VERSION = re.compile( 14 | r".*__version__ = '(.*?)'", re.S 15 | ).match(app_file.read()).group(1) 16 | 17 | with open("README.md") as readme: 18 | long_description = readme.read() 19 | 20 | setup( 21 | name='stellar', 22 | description=( 23 | 'stellar is a tool for creating and restoring database snapshots' 24 | ), 25 | long_description=long_description, 26 | version=VERSION, 27 | url='https://github.com/fastmonkeys/stellar', 28 | license='BSD', 29 | author=u'Teemu Kokkonen, Pekka Pöyry', 30 | author_email='teemu@fastmonkeys.com, pekka@fastmonkeys.com', 31 | packages=find_packages('.', exclude=['examples*', 'test*']), 32 | entry_points={ 33 | 'console_scripts': [ 'stellar = stellar.command:main' ], 34 | }, 35 | zip_safe=False, 36 | include_package_data=True, 37 | platforms='any', 38 | classifiers=[ 39 | 'Intended Audience :: Developers', 40 | 'License :: OSI Approved :: BSD License', 41 | 'Operating System :: POSIX', 42 | 'Operating System :: Microsoft :: Windows', 43 | 'Operating System :: MacOS :: MacOS X', 44 | 'Topic :: Utilities', 45 | 'Programming Language :: Python :: 2', 46 | 'Programming Language :: Python :: 3', 47 | 'Topic :: Database', 48 | 'Topic :: Software Development :: Version Control', 49 | ], 50 | install_requires = [ 51 | 'PyYAML>=3.11', 52 | 'SQLAlchemy>=0.9.6', 53 | 'humanize>=0.5.1', 54 | 'schema>=0.3.1', 55 | 'click>=3.1', 56 | 'SQLAlchemy-Utils>=0.26.11', 57 | 'psutil>=2.1.1', 58 | ] 59 | ) 60 | -------------------------------------------------------------------------------- /stellar/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import yaml 4 | from schema import Use, Schema, SchemaError, Optional 5 | 6 | 7 | class InvalidConfig(Exception): 8 | pass 9 | 10 | 11 | class MissingConfig(Exception): 12 | pass 13 | 14 | 15 | default_config = { 16 | 'logging': 30, 17 | 'migrate_from_0_3_2': True 18 | } 19 | schema = Schema({ 20 | 'stellar_url': Use(str), 21 | 'url': Use(str), 22 | 'project_name': Use(str), 23 | 'tracked_databases': [Use(str)], 24 | Optional('logging'): int, 25 | Optional('migrate_from_0_3_2'): bool 26 | }) 27 | 28 | 29 | def get_config_path(): 30 | current_directory = os.getcwd() 31 | while True: 32 | try: 33 | with open( 34 | os.path.join(current_directory, 'stellar.yaml'), 35 | 'rb' 36 | ) as fp: 37 | return os.path.join(current_directory, 'stellar.yaml') 38 | except IOError: 39 | pass 40 | 41 | current_directory = os.path.abspath( 42 | os.path.join(current_directory, '..') 43 | ) 44 | if current_directory == '/': 45 | return None 46 | 47 | 48 | def load_config(): 49 | config = {} 50 | current_directory = os.getcwd() 51 | while True: 52 | try: 53 | with open( 54 | os.path.join(current_directory, 'stellar.yaml'), 55 | 'rb' 56 | ) as fp: 57 | config = yaml.safe_load(fp) 58 | break 59 | except IOError: 60 | pass 61 | current_directory = os.path.abspath( 62 | os.path.join(current_directory, '..') 63 | ) 64 | 65 | if current_directory == '/': 66 | break 67 | 68 | if not config: 69 | raise MissingConfig() 70 | 71 | for k, v in default_config.items(): 72 | if k not in config: 73 | config[k] = v 74 | 75 | try: 76 | return schema.validate(config) 77 | except SchemaError as e: 78 | raise InvalidConfig(e) 79 | 80 | 81 | def save_config(config): 82 | logging.getLogger(__name__).debug('save_config()') 83 | with open(get_config_path(), "w") as fp: 84 | yaml.dump(config, fp) 85 | -------------------------------------------------------------------------------- /stellar/models.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import uuid 3 | from datetime import datetime 4 | 5 | import sqlalchemy as sa 6 | from sqlalchemy.ext.declarative import declarative_base 7 | 8 | Base = declarative_base() 9 | 10 | 11 | def get_unique_hash(): 12 | return hashlib.md5(str(uuid.uuid4()).encode('utf-8')).hexdigest() 13 | 14 | 15 | class Snapshot(Base): 16 | __tablename__ = 'snapshot' 17 | id = sa.Column( 18 | sa.Integer, 19 | sa.Sequence('snapshot_id_seq'), 20 | primary_key=True 21 | ) 22 | snapshot_name = sa.Column(sa.String(255), nullable=False) 23 | project_name = sa.Column(sa.String(255), nullable=False) 24 | hash = sa.Column(sa.String(32), nullable=False, default=get_unique_hash) 25 | created_at = sa.Column(sa.DateTime, default=datetime.utcnow) 26 | worker_pid = sa.Column(sa.Integer, nullable=True) 27 | 28 | @property 29 | def slaves_ready(self): 30 | return self.worker_pid is None 31 | 32 | def __repr__(self): 33 | return "" % ( 34 | self.snapshot_name 35 | ) 36 | 37 | 38 | class Table(Base): 39 | __tablename__ = 'table' 40 | id = sa.Column(sa.Integer, sa.Sequence('table_id_seq'), primary_key=True) 41 | table_name = sa.Column(sa.String(255), nullable=False) 42 | snapshot_id = sa.Column( 43 | sa.Integer, sa.ForeignKey(Snapshot.id), nullable=False 44 | ) 45 | snapshot = sa.orm.relationship(Snapshot, backref='tables') 46 | 47 | def get_table_name(self, postfix, old=False): 48 | if not self.snapshot: 49 | raise Exception('Table name requires snapshot') 50 | if not self.snapshot.hash: 51 | raise Exception('Snapshot hash is empty.') 52 | 53 | if old: 54 | return 'stellar_%s_%s_%s' % ( 55 | self.table_name, 56 | self.snapshot.hash, 57 | postfix 58 | ) 59 | else: 60 | return 'stellar_%s' % hashlib.md5( 61 | ('%s|%s|%s' % ( 62 | self.table_name, 63 | self.snapshot.hash, 64 | postfix 65 | )).encode('utf-8') 66 | ).hexdigest()[0:16] 67 | 68 | def __repr__(self): 69 | return "" % ( 70 | self.table_name, 71 | ) 72 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Stellar - Fast database snapshot and restore tool for development. 2 | ======= 3 | 4 | [![Build Status](https://travis-ci.org/fastmonkeys/stellar.svg?branch=master)](https://travis-ci.org/fastmonkeys/stellar)  5 | ![](http://img.shields.io/pypi/dm/stellar.svg)  6 | ![](http://img.shields.io/pypi/v/stellar.svg) 7 | 8 | 9 | Stellar allows you to quickly restore database when you are e.g. writing database migrations, switching branches or messing with SQL. PostgreSQL and MySQL (partially) are supported. 10 | 11 | ![Screenshot of Stellar terminal window](http://imgur.com/0fXXdcx.png) 12 | 13 | 14 | Benchmarks 15 | ------- 16 | Stellar is fast. It can restore a database ~140 times faster than using the usual 17 | pg_dump & pg_restore. 18 | 19 | ![Benchmarking database restore speed](http://imgur.com/Md1AHXa.png) 20 | 21 | How it works 22 | ------- 23 | 24 | Stellar works by storing copies of the database in the RDBMS (named as stellar_xxx_master and stellar_xxxx_slave). When restoring the database, Stellar simply renames the database making it lot faster than the usual SQL dump. However, Stellar uses lots of storage space so you probably don't want to make too many snapshots or you will eventually run out of storage space. 25 | 26 | **Warning: Please don't use Stellar if you can't afford data loss.** It's great for developing but not meant for production. 27 | 28 | How to get started 29 | ------- 30 | 31 | You can install Stellar with `pip`. 32 | 33 | ```$ pip install stellar``` 34 | 35 | After that, you should go to your project folder and initialize Stellar settings. Stellar initialization wizard will help you with that. 36 | 37 | ```$ stellar init``` 38 | 39 | Stellar settings are saved as 'stellar.yaml' so you probably want to add that to your `.gitignore`. 40 | 41 | ```$ echo stellar.yaml >> .gitignore``` 42 | 43 | Done! :dancers: 44 | 45 | 46 | How to take a snapshot 47 | ------- 48 | 49 | ```$ stellar snapshot SNAPSHOT_NAME``` 50 | 51 | How to restore from a snapshot 52 | ------- 53 | 54 | ```$ stellar restore SNAPSHOT_NAME``` 55 | 56 | Common issues 57 | ------- 58 | 59 | ```` 60 | sqlalchemy.exc.OperationalError: (OperationalError) (1044, u"Access denied for user 'my_db_username'@'localhost' to database 'stellar_data'") "CREATE DATABASE stellar_data CHARACTER SET = 'utf8'" () 61 | ````` 62 | 63 | Make sure you have the rights to create new databases. See [Issue 10](https://github.com/fastmonkeys/stellar/issues/10) for discussion 64 | 65 | If you are using PostgreSQL, make sure you have a database that is named the same as the unix username. You can test this by running just `psql`. (See [issue #44](https://github.com/fastmonkeys/stellar/issues/44) for details) 66 | -------------------------------------------------------------------------------- /stellar/operations.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | 4 | import sqlalchemy_utils 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | SUPPORTED_DIALECTS = ( 10 | 'postgresql', 11 | 'mysql' 12 | ) 13 | 14 | class NotSupportedDatabase(Exception): 15 | pass 16 | 17 | 18 | def get_engine_url(raw_conn, database): 19 | url = str(raw_conn.engine.url) 20 | if url.count('/') == 3 and url.endswith('/'): 21 | return '%s%s' % (url, database) 22 | else: 23 | if not url.endswith('/'): 24 | url += '/' 25 | return '%s/%s' % ('/'.join(url.split('/')[0:-2]), database) 26 | 27 | 28 | def _get_pid_column(raw_conn): 29 | # Some distros (e.g Debian) may inject their branding into server_version 30 | server_version = raw_conn.execute('SHOW server_version;').first()[0] 31 | version_string = re.search('^(\d+\.\d+)', server_version).group(0) 32 | version = [int(x) for x in version_string.split('.')] 33 | return 'pid' if version >= [9, 2] else 'procpid' 34 | 35 | 36 | def terminate_database_connections(raw_conn, database): 37 | logger.debug('terminate_database_connections(%r)', database) 38 | if raw_conn.engine.dialect.name == 'postgresql': 39 | pid_column = _get_pid_column(raw_conn) 40 | 41 | raw_conn.execute( 42 | ''' 43 | SELECT pg_terminate_backend(pg_stat_activity.%(pid_column)s) 44 | FROM pg_stat_activity 45 | WHERE 46 | pg_stat_activity.datname = '%(database)s' AND 47 | %(pid_column)s <> pg_backend_pid(); 48 | ''' % {'pid_column': pid_column, 'database': database} 49 | ) 50 | else: 51 | # NotYetImplemented 52 | pass 53 | 54 | 55 | def create_database(raw_conn, database): 56 | logger.debug('create_database(%r)', database) 57 | return sqlalchemy_utils.functions.create_database( 58 | get_engine_url(raw_conn, database) 59 | ) 60 | 61 | 62 | def copy_database(raw_conn, from_database, to_database): 63 | logger.debug('copy_database(%r, %r)', from_database, to_database) 64 | terminate_database_connections(raw_conn, from_database) 65 | 66 | if raw_conn.engine.dialect.name == 'postgresql': 67 | sqlalchemy_utils.functions.create_database( 68 | '%s%s' % (raw_conn.engine.url, to_database), 69 | template=from_database 70 | ) 71 | elif raw_conn.engine.dialect.name == 'mysql': 72 | # Horribly slow implementation. 73 | create_database(raw_conn, to_database) 74 | for row in raw_conn.execute('SHOW TABLES in %s;' % from_database): 75 | raw_conn.execute(''' 76 | CREATE TABLE %s.%s LIKE %s.%s 77 | ''' % ( 78 | to_database, 79 | row[0], 80 | from_database, 81 | row[0] 82 | )) 83 | raw_conn.execute('ALTER TABLE %s.%s DISABLE KEYS' % ( 84 | to_database, 85 | row[0] 86 | )) 87 | raw_conn.execute(''' 88 | INSERT INTO %s.%s SELECT * FROM %s.%s 89 | ''' % ( 90 | to_database, 91 | row[0], 92 | from_database, 93 | row[0] 94 | )) 95 | raw_conn.execute('ALTER TABLE %s.%s ENABLE KEYS' % ( 96 | to_database, 97 | row[0] 98 | )) 99 | else: 100 | raise NotSupportedDatabase() 101 | 102 | 103 | def database_exists(raw_conn, database): 104 | logger.debug('database_exists(%r)', database) 105 | return sqlalchemy_utils.functions.database_exists( 106 | get_engine_url(raw_conn, database) 107 | ) 108 | 109 | 110 | def remove_database(raw_conn, database): 111 | logger.debug('remove_database(%r)', database) 112 | terminate_database_connections(raw_conn, database) 113 | return sqlalchemy_utils.functions.drop_database( 114 | get_engine_url(raw_conn, database) 115 | ) 116 | 117 | 118 | def rename_database(raw_conn, from_database, to_database): 119 | logger.debug('rename_database(%r, %r)', from_database, to_database) 120 | terminate_database_connections(raw_conn, from_database) 121 | if raw_conn.engine.dialect.name == 'postgresql': 122 | raw_conn.execute( 123 | ''' 124 | ALTER DATABASE "%s" RENAME TO "%s" 125 | ''' % 126 | ( 127 | from_database, 128 | to_database 129 | ) 130 | ) 131 | elif raw_conn.engine.dialect.name == 'mysql': 132 | create_database(raw_conn, to_database) 133 | for row in raw_conn.execute('SHOW TABLES in %s;' % from_database): 134 | raw_conn.execute(''' 135 | RENAME TABLE %s.%s TO %s.%s; 136 | ''' % ( 137 | from_database, 138 | row[0], 139 | to_database, 140 | row[0] 141 | )) 142 | remove_database(raw_conn, from_database) 143 | else: 144 | raise NotSupportedDatabase() 145 | 146 | 147 | def list_of_databases(raw_conn): 148 | logger.debug('list_of_databases()') 149 | if raw_conn.engine.dialect.name == 'postgresql': 150 | return [ 151 | row[0] 152 | for row in raw_conn.execute(''' 153 | SELECT datname FROM pg_database 154 | WHERE datistemplate = false 155 | ''') 156 | ] 157 | elif raw_conn.engine.dialect.name == 'mysql': 158 | return [ 159 | row[0] 160 | for row in raw_conn.execute('''SHOW DATABASES''') 161 | ] 162 | else: 163 | raise NotSupportedDatabase() 164 | 165 | -------------------------------------------------------------------------------- /stellar/app.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import sys 4 | import click 5 | from functools import partial 6 | 7 | from .config import load_config 8 | from .models import Snapshot, Table, Base 9 | from .operations import ( 10 | copy_database, 11 | create_database, 12 | database_exists, 13 | remove_database, 14 | rename_database, 15 | terminate_database_connections, 16 | list_of_databases, 17 | ) 18 | from sqlalchemy import create_engine 19 | from sqlalchemy.orm import sessionmaker 20 | from sqlalchemy.exc import ProgrammingError 21 | from psutil import pid_exists 22 | 23 | 24 | __version__ = '0.4.5' 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | class Operations(object): 29 | def __init__(self, raw_connection, config): 30 | self.terminate_database_connections = partial( 31 | terminate_database_connections, raw_connection 32 | ) 33 | self.create_database = partial(create_database, raw_connection) 34 | self.copy_database = partial(copy_database, raw_connection) 35 | self.database_exists = partial(database_exists, raw_connection) 36 | self.rename_database = partial(rename_database, raw_connection) 37 | self.remove_database = partial(remove_database, raw_connection) 38 | self.list_of_databases = partial(list_of_databases, raw_connection) 39 | 40 | 41 | class Stellar(object): 42 | def __init__(self): 43 | logger.debug('Initialized Stellar()') 44 | self.load_config() 45 | self.init_database() 46 | 47 | def load_config(self): 48 | self.config = load_config() 49 | logging.basicConfig(level=self.config['logging']) 50 | 51 | def init_database(self): 52 | self.raw_db = create_engine(self.config['url'], echo=False) 53 | self.raw_conn = self.raw_db.connect() 54 | self.operations = Operations(self.raw_conn, self.config) 55 | 56 | try: 57 | self.raw_conn.connection.set_isolation_level(0) 58 | except AttributeError: 59 | logger.info('Could not set isolation level to 0') 60 | 61 | self.db = create_engine(self.config['stellar_url'], echo=False) 62 | self.db.session = sessionmaker(bind=self.db)() 63 | self.raw_db.session = sessionmaker(bind=self.raw_db)() 64 | tables_missing = self.create_stellar_database() 65 | 66 | self.create_stellar_tables() 67 | 68 | # logger.getLogger('sqlalchemy.engine').setLevel(logger.WARN) 69 | 70 | def create_stellar_database(self): 71 | if not self.operations.database_exists('stellar_data'): 72 | self.operations.create_database('stellar_data') 73 | return True 74 | else: 75 | return False 76 | 77 | def create_stellar_tables(self): 78 | Base.metadata.create_all(self.db) 79 | self.db.session.commit() 80 | 81 | def get_snapshot(self, snapshot_name): 82 | return self.db.session.query(Snapshot).filter( 83 | Snapshot.snapshot_name == snapshot_name, 84 | Snapshot.project_name == self.config['project_name'] 85 | ).first() 86 | 87 | def get_snapshots(self): 88 | return self.db.session.query(Snapshot).filter( 89 | Snapshot.project_name == self.config['project_name'] 90 | ).order_by( 91 | Snapshot.created_at.desc() 92 | ).all() 93 | 94 | def get_latest_snapshot(self): 95 | return self.db.session.query(Snapshot).filter( 96 | Snapshot.project_name == self.config['project_name'] 97 | ).order_by(Snapshot.created_at.desc()).first() 98 | 99 | def create_snapshot(self, snapshot_name, before_copy=None): 100 | snapshot = Snapshot( 101 | snapshot_name=snapshot_name, 102 | project_name=self.config['project_name'] 103 | ) 104 | self.db.session.add(snapshot) 105 | self.db.session.flush() 106 | 107 | for table_name in self.config['tracked_databases']: 108 | if before_copy: 109 | before_copy(table_name) 110 | table = Table( 111 | table_name=table_name, 112 | snapshot=snapshot 113 | ) 114 | logger.debug('Copying %s to %s' % ( 115 | table_name, 116 | table.get_table_name('master') 117 | )) 118 | self.operations.copy_database( 119 | table_name, 120 | table.get_table_name('master') 121 | ) 122 | self.db.session.add(table) 123 | self.db.session.commit() 124 | 125 | self.start_background_slave_copy(snapshot) 126 | 127 | def remove_snapshot(self, snapshot): 128 | for table in snapshot.tables: 129 | try: 130 | self.operations.remove_database( 131 | table.get_table_name('master') 132 | ) 133 | except ProgrammingError: 134 | pass 135 | try: 136 | self.operations.remove_database( 137 | table.get_table_name('slave') 138 | ) 139 | except ProgrammingError: 140 | pass 141 | self.db.session.delete(table) 142 | self.db.session.delete(snapshot) 143 | self.db.session.commit() 144 | 145 | def rename_snapshot(self, snapshot, new_name): 146 | snapshot.snapshot_name = new_name 147 | self.db.session.commit() 148 | 149 | def restore(self, snapshot): 150 | for table in snapshot.tables: 151 | click.echo("Restoring database %s" % table.table_name) 152 | if not self.operations.database_exists( 153 | table.get_table_name('slave') 154 | ): 155 | click.echo( 156 | "Database %s does not exist." 157 | % table.get_table_name('slave') 158 | ) 159 | sys.exit(1) 160 | try: 161 | self.operations.remove_database(table.table_name) 162 | except ProgrammingError: 163 | logger.warn('Database %s does not exist.' % table.table_name) 164 | self.operations.rename_database( 165 | table.get_table_name('slave'), 166 | table.table_name 167 | ) 168 | snapshot.worker_pid = 1 169 | self.db.session.commit() 170 | 171 | self.start_background_slave_copy(snapshot) 172 | 173 | def start_background_slave_copy(self, snapshot): 174 | logger.debug('Starting background slave copy') 175 | snapshot_id = snapshot.id 176 | 177 | self.raw_conn.close() 178 | self.raw_db.session.close() 179 | self.db.session.close() 180 | 181 | pid = os.fork() if hasattr(os, 'fork') else None 182 | if pid: 183 | return 184 | 185 | self.init_database() 186 | self.operations = Operations(self.raw_conn, self.config) 187 | 188 | snapshot = self.db.session.query(Snapshot).get(snapshot_id) 189 | snapshot.worker_pid = os.getpid() 190 | self.db.session.commit() 191 | self.inline_slave_copy(snapshot) 192 | sys.exit() 193 | 194 | def inline_slave_copy(self, snapshot): 195 | for table in snapshot.tables: 196 | self.operations.copy_database( 197 | table.get_table_name('master'), 198 | table.get_table_name('slave') 199 | ) 200 | snapshot.worker_pid = None 201 | self.db.session.commit() 202 | 203 | def is_copy_process_running(self, snapshot): 204 | return pid_exists(snapshot.worker_pid) 205 | 206 | def is_old_database(self): 207 | for snapshot in self.db.session.query(Snapshot): 208 | for table in snapshot.tables: 209 | for postfix in ('master', 'slave'): 210 | old_name = table.get_table_name(postfix=postfix, old=True) 211 | if self.operations.database_exists(old_name): 212 | return True 213 | return False 214 | 215 | def update_database_names_to_new_version(self, after_rename=None): 216 | for snapshot in self.db.session.query(Snapshot): 217 | for table in snapshot.tables: 218 | for postfix in ('master', 'slave'): 219 | old_name = table.get_table_name(postfix=postfix, old=True) 220 | new_name = table.get_table_name(postfix=postfix, old=False) 221 | if self.operations.database_exists(old_name): 222 | self.operations.rename_database(old_name, new_name) 223 | if after_rename: 224 | after_rename(old_name, new_name) 225 | 226 | def delete_orphan_snapshots(self, after_delete=None): 227 | stellar_databases = set() 228 | for snapshot in self.db.session.query(Snapshot): 229 | for table in snapshot.tables: 230 | stellar_databases.add(table.get_table_name('master')) 231 | stellar_databases.add(table.get_table_name('slave')) 232 | 233 | databases = set(self.operations.list_of_databases()) 234 | 235 | for database in filter( 236 | lambda database: ( 237 | database.startswith('stellar_') and 238 | database != 'stellar_data' 239 | ), 240 | (databases-stellar_databases) 241 | ): 242 | self.operations.remove_database(database) 243 | if after_delete: 244 | after_delete(database) 245 | 246 | @property 247 | def default_snapshot_name(self): 248 | n = 1 249 | while self.db.session.query(Snapshot).filter( 250 | Snapshot.snapshot_name == 'snap%d' % n, 251 | Snapshot.project_name == self.config['project_name'] 252 | ).count(): 253 | n += 1 254 | return 'snap%d' % n 255 | -------------------------------------------------------------------------------- /stellar/command.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from datetime import datetime 3 | from time import sleep 4 | 5 | import humanize 6 | import click 7 | import logging 8 | from sqlalchemy import create_engine 9 | from sqlalchemy.exc import OperationalError 10 | 11 | from .app import Stellar, __version__ 12 | from .config import InvalidConfig, MissingConfig, load_config, save_config 13 | from .operations import database_exists, list_of_databases, SUPPORTED_DIALECTS 14 | 15 | 16 | def upgrade_from_old_version(app): 17 | if app.config['migrate_from_0_3_2']: 18 | if app.is_old_database(): 19 | click.echo('Upgrading from old Stellar version...') 20 | def after_rename(old_name, new_name): 21 | click.echo('* Renamed %s to %s' % (old_name, new_name)) 22 | app.update_database_names_to_new_version(after_rename=after_rename) 23 | 24 | app.config['migrate_from_0_3_2'] = False 25 | save_config(app.config) 26 | 27 | def get_app(): 28 | app = Stellar() 29 | upgrade_from_old_version(app) 30 | return app 31 | 32 | 33 | @click.group() 34 | def stellar(): 35 | """Fast database snapshots for development. It's like Git for databases.""" 36 | pass 37 | 38 | 39 | @stellar.command() 40 | def version(): 41 | """Shows version number""" 42 | click.echo("Stellar %s" % __version__) 43 | 44 | 45 | @stellar.command() 46 | def gc(): 47 | """Deletes old stellar tables that are not used anymore""" 48 | def after_delete(database): 49 | click.echo("Deleted table %s" % database) 50 | 51 | app = get_app() 52 | upgrade_from_old_version(app) 53 | app.delete_orphan_snapshots(after_delete) 54 | 55 | 56 | @stellar.command() 57 | @click.argument('name', required=False) 58 | def snapshot(name): 59 | """Takes a snapshot of the database""" 60 | app = get_app() 61 | upgrade_from_old_version(app) 62 | name = name or app.default_snapshot_name 63 | 64 | if app.get_snapshot(name): 65 | click.echo("Snapshot with name %s already exists" % name) 66 | sys.exit(1) 67 | else: 68 | def before_copy(table_name): 69 | click.echo("Snapshotting database %s" % table_name) 70 | app.create_snapshot(name, before_copy=before_copy) 71 | 72 | 73 | @stellar.command() 74 | def list(): 75 | """Returns a list of snapshots""" 76 | snapshots = get_app().get_snapshots() 77 | 78 | click.echo('\n'.join( 79 | '%s: %s' % ( 80 | s.snapshot_name, 81 | humanize.naturaltime(datetime.utcnow() - s.created_at) 82 | ) 83 | for s in snapshots 84 | )) 85 | 86 | 87 | @stellar.command() 88 | @click.argument('name', required=False) 89 | def restore(name): 90 | """Restores the database from a snapshot""" 91 | app = get_app() 92 | 93 | if not name: 94 | snapshot = app.get_latest_snapshot() 95 | if not snapshot: 96 | click.echo( 97 | "Couldn't find any snapshots for project %s" % 98 | load_config()['project_name'] 99 | ) 100 | sys.exit(1) 101 | else: 102 | snapshot = app.get_snapshot(name) 103 | if not snapshot: 104 | click.echo( 105 | "Couldn't find snapshot with name %s.\n" 106 | "You can list snapshots with 'stellar list'" % name 107 | ) 108 | sys.exit(1) 109 | 110 | # Check if slaves are ready 111 | if not snapshot.slaves_ready: 112 | if app.is_copy_process_running(snapshot): 113 | sys.stdout.write( 114 | 'Waiting for background process(%s) to finish' % 115 | snapshot.worker_pid 116 | ) 117 | sys.stdout.flush() 118 | while not snapshot.slaves_ready: 119 | sys.stdout.write('.') 120 | sys.stdout.flush() 121 | sleep(1) 122 | app.db.session.refresh(snapshot) 123 | click.echo('') 124 | else: 125 | click.echo('Background process missing, doing slow restore.') 126 | app.inline_slave_copy(snapshot) 127 | 128 | app.restore(snapshot) 129 | click.echo('Restore complete.') 130 | 131 | 132 | @stellar.command() 133 | @click.argument('name') 134 | def remove(name): 135 | """Removes a snapshot""" 136 | app = get_app() 137 | 138 | snapshot = app.get_snapshot(name) 139 | if not snapshot: 140 | click.echo("Couldn't find snapshot %s" % name) 141 | sys.exit(1) 142 | 143 | click.echo("Deleting snapshot %s" % name) 144 | app.remove_snapshot(snapshot) 145 | click.echo("Deleted") 146 | 147 | 148 | @stellar.command() 149 | @click.argument('old_name') 150 | @click.argument('new_name') 151 | def rename(old_name, new_name): 152 | """Renames a snapshot""" 153 | app = get_app() 154 | 155 | snapshot = app.get_snapshot(old_name) 156 | if not snapshot: 157 | click.echo("Couldn't find snapshot %s" % old_name) 158 | sys.exit(1) 159 | 160 | new_snapshot = app.get_snapshot(new_name) 161 | if new_snapshot: 162 | click.echo("Snapshot with name %s already exists" % new_name) 163 | sys.exit(1) 164 | 165 | app.rename_snapshot(snapshot, new_name) 166 | click.echo("Renamed snapshot %s to %s" % (old_name, new_name)) 167 | 168 | 169 | @stellar.command() 170 | @click.argument('name') 171 | def replace(name): 172 | """Replaces a snapshot""" 173 | app = get_app() 174 | 175 | snapshot = app.get_snapshot(name) 176 | if not snapshot: 177 | click.echo("Couldn't find snapshot %s" % name) 178 | sys.exit(1) 179 | 180 | app.remove_snapshot(snapshot) 181 | app.create_snapshot(name) 182 | click.echo("Replaced snapshot %s" % name) 183 | 184 | 185 | @stellar.command() 186 | def init(): 187 | """Initializes Stellar configuration.""" 188 | while True: 189 | url = click.prompt( 190 | "Please enter the url for your database.\n\n" 191 | "For example:\n" 192 | "PostgreSQL: postgresql://localhost:5432/\n" 193 | "MySQL: mysql+pymysql://root@localhost/" 194 | ) 195 | if url.count('/') == 2 and not url.endswith('/'): 196 | url = url + '/' 197 | 198 | if ( 199 | url.count('/') == 3 and 200 | url.endswith('/') and 201 | url.startswith('postgresql://') 202 | ): 203 | connection_url = url + 'template1' 204 | else: 205 | connection_url = url 206 | 207 | engine = create_engine(connection_url, echo=False) 208 | try: 209 | conn = engine.connect() 210 | except OperationalError as err: 211 | click.echo("Could not connect to database: %s" % url) 212 | click.echo("Error message: %s" % err.message) 213 | click.echo('') 214 | else: 215 | break 216 | 217 | if engine.dialect.name not in SUPPORTED_DIALECTS: 218 | click.echo("Your engine dialect %s is not supported." % ( 219 | engine.dialect.name 220 | )) 221 | click.echo("Supported dialects: %s" % ( 222 | ', '.join(SUPPORTED_DIALECTS) 223 | )) 224 | 225 | if url.count('/') == 3 and url.endswith('/'): 226 | while True: 227 | click.echo("You have the following databases: %s" % ', '.join([ 228 | db for db in list_of_databases(conn) 229 | if not db.startswith('stellar_') 230 | ])) 231 | 232 | db_name = click.prompt( 233 | "Please enter the name of the database (eg. projectdb)" 234 | ) 235 | if database_exists(conn, db_name): 236 | break 237 | else: 238 | click.echo("Could not find database %s" % db_name) 239 | click.echo('') 240 | else: 241 | db_name = url.rsplit('/', 1)[-1] 242 | url = url.rsplit('/', 1)[0] + '/' 243 | 244 | name = click.prompt( 245 | 'Please enter your project name (used internally, eg. %s)' % db_name, 246 | default=db_name 247 | ) 248 | 249 | raw_url = url 250 | 251 | if engine.dialect.name == 'postgresql': 252 | raw_url = raw_url + 'template1' 253 | 254 | with open('stellar.yaml', 'w') as project_file: 255 | project_file.write( 256 | """ 257 | project_name: '%(name)s' 258 | tracked_databases: ['%(db_name)s'] 259 | url: '%(raw_url)s' 260 | stellar_url: '%(url)sstellar_data' 261 | """.strip() % 262 | { 263 | 'name': name, 264 | 'raw_url': raw_url, 265 | 'url': url, 266 | 'db_name': db_name 267 | } 268 | ) 269 | 270 | click.echo("Wrote stellar.yaml") 271 | click.echo('') 272 | if engine.dialect.name == 'mysql': 273 | click.echo("Warning: MySQL support is still in beta.") 274 | click.echo("Tip: You probably want to take a snapshot: stellar snapshot") 275 | 276 | 277 | def main(): 278 | try: 279 | stellar() 280 | except MissingConfig: 281 | click.echo("You don't have stellar.yaml configuration yet.") 282 | click.echo("Initialize it by running: stellar init") 283 | sys.exit(1) 284 | except InvalidConfig as e: 285 | click.echo("Your stellar.yaml configuration is wrong: %s" % e.message) 286 | sys.exit(1) 287 | except ImportError as e: 288 | libraries = { 289 | 'psycopg2': 'PostreSQL', 290 | 'pymysql': 'MySQL', 291 | } 292 | for library, name in libraries.items(): 293 | if 'No module named' in str(e) and library in str(e): 294 | click.echo( 295 | "Python library %s is required for %s support." % 296 | (library, name) 297 | ) 298 | click.echo("You can install it with pip:") 299 | click.echo("pip install %s" % library) 300 | sys.exit(1) 301 | elif 'No module named' in str(e) and 'MySQLdb' in str(e): 302 | click.echo( 303 | "MySQLdb binary drivers are required for MySQL support. " 304 | "You can try installing it with these instructions: " 305 | "http://stackoverflow.com/questions/454854/no-module-named" 306 | "-mysqldb" 307 | ) 308 | click.echo('') 309 | click.echo("Alternatively you can use pymysql instead:") 310 | click.echo("1. Install it first: pip install pymysql") 311 | click.echo( 312 | "2. Specify database url as " 313 | "mysql+pymysql://root@localhost/ and not as " 314 | "mysql://root@localhost/" 315 | ) 316 | sys.exit(1) 317 | raise 318 | 319 | if __name__ == '__main__': 320 | main() 321 | --------------------------------------------------------------------------------