├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.rst ├── backupdb ├── __init__.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── backupdb.py │ │ └── restoredb.py ├── tests │ ├── __init__.py │ ├── app │ │ ├── models.py │ │ └── urls.py │ ├── commands.py │ ├── files.py │ ├── log.py │ ├── processes.py │ └── utils.py └── utils │ ├── __init__.py │ ├── commands.py │ ├── exceptions.py │ ├── files.py │ ├── log.py │ ├── processes.py │ └── settings.py ├── end_to_end_requirements.txt ├── setup.py ├── test_settings.py └── unit_tests_scratch └── .gitkeep /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | doc/_build 4 | *.egg 5 | *.egg-info 6 | build/ 7 | dist/ 8 | test_backupdb_database 9 | backups/ 10 | .coverage 11 | .b_hook 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.3" 4 | - "2.7" 5 | - "2.6" 6 | env: 7 | - DJANGO="Django==1.4" 8 | - DJANGO="Django==1.5" 9 | - DJANGO="Django==1.6" 10 | - DJANGO="https://github.com/django/django/archive/1.7a2.tar.gz#egg=Django" 11 | matrix: 12 | exclude: 13 | # Python 2.6 support has been dropped in Django 1.7 14 | - python: "2.6" 15 | env: DJANGO="https://github.com/django/django/archive/1.7a2.tar.gz#egg=Django" 16 | # Support for python 3 was added in Django 1.5 17 | - python: "3.3" 18 | env: DJANGO="Django==1.4" 19 | install: pip install $DJANGO 20 | script: python setup.py test 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, 2013, 2014, Fusionbox, Inc. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this 7 | list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 17 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 20 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 21 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 22 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include LICENSE 3 | include test_settings.py 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | django-backupdb 3 | =============== 4 | 5 | Management commands for automatically backing up and restoring all databases 6 | defined in ``settings.DATABASES``. 7 | 8 | License 9 | ------- 10 | 11 | Please take a moment to read the license specified in the ``LICENSE`` file. 12 | -------------------------------------------------------------------------------- /backupdb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fusionbox/django-backupdb/db4aa73049303245ef0182cda5c76b1dd194cd00/backupdb/__init__.py -------------------------------------------------------------------------------- /backupdb/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fusionbox/django-backupdb/db4aa73049303245ef0182cda5c76b1dd194cd00/backupdb/management/__init__.py -------------------------------------------------------------------------------- /backupdb/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fusionbox/django-backupdb/db4aa73049303245ef0182cda5c76b1dd194cd00/backupdb/management/commands/__init__.py -------------------------------------------------------------------------------- /backupdb/management/commands/backupdb.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from optparse import make_option 3 | from subprocess import CalledProcessError 4 | import logging 5 | import os 6 | import time 7 | 8 | from backupdb.utils.commands import BaseBackupDbCommand, do_postgresql_backup 9 | from backupdb.utils.exceptions import BackupError 10 | from backupdb.utils.log import section, SectionError, SectionWarning 11 | from backupdb.utils.settings import BACKUP_DIR, BACKUP_CONFIG 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class Command(BaseBackupDbCommand): 17 | help = 'Backs up each database in settings.DATABASES.' 18 | 19 | def add_arguments(self, parser): 20 | parser.add_argument( 21 | '--backup-name', 22 | help=( 23 | 'Specify a name for the backup. Defaults to the current ' 24 | 'timestamp. Example: `--backup-name=test` will create backup ' 25 | 'files that look like "default-test.pgsql.gz".' 26 | ), 27 | ) 28 | parser.add_argument( 29 | '--pg-dump-options', 30 | help=( 31 | 'For postgres backups, a list of additional options that will ' 32 | 'be passed through to the pg_dump utility. Example: ' 33 | '`--pg-dump-options="--inserts --no-owner"`' 34 | ), 35 | ) 36 | parser.add_argument( 37 | '--show-output', 38 | action='store_true', 39 | default=False, 40 | help=( 41 | 'Display the output of stderr and stdout (apart from data which ' 42 | 'is piped from one process to another) for processes that are ' 43 | 'run while backing up databases. These are command-line ' 44 | 'programs such as `psql` or `mysql`. This can be useful for ' 45 | 'understanding how backups are failing in the case that they ' 46 | 'are.' 47 | ), 48 | ) 49 | 50 | def handle(self, *args, **options): 51 | super(Command, self).handle(*args, **options) 52 | 53 | from django.conf import settings 54 | 55 | current_time = time.strftime('%F-%s') 56 | backup_name = options['backup_name'] or current_time 57 | show_output = options['show_output'] 58 | 59 | # Ensure backup dir present 60 | if not os.path.exists(BACKUP_DIR): 61 | os.makedirs(BACKUP_DIR) 62 | 63 | # Loop through databases 64 | for db_name, db_config in settings.DATABASES.items(): 65 | with section("Backing up '{0}'...".format(db_name)): 66 | # Get backup config for this engine type 67 | engine = db_config['ENGINE'] 68 | backup_config = BACKUP_CONFIG.get(engine) 69 | if not backup_config: 70 | raise SectionWarning("Backup for '{0}' engine not implemented".format(engine)) 71 | 72 | # Get backup file name 73 | backup_base_name = '{db_name}-{backup_name}.{backup_extension}.gz'.format( 74 | db_name=db_name, 75 | backup_name=backup_name, 76 | backup_extension=backup_config['backup_extension'], 77 | ) 78 | backup_file = os.path.join(BACKUP_DIR, backup_base_name) 79 | 80 | # Find backup command and get kwargs 81 | backup_func = backup_config['backup_func'] 82 | backup_kwargs = { 83 | 'backup_file': backup_file, 84 | 'db_config': db_config, 85 | 'show_output': show_output, 86 | } 87 | if backup_func is do_postgresql_backup: 88 | backup_kwargs['pg_dump_options'] = options['pg_dump_options'] 89 | 90 | # Run backup command 91 | try: 92 | backup_func(**backup_kwargs) 93 | logger.info("Backup of '{db_name}' saved in '{backup_file}'".format( 94 | db_name=db_name, 95 | backup_file=backup_file)) 96 | except (BackupError, CalledProcessError) as e: 97 | raise SectionError(e) 98 | -------------------------------------------------------------------------------- /backupdb/management/commands/restoredb.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from optparse import make_option 3 | from subprocess import CalledProcessError 4 | import logging 5 | import os 6 | 7 | from django.core.management.base import CommandError 8 | from django.conf import settings 9 | 10 | from backupdb.utils.commands import BaseBackupDbCommand 11 | from backupdb.utils.exceptions import RestoreError 12 | from backupdb.utils.files import get_latest_timestamped_file 13 | from backupdb.utils.log import section, SectionError, SectionWarning 14 | from backupdb.utils.settings import BACKUP_DIR, BACKUP_CONFIG 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class Command(BaseBackupDbCommand): 20 | help = 'Restores each database in settings.DATABASES from latest db backup.' 21 | 22 | def add_arguments(self, parser): 23 | 24 | parser.add_argument( 25 | '--backup-name', 26 | help=( 27 | 'Name of backup to restore from. Example: ' 28 | '`--backup-name=mybackup` will restore any backups that look ' 29 | 'like "default-mybackup.pgsql.gz". Defaults to latest ' 30 | 'timestamped backup name.' 31 | ), 32 | ) 33 | parser.add_argument( 34 | '--drop-tables', 35 | action='store_true', 36 | default=False, 37 | help=( 38 | '** USE WITH CAUTION ** Drop all tables in databases before ' 39 | 'restoring them. The SQL dumps which are created by backupdb ' 40 | 'have drop statements so, usually, this option is not ' 41 | 'necessary.' 42 | ), 43 | ) 44 | parser.add_argument( 45 | '--show-output', 46 | action='store_true', 47 | default=False, 48 | help=( 49 | 'Display the output of stderr and stdout (apart from data which ' 50 | 'is piped from one process to another) for processes that are ' 51 | 'run while restoring databases. These are command-line ' 52 | 'programs such as `psql` or `mysql`. This can be useful for ' 53 | 'understanding how restoration is failing in the case that it ' 54 | 'is.' 55 | ), 56 | ) 57 | 58 | def handle(self, *args, **options): 59 | super(Command, self).handle(*args, **options) 60 | 61 | # Ensure backup dir present 62 | if not os.path.exists(BACKUP_DIR): 63 | raise CommandError("Backup dir '{0}' does not exist!".format(BACKUP_DIR)) 64 | 65 | backup_name = options['backup_name'] 66 | drop_tables = options['drop_tables'] 67 | show_output = options['show_output'] 68 | 69 | # Loop through databases 70 | for db_name, db_config in settings.DATABASES.items(): 71 | with section("Restoring '{0}'...".format(db_name)): 72 | # Get backup config for this engine type 73 | engine = db_config['ENGINE'] 74 | backup_config = BACKUP_CONFIG.get(engine) 75 | if not backup_config: 76 | raise SectionWarning("Restore for '{0}' engine not implemented".format(engine)) 77 | 78 | # Get backup file name 79 | backup_extension = backup_config['backup_extension'] 80 | if backup_name: 81 | backup_file = '{dir}/{db_name}-{backup_name}.{ext}.gz'.format( 82 | dir=BACKUP_DIR, 83 | db_name=db_name, 84 | backup_name=backup_name, 85 | ext=backup_extension, 86 | ) 87 | else: 88 | try: 89 | backup_file = get_latest_timestamped_file(backup_extension) 90 | except RestoreError as e: 91 | raise SectionError(e) 92 | 93 | # Find restore command and get kwargs 94 | restore_func = backup_config['restore_func'] 95 | restore_kwargs = { 96 | 'backup_file': backup_file, 97 | 'db_config': db_config, 98 | 'drop_tables': drop_tables, 99 | 'show_output': show_output, 100 | } 101 | 102 | # Run restore command 103 | try: 104 | restore_func(**restore_kwargs) 105 | logger.info("Restored '{db_name}' from '{backup_file}'".format( 106 | db_name=db_name, 107 | backup_file=backup_file)) 108 | except (RestoreError, CalledProcessError) as e: 109 | raise SectionError(e) 110 | -------------------------------------------------------------------------------- /backupdb/tests/__init__.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from . import commands 4 | from . import files 5 | from . import log 6 | from . import processes 7 | 8 | 9 | loader = unittest.TestLoader() 10 | 11 | commands_tests = loader.loadTestsFromModule(commands) 12 | files_tests = loader.loadTestsFromModule(files) 13 | log_tests = loader.loadTestsFromModule(log) 14 | processes_tests = loader.loadTestsFromModule(processes) 15 | 16 | all_tests = unittest.TestSuite([ 17 | commands_tests, 18 | files_tests, 19 | log_tests, 20 | processes_tests, 21 | ]) 22 | -------------------------------------------------------------------------------- /backupdb/tests/app/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fusionbox/django-backupdb/db4aa73049303245ef0182cda5c76b1dd194cd00/backupdb/tests/app/models.py -------------------------------------------------------------------------------- /backupdb/tests/app/urls.py: -------------------------------------------------------------------------------- 1 | 2 | urlpatterns = () 3 | -------------------------------------------------------------------------------- /backupdb/tests/commands.py: -------------------------------------------------------------------------------- 1 | from mock import call, patch 2 | import unittest 3 | 4 | from backupdb.utils.commands import ( 5 | PG_DROP_SQL, 6 | get_mysql_args, 7 | get_postgresql_args, 8 | get_postgresql_env, 9 | do_mysql_backup, 10 | do_postgresql_backup, 11 | do_sqlite_backup, 12 | do_mysql_restore, 13 | do_postgresql_restore, 14 | do_sqlite_restore, 15 | ) 16 | from backupdb.utils.exceptions import RestoreError 17 | 18 | 19 | DB_CONFIG = { 20 | 'NAME': 'test_db', 21 | 'USER': 'test_user', 22 | 'PASSWORD': 'test_password', 23 | 'HOST': 'test_host', 24 | 'PORT': 12345, 25 | } 26 | 27 | 28 | def make_db_config(*keys): 29 | new_dict = {} 30 | for k in keys: 31 | new_dict[k] = DB_CONFIG[k] 32 | return new_dict 33 | 34 | 35 | class MockOsPathExists(object): 36 | """ 37 | Used as a mock object for os.path.exists. 38 | """ 39 | def __init__(self, *args, **kwargs): 40 | pass 41 | 42 | def __call__(self, *args, **kwargs): 43 | return True 44 | 45 | 46 | class PatchPipeCommandsTestCase(unittest.TestCase): 47 | """ 48 | Used for testing of pipe_commands and pipe_commands_to_file. 49 | """ 50 | def setUp(self): 51 | self.pipe_commands_patcher = patch('backupdb.utils.commands.pipe_commands') 52 | self.pipe_commands_to_file_patcher = patch('backupdb.utils.commands.pipe_commands_to_file') 53 | self.os_patcher = patch('os.path.exists', new_callable=MockOsPathExists) 54 | 55 | self.mock_pipe_commands = self.pipe_commands_patcher.start() 56 | self.mock_pipe_commands_to_file = self.pipe_commands_to_file_patcher.start() 57 | self.mock_os = self.os_patcher.start() 58 | 59 | def tearDown(self): 60 | self.pipe_commands_patcher.stop() 61 | self.pipe_commands_to_file_patcher.stop() 62 | self.os_patcher.stop() 63 | 64 | def assertPipeCommandsCallsEqual(self, *args): 65 | self.assertEqual(self.mock_pipe_commands.call_args_list, list(args)) 66 | 67 | def assertPipeCommandsToFileCallsEqual(self, *args): 68 | self.assertEqual(self.mock_pipe_commands_to_file.call_args_list, list(args)) 69 | 70 | 71 | class RequireBackupExistsTestCase(unittest.TestCase): 72 | def test_it_raises_an_exception_when_the_path_in_backup_file_arg_doesnt_exist(self): 73 | self.assertRaises(RestoreError, do_mysql_restore, backup_file='i_dont_exist', db_config={}) 74 | self.assertRaises(RestoreError, do_postgresql_restore, backup_file='i_dont_exist', db_config={}) 75 | self.assertRaises(RestoreError, do_sqlite_restore, backup_file='i_dont_exist', db_config={}) 76 | 77 | 78 | class GetMysqlArgsTestCase(unittest.TestCase): 79 | def test_it_builds_the_correct_args(self): 80 | self.assertEqual( 81 | get_mysql_args(make_db_config('NAME', 'USER')), 82 | [ 83 | '--user=test_user', 84 | 'test_db', 85 | ], 86 | ) 87 | self.assertEqual( 88 | get_mysql_args(make_db_config('NAME', 'USER', 'PASSWORD')), 89 | [ 90 | '--user=test_user', 91 | '--password=test_password', 92 | 'test_db', 93 | ], 94 | ) 95 | self.assertEqual( 96 | get_mysql_args(make_db_config('NAME', 'USER', 'PASSWORD', 'HOST')), 97 | [ 98 | '--user=test_user', 99 | '--password=test_password', 100 | '--host=test_host', 101 | 'test_db', 102 | ], 103 | ) 104 | self.assertEqual( 105 | get_mysql_args(make_db_config('NAME', 'USER', 'PASSWORD', 'HOST', 'PORT')), 106 | [ 107 | '--user=test_user', 108 | '--password=test_password', 109 | '--host=test_host', 110 | '--port=12345', 111 | 'test_db', 112 | ], 113 | ) 114 | 115 | 116 | class GetPostgresqlArgsTestCase(unittest.TestCase): 117 | def test_it_builds_the_correct_args(self): 118 | self.assertEqual( 119 | get_postgresql_args(make_db_config('NAME', 'USER')), 120 | [ 121 | '--username=test_user', 122 | 'test_db', 123 | ], 124 | ) 125 | self.assertEqual( 126 | get_postgresql_args(make_db_config('NAME', 'USER', 'HOST')), 127 | [ 128 | '--username=test_user', 129 | '--host=test_host', 130 | 'test_db', 131 | ], 132 | ) 133 | self.assertEqual( 134 | get_postgresql_args(make_db_config('NAME', 'USER', 'HOST', 'PORT')), 135 | [ 136 | '--username=test_user', 137 | '--host=test_host', 138 | '--port=12345', 139 | 'test_db', 140 | ], 141 | ) 142 | 143 | def test_it_correctly_includes_extra_args(self): 144 | self.assertEqual( 145 | get_postgresql_args( 146 | make_db_config('NAME', 'USER'), 147 | extra_args='--no-owner --no-privileges', 148 | ), 149 | [ 150 | '--username=test_user', 151 | '--no-owner', 152 | '--no-privileges', 153 | 'test_db', 154 | ], 155 | ) 156 | 157 | 158 | class GetPostgresqlEnvTestCase(unittest.TestCase): 159 | def test_it_builds_the_correct_env_dict(self): 160 | self.assertTrue( 161 | get_postgresql_env(make_db_config('USER', 'NAME')) is None, 162 | ) 163 | self.assertEqual( 164 | get_postgresql_env(make_db_config('USER', 'NAME', 'PASSWORD')), 165 | {'PGPASSWORD': 'test_password'}, 166 | ) 167 | 168 | 169 | class DoMysqlBackupTestCase(PatchPipeCommandsTestCase): 170 | def test_it_makes_correct_calls_to_processes_api(self): 171 | do_mysql_backup('test.mysql.gz', DB_CONFIG) 172 | 173 | self.assertPipeCommandsToFileCallsEqual(call( 174 | [ 175 | [ 176 | 'mysqldump', 177 | '--user=test_user', 178 | '--password=test_password', 179 | '--host=test_host', 180 | '--port=12345', 181 | 'test_db', 182 | ], 183 | ['gzip'], 184 | ], 185 | path='test.mysql.gz', 186 | show_stderr=False, 187 | )) 188 | 189 | 190 | class DoPostgresqlBackupTestCase(PatchPipeCommandsTestCase): 191 | def test_it_makes_correct_calls_to_processes_api(self): 192 | do_postgresql_backup('test.pgsql.gz', DB_CONFIG) 193 | 194 | self.assertPipeCommandsToFileCallsEqual(call( 195 | [ 196 | [ 197 | 'pg_dump', 198 | '--clean', 199 | '--username=test_user', 200 | '--host=test_host', 201 | '--port=12345', 202 | 'test_db', 203 | ], 204 | ['gzip'], 205 | ], 206 | path='test.pgsql.gz', 207 | extra_env={'PGPASSWORD': 'test_password'}, 208 | show_stderr=False, 209 | )) 210 | 211 | 212 | class DoSqliteBackupTestCase(PatchPipeCommandsTestCase): 213 | def test_it_makes_correct_calls_to_processes_api(self): 214 | do_sqlite_backup('test.sqlite.gz', DB_CONFIG) 215 | 216 | self.assertPipeCommandsToFileCallsEqual(call( 217 | [['cat', 'test_db'], ['gzip']], 218 | path='test.sqlite.gz', 219 | show_stderr=False, 220 | )) 221 | 222 | 223 | class DoMysqlRestoreTestCase(PatchPipeCommandsTestCase): 224 | def test_it_makes_correct_calls_to_processes_api(self): 225 | do_mysql_restore(backup_file='test.mysql.gz', db_config=DB_CONFIG) 226 | 227 | self.assertPipeCommandsCallsEqual(call( 228 | [ 229 | ['cat', 'test.mysql.gz'], 230 | ['gunzip'], 231 | [ 232 | 'mysql', 233 | '--user=test_user', 234 | '--password=test_password', 235 | '--host=test_host', 236 | '--port=12345', 237 | 'test_db', 238 | ], 239 | ], 240 | show_stderr=False, 241 | show_last_stdout=False, 242 | )) 243 | 244 | def test_it_drops_tables_before_restoring_if_specified(self): 245 | do_mysql_restore(backup_file='test.mysql.gz', db_config=DB_CONFIG, drop_tables=True) 246 | 247 | self.assertPipeCommandsCallsEqual( 248 | call( 249 | [ 250 | [ 251 | 'mysqldump', 252 | '--user=test_user', 253 | '--password=test_password', 254 | '--host=test_host', 255 | '--port=12345', 256 | 'test_db', 257 | '--no-data', 258 | ], 259 | ['grep', '^DROP'], 260 | [ 261 | 'mysql', 262 | '--user=test_user', 263 | '--password=test_password', 264 | '--host=test_host', 265 | '--port=12345', 266 | 'test_db', 267 | ], 268 | ], 269 | show_stderr=False, 270 | show_last_stdout=False, 271 | ), 272 | call( 273 | [ 274 | ['cat', 'test.mysql.gz'], 275 | ['gunzip'], 276 | [ 277 | 'mysql', 278 | '--user=test_user', 279 | '--password=test_password', 280 | '--host=test_host', 281 | '--port=12345', 282 | 'test_db', 283 | ], 284 | ], 285 | show_stderr=False, 286 | show_last_stdout=False, 287 | ), 288 | ) 289 | 290 | 291 | class DoPostgresqlRestoreTestCase(PatchPipeCommandsTestCase): 292 | def test_it_makes_correct_calls_to_processes_api(self): 293 | do_postgresql_restore(backup_file='test.pgsql.gz', db_config=DB_CONFIG) 294 | 295 | self.assertPipeCommandsCallsEqual(call( 296 | [ 297 | ['cat', 'test.pgsql.gz'], 298 | ['gunzip'], 299 | [ 300 | 'psql', 301 | '--username=test_user', 302 | '--host=test_host', 303 | '--port=12345', 304 | 'test_db', 305 | ], 306 | ], 307 | extra_env={'PGPASSWORD': 'test_password'}, 308 | show_stderr=False, 309 | show_last_stdout=False, 310 | )) 311 | 312 | def test_it_drops_tables_before_restoring_if_specified(self): 313 | do_postgresql_restore(backup_file='test.pgsql.gz', db_config=DB_CONFIG, drop_tables=True) 314 | 315 | self.assertPipeCommandsCallsEqual( 316 | call( 317 | [ 318 | [ 319 | 'psql', 320 | '--username=test_user', 321 | '--host=test_host', 322 | '--port=12345', 323 | 'test_db', 324 | '-t', 325 | '-c', 326 | PG_DROP_SQL, 327 | ], 328 | [ 329 | 'psql', 330 | '--username=test_user', 331 | '--host=test_host', 332 | '--port=12345', 333 | 'test_db', 334 | ], 335 | ], 336 | extra_env={'PGPASSWORD': 'test_password'}, 337 | show_stderr=False, 338 | show_last_stdout=False, 339 | ), 340 | call( 341 | [ 342 | ['cat', 'test.pgsql.gz'], 343 | ['gunzip'], 344 | [ 345 | 'psql', 346 | '--username=test_user', 347 | '--host=test_host', 348 | '--port=12345', 349 | 'test_db', 350 | ], 351 | ], 352 | extra_env={'PGPASSWORD': 'test_password'}, 353 | show_stderr=False, 354 | show_last_stdout=False, 355 | ), 356 | ) 357 | 358 | 359 | class DoSqliteRestoreTestCase(PatchPipeCommandsTestCase): 360 | def test_it_makes_correct_calls_to_processes_api(self): 361 | do_sqlite_restore(backup_file='test.sqlite.gz', db_config=DB_CONFIG) 362 | 363 | self.assertPipeCommandsToFileCallsEqual(call( 364 | [['cat', 'test.sqlite.gz'], ['gunzip']], 365 | path='test_db', 366 | show_stderr=False, 367 | )) 368 | -------------------------------------------------------------------------------- /backupdb/tests/files.py: -------------------------------------------------------------------------------- 1 | from backupdb.utils.exceptions import RestoreError 2 | from backupdb.utils.files import get_latest_timestamped_file 3 | 4 | from backupdb.tests.utils import FileSystemScratchTestCase 5 | 6 | 7 | class GetLatestTimestampedFileTestCase(FileSystemScratchTestCase): 8 | def create_files(self, *args): 9 | for file in args: 10 | open(self.get_path(file), 'a').close() 11 | 12 | def test_it_returns_the_latest_timestamped_file_with_ext(self): 13 | self.create_files( 14 | 'default-2013-05-02-1367553089.sqlite.gz', 15 | 'default-2013-06-06-1370570260.sqlite.gz', 16 | 'default-2013-06-06-1370580510.sqlite.gz', 17 | 18 | 'default-2013-05-02-1367553089.mysql.gz', 19 | 'default-2013-06-06-1370570260.mysql.gz', 20 | 'default-2013-06-06-1370580510.mysql.gz', 21 | 22 | 'default-2013-05-02-1367553089.pgsql.gz', 23 | 'default-2013-06-06-1370570260.pgsql.gz', 24 | 'default-2013-06-06-1370580510.pgsql.gz', 25 | ) 26 | 27 | sqlite_file = get_latest_timestamped_file('sqlite', dir=self.SCRATCH_DIR) 28 | mysql_file = get_latest_timestamped_file('mysql', dir=self.SCRATCH_DIR) 29 | pgsql_file = get_latest_timestamped_file('pgsql', dir=self.SCRATCH_DIR) 30 | 31 | self.assertEqual(sqlite_file, self.get_path('default-2013-06-06-1370580510.sqlite.gz')) 32 | self.assertEqual(mysql_file, self.get_path('default-2013-06-06-1370580510.mysql.gz')) 33 | self.assertEqual(pgsql_file, self.get_path('default-2013-06-06-1370580510.pgsql.gz')) 34 | 35 | def test_it_raises_an_exception_when_no_files_found(self): 36 | self.assertRaises(RestoreError, get_latest_timestamped_file, '', dir=self.SCRATCH_DIR) 37 | self.assertRaises(RestoreError, get_latest_timestamped_file, 'mysql', dir=self.SCRATCH_DIR) 38 | -------------------------------------------------------------------------------- /backupdb/tests/log.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from backupdb.utils.log import bar 4 | 5 | 6 | class BarTestCase(unittest.TestCase): 7 | def test_it_should_return_a_string_with_text_centered_in_a_bar(self): 8 | self.assertEqual( 9 | bar('test', width=70), 10 | '================================ test ================================', 11 | ) 12 | 13 | def test_it_should_return_a_bar_string_with_the_specified_width(self): 14 | test_bar1 = r'=========================== test ===========================' 15 | test_bar2 = r'//========================= test =========================\\' 16 | test_bar3 = r'\\========================= test =========================//' 17 | 18 | self.assertEqual(len(test_bar1), 60) 19 | self.assertEqual(len(test_bar2), 60) 20 | self.assertEqual(len(test_bar3), 60) 21 | 22 | self.assertEqual(bar('test', width=60), test_bar1) 23 | self.assertEqual(bar('test', width=60, position='top'), test_bar2) 24 | self.assertEqual(bar('test', width=60, position='bottom'), test_bar3) 25 | 26 | def test_it_should_work_even_if_the_given_width_is_less_than_the_message_length(self): 27 | self.assertEqual(bar('test', width=0), '== test ==') 28 | self.assertEqual(bar('test', width=0, position='top'), r'// test \\') 29 | self.assertEqual(bar('test', width=0, position='bottom'), r'\\ test //') 30 | 31 | def test_it_should_render_a_top_bottom_or_plain_bar_depending_on_the_position_argument(self): 32 | test_bar1 = r'================================ test ================================' 33 | test_bar2 = r'//============================== test ==============================\\' 34 | test_bar3 = r'\\============================== test ==============================//' 35 | 36 | self.assertEqual(bar('test', width=70), test_bar1) 37 | self.assertEqual(bar('test', width=70, position='top'), test_bar2) 38 | self.assertEqual(bar('test', width=70, position='bottom'), test_bar3) 39 | 40 | def test_it_should_allow_the_message_to_be_blank(self): 41 | test_bar1 = r'======================================================================' 42 | test_bar2 = r'//==================================================================\\' 43 | test_bar3 = r'\\==================================================================//' 44 | 45 | self.assertEqual(bar(width=70), test_bar1) 46 | self.assertEqual(bar(width=70, position='top'), test_bar2) 47 | self.assertEqual(bar(width=70, position='bottom'), test_bar3) 48 | -------------------------------------------------------------------------------- /backupdb/tests/processes.py: -------------------------------------------------------------------------------- 1 | from subprocess import CalledProcessError 2 | import os 3 | import unittest 4 | 5 | try: 6 | from collections import OrderedDict 7 | except ImportError: 8 | # This should only happen in Python 2.6 9 | # SortedDict is deprecated in Django 1.7 and will be removed in Django 1.9 10 | from django.utils.datastructures import SortedDict as OrderedDict 11 | 12 | from backupdb.utils.processes import ( 13 | extend_env, 14 | get_env_str, 15 | pipe_commands, 16 | pipe_commands_to_file, 17 | ) 18 | 19 | from .utils import FileSystemScratchTestCase 20 | 21 | 22 | class ExtendEnvTestCase(unittest.TestCase): 23 | def test_extend_env_creates_a_copy_of_the_current_env(self): 24 | env = extend_env({'BACKUPDB_TEST_ENV_SETTING': 1234}) 25 | self.assertFalse(env is os.environ) 26 | 27 | def test_extend_env_adds_keys_to_a_copy_of_the_current_env(self): 28 | env = extend_env({ 29 | 'BACKUPDB_TEST_ENV_SETTING_1': 1234, 30 | 'BACKUPDB_TEST_ENV_SETTING_2': 1234, 31 | }) 32 | 33 | orig_keys = set(os.environ.keys()) 34 | curr_keys = set(env.keys()) 35 | diff_keys = curr_keys - orig_keys 36 | 37 | self.assertEqual(diff_keys, set([ 38 | 'BACKUPDB_TEST_ENV_SETTING_1', 39 | 'BACKUPDB_TEST_ENV_SETTING_2', 40 | ])) 41 | 42 | 43 | class GetEnvStrTestCase(unittest.TestCase): 44 | def test_get_env_str_works_for_empty_dicts(self): 45 | self.assertEqual(get_env_str({}), '') 46 | 47 | def test_get_env_str_works_for_non_empty_dicts(self): 48 | self.assertEqual( 49 | get_env_str({'VAR_1': 1234}), 50 | "VAR_1='1234'", 51 | ) 52 | self.assertEqual( 53 | get_env_str(OrderedDict([ 54 | ('VAR_1', 1234), 55 | ('VAR_2', 'arst'), 56 | ])), 57 | "VAR_1='1234' VAR_2='arst'", 58 | ) 59 | self.assertEqual( 60 | get_env_str(OrderedDict([ 61 | ('VAR_1', 1234), 62 | ('VAR_2', 'arst'), 63 | ('VAR_3', 'zxcv'), 64 | ])), 65 | "VAR_1='1234' VAR_2='arst' VAR_3='zxcv'", 66 | ) 67 | 68 | 69 | class PipeCommandsTestCase(FileSystemScratchTestCase): 70 | def test_it_pipes_a_list_of_commands_into_each_other(self): 71 | pipe_commands([ 72 | ['echo', r""" 73 | import sys 74 | for i in range(4): 75 | sys.stdout.write('spam\n') 76 | """], 77 | ['python'], 78 | ['tee', self.get_path('pipe_commands.out')], 79 | ]) 80 | 81 | self.assertFileExists('pipe_commands.out') 82 | self.assertFileHasContent( 83 | 'pipe_commands.out', 84 | 'spam\nspam\nspam\nspam\n', 85 | ) 86 | 87 | def test_it_works_when_large_amounts_of_data_are_being_piped(self): 88 | pipe_commands([ 89 | ['echo', r""" 90 | import sys 91 | for i in range(400000): 92 | sys.stdout.write('spam\n') 93 | """], 94 | ['python'], 95 | ['tee', self.get_path('pipe_commands.out')], 96 | ]) 97 | 98 | self.assertFileExists('pipe_commands.out') 99 | self.assertFileHasLength('pipe_commands.out', 2000000) 100 | self.assertInFile('pipe_commands.out', 'spam\nspam\nspam\n') 101 | 102 | def test_it_allows_you_to_specify_extra_environment_variables(self): 103 | pipe_commands([ 104 | ['echo', """ 105 | import os 106 | import sys 107 | sys.stdout.write(os.environ['TEST_VAR']) 108 | """], 109 | ['python'], 110 | ['tee', self.get_path('pipe_commands.out')], 111 | ], extra_env={'TEST_VAR': 'spam'}) 112 | 113 | self.assertFileExists('pipe_commands.out') 114 | self.assertFileHasContent('pipe_commands.out', 'spam') 115 | 116 | def test_it_correctly_raises_a_called_process_error_when_necessary(self): 117 | self.assertRaises(CalledProcessError, pipe_commands, [['false'], ['true']]) 118 | 119 | 120 | class PipeCommandsToFileTestCase(FileSystemScratchTestCase): 121 | def test_it_pipes_a_list_of_commands_into_each_other_and_then_into_a_file(self): 122 | pipe_commands_to_file([ 123 | ['echo', r""" 124 | import sys 125 | for i in range(4): 126 | sys.stdout.write('spam\n') 127 | """], 128 | ['python'], 129 | ], self.get_path('pipe_commands.out')) 130 | 131 | self.assertFileExists('pipe_commands.out') 132 | self.assertFileHasContent( 133 | 'pipe_commands.out', 134 | 'spam\nspam\nspam\nspam\n', 135 | ) 136 | 137 | def test_it_works_when_large_amounts_of_data_are_being_piped(self): 138 | pipe_commands_to_file([ 139 | ['echo', r""" 140 | import sys 141 | for i in range(400000): 142 | sys.stdout.write('spam\n') 143 | """], 144 | ['python'], 145 | ], self.get_path('pipe_commands.out')) 146 | 147 | self.assertFileExists('pipe_commands.out') 148 | self.assertFileHasLength('pipe_commands.out', 2000000) 149 | self.assertInFile('pipe_commands.out', 'spam\nspam\nspam\n') 150 | 151 | def test_it_allows_you_to_specify_extra_environment_variables(self): 152 | pipe_commands_to_file([ 153 | ['echo', """ 154 | import os 155 | import sys 156 | sys.stdout.write(os.environ['TEST_VAR']) 157 | """], 158 | ['python'], 159 | ], self.get_path('pipe_commands.out'), extra_env={'TEST_VAR': 'spam'}) 160 | 161 | self.assertFileExists('pipe_commands.out') 162 | self.assertFileHasContent('pipe_commands.out', 'spam') 163 | 164 | def test_it_correctly_raises_a_called_process_error_when_necessary(self): 165 | self.assertRaises( 166 | CalledProcessError, 167 | pipe_commands_to_file, 168 | [['false'], ['true']], 169 | self.get_path('pipe_commands.out'), 170 | ) 171 | -------------------------------------------------------------------------------- /backupdb/tests/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | 4 | 5 | class FileSystemScratchTestCase(unittest.TestCase): 6 | SCRATCH_DIR = 'unit_tests_scratch' 7 | 8 | @classmethod 9 | def get_path(cls, file): 10 | return os.path.join(cls.SCRATCH_DIR, file) 11 | 12 | @classmethod 13 | def clear_scratch_dir(cls): 14 | """ 15 | Deletes all scratch files in the tests scratch directory. 16 | """ 17 | for file in os.listdir(cls.SCRATCH_DIR): 18 | if file != '.gitkeep': 19 | os.remove(cls.get_path(file)) 20 | 21 | def setUp(self): 22 | self.clear_scratch_dir() 23 | 24 | def tearDown(self): 25 | self.clear_scratch_dir() 26 | 27 | def assertFileExists(self, file): 28 | self.assertTrue(os.path.exists(self.get_path(file))) 29 | 30 | def assertFileHasLength(self, file, length): 31 | with open(self.get_path(file)) as f: 32 | content = f.read() 33 | self.assertEqual(len(content), length) 34 | 35 | def assertFileHasContent(self, file, expected_content): 36 | with open(self.get_path(file)) as f: 37 | actual_content = f.read() 38 | self.assertEqual(actual_content, expected_content) 39 | 40 | def assertInFile(self, file, expected_content): 41 | with open(self.get_path(file)) as f: 42 | actual_content = f.read() 43 | self.assertTrue(expected_content in actual_content) 44 | -------------------------------------------------------------------------------- /backupdb/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fusionbox/django-backupdb/db4aa73049303245ef0182cda5c76b1dd194cd00/backupdb/utils/__init__.py -------------------------------------------------------------------------------- /backupdb/utils/commands.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import shlex 4 | 5 | from django.core.management.base import BaseCommand 6 | 7 | from .exceptions import RestoreError 8 | from .processes import pipe_commands, pipe_commands_to_file 9 | 10 | 11 | PG_DROP_SQL = """SELECT 'DROP TABLE IF EXISTS "' || tablename || '" CASCADE;' FROM pg_tables WHERE schemaname = 'public';""" 12 | 13 | 14 | class BaseBackupDbCommand(BaseCommand): 15 | can_import_settings = True 16 | 17 | LOG_LEVELS = { 18 | 0: logging.ERROR, 19 | 1: logging.INFO, 20 | 2: logging.DEBUG, 21 | 3: logging.DEBUG, 22 | } 23 | LOG_FORMAT = '%(asctime)s - %(levelname)-8s: %(message)s' 24 | 25 | def _setup_logging(self, level): 26 | level = int(level) 27 | logging.basicConfig(format=self.LOG_FORMAT, level=self.LOG_LEVELS[level]) 28 | 29 | def handle(self, *args, **options): 30 | self._setup_logging(options['verbosity']) 31 | 32 | 33 | def apply_arg_values(arg_values): 34 | """ 35 | Apply argument to values. 36 | 37 | l = [('--name={0}', 'name'), 38 | ('--password={0}', 'password'), 39 | ('--level={0}', ''), 40 | ('--last={0}', None)] 41 | assert apply_arg_values(l) == ['--name=name', '--password=password'] 42 | """ 43 | return [a.format(v) for a, v in arg_values if v] 44 | 45 | 46 | def require_backup_exists(func): 47 | """ 48 | Requires that the file referred to by `backup_file` exists in the file 49 | system before running the decorated function. 50 | """ 51 | def new_func(*args, **kwargs): 52 | backup_file = kwargs['backup_file'] 53 | if not os.path.exists(backup_file): 54 | raise RestoreError("Could not find file '{0}'".format(backup_file)) 55 | return func(*args, **kwargs) 56 | return new_func 57 | 58 | 59 | def get_mysql_args(db_config): 60 | """ 61 | Returns an array of argument values that will be passed to a `mysql` or 62 | `mysqldump` process when it is started based on the given database 63 | configuration. 64 | """ 65 | db = db_config['NAME'] 66 | 67 | mapping = [('--user={0}', db_config.get('USER')), 68 | ('--password={0}', db_config.get('PASSWORD')), 69 | ('--host={0}', db_config.get('HOST')), 70 | ('--port={0}', db_config.get('PORT'))] 71 | args = apply_arg_values(mapping) 72 | args.append(db) 73 | 74 | return args 75 | 76 | 77 | def get_postgresql_args(db_config, extra_args=None): 78 | """ 79 | Returns an array of argument values that will be passed to a `psql` or 80 | `pg_dump` process when it is started based on the given database 81 | configuration. 82 | """ 83 | db = db_config['NAME'] 84 | 85 | mapping = [('--username={0}', db_config.get('USER')), 86 | ('--host={0}', db_config.get('HOST')), 87 | ('--port={0}', db_config.get('PORT'))] 88 | args = apply_arg_values(mapping) 89 | 90 | if extra_args is not None: 91 | args.extend(shlex.split(extra_args)) 92 | args.append(db) 93 | 94 | return args 95 | 96 | 97 | def get_postgresql_env(db_config): 98 | """ 99 | Returns a dict containing extra environment variable values that will be 100 | added to the environment of the `psql` or `pg_dump` process when it is 101 | started based on the given database configuration. 102 | """ 103 | password = db_config.get('PASSWORD') 104 | return {'PGPASSWORD': password} if password else None 105 | 106 | 107 | def do_mysql_backup(backup_file, db_config, show_output=False): 108 | args = get_mysql_args(db_config) 109 | 110 | cmd = ['mysqldump'] + args 111 | pipe_commands_to_file([cmd, ['gzip']], path=backup_file, show_stderr=show_output) 112 | 113 | 114 | def do_postgresql_backup(backup_file, db_config, pg_dump_options=None, show_output=False): 115 | env = get_postgresql_env(db_config) 116 | args = get_postgresql_args(db_config, pg_dump_options) 117 | 118 | cmd = ['pg_dump', '--clean'] + args 119 | pipe_commands_to_file([cmd, ['gzip']], path=backup_file, extra_env=env, show_stderr=show_output) 120 | 121 | 122 | def do_sqlite_backup(backup_file, db_config, show_output=False): 123 | db_file = db_config['NAME'] 124 | 125 | cmd = ['cat', db_file] 126 | pipe_commands_to_file([cmd, ['gzip']], path=backup_file, show_stderr=show_output) 127 | 128 | 129 | @require_backup_exists 130 | def do_mysql_restore(backup_file, db_config, drop_tables=False, show_output=False): 131 | args = get_mysql_args(db_config) 132 | mysql_cmd = ['mysql'] + args 133 | 134 | kwargs = {'show_stderr': show_output, 'show_last_stdout': show_output} 135 | 136 | if drop_tables: 137 | dump_cmd = ['mysqldump'] + args + ['--no-data'] 138 | pipe_commands([dump_cmd, ['grep', '^DROP'], mysql_cmd], **kwargs) 139 | 140 | pipe_commands([['cat', backup_file], ['gunzip'], mysql_cmd], **kwargs) 141 | 142 | 143 | @require_backup_exists 144 | def do_postgresql_restore(backup_file, db_config, drop_tables=False, show_output=False): 145 | env = get_postgresql_env(db_config) 146 | args = get_postgresql_args(db_config) 147 | psql_cmd = ['psql'] + args 148 | 149 | kwargs = {'extra_env': env, 'show_stderr': show_output, 'show_last_stdout': show_output} 150 | 151 | if drop_tables: 152 | gen_drop_sql_cmd = psql_cmd + ['-t', '-c', PG_DROP_SQL] 153 | pipe_commands([gen_drop_sql_cmd, psql_cmd], **kwargs) 154 | 155 | pipe_commands([['cat', backup_file], ['gunzip'], psql_cmd], **kwargs) 156 | 157 | 158 | @require_backup_exists 159 | def do_sqlite_restore(backup_file, db_config, drop_tables=False, show_output=False): 160 | db_file = db_config['NAME'] 161 | 162 | cmd = ['cat', backup_file] 163 | pipe_commands_to_file([cmd, ['gunzip']], path=db_file, show_stderr=show_output) 164 | -------------------------------------------------------------------------------- /backupdb/utils/exceptions.py: -------------------------------------------------------------------------------- 1 | class BackupError(Exception): 2 | pass 3 | 4 | 5 | class RestoreError(Exception): 6 | pass 7 | -------------------------------------------------------------------------------- /backupdb/utils/files.py: -------------------------------------------------------------------------------- 1 | import glob 2 | 3 | from .exceptions import RestoreError 4 | from .settings import BACKUP_DIR, BACKUP_TIMESTAMP_PATTERN 5 | 6 | 7 | def get_latest_timestamped_file(ext, dir=BACKUP_DIR, pattern=BACKUP_TIMESTAMP_PATTERN): 8 | """ 9 | Gets the latest timestamped backup file name with the given database type 10 | extension. 11 | """ 12 | pattern = '{dir}/{pattern}.{ext}.gz'.format( 13 | dir=dir, 14 | pattern=pattern, 15 | ext=ext, 16 | ) 17 | 18 | l = glob.glob(pattern) 19 | l.sort() 20 | l.reverse() 21 | 22 | if not l: 23 | raise RestoreError("No backups found matching '{0}' pattern".format(pattern)) 24 | 25 | return l[0] 26 | -------------------------------------------------------------------------------- /backupdb/utils/log.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import logging 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | 7 | def bar(msg='', width=40, position=None): 8 | r""" 9 | Returns a string with text centered in a bar caption. 10 | 11 | Examples: 12 | >>> bar('test', width=10) 13 | '== test ==' 14 | >>> bar(width=10) 15 | '==========' 16 | >>> bar('Richard Dean Anderson is...', position='top', width=50) 17 | '//========= Richard Dean Anderson is... ========\\\\' 18 | >>> bar('...MacGyver', position='bottom', width=50) 19 | '\\\\================= ...MacGyver ================//' 20 | """ 21 | if position == 'top': 22 | start_bar = '//' 23 | end_bar = r'\\' 24 | elif position == 'bottom': 25 | start_bar = r'\\' 26 | end_bar = '//' 27 | else: 28 | start_bar = end_bar = '==' 29 | 30 | if msg: 31 | msg = ' ' + msg + ' ' 32 | 33 | width -= 4 34 | 35 | return start_bar + msg.center(width, '=') + end_bar 36 | 37 | 38 | class SectionError(Exception): 39 | pass 40 | 41 | 42 | class SectionWarning(Exception): 43 | pass 44 | 45 | 46 | @contextlib.contextmanager 47 | def section(msg): 48 | """ 49 | Context manager that prints a top bar to stderr upon entering and a bottom 50 | bar upon exiting. The caption of the top bar is specified by `msg`. The 51 | caption of the bottom bar is '...done!' if the context manager exits 52 | successfully. If a SectionError or SectionWarning is raised inside of the 53 | context manager, SectionError.message or SectionWarning.message is passed 54 | to logging.error or logging.warning respectively and the bottom bar caption 55 | becomes '...skipped.'. 56 | """ 57 | logger.info(bar(msg, position='top')) 58 | try: 59 | yield 60 | except SectionError as e: 61 | logger.error(e) 62 | logger.info(bar('...skipped.', position='bottom')) 63 | except SectionWarning as e: 64 | logger.warning(e) 65 | logger.info(bar('...skipped.', position='bottom')) 66 | else: 67 | logger.info(bar('...done!', position='bottom')) 68 | -------------------------------------------------------------------------------- /backupdb/utils/processes.py: -------------------------------------------------------------------------------- 1 | from subprocess import Popen, PIPE, CalledProcessError 2 | import logging 3 | import os 4 | import shutil 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | def extend_env(extra_env): 10 | """ 11 | Copies and extends the current environment with the values present in 12 | `extra_env`. 13 | """ 14 | env = os.environ.copy() 15 | env.update(extra_env) 16 | return env 17 | 18 | 19 | def get_env_str(env): 20 | """ 21 | Gets a string representation of a dict as though it contained environment 22 | variable values. 23 | """ 24 | return ' '.join("{0}='{1}'".format(k, v) for k, v in env.items()) 25 | 26 | 27 | def pipe_commands(cmds, extra_env=None, show_stderr=False, show_last_stdout=False): 28 | """ 29 | Executes the list of commands piping each one into the next. 30 | """ 31 | env = extend_env(extra_env) if extra_env else None 32 | env_str = (get_env_str(extra_env) + ' ') if extra_env else '' 33 | cmd_strs = [env_str + ' '.join(cmd) for cmd in cmds] 34 | 35 | logger.info('Running `{0}`'.format(' | '.join(cmd_strs))) 36 | 37 | with open('/dev/null', 'w') as NULL: 38 | # Start processes 39 | processes = [] 40 | last_i = len(cmds) - 1 41 | for i, (cmd_str, cmd) in enumerate(zip(cmd_strs, cmds)): 42 | if i == last_i: 43 | p_stdout = None if show_last_stdout else NULL 44 | else: 45 | p_stdout = PIPE 46 | p_stdin = processes[-1][1].stdout if processes else None 47 | p_stderr = None if show_stderr else NULL 48 | 49 | p = Popen(cmd, env=env, stdout=p_stdout, stdin=p_stdin, stderr=p_stderr) 50 | processes.append((cmd_str, p)) 51 | 52 | # Close processes 53 | error = False 54 | for cmd_str, p in processes: 55 | if p.stdout: 56 | p.stdout.close() 57 | if p.wait() != 0: 58 | error = True 59 | if error: 60 | raise CalledProcessError(cmd=cmd_str, returncode=p.returncode) 61 | 62 | 63 | def pipe_commands_to_file(cmds, path, extra_env=None, show_stderr=False): 64 | """ 65 | Executes the list of commands piping each one into the next and writing 66 | stdout of the last process into a file at the given path. 67 | """ 68 | env = extend_env(extra_env) if extra_env else None 69 | env_str = (get_env_str(extra_env) + ' ') if extra_env else '' 70 | cmd_strs = [env_str + ' '.join(cmd) for cmd in cmds] 71 | 72 | logger.info('Saving output of `{0}`'.format(' | '.join(cmd_strs))) 73 | 74 | with open('/dev/null', 'w') as NULL: 75 | # Start processes 76 | processes = [] 77 | for cmd_str, cmd in zip(cmd_strs, cmds): 78 | p_stdin = processes[-1][1].stdout if processes else None 79 | p_stderr = None if show_stderr else NULL 80 | 81 | p = Popen(cmd, env=env, stdout=PIPE, stdin=p_stdin, stderr=p_stderr) 82 | processes.append((cmd_str, p)) 83 | 84 | p_last = processes[-1][1] 85 | 86 | with open(path, 'wb') as f: 87 | shutil.copyfileobj(p_last.stdout, f) 88 | 89 | # Close processes 90 | error = False 91 | for cmd_str, p in processes: 92 | p.stdout.close() 93 | if p.wait() != 0: 94 | error = True 95 | if error: 96 | raise CalledProcessError(cmd=cmd_str, returncode=p.returncode) 97 | -------------------------------------------------------------------------------- /backupdb/utils/settings.py: -------------------------------------------------------------------------------- 1 | from .commands import ( 2 | do_mysql_backup, 3 | do_mysql_restore, 4 | do_postgresql_backup, 5 | do_postgresql_restore, 6 | do_sqlite_backup, 7 | do_sqlite_restore, 8 | ) 9 | from django.conf import settings 10 | 11 | 12 | DEFAULT_BACKUP_DIR = 'backups' 13 | BACKUP_DIR = getattr(settings, 'BACKUPDB_DIRECTORY', DEFAULT_BACKUP_DIR) 14 | BACKUP_TIMESTAMP_PATTERN = '*-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' 15 | BACKUP_CONFIG = { 16 | 'django.db.backends.mysql': { 17 | 'backup_extension': 'mysql', 18 | 'backup_func': do_mysql_backup, 19 | 'restore_func': do_mysql_restore, 20 | }, 21 | 'django.db.backends.postgresql_psycopg2': { 22 | 'backup_extension': 'pgsql', 23 | 'backup_func': do_postgresql_backup, 24 | 'restore_func': do_postgresql_restore, 25 | }, 26 | 'django.contrib.gis.db.backends.postgis': { 27 | 'backup_extension': 'pgsql', 28 | 'backup_func': do_postgresql_backup, 29 | 'restore_func': do_postgresql_restore, 30 | }, 31 | 'django.db.backends.sqlite3': { 32 | 'backup_extension': 'sqlite', 33 | 'backup_func': do_sqlite_backup, 34 | 'restore_func': do_sqlite_restore, 35 | }, 36 | } 37 | -------------------------------------------------------------------------------- /end_to_end_requirements.txt: -------------------------------------------------------------------------------- 1 | Django 2 | MySQL-python==1.2.4 3 | psycopg2==2.5 4 | wsgiref==0.1.2 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import os 4 | 5 | from setuptools import setup 6 | 7 | version = '0.7.1.dev0' 8 | 9 | current_path = os.path.dirname(__file__) 10 | 11 | sys.path.append(current_path) 12 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_settings') 13 | 14 | 15 | def get_readme(): 16 | with open(os.path.join(current_path, 'README.rst'), 'r') as f: 17 | return f.read() 18 | 19 | setup( 20 | name='django-backupdb', 21 | version=version, 22 | description='Management commands for backing up and restoring databases in Django.', 23 | long_description=get_readme(), 24 | author='Fusionbox programmers', 25 | author_email='programmers@fusionbox.com', 26 | keywords='django database backup', 27 | url='https://github.com/fusionbox/django-backupdb', 28 | packages=['backupdb', 'backupdb.utils', 'backupdb.management', 29 | 'backupdb.management.commands', 'backupdb.tests', 30 | 'backupdb.tests.app'], 31 | platforms='any', 32 | license='Fusionbox', 33 | test_suite='backupdb.tests.all_tests', 34 | tests_require=[ 35 | 'mock>=1.0.1', 36 | 'dj_database_url==0.3.0', 37 | ], 38 | install_requires=[ 39 | 'Django>=1.4', 40 | ], 41 | classifiers=[ 42 | 'Development Status :: 5 - Production/Stable', 43 | 'Environment :: Web Environment', 44 | 'Framework :: Django', 45 | ], 46 | ) 47 | -------------------------------------------------------------------------------- /test_settings.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | import dj_database_url 3 | import os 4 | import shutil 5 | import tempfile 6 | 7 | DEBUG = True 8 | 9 | DATABASE = { 10 | 'default': dj_database_url.config(default='sqlite://:memory:'), 11 | } 12 | 13 | INSTALLED_APPS = ( 14 | 'backupdb', 15 | 'backupdb.tests.app', 16 | ) 17 | 18 | SECRET_KEY = 'this is a secret!' 19 | 20 | BACKUPDB_DIRECTORY = tempfile.mkdtemp() 21 | 22 | # Cleanup after itself 23 | atexit.register(shutil.rmtree, BACKUPDB_DIRECTORY) 24 | -------------------------------------------------------------------------------- /unit_tests_scratch/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fusionbox/django-backupdb/db4aa73049303245ef0182cda5c76b1dd194cd00/unit_tests_scratch/.gitkeep --------------------------------------------------------------------------------