├── .gitignore ├── LICENSE.txt ├── README.md ├── Vagrantfile ├── __init__.py ├── autoxtrabackup.py ├── backup_prepare ├── __init__.py └── prepare.py ├── docs ├── Makefile ├── advance_features.rst ├── backup_tags.rst ├── basic_features.rst ├── basic_overview.rst ├── conf.py ├── config_file.rst ├── index.rst ├── installation.rst ├── intro.rst ├── option_reference.rst └── test_mode.rst ├── general_conf ├── __init__.py ├── bck.conf ├── check_env.py └── generalops.py ├── master_backup_script ├── __init__.py └── backuper.py ├── partial_recovery ├── __init__.py └── partial.py ├── prepare_env_test_mode ├── __init__.py ├── build_2.3_pxb.sh ├── build_2.4_pxb.sh ├── call_create_index_temp.sh ├── call_ddl_test.sh ├── call_innodb_online_alter_encryption_alters.sh ├── call_innodb_online_alter_encryption_sql.sh ├── call_temp_table_test.sh ├── clone_build_start_server.py ├── config_generator.py ├── create_index_temp.sh ├── ddl_test.sh ├── innodb_online_alter_encryption.sql ├── innodb_online_alter_encryption_alters.sql ├── prepare_backup.py ├── run_benchmark.py ├── run_sql_queries.sh ├── runner_test_mode.py ├── start_dynamic.sh ├── take_backup.py ├── temp_table_test.sh └── test_check_env.py ├── setup.cfg ├── setup.py └── test ├── README.md ├── __init__.py ├── conftest.py ├── prepare_env.bats ├── test_backup.py ├── test_build_pxb.bats ├── test_build_server.bats ├── test_clone_build_start_server.py ├── test_clone_percona_qa.bats ├── test_clone_ps_server_from_conf.bats ├── test_clone_pxb.bats ├── test_config_generator.py ├── test_extract_xb_archive.bats ├── test_generate_config_files.bats ├── test_option_combinations.py ├── test_partialRecovery.py ├── test_prepare_start_dynamic.bats ├── test_prepare_startup.bats ├── test_rename_basedirs.bats ├── test_run_benchmark.py ├── test_runner_test_mode.py ├── test_start_server.bats ├── test_test_prepare_backup.py └── test_test_take_backup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # PyCharm stuff 54 | .idea 55 | .vagrant 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | .venv 88 | venv/ 89 | ENV/ 90 | 91 | # Spyder project settings 92 | .spyderproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Shahriyar Rzayev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | MySQL-AutoXtrabackup 2 | ==================== 3 | 4 | MySQL AutoXtrabackup commandline tool written in Python 3. 5 | For community from Azerbaijan MySQL User Community: [MySQL Azerbaijan Community](http://mysql.az/). 6 | For any question please ask: [Email](mailto:rzayev.shahriyar@yandex.com) 7 | 8 | Read full documentation here: 9 | ----------------------------- 10 | 11 | [**MySQL-AutoXtrabackup documentation!**](http://mysql-autoxtrabackup.readthedocs.io/en/latest/index.html) 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # All Vagrant configuration is done below. The "2" in Vagrant.configure 5 | # configures the configuration version (we support older styles for 6 | # backwards compatibility). Please don't change it unless you know what 7 | # you're doing. 8 | Vagrant.configure("2") do |config| 9 | # The most common configuration options are documented and commented below. 10 | # For a complete reference, please see the online documentation at 11 | # https://docs.vagrantup.com. 12 | 13 | # Every Vagrant development environment requires a box. You can search for 14 | # boxes at https://atlas.hashicorp.com/search. 15 | config.vm.box = "centos/7" 16 | config.disksize.size = '50GB' 17 | 18 | # Disable automatic box update checking. If you disable this, then 19 | # boxes will only be checked for updates when the user runs 20 | # `vagrant box outdated`. This is not recommended. 21 | # config.vm.box_check_update = false 22 | 23 | # Create a forwarded port mapping which allows access to a specific port 24 | # within the machine from a port on the host machine. In the example below, 25 | # accessing "localhost:8080" will access port 80 on the guest machine. 26 | # config.vm.network "forwarded_port", guest: 80, host: 8080 27 | 28 | # Create a private network, which allows host-only access to the machine 29 | # using a specific IP. 30 | config.vm.network "public_network", 31 | use_dhcp_assigned_default_route: true 32 | 33 | # Create a public network, which generally matched to bridged network. 34 | # Bridged networks make the machine appear as another physical device on 35 | # your network. 36 | # config.vm.network "public_network" 37 | 38 | # Share an additional folder to the guest VM. The first argument is 39 | # the path on the host to the actual folder. The second argument is 40 | # the path on the guest to mount the folder. And the optional third 41 | # argument is a set of non-required options. 42 | # config.vm.synced_folder "../data", "/vagrant_data" 43 | 44 | # Provider-specific configuration so you can fine-tune various 45 | # backing providers for Vagrant. These expose provider-specific options. 46 | # Example for VirtualBox: 47 | # 48 | config.vm.provider "virtualbox" do |vb| 49 | # # Display the VirtualBox GUI when booting the machine 50 | vb.gui = true 51 | # 52 | # # Customize the amount of memory on the VM: 53 | vb.memory = "6144" 54 | vb.cpus = 4 55 | end 56 | # 57 | # View the documentation for the provider you are using for more 58 | # information on available options. 59 | 60 | # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies 61 | # such as FTP and Heroku are also available. See the documentation at 62 | # https://docs.vagrantup.com/v2/push/atlas.html for more information. 63 | # config.push.define "atlas" do |push| 64 | # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" 65 | # end 66 | 67 | #config.ssh.username = 'root' 68 | #config.ssh.password = 'vagrant' 69 | #config.ssh.insert_key = 'true' 70 | 71 | # Enable provisioning with a shell script. Additional provisioners such as 72 | # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the 73 | # documentation for more information about their specific syntax and use. 74 | config.vm.provision "shell", inline: <<-SHELL 75 | sudo yum -y update 76 | sudo yum -y install yum-utils 77 | sudo yum -y groupinstall development 78 | sudo yum -y install cmake 79 | sudo yum -y install libaio-devel 80 | sudo yum -y install ncurses-devel 81 | sudo yum -y install readline readline-devel 82 | sudo yum -y install pam pam-devel 83 | sudo yum -y install openssl openssl-devel 84 | sudo yum -y install libev libev-devel 85 | sudo yum -y install libgcrypt libgcrypt-devel 86 | sudo yum -y install libcurl libcurl-devel 87 | sudo yum -y install wget 88 | sudo yum -y install vim 89 | sudo yum -y install https://centos7.iuscommunity.org/ius-release.rpm 90 | sudo yum -y install http://www.percona.com/downloads/percona-release/redhat/0.1-4/percona-release-0.1-4.noarch.rpm 91 | sudo yum -y update 92 | sudo yum -y install qpress 93 | sudo yum -y install jemalloc 94 | sudo yum -y install percona-toolkit 95 | sudo yum -y install sysbench 96 | sudo yum -y install python35u 97 | sudo yum -y install python35u-pip 98 | sudo yum -y install python35u-devel 99 | sudo yum -y install python35u-tkinter 100 | sudo yum -y install xauth 101 | sudo pip3.5 install setuptools -U pip setuptools 102 | sudo pip3.5 install memory_profiler 103 | sudo pip3.5 install psutil 104 | sudo pip3.5 install matplotlib 105 | sudo pip3.5 install virtualenv 106 | cd /home/vagrant 107 | virtualenv -p /usr/bin/python3.5 py_3_5_autoxtrabackup 108 | cd /vagrant 109 | sudo /home/vagrant/py_3_5_autoxtrabackup/bin/python setup.py install 110 | sudo sed -i "0,/^[ \t]*testpath[ \t]*=.*$/s|^[ \t]*testpath[ \t]*=.*$|testpath=\/home\/vagrant\/XB_TEST\/server_dir|" /etc/bck.conf 111 | cd /home/vagrant 112 | sudo chown -R vagrant:vagrant * 113 | touch python-sudo.sh 114 | echo "#!/bin/bash" > python-sudo.sh 115 | echo 'sudo /usr/bin/python3.5 "$@"' >> python-sudo.sh 116 | chmod +x python-sudo.sh 117 | git clone https://github.com/sstephenson/bats.git 118 | cd bats 119 | ./install.sh /usr/local 120 | cd /tmp 121 | sudo chown -R vagrant:vagrant * 122 | cd /vagrant/test 123 | source /home/vagrant/py_3_5_autoxtrabackup/bin/activate 124 | /usr/local/bin/bats prepare_env.bats 125 | chown -R vagrant:vagrant * 126 | SHELL 127 | end -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | from general_conf.generalops import GeneralClass 5 | from master_backup_script.backuper import Backup 6 | from general_conf.check_env import CheckEnv 7 | from prepare_env_test_mode.runner_test_mode import RunnerTestMode 8 | from .autoxtrabackup import * 9 | 10 | -------------------------------------------------------------------------------- /autoxtrabackup.py: -------------------------------------------------------------------------------- 1 | import click 2 | from master_backup_script.backuper import Backup 3 | from backup_prepare.prepare import Prepare 4 | from partial_recovery.partial import PartialRecovery 5 | from general_conf.generalops import GeneralClass 6 | from prepare_env_test_mode.runner_test_mode import RunnerTestMode 7 | from sys import platform as _platform 8 | from sys import exit 9 | import pid 10 | import time 11 | import re 12 | import os 13 | import humanfriendly 14 | import logging 15 | import logging.handlers 16 | from logging.handlers import RotatingFileHandler 17 | 18 | 19 | logger = logging.getLogger('') 20 | 21 | 22 | destinations_hash = {'linux':'/dev/log', 'linux2': '/dev/log', 'darwin':'/var/run/syslog'} 23 | 24 | def address_matcher(plt): 25 | return destinations_hash.get(plt, ('localhost', 514)) 26 | 27 | handler = logging.handlers.SysLogHandler(address=address_matcher(_platform)) 28 | 29 | # Set syslog for the root logger 30 | logger.addHandler(handler) 31 | 32 | 33 | def print_help(ctx, param, value): 34 | if value is False: 35 | return 36 | click.echo(ctx.get_help()) 37 | ctx.exit() 38 | 39 | def print_version(ctx, param, value): 40 | if not value or ctx.resilient_parsing: 41 | return 42 | click.echo( 43 | "Developed by Shahriyar Rzayev from Azerbaijan MUG(http://mysql.az)") 44 | click.echo("Link : https://github.com/ShahriyarR/MySQL-AutoXtraBackup") 45 | click.echo("Email: rzayev.shahriyar@yandex.com") 46 | click.echo( 47 | "Based on Percona XtraBackup: https://github.com/percona/percona-xtrabackup/") 48 | click.echo('MySQL-AutoXtraBackup Version: 1.5.2') 49 | ctx.exit() 50 | 51 | 52 | def check_file_content(file): 53 | """Check if all mandatory headers and keys exist in file""" 54 | with open(file, 'r') as config_file: 55 | file_content = config_file.read() 56 | 57 | config_headers = ["MySQL", "Backup", "Encrypt", "Compress", "Commands"] 58 | config_keys = [ 59 | "mysql", 60 | "mycnf", 61 | "mysqladmin", 62 | "mysql_user", 63 | "mysql_password", 64 | "mysql_host", 65 | "datadir", 66 | "tmpdir", 67 | "backupdir", 68 | "backup_tool", 69 | "xtra_prepare", 70 | "start_mysql_command", 71 | "stop_mysql_command", 72 | "chown_command"] 73 | 74 | for header in config_headers: 75 | if header not in file_content: 76 | raise KeyError( 77 | "Mandatory header [%s] doesn't exist in %s" % 78 | (header, file)) 79 | 80 | for key in config_keys: 81 | if key not in file_content: 82 | raise KeyError( 83 | "Mandatory key \'%s\' doesn't exists in %s." % 84 | (key, file)) 85 | 86 | return True 87 | 88 | 89 | def validate_file(file): 90 | """ 91 | Check for validity of the file given in file path. If file doesn't exist or invalid 92 | configuration file, throw error. 93 | """ 94 | if os.path.isfile(file): 95 | # filename extension should be .conf 96 | pattern = re.compile(r'.*\.conf') 97 | 98 | if pattern.match(file): 99 | # Lastly the file should have all 5 required headers 100 | if check_file_content(file): 101 | return 102 | else: 103 | raise ValueError("Invalid file extension. Expecting .conf") 104 | else: 105 | raise FileNotFoundError("Specified file does not exist.") 106 | 107 | 108 | @click.command() 109 | @click.option('--dry_run', is_flag=True, help="Enable the dry run.") 110 | @click.option('--prepare', is_flag=True, help="Prepare/recover backups.") 111 | @click.option('--backup', 112 | is_flag=True, 113 | help="Take full and incremental backups.") 114 | @click.option('--partial', 115 | is_flag=True, 116 | help="Recover specified table (partial recovery).") 117 | @click.option('--version', 118 | is_flag=True, 119 | callback=print_version, 120 | expose_value=False, 121 | is_eager=True, 122 | help="Version information.") 123 | @click.option('--defaults_file', 124 | default='/etc/bck.conf', 125 | show_default=True, 126 | help="Read options from the given file") 127 | @click.option('--tag', 128 | help="Pass the tag string for each backup") 129 | @click.option('--show_tags', 130 | is_flag=True, 131 | help="Show backup tags and exit") 132 | @click.option('-v', '--verbose', is_flag=True, 133 | help="Be verbose (print to console)") 134 | @click.option('-lf', 135 | '--log_file', 136 | default='/var/log/autoxtrabackup.log', 137 | show_default=True, 138 | help="Set log file") 139 | @click.option('-l', 140 | '--log', 141 | default='WARNING', 142 | show_default=True, 143 | type=click.Choice(['DEBUG', 144 | 'INFO', 145 | 'WARNING', 146 | 'ERROR', 147 | 'CRITICAL']), 148 | help="Set log level") 149 | @click.option('--log_file_max_bytes', 150 | default=1073741824, 151 | show_default=True, 152 | nargs=1, 153 | type=int, 154 | help="Set log file max size in bytes") 155 | @click.option('--log_file_backup_count', 156 | default=7, 157 | show_default=True, 158 | nargs=1, 159 | type=int, 160 | help="Set log file backup count") 161 | @click.option('--keyring_vault', 162 | default=0, 163 | show_default=True, 164 | nargs=1, 165 | type=int, 166 | help="Enable this when you pass keyring_vault options in default mysqld options in config" 167 | "[Only for using with --test_mode]") 168 | @click.option('--test_mode', 169 | is_flag=True, 170 | help="Enable test mode. Must be used with --defaults_file and only for TESTs for XtraBackup") 171 | @click.option('--help', 172 | is_flag=True, 173 | callback=print_help, 174 | expose_value=False, 175 | is_eager=False, 176 | help="Print help message and exit.") 177 | @click.pass_context 178 | def all_procedure(ctx, prepare, backup, partial, tag, show_tags, 179 | verbose, log_file, log, defaults_file, 180 | dry_run, test_mode, log_file_max_bytes, 181 | log_file_backup_count, keyring_vault): 182 | logger.setLevel(log) 183 | formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s', 184 | datefmt='%Y-%m-%d %H:%M:%S') 185 | 186 | if verbose: 187 | ch = logging.StreamHandler() 188 | ch.setFormatter(formatter) 189 | logger.addHandler(ch) 190 | 191 | if log_file: 192 | try: 193 | file_handler = RotatingFileHandler(log_file, mode='a', 194 | maxBytes=log_file_max_bytes, backupCount=log_file_backup_count) 195 | file_handler.setFormatter(formatter) 196 | logger.addHandler(file_handler) 197 | except PermissionError as err: 198 | exit("{} Please consider to run as root or sudo".format(err)) 199 | 200 | validate_file(defaults_file) 201 | config = GeneralClass(defaults_file) 202 | 203 | pid_file = pid.PidFile(piddir=config.pid_dir) 204 | 205 | try: 206 | with pid_file: # User PidFile for locking to single instance 207 | if (prepare is False and 208 | backup is False and 209 | partial is False and 210 | verbose is False and 211 | dry_run is False and 212 | test_mode is False and 213 | show_tags is False): 214 | print_help(ctx, None, value=True) 215 | elif show_tags and defaults_file: 216 | b = Backup(config=defaults_file) 217 | b.show_tags(backup_dir=b.backupdir) 218 | elif test_mode and defaults_file: 219 | # TODO: do staff here to implement all in one things for running test mode 220 | logger.warning("Enabled Test Mode!!!") 221 | logger.debug("Starting Test Mode") 222 | test_obj = RunnerTestMode(config=defaults_file) 223 | for basedir in test_obj.basedirs: 224 | if ('5.7' in basedir) and ('2_4_ps_5_7' in defaults_file): 225 | if keyring_vault == 1: 226 | test_obj.wipe_backup_prepare_copyback(basedir=basedir, keyring_vault=1) 227 | else: 228 | test_obj.wipe_backup_prepare_copyback(basedir=basedir) 229 | elif ('5.6' in basedir) and ('2_4_ps_5_6' in defaults_file): 230 | test_obj.wipe_backup_prepare_copyback(basedir=basedir) 231 | elif ('5.6' in basedir) and ('2_3_ps_5_6' in defaults_file): 232 | test_obj.wipe_backup_prepare_copyback(basedir=basedir) 233 | elif ('5.5' in basedir) and ('2_3_ps_5_5' in defaults_file): 234 | test_obj.wipe_backup_prepare_copyback(basedir=basedir) 235 | elif ('5.5' in basedir) and ('2_4_ps_5_5' in defaults_file): 236 | test_obj.wipe_backup_prepare_copyback(basedir=basedir) 237 | else: 238 | logger.error("Please pass proper already generated config file!") 239 | logger.error("Please check also if you have run prepare_env.bats file") 240 | elif prepare and not test_mode: 241 | if not dry_run: 242 | if tag: 243 | a = Prepare(config=defaults_file, tag=tag) 244 | a.prepare_backup_and_copy_back() 245 | else: 246 | a = Prepare(config=defaults_file) 247 | a.prepare_backup_and_copy_back() 248 | else: 249 | logger.warning("Dry run enabled!") 250 | logger.warning("Do not recover/copy-back in this mode!") 251 | if tag: 252 | a = Prepare(config=defaults_file, dry_run=1, tag=tag) 253 | a.prepare_backup_and_copy_back() 254 | else: 255 | a = Prepare(config=defaults_file, dry_run=1) 256 | a.prepare_backup_and_copy_back() 257 | elif backup and not test_mode: 258 | if not dry_run: 259 | if tag: 260 | b = Backup(config=defaults_file, tag=tag) 261 | b.all_backup() 262 | else: 263 | b = Backup(config=defaults_file) 264 | b.all_backup() 265 | else: 266 | logger.warning("Dry run enabled!") 267 | if tag: 268 | b = Backup(config=defaults_file, dry_run=1, tag=tag) 269 | b.all_backup() 270 | else: 271 | b = Backup(config=defaults_file, dry_run=1) 272 | b.all_backup() 273 | elif partial: 274 | if not dry_run: 275 | c = PartialRecovery(config=defaults_file) 276 | c.final_actions() 277 | else: 278 | logger.critical("Dry run is not implemented for partial recovery!") 279 | except pid.PidFileAlreadyLockedError as error: 280 | if hasattr(config, 'pid_runtime_warning'): 281 | if time.time() - os.stat(pid_file.filename).st_ctime > config.pid_runtime_warning: 282 | pid.fh.seek(0) 283 | pid_str = pid.fh.read(16).split("\n", 1)[0].strip() 284 | logger.critical( 285 | "Backup (pid: " + pid_str + ") has been running for logger than: " + str( 286 | humanfriendly.format_timespan( 287 | config.pid_runtime_warning))) 288 | # logger.warn("Pid file already exists: " + str(error)) 289 | except pid.PidFileAlreadyRunningError as error: 290 | if hasattr(config, 'pid_runtime_warning'): 291 | if time.time() - os.stat(pid_file.filename).st_ctime > config.pid_runtime_warning: 292 | pid.fh.seek(0) 293 | pid_str = pid.fh.read(16).split("\n", 1)[0].strip() 294 | logger.critical( 295 | "Backup (pid: " + pid_str + ") has been running for logger than: " + str( 296 | humanfriendly.format_timespan( 297 | config.pid_runtime_warning))) 298 | # logger.warn("Pid already running: " + str(error)) 299 | except pid.PidFileUnreadableError as error: 300 | logger.warning("Pid file can not be read: " + str(error)) 301 | except pid.PidFileError as error: 302 | logger.warning("Generic error with pid file: " + str(error)) 303 | 304 | 305 | if __name__ == "__main__": 306 | all_procedure() 307 | -------------------------------------------------------------------------------- /backup_prepare/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Percona-Lab/MySQL-AutoXtraBackup/8ae7927e72c03cf4e685d26f2c2d0d2580eeac52/backup_prepare/__init__.py -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = MySQLAutoXtrabackup 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/advance_features.rst: -------------------------------------------------------------------------------- 1 | Advanced features 2 | ================= 3 | 4 | Compressed backups: 5 | ------------------- 6 | 7 | To enable compression support just uncomment the options under 8 | [Compress] category inside main configuration file: 9 | 10 | .. code-block:: shell 11 | 12 | [Compress] 13 | #Optional 14 | #Enable only if you want to use compression. 15 | compress=quicklz 16 | compress_chunk_size=65536 17 | compress_threads=4 18 | decompress=TRUE 19 | #Enable if you want to remove .qp files after decompression.(Not available yet, will be released with XB 2.3.7 and 2.4.6) 20 | remove_original=FALSE 21 | 22 | 23 | Encrypted backups 24 | ----------------- 25 | 26 | To enable encryption support uncomment the options under [Encryption] 27 | category: 28 | 29 | :: 30 | 31 | [Encrypt] 32 | #Optional 33 | #Enable only if you want to create encrypted backups 34 | xbcrypt=/home/shahriyar.rzaev/Percona_Xtrabackups/xb_2.4/usr/local/xtrabackup/bin/xbcrypt 35 | encrypt=AES256 36 | # Please note that --encrypt-key and --encrypt-key-file are mutually exclusive 37 | encrypt_key='VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K' 38 | #encrypt_key_file=/path/to/file/with_encrypt_key 39 | encrypt_threads=4 40 | encrypt_chunk_size=65536 41 | decrypt=AES256 42 | #Enable if you want to remove .qp files after decompression.(Not available yet, will be released with XB 2.3.7 and 2.4.6) 43 | remove_original=True 44 | 45 | 46 | Partial backups 47 | --------------- 48 | 49 | It is possible to take partial full and incremental backups. The idea is, to take specified table(or database) as full backup, 50 | then to take incremental backups based on this one table. 51 | You can achieve this by enabling ``partial_list`` option from config file: 52 | 53 | 54 | :: 55 | 56 | [Backup] 57 | #Optional: set pid directory 58 | pid_dir=/tmp/MySQL-AutoXtraBackup 59 | tmpdir=/home/shahriyar.rzaev/XB_TEST/mysql_datadirs 60 | #Optional: set warning if pid of backup us running for longer than X 61 | pid_runtime_warning=2 Hours 62 | backupdir=/home/shahriyar.rzaev/XB_TEST/backup_dir 63 | backup_tool=/usr/bin/xtrabackup 64 | #Optional: specify different path/version of xtrabackup here for prepare 65 | #prepare_tool= 66 | xtra_prepare=--apply-log-only 67 | #Optional: pass additional options for backup stage 68 | #xtra_backup=--compact 69 | #Optional: pass additional options for prepare stage 70 | #xtra_prepare_options=--rebuild-indexes 71 | #Optional: pass general additional options; it will go to both for backup and prepare 72 | #xtra_options=--binlog-info=ON --galera-info 73 | xtra_options=--no-version-check 74 | #Optional: set archive and rotation 75 | #archive_dir=/home/shahriyar.rzaev/XB_TEST/backup_archives 76 | #full_backup_interval=1 day 77 | #max_archive_size=100GiB 78 | #max_archive_duration=4 Days 79 | #Optional WARNING(Enable this if you want to take partial backups). Specify database names or table names. 80 | partial_list=test.t1 test.t2 dbtest 81 | 82 | 83 | Run it and notice that backup command has changed (see ``--databases`` option for xtrabackup command): 84 | 85 | In the same way you can take incremental backup of specified tables. 86 | 87 | The prepare process is the same as ordinary prepare, just run autoxtrabackup with ``--prepare`` option, you can even restore this single table using ``--partial`` option. 88 | 89 | Decompressing and Decrypting backups 90 | ------------------------------------ 91 | 92 | We took Compressed and Encrypted backups. 93 | It is time to prepare them. 94 | autoxtrabackup will prepare all backups automatically, by first decrypting then 95 | decompressing step-by-step. 96 | All backups first will be decrypted then decompressed and then 97 | prepared. 98 | You can also optionally enable ``--remove-original`` option to 99 | remove ``.xbcrypt`` and ``.qp`` files from backup directory during prepare 100 | process. Read about this option here -> `--remove-original `_ 101 | 102 | Restoring single table after drop 103 | --------------------------------- 104 | 105 | Let's explain a bit, how we can restore single table from full backup? 106 | This is the part of "Transportable Tablespace" concept which you can read more: `Transportable Tablespace `_ 107 | 108 | The basic idea is: 109 | 110 | - Discard available tablespace of table 111 | - Copy the .ibd file from backup to current database directory 112 | - Import tablespace 113 | - You have restored the table. 114 | 115 | Previously we have mentioned about that, we can restore single table 116 | after deleting data. The situation there, was quite clear because the 117 | table structure was available(i.e table was not dropped). 118 | 119 | The problem is getting interesting, if table was dropped or even the 120 | whole database dropped. We should figure out how to find table structure 121 | and create it. 122 | 123 | The basic plan for this situation is: 124 | 125 | - Find the dropped table structure(i.e create statement) 126 | - Create dropped table again 127 | - Discard tablespace of newly created table 128 | - Copy the .ibd file from backup to current database directory 129 | - Import tablespace 130 | - You have restored the table. 131 | 132 | I found a way,by using ``mysqlfrm`` tool for extracting create statement 133 | from table's .frm file, which is stored in backup directory. So this is 134 | also automated. Let's see it in action. We have a dbtest database and t1 table: 135 | 136 | Dropping the database: 137 | 138 | :: 139 | 140 | > drop database test; 141 | Query OK, 1 row affected (1.08 sec) 142 | 143 | 144 | Trying to restore t1 table: It will figure out that specified database is missing and will prompt to create it. 145 | 146 | :: 147 | 148 | 149 | $ autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_3_5_6.log \ 150 | -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --partial 151 | 2017-11-16 20:38:16 DEBUG entering setup 152 | 2017-11-16 20:38:16 DEBUG create pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 153 | 2017-11-16 20:38:16 DEBUG check pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 154 | Type Database name: test 155 | Type Table name: t1 156 | 2017-11-16 20:38:19 DEBUG Running mysqladmin command -> /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysqladmin --defaults-file= --user=root --password= status --socket=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/socket.sock 157 | mysqladmin: [Warning] Using a password on the command line interface can be insecure. 158 | 2017-11-16 20:38:19 DEBUG OK: Server is Up and running 159 | 2017-11-16 20:38:19 DEBUG Checking if innodb_file_per_table is enabled 160 | 2017-11-16 20:38:19 DEBUG OK: innodb_file_per_table is enabled! 161 | 2017-11-16 20:38:19 DEBUG Checking MySQL version 162 | 2017-11-16 20:38:19 DEBUG You have correct version of MySQL 163 | 2017-11-16 20:38:19 DEBUG Checking if database exists in MySQL 164 | 2017-11-16 20:38:19 DEBUG There is no such database! 165 | 2017-11-16 20:38:19 DEBUG Create Specified Database in MySQL Server, before restoring single table 166 | We can create it for you do you want? (yes/no): yes 167 | 2017-11-16 20:38:26 DEBUG Creating specified database 168 | 2017-11-16 20:38:26 DEBUG OK: test database created 169 | 2017-11-16 20:38:26 DEBUG Checking if table exists in MySQL Server 170 | 2017-11-16 20:38:26 DEBUG Table does not exist in MySQL Server. 171 | 2017-11-16 20:38:26 DEBUG You can not restore table, with not existing tablespace file(.ibd)! 172 | 2017-11-16 20:38:26 DEBUG We will try to extract table create statement from .frm file, from backup folder 173 | 2017-11-16 20:38:26 DEBUG Running mysqlfrm tool 174 | 2017-11-16 20:38:26 DEBUG OK: Success to run mysqlfrm 175 | 2017-11-16 20:38:26 DEBUG Table Created from .frm file! 176 | 2017-11-16 20:38:26 DEBUG Applying write lock! 177 | 2017-11-16 20:38:26 DEBUG OK: Table is locked 178 | 2017-11-16 20:38:26 DEBUG Discarding tablespace 179 | 2017-11-16 20:38:26 DEBUG OK: Tablespace discarded successfully 180 | 2017-11-16 20:38:26 DEBUG OK: Copying .ibd file back 181 | 2017-11-16 20:38:26 DEBUG Running chown command! 182 | 2017-11-16 20:38:26 DEBUG OK: Chown command completed 183 | 2017-11-16 20:38:26 DEBUG Importing Tablespace! 184 | 2017-11-16 20:38:26 DEBUG OK: Tablespace imported 185 | 2017-11-16 20:38:26 DEBUG Unlocking tables! 186 | 2017-11-16 20:38:26 DEBUG OK: Unlocked! 187 | 2017-11-16 20:38:26 DEBUG OK: Table Recovered! ... 188 | 2017-11-16 20:38:26 DEBUG closing pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 189 | 2017-11-16 20:38:26 DEBUG closing pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 190 | 191 | As you noticed, the ``mysqlfrm`` tool did the job and table is restored after drop: 192 | 193 | :: 194 | 195 | > select * from dbtest.t1; 196 | +----+ 197 | | id | 198 | +----+ 199 | | 1 | 200 | | 1 | 201 | | 2 | 202 | | 1 | 203 | | 2 | 204 | | 3 | 205 | +----+ 206 | 6 rows in set (0.00 sec) 207 | 208 | 209 | autoxtrabackup with --dry_run option 210 | ------------------------------------ 211 | 212 | For testing purposes or just to show what is going on, with autoxtrabackup backup and prepare steps. 213 | You can append ``--dry_run`` option, to show commands but not to run them. 214 | Taking backup: 215 | 216 | :: 217 | 218 | 219 | $ autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_3_5_6.log -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup --dry_run 220 | 2017-11-16 20:40:47 DEBUG entering setup 221 | 2017-11-16 20:40:47 DEBUG create pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 222 | 2017-11-16 20:40:47 DEBUG check pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 223 | 2017-11-16 20:40:47 WARNING Dry run enabled! 224 | 2017-11-16 20:40:47 DEBUG Running mysqladmin command -> /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysqladmin --defaults-file= --user=root --password= status --socket=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/socket.sock 225 | mysqladmin: [Warning] Using a password on the command line interface can be insecure. 226 | 2017-11-16 20:40:47 DEBUG OK: Server is Up and running 227 | 2017-11-16 20:40:47 DEBUG OK: /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysql exists 228 | 2017-11-16 20:40:47 DEBUG OK: /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysqladmin exists 229 | 2017-11-16 20:40:47 DEBUG Skipping my.cnf check, because it is not specified 230 | 2017-11-16 20:40:47 DEBUG OK: XtraBackup exists 231 | 2017-11-16 20:40:47 DEBUG OK: Main backup directory exists 232 | 2017-11-16 20:40:47 DEBUG OK: Full Backup directory exists 233 | 2017-11-16 20:40:47 DEBUG OK: Increment directory exists 234 | 2017-11-16 20:40:47 DEBUG OK: Check status 235 | 2017-11-16 20:40:47 DEBUG - - - - You have a full backup that is less than 86400 seconds old. - - - - 236 | 2017-11-16 20:40:47 DEBUG - - - - We will take an incremental one based on recent Full Backup - - - - 237 | 2017-11-16 20:40:50 DEBUG Using xbstream to extract and decrypt from inc_backup.stream! 238 | 2017-11-16 20:40:50 DEBUG The following xbstream command will be executed /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xbstream -x --parallel=100 --decrypt=AES256 --encrypt-key=VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K --encrypt-threads=4 < /home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-13-39/inc_backup.stream -C /home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-13-39 239 | 2017-11-16 20:40:50 WARNING Streaming is enabled! 240 | 2017-11-16 20:40:50 DEBUG The following backup command will be executed /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --defaults-file= --user=root --password='' --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-40-50 --incremental-basedir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-13-39 --backup --socket=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/socket.sock --compress=quicklz --compress_chunk_size=65536 --encrypt=AES256 --encrypt-key=VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K --encrypt-threads=4 --encrypt-chunk-size=65536 --slave-info --no-version-check --core-file --parallel=1 --throttle=40 --keyring-file-data=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/mysql-keyring/keyring --stream="xbstream" > /home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-40-50/inc_backup.stream 241 | 2017-11-16 20:40:50 DEBUG closing pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 242 | 2017-11-16 20:40:50 DEBUG closing pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 243 | 244 | Preparing backups: 245 | 246 | :: 247 | 248 | 249 | $ autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_3_5_6.log -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --prepare --dry_run 250 | 2017-11-16 20:41:49 DEBUG entering setup 251 | 2017-11-16 20:41:49 DEBUG create pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 252 | 2017-11-16 20:41:49 DEBUG check pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 253 | 2017-11-16 20:41:49 WARNING Dry run enabled! 254 | 2017-11-16 20:41:49 WARNING Do not recover/copy-back in this mode! 255 | - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 256 | 257 | Preparing full/inc backups! 258 | What do you want to do? 259 | 1. Prepare Backups and keep for future usage. NOTE('Once Prepared Backups Can not be prepared Again') 260 | 2. Prepare Backups and restore/recover/copy-back immediately 261 | 3. Just copy-back previously prepared backups 262 | Please Choose one of options and type 1 or 2 or 3: 1 263 | 264 | - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 265 | 2017-11-16 20:41:53 DEBUG - - - - You have Incremental backups. - - - - 266 | 2017-11-16 20:41:53 DEBUG - - - - Preparing Full backup for incrementals - - - - 267 | 2017-11-16 20:41:53 DEBUG - - - - Final prepare,will occur after preparing all inc backups - - - - 268 | 2017-11-16 20:41:56 DEBUG Trying to decrypt backup 269 | 2017-11-16 20:41:56 DEBUG Running decrypt command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --decrypt=AES256 --encrypt-key=VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_20-10-53 --remove-original 270 | 2017-11-16 20:41:56 DEBUG Trying to decompress backup 271 | 2017-11-16 20:41:56 DEBUG Running decompress command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --decompress=TRUE --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_20-10-53 --remove-original 272 | 2017-11-16 20:41:56 DEBUG Running prepare command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --prepare --apply-log-only --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_20-10-53 --slave-info --no-version-check --core-file --parallel=1 --throttle=40 --keyring-file-data=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/mysql-keyring/keyring 273 | 2017-11-16 20:41:56 DEBUG Preparing Incs: 274 | 2017-11-16 20:41:56 DEBUG Preparing inc backups in sequence. inc backup dir/name is 2017-11-16_20-12-23 275 | 2017-11-16 20:41:56 DEBUG Trying to decrypt backup 276 | 2017-11-16 20:41:56 DEBUG Running decrypt command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --decrypt=AES256 --encrypt-key=VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-12-23 --remove-original 277 | 2017-11-16 20:41:56 DEBUG Trying to decompress backup 278 | 2017-11-16 20:41:56 DEBUG Running decompress command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --decompress=TRUE --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-12-23 --remove-original 279 | 2017-11-16 20:41:56 DEBUG Running prepare command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --prepare --apply-log-only --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_20-10-53 --incremental-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-12-23 --slave-info --no-version-check --core-file --parallel=1 --throttle=40 --keyring-file-data=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/mysql-keyring/keyring 280 | 2017-11-16 20:41:56 DEBUG Preparing inc backups in sequence. inc backup dir/name is 2017-11-16_20-13-39 281 | 2017-11-16 20:41:56 DEBUG Trying to decrypt backup 282 | 2017-11-16 20:41:56 DEBUG Running decrypt command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --decrypt=AES256 --encrypt-key=VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-13-39 --remove-original 283 | 2017-11-16 20:41:56 DEBUG Trying to decompress backup 284 | 2017-11-16 20:41:56 DEBUG Running decompress command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --decompress=TRUE --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-13-39 --remove-original 285 | 2017-11-16 20:41:56 DEBUG Running prepare command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --prepare --apply-log-only --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_20-10-53 --incremental-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-13-39 --slave-info --no-version-check --core-file --parallel=1 --throttle=40 --keyring-file-data=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/mysql-keyring/keyring 286 | 2017-11-16 20:41:56 DEBUG Preparing last incremental backup, inc backup dir/name is 2017-11-16_20-40-50 287 | 2017-11-16 20:41:56 DEBUG Using xbstream to extract from inc_backup.stream! 288 | 2017-11-16 20:41:56 DEBUG The following xbstream command will be executed /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xbstream -x --parallel=100 < /home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-40-50/inc_backup.stream -C /home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-40-50 289 | 2017-11-16 20:41:56 DEBUG Trying to decrypt backup 290 | 2017-11-16 20:41:56 DEBUG Running decrypt command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --decrypt=AES256 --encrypt-key=VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-40-50 --remove-original 291 | 2017-11-16 20:41:56 DEBUG Trying to decompress backup 292 | 2017-11-16 20:41:56 DEBUG Running decompress command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --decompress=TRUE --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-40-50 --remove-original 293 | 2017-11-16 20:41:56 DEBUG Running prepare command -> /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --prepare --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_20-10-53 --incremental-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_20-40-50 --slave-info --no-version-check --core-file --parallel=1 --throttle=40 --keyring-file-data=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/mysql-keyring/keyring 294 | 2017-11-16 20:41:56 DEBUG - - - - The end of the Prepare Stage. - - - - 295 | 2017-11-16 20:41:56 DEBUG closing pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 296 | 2017-11-16 20:41:56 DEBUG closing pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 297 | 298 | The end. 299 | -------------------------------------------------------------------------------- /docs/backup_tags.rst: -------------------------------------------------------------------------------- 1 | Backup Tags 2 | =========== 3 | 4 | The backup tags actually is a result of feature requests by community member `Yusif Yusifli `_. 5 | Read discussions about feature requests below: 6 | 7 | `#163 `_. 8 | `#164 `_. 9 | `#210 `_. 10 | 11 | So basically how to take backups and create a tag for it? 12 | 13 | Taking full backup: 14 | ------------------ 15 | 16 | :: 17 | 18 | $ sudo autoxtrabackup --tag="My Full backup" -v \ 19 | -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ 20 | -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup 21 | 22 | Taking incremental one: 23 | ---------------------- 24 | 25 | :: 26 | 27 | $ autoxtrabackup --tag="First incremental backup" -v \ 28 | -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ 29 | -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup 30 | 31 | Taking second incremental: 32 | ------------------------- 33 | 34 | :: 35 | 36 | $ autoxtrabackup --tag="Second incremental backup" -v \ 37 | -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ 38 | -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup 39 | 40 | To list available tags(backups): 41 | ------------------------------- 42 | For eg, if full backup failed, the result will be something like this: 43 | 44 | :: 45 | 46 | $ sudo autoxtrabackup --show_tags \ 47 | --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf 48 | Backup Type Status Completion_time Size TAG 49 | ---------------------------------------------------------------------------------- 50 | 2017-12-14_12-01-11 Full FAILED 2017-12-14_12-01-11 4,0K 'My Full backup' 51 | 52 | 53 | backup_tags.txt file 54 | -------------------- 55 | All tags are stored inside backup_tags.txt file, which will be created in backup directory: 56 | 57 | :: 58 | 59 | [vagrant@localhost ps_5_7_x_2_4]$ ls 60 | backup_tags.txt full inc 61 | [vagrant@localhost ps_5_7_x_2_4]$ cat backup_tags.txt 62 | $ cat backup_tags.txt 63 | 2017-12-14_12-01-11 Full FAILED 2017-12-14_12-01-11 4,0K 'My Full backup' 64 | 65 | Preparing with tag 66 | ------------------ 67 | 68 | That's very nice. Now you can use those tags to prepare your backups. 69 | Say you want to prepare only first incremental and ignore second one(or others). 70 | 71 | :: 72 | 73 | $ autoxtrabackup --tag="First incremental backup" -v \ 74 | -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ 75 | -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --prepare 76 | 2017-11-16 20:18:16 DEBUG entering setup 77 | 2017-11-16 20:18:16 DEBUG create pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 78 | 2017-11-16 20:18:16 DEBUG check pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 79 | - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 80 | 81 | Preparing full/inc backups! 82 | What do you want to do? 83 | 1. Prepare Backups and keep for future usage. NOTE('Once Prepared Backups Can not be prepared Again') 84 | 2. Prepare Backups and restore/recover/copy-back immediately 85 | 3. Just copy-back previously prepared backups 86 | Please Choose one of options and type 1 or 2 or 3: 1 87 | 88 | - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 89 | 2017-11-16 20:18:20 DEBUG Backup tag will be used to prepare backups 90 | 2017-11-16 20:18:20 DEBUG - - - - You have Incremental backups. - - - - 91 | 2017-11-16 20:18:20 DEBUG - - - - Preparing Full backup for incrementals - - - - 92 | 2017-11-16 20:18:20 DEBUG - - - - Final prepare,will occur after preparing all inc backups - - - - 93 | . 94 | . 95 | . 96 | 2017-11-16 20:18:26 DEBUG Preparing Incs: 97 | 2017-11-16 20:18:26 DEBUG Preparing last incremental backup, inc backup dir/name is 2017-11-16_20-12-23 98 | 99 | As you see it will mark given incremental backup as last one, because you have specified it in --tag option. 100 | 101 | **If you pass wrong/non-existing tag name the tool will raise RuntimeError.** 102 | 103 | -------------------------------------------------------------------------------- /docs/basic_features.rst: -------------------------------------------------------------------------------- 1 | Basic features 2 | ============== 3 | 4 | Backup 5 | ------ 6 | 7 | Yes you are right, this tool is for taking backups. 8 | It should take care for automating this process for you. 9 | You can specify the backup directory in config file (default /etc/bck.conf) under [Backup] category. 10 | So you have prepared your config and now you are ready for start. 11 | 12 | The command for taking full backup with DEBUG enabled, i.e first run of the tool. 13 | 14 | :: 15 | 16 | $ sudo autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ 17 | -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup 18 | 19 | 2017-11-16 19:37:52 DEBUG entering setup 20 | 2017-11-16 19:37:52 DEBUG create pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 21 | 2017-11-16 19:37:52 DEBUG check pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 22 | 2017-11-16 19:37:52 DEBUG Running mysqladmin command -> /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysqladmin --defaults-file= --user=root --password= status --socket=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/socket.sock 23 | mysqladmin: [Warning] Using a password on the command line interface can be insecure. 24 | 2017-11-16 19:37:52 DEBUG OK: Server is Up and running 25 | 2017-11-16 19:37:52 DEBUG OK: /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysql exists 26 | 2017-11-16 19:37:52 DEBUG OK: /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysqladmin exists 27 | 2017-11-16 19:37:52 DEBUG Skipping my.cnf check, because it is not specified 28 | 2017-11-16 19:37:52 DEBUG OK: XtraBackup exists 29 | 2017-11-16 19:37:52 DEBUG Main backup directory does not exist 30 | 2017-11-16 19:37:52 DEBUG Creating Main Backup folder... 31 | 2017-11-16 19:37:52 DEBUG OK: Created 32 | 2017-11-16 19:37:52 DEBUG Full Backup directory does not exist 33 | 2017-11-16 19:37:52 DEBUG Creating full backup directory... 34 | 2017-11-16 19:37:52 DEBUG OK: Created 35 | 2017-11-16 19:37:52 DEBUG Increment directory does not exist 36 | 2017-11-16 19:37:52 DEBUG Creating increment backup directory... 37 | 2017-11-16 19:37:52 DEBUG OK: Created 38 | 2017-11-16 19:37:52 DEBUG OK: Check status 39 | 2017-11-16 19:37:52 DEBUG - - - - You have no backups : Taking very first Full Backup! - - - - 40 | 2017-11-16 19:37:52 DEBUG Trying to flush logs 41 | 2017-11-16 19:37:52 DEBUG OK: Log flushing completed 42 | 2017-11-16 19:37:52 WARNING Streaming is enabled! 43 | 2017-11-16 19:37:52 DEBUG The following backup command will be executed /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --defaults-file= --user=root --password='' --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_19-37-52 --backup --socket=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/socket.sock --compress=quicklz --compress-chunk-size=65536 --compress-threads=4 --encrypt=AES256 --encrypt-key=VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K --encrypt-threads=4 --encrypt-chunk-size=65536 --slave-info --no-version-check --core-file --parallel=1 --throttle=40 --keyring-file-data=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/mysql-keyring/keyring --stream="xbstream" > /home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_19-37-52/full_backup.stream 44 | 45 | The result of second run; it will take an incremental backup. 46 | 47 | :: 48 | 49 | $ sudo autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ 50 | -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --backup 51 | 2017-11-16 19:43:07 DEBUG entering setup 52 | 2017-11-16 19:43:07 DEBUG create pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 53 | 2017-11-16 19:43:07 DEBUG check pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 54 | 2017-11-16 19:43:07 DEBUG Running mysqladmin command -> /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysqladmin --defaults-file= --user=root --password= status --socket=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/socket.sock 55 | mysqladmin: [Warning] Using a password on the command line interface can be insecure. 56 | 2017-11-16 19:43:07 DEBUG OK: Server is Up and running 57 | 2017-11-16 19:43:07 DEBUG OK: /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysql exists 58 | 2017-11-16 19:43:07 DEBUG OK: /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysqladmin exists 59 | 2017-11-16 19:43:07 DEBUG Skipping my.cnf check, because it is not specified 60 | 2017-11-16 19:43:07 DEBUG OK: XtraBackup exists 61 | 2017-11-16 19:43:07 DEBUG OK: Main backup directory exists 62 | 2017-11-16 19:43:07 DEBUG OK: Full Backup directory exists 63 | 2017-11-16 19:43:07 DEBUG OK: Increment directory exists 64 | 2017-11-16 19:43:07 DEBUG OK: Check status 65 | 2017-11-16 19:43:07 DEBUG - - - - You have a full backup that is less than 86400 seconds old. - - - - 66 | 2017-11-16 19:43:07 DEBUG - - - - We will take an incremental one based on recent Full Backup - - - - 67 | 2017-11-16 19:43:10 DEBUG Using xbstream to extract and decrypt from full_backup.stream! 68 | 2017-11-16 19:43:10 DEBUG The following xbstream command will be executed /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xbstream -x --parallel=100 --decrypt=AES256 --encrypt-key=VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K --encrypt-threads=4 < /home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_19-37-52/full_backup.stream -C /home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_19-37-52 69 | 2017-11-16 19:43:10 DEBUG OK: XBSTREAM command succeeded. 70 | 2017-11-16 19:43:10 WARNING Streaming is enabled! 71 | 2017-11-16 19:43:10 DEBUG The following backup command will be executed /home/shahriyar.rzaev/XB_TEST/server_dir/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup --defaults-file= --user=root --password='' --target-dir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_19-43-10 --incremental-basedir=/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/full/2017-11-16_19-37-52 --backup --socket=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/socket.sock --compress=quicklz --compress-chunk-size=65536 --compress-threads=4 --encrypt=AES256 --encrypt-key=VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K --encrypt-threads=4 --encrypt-chunk-size=65536 --slave-info --no-version-check --core-file --parallel=1 --throttle=40 --keyring-file-data=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/mysql-keyring/keyring --stream="xbstream" > /home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4/inc/2017-11-16_19-43-10/inc_backup.stream 72 | 73 | 74 | 75 | You will have 2 separate folders inside backup directory: 76 | 77 | :: 78 | 79 | [vagrant@localhost backup_dir]$ ls 80 | ps_5_7_x_2_4 81 | [vagrant@localhost backup_dir]$ cd ps_5_7_x_2_4/ 82 | [vagrant@localhost ps_5_7_x_2_4]$ ls 83 | full inc 84 | 85 | 86 | We took full backup and it should be under the ``full`` directory: 87 | 88 | :: 89 | 90 | [vagrant@localhost ps_5_7_x_2_4]$ ls full/ 91 | 2017-11-16_19-37-52 92 | 93 | Incremental backups are inside ``inc`` directory: 94 | 95 | :: 96 | 97 | [vagrant@localhost ps_5_7_x_2_4]$ ls inc/ 98 | 2017-11-16_19-43-10 99 | 100 | If you want more incremental backups just run the same command again and again. 101 | 102 | 103 | Prepare 104 | ------- 105 | For preparing backups just use --prepare option. For our case we have a 106 | full and 2 incremental backups: 107 | 108 | :: 109 | 110 | [vagrant@localhost ps_5_7_x_2_4]$ ls full/ 111 | 2017-11-16_19-37-52 112 | [vagrant@localhost ps_5_7_x_2_4]$ ls inc/ 113 | 2017-11-16_19-43-10 2017-11-16_19-48-23 114 | 115 | All backups will be prepared 116 | automatically. 117 | 118 | You are going to have 3 options to choose: 119 | 120 | 1. Only prepare backups. 121 | 2. Prepare backups and restore immediately. 122 | 3. Restore from already prepared backup. 123 | 124 | For now let's choose 1: 125 | 126 | :: 127 | 128 | $ autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ 129 | -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --prepare 130 | 2017-11-16 19:50:33 DEBUG entering setup 131 | 2017-11-16 19:50:33 DEBUG create pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 132 | 2017-11-16 19:50:33 DEBUG check pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 133 | - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 134 | 135 | Preparing full/inc backups! 136 | What do you want to do? 137 | 1. Prepare Backups and keep for future usage. NOTE('Once Prepared Backups Can not be prepared Again') 138 | 2. Prepare Backups and restore/recover/copy-back immediately 139 | 3. Just copy-back previously prepared backups 140 | Please Choose one of options and type 1 or 2 or 3: 1 141 | 142 | 143 | That's it. Your backup is ready to restore/recovery. 144 | 145 | 146 | 147 | Restore single table 148 | -------------------- 149 | 150 | If you have deleted table data and you have already prepared full server backup. 151 | You can restore single table as displayed here: 152 | 153 | :: 154 | 155 | > show create table t1\G 156 | *************************** 1. row *************************** 157 | Table: t1 158 | Create Table: CREATE TABLE t1 ( 159 | id int(11) NOT NULL 160 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1 161 | 1 row in set (0.01 sec) 162 | 163 | > select * from t1; 164 | +----+ 165 | | id | 166 | +----+ 167 | | 1 | 168 | | 1 | 169 | | 2 | 170 | | 1 | 171 | | 2 | 172 | | 3 | 173 | +----+ 174 | 6 rows in set (0.01 sec) 175 | 176 | > delete from t1; 177 | Query OK, 6 rows affected (0.12 sec) 178 | 179 | 180 | Restoring single table, the ``--partial`` option must be used for this: 181 | 182 | :: 183 | 184 | 185 | $ sudo autoxtrabackup -v -lf /home/shahriyar.rzaev/autoxtrabackup_2_4_5_7.log \ 186 | -l DEBUG --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf --partial 187 | 2017-11-16 19:56:32 DEBUG entering setup 188 | 2017-11-16 19:56:32 DEBUG create pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 189 | 2017-11-16 19:56:32 DEBUG check pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 190 | Type Database name: test 191 | Type Table name: t1 192 | 2017-11-16 19:56:56 DEBUG Running mysqladmin command -> /home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/bin/mysqladmin --defaults-file= --user=root --password= status --socket=/home/shahriyar.rzaev/XB_TEST/server_dir/PS131117-percona-server-5.7.19-17-linux-x86_64/socket.sock 193 | mysqladmin: [Warning] Using a password on the command line interface can be insecure. 194 | 2017-11-16 19:56:56 DEBUG OK: Server is Up and running 195 | 2017-11-16 19:56:56 DEBUG Checking if innodb_file_per_table is enabled 196 | 2017-11-16 19:56:56 DEBUG OK: innodb_file_per_table is enabled! 197 | 2017-11-16 19:56:56 DEBUG Checking MySQL version 198 | 2017-11-16 19:56:56 DEBUG You have correct version of MySQL 199 | 2017-11-16 19:56:56 DEBUG Checking if database exists in MySQL 200 | 2017-11-16 19:56:56 DEBUG Database exists! 201 | 2017-11-16 19:56:56 DEBUG Checking if table exists in MySQL Server 202 | 2017-11-16 19:56:57 DEBUG Table exists in MySQL Server. 203 | 2017-11-16 19:56:57 DEBUG Applying write lock! 204 | 2017-11-16 19:56:57 DEBUG OK: Table is locked 205 | 2017-11-16 19:56:57 DEBUG Discarding tablespace 206 | 2017-11-16 19:56:57 DEBUG OK: Tablespace discarded successfully 207 | 2017-11-16 19:56:57 DEBUG OK: Copying .ibd file back 208 | 2017-11-16 19:56:57 DEBUG Running chown command! 209 | 2017-11-16 19:56:57 DEBUG OK: Chown command completed 210 | 2017-11-16 19:56:57 DEBUG Importing Tablespace! 211 | 2017-11-16 19:56:57 DEBUG OK: Tablespace imported 212 | 2017-11-16 19:56:57 DEBUG Unlocking tables! 213 | 2017-11-16 19:56:57 DEBUG OK: Unlocked! 214 | 2017-11-16 19:56:57 DEBUG OK: Table Recovered! ... 215 | 2017-11-16 19:56:57 DEBUG closing pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 216 | 2017-11-16 19:56:57 DEBUG closing pidfile: /tmp/MySQL-AutoXtraBackup/autoxtrabackup.pid 217 | 218 | Congratulations you have restored table: 219 | 220 | :: 221 | 222 | > select * from t1; 223 | +----+ 224 | | id | 225 | +----+ 226 | | 1 | 227 | | 1 | 228 | | 2 | 229 | | 1 | 230 | | 2 | 231 | | 3 | 232 | +----+ 233 | 6 rows in set (0.00 sec) 234 | -------------------------------------------------------------------------------- /docs/basic_overview.rst: -------------------------------------------------------------------------------- 1 | Basic Overview 2 | ============== 3 | 4 | Project Structure 5 | ----------------- 6 | 7 | XtraBackup is a powerful open-source hot online backup tool for MySQL 8 | from Percona. This script is using XtraBackup for full and incremental 9 | backups, also for preparing backups, as well as to restore. Here is project path tree: 10 | 11 | :: 12 | 13 | * master_backup_script -- Full and Incremental backup taker script. 14 | * backup_prepare -- Backup prepare and restore script. 15 | * partial_recovery -- Partial table recovery script. 16 | * general_conf -- All-in-one config file's and config reader class folder. 17 | * prepare_env_test_mode -- The directory for --test_mode actions. 18 | * test -- The directory for test things. 19 | * setup.py -- Setuptools Setup file. 20 | * autoxtrabackup.py -- Commandline Tool provider script. 21 | * VagrantFile -- The Vagrant thing for starting using this tool[will be useful to contributors]. 22 | * /etc/bck.conf -- Config file will be created from general_conf/bck.conf 23 | 24 | 25 | Available Options 26 | ----------------- 27 | 28 | .. code-block:: shell 29 | 30 | $ sudo autoxtrabackup 31 | Usage: autoxtrabackup [OPTIONS] 32 | 33 | Options: 34 | --dry_run Enable the dry run. 35 | --prepare Prepare/recover backups. 36 | --backup Take full and incremental backups. 37 | --partial Recover specified table (partial recovery). 38 | --version Version information. 39 | --defaults_file TEXT Read options from the given file [default: 40 | /etc/bck.conf] 41 | --tag TEXT Pass the tag string for each backup 42 | --show_tags Show backup tags and exit 43 | -v, --verbose Be verbose (print to console) 44 | -lf, --log_file TEXT Set log file [default: 45 | /var/log/autoxtrabackup.log] 46 | -l, --log [DEBUG|INFO|WARNING|ERROR|CRITICAL] 47 | Set log level [default: WARNING] 48 | --test_mode Enable test mode.Must be used with 49 | --defaults_file and only for TESTs for 50 | XtraBackup 51 | --help Print help message and exit. 52 | 53 | 54 | 55 | 56 | 57 | Usage 58 | ----- 59 | 60 | :: 61 | 62 | 1. Install it. 63 | 2. Edit /etc/bck.conf file to reflect your environment or create your own config. 64 | 3. Pass this config file to autoxtrabackup with --defaults_file and begin to backup/prepare/restore. 65 | 66 | 67 | 68 | 69 | Logging 70 | -------- 71 | 72 | The logging mechanism is using Python3 logging. 73 | It lets to log directly to console and also to file. 74 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # MySQL AutoXtrabackup documentation build configuration file, created by 4 | # sphinx-quickstart on Fri Feb 24 23:39:55 2017. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | # 19 | # import os 20 | # import sys 21 | # sys.path.insert(0, os.path.abspath('.')) 22 | import sphinx_rtd_theme 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | # 28 | # needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = ['sphinx.ext.autodoc'] 34 | 35 | # Add any paths that contain templates here, relative to this directory. 36 | templates_path = ['_templates'] 37 | 38 | # The suffix(es) of source filenames. 39 | # You can specify multiple suffix as a list of string: 40 | # 41 | # source_suffix = ['.rst', '.md'] 42 | source_suffix = '.rst' 43 | 44 | # The master toctree document. 45 | master_doc = 'index' 46 | 47 | # General information about the project. 48 | project = u'MySQL-AutoXtrabackup' 49 | copyright = u'2017, Shahriyar Rzayev' 50 | author = u'Shahriyar Rzayev' 51 | 52 | # The version info for the project you're documenting, acts as replacement for 53 | # |version| and |release|, also used in various other places throughout the 54 | # built documents. 55 | # 56 | # The short X.Y version. 57 | version = u'1.5.0' 58 | # The full version, including alpha/beta/rc tags. 59 | release = u'1.5.0' 60 | 61 | # The language for content autogenerated by Sphinx. Refer to documentation 62 | # for a list of supported languages. 63 | # 64 | # This is also used if you do content translation via gettext catalogs. 65 | # Usually you set "language" from the command line for these cases. 66 | language = None 67 | 68 | # List of patterns, relative to source directory, that match files and 69 | # directories to ignore when looking for source files. 70 | # This patterns also effect to html_static_path and html_extra_path 71 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 72 | 73 | # The name of the Pygments (syntax highlighting) style to use. 74 | pygments_style = 'sphinx' 75 | 76 | # If true, `todo` and `todoList` produce output, else they produce nothing. 77 | todo_include_todos = False 78 | 79 | 80 | # -- Options for HTML output ---------------------------------------------- 81 | 82 | # The theme to use for HTML and HTML Help pages. See the documentation for 83 | # a list of builtin themes. 84 | # 85 | html_theme = "sphinx_rtd_theme" 86 | 87 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 88 | 89 | # Theme options are theme-specific and customize the look and feel of a theme 90 | # further. For a list of options available for each theme, see the 91 | # documentation. 92 | # 93 | # html_theme_options = {} 94 | 95 | # Add any paths that contain custom static files (such as style sheets) here, 96 | # relative to this directory. They are copied after the builtin static files, 97 | # so a file named "default.css" will overwrite the builtin "default.css". 98 | html_static_path = ['_static'] 99 | 100 | 101 | # -- Options for HTMLHelp output ------------------------------------------ 102 | 103 | # Output file base name for HTML help builder. 104 | htmlhelp_basename = 'MySQLAutoXtrabackupdoc' 105 | 106 | 107 | # -- Options for LaTeX output --------------------------------------------- 108 | 109 | latex_elements = { 110 | # The paper size ('letterpaper' or 'a4paper'). 111 | # 112 | # 'papersize': 'letterpaper', 113 | 114 | # The font size ('10pt', '11pt' or '12pt'). 115 | # 116 | # 'pointsize': '10pt', 117 | 118 | # Additional stuff for the LaTeX preamble. 119 | # 120 | # 'preamble': '', 121 | 122 | # Latex figure (float) alignment 123 | # 124 | # 'figure_align': 'htbp', 125 | } 126 | 127 | # Grouping the document tree into LaTeX files. List of tuples 128 | # (source start file, target name, title, 129 | # author, documentclass [howto, manual, or own class]). 130 | latex_documents = [ 131 | (master_doc, 'MySQLAutoXtrabackup.tex', u'MySQL AutoXtrabackup Documentation', 132 | u'Shahriyar Rzayev', 'manual'), 133 | ] 134 | 135 | 136 | # -- Options for manual page output --------------------------------------- 137 | 138 | # One entry per manual page. List of tuples 139 | # (source start file, name, description, authors, manual section). 140 | man_pages = [ 141 | (master_doc, 'mysqlautoxtrabackup', u'MySQL AutoXtrabackup Documentation', 142 | [author], 1) 143 | ] 144 | 145 | 146 | # -- Options for Texinfo output ------------------------------------------- 147 | 148 | # Grouping the document tree into Texinfo files. List of tuples 149 | # (source start file, target name, title, author, 150 | # dir menu entry, description, category) 151 | texinfo_documents = [ 152 | (master_doc, 'MySQLAutoXtrabackup', u'MySQL AutoXtrabackup Documentation', 153 | author, 'MySQLAutoXtrabackup', 'One line description of project.', 154 | 'Miscellaneous'), 155 | ] 156 | 157 | 158 | 159 | -------------------------------------------------------------------------------- /docs/config_file.rst: -------------------------------------------------------------------------------- 1 | The structure of configuration file 2 | =================================== 3 | 4 | Defaults file explained 5 | ----------------------- 6 | 7 | The default config file is located /etc/bck.conf. 8 | The available options are divided into optional and primary options. 9 | Options are quite self-explanatory. 10 | I have tried to make them similar to existing options in XtraBackup and MySQL. 11 | You can use another configuration file using ``--defaults_file`` option. 12 | 13 | Let's clarify the config file structure a bit. 14 | The [MySQL] category is for specifying information about MySQL instance. 15 | 16 | :: 17 | 18 | [MySQL] 19 | mysql=/usr/bin/mysql 20 | mycnf=/etc/my.cnf 21 | mysqladmin=/usr/bin/mysqladmin 22 | mysql_user=root 23 | mysql_password=12345 24 | #Set either mysql_socket or host and post. If both are set socket is used 25 | #mysql_socket=/var/lib/mysql/mysql.sock 26 | mysql_host=127.0.0.1 27 | mysql_port=3306 28 | datadir=/var/lib/mysql 29 | 30 | 31 | The [Backup] category is for specifying information about backup/prepare process itself. 32 | 33 | :: 34 | 35 | [Backup] 36 | #Optional: set pid directory 37 | pid_dir=/tmp/MySQL-AutoXtraBackup 38 | tmpdir=/home/shahriyar.rzaev/XB_TEST/mysql_datadirs 39 | #Optional: set warning if pid of backup us running for longer than X 40 | pid_runtime_warning=2 Hours 41 | backupdir=/home/shahriyar.rzaev/XB_TEST/backup_dir 42 | backup_tool=/usr/bin/xtrabackup 43 | #Optional: specify different path/version of xtrabackup here for prepare 44 | #prepare_tool= 45 | xtra_prepare=--apply-log-only 46 | #Optional: pass additional options for backup stage 47 | #xtra_backup=--compact 48 | #Optional: pass additional options for prepare stage 49 | #xtra_prepare_options=--rebuild-indexes 50 | #Optional: pass general additional options; it will go to both for backup and prepare 51 | #xtra_options=--binlog-info=ON --galera-info 52 | xtra_options=--no-version-check 53 | #Optional: set archive and rotation 54 | #archive_dir=/home/shahriyar.rzaev/XB_TEST/backup_archives 55 | #full_backup_interval=1 day 56 | #max_archive_size=100GiB 57 | #max_archive_duration=4 Days 58 | #Optional WARNING(Enable this if you want to take partial backups). Specify database names or table names. 59 | #partial_list=test.t1 test.t2 dbtest 60 | 61 | 62 | The [Compress] category is for enabling backup compression. 63 | The options will be passed to XtraBackup. 64 | 65 | :: 66 | 67 | [Compress] 68 | #Optional 69 | #Enable only if you want to use compression. 70 | #compress=quicklz 71 | #compress_chunk_size=65536 72 | #compress_threads=4 73 | #decompress=TRUE 74 | #Enable if you want to remove .qp files after decompression.(Not available yet, will be released with XB 2.3.7 and 2.4.6) 75 | #remove_original=FALSE 76 | 77 | The [Encrypt] category is for enabling backup encryption. 78 | The options will be passed to XtraBackup. 79 | 80 | :: 81 | 82 | [Encrypt] 83 | #Optional 84 | #Enable only if you want to create encrypted backups 85 | #xbcrypt=/usr/bin/xbcrypt 86 | #encrypt=AES256 87 | # Please note that --encrypt-key and --encrypt-key-file are mutually exclusive 88 | #encrypt_key='VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K' 89 | #encrypt_key_file=/path/to/file/with_encrypt_key 90 | #encrypt_threads=4 91 | #encrypt_chunk_size=65536 92 | #decrypt=AES256 93 | #Enable if you want to remove .qp files after decompression.(Not available yet, will be released with XB 2.3.7 and 2.4.6) 94 | #remove_original=FALSE 95 | 96 | The [Xbstream] category is for enabling backup streaming. 97 | The options will be passed to XtraBackup. 98 | 99 | :: 100 | 101 | [Xbstream] 102 | #EXPERIMENTAL 103 | # Enable this, if you want to stream your backups 104 | #xbstream=/usr/bin/xbstream 105 | #stream=xbstream 106 | #Optional 107 | #Please enable this and disable all other options here, for tar streaming 108 | #stream=tar 109 | #xbstream_options=-x --parallel=100 110 | #xbs_decrypt=1 111 | # WARN, enable this, if you want to stream your backups to remote host 112 | #remote_stream=ssh xxx.xxx.xxx.xxx 113 | 114 | 115 | Deprecated feature, will be removed in next releases 116 | 117 | :: 118 | 119 | #Optional remote syncing 120 | #[Remote] 121 | #remote_conn=root@xxx.xxx.xxx.xxx 122 | #remote_dir=/home/sh/Documents 123 | 124 | The [Commands] category is for specifying some options for copy-back/restore actions. 125 | 126 | :: 127 | 128 | [Commands] 129 | start_mysql_command=service mysql start 130 | stop_mysql_command=service mysql stop 131 | #Change user:group respectively 132 | chown_command=chown -R mysql:mysql 133 | 134 | The [TestConf] category is part of XtraBackup testing procedures and is not for daily usage. 135 | So just ignore this, it is actually for myself :) 136 | 137 | :: 138 | 139 | # Do not touch; this is for --test_mode, which is testing for XtraBackup itself. 140 | [TestConf] 141 | ps_branches=5.5 5.6 5.7 142 | pxb_branches=2.3 2.4 143 | gitcmd=--recursive --depth=1 https://github.com/percona/percona-server.git 144 | pxb_gitcmd=https://github.com/percona/percona-xtrabackup.git 145 | testpath=/home/shahriyar.rzaev/XB_TEST/server_dir 146 | incremental_count=3 147 | #make_slaves=1 148 | xb_configs=xb_2_4_ps_5_6.conf xb_2_4_ps_5_7.conf xb_2_3_ps_5_6.conf xb_2_3_ps_5_5.conf xb_2_4_ps_5_5.conf 149 | default_mysql_options=--log-bin=mysql-bin,--log-slave-updates,--server-id={},--gtid-mode=ON,--enforce-gtid-consistency,--binlog-format=row 150 | mysql_options=--innodb_buffer_pool_size=1G 2G 3G,--innodb_log_file_size=1G 2G 3G,--innodb_page_size=4K 8K 16K 32K 64K 151 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. MySQL AutoXtrabackup documentation master file, created by 2 | sphinx-quickstart on Fri Feb 24 23:39:55 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to MySQL-AutoXtrabackup's documentation! 7 | ================================================ 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | intro 13 | basic_overview 14 | installation 15 | config_file 16 | basic_features 17 | backup_tags 18 | advance_features 19 | test_mode 20 | option_reference 21 | 22 | Indices and tables 23 | ================== 24 | 25 | * :ref:`genindex` 26 | * :ref:`modindex` 27 | * :ref:`search` 28 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | System requirements 5 | ------------------- 6 | 7 | Following packages should be already there: 8 | 9 | - Percona Xtrabackup (>= 2.3.5) 10 | - Python 3 (tested version 3.5.3 on CentOS 7) 11 | - mysql-utilities (>=1.5.4) 12 | 13 | Preparing the system 14 | -------------------- 15 | 16 | Installing dependencies: 17 | 18 | :: 19 | 20 | yum install openssl openssl-devel zlib zlib-devel 21 | 22 | Installing latest XtraBackup: 23 | For more options please refer to official documentation -> `Installing Percona XtraBackup 2.4 `_ 24 | 25 | :: 26 | 27 | yum install http://www.percona.com/downloads/percona-release/redhat/0.1-3/percona-release-0.1-3.noarch.rpm 28 | yum install percona-xtrabackup 29 | or 30 | yum install percona-xtrabackup-24 31 | 32 | Installing Python 3 from source: 33 | 34 | :: 35 | 36 | wget https://www.python.org/ftp/python/3.5.3/Python-3.5.3.tgz 37 | tar -xf Python-3.5.3.tgz 38 | cd Python-3.5.3 39 | 40 | -- Open Setup.dist file and search for zlib, uncomment zlib notes: 41 | * nano Modules/Setup.dist 42 | # See http://www.gzip.org/zlib/ 43 | zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz 44 | 45 | -- Also search for ssl and uncomment ssl section: 46 | 47 | # socket line above, and possibly edit the SSL variable: 48 | SSL=/usr/local/ssl 49 | _ssl _ssl.c \ 50 | -DUSE_SSL -I$(SSL)/include -I$(SSL)/include/openssl \ 51 | -L$(SSL)/lib -lssl -lcrypto 52 | 53 | mkdir /opt/Python-3.5.3 54 | 55 | ./configure --prefix=/opt/Python-3.5.3 56 | make 57 | make install 58 | 59 | Installing mysql-connector-python and mysql-utilities: 60 | (For CentOS 7). 61 | :: 62 | 63 | wget https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-2.1.5-1.el7.x86_64.rpm 64 | yum install mysql-connector-python-2.1.5-1.el7.x86_64.rpm 65 | 66 | :: 67 | 68 | wget https://dev.mysql.com/get/Downloads/MySQLGUITools/mysql-utilities-1.6.5-1.el7.noarch.rpm 69 | yum install mysql-utilities-1.6.5-1.el7.noarch.rpm 70 | 71 | Installing MySQL-AutoXtraBackup 72 | ------------------------------- 73 | 74 | Using pip3: 75 | 76 | :: 77 | 78 | pip3 install mysql-autoxtrabackup 79 | 80 | Installing from source: 81 | 82 | :: 83 | 84 | cd /home git clone https://github.com/ShahriyarR/MySQL-AutoXtraBackup.git 85 | cd /home/MySQL-AutoXtraBackup/ 86 | sudo python3 setup.py install 87 | -------------------------------------------------------------------------------- /docs/intro.rst: -------------------------------------------------------------------------------- 1 | Intro 2 | ===== 3 | 4 | What is this? 5 | ------------- 6 | 7 | MySQL-AutoXtraBackup is a commandline tool written in Python3 based on 8 | Percona XtraBackup. 9 | It is aimed to simplify the usage of XtraBackup in 10 | daily basis and to help backup admin in certain tasks. 11 | 12 | Why you need this? 13 | ------------------ 14 | 15 | The idea for this tool, came from my hard times after accidentally 16 | deleting the table data. 17 | There was a full backup and 12 incremental backups. 18 | It took me 20 minutes to prepare necessary commands for preparing 19 | backups. If you have compressed + encrypted backups you need also, 20 | decrypt and decompress, which is going to add extra time for preparing 21 | backups. Then I decided to automate this process. In other words, 22 | preparing necessary commands for backup and prepare stage were 23 | automated. 24 | 25 | After a while, there was an issue, the log table was dropped on test 26 | server for testing our reaction speed :) 27 | I have decided to recover only that table(partial recovery). 28 | But still did a bunch of manual tasks, which was quite time consuming process. That was the reason for 29 | implementing partial recovery functionality. So the necessary actions 30 | for restoring single table was added to automate this process as well. 31 | 32 | The recent thing is, --test_mode for testing XtraBackup[So it is for me, or for somebody who wants to find bug in XtraBackup] 33 | 34 | If you think that those reasons are not enough - Then just believe me 35 | you need this :) 36 | 37 | Is it production ready? 38 | ----------------------- 39 | 40 | Well, we have famous answer for this - "It depends!". 41 | Basically this tool is based on Percona XtraBackup and it is using XtraBackup's 42 | functionalities, there is also a 'mysqlfrm' (from mysql-utilities) tool 43 | usage which is quite safe, happy tool. 44 | For me, I have used it in production environment after testing for a while in test servers. 45 | But to be clear, just test it enough to be confident. 46 | -------------------------------------------------------------------------------- /docs/option_reference.rst: -------------------------------------------------------------------------------- 1 | Option Reference 2 | ================= 3 | 4 | The command line options to use: 5 | 6 | .. code-block:: shell 7 | 8 | 9 | $ sudo autoxtrabackup 10 | Usage: autoxtrabackup [OPTIONS] 11 | 12 | Options: 13 | --dry_run Enable the dry run. 14 | --prepare Prepare/recover backups. 15 | --backup Take full and incremental backups. 16 | --partial Recover specified table (partial recovery). 17 | --version Version information. 18 | --defaults_file TEXT Read options from the given file [default: 19 | /etc/bck.conf] 20 | --tag TEXT Pass the tag string for each backup 21 | --show_tags Show backup tags and exit 22 | -v, --verbose Be verbose (print to console) 23 | -lf, --log_file TEXT Set log file [default: 24 | /var/log/autoxtrabackup.log] 25 | -l, --log [DEBUG|INFO|WARNING|ERROR|CRITICAL] 26 | Set log level [default: WARNING] 27 | --test_mode Enable test mode.Must be used with 28 | --defaults_file and only for TESTs for 29 | XtraBackup 30 | --help Print help message and exit. 31 | 32 | 33 | dry_run 34 | ------- 35 | 36 | --dry-run 37 | Use this option to enable dry running. If you enable this, actual commands will not be executed but, will only be shown. 38 | It is useful, when you want to see what is going to happen, if you run the actual process. 39 | 40 | prepare 41 | ------- 42 | 43 | --prepare 44 | This option is for prepare and copy-back(recover) the backup. 45 | 46 | 47 | backup 48 | ------ 49 | 50 | --backup 51 | This option for taking backups. If it is first run, it will take full backup. 52 | If you want incremental backups, just run same command as much as you want take incremental backups. 53 | 54 | partial 55 | ------- 56 | 57 | This option will let you recover only desired tables. The version of MySQL must be > 5.6. 58 | It is using transportable tablespace concept. 59 | 60 | version 61 | ------- 62 | 63 | --version 64 | Prints version information. 65 | 66 | defaults_file 67 | ------------- 68 | 69 | --defaults_file 70 | The main config file to path to ``autoxtrabackup``. The default one is ``/etc/bck.conf``. 71 | In default config, the compression, encryption and streaming backups are disabled by defualt. 72 | 73 | tag 74 | ---- 75 | --tag 76 | This option enables creation of tags for backups. 77 | The backup_tags.txt file will be created and stored inside backup directory. 78 | 79 | show_tags 80 | --------- 81 | It will show the backup tags and exit. 82 | 83 | verbose, v 84 | ---------- 85 | 86 | --verbose, -v 87 | This option enables to print to console the logging messages. 88 | 89 | log_file, lf 90 | ------------ 91 | 92 | -lf, --log_file 93 | Pass, the path for log file, for autoxtrabackup. Default is ``/var/log/autoxtrabackup.log`` 94 | 95 | log 96 | ---- 97 | 98 | -l, --log 99 | 100 | Set the log level for tool. 101 | 102 | test_mode 103 | --------- 104 | 105 | --test_mode 106 | This option enables Test Mode and must be used with --defaults_file option. 107 | WARNING: It is not for daily usage. It is only and only for testing XtraBackup. 108 | 109 | 110 | help 111 | ---- 112 | 113 | --help 114 | As name indicates. 115 | 116 | -------------------------------------------------------------------------------- /docs/test_mode.rst: -------------------------------------------------------------------------------- 1 | Using --test_mode 2 | ================ 3 | 4 | I have added --test_mode option for testing XtraBackup itself. 5 | 6 | > This is not for general usage. 7 | 8 | Here is the brief flow for this: 9 | 10 | * Clone percona-qa repo 11 | * Clone Percona Server 5.6 and 5.7 from github. 12 | * Build PS servers in debug mode. 13 | * Get 2.3 and 2.4 versions of XtraBackup 14 | * Generate autoxtrabackup .conf files for each version PS and XtraBackup 15 | * Pass different combination of options to PS start command; Initialize PS servers each time with different options. 16 | * Run sysbench against each started PS server 17 | * Took backup in cycles for each started PS + prepare 18 | * If make_slaves is defined then create slave1 server from this backup(i.e copy-back to another directory and start slave from it) 19 | * Then take backup, prepare and copy-back from this new slave1 and create slave2 20 | * Run pt-table-checksum on master to check backup consistency 21 | 22 | Now let's talk a bit more here. 23 | 24 | Preparing test environment 25 | -------------------------- 26 | 27 | For simplicity I have created ``prepare_env.bats`` file for preparing test environment in fully automated manner. 28 | You need BATS_. for this. 29 | 30 | .. _BATS: https://github.com/sstephenson/bats 31 | 32 | Prior actions can be taken to use setup in advance: 33 | 34 | * Edit /etc/bck.conf and specify testpath under [TestConf] - in which path the test environment should be constructed, created. 35 | * ps_branches - specify which branches of PS should be cloned and build. 36 | * pxb_branches - specify which branches of PXB should be cloned and build. 37 | * gitcmd - specify a link of PS repo. 38 | * pxb_gitcmd - specify a link of PXB repo. 39 | 40 | **Running whole environment setup** 41 | 42 | The thing you need is to run: 43 | 44 | ``bats prepare_env.bats`` 45 | 46 | That's it, it will do the all job. 47 | 48 | **Running specific bats files** 49 | 50 | I have prepared separate bats files to run specific things: 51 | 52 | * ``test_clone_percona_qa.bats`` -> will clone percona-qa repo. 53 | * ``test_clone_ps_server_from_conf.bats`` -> will clone PS servers based on specified branches. 54 | * ``test_clone_pxb.bats`` -> will clone specified PXB based on specified branches. 55 | * ``test_build_pxb.bats`` -> will build cloned PXBs and create separate binary archives for each branch. 56 | * ``test_build_server.bats`` -> will build PS servers. 57 | * ``test_prepare_startup.bats`` -> will run startup.sh from percona-qa inside PS basedirs. 58 | * ``test_prepare_start_dynamic.bats`` -> will create start_dynamic scripts inside PS basedirs, which is going to be used in slave setup etc. 59 | * ``test_start_server.bats`` -> will start PS servers with default values(executing start script inside basedirs). 60 | * ``test_extract_xb_archive.bats`` -> will extracting PXB binary archives to the target folder inside testpath(which is grabbed from config file). 61 | * ``test_generate_config_files.bats`` -> will generate specific config files based on PXB and PS versions. 62 | 63 | After running this you will likely have something like in your test path: 64 | 65 | 66 | .. code-block:: shell 67 | 68 | [shahriyar.rzaev@qaserver-02 ~]$ cd XB_TEST/ 69 | [shahriyar.rzaev@qaserver-02 XB_TEST]$ ls 70 | server_dir 71 | [shahriyar.rzaev@qaserver-02 XB_TEST]$ ls server_dir/ 72 | percona-qa PS231017-percona-server-5.6.37-82.2-linux-x86_64-debug.tar.gz PS-5.6-trunk_dbg xb_2_3_ps_5_6.conf 73 | percona-xtrabackup-2.3.x-debug.tar.gz PS231017-percona-server-5.7.19-17-linux-x86_64-debug PS-5.7-trunk xb_2_4_ps_5_6.conf 74 | percona-xtrabackup-2.4.x-debug.tar.gz PS231017-percona-server-5.7.19-17-linux-x86_64-debug.tar.gz PS-5.7-trunk_dbg xb_2_4_ps_5_7.conf 75 | PS231017-percona-server-5.6.37-82.2-linux-x86_64-debug PS-5.6-trunk target 76 | 77 | So you have everything you need to run combination of tests for XtraBackup. Even configs are generated for you. 78 | 79 | 80 | Running test mode 81 | ----------------- 82 | 83 | For this, just run autoxtrabackup with respective config file: 84 | 85 | .. code-block:: shell 86 | 87 | autoxtrabackup -v -l DEBUG --test_mode \ 88 | --defaults_file=/home/shahriyar.rzaev/XB_TEST/server_dir/xb_2_4_ps_5_7.conf 89 | 90 | This will start autoxtrabackup in test mode and will run full cycle based on combinations of mysql options passed to PS. 91 | To be clear, for eg, we have 50 different combinations of starting PS, then we will have 50 cycles of backup/restore process. 92 | 93 | 94 | Where I can add more mysql options? 95 | ----------------------------------- 96 | 97 | In generated configs you can add more PS(mysql) startup/initialization options. 98 | For test mode [TestConf] category is relevant. Let's go through options 99 | 100 | :: 101 | 102 | # Do not touch; this is for --test_mode, which is testing for XtraBackup itself. 103 | [TestConf] 104 | ps_branches=5.6 5.7 105 | pxb_branches=2.3 2.4 106 | gitcmd=--recursive --depth=1 https://github.com/percona/percona-server.git 107 | pxb_gitcmd=https://github.com/percona/percona-xtrabackup.git 108 | testpath=/home/shahriyar.rzaev/XB_TEST/server_dir 109 | incremental_count=3 110 | #make_slaves=1 111 | xb_configs=xb_2_4_ps_5_6.conf xb_2_4_ps_5_7.conf xb_2_3_ps_5_6.conf 112 | default_mysql_options=--log-bin=mysql-bin,--log-slave-updates,--server-id={},--gtid-mode=ON,--enforce-gtid-consistency,--binlog-format=row 113 | mysql_options=--innodb_buffer_pool_size=1G 2G 3G,--innodb_log_file_size=1G 2G 3G,--innodb_page_size=4K 8K 16K 32K 64K 114 | 115 | ``ps_branches`` is for specifying PS branches. 116 | 117 | ``pxb_branches`` is for specifying PXB branches. 118 | 119 | ``pxb_gitcmd`` is for passing repo link. 120 | 121 | ``gitcmd`` is for passing git command for git clone. 122 | 123 | ``testpath`` is for passing the path for test mode. 124 | 125 | ``incremental_count`` specify how many incremental backups the tool should take. 126 | 127 | ``make_slaves`` specify if you want to create slave servers. 128 | 129 | ``xb_configs`` is for passing config files to be generated. 130 | 131 | ``default_mysql_options`` default mysql options to pass to PS start script. 132 | 133 | ``mysql_options`` option combinations are for passing mysql startup/initialization options to PS start script. 134 | 135 | Internally, based on mysql options, the combination of those options will be created. 136 | So just add more options to ``mysql_options`` if you want more. 137 | 138 | 139 | Important things to remember 140 | ----------------------------- 141 | 142 | This is tested only with Percona Servers, but can be expanded. 143 | Also --test_mode option is mutually exclusive with other options such as --backup and --prepare. 144 | So basically do not touch this, if you are not testing XtraBackup. -------------------------------------------------------------------------------- /general_conf/__init__.py: -------------------------------------------------------------------------------- 1 | from general_conf import generalops 2 | from general_conf import check_env -------------------------------------------------------------------------------- /general_conf/bck.conf: -------------------------------------------------------------------------------- 1 | [MySQL] 2 | mysql=/usr/bin/mysql 3 | mycnf=/etc/my.cnf 4 | mysqladmin=/usr/bin/mysqladmin 5 | mysql_user=root 6 | mysql_password=12345 7 | #Set either mysql_socket or host and post. If both are set socket is used 8 | #mysql_socket=/var/lib/mysql/mysql.sock 9 | mysql_host=127.0.0.1 10 | mysql_port=3306 11 | datadir=/var/lib/mysql 12 | 13 | [Backup] 14 | #Optional: set pid directory 15 | pid_dir=/tmp/MySQL-AutoXtraBackup 16 | tmpdir=/home/shahriyar.rzaev/XB_TEST/mysql_datadirs 17 | #Optional: set warning if pid of backup us running for longer than X 18 | pid_runtime_warning=2 Hours 19 | backupdir=/home/shahriyar.rzaev/XB_TEST/backup_dir 20 | backup_tool=/usr/bin/xtrabackup 21 | #Optional: specify different path/version of xtrabackup here for prepare 22 | #prepare_tool= 23 | xtra_prepare=--apply-log-only 24 | #Optional: pass additional options for backup stage 25 | #xtra_backup=--compact 26 | #Optional: pass additional options for prepare stage 27 | #xtra_prepare_options=--rebuild-indexes 28 | #Optional: pass general additional options; it will go to both for backup and prepare 29 | #xtra_options=--binlog-info=ON --galera-info 30 | xtra_options=--no-version-check 31 | #Optional: set archive and rotation 32 | #archive_dir=/home/shahriyar.rzaev/XB_TEST/backup_archives 33 | #prepare_archive=1 34 | #move_archive=0 35 | #full_backup_interval=1 day 36 | #max_archive_size=100GiB 37 | #max_archive_duration=4 Days 38 | #Optional WARNING(Enable this if you want to take partial backups). Specify database names or table names. 39 | #partial_list=test.t1 test.t2 dbtest 40 | 41 | [Compress] 42 | #Optional 43 | #Enable only if you want to use compression. 44 | #compress=quicklz 45 | #compress_chunk_size=65536 46 | #compress_threads=4 47 | #decompress=TRUE 48 | #Enable if you want to remove .qp files after decompression.(Not available yet, will be released with XB 2.3.7 and 2.4.6) 49 | #remove_original=FALSE 50 | 51 | [Encrypt] 52 | #Optional 53 | #Enable only if you want to create encrypted backups 54 | #xbcrypt=/usr/bin/xbcrypt 55 | #encrypt=AES256 56 | # Please note that --encrypt-key and --encrypt-key-file are mutually exclusive 57 | #encrypt_key='VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K' 58 | #encrypt_key_file=/path/to/file/with_encrypt_key 59 | #encrypt_threads=4 60 | #encrypt_chunk_size=65536 61 | #decrypt=AES256 62 | #Enable if you want to remove .qp files after decompression.(Not available yet, will be released with XB 2.3.7 and 2.4.6) 63 | #remove_original=FALSE 64 | 65 | [Xbstream] 66 | #EXPERIMENTAL 67 | # Enable this, if you want to stream your backups 68 | #xbstream=/usr/bin/xbstream 69 | #stream=xbstream 70 | #Optional 71 | #Please enable this and disable all other options here, for tar streaming 72 | #stream=tar 73 | #xbstream_options=-x --parallel=100 74 | #xbs_decrypt=1 75 | # WARN, enable this, if you want to stream your backups to remote host 76 | #remote_stream=ssh xxx.xxx.xxx.xxx 77 | 78 | #Optional remote syncing 79 | #[Remote] 80 | #remote_conn=root@xxx.xxx.xxx.xxx 81 | #remote_dir=/home/sh/Documents 82 | 83 | 84 | [Commands] 85 | start_mysql_command=service mysql start 86 | stop_mysql_command=service mysql stop 87 | #Change user:group respectively 88 | chown_command=chown -R mysql:mysql 89 | 90 | 91 | # Do not touch; this is for --test_mode, which is testing for XtraBackup itself. 92 | #[TestConf] 93 | #ps_branches=5.5 5.6 5.7 94 | #pxb_branches=2.3 2.4 95 | #gitcmd=--recursive --depth=1 https://github.com/percona/percona-server.git 96 | #pxb_gitcmd=https://github.com/percona/percona-xtrabackup.git 97 | #testpath=/home/shahriyar.rzaev/XB_TEST/server_dir 98 | #incremental_count=3 99 | #make_slaves=1 100 | #xb_configs=xb_2_4_ps_5_6.conf xb_2_4_ps_5_7.conf xb_2_3_ps_5_6.conf xb_2_3_ps_5_5.conf xb_2_4_ps_5_5.conf 101 | #default_mysql_options=--log-bin=mysql-bin,--log-slave-updates,--server-id={},--gtid-mode=ON,--enforce-gtid-consistency,--binlog-format=row 102 | #mysql_options=--innodb_buffer_pool_size=1G 2G 3G,--innodb_log_file_size=1G 2G 3G,--innodb_page_size=4K 8K 16K 32K 64K -------------------------------------------------------------------------------- /general_conf/check_env.py: -------------------------------------------------------------------------------- 1 | #!/opt/Python-3.3.2/bin/python3 2 | 3 | import shlex 4 | import subprocess 5 | import os 6 | import time 7 | from general_conf.generalops import GeneralClass 8 | 9 | import logging 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class CheckEnv(GeneralClass): 14 | 15 | def __init__(self, config='/etc/bck.conf', full_dir=None, inc_dir=None): 16 | self.conf = config 17 | GeneralClass.__init__(self, self.conf) 18 | if full_dir is not None: 19 | self.full_dir = full_dir 20 | if inc_dir is not None: 21 | self.inc_dir = inc_dir 22 | 23 | def check_mysql_uptime(self, options=None): 24 | ''' 25 | Method for checking if MySQL server is up or not. 26 | :param options: Passed options to connect to MySQL server if None, then going to get it from conf file 27 | :return: True on success, raise RuntimeError on error. 28 | ''' 29 | if options is None: 30 | 31 | statusargs = '{} --defaults-file={} --user={} --password={} status'.format(self.mysqladmin, 32 | self.mycnf, 33 | self.mysql_user, 34 | self.mysql_password) 35 | 36 | if hasattr(self, 'mysql_socket'): 37 | statusargs += " --socket={}".format(self.mysql_socket) 38 | elif hasattr(self, 'mysql_host') and hasattr(self, 'mysql_port'): 39 | statusargs += " --host={}".format(self.mysql_host) 40 | statusargs += " --port={}".format(self.mysql_port) 41 | else: 42 | logger.critical("Neither mysql_socket nor mysql_host and mysql_port are defined in config!") 43 | raise RuntimeError("Neither mysql_socket nor mysql_host and mysql_port are defined in config!") 44 | else: 45 | statusargs = "{} {} status".format(self.mysqladmin, options) 46 | 47 | logger.debug("Running mysqladmin command -> {}".format(statusargs)) 48 | #statusargs = shlex.split(statusargs) 49 | status, output = subprocess.getstatusoutput(statusargs) 50 | #myadmin = subprocess.Popen(statusargs, stdout=subprocess.PIPE) 51 | 52 | if status == 0: 53 | logger.debug('OK: Server is Up and running') 54 | return True 55 | else: 56 | logger.error('FAILED: Server is NOT Up') 57 | raise RuntimeError('FAILED: Server is NOT Up') 58 | 59 | # if not ('Uptime' in str(myadmin.stdout.read())): 60 | # logger.error('FAILED: Server is NOT Up') 61 | # raise RuntimeError('FAILED: Server is NOT Up') 62 | # else: 63 | # logger.debug('OK: Server is Up and running') 64 | # return True 65 | 66 | def check_mysql_conf(self): 67 | ''' 68 | Method for checking passed MySQL my.cnf defaults file. If it is not passed then skip this check 69 | :return: True on success, raise RuntimeError on error. 70 | ''' 71 | if self.mycnf is None or self.mycnf == '': 72 | logger.debug("Skipping my.cnf check, because it is not specified") 73 | return True 74 | elif not os.path.exists(self.mycnf) and (self.mycnf is not None): 75 | logger.error('FAILED: MySQL configuration file path does NOT exist') 76 | raise RuntimeError('FAILED: MySQL configuration file path does NOT exist') 77 | else: 78 | logger.debug('OK: MySQL configuration file exists') 79 | return True 80 | 81 | def check_mysql_mysql(self): 82 | ''' 83 | Method for checking mysql client path 84 | :return: True on success, raise RuntimeError on error. 85 | ''' 86 | if not os.path.exists(self.mysql): 87 | logger.error('FAILED: {} doest NOT exist'.format(self.mysql)) 88 | raise RuntimeError('FAILED: {} doest NOT exist'.format(self.mysql)) 89 | else: 90 | logger.debug('OK: {} exists'.format(self.mysql)) 91 | return True 92 | 93 | def check_mysql_mysqladmin(self): 94 | ''' 95 | Method for checking mysqladmin path 96 | :return: True on success, raise RuntimeError on error. 97 | ''' 98 | if not os.path.exists(self.mysqladmin): 99 | logger.error('FAILED: {} does NOT exist'.format(self.mysqladmin)) 100 | raise RuntimeError('FAILED: {} does NOT exist'.format(self.mysqladmin)) 101 | else: 102 | logger.debug('OK: {} exists'.format(self.mysqladmin)) 103 | return True 104 | 105 | def check_mysql_backuptool(self): 106 | if not os.path.exists(self.backup_tool): 107 | logger.error('FAILED: XtraBackup does NOT exist') 108 | raise RuntimeError('FAILED: XtraBackup does NOT exist') 109 | else: 110 | logger.debug('OK: XtraBackup exists') 111 | return True 112 | 113 | def check_mysql_backupdir(self): 114 | ''' 115 | Check for MySQL backup directory. 116 | If directory exists already then, return True. If not, try to create it. 117 | :return: True on success. 118 | ''' 119 | if not (os.path.exists(self.backupdir)): 120 | logger.debug('Main backup directory does not exist') 121 | logger.debug('Creating Main Backup folder...') 122 | os.makedirs(self.backupdir) 123 | logger.debug('OK: Created') 124 | return True 125 | else: 126 | logger.debug('OK: Main backup directory exists') 127 | return True 128 | 129 | def check_mysql_archive_dir(self): 130 | ''' 131 | Check for archive directory. 132 | If archive_dir is given in config file and if it is does not exist, try to create. 133 | :return: True on success. 134 | ''' 135 | if hasattr(self, 'archive_dir'): 136 | if not (os.path.exists(self.archive_dir)): 137 | logger.debug('Archive backup directory does not exist') 138 | logger.debug('Creating archive folder...') 139 | os.makedirs(self.archive_dir) 140 | logger.debug('OK: Created') 141 | return True 142 | else: 143 | logger.debug('OK: Archive folder directory exists') 144 | return True 145 | else: 146 | return True 147 | 148 | def check_mysql_fullbackupdir(self): 149 | ''' 150 | Check full backup directory path. 151 | If this path exists return True if not try to create. 152 | :return: True on success. 153 | ''' 154 | if not (os.path.exists(self.full_dir)): 155 | logger.debug('Full Backup directory does not exist') 156 | logger.debug('Creating full backup directory...') 157 | os.makedirs(self.full_dir) 158 | logger.debug('OK: Created') 159 | return True 160 | else: 161 | logger.debug("OK: Full Backup directory exists") 162 | return True 163 | 164 | def check_mysql_incbackupdir(self): 165 | ''' 166 | Check incremental backup directory path. 167 | If this path exists return True if not try to create. 168 | :return: True on success. 169 | ''' 170 | if not (os.path.exists(self.inc_dir)): 171 | logger.debug('Increment directory does not exist') 172 | logger.debug('Creating increment backup directory...') 173 | os.makedirs(self.inc_dir) 174 | logger.debug('OK: Created') 175 | return True 176 | else: 177 | logger.debug('OK: Increment directory exists') 178 | return True 179 | 180 | def check_all_env(self): 181 | ''' 182 | Method for running all checks 183 | :return: True on success, raise RuntimeError on error. 184 | ''' 185 | try: 186 | self.check_mysql_uptime() 187 | self.check_mysql_mysql() 188 | self.check_mysql_mysqladmin() 189 | self.check_mysql_conf() 190 | self.check_mysql_backuptool() 191 | self.check_mysql_backupdir() 192 | self.check_mysql_fullbackupdir() 193 | self.check_mysql_incbackupdir() 194 | self.check_mysql_archive_dir() 195 | except Exception as err: 196 | logger.critical("FAILED: Check status") 197 | logger.error(err) 198 | raise RuntimeError("FAILED: Check status") 199 | else: 200 | logger.debug("OK: Check status") 201 | return True -------------------------------------------------------------------------------- /general_conf/generalops.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | from os.path import isfile 3 | import humanfriendly 4 | 5 | import logging 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class GeneralClass: 10 | 11 | def __init__(self, config='/etc/bck.conf'): 12 | 13 | if isfile(config): 14 | con = configparser.ConfigParser() 15 | con.read(config) 16 | 17 | DB = con['MySQL'] 18 | self.mysql = DB['mysql'] 19 | self.mycnf = DB['mycnf'] 20 | self.mysqladmin = DB['mysqladmin'] 21 | self.mysql_user = DB['mysql_user'] 22 | self.mysql_password = DB['mysql_password'] 23 | if 'mysql_socket' in DB: 24 | self.mysql_socket = DB['mysql_socket'] 25 | if 'mysql_host' in DB: 26 | self.mysql_host = DB['mysql_host'] 27 | if 'mysql_port' in DB: 28 | self.mysql_port = DB['mysql_port'] 29 | self.datadir = DB['datadir'] 30 | # self.tmpdir = DB['tmpdir'] 31 | # self.tmp = DB['tmp'] 32 | 33 | BCK = con['Backup'] 34 | if 'pid_dir' in BCK: 35 | self.pid_dir = BCK['pid_dir'] 36 | else: 37 | self.pid_dir = "/tmp/" 38 | self.tmpdir = BCK['tmpdir'] 39 | if 'pid_runtime_warning' in BCK: 40 | self.pid_runtime_warning = humanfriendly.parse_timespan( 41 | BCK['pid_runtime_warning']) 42 | self.backupdir = BCK['backupdir'] 43 | self.full_dir = self.backupdir + '/full' 44 | self.inc_dir = self.backupdir + '/inc' 45 | self.backup_tool = BCK['backup_tool'] 46 | if 'prepare_tool' in BCK: 47 | self.prepare_tool = BCK['prepare_tool'] 48 | self.xtrabck_prepare = BCK['xtra_prepare'] 49 | if 'xtra_backup' in BCK: 50 | self.xtra_backup = BCK['xtra_backup'] 51 | if 'xtra_prepare_options' in BCK: 52 | self.xtra_prepare_options = BCK['xtra_prepare_options'] 53 | if 'xtra_options' in BCK: 54 | self.xtra_options = BCK['xtra_options'] 55 | if 'full_backup_interval' in BCK: 56 | self.full_backup_interval = humanfriendly.parse_timespan( 57 | BCK['full_backup_interval']) 58 | else: 59 | self.full_backup_interval = 86400 60 | if 'archive_dir' in BCK: 61 | self.archive_dir = BCK['archive_dir'] 62 | if 'prepare_archive' in BCK: 63 | self.prepare_archive = BCK['prepare_archive'] 64 | if 'move_archive' in BCK: 65 | self.move_archive = BCK['move_archive'] 66 | if 'max_archive_size' in BCK: 67 | self.max_archive_size = humanfriendly.parse_size( 68 | BCK['max_archive_size']) 69 | if 'max_archive_duration' in BCK: 70 | self.max_archive_duration = humanfriendly.parse_timespan( 71 | BCK['max_archive_duration']) 72 | if 'partial_list' in BCK: 73 | self.partial_list = BCK['partial_list'] 74 | 75 | if 'Remote' in con: 76 | RM = con['Remote'] 77 | if 'remote_conn' in RM: 78 | self.remote_conn = RM['remote_conn'] 79 | if 'remote_dir' in RM: 80 | self.remote_dir = RM['remote_dir'] 81 | 82 | COM = con['Compress'] 83 | if 'compress' in COM: 84 | self.compress = COM['compress'] 85 | if 'compress_chunk_size' in COM: 86 | self.compress_chunk_size = COM['compress_chunk_size'] 87 | if 'compress_threads' in COM: 88 | self.compress_threads = COM['compress_threads'] 89 | if 'decompress' in COM: 90 | self.decompress = COM['decompress'] 91 | if 'remove_original' in COM: 92 | self.remove_original_comp = COM['remove_original'] 93 | 94 | ENC = con['Encrypt'] 95 | if 'xbcrypt' in ENC: 96 | self.xbcrypt = ENC['xbcrypt'] 97 | if 'encrypt' in ENC: 98 | self.encrypt = ENC['encrypt'] 99 | if 'encrypt_key' in ENC: 100 | self.encrypt_key = ENC['encrypt_key'] 101 | if 'encrypt_key_file' in ENC: 102 | self.encrypt_key_file = ENC['encrypt_key_file'] 103 | if 'encrypt_threads' in ENC: 104 | self.encrypt_threads = ENC['encrypt_threads'] 105 | if 'encrypt_chunk_size' in ENC: 106 | self.encrypt_chunk_size = ENC['encrypt_chunk_size'] 107 | if 'decrypt' in ENC: 108 | self.decrypt = ENC['decrypt'] 109 | if 'remove_original' in ENC: 110 | self.remove_original_enc = ENC['remove_original'] 111 | 112 | XBS = con['Xbstream'] 113 | if 'xbstream' in XBS: 114 | self.xbstream = XBS['xbstream'] 115 | if 'stream' in XBS: 116 | self.stream = XBS['stream'] 117 | if 'xbstream_options' in XBS: 118 | self.xbstream_options = XBS['xbstream_options'] 119 | if 'xbs_decrypt' in XBS: 120 | self.xbs_decrypt = XBS['xbs_decrypt'] 121 | 122 | CM = con['Commands'] 123 | self.start_mysql = CM['start_mysql_command'] 124 | self.stop_mysql = CM['stop_mysql_command'] 125 | self.chown_command = CM['chown_command'] 126 | 127 | if 'TestConf' in con: 128 | TEST = con['TestConf'] 129 | if 'ps_branches' in TEST: 130 | self.ps_branches = TEST['ps_branches'] 131 | if 'pxb_branches' in TEST: 132 | self.pxb_branches = TEST['pxb_branches'] 133 | if 'gitcmd' in TEST: 134 | self.gitcmd = TEST['gitcmd'] 135 | if 'pxb_gitcmd' in TEST: 136 | self.pxb_gitcmd = TEST['pxb_gitcmd'] 137 | if 'testpath' in TEST: 138 | self.testpath = TEST['testpath'] 139 | if 'incremental_count' in TEST: 140 | self.incremental_count = TEST['incremental_count'] 141 | if 'xb_configs' in TEST: 142 | self.xb_configs = TEST['xb_configs'] 143 | if 'default_mysql_options' in TEST: 144 | self.default_mysql_options = TEST['default_mysql_options'] 145 | if 'mysql_options' in TEST: 146 | self.mysql_options = TEST['mysql_options'] 147 | if 'make_slaves' in TEST: 148 | self.make_slaves = TEST['make_slaves'] 149 | 150 | else: 151 | logger.critical("Missing config file : /etc/bck.conf") 152 | -------------------------------------------------------------------------------- /master_backup_script/__init__.py: -------------------------------------------------------------------------------- 1 | from master_backup_script import backuper 2 | -------------------------------------------------------------------------------- /partial_recovery/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Percona-Lab/MySQL-AutoXtraBackup/8ae7927e72c03cf4e685d26f2c2d0d2580eeac52/partial_recovery/__init__.py -------------------------------------------------------------------------------- /partial_recovery/partial.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import subprocess 4 | from general_conf.generalops import GeneralClass 5 | import re 6 | from general_conf import check_env 7 | import sys 8 | 9 | import logging 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class PartialRecovery(GeneralClass): 14 | 15 | def __init__(self, config='/etc/bck.conf'): 16 | self.conf = config 17 | GeneralClass.__init__(self, self.conf) 18 | if shutil.which('mysqlfrm') is None: 19 | logger.critical("Could not find mysqlfrm! Please install it or check if it is in PATH") 20 | raise RuntimeError("Could not find mysqlfrm! Please install it or check if it is in PATH") 21 | 22 | def create_mysql_client_command(self, statement): 23 | command_connection = '{} --defaults-file={} -u{} --password={}' 24 | command_execute = ' -e "{}"' 25 | 26 | if hasattr(self, 'mysql_socket'): 27 | command_connection += ' --socket={}' 28 | command_connection += command_execute 29 | new_command = command_connection.format( 30 | self.mysql, 31 | self.mycnf, 32 | self.mysql_user, 33 | self.mysql_password, 34 | self.mysql_socket, 35 | statement) 36 | return new_command 37 | else: 38 | command_connection += ' --port={}' 39 | command_connection += command_execute 40 | new_command = command_connection.format( 41 | self.mysql, 42 | self.mycnf, 43 | self.mysql_user, 44 | self.mysql_password, 45 | self.mysql_host, 46 | self.mysql_port, 47 | statement) 48 | return new_command 49 | 50 | def check_innodb_file_per_table(self): 51 | """ 52 | Function for checking MySQL innodb_file_per_table option. 53 | It is needed for "Transportable Tablespace" concept. 54 | :return: True/False 55 | """ 56 | statement = "select @@global.innodb_file_per_table" 57 | run_command = self.create_mysql_client_command(statement=statement) 58 | 59 | logger.debug("Checking if innodb_file_per_table is enabled") 60 | status, output = subprocess.getstatusoutput(run_command) 61 | 62 | if status == 0 and int(output[-1]) == 1: 63 | logger.debug("OK: innodb_file_per_table is enabled!") 64 | return True 65 | elif status == 0 and int(output[-1]) == 0: 66 | logger.debug("OK: innodb_file_per_table is disabled!") 67 | return False 68 | else: 69 | logger.error("FAILED: InnoDB file per-table Check") 70 | logger.error(output) 71 | raise RuntimeError("FAILED: InnoDB file per-table Check") 72 | 73 | def check_mysql_version(self): 74 | """ 75 | Function for checking MySQL version. 76 | Version must be >= 5.6 for using "Transportable Tablespace" concept. 77 | :return: True/False 78 | """ 79 | statement = "select @@version" 80 | run_command = self.create_mysql_client_command(statement=statement) 81 | 82 | logger.debug("Checking MySQL version") 83 | status, output = subprocess.getstatusoutput(run_command) 84 | 85 | if status == 0 and ('5.6' in output): 86 | logger.debug("You have correct version of MySQL") 87 | return True 88 | elif status == 0 and ('5.7' in output): 89 | logger.debug("You have correct version of MySQL") 90 | return True 91 | elif status == 0 and ('5.7' not in output) and ('5.6' not in output): 92 | logger.error("Your MySQL server is not supported. MySQL version must be >= 5.6") 93 | raise RuntimeError("Your MySQL server is not supported. MySQL version must be >= 5.6") 94 | else: 95 | logger.error("FAILED: MySQL version check") 96 | logger.error(output) 97 | raise RuntimeError("FAILED: MySQL version check") 98 | 99 | def check_database_exists_on_mysql(self, database_name): 100 | """ 101 | Function check if this database already exists in MySQL Server.(.frm and .ibd files are exist) 102 | In other words database is not dropped. If there is no such database, there is an input for creation. 103 | :param database_name: Specified database name 104 | :return: True/False 105 | """ 106 | statement = "SELECT count(*) FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = '%s'" % database_name 107 | run_command = self.create_mysql_client_command(statement=statement) 108 | 109 | logger.debug("Checking if database exists in MySQL") 110 | status, output = subprocess.getstatusoutput(run_command) 111 | if status == 0 and int(output[-1]) == 1: 112 | logger.debug("Database exists!") 113 | return True 114 | if status == 0 and int(output[-1]) == 0: 115 | logger.debug("There is no such database!") 116 | logger.debug("Create Specified Database in MySQL Server, before restoring single table") 117 | answer = input("We can create it for you do you want? (yes/no): ") 118 | if answer == 'yes': 119 | create_db = "create database %s" % database_name 120 | run_command = self.create_mysql_client_command(statement=create_db) 121 | logger.debug("Creating specified database") 122 | status, output = subprocess.getstatusoutput(run_command) 123 | if status == 0: 124 | logger.debug("OK: {} database created".format(database_name)) 125 | return True 126 | else: 127 | logger.error("FAILED: to create database!") 128 | logger.error(output) 129 | raise RuntimeError("FAILED: to create database!") 130 | else: # if you type non-yes word 131 | logger.error("Exited!") 132 | return False 133 | 134 | else: 135 | logger.error("FAILED: Check for database") 136 | logger.error(output) 137 | raise RuntimeError("FAILED: Check for database") 138 | 139 | def check_table_exists_on_mysql( 140 | self, 141 | path_to_frm_file, 142 | database_name, 143 | table_name): 144 | """ 145 | Function to check if table exists on MySQL. 146 | If it is dropped, we will try to extract table create statement from .frm file from backup file. 147 | :param path_to_frm_file: Path for .frm file 148 | :param database_name: Specified database name 149 | :param table_name: Specified table name 150 | :return: True/False 151 | """ 152 | 153 | statement = "select count(*) from INFORMATION_SCHEMA.tables " \ 154 | "where table_schema = '%s'" \ 155 | "and table_name = '%s'" % (database_name, table_name) 156 | 157 | run_command = self.create_mysql_client_command(statement=statement) 158 | logger.debug("Checking if table exists in MySQL Server") 159 | status, output = subprocess.getstatusoutput(run_command) 160 | if status == 0 and int(output[-1]) == 1: 161 | logger.debug("Table exists in MySQL Server.") 162 | return True 163 | elif status == 0 and int(output[-1]) == 0: 164 | logger.debug("Table does not exist in MySQL Server.") 165 | logger.debug("You can not restore table, with not existing tablespace file(.ibd)!") 166 | logger.debug("We will try to extract table create statement from .frm file, from backup folder") 167 | create = self.run_mysqlfrm_utility(path_to_frm_file=path_to_frm_file) 168 | regex = re.compile(r'((\n)CREATE((?!#).)*ENGINE=\w+)', re.DOTALL) 169 | matches = [m.groups() for m in regex.finditer(create)] 170 | for m in matches: 171 | create_table = m[0] 172 | new_create_table = create_table.replace("`", "") 173 | run_command = self.create_mysql_client_command(statement=new_create_table) 174 | status, output = subprocess.getstatusoutput(run_command) 175 | if status == 0: 176 | logger.debug("Table Created from .frm file!") 177 | return True 178 | else: 179 | logger.error("Failed to create table from .frm file!") 180 | logger.error(output) 181 | raise RuntimeError("Failed to create table from .frm file!") 182 | else: 183 | logger.error("FAILED: Check if table exists") 184 | logger.error(output) 185 | raise RuntimeError("FAILED: Check if table exists") 186 | 187 | @staticmethod 188 | def run_mysqlfrm_utility(path_to_frm_file): 189 | command = '/usr/bin/mysqlfrm --diagnostic %s' % path_to_frm_file 190 | logger.debug("Running mysqlfrm tool") 191 | status, output = subprocess.getstatusoutput(command) 192 | if status == 0: 193 | logger.debug("OK: Success to run mysqlfrm") 194 | return output 195 | else: 196 | logger.error("FAILED: run mysqlfrm") 197 | logger.error(output) 198 | raise RuntimeError("FAILED: run mysqlfrm") 199 | 200 | def get_table_ibd_file(self, database_name, table_name): 201 | """ 202 | Locate backed up database and table. 203 | Exactly we are looking for .ibd file. 204 | .ibd file is a tablespace file where table data located. 205 | :param database_name: Specified database name 206 | :param table_name: Specified table name 207 | :return .ibd file full path / False if not exists 208 | """ 209 | 210 | database_dir_list = [] 211 | database_objects_full_path = [] 212 | find_objects_full_path = [] 213 | table_dir_list = [] 214 | 215 | # Look for all files in database directory 216 | for i in os.listdir(self.full_dir): 217 | for x in os.listdir(self.full_dir + "/" + i): 218 | if os.path.isdir( 219 | self.full_dir + 220 | "/" + 221 | i + 222 | "/" + 223 | x) and x == database_name: 224 | for z in os.listdir(self.full_dir + "/" + i + "/" + x): 225 | database_dir_list.append(z) 226 | database_objects_full_path.append( 227 | self.full_dir + "/" + i + "/" + x + "/" + z) 228 | 229 | # If database directory exists find already provided table in database 230 | # directory 231 | if len(database_dir_list) > 0: 232 | for i in database_dir_list: 233 | base_file = os.path.splitext(i)[0] 234 | ext = os.path.splitext(i)[1] 235 | 236 | if table_name == base_file: 237 | table_dir_list.append(i) 238 | 239 | # If table name from input is valid and it is located in database 240 | # directory return .ibd file name 241 | if len(database_dir_list) > 0 and len( 242 | table_dir_list) == 2: # Why 2? because every InnoDB table must have .frm and .ibd file 243 | for i in table_dir_list: 244 | ext = os.path.splitext(i)[1] 245 | if ext == '.ibd': 246 | for a in database_objects_full_path: 247 | if i in a: 248 | find_objects_full_path.append(a) 249 | 250 | if len(find_objects_full_path) > 0: 251 | for x in find_objects_full_path: 252 | return x 253 | else: 254 | logger.error("Sorry, There is no such Database or Table in backup directory") 255 | logger.error("Or maybe table storage engine is not InnoDB") 256 | raise RuntimeError("Sorry, There is no such Database or Table in backup directory " 257 | "Or maybe table storage engine is not InnoDB ") 258 | 259 | def lock_table(self, database_name, table_name): 260 | # Executing lock tables write on specified table 261 | statement = "LOCK TABLES %s.%s WRITE" % (database_name, table_name) 262 | run_command = self.create_mysql_client_command(statement=statement) 263 | status, output = subprocess.getstatusoutput(run_command) 264 | logger.debug("Applying write lock!") 265 | if status == 0: 266 | logger.debug("OK: Table is locked") 267 | return True 268 | else: 269 | logger.error("FAILED: to LOCK!") 270 | logger.error(output) 271 | raise RuntimeError("FAILED: to LOCK!") 272 | 273 | def alter_tablespace(self, database_name, table_name): 274 | # Running alter table discard tablespace here 275 | statement = "ALTER TABLE %s.%s DISCARD TABLESPACE" % ( 276 | database_name, table_name) 277 | run_command = self.create_mysql_client_command(statement=statement) 278 | status, output = subprocess.getstatusoutput(run_command) 279 | logger.debug("Discarding tablespace") 280 | if status == 0: 281 | logger.debug("OK: Tablespace discarded successfully") 282 | return True 283 | else: 284 | logger.error("FAILED: discard tablespace!") 285 | logger.error(output) 286 | raise RuntimeError("FAILED: discard tablespace!") 287 | 288 | @staticmethod 289 | def copy_ibd_file_back(path_of_ibd_file, path_to_mysql_database_dir): 290 | # Copy .ibd file back 291 | try: 292 | logger.debug("OK: Copying .ibd file back") 293 | shutil.copy(path_of_ibd_file, path_to_mysql_database_dir) 294 | return True 295 | except Exception as err: 296 | logger.error("FAILED: copy .ibd file back") 297 | logger.error(err) 298 | raise RuntimeError("FAILED: copy .ibd file back") 299 | 300 | def give_chown(self, path_to_mysql_database_dir): 301 | # run chown command 302 | comm = '%s %s' % (self.chown_command, path_to_mysql_database_dir) 303 | status, output = subprocess.getstatusoutput(comm) 304 | logger.debug("Running chown command!") 305 | if status == 0: 306 | logger.debug("OK: Chown command completed") 307 | return True 308 | else: 309 | logger.error("FAILED: Chown Command") 310 | raise RuntimeError("FAILED: Chown Command") 311 | 312 | def import_tablespace(self, database_name, table_name): 313 | # Running alter table import tablespace 314 | statement = "ALTER TABLE %s.%s IMPORT TABLESPACE" % ( 315 | database_name, table_name) 316 | run_command = self.create_mysql_client_command(statement=statement) 317 | status, output = subprocess.getstatusoutput(run_command) 318 | logger.debug("Importing Tablespace!") 319 | if status == 0: 320 | logger.debug("OK: Tablespace imported") 321 | return True 322 | else: 323 | logger.error("FAILED: Tablespace import") 324 | logger.error(output) 325 | raise RuntimeError("FAILED: Tablespace import") 326 | 327 | def unlock_tables(self): 328 | # Run unlock tables command 329 | statement = "unlock tables" 330 | run_command = self.create_mysql_client_command(statement=statement) 331 | status, output = subprocess.getstatusoutput(run_command) 332 | logger.debug("Unlocking tables!") 333 | if status == 0: 334 | logger.debug("OK: Unlocked!") 335 | return True 336 | else: 337 | logger.error("FAILED: Unlocking") 338 | logger.error(output) 339 | raise RuntimeError("FAILED: Unlocking") 340 | 341 | def final_actions(self): 342 | # Type Database name of table which you want to restore 343 | database_name = input("Type Database name: ") 344 | # Type name of table which you want to restore 345 | table_name = input("Type Table name: ") 346 | path = self.get_table_ibd_file( 347 | database_name=database_name, 348 | table_name=table_name) 349 | path_to_mysql_datadir = self.datadir + "/" + database_name 350 | 351 | if path: 352 | path_to_frm_file = path[:-3] + 'frm' 353 | 354 | obj_check_env = check_env.CheckEnv(self.conf) 355 | 356 | if path: 357 | try: 358 | obj_check_env.check_mysql_uptime() 359 | self.check_innodb_file_per_table() 360 | self.check_mysql_version() 361 | self.check_database_exists_on_mysql( 362 | database_name=database_name) 363 | self.check_table_exists_on_mysql( 364 | path_to_frm_file=path_to_frm_file, 365 | database_name=database_name, 366 | table_name=table_name) 367 | self.lock_table(database_name=database_name, table_name=table_name) 368 | self.alter_tablespace(database_name=database_name, table_name=table_name) 369 | self.copy_ibd_file_back(path_of_ibd_file=path, path_to_mysql_database_dir=path_to_mysql_datadir) 370 | self.give_chown(path_to_mysql_database_dir=path_to_mysql_datadir) 371 | self.import_tablespace(database_name=database_name, table_name=table_name) 372 | self.unlock_tables() 373 | except Exception as err: 374 | logger.error("FAILED: Table is not recovered") 375 | logger.error(err) 376 | raise RuntimeError("FAILED: Table is not recovered") 377 | else: 378 | logger.debug("OK: Table Recovered! ...") 379 | return True 380 | -------------------------------------------------------------------------------- /prepare_env_test_mode/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Percona-Lab/MySQL-AutoXtraBackup/8ae7927e72c03cf4e685d26f2c2d0d2580eeac52/prepare_env_test_mode/__init__.py -------------------------------------------------------------------------------- /prepare_env_test_mode/build_2.3_pxb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TESTPATH=$1 4 | BRANCH=$2 5 | 6 | rm -rf target 7 | mkdir -p target 8 | 9 | cmake -DCMAKE_INSTALL_PREFIX=./target -DWITH_DEBUG=1 -DWITH_SSL=system -DWITH_MAN_PAGES=OFF -DDOWNLOAD_BOOST=1 -DWITH_BOOST=/tmp/boost_23/ && make -j2 10 | make install 11 | mkdir ./target/percona-xtrabackup-${BRANCH}.x-debug 12 | mv ./target/bin ./target/xtrabackup-test ./target/percona-xtrabackup-${BRANCH}.x-debug/ 13 | tar -zcf ${TESTPATH}/percona-xtrabackup-${BRANCH}.x-debug.tar.gz ./target/percona-xtrabackup-${BRANCH}.x-debug -------------------------------------------------------------------------------- /prepare_env_test_mode/build_2.4_pxb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TESTPATH=$1 4 | BRANCH=$2 5 | 6 | rm -rf target 7 | mkdir -p target 8 | 9 | cmake -DCMAKE_INSTALL_PREFIX=./target -DWITH_DEBUG=1 -DWITH_SSL=system -DWITH_MAN_PAGES=OFF -DDOWNLOAD_BOOST=1 -DWITH_BOOST=/tmp/boost_23/ && make -j2 10 | make install 11 | mkdir ./target/percona-xtrabackup-${BRANCH}.x-debug 12 | mv ./target/bin ./target/xtrabackup-test ./target/percona-xtrabackup-${BRANCH}.x-debug/ 13 | tar -zcf ${TESTPATH}/percona-xtrabackup-${BRANCH}.x-debug.tar.gz ./target/percona-xtrabackup-${BRANCH}.x-debug -------------------------------------------------------------------------------- /prepare_env_test_mode/call_create_index_temp.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PATH=$1 4 | BASEDIR=$2 5 | MYSQL_SOCK=$3 6 | 7 | while true; do /usr/bin/bash ${PATH}/create_index_temp.sh ${BASEDIR} ${MYSQL_SOCK}; done -------------------------------------------------------------------------------- /prepare_env_test_mode/call_ddl_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PATH=$1 4 | BASEDIR=$2 5 | MYSQL_SOCK=$3 6 | 7 | while true; do /usr/bin/bash ${PATH}/ddl_test.sh ${BASEDIR} ${MYSQL_SOCK}; done -------------------------------------------------------------------------------- /prepare_env_test_mode/call_innodb_online_alter_encryption_alters.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This bash file is for creating necessary environment for running tests 4 | # See https://jira.percona.com/browse/PS-3819 5 | 6 | BASEDIR=$1 7 | MYSQL_SOCK=$2 8 | FILE_DIR=$3 9 | MYSQL_USER=root 10 | 11 | while true; do ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} < ${FILE_DIR}/innodb_online_alter_encryption_alters.sql >> /dev/null 2>&1; done -------------------------------------------------------------------------------- /prepare_env_test_mode/call_innodb_online_alter_encryption_sql.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This bash file is for creating necessary environment for running tests 4 | # See https://jira.percona.com/browse/PS-3819 5 | 6 | BASEDIR=$1 7 | MYSQL_SOCK=$2 8 | FILE_DIR=$3 9 | MYSQL_USER=root 10 | 11 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} < ${FILE_DIR}/innodb_online_alter_encryption.sql -------------------------------------------------------------------------------- /prepare_env_test_mode/call_temp_table_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PATH=$1 4 | BASEDIR=$2 5 | MYSQL_SOCK=$3 6 | 7 | while true; do /usr/bin/bash ${PATH}/temp_table_test.sh ${BASEDIR} ${MYSQL_SOCK}; done -------------------------------------------------------------------------------- /prepare_env_test_mode/clone_build_start_server.py: -------------------------------------------------------------------------------- 1 | from prepare_env_test_mode.test_check_env import TestModeConfCheck 2 | from shutil import rmtree 3 | import subprocess 4 | import os 5 | import re 6 | import logging 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class CloneBuildStartServer(TestModeConfCheck): 11 | """ 12 | Class for cloning from git, building server from source and starting test server etc. 13 | This class will include all necessary actions for preparing test environment. 14 | Please see specific methods for clarity. 15 | """ 16 | def __init__(self, config='/etc/bck.conf'): 17 | self.conf = config 18 | super().__init__(config=self.conf) 19 | #self.git_cmd = GeneralClass().gitcmd 20 | #self.xb_configs = GeneralClass().xb_configs 21 | # Creating needed path here 22 | t_obj = TestModeConfCheck(config=self.conf) 23 | if t_obj.check_test_path(t_obj.testpath): 24 | self.testpath = t_obj.testpath 25 | 26 | def clone_percona_qa(self): 27 | # Clone percona-qa repo for using existing bash scripts 28 | clone_cmd = "git clone https://github.com/Percona-QA/percona-qa.git {}/percona-qa" 29 | if not os.path.exists("{}/percona-qa".format(self.testpath)): 30 | logger.debug("Started to clone percona-qa...") 31 | status, output = subprocess.getstatusoutput(clone_cmd.format(self.testpath)) 32 | if status == 0: 33 | logger.debug("percona-qa ready to use") 34 | return True 35 | else: 36 | logger.error("Cloning percona-qa repo failed") 37 | logger.error(output) 38 | return False 39 | else: 40 | return True 41 | 42 | def clone_ps_server_from_conf(self): 43 | # Clone PS server[the value coming from config file] 44 | ps_branches = self.ps_branches.split() 45 | for branch in ps_branches: 46 | if branch != '5.5': 47 | clone_cmd = "git clone {} -b {} {}/PS-{}-trunk".format(self.gitcmd, branch, self.testpath, branch) 48 | else: 49 | clone_cmd = "git clone {} -b {} {}/PS-{}-trunk".format(self.gitcmd.split()[-1], branch, self.testpath, branch) 50 | if not os.path.exists("{}/PS-{}-trunk".format(self.testpath, branch)): 51 | logger.debug("Started to clone Percona Server...") 52 | status, output = subprocess.getstatusoutput(clone_cmd) 53 | if status == 0: 54 | logger.debug("PS-{} cloned ready to build".format(branch)) 55 | else: 56 | logger.error("Cloning PS-{} failed".format(branch)) 57 | logger.error(output) 58 | return False 59 | 60 | return True 61 | 62 | def clone_pxb(self): 63 | # Clone PXB 64 | pxb_branches = self.pxb_branches.split() 65 | for branch in pxb_branches: 66 | clone_cmd = "git clone {} -b {} {}/PXB-{}" 67 | if not os.path.exists("{}/PXB-{}".format(self.testpath, branch)): 68 | logger.debug("Started to clone PXB...") 69 | status, output = subprocess.getstatusoutput( 70 | clone_cmd.format(self.pxb_gitcmd, branch, self.testpath, branch)) 71 | if status == 0: 72 | logger.debug("PXB-{} cloned ready to build".format(branch)) 73 | else: 74 | logger.error("Cloning PXB-{} failed".format(branch)) 75 | logger.error(output) 76 | return False 77 | return True 78 | 79 | def build_pxb(self): 80 | # Building pxb from source 81 | # For this purpose will use build_{}_pxb.sh scripts from this folder 82 | pxb_branches = self.pxb_branches.split() 83 | saved_path = os.getcwd() 84 | dir_path = os.path.dirname(os.path.realpath(__file__)) 85 | for branch in pxb_branches: 86 | pxb_path = "{}/PXB-{}".format(self.testpath, branch) 87 | os.chdir(pxb_path) 88 | if '2.3' in branch: 89 | build_cmd = "{}/build_{}_pxb.sh {} {}".format(dir_path, '2.3', self.testpath, branch) 90 | elif '2.4' in branch: 91 | build_cmd = "{}/build_{}_pxb.sh {} {}".format(dir_path, '2.4', self.testpath, branch) 92 | status, output = subprocess.getstatusoutput(build_cmd) 93 | if status == 0: 94 | logger.debug("PXB build succeeded") 95 | os.chdir(saved_path) 96 | else: 97 | logger.error("PXB build failed") 98 | logger.error(output) 99 | os.chdir(saved_path) 100 | return False 101 | 102 | return True 103 | 104 | def build_server(self): 105 | # Building server from source 106 | # For this purpose; I am going to use build_5.x_debug.sh script from percona-qa 107 | saved_path = os.getcwd() 108 | # Specify here the cloned PS path; for me it is PS-5.7-trunk(which I have hard coded in method above) 109 | ps_branches = self.ps_branches.split() 110 | for branch in ps_branches: 111 | new_path = "{}/PS-{}-trunk" 112 | os.chdir(new_path.format(self.testpath, branch)) 113 | if '5.5' in branch: 114 | # Use same script with 5.5 and 5.6 versions 115 | build_cmd = "{}/percona-qa/build_5.x_debug_5.6_for_pxb_tests.sh" 116 | else: 117 | build_cmd = "{}/percona-qa/build_5.x_debug_{}_for_pxb_tests.sh" 118 | logger.debug("Started to build Percon Server from source...") 119 | status, output = subprocess.getstatusoutput(build_cmd.format(self.testpath, branch)) 120 | if status == 0: 121 | logger.debug("PS build succeeded") 122 | os.chdir(saved_path) 123 | else: 124 | logger.error("PS build failed") 125 | logger.error(output) 126 | os.chdir(saved_path) 127 | return False 128 | 129 | return True 130 | 131 | def rename_basedirs(self): 132 | logger.debug("Renaming basedir folder name...") 133 | basedirs = [] 134 | for root, dirs, files in os.walk(self.testpath): 135 | for dir_name in dirs: 136 | obj = re.search('PS[0-9]', dir_name) 137 | if obj: 138 | basedir_path = "{}/{}" 139 | basedirs.append(basedir_path.format(self.testpath, dir_name)) 140 | if len(basedirs) > 0: 141 | for i in basedirs: 142 | os.rename(i, i.replace('-percona-server', '')) 143 | return True 144 | else: 145 | logger.warning("Could not get PS basedir path...") 146 | logger.debug("It looks like you should build server first...") 147 | return False 148 | 149 | def get_basedir(self): 150 | # Method for getting PS basedir path 151 | logger.debug("Trying to get basedir path...") 152 | basedirs = [] 153 | for root, dirs, files in os.walk(self.testpath): 154 | for dir_name in dirs: 155 | obj = re.search('PS[0-9]', dir_name) 156 | if obj: 157 | basedir_path = "{}/{}" 158 | basedirs.append(basedir_path.format(self.testpath, dir_name)) 159 | # return basedir_path.format(self.testpath, dir_name) 160 | if len(basedirs) > 0: 161 | logger.debug("Could get PS basedir path...") 162 | return basedirs 163 | else: 164 | logger.warning("Could not get PS basedir path...") 165 | logger.debug("It looks like you should build server first...") 166 | return False 167 | 168 | def prepare_startup(self, basedir_path): 169 | # Method for calling startup.sh file from percona-qa folder 170 | saved_path = os.getcwd() 171 | os.chdir(basedir_path) 172 | 173 | startup_cmd = "{}/percona-qa/startup.sh" 174 | logger.debug("Started to run startup.sh file...") 175 | status, output = subprocess.getstatusoutput(startup_cmd.format(self.testpath)) 176 | if status == 0: 177 | logger.debug("Running startup.sh succeeded") 178 | os.chdir(saved_path) 179 | return True 180 | else: 181 | logger.error("Running startup.sh failed") 182 | logger.error(output) 183 | os.chdir(saved_path) 184 | return False 185 | 186 | @staticmethod 187 | def prepare_start_dynamic(basedir_path): 188 | # Method for calling start_dynamic.sh from basedir. 189 | # It will generated start_dynamuc executables in basedir. 190 | saved_path = os.getcwd() 191 | dir_path = os.path.dirname(os.path.realpath(__file__)) 192 | os.chdir(basedir_path) 193 | start_dynamic = "{}/start_dynamic.sh".format(dir_path) 194 | logger.debug("Running start_dynamic.sh here...") 195 | status, output = subprocess.getstatusoutput(start_dynamic) 196 | if status == 0: 197 | logger.debug("Running start_dynamic.sh succeeded") 198 | os.chdir(saved_path) 199 | return True 200 | else: 201 | logger.error("Running start_dynamic.sh failed") 202 | logger.error(output) 203 | os.chdir(saved_path) 204 | return False 205 | 206 | @staticmethod 207 | def start_server(basedir_path, options=None): 208 | # Method for calling start script which is created inside PS basedir 209 | logger.debug("Using start script here...") 210 | if options is not None: 211 | start_cmd = "{}/start {}" 212 | status, output = subprocess.getstatusoutput(start_cmd.format(basedir_path, options)) 213 | else: 214 | start_cmd = "{}/start" 215 | status, output = subprocess.getstatusoutput(start_cmd.format(basedir_path)) 216 | if status == 0: 217 | logger.debug("Server started!") 218 | return True 219 | else: 220 | logger.error("Server start failed") 221 | logger.error(output) 222 | return False 223 | 224 | @staticmethod 225 | def wipe_server_all(basedir_path, options=None): 226 | # Method for calling "all" script which is created inside PS basedir 227 | saved_path = os.getcwd() 228 | os.chdir(basedir_path) 229 | logger.debug("Using all_no_cl script here...") 230 | if options is not None: 231 | all_cmd = "./all_no_cl {}" 232 | status, output = subprocess.getstatusoutput(all_cmd.format(options)) 233 | else: 234 | all_cmd = "./all_no_cl" 235 | status, output = subprocess.getstatusoutput(all_cmd) 236 | if status == 0: 237 | logger.debug("Server wiped for fresh start!") 238 | os.chdir(saved_path) 239 | return True 240 | else: 241 | logger.error("All script run failed") 242 | logger.error(output) 243 | os.chdir(saved_path) 244 | return False 245 | 246 | def extract_xb_archive(self, file_name): 247 | # General method for extracting XB archives 248 | # It will create target folder inside test path 249 | extract_cmd = "tar -xf {}/{} -C {}" 250 | if os.path.isfile("{}/{}".format(self.testpath, file_name)): 251 | if not os.path.isdir("{}/target/{}".format(self.testpath, file_name[:-7])): 252 | status, output = subprocess.getstatusoutput(extract_cmd.format(self.testpath, file_name, self.testpath)) 253 | if status == 0: 254 | logger.debug("Extracted from {}".format(file_name)) 255 | return True 256 | else: 257 | logger.error("Failed to extract from {}".format(file_name)) 258 | logger.error(output) 259 | return False 260 | else: 261 | logger.debug("The 'target' folder already there...") 262 | return True 263 | else: 264 | logger.debug("Could not find {}".format(file_name)) 265 | return False 266 | 267 | 268 | -------------------------------------------------------------------------------- /prepare_env_test_mode/config_generator.py: -------------------------------------------------------------------------------- 1 | from prepare_env_test_mode.clone_build_start_server import CloneBuildStartServer 2 | from prepare_env_test_mode.run_benchmark import RunBenchmark 3 | import configparser 4 | from itertools import product 5 | import logging 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class ConfigGenerator(CloneBuildStartServer): 10 | 11 | def __init__(self, config='/etc/bck.conf'): 12 | self.conf = config 13 | super().__init__(config=self.conf) 14 | # For getting socket file path using RunBenchmark() 15 | self.benchmark_obj = RunBenchmark() 16 | 17 | @staticmethod 18 | def generate_config_files(test_path, conf_file, basedir, datadir, sock_file, backup_path=None): 19 | # This method for generating separate config files for each XB versions based on PS versions 20 | try: 21 | if backup_path is None: 22 | conf_path = "{}/{}".format(test_path, conf_file) 23 | else: 24 | conf_path = "{}/{}".format(backup_path, conf_file) 25 | with open(conf_path, 'w+') as cfgfile: 26 | config = configparser.ConfigParser(allow_no_value=True) 27 | section1 = 'MySQL' 28 | config.add_section(section1) 29 | config.set(section1, "mysql", "{}/bin/mysql".format(basedir)) 30 | config.set(section1, "mycnf", "") 31 | config.set(section1, "mysqladmin", "{}/bin/mysqladmin".format(basedir)) 32 | config.set(section1, "mysql_user", "root") 33 | config.set(section1, "mysql_password", "") 34 | config.set(section1, "#Use either socket or port + host combination") 35 | config.set(section1, "mysql_socket", "{}".format(sock_file)) 36 | config.set(section1, "#mysql_host", "127.0.0.1") 37 | config.set(section1, "#mysql_port", "3306") 38 | config.set(section1, "datadir", "{}/{}".format(basedir, datadir)) 39 | 40 | section2 = 'Backup' 41 | config.add_section(section2) 42 | config.set(section2, "#Optional: set pid directory") 43 | config.set(section2, "pid_dir", "/tmp/MySQL-AutoXtraBackup") 44 | config.set(section2, "tmpdir", "/home/shahriyar.rzaev/XB_TEST/mysql_datadirs") 45 | config.set(section2, "#Optional: set warning if pid of backup us running for longer than X") 46 | config.set(section2, "pid_runtime_warning", "2 Hours") 47 | if ('5.7' in basedir) and ('2_4' in conf_file): 48 | config.set(section2, "backupdir", "/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_7_x_2_4") 49 | elif ('5.6' in basedir) and ('2_4' in conf_file): 50 | config.set(section2, "backupdir", "/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_6_x_2_4") 51 | elif ('5.6' in basedir) and ('2_3' in conf_file): 52 | config.set(section2, "backupdir", "/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_6_x_2_3") 53 | elif ('5.5' in basedir) and ('2_3' in conf_file): 54 | config.set(section2, "backupdir", "/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_5_x_2_3") 55 | elif ('5.5' in basedir) and ('2_4' in conf_file): 56 | config.set(section2, "backupdir", "/home/shahriyar.rzaev/XB_TEST/backup_dir/ps_5_5_x_2_4") 57 | if '2_4' in conf_file: 58 | config.set(section2, "backup_tool", 59 | "{}/target/percona-xtrabackup-2.4.x-debug/bin/xtrabackup".format(test_path)) 60 | else: 61 | config.set(section2, "backup_tool", 62 | "{}/target/percona-xtrabackup-2.3.x-debug/bin/xtrabackup".format(test_path)) 63 | config.set(section2, "#Optional: specify different path/version of xtrabackup here for prepare") 64 | config.set(section2, "#prepare_tool", "") 65 | config.set(section2, "xtra_prepare", "--apply-log-only") 66 | config.set(section2, "#Optional: pass additional options for backup stage") 67 | config.set(section2, "#xtra_backup", "--compact") 68 | config.set(section2, "#Optional: pass additional options for prepare stage") 69 | config.set(section2, "#xtra_prepare_options", "--rebuild-indexes") 70 | config.set(section2, 71 | "#Optional: pass general additional options; it will go to both for backup and prepare") 72 | config.set(section2, "#xtra_options", "--binlog-info=ON --galera-info") 73 | if '5.7' in basedir: 74 | config.set(section2, "xtra_options", "--slave-info --no-version-check --core-file " 75 | "--parallel=10 --throttle=40 --check-privileges " 76 | "--ftwrl-wait-timeout=0 " 77 | "--ftwrl-wait-query-type=all " 78 | "--ftwrl-wait-threshold=1 " 79 | #"--lock-wait-timeout=0 " 80 | #"--lock-wait-query-type=all " 81 | #"--lock-wait-threshold=1 " 82 | "--kill-long-queries-timeout=1 " 83 | "--kill-wait-query-type=all " 84 | "--kill-long-query-type=all " 85 | "--no-backup-locks " 86 | #"--lock-ddl-per-table " 87 | #"--lock-ddl " 88 | "--keyring-file-data={}/mysql-keyring/keyring ".format(basedir)) 89 | else: 90 | config.set(section2, "xtra_options", "--slave-info --no-version-check --core-file " 91 | "--ftwrl-wait-timeout=0 " 92 | "--ftwrl-wait-query-type=all " 93 | "--ftwrl-wait-threshold=1 " 94 | #"--lock-wait-timeout=0 " 95 | #"--lock-wait-query-type=all " 96 | #"--lock-wait-threshold=1 " 97 | "--kill-long-queries-timeout=1 " 98 | "--kill-wait-query-type=all " 99 | "--kill-long-query-type=all " 100 | "--no-backup-locks " 101 | "--parallel=10 --throttle=40 --check-privileges ") 102 | config.set(section2, "#Optional: set archive and rotation") 103 | config.set(section2, "#archive_dir", "/home/shahriyar.rzaev/XB_TEST/backup_archives") 104 | config.set(section2, "#prepare_archive", "1") 105 | config.set(section2, "#move_archive", "0") 106 | config.set(section2, "#full_backup_interval", "1 day") 107 | config.set(section2, "#max_archive_size", "100GiB") 108 | config.set(section2, "#max_archive_duration", "4 Days") 109 | config.set(section2, "#Optional: WARNING(Enable this if you want to take partial backups). " 110 | "Specify database names or table names.") 111 | config.set(section2, "#partial_list", "test.t1 test.t2 dbtest") 112 | 113 | section3 = "Compress" 114 | config.add_section(section3) 115 | config.set(section3, "#optional") 116 | config.set(section3, "#Enable only if you want to use compression.") 117 | config.set(section3, "compress", "quicklz") 118 | config.set(section3, "compress_chunk_size", "65536") 119 | config.set(section3, "compress_threads", "4") 120 | config.set(section3, "decompress", "TRUE") 121 | config.set(section3, "#Enable if you want to remove .qp files after decompression." 122 | "(Not available yet, will be released with XB 2.3.7 and 2.4.6)") 123 | config.set(section3, "remove_original", "FALSE") 124 | 125 | section4 = "Encrypt" 126 | config.add_section(section4) 127 | config.set(section4, "#Optional") 128 | config.set(section4, "#Enable only if you want to create encrypted backups") 129 | if '2_4' in conf_file: 130 | config.set(section4, "xbcrypt", 131 | "{}/target/percona-xtrabackup-2.4.x-debug/bin/xbcrypt".format(test_path)) 132 | else: 133 | config.set(section4, "xbcrypt", 134 | "{}/target/percona-xtrabackup-2.3.x-debug/bin/xbcrypt".format(test_path)) 135 | config.set(section4, "encrypt", "AES256") 136 | config.set(section4, "#Please note that --encrypt-key and --encrypt-key-file are mutually exclusive") 137 | config.set(section4, "encrypt_key", 'VVTBwgM4UhwkTTV98fhuj+D1zyWoA89K') 138 | config.set(section4, "#encrypt_key_file", "/path/to/file/with_encrypt_key") 139 | config.set(section4, "encrypt_threads", "4") 140 | config.set(section4, "encrypt_chunk_size", "65536") 141 | config.set(section4, "decrypt", "AES256") 142 | config.set(section4, "#Enable if you want to remove .qp files after decompression." 143 | "(Not available yet, will be released with XB 2.3.7 and 2.4.6)") 144 | config.set(section4, "remove_original", "FALSE") 145 | 146 | section5 = "Xbstream" 147 | config.add_section(section5) 148 | config.set(section5, "#EXPERIMENTAL") 149 | config.set(section5, "#Enable this, if you want to stream your backups") 150 | if '2_4' in conf_file: 151 | config.set(section5, "xbstream", 152 | "{}/target/percona-xtrabackup-2.4.x-debug/bin/xbstream".format(test_path)) 153 | else: 154 | config.set(section5, "xbstream", 155 | "{}/target/percona-xtrabackup-2.3.x-debug/bin/xbstream".format(test_path)) 156 | config.set(section5, "stream", "xbstream") 157 | config.set(section5, "xbstream_options", "-x --parallel=100") 158 | config.set(section5, "xbs_decrypt", "1") 159 | config.set(section5, "# WARN, enable this, if you want to stream your backups to remote host") 160 | config.set(section5, "#remote_stream", "ssh xxx.xxx.xxx.xxx") 161 | 162 | section6 = "Remote" 163 | config.add_section(section6) 164 | config.set(section6, "#Optional remote syncing") 165 | config.set(section6, "#remote_conn", "root@xxx.xxx.xxx.xxx") 166 | config.set(section6, "#remote_dir", "/home/sh/Documents") 167 | 168 | section7 = "Commands" 169 | config.add_section(section7) 170 | config.set(section7, "start_mysql_command", "{}/start".format(basedir)) 171 | config.set(section7, "stop_mysql_command", "{}/stop".format(basedir)) 172 | config.set(section7, "chown_command", "chown -R shahriyar.rzaev:shahriyar.rzaev") 173 | 174 | section8 = "TestConf" 175 | config.add_section(section8) 176 | config.set(section8, "ps_branches", "5.5 5.6 5.7") 177 | config.set(section8, "pxb_branches", "2.3 2.4") 178 | config.set(section8, "gitcmd", 179 | "--recursive --depth=1 https://github.com/percona/percona-server.git") 180 | config.set(section8, "pxb_gitcmd", "https://github.com/percona/percona-xtrabackup.git") 181 | config.set(section8, "testpath", "/home/shahriyar.rzaev/XB_TEST/server_dir") 182 | config.set(section8, "incremental_count", "3") 183 | config.set(section8, "xb_configs", "xb_2_4_ps_5_6.conf xb_2_4_ps_5_7.conf xb_2_3_ps_5_6.conf xb_2_3_ps_5_5.conf xb_2_4_ps_5_5.conf") 184 | config.set(section8, "make_slaves", "1") 185 | if '5_7' in conf_file: 186 | config.set(section8, "default_mysql_options", 187 | "--early-plugin-load=keyring_file.so," 188 | "--keyring_file_data={}/mysql-keyring/keyring," 189 | "--log-bin=mysql-bin,--log-slave-updates,--server-id={}," 190 | "--gtid-mode=ON,--enforce-gtid-consistency,--binlog-format=row," 191 | "--encrypt_binlog=ON,--master_verify_checksum=ON,--binlog_checksum=CRC32," 192 | "--innodb_encrypt_tables=ON," 193 | "--innodb_encrypt_online_alter_logs=ON," 194 | "--innodb_temp_tablespace_encrypt=ON") 195 | elif '5_6' in conf_file: 196 | config.set(section8, "default_mysql_options", 197 | "--log-bin=mysql-bin,--log-slave-updates,--server-id={}," 198 | "--gtid-mode=ON,--enforce-gtid-consistency,--binlog-format=row") 199 | elif '5_5' in conf_file: 200 | config.set(section8, "default_mysql_options", 201 | "--log-bin=mysql-bin,--log-slave-updates,--server-id={}," 202 | "--binlog-format=row") 203 | 204 | if '5_7' in conf_file: 205 | config.set(section8, "mysql_options", 206 | "--innodb_buffer_pool_size=1G 2G 3G,--innodb_log_file_size=1G 2G 3G," 207 | "--innodb_page_size=4K 8K 16K 32K") 208 | elif '5_6' in conf_file: 209 | config.set(section8, "mysql_options", 210 | "--innodb_buffer_pool_size=1G 2G 3G,--innodb_log_file_size=1G 2G 3G," 211 | "--innodb_page_size=4K 8K 16K") 212 | elif '5_5' in conf_file: 213 | config.set(section8, "mysql_options", 214 | "--innodb_buffer_pool_size=1G 2G 3G,--innodb_log_file_size=1G 2G 3G," 215 | "--innodb_page_size=4K 8K 16K") 216 | 217 | config.write(cfgfile) 218 | 219 | except Exception as err: 220 | logger.error("Failed to generate config file...") 221 | logger.error(err) 222 | return False 223 | else: 224 | logger.debug("Config file generated successfully...") 225 | return True 226 | 227 | def the_main_generator(self): 228 | # The method for calling config generator based on if statements 229 | conf_list = self.xb_configs.split() 230 | basedirs = self.get_basedir() 231 | print(basedirs) 232 | print(conf_list) 233 | for basedir in basedirs: 234 | for conf_file in conf_list: 235 | if ('5.7' in basedir) and ('2_4_ps_5_7' in conf_file): 236 | self.generate_config_files(test_path=self.testpath, 237 | conf_file=conf_file, 238 | basedir=basedir, 239 | datadir='data', 240 | sock_file=self.benchmark_obj.get_sock(basedir=basedir)) 241 | elif ('5.6' in basedir) and ('2_4_ps_5_6' in conf_file): 242 | self.generate_config_files(test_path=self.testpath, 243 | conf_file=conf_file, 244 | basedir=basedir, 245 | datadir='data', 246 | sock_file=self.benchmark_obj.get_sock(basedir=basedir)) 247 | elif ('5.6' in basedir) and ('2_3_ps_5_6' in conf_file): 248 | self.generate_config_files(test_path=self.testpath, 249 | conf_file=conf_file, 250 | basedir=basedir, 251 | datadir='data', 252 | sock_file=self.benchmark_obj.get_sock(basedir=basedir)) 253 | elif ('5.5' in basedir) and ('2_4_ps_5_5' in conf_file): 254 | self.generate_config_files(test_path=self.testpath, 255 | conf_file=conf_file, 256 | basedir=basedir, 257 | datadir='data', 258 | sock_file=self.benchmark_obj.get_sock(basedir=basedir)) 259 | elif ('5.5' in basedir) and ('2_3_ps_5_5' in conf_file): 260 | self.generate_config_files(test_path=self.testpath, 261 | conf_file=conf_file, 262 | basedir=basedir, 263 | datadir='data', 264 | sock_file=self.benchmark_obj.get_sock(basedir=basedir)) 265 | else: 266 | continue 267 | 268 | return True 269 | 270 | @staticmethod 271 | def options_combination_generator(initial_str): 272 | ''' 273 | Option parser method for creating option combinarotics 274 | :param initial_str -> mysql_options initial string from config file 275 | :return List of tuples with option combinations 276 | ''' 277 | separated_values_list = [] 278 | 279 | for i in initial_str.split(','): 280 | separated_values_list.append(i.split('=')) 281 | 282 | all_new_list = [] 283 | 284 | for i in separated_values_list: 285 | k = ["{}={}".format(i[0], j) for j in i[1].split()] 286 | all_new_list.append(k) 287 | 288 | option_combinations = [] 289 | 290 | for i in product(*all_new_list): 291 | option_combinations.append(i) 292 | 293 | return option_combinations 294 | -------------------------------------------------------------------------------- /prepare_env_test_mode/create_index_temp.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | BASEDIR=$1 4 | MYSQL_SOCK=$2 5 | MYSQL_USER=root 6 | #CONN_STR=${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} 7 | 8 | 9 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "CREATE INDEX t10_b ON sysbench_test_db.t10 (b)" 10 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "DROP INDEX t10_b ON sysbench_test_db.t10" 11 | 12 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "CREATE INDEX t10_b ON sysbench_test_db.t10 (b) ALGORITHM=COPY" 13 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "DROP INDEX t10_b ON sysbench_test_db.t10 ALGORITHM=COPY" 14 | -------------------------------------------------------------------------------- /prepare_env_test_mode/ddl_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | BASEDIR=$1 4 | MYSQL_SOCK=$2 5 | MYSQL_USER=root 6 | #CONN_STR=${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} 7 | 8 | 9 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "drop table if exists sysbench_test_db.sb1" 10 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "create table sysbench_test_db.sb1(id int(11) NOT NULL, c char(120) NOT NULL DEFAULT '')" 11 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "insert into sysbench_test_db.sb1 select id, c from sysbench_test_db.sbtest1" 12 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "create unique index ix on sysbench_test_db.sb1 (id)" 13 | #echo "drop table if exists sysbencht_test_db.sb1"|./use 14 | #echo "create table sb1 as select id,c from sbtest1 where id < 150000;"|./use db1 15 | #echo "create unique index ix on sb1 (id)"|./use db1 16 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "drop table if exists sysbench_test_db.sb2" 17 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "create table sysbench_test_db.sb2(id int(11) NOT NULL, c char(120) NOT NULL DEFAULT '')" 18 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "insert into sysbench_test_db.sb2 select id, c from sysbench_test_db.sbtest1" 19 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "create unique index ix on sysbench_test_db.sb2 (id)" 20 | #echo "drop table if exists db2.sb1"|./use 21 | #echo "create table sb1 as select id,c from sbtest1 where id < 150000;"|./use db2 22 | #echo "create unique index ix on sb1 (id)"|./use db2 -------------------------------------------------------------------------------- /prepare_env_test_mode/innodb_online_alter_encryption.sql: -------------------------------------------------------------------------------- 1 | create database alter_enc_inc; 2 | use alter_enc_inc; 3 | 4 | 5 | CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB ENCRYPTION='y'; 6 | CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB; 7 | CREATE TABLE t3 (id INT, a VARCHAR(255)) ENGINE=InnoDB ENCRYPTION='y'; 8 | CREATE TABLE t4 (id INT, a VARCHAR(255)) ENGINE=InnoDB; 9 | CREATE TABLE t5 (id INT NOT NULL PRIMARY KEY, a TEXT(500), b VARCHAR(255), FULLTEXT(b)) ENGINE=InnoDB ENCRYPTION='y'; 10 | CREATE TABLE t6 (id INT, a TEXT(500), b VARCHAR(255), FULLTEXT(b)) ENGINE=InnoDB; 11 | CREATE TABLE t7 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTION='y'; 12 | 13 | DELIMITER // 14 | CREATE PROCEDURE innodb_insert_proc (repeat_count INT) 15 | BEGIN 16 | DECLARE current_num INT; 17 | SET current_num = 0; 18 | WHILE current_num < repeat_count DO 19 | INSERT INTO t1 VALUES (current_num, REPEAT('foobar', 42)); 20 | INSERT INTO t2 VALUES (current_num, REPEAT('temp', 42)); 21 | INSERT INTO t3 VALUES (current_num, REPEAT('barfoo', 42)); 22 | INSERT INTO t4 VALUES (current_num, REPEAT('repeat', 42)); 23 | INSERT INTO t5 VALUES (current_num, SUBSTRING('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', RAND() * 36 + 1, 100), REPEAT('author new', 22)); 24 | INSERT INTO t6 VALUES (current_num, SUBSTRING('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', RAND() * 36 + 1, 100), REPEAT('mangled old', 22)); 25 | INSERT INTO t7 VALUES (current_num, REPEAT('mysql', 42)); 26 | SET current_num = current_num + 1; 27 | END WHILE; 28 | END// 29 | DELIMITER ; 30 | COMMIT; 31 | 32 | SET autocommit=0; 33 | CALL innodb_insert_proc(15000); 34 | COMMIT; 35 | SET autocommit=1; -------------------------------------------------------------------------------- /prepare_env_test_mode/innodb_online_alter_encryption_alters.sql: -------------------------------------------------------------------------------- 1 | use alter_enc_inc; 2 | 3 | ALTER TABLE t1 ADD COLUMN b INT DEFAULT 2; 4 | ALTER TABLE t2 ADD COLUMN b INT DEFAULT 2; 5 | ALTER TABLE t7 ADD COLUMN b INT DEFAULT 2; 6 | ALTER TABLE t1 ADD KEY a(a), ADD KEY b(b); 7 | ALTER TABLE t2 ADD KEY a(a), ADD KEY b(b); 8 | ALTER TABLE t3 ADD COLUMN c INT DEFAULT 5; 9 | ALTER TABLE t4 ADD COLUMN c INT DEFAULT 5; 10 | ALTER TABLE t3 ADD KEY (a), ADD KEY c(c); 11 | ALTER TABLE t4 ADD KEY (a), ADD KEY c(c); 12 | ALTER TABLE t5 ADD FULLTEXT(a); 13 | ALTER TABLE t6 ADD FULLTEXT(a); 14 | ALTER TABLE t7 ADD KEY a(a), ADD KEY b(b); 15 | 16 | alter table t1 drop column b; 17 | alter table t2 drop column b; 18 | alter table t7 drop column b; 19 | 20 | ALTER TABLE t1 drop KEY a; 21 | ALTER TABLE t1 drop KEY b; 22 | ALTER TABLE t2 drop KEY a; 23 | ALTER TABLE t2 drop KEY b; 24 | 25 | ALTER TABLE t3 drop COLUMN c; 26 | ALTER TABLE t4 drop COLUMN c; 27 | 28 | ALTER TABLE t3 drop KEY a; 29 | ALTER TABLE t3 drop KEY c; 30 | 31 | ALTER TABLE t4 drop KEY a; 32 | ALTER TABLE t4 drop KEY c; 33 | alter table t5 drop key a; 34 | alter table t6 drop key a; 35 | 36 | ALTER TABLE t7 drop KEY a; 37 | ALTER TABLE t7 drop KEY c; 38 | 39 | -------------------------------------------------------------------------------- /prepare_env_test_mode/prepare_backup.py: -------------------------------------------------------------------------------- 1 | from backup_prepare.prepare import Prepare 2 | 3 | 4 | class WrapperForPrepareTest(Prepare): 5 | 6 | def __init__(self, config='/etc/bck.conf', full_dir=None, inc_dir=None): 7 | self.conf = config 8 | super().__init__(config=self.conf) 9 | if full_dir is not None: 10 | self.full_dir = full_dir 11 | if inc_dir is not None: 12 | self.inc_dir = inc_dir 13 | 14 | def run_prepare_backup(self): 15 | self.prepare_inc_full_backups() 16 | return True 17 | 18 | def run_copy_back(self): 19 | self.copy_back_action() 20 | return True -------------------------------------------------------------------------------- /prepare_env_test_mode/run_benchmark.py: -------------------------------------------------------------------------------- 1 | from general_conf.generalops import GeneralClass 2 | from prepare_env_test_mode.clone_build_start_server import CloneBuildStartServer 3 | import subprocess 4 | import logging 5 | logger = logging.getLogger(__name__) 6 | 7 | 8 | class RunBenchmark: 9 | """ 10 | General class for running all kind of Benchmarks; For now running sysbench against started server. 11 | """ 12 | 13 | def __init__(self, config="/etc/bck.conf"): 14 | self.conf = config 15 | self.testpath = GeneralClass(self.conf).testpath 16 | self.basedir = CloneBuildStartServer(self.conf).get_basedir() 17 | 18 | @staticmethod 19 | def get_sock(basedir): 20 | # Get socket connection path from PS basedir(Pythonic way) 21 | logger.debug("Trying to get socket file...") 22 | file_name = "{}/cl_noprompt_nobinary" 23 | with open(file_name.format(basedir)) as config: 24 | sock_file = config.read().split()[3][2:] 25 | 26 | return sock_file 27 | 28 | @staticmethod 29 | def get_mysql_conn(basedir, file_name=None): 30 | # Get mysql client connection 31 | logger.debug("Trying to get mysql client connection...") 32 | if file_name is None: 33 | get_conn = "cat {}/cl_noprompt_nobinary" 34 | status, output = subprocess.getstatusoutput(get_conn.format(basedir)) 35 | else: 36 | get_conn = "cat {}/{}" 37 | status, output = subprocess.getstatusoutput(get_conn.format(basedir, file_name)) 38 | if status == 0: 39 | logger.debug("Could get mysql client") 40 | return output 41 | else: 42 | logger.error("Failed to get mysql client connection") 43 | logger.error(output) 44 | raise RuntimeError("Failed to get mysql client connection") 45 | 46 | @staticmethod 47 | def run_sql_statement(basedir, sql_statement): 48 | sql = '{} -e \"{}\"'.format(RunBenchmark.get_mysql_conn(basedir), sql_statement) 49 | status, output = subprocess.getstatusoutput(sql) 50 | if status == 0: 51 | logger.debug("OK: Running -> {}".format(sql)) 52 | return True 53 | else: 54 | logger.error("FAILED: running SQL -> {}".format(sql)) 55 | logger.error(output) 56 | raise RuntimeError("FAILED: running SQL -> {}".format(sql)) 57 | 58 | def create_db(self, db_name, basedir): 59 | # Creating DB using mysql client 60 | conn = self.get_mysql_conn(basedir) 61 | sql = "{} -e 'create database if not exists {} '" 62 | logger.debug("Trying to create DB...") 63 | status, output = subprocess.getstatusoutput(sql.format(conn, db_name)) 64 | if status == 0: 65 | logger.debug("Given DB is created") 66 | return True 67 | else: 68 | logger.error("Failed to create DB") 69 | logger.error(output) 70 | return False 71 | 72 | def run_sysbench_prepare(self, basedir): 73 | # Running sysbench prepare here; The parameters hard coded here, should figure out how to pass them also 74 | # TODO: make sysbench run with dynamic values 75 | # TODO: make sysbench different possible kind of runs 76 | 77 | # Created sysbench DB 78 | db_name = "sysbench_test_db" 79 | self.create_db(db_name=db_name, basedir=basedir) 80 | 81 | # Likely to fail here! Pay attention 82 | sock_name = self.get_sock(basedir) 83 | 84 | sysbench_cmd = "sysbench /usr/share/sysbench/oltp_insert.lua " \ 85 | "--table-size={} " \ 86 | "--tables={} " \ 87 | "--mysql-db={} " \ 88 | "--mysql-user=root " \ 89 | "--threads={} " \ 90 | "--db-driver=mysql " \ 91 | "--mysql-socket={} prepare" 92 | 93 | logger.debug("Running command -> {}".format(sysbench_cmd.format(1000, 94 | 35, 95 | db_name, 96 | 100, 97 | sock_name))) 98 | 99 | status, output = subprocess.getstatusoutput(sysbench_cmd.format(1000, 100 | 35, 101 | db_name, 102 | 100, 103 | sock_name)) 104 | 105 | if status == 0: 106 | logger.debug("Sysbench succeeded!") 107 | return True 108 | else: 109 | logger.error("Failed to run sysbench") 110 | logger.error(output) 111 | raise RuntimeError("Failed to run sysbench") 112 | 113 | def run_sysbench_run(self, basedir): 114 | # Running sysbench run here 115 | db_name = "sysbench_test_db" 116 | 117 | sock_name = self.get_sock(basedir) 118 | 119 | sysbench_cmd = "sysbench /usr/share/sysbench/oltp_update_non_index.lua " \ 120 | "--table-size={} " \ 121 | "--tables={} " \ 122 | "--mysql-db={} " \ 123 | "--mysql-user=root " \ 124 | "--threads={} " \ 125 | "--db-driver=mysql " \ 126 | "--mysql-socket={} run" 127 | 128 | logger.debug("Running command -> {}".format(sysbench_cmd.format(1000, 129 | 35, 130 | db_name, 131 | 100, 132 | sock_name))) 133 | 134 | status, output = subprocess.getstatusoutput(sysbench_cmd.format(1000, 135 | 35, 136 | db_name, 137 | 100, 138 | sock_name)) 139 | 140 | if status == 0: 141 | logger.debug("Sysbench succeeded!") 142 | return True 143 | else: 144 | logger.error("Failed to run sysbench") 145 | logger.error(output) 146 | raise RuntimeError("Failed to run sysbench") 147 | -------------------------------------------------------------------------------- /prepare_env_test_mode/run_sql_queries.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Executed as run_sql_queries.sh BASEDIR_PATH 50 SOCKET 3 | BASEDIR=$1 4 | MYSQL_SOCK=$2 5 | SQL=$3 6 | MYSQL_USER=root 7 | 8 | #echo "Running queries using MYSQL_SOCK=${MYSQL_SOCK}" 9 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "${SQL}" > /dev/null 2>&1 10 | -------------------------------------------------------------------------------- /prepare_env_test_mode/start_dynamic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Call this inside basedir, it will create start_dynamic executable file. 3 | 4 | # Get version specific options 5 | BIN= 6 | if [ -r ${PWD}/bin/mysqld-debug ]; then BIN="${PWD}/bin/mysqld-debug"; fi # Needs to come first so it's overwritten in next line if both exist 7 | if [ -r ${PWD}/bin/mysqld ]; then BIN="${PWD}/bin/mysqld"; fi 8 | if [ "${BIN}" == "" ]; then echo "Assert: no mysqld or mysqld-debug binary was found!"; fi 9 | 10 | JE1="if [ -r /usr/lib64/libjemalloc.so.1 ]; then export LD_PRELOAD=/usr/lib64/libjemalloc.so.1" 11 | JE2=" elif [ -r /usr/lib/x86_64-linux-gnu/libjemalloc.so.1 ]; then export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1" 12 | JE3=" elif [ -r /usr/local/lib/libjemalloc.so ]; then export LD_PRELOAD=/usr/local/lib/libjemalloc.so" 13 | JE4=" elif [ -r ${PWD}/lib/mysql/libjemalloc.so.1 ]; then export LD_PRELOAD=${PWD}/lib/mysql/libjemalloc.so.1" 14 | JE5=" else echo 'Error: jemalloc not found, please install it first'; exit 1; fi" 15 | 16 | START_OPT="--core-file" # Compatible with 5.6,5.7,8.0 17 | 18 | if [ -r ${PWD}/lib/mysql/plugin/ha_tokudb.so ]; then 19 | TOKUDB="--plugin-load-add=tokudb=ha_tokudb.so --tokudb-check-jemalloc=0" 20 | else 21 | TOKUDB="" 22 | fi 23 | if [ -r ${PWD}/lib/mysql/plugin/ha_rocksdb.so ]; then 24 | ROCKSDB="--plugin-load-add=rocksdb=ha_rocksdb.so" 25 | else 26 | ROCKSDB="" 27 | fi 28 | 29 | echo "MYEXTRA_OPT=\"\$*\"" > start_dynamic 30 | echo 'MYEXTRA=" --no-defaults --secure-file-priv="' >> start_dynamic 31 | echo '#MYEXTRA=" --no-defaults --sql_mode="' >> start_dynamic 32 | echo "#MYEXTRA=\" --no-defaults --performance-schema --performance-schema-instrument='%=on'\" # For PMM" >> start_dynamic 33 | echo '#MYEXTRA=" --no-defaults --default-tmp-storage-engine=MyISAM --rocksdb --skip-innodb --default-storage-engine=RocksDB"' >> start_dynamic 34 | echo '#MYEXTRA=" --no-defaults --event-scheduler=ON --maximum-bulk_insert_buffer_size=1M --maximum-join_buffer_size=1M --maximum-max_heap_table_size=1M --maximum-max_join_size=1M --maximum-myisam_max_sort_file_size=1M --maximum-myisam_mmap_size=1M --maximum-myisam_sort_buffer_size=1M --maximum-optimizer_trace_max_mem_size=1M --maximum-preload_buffer_size=1M --maximum-query_alloc_block_size=1M --maximum-query_prealloc_size=1M --maximum-range_alloc_block_size=1M --maximum-read_buffer_size=1M --maximum-read_rnd_buffer_size=1M --maximum-sort_buffer_size=1M --maximum-tmp_table_size=1M --maximum-transaction_alloc_block_size=1M --maximum-transaction_prealloc_size=1M --log-output=none --sql_mode=ONLY_FULL_GROUP_BY"' >> start_dynamic 35 | 36 | echo $JE1 >> start_dynamic; echo $JE2 >> start_dynamic; echo $JE3 >> start_dynamic; echo $JE4 >> start_dynamic; echo $JE5 >> start_dynamic 37 | echo "$BIN \${MYEXTRA} \${MYEXTRA_OPT} ${START_OPT} --basedir=${PWD} ${TOKUDB} ${ROCKSDB} 2>&1 &" >> start_dynamic 38 | echo "sleep 10" >> start_dynamic 39 | 40 | chmod u+x start_dynamic 2>/dev/null -------------------------------------------------------------------------------- /prepare_env_test_mode/temp_table_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | BASEDIR=$1 4 | MYSQL_SOCK=$2 5 | MYSQL_USER=root 6 | #CONN_STR=${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} 7 | 8 | 9 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "CREATE TEMPORARY TABLE sysbench_test_db.t04 (a TEXT) ENGINE=InnoDB encryption='N';INSERT INTO sysbench_test_db.t04 VALUES ('Praesent tristique eros a tempus fringilla');" 10 | ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "CREATE TEMPORARY TABLE sysbench_test_db.t03 (a TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED encryption='N';INSERT INTO sysbench_test_db.t03 VALUES ('Praesent tristique eros a tempus fringilla');" 11 | -------------------------------------------------------------------------------- /prepare_env_test_mode/test_check_env.py: -------------------------------------------------------------------------------- 1 | from general_conf.generalops import GeneralClass 2 | import os 3 | import sys 4 | 5 | import logging 6 | logger = logging.getLogger(__name__) 7 | 8 | class TestModeConfCheck(GeneralClass): 9 | """ 10 | Class for checking environment for running Test Mode. 11 | """ 12 | def __init__(self, config='/etc/bck.conf'): 13 | self.conf = config 14 | super().__init__(config=self.conf) 15 | if hasattr(self, 'gitcmd') and hasattr(self, 'testpath'): 16 | pass 17 | else: 18 | logger.critical("Missing needed variables from config file") 19 | sys.exit(-1) 20 | 21 | def check_test_path(self, path): 22 | if not os.path.exists(path): 23 | try: 24 | logger.debug("Test dir does not exist") 25 | logger.debug("Creating test mode directory") 26 | os.makedirs(self.testpath) 27 | return True 28 | except Exception as err: 29 | logger.error(err) 30 | return False 31 | else: 32 | return True 33 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | datafiles = [('//etc', ['general_conf/bck.conf'])] 4 | 5 | setup( 6 | name='mysql-autoxtrabackup', 7 | version='1.5.2', 8 | packages=['general_conf', 'backup_prepare', 'partial_recovery', 'master_backup_script', 'prepare_env_test_mode'], 9 | package_data={ 10 | 'prepare_env_test_mode': ['*.sh', '*.sql'] 11 | }, 12 | py_modules=['autoxtrabackup'], 13 | url='https://github.com/ShahriyarR/MySQL-AutoXtraBackup', 14 | download_url='https://github.com/ShahriyarR/MySQL-AutoXtraBackup/archive/v1.5.2.zip', 15 | license='MIT', 16 | author='Shahriyar Rzayev', 17 | author_email='rzayev.shahriyar@yandex.com', 18 | description='Commandline tool written in Python 3 for using Percona XtraBackup', 19 | install_requires=[ 20 | 'click>=3.3', 21 | 'mysql-connector==2.1.4', 22 | 'pid>=2.0', 23 | 'humanfriendly>=2.0', 24 | 'pytest' 25 | ], 26 | dependency_links=['https://dev.mysql.com/get/Downloads/Connector-Python/mysql-connector-python-2.1.4.tar.gz'], 27 | entry_points=''' 28 | [console_scripts] 29 | autoxtrabackup=autoxtrabackup:all_procedure 30 | ''', 31 | data_files=datafiles, 32 | ) 33 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # Special information on using autoxtrabackup --test_mode 2 | 3 | ## Preparing environment: 4 | 5 | * Edit /etc/bck.conf and specify testpath under [TestConf] - in which path the test environment should be constructed, created. 6 | * ps_branches - specify which branches of PS should be cloned and build. 7 | * pxb_branches - specify which branches of PXB should be cloned and build. 8 | * gitcmd - specify a link of PS repo. 9 | * pxb_gitcmd - specify a link of PXB repo. 10 | 11 | **Running with bats**: 12 | 13 | The reason I choose to call PyTests from bats is, it gives an elegant way to constructs tests. 14 | 15 | So what you need basically the BATS framework: 16 | 17 | [installing-bats-from-source](https://github.com/sstephenson/bats#installing-bats-from-source) 18 | 19 | **Running whole environment setup** 20 | 21 | The thing you need is to run: 22 | 23 | `bats prepare_env.bats` 24 | 25 | That's it, it will do the all job. 26 | 27 | **Running specific bats files** 28 | 29 | I have prepared separate bats files to run specific things: 30 | 31 | * `test_clone_percona_qa.bats` -> will clone percona-qa repo. 32 | * `test_clone_ps_server_from_conf.bats` -> will clone PS servers based on specified branches. 33 | * `test_clone_pxb.bats` -> will clone specified PXB based on specified branches. 34 | * `test_build_pxb.bats` -> will build cloned PXBs and create separate binary archives for each branch. 35 | * `test_build_server.bats` -> will build PS servers. 36 | * `test_prepare_startup.bats` -> will run startup.sh from percona-qa inside PS basedirs. 37 | * `test_prepare_start_dynamic.bats` -> will create start_dynamic scripts inside PS basedirs, which is going to be used in slave setup etc. 38 | * `test_start_server.bats` -> will start PS servers with default values(executing start script inside basedirs). 39 | * `test_extract_xb_archive.bats` -> will extracting PXB binary archives to the target folder inside testpath(which is grabbed from config file). 40 | * `test_generate_config_files.bats` -> will generate specific config files based on PXB and PS versions. 41 | 42 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Percona-Lab/MySQL-AutoXtraBackup/8ae7927e72c03cf4e685d26f2c2d0d2580eeac52/test/__init__.py -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from prepare_env_test_mode.clone_build_start_server import CloneBuildStartServer 3 | from prepare_env_test_mode.run_benchmark import RunBenchmark 4 | from prepare_env_test_mode.config_generator import ConfigGenerator 5 | from prepare_env_test_mode.runner_test_mode import RunnerTestMode 6 | from master_backup_script.backuper import Backup 7 | 8 | clb_obj = CloneBuildStartServer() 9 | 10 | 11 | @pytest.fixture() 12 | def return_clone_obj(): 13 | return clb_obj 14 | 15 | 16 | @pytest.fixture() 17 | def return_basedir(): 18 | basedir = clb_obj.get_basedir() 19 | return basedir 20 | 21 | rb_obj = RunBenchmark() 22 | 23 | 24 | @pytest.fixture() 25 | def return_run_benchmark_obj(): 26 | return rb_obj 27 | 28 | cg_obj = ConfigGenerator() 29 | 30 | 31 | @pytest.fixture() 32 | def return_config_generator_obj(): 33 | return cg_obj 34 | 35 | @pytest.fixture() 36 | def return_runner_test_mode_obj_5_6_xb_2_3(): 37 | obj = RunnerTestMode(config="{}/{}".format(clb_obj.testpath, 'xb_2_3_ps_5_6.conf')) 38 | return obj 39 | 40 | @pytest.fixture() 41 | def return_runner_test_mode_obj_5_6_xb_2_4(): 42 | obj = RunnerTestMode(config="{}/{}".format(clb_obj.testpath, 'xb_2_4_ps_5_6.conf')) 43 | return obj 44 | 45 | @pytest.fixture() 46 | def return_runner_test_mode_obj_5_7_xb_2_4(): 47 | obj = RunnerTestMode(config="{}/{}".format(clb_obj.testpath, 'xb_2_4_ps_5_7.conf')) 48 | return obj -------------------------------------------------------------------------------- /test/prepare_env.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Run this BATS file to prepare full test environment 4 | 5 | DIRNAME=$BATS_TEST_DIRNAME 6 | 7 | @test "Running test_clone_percona_qa" { 8 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_clone_percona_qa 9 | echo $output 10 | [ $status -eq 0 ] 11 | } 12 | 13 | @test "Running test_clone_ps_server_from_conf" { 14 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_clone_ps_server_from_conf 15 | echo $output 16 | [ $status -eq 0 ] 17 | } 18 | 19 | @test "Running test_clone_pxb" { 20 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_clone_pxb 21 | echo $output 22 | [ $status -eq 0 ] 23 | } 24 | 25 | @test "Running test_build_pxb" { 26 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_build_pxb 27 | echo $output 28 | [ $status -eq 0 ] 29 | } 30 | 31 | @test "Running test_build_server" { 32 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_build_server 33 | echo $output 34 | [ $status -eq 0 ] 35 | } 36 | 37 | @test "Running test_rename_basedirs" { 38 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_rename_basedirs 39 | echo $output 40 | [ $status -eq 0 ] 41 | } 42 | 43 | @test "Running test_prepare_startup" { 44 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_prepare_startup 45 | echo $output 46 | [ $status -eq 0 ] 47 | } 48 | 49 | @test "Running test_prepare_start_dynamic" { 50 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_prepare_start_dynamic 51 | echo $output 52 | [ $status -eq 0 ] 53 | } 54 | 55 | @test "Running test_start_server" { 56 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_start_server 57 | echo $output 58 | [ $status -eq 0 ] 59 | } 60 | 61 | @test "Running test_extract_xb_archive" { 62 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_extract_xb_archive 63 | echo $output 64 | [ $status -eq 0 ] 65 | } 66 | 67 | @test "Running test_generate_config_files" { 68 | run python -m pytest -vv ${DIRNAME}/test_config_generator.py::TestConfigGenerator::test_generate_config_files 69 | echo $output 70 | [ $status -eq 0 ] 71 | } -------------------------------------------------------------------------------- /test/test_backup.py: -------------------------------------------------------------------------------- 1 | # PyTest file for testing Backup class 2 | from master_backup_script.backuper import Backup 3 | from general_conf.generalops import GeneralClass 4 | 5 | class TestBackup: 6 | 7 | def test_add_tag(self): 8 | # Method for checking the add_tag() static method. All parameters are hard coded. 9 | gen_obj = GeneralClass() 10 | for conf_files in gen_obj.xb_configs.split(): 11 | if '2_3' in conf_files and '5_6' in conf_files: 12 | obj = Backup(config='{}/{}'.format(gen_obj.testpath, conf_files), dry_run=0, tag="My first full backup") 13 | backup_name = obj.recent_full_backup_file() 14 | obj.add_tag(backup_dir=obj.backupdir, backup_name=backup_name, type='Full', tag_string=obj.tag) 15 | 16 | def test_show_tags(self): 17 | gen_obj = GeneralClass() 18 | for conf_files in gen_obj.xb_configs.split(): 19 | if '2_3' in conf_files and '5_6' in conf_files: 20 | obj = Backup(config='{}/{}'.format(gen_obj.testpath, conf_files)) 21 | obj.show_tags(obj.backupdir) 22 | 23 | def test_full_backup(self): 24 | # Method for running full_backup() 25 | gen_obj = GeneralClass() 26 | for conf_files in gen_obj.xb_configs.split(): 27 | if '2_3' in conf_files and '5_6' in conf_files: 28 | obj = Backup(config='{}/{}'.format(gen_obj.testpath, conf_files), dry_run=0, tag="My first full backup") 29 | obj.full_backup() 30 | 31 | def test_inc_backup(self): 32 | # Method for running inc_backup() 33 | gen_obj = GeneralClass() 34 | for conf_files in gen_obj.xb_configs.split(): 35 | if '2_3' in conf_files and '5_6' in conf_files: 36 | obj = Backup(config='{}/{}'.format(gen_obj.testpath, conf_files), dry_run=0, tag="My first inc backup") 37 | obj.inc_backup() 38 | -------------------------------------------------------------------------------- /test/test_build_pxb.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Run this after running test_clone_pxb.bats 4 | # It will build the cloned/specified branches and will create separate binary .tar.gz files for each branch. 5 | 6 | DIRNAME=$BATS_TEST_DIRNAME 7 | 8 | @test "Running test_build_pxb" { 9 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_build_pxb 10 | echo $output 11 | [ $status -eq 0 ] 12 | } -------------------------------------------------------------------------------- /test/test_build_server.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Run this BATS file to build PS servers from source. 4 | 5 | DIRNAME=$BATS_TEST_DIRNAME 6 | 7 | @test "Running test_build_server" { 8 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_build_server 9 | echo $output 10 | [ $status -eq 0 ] 11 | } -------------------------------------------------------------------------------- /test/test_clone_build_start_server.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | @pytest.mark.usefixtures("return_clone_obj") 4 | class TestCloneBuildStartServer: 5 | """ 6 | Tests for clone_build_start_server.py module 7 | """ 8 | 9 | def test_clone_percona_qa(self, return_clone_obj): 10 | assert return_clone_obj.clone_percona_qa() is True 11 | 12 | def test_clone_ps_server_from_conf(self, return_clone_obj): 13 | assert return_clone_obj.clone_ps_server_from_conf() is True 14 | 15 | def test_clone_pxb(self, return_clone_obj): 16 | assert return_clone_obj.clone_pxb() is True 17 | 18 | def test_build_pxb(self, return_clone_obj): 19 | assert return_clone_obj.build_pxb() is True 20 | 21 | def test_build_server(self, return_clone_obj): 22 | assert return_clone_obj.build_server() is True 23 | 24 | def test_rename_basedirs(self, return_clone_obj): 25 | assert return_clone_obj.rename_basedirs() is True 26 | 27 | def test_get_basedir(self, return_clone_obj): 28 | assert isinstance(return_clone_obj.get_basedir(), list) 29 | assert len(return_clone_obj.get_basedir()) > 0 30 | 31 | @pytest.mark.usefixtures("return_basedir") 32 | def test_prepare_startup(self, return_clone_obj, return_basedir): 33 | basedir_list = return_basedir 34 | for basedir in basedir_list: 35 | assert return_clone_obj.prepare_startup(basedir_path=basedir) is True 36 | 37 | @pytest.mark.usefixtures("return_basedir") 38 | def test_prepare_start_dynamic(self, return_clone_obj, return_basedir): 39 | basedir_list = return_basedir 40 | for basedir in basedir_list: 41 | assert return_clone_obj.prepare_start_dynamic(basedir_path=basedir) is True 42 | 43 | @pytest.mark.usefixtures("return_basedir") 44 | def test_start_server(self, return_clone_obj, return_basedir): 45 | basedirs = return_basedir 46 | for basedir in basedirs: 47 | assert return_clone_obj.start_server(basedir_path=basedir) 48 | 49 | @pytest.mark.usefixtures("return_basedir") 50 | def test_wipe_server_all(self, return_clone_obj, return_basedir): 51 | basedirs = return_basedir 52 | for basedir in basedirs: 53 | assert return_clone_obj.wipe_server_all(basedir_path=basedir) is True 54 | 55 | def test_extract_xb_archive(self, return_clone_obj): 56 | for branch in return_clone_obj.pxb_branches.split(): 57 | archive_name = "percona-xtrabackup-{}.x-debug.tar.gz".format(branch) 58 | assert return_clone_obj.extract_xb_archive(file_name=archive_name) is True -------------------------------------------------------------------------------- /test/test_clone_percona_qa.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # This is for cloning percona-qa repo. 4 | # It is crucial because we are going to use some setup scripts from this repo. 5 | 6 | DIRNAME=$BATS_TEST_DIRNAME 7 | 8 | @test "Running test_clone_percona_qa" { 9 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_clone_percona_qa 10 | echo $output 11 | [ $status -eq 0 ] 12 | } -------------------------------------------------------------------------------- /test/test_clone_ps_server_from_conf.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Run this to clone PS(percona server) branches. By default 5.5 , 5,6 and 5.7 4 | 5 | DIRNAME=$BATS_TEST_DIRNAME 6 | 7 | @test "Running test_clone_ps_server_from_conf" { 8 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_clone_ps_server_from_conf 9 | echo $output 10 | [ $status -eq 0 ] 11 | } -------------------------------------------------------------------------------- /test/test_clone_pxb.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Edit /etc/bck.conf and specify PXB branches and github link to clone. 4 | # It will create separate folders for each branch. 5 | 6 | DIRNAME=$BATS_TEST_DIRNAME 7 | 8 | @test "Running test_clone_pxb" { 9 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_clone_pxb 10 | echo $output 11 | [ $status -eq 0 ] 12 | } -------------------------------------------------------------------------------- /test/test_config_generator.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from general_conf.generalops import GeneralClass 3 | 4 | @pytest.mark.usefixtures("return_config_generator_obj") 5 | class TestConfigGenerator: 6 | # Class for testing config_generator.py 7 | 8 | def test_generate_config_files(self, return_config_generator_obj): 9 | assert return_config_generator_obj.the_main_generator() is True 10 | 11 | def test_options_combination_generator(self, return_config_generator_obj): 12 | mysql_options = GeneralClass(config='{}/{}'.format(return_config_generator_obj.testpath, 'xb_2_4_ps_5_6.conf')).mysql_options 13 | assert len(return_config_generator_obj.options_combination_generator(mysql_options)) > 0 14 | assert isinstance(return_config_generator_obj.options_combination_generator(mysql_options), list) -------------------------------------------------------------------------------- /test/test_extract_xb_archive.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Extracting PXB binary archives to the target folder inside testpath(which is grabbed from config file). 4 | 5 | DIRNAME=$BATS_TEST_DIRNAME 6 | 7 | @test "Running test_extract_xb_archive" { 8 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_extract_xb_archive 9 | echo $output 10 | [ $status -eq 0 ] 11 | } -------------------------------------------------------------------------------- /test/test_generate_config_files.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Generating specific config files based on PXB and PS versions. 4 | # Those config files can be used with autoxtrabackup by passing to --defaults_file option. 5 | 6 | DIRNAME=$BATS_TEST_DIRNAME 7 | 8 | @test "Running test_generate_config_files" { 9 | run python -m pytest -vv ${DIRNAME}/test_config_generator.py::TestConfigGenerator::test_generate_config_files 10 | echo $output 11 | [ $status -eq 0 ] 12 | } -------------------------------------------------------------------------------- /test/test_option_combinations.py: -------------------------------------------------------------------------------- 1 | from general_conf.generalops import GeneralClass 2 | 3 | class TestOptionCombinations: 4 | # Class for testing starting/initializing mysqld with different option combinations 5 | 6 | def test_combination_creation(self): 7 | pass -------------------------------------------------------------------------------- /test/test_partialRecovery.py: -------------------------------------------------------------------------------- 1 | from partial_recovery.partial import PartialRecovery 2 | import pytest 3 | 4 | class TestPartialRecovery: 5 | """Tests for PartialRecovery class and methods""" 6 | 7 | def test_create_mysql_client_command(self): 8 | """Checking return value type""" 9 | print("\nIn test_create_mysql_client_command()...") 10 | return_value = PartialRecovery().create_mysql_client_command("") 11 | assert type(return_value) == str 12 | 13 | def test_create_mysql_client_command01(self): 14 | """Checking if TypeError raised""" 15 | print("\nIn test_create_mysql_client_command01()...") 16 | with pytest.raises(TypeError): 17 | PartialRecovery().create_mysql_client_command() 18 | 19 | def test_check_innodb_file_per_table(self): 20 | """Checking return value type""" 21 | print("\nIn test_check_innodb_file_per_table()...") 22 | assert type(PartialRecovery().check_innodb_file_per_table()) == bool 23 | 24 | def test_check_mysql_version(self): 25 | """Checking return value type""" 26 | print("\nIn test_check_mysql_version()...") 27 | assert type(PartialRecovery().check_mysql_version()) == bool 28 | 29 | def test_check_database_exists_on_mysql(self): 30 | """Checking return value type""" 31 | print("\nIn test_check_mysql_version()...") 32 | assert type(PartialRecovery().check_database_exists_on_mysql("")) == bool 33 | 34 | def test_check_database_exists_on_mysql01(self): 35 | """Checking if TypeError raised""" 36 | print("\nIn test_check_mysql_version01()...") 37 | with pytest.raises(TypeError): 38 | PartialRecovery().check_database_exists_on_mysql() 39 | 40 | def test_check_table_exists_on_mysql(self): 41 | """Checking return value type""" 42 | print("\nIn test_check_table_exists_on_mysql()...") 43 | assert type(PartialRecovery().check_table_exists_on_mysql("","","")) == bool 44 | 45 | def test_check_table_exists_on_mysql01(self): 46 | """Checking if TypeError raised""" 47 | print("\nIn test_check_table_exists_on_mysql01()...") 48 | with pytest.raises(TypeError): 49 | PartialRecovery().check_table_exists_on_mysql() 50 | 51 | def test_run_mysqlfrm_utility(self): 52 | """Checking return value for None""" 53 | print("\nIn test_run_mysqlfrm_utility()...") 54 | assert PartialRecovery().run_mysqlfrm_utility("") is None 55 | 56 | def test_run_mysqlfrm_utility01(self): 57 | """Checking return value for None""" 58 | print("\nIn test_run_mysqlfrm_utility01()...") 59 | assert PartialRecovery().run_mysqlfrm_utility("/etc/passwd") is None 60 | 61 | def test_get_table_ibd_file(self): 62 | """Checking if TypeError raised""" 63 | print("\nIn test_get_table_ibd_file()...") 64 | with pytest.raises(TypeError): 65 | PartialRecovery().get_table_ibd_file() 66 | 67 | def test_get_table_ibd_file01(self): 68 | """Checking return value type""" 69 | print("\nIn test_get_table_ibd_file01()...") 70 | assert type(PartialRecovery().get_table_ibd_file("","")) == bool 71 | 72 | def test_lock_table(self): 73 | """Checking return value type""" 74 | print("\nIn test_lock_table()...") 75 | assert type(PartialRecovery().lock_table("","")) == bool 76 | 77 | def test_lock_table01(self): 78 | """Checking if TypeError raised""" 79 | print("\nIn test_lock_table01()...") 80 | with pytest.raises(TypeError): 81 | PartialRecovery().lock_table() 82 | 83 | def test_alter_tablespace(self): 84 | """Checking return value type""" 85 | print("\nIn test_alter_tablespace()...") 86 | assert type(PartialRecovery().alter_tablespace("","")) == bool 87 | 88 | def test_alter_tablespace01(self): 89 | """Checking if TypeError raised""" 90 | print("\nIn test_alter_tablespace01()...") 91 | with pytest.raises(TypeError): 92 | PartialRecovery().alter_tablespace() 93 | 94 | def test_copy_ibd_file_back(self): 95 | """Checking return value type""" 96 | print("\nIn test_copy_ibd_file_back()...") 97 | assert type(PartialRecovery().copy_ibd_file_back("","")) == bool 98 | 99 | def test_copy_ibd_file_back01(self): 100 | """Checking if TypeError raised""" 101 | print("\nIn test_copy_ibd_file_back01()...") 102 | with pytest.raises(TypeError): 103 | PartialRecovery().copy_ibd_file_back() 104 | 105 | def test_give_chown(self): 106 | """Checking return value type""" 107 | print("\nIn test_give_chown()...") 108 | assert type(PartialRecovery().give_chown("")) == bool 109 | 110 | def test_give_chown01(self): 111 | """Checking if TypeError raised""" 112 | print("\nIn test_give_chown01()...") 113 | with pytest.raises(TypeError): 114 | PartialRecovery().give_chown() 115 | 116 | def test_import_tablespace(self): 117 | """Checking return value type""" 118 | print("\nIn test_import_tablespace()...") 119 | assert type(PartialRecovery().import_tablespace("", "")) == bool 120 | 121 | def test_import_tablespace01(self): 122 | """Checking if TypeError raised""" 123 | print("\nIn test_import_tablespace01()...") 124 | with pytest.raises(TypeError): 125 | PartialRecovery().import_tablespace() 126 | 127 | def test_unlock_tables(self): 128 | """Checking return value type""" 129 | print("\nIn test_unlock_tables()...") 130 | assert type(PartialRecovery().unlock_tables()) == bool 131 | 132 | 133 | # def test_final_actions(self): 134 | # print("\nIn test_final_actions()...") 135 | # PartialRecovery().final_actions() 136 | -------------------------------------------------------------------------------- /test/test_prepare_start_dynamic.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Running this to create start_dynamic scripts in basedirs, it is needed for slave startup etc. 4 | 5 | DIRNAME=$BATS_TEST_DIRNAME 6 | 7 | @test "Running test_prepare_start_dynamic" { 8 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_prepare_start_dynamic 9 | echo $output 10 | [ $status -eq 0 ] 11 | } -------------------------------------------------------------------------------- /test/test_prepare_startup.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Running startup.sh file from percona-qa repo inside each PS basedir. 4 | 5 | DIRNAME=$BATS_TEST_DIRNAME 6 | 7 | @test "Running test_prepare_startup" { 8 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_prepare_startup 9 | echo $output 10 | [ $status -eq 0 ] 11 | } -------------------------------------------------------------------------------- /test/test_rename_basedirs.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Renaming basedirs in testpath 4 | 5 | DIRNAME=$BATS_TEST_DIRNAME 6 | 7 | @test "Running test_rename_basedirs" { 8 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_rename_basedirs 9 | echo $output 10 | [ $status -eq 0 ] 11 | } -------------------------------------------------------------------------------- /test/test_run_benchmark.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | @pytest.mark.usefixtures("return_run_benchmark_obj") 4 | class TestRunBenchmark: 5 | """ 6 | Test class for run_benchmark.py 7 | """ 8 | 9 | def test_get_sock(self, return_run_benchmark_obj): 10 | for basedir in return_run_benchmark_obj.basedir: 11 | assert "sock" in return_run_benchmark_obj.get_sock(basedir=basedir) 12 | 13 | def test_get_mysql_conn(self, return_run_benchmark_obj): 14 | for basedir in return_run_benchmark_obj.basedir: 15 | assert "mysql" in return_run_benchmark_obj.get_mysql_conn(basedir=basedir) 16 | 17 | def test_create_db(self, return_run_benchmark_obj): 18 | for basedir in return_run_benchmark_obj.basedir: 19 | assert return_run_benchmark_obj.create_db(db_name="test_run_benchmark_db", basedir=basedir) is True 20 | 21 | def test_run_sysbench_prepare(self, return_run_benchmark_obj): 22 | for basedir in return_run_benchmark_obj.basedir: 23 | assert return_run_benchmark_obj.run_sysbench_prepare(basedir=basedir) is True 24 | 25 | def test_run_sysbench_run(self, return_run_benchmark_obj): 26 | for basedir in return_run_benchmark_obj.basedir: 27 | assert return_run_benchmark_obj.run_sysbench_run(basedir=basedir) is True -------------------------------------------------------------------------------- /test/test_runner_test_mode.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from prepare_env_test_mode.run_benchmark import RunBenchmark 3 | 4 | class TestRunnerTestMode: 5 | 6 | # No matter here which fixture to choose I need an object 7 | @pytest.mark.usefixtures("return_runner_test_mode_obj_5_6_xb_2_3") 8 | def test_get_free_tcp_port(self, return_runner_test_mode_obj_5_6_xb_2_3): 9 | return_runner_test_mode_obj_5_6_xb_2_3.get_free_tcp_port() 10 | 11 | @pytest.mark.usefixtures("return_runner_test_mode_obj_5_6_xb_2_3") 12 | def test_create_dsns_table(self, return_runner_test_mode_obj_5_6_xb_2_3): 13 | for basedir in return_runner_test_mode_obj_5_6_xb_2_3.basedirs: 14 | if '5.6' in basedir: 15 | mysql_master_client_cmd = RunBenchmark(config=return_runner_test_mode_obj_5_6_xb_2_3.conf).get_mysql_conn( 16 | basedir=basedir) 17 | assert return_runner_test_mode_obj_5_6_xb_2_3.create_dsns_table(mysql_master_client_cmd) 18 | 19 | @pytest.mark.usefixtures("return_runner_test_mode_obj_5_6_xb_2_3") 20 | def test_populate_dsns_table(self, return_runner_test_mode_obj_5_6_xb_2_3): 21 | for basedir in return_runner_test_mode_obj_5_6_xb_2_3.basedirs: 22 | if '5.6' in basedir: 23 | mysql_master_client_cmd = RunBenchmark(config=return_runner_test_mode_obj_5_6_xb_2_3.conf).get_mysql_conn( 24 | basedir=basedir) 25 | file_name = "cl_node0" 26 | mysql_slave_client_cmd = RunBenchmark(config=return_runner_test_mode_obj_5_6_xb_2_3.conf).get_mysql_conn(basedir=basedir, 27 | file_name=file_name) 28 | # Get slave port here 29 | sql = "{} -e 'select @@port'".format(mysql_slave_client_cmd) 30 | port = return_runner_test_mode_obj_5_6_xb_2_3.run_sql_command(sql_command=sql) 31 | #slave_sock = "{}/sock0.sock".format(basedir) 32 | assert return_runner_test_mode_obj_5_6_xb_2_3.populate_dsns_table(sql_conn=mysql_master_client_cmd, slave_port=port[7:]) 33 | 34 | @pytest.mark.usefixtures("return_runner_test_mode_obj_5_6_xb_2_3") 35 | def test_run_pt_table_checksum(self, return_runner_test_mode_obj_5_6_xb_2_3): 36 | for basedir in return_runner_test_mode_obj_5_6_xb_2_3.basedirs: 37 | if '5.6' in basedir: 38 | # socket = return_run_benchmark_obj.get_sock(basedir=basedir) 39 | # conn_options = "--user={} --socket={}".format('root', socket) 40 | assert return_runner_test_mode_obj_5_6_xb_2_3.run_pt_table_checksum(basedir=basedir) 41 | 42 | @pytest.mark.usefixtures("return_runner_test_mode_obj_5_6_xb_2_3") 43 | def test_run_change_master(self, return_runner_test_mode_obj_5_6_xb_2_3): 44 | for basedir in return_runner_test_mode_obj_5_6_xb_2_3.basedirs: 45 | if '5.6' in basedir: 46 | assert return_runner_test_mode_obj_5_6_xb_2_3.run_change_master(basedir=basedir, file_name='cl_node0') 47 | 48 | @pytest.mark.usefixtures("return_runner_test_mode_obj_5_6_xb_2_3") 49 | def test_drop_blank_mysql_users(self, return_runner_test_mode_obj_5_6_xb_2_3): 50 | for basedir in return_runner_test_mode_obj_5_6_xb_2_3.basedirs: 51 | if '5.6' in basedir: 52 | mysql_master_client_cmd = RunBenchmark(config=return_runner_test_mode_obj_5_6_xb_2_3.conf).get_mysql_conn(basedir=basedir) 53 | assert return_runner_test_mode_obj_5_6_xb_2_3.drop_blank_mysql_users(mysql_master_client_cmd) 54 | 55 | @pytest.mark.usefixtures("return_runner_test_mode_obj_5_6_xb_2_3") 56 | def test_check_slave_status(self, return_runner_test_mode_obj_5_6_xb_2_3): 57 | for basedir in return_runner_test_mode_obj_5_6_xb_2_3.basedirs: 58 | if '5.6' in basedir: 59 | mysql_slave_client_cmd = RunBenchmark(config=return_runner_test_mode_obj_5_6_xb_2_3.conf).get_mysql_conn(basedir=basedir, file_name="cl_node0") 60 | show_slave_status = "{} -e 'show slave status\G'" 61 | assert return_runner_test_mode_obj_5_6_xb_2_3.check_slave_status(show_slave_status.format(mysql_slave_client_cmd)) is None 62 | 63 | 64 | @pytest.mark.usefixtures("return_runner_test_mode_obj_5_6_xb_2_3") 65 | def test_wipe_backup_prepare_copyback_5_6_xb_2_3(self, return_runner_test_mode_obj_5_6_xb_2_3): 66 | for basedir in return_runner_test_mode_obj_5_6_xb_2_3.basedirs: 67 | if '5.6' in basedir: 68 | return_runner_test_mode_obj_5_6_xb_2_3.wipe_backup_prepare_copyback(basedir=basedir) 69 | 70 | @pytest.mark.usefixtures("return_runner_test_mode_obj_5_6_xb_2_4") 71 | def test_wipe_backup_prepare_copyback_5_6_xb_2_4(self, return_runner_test_mode_obj_5_6_xb_2_4): 72 | for basedir in return_runner_test_mode_obj_5_6_xb_2_4.basedirs: 73 | if '5.6' in basedir: 74 | return_runner_test_mode_obj_5_6_xb_2_4.wipe_backup_prepare_copyback(basedir=basedir) 75 | 76 | @pytest.mark.usefixtures("return_runner_test_mode_obj_5_7_xb_2_4") 77 | def test_wipe_backup_prepare_copyback_5_7_xb_2_4(self, return_runner_test_mode_obj_5_7_xb_2_4): 78 | for basedir in return_runner_test_mode_obj_5_7_xb_2_4.basedirs: 79 | if '5.7' in basedir: 80 | return_runner_test_mode_obj_5_7_xb_2_4.wipe_backup_prepare_copyback(basedir=basedir) 81 | -------------------------------------------------------------------------------- /test/test_start_server.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # Starting PS servers with default values(executing start script inside basedirs). 4 | 5 | DIRNAME=$BATS_TEST_DIRNAME 6 | 7 | @test "Running test_start_server" { 8 | run python -m pytest -vv ${DIRNAME}/test_clone_build_start_server.py::TestCloneBuildStartServer::test_start_server 9 | echo $output 10 | [ $status -eq 0 ] 11 | } -------------------------------------------------------------------------------- /test/test_test_prepare_backup.py: -------------------------------------------------------------------------------- 1 | from general_conf.generalops import GeneralClass 2 | from prepare_env_test_mode.prepare_backup import WrapperForPrepareTest 3 | 4 | # Part of TEST MODE 5 | 6 | class TestPrepareBackup: 7 | # class for prepare_env_test_mode.prepare_backup.py 8 | def test_run_prepare_backup(self): 9 | gen_obj = GeneralClass() 10 | for conf_files in gen_obj.xb_configs.split(): 11 | obj = WrapperForPrepareTest(config='{}/{}'.format(gen_obj.testpath, conf_files)) 12 | obj.run_prepare_backup() 13 | 14 | def test_run_copy_back(self): 15 | gen_obj = GeneralClass() 16 | for conf_files in gen_obj.xb_configs.split(): 17 | obj = WrapperForPrepareTest(config='{}/{}'.format(gen_obj.testpath, conf_files)) 18 | obj.run_copy_back() 19 | 20 | -------------------------------------------------------------------------------- /test/test_test_take_backup.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from prepare_env_test_mode.take_backup import WrapperForBackupTest 3 | from general_conf.generalops import GeneralClass 4 | 5 | # Part of TEST MODE 6 | 7 | class TestTakeBackup: 8 | # Test class for take_backup.py 9 | 10 | def test_run_all_backup(self): 11 | gen_obj = GeneralClass() 12 | for conf_files in gen_obj.xb_configs.split(): 13 | obj = WrapperForBackupTest(config='{}/{}'.format(gen_obj.testpath, conf_files)) 14 | obj.run_all_backup() 15 | 16 | 17 | --------------------------------------------------------------------------------