├── docs ├── authors.rst ├── history.rst ├── readme.rst ├── contributing.rst ├── usage.rst ├── installation.rst ├── index.rst ├── Makefile ├── make.bat └── conf.py ├── tests ├── unit │ ├── __init__.py │ ├── test_email_helper.py │ ├── test_ssh_helper.py │ ├── test_failover_report.py │ ├── run-tests.sh │ ├── test_vip_metal_helper.py │ ├── test_mysql_helper.py │ ├── test_config_helper.py │ ├── test_master_ip_online_failover_helper.py │ └── test_master_ip_hard_failover_helper.py └── integration │ ├── test │ ├── integration │ │ ├── manager │ │ │ └── serverspec │ │ │ │ ├── spec_helper.rb │ │ │ │ └── default_spec.rb │ │ ├── environments │ │ │ ├── prod.json │ │ │ └── vagrant.json │ │ ├── nodes │ │ │ ├── standalone.json │ │ │ ├── slave02.json │ │ │ ├── master.json │ │ │ └── slave01.json │ │ ├── encrypted_data_bag_secret │ │ └── data_bags │ │ │ ├── mysql_mha_config │ │ │ └── test_pod.json │ │ │ └── mysql_mha_secrets │ │ │ └── test_pod.json │ └── fixtures │ │ └── cookbooks │ │ ├── base_test_setup │ │ ├── README.md │ │ ├── .gitignore │ │ ├── .kitchen.yml │ │ ├── metadata.rb │ │ ├── recipes │ │ │ ├── default.rb │ │ │ ├── dev_install.rb │ │ │ └── sysbench.rb │ │ └── chefignore │ │ └── mysql_replication │ │ ├── templates │ │ └── default │ │ │ ├── my.cnf.root.erb │ │ │ └── my.cnf.erb │ │ ├── recipes │ │ ├── client.rb │ │ ├── test_db.rb │ │ ├── mha_user.rb │ │ ├── replication_user.rb │ │ ├── slave.rb │ │ ├── _server.rb │ │ └── master.rb │ │ ├── metadata.rb │ │ └── attributes │ │ └── default.rb │ ├── .gitignore │ ├── Berksfile │ ├── Gemfile │ ├── Vagrantfile.erb │ ├── Rakefile │ └── .kitchen.yml ├── requirements.txt ├── setup.cfg ├── MANIFEST.in ├── AUTHORS.rst ├── .editorconfig ├── .travis.yml ├── .gitignore ├── HISTORY.rst ├── mha_helper ├── __init__.py ├── email_helper.py ├── vip_metal_helper.py ├── unix_daemon.py ├── ssh_helper.py ├── mysql_helper.py ├── config_helper.py └── mha_helper.py ├── scripts ├── mysql_online_failover ├── mysql_failover ├── master_failover_report ├── master_ip_hard_failover_helper └── master_ip_online_failover_helper ├── TESTING.rst ├── support-files ├── mha_manager_daemon-test_cluster-init └── mha_manager_daemon ├── setup.py ├── CONTRIBUTING.rst └── README.rst /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | wheel==0.23.0 2 | nose==1.3.4 3 | PyMySQL>=0.6.3 4 | paramiko>=1.10.0 5 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Usage 3 | ======== 4 | 5 | To use mha_helper in a project:: 6 | 7 | import mha_helper 8 | -------------------------------------------------------------------------------- /tests/integration/test/integration/manager/serverspec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'serverspec' 2 | 3 | set :backend, :exec 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | 4 | [flake8] 5 | ignore = E226,E301,E701 6 | exclude = tests,build 7 | max-line-length = 99 8 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/base_test_setup/README.md: -------------------------------------------------------------------------------- 1 | # base_test_setup 2 | 3 | TODO: Enter the cookbook description here. 4 | 5 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/templates/default/my.cnf.root.erb: -------------------------------------------------------------------------------- 1 | [client] 2 | user=root 3 | password='<%= @root_password %>' 4 | socket=<%= @mysql_socket %> 5 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst 2 | recursive-include scripts * 3 | recursive-include mha_helper *.py 4 | recursive-include docs *.rst 5 | recursive-include support-files * 6 | include setup.py 7 | include setup.cfg 8 | 9 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Ovais Tariq 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? 14 | -------------------------------------------------------------------------------- /tests/integration/.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | Berksfile.lock 3 | *~ 4 | *# 5 | .#* 6 | \#*# 7 | .*.sw[a-z] 8 | *.un~ 9 | 10 | # Bundler 11 | Gemfile.lock 12 | bin/* 13 | .bundle/* 14 | 15 | .kitchen/ 16 | .kitchen.local.yml 17 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/base_test_setup/.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | Berksfile.lock 3 | *~ 4 | *# 5 | .#* 6 | \#*# 7 | .*.sw[a-z] 8 | *.un~ 9 | 10 | # Bundler 11 | Gemfile.lock 12 | bin/* 13 | .bundle/* 14 | 15 | .kitchen/ 16 | .kitchen.local.yml 17 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | At the command line:: 6 | 7 | $ easy_install mha_helper 8 | 9 | Or, if you have virtualenvwrapper installed:: 10 | 11 | $ mkvirtualenv mha_helper 12 | $ pip install mha_helper 13 | -------------------------------------------------------------------------------- /tests/integration/test/integration/environments/prod.json: -------------------------------------------------------------------------------- 1 | { 2 | "override_attributes": { 3 | }, 4 | "cookbook_versions": { 5 | }, 6 | "description": "", 7 | "name": "prod", 8 | "json_class": "Chef::Environment", 9 | "default_attributes": { 10 | }, 11 | "chef_type": "environment" 12 | } 13 | -------------------------------------------------------------------------------- /tests/integration/test/integration/environments/vagrant.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vagrant", 3 | "default_attributes": { 4 | }, 5 | "json_class": "Chef::Environment", 6 | "description": "", 7 | "cookbook_versions": { 8 | }, 9 | "override_attributes": { 10 | }, 11 | "chef_type": "environment" 12 | } 13 | -------------------------------------------------------------------------------- /tests/integration/test/integration/nodes/standalone.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "standalone", 3 | "chef_environment": "prod", 4 | "normal": { 5 | "tags": [ 6 | ] 7 | }, 8 | "automatic": { 9 | "ipaddress": "192.168.30.10", 10 | "hostname": "standalone", 11 | "fqdn": "standalone.localhost" 12 | }, 13 | "run_list": [] 14 | } 15 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/base_test_setup/.kitchen.yml: -------------------------------------------------------------------------------- 1 | --- 2 | driver: 3 | name: vagrant 4 | 5 | provisioner: 6 | name: chef_zero 7 | 8 | platforms: 9 | - name: ubuntu-12.04 10 | - name: centos-6.5 11 | 12 | suites: 13 | - name: default 14 | run_list: 15 | - recipe[base_test_setup::default] 16 | attributes: 17 | -------------------------------------------------------------------------------- /tests/integration/Berksfile: -------------------------------------------------------------------------------- 1 | source "https://supermarket.chef.io" 2 | 3 | cookbook "mysql-mha", git: "git://github.com/ovaistariq/cookbook-mysql-mha.git" 4 | 5 | # Testing cookbook used by test-kitchen 6 | cookbook "base_test_setup", path: "test/fixtures/cookbooks/base_test_setup" 7 | cookbook "mysql_replication", path: "test/fixtures/cookbooks/mysql_replication" 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Config file for automatic testing at travis-ci.org 2 | 3 | language: python 4 | 5 | python: 6 | - "3.4" 7 | - "3.3" 8 | - "2.7" 9 | - "2.6" 10 | - "pypy" 11 | 12 | # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors 13 | install: pip install -r requirements.txt 14 | 15 | # command to run tests, e.g. python setup.py test 16 | script: python setup.py test 17 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/base_test_setup/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'base_test_setup' 2 | maintainer 'Ovais Tariq' 3 | maintainer_email 'me@ovaistariq.net' 4 | license 'Apache 2.0' 5 | description 'Installs/Configures base_test_setup' 6 | long_description 'Installs/Configures base_test_setup' 7 | version '0.1.0' 8 | 9 | depends 'build-essential', '~> 2.2.4' 10 | depends 'hostsfile' 11 | depends 'mysql-mha' 12 | depends 'python' 13 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. mha_helper documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to mha_helper's documentation! 7 | ====================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | readme 15 | installation 16 | usage 17 | contributing 18 | authors 19 | history 20 | 21 | Indices and tables 22 | ================== 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | 28 | -------------------------------------------------------------------------------- /tests/integration/test/integration/nodes/slave02.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "slave02", 3 | "chef_environment": "prod", 4 | "normal": { 5 | "tags": [ 6 | "mha_managed", 7 | "pod:test_pod" 8 | ], 9 | "mysql_mha": { 10 | "pod_name": "test_pod", 11 | "node": { 12 | "no_master": "1", 13 | "mysql_port": "3307", 14 | "mysql_binlog_dir": "/var/log/mysql-default", 15 | "ssh_port": "22" 16 | } 17 | } 18 | }, 19 | "automatic": { 20 | "ipaddress": "192.168.30.13", 21 | "hostname": "slave02", 22 | "fqdn": "slave02.localhost" 23 | }, 24 | "run_list": [] 25 | } 26 | -------------------------------------------------------------------------------- /tests/integration/test/integration/nodes/master.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "master", 3 | "chef_environment": "prod", 4 | "normal": { 5 | "tags": [ 6 | "mha_managed", 7 | "pod:test_pod" 8 | ], 9 | "mysql_mha": { 10 | "pod_name": "test_pod", 11 | "node": { 12 | "candidate_master": "1", 13 | "check_repl_delay": "0", 14 | "mysql_port": "3306", 15 | "mysql_binlog_dir": "/var/lib/mysql", 16 | "ssh_port": "22" 17 | } 18 | } 19 | }, 20 | "automatic": { 21 | "ipaddress": "192.168.30.11", 22 | "hostname": "master", 23 | "fqdn": "master.localhost" 24 | }, 25 | "run_list": [] 26 | } 27 | -------------------------------------------------------------------------------- /tests/integration/test/integration/nodes/slave01.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "slave01", 3 | "chef_environment": "prod", 4 | "normal": { 5 | "tags": [ 6 | "mha_managed", 7 | "pod:test_pod" 8 | ], 9 | "mysql_mha": { 10 | "pod_name": "test_pod", 11 | "node": { 12 | "candidate_master": "1", 13 | "check_repl_delay": "0", 14 | "mysql_port": "3306", 15 | "mysql_binlog_dir": "/var/log/mysql-default", 16 | "ssh_port": "22" 17 | } 18 | } 19 | }, 20 | "automatic": { 21 | "ipaddress": "192.168.30.12", 22 | "hostname": "slave01", 23 | "fqdn": "slave01.localhost" 24 | }, 25 | "run_list": [] 26 | } 27 | -------------------------------------------------------------------------------- /tests/integration/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | group :lint do 4 | gem 'foodcritic', '~> 5.0' 5 | gem 'rubocop', '~> 0.18' 6 | gem 'rainbow', '< 2.0' 7 | gem 'rake' 8 | end 9 | 10 | group :unit do 11 | gem 'berkshelf', '~> 3.2' 12 | gem 'chefspec', '~> 4.2' 13 | end 14 | 15 | group :kitchen_common do 16 | gem 'test-kitchen', '~> 1.4' 17 | end 18 | 19 | group :kitchen_vagrant do 20 | gem 'kitchen-vagrant', '~> 0.16' 21 | end 22 | 23 | group :development do 24 | gem 'ruby_gntp' 25 | gem 'growl' 26 | gem 'rb-fsevent' 27 | gem 'guard', '~> 2.4' 28 | gem 'guard-kitchen' 29 | gem 'guard-foodcritic' 30 | gem 'guard-rspec' 31 | gem 'guard-rubocop' 32 | end 33 | -------------------------------------------------------------------------------- /tests/integration/test/integration/encrypted_data_bag_secret: -------------------------------------------------------------------------------- 1 | Xftudi6R3nBZPNMWt5dUR/lok7E69Ig4t6F2zGXvLT63W+7WjeprvLOuSq1S9Zjq4vgcdfrDky/S7YrzcdXKhdqeoPgYjdVwWOTYXt0ZybohfIdvUqOyB0xtI4BY8N7iFGgWjSqS9cLO/QvQcZ6M1sfPNogAXBLZs3HGdvnGn8yZ1j6h+3tIv6A6nnH1vULELnV2ZnsDhGFc9YmoKgXob7Zv4tnhCNfABL06al+uRTClZTWMhQWUtMYsnAxQyI5/rJGAiaeY+YaTabRZCzUT9WktssR1BKLJX0E3uOkEhYzYcoZQAhA6fIRtcPihgAklUeuVxEG/6XgtbN2bO7sfQl6RO0Em179RMX5jBmbIA30aWeRc8qv90Y04IhR2/tQjdOtdbJpWL6kBvtM9FJD1FZiTYlEof/V/IU9e6fhzOJCWpHAZYjaBU4pkDzPjcv9Mw8t3tnDrj0coE4M5S1d+hfoytMx/IBGcZ4RYsCB8U1t8xqlxlu7yoPZUQFF75XNTITxRKprTO2ja+m+zRRgwvr6PXgl635ejiydaPA9uW9lYWg/NgcTFjTgTvrGZIYcAyZVs7hrWWXzDP59vF8sdVFUVyUlbcqhOFT3/4HhRNPwDioHW88X3/AwOlZxNm4tRXcDs0HxpsnaS0OT2zihWdqK2vUFk7FpVm+RmTXkZuRI= -------------------------------------------------------------------------------- /tests/integration/test/integration/manager/serverspec/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | set :backend, :exec 4 | 5 | puts "os: #{os}" 6 | 7 | describe command('whoami') do 8 | its(:stdout) { should match /root/ } 9 | end 10 | 11 | describe 'mha4mysql-node package is installed' do 12 | describe package('mha4mysql-node') do 13 | it { should be_installed } 14 | end 15 | end 16 | 17 | describe 'mha4mysql-manager package is installed' do 18 | describe package('mha4mysql-manager') do 19 | it { should be_installed } 20 | end 21 | end 22 | 23 | describe 'MySQL replication is setup correctly and MHA can reach MySQL and SSH' do 24 | describe command("/usr/bin/masterha_check_repl --conf /etc/mha/test_pod.conf") do 25 | its(:stdout) { should match /MySQL Replication Health is OK./ } 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/recipes/client.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: mysql_replication 3 | # Recipe:: client 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | mysql_client 'default' do 21 | action :create 22 | end 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | bin 12 | eggs 13 | parts 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | htmlcov 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | .idea 38 | 39 | # Complexity 40 | output/*.html 41 | output/*/index.html 42 | 43 | # Sphinx 44 | docs/_build 45 | 46 | # Key files 47 | id_rsa 48 | id_rsa.pub 49 | 50 | # Configuration files 51 | *.conf 52 | 53 | # Logs and databases 54 | *.log 55 | *.sql 56 | *.sqlite 57 | 58 | # OS generated files 59 | 60 | .DS_Store 61 | .DS_Store? 62 | ._* 63 | .Spotlight-V100 64 | .Trashes 65 | Icon? 66 | ehthumbs.db 67 | Thumbs.db 68 | *.swp 69 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/base_test_setup/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: base_test_setup 3 | # Recipe:: default 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | node['hosts_list'].each do |host| 21 | hostsfile_entry host['ip'] do 22 | hostname host['hostname'] 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | History 4 | ------- 5 | 6 | 0.4.0 ("2015-11-01") 7 | -------------------- 8 | 9 | * Completely redesigned MHA Helper. 10 | * MHA Helper is now a Python module. 11 | * MHA Helper now installs scripts inside /usr/bin/ 12 | * Paramiko is now used for SSH based communication. 13 | * User's ssh-config file is now parsed to read in additional SSH options if set. 14 | * PyMySQL is now used instead of MySQLdb as the Python MySQL driver. 15 | * Configuration files are now located in /etc/mha-helper with one file per MySQL replication cluster. 16 | * MHA Helper now kills the sleeping threads from the old master after failover. 17 | * MHA Helper writer Virtual IP support is now pluggable and can support more than one implementations. Currently only traditional Virtual IP support has been implemented which involves moving an IP from one machine to the other and broadcasting ARP packets. 18 | 19 | 0.3.0 ("2013-10-26") 20 | -------------------- 21 | 22 | * Initial release. 23 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/templates/default/my.cnf.erb: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | 3 | # BINARY LOGGING # 4 | log-bin = <%= node["mysql"]["server"]["log_bin"] %> 5 | sync-binlog = <%= node["mysql"]["server"]["sync_binlog"] %> 6 | 7 | # REPLICATION # 8 | server-id = <%= node["mysql"]["server"]["server_id"] %> 9 | read-only = <%= node["mysql"]["server"]["read_only"] %> 10 | log-slave-updates = <%= node["mysql"]["server"]["log_slave_updates"] %> 11 | relay-log = <%= node["mysql"]["server"]["relay_log"] %> 12 | slave-net-timeout = <%= node["mysql"]["server"]["slave_net_timeout"] %> 13 | report-host = <%= node["hostname"] %> 14 | 15 | # INNODB # 16 | innodb-buffer-pool-size = <%= node["mysql"]["server"]["innodb_buffer_pool_size"] %> 17 | 18 | # LOGGING # 19 | log-warnings = <%= node["mysql"]["server"]["log_warnings"] %> -------------------------------------------------------------------------------- /mha_helper/__init__.py: -------------------------------------------------------------------------------- 1 | # (c) 2013, Ovais Tariq 2 | # 3 | # This file is part of mha_helper 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | # -*- coding: utf-8 -*- 19 | 20 | __name__ = 'mha_helper' 21 | __author__ = 'Ovais Tariq' 22 | __email__ = 'me@ovaistariq.net' 23 | __version__ = '0.4.2' 24 | __url__ = 'https://github.com/ovaistariq/mha-helper' 25 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'mysql_replication' 2 | maintainer 'Ovais Tariq' 3 | maintainer_email 'me@ovaistariq.net' 4 | license 'All rights reserved' 5 | description 'Installs/Configures mysql_replication' 6 | long_description 'Installs/Configures mysql_replication' 7 | version '0.1.0' 8 | 9 | recipe 'mysql_replication::master', 'Installs and configures a MySQL server as a replication master' 10 | recipe 'mysql_replication::slave', 'Installs and configures a MySQL server as a replication slave' 11 | recipe 'mysql_replication::replication_user', 'Configures replication users that are used by slaves to connect to master' 12 | recipe 'mysql_replication::_server', 'Installs and configures the MySQL server, should not be added to the runlist directly' 13 | recipe 'mysql_replication::client', 'Installs and configures the MySQL client' 14 | 15 | depends 'mysql', '~> 6.1.2' 16 | depends 'database' 17 | depends 'mysql2_chef_gem' 18 | 19 | supports 'centos' 20 | supports 'redhat' 21 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/base_test_setup/recipes/dev_install.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: base_test_setup 3 | # Recipe:: dev_install 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe 'python::pip' 21 | 22 | # Remove the mha-helper package installed via yum 23 | package node["mysql_mha"]["helper"]["package"] do 24 | action :remove 25 | end 26 | 27 | python_pip '/tmp/mha-helper' do 28 | version 'latest' 29 | options '-e' 30 | end 31 | 32 | include_recipe 'base_test_setup::sysbench' 33 | -------------------------------------------------------------------------------- /tests/integration/test/integration/data_bags/mysql_mha_config/test_pod.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "test_pod", 3 | "comment": "MHA replication pod 'test_pod' configuration", 4 | "mysql": { 5 | "user": "mha", 6 | "repl_user": "repl" 7 | }, 8 | "remote_user": { 9 | "id": "mha", 10 | "comment": "user used by MHA to login to managed nodes", 11 | "home": "/home/mha", 12 | "uid": "10011", 13 | "groups": ["mysql"], 14 | "ssh_keys": [ 15 | "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAwHjA7iaA0/+TD0roAmyOqYFp3DMpuJ/Xee260gio5igLeV6DHyPBskHhcCOWFcZ+uCAOGIm+Yye9nxuspWFEWCa4L1SxniBEpRGtGyorbgz7Zmh/6VlcZOUBTe1GtaprokGAtzP2gaQbRpJ1c0oX3JZ4lVH6Oro3keCGyncZMFJ2nTu0hOgbJPA3XZkRwO0DhB/8IbPu6NwXVcDjaMfTmpj4kp722RFuEUgGrDBwx/vasakpcBMHF+a6QL0gAODHMttQB1kk1hCV4fQtiTSrbG97jldlcC7VvSqK79twpQNe9y06jnMah8xdvZ69mw/4k3Av+Vv3I4KHxvN9wE59Tw== root@manager-centos-66" 16 | ] 17 | }, 18 | "vip_type": "metal", 19 | "writer_vip_cidr": "192.168.30.100/24", 20 | "report_email": "ovaistariq@gmail.com", 21 | "smtp_host": "localhost", 22 | "requires_sudo": "yes", 23 | "requires_arping": "yes", 24 | "super_read_only": "no", 25 | "cluster_interface": "eth1", 26 | "kill_after_timeout": 5 27 | } 28 | -------------------------------------------------------------------------------- /tests/unit/test_email_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import os 6 | from mha_helper.config_helper import ConfigHelper 7 | from mha_helper.email_helper import EmailHelper 8 | 9 | __author__ = 'ovais.tariq' 10 | __project__ = 'mha_helper' 11 | 12 | 13 | class TestEmailHelper(unittest.TestCase): 14 | def setUp(self): 15 | self.root_directory = os.path.dirname(os.path.realpath(__file__)) 16 | 17 | # Test with the correct config 18 | mha_helper_config_dir = os.path.join(self.root_directory, 'conf', 'good') 19 | if not mha_helper_config_dir: 20 | self.fail(msg='mha_helper configuration dir not set') 21 | 22 | ConfigHelper.MHA_HELPER_CONFIG_DIR = mha_helper_config_dir 23 | if not ConfigHelper.load_config(): 24 | self.fail(msg='Could not load mha_helper configuration from %s' % mha_helper_config_dir) 25 | 26 | def test_send_email(self): 27 | subject = "Testing Emails via %s" % self.__class__.__name__ 28 | message = "Test message sent through %s" % self.__class__.__name__ 29 | 30 | email_sender = EmailHelper('master') 31 | self.assertTrue(email_sender.send_email(subject, message)) 32 | 33 | if __name__ == '__main__': 34 | unittest.main() 35 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/recipes/test_db.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: mysql_replication 3 | # Recipe:: test_db 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Setup credentials that are used by the database cook LWRPs 21 | connection_info = { 22 | host: "127.0.0.1", 23 | username: "root", 24 | password: node["mysql"]["root_password"], 25 | port: node["mysql"]["server"]["port"] 26 | } 27 | 28 | # This is a prerequisite for the database cookbook 29 | mysql2_chef_gem "default" do 30 | client_version node["mysql"]["version"] 31 | action :install 32 | end 33 | 34 | mysql_database 'test' do 35 | connection connection_info 36 | action :create 37 | end 38 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/base_test_setup/chefignore: -------------------------------------------------------------------------------- 1 | # Put files/directories that should be ignored in this file when uploading 2 | # or sharing to the community site. 3 | # Lines that start with '# ' are comments. 4 | 5 | # OS generated files # 6 | ###################### 7 | .DS_Store 8 | Icon? 9 | nohup.out 10 | ehthumbs.db 11 | Thumbs.db 12 | 13 | # SASS # 14 | ######## 15 | .sass-cache 16 | 17 | # EDITORS # 18 | ########### 19 | \#* 20 | .#* 21 | *~ 22 | *.sw[a-z] 23 | *.bak 24 | REVISION 25 | TAGS* 26 | tmtags 27 | *_flymake.* 28 | *_flymake 29 | *.tmproj 30 | .project 31 | .settings 32 | mkmf.log 33 | 34 | ## COMPILED ## 35 | ############## 36 | a.out 37 | *.o 38 | *.pyc 39 | *.so 40 | *.com 41 | *.class 42 | *.dll 43 | *.exe 44 | */rdoc/ 45 | 46 | # Testing # 47 | ########### 48 | .watchr 49 | .rspec 50 | spec/* 51 | spec/fixtures/* 52 | test/* 53 | features/* 54 | Guardfile 55 | Procfile 56 | 57 | # SCM # 58 | ####### 59 | .git 60 | */.git 61 | .gitignore 62 | .gitmodules 63 | .gitconfig 64 | .gitattributes 65 | .svn 66 | */.bzr/* 67 | */.hg/* 68 | */.svn/* 69 | 70 | # Berkshelf # 71 | ############# 72 | Berksfile 73 | Berksfile.lock 74 | cookbooks/* 75 | tmp 76 | 77 | # Cookbooks # 78 | ############# 79 | CONTRIBUTING 80 | 81 | # Strainer # 82 | ############ 83 | Colanderfile 84 | Strainerfile 85 | .colander 86 | .strainer 87 | 88 | # Vagrant # 89 | ########### 90 | .vagrant 91 | Vagrantfile 92 | 93 | # Travis # 94 | ########## 95 | .travis.yml 96 | -------------------------------------------------------------------------------- /scripts/mysql_online_failover: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # (c) 2013, Ovais Tariq 4 | # 5 | # This file is part of mha_helper 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | 20 | conf="" 21 | 22 | OPTIND=1 23 | while getopts "h?c:" opt; do 24 | case "$opt" in 25 | h|\?) 26 | #show_help 27 | exit 0 28 | ;; 29 | c) conf=$OPTARG 30 | ;; 31 | esac 32 | done 33 | shift $((OPTIND-1)) # Shift off the options and optional --. 34 | 35 | if [[ -z "$conf" ]] 36 | then 37 | echo "ERROR: option '-c CONF' not given. See -h" >&2 38 | exit 1 39 | fi 40 | 41 | if [[ ! -e "$conf" ]] 42 | then 43 | echo "ERROR: $conf does not exist" 44 | exit 1 45 | fi 46 | 47 | 48 | /usr/bin/masterha_master_switch \ 49 | --conf=${conf} \ 50 | --master_state=alive \ 51 | --orig_master_is_new_slave \ 52 | --interactive=0 53 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/recipes/mha_user.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: mysql_replication 3 | # Recipe:: mha_user 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Setup credentials that are used by the database cook LWRPs 21 | connection_info = { 22 | host: "127.0.0.1", 23 | username: "root", 24 | password: node["mysql"]["root_password"], 25 | port: node["mysql"]["server"]["port"] 26 | } 27 | 28 | # This is a prerequisite for the database cookbook 29 | mysql2_chef_gem "default" do 30 | client_version node["mysql"]["version"] 31 | action :install 32 | end 33 | 34 | # Setup the grants for the slave users 35 | # But create the users only if we are running on the master 36 | mysql_database_user node["mysql"]["mha"]["user"] do 37 | connection connection_info 38 | password node["mysql"]["mha"]["password"] 39 | host '%' 40 | privileges [:all] 41 | action :grant 42 | end 43 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/base_test_setup/recipes/sysbench.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: base_test_setup 3 | # Recipe:: sysbench 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe 'build-essential::default' 21 | 22 | %w(automake libtool libaio libaio-devel git rubygems ruby-devel rpm-build python-pip).each do |pkg| 23 | package pkg do 24 | action :install 25 | end 26 | end 27 | 28 | git "#{Chef::Config[:file_cache_path]}/sysbench" do 29 | repository 'https://github.com/akopytov/sysbench.git' 30 | revision '0.5' 31 | action :sync 32 | notifies :run, 'bash[sysbench :install from source]', :immediately 33 | not_if 'which sysbench' 34 | end 35 | 36 | bash 'sysbench :install from source' do 37 | user 'root' 38 | cwd "#{Chef::Config[:file_cache_path]}/sysbench" 39 | code <<-EOH 40 | ./autogen.sh 41 | ./configure 42 | make 43 | make install 44 | EOH 45 | action :nothing 46 | end 47 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/recipes/replication_user.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: mysql_replication 3 | # Recipe:: replication_user 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Setup credentials that are used by the database cook LWRPs 21 | connection_info = { 22 | host: "127.0.0.1", 23 | username: "root", 24 | password: node["mysql"]["root_password"], 25 | port: node["mysql"]["server"]["port"] 26 | } 27 | 28 | # This is a prerequisite for the database cookbook 29 | mysql2_chef_gem "default" do 30 | client_version node["mysql"]["version"] 31 | action :install 32 | end 33 | 34 | # Setup the grants for the slave users 35 | # But create the users only if we are running on the master 36 | mysql_database_user node["mysql"]["replication"]["user"] do 37 | connection connection_info 38 | password node["mysql"]["replication"]["password"] 39 | host '%' 40 | privileges [:"replication slave", :"replication client"] 41 | action :grant 42 | end 43 | -------------------------------------------------------------------------------- /tests/unit/test_ssh_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import os 6 | from mha_helper.ssh_helper import SSHHelper 7 | 8 | __author__ = 'ovais.tariq' 9 | __project__ = 'mha_helper' 10 | 11 | 12 | class TestSSHHelper(unittest.TestCase): 13 | def setUp(self): 14 | self.ssh_host = os.getenv('SSH_TEST_HOST') 15 | self.ssh_host_ip = os.getenv('SSH_TEST_IP') 16 | self.ssh_user = os.getenv('SSH_TEST_USER') 17 | self.ssh_port = os.getenv('SSH_TEST_PORT') 18 | 19 | if not self.ssh_host or not self.ssh_host_ip or not self.ssh_user or not self.ssh_port: 20 | self.fail(msg='SSH connection information not set') 21 | 22 | self.ssh_client = SSHHelper(host=self.ssh_host, host_ip=self.ssh_host_ip, ssh_user=self.ssh_user, 23 | ssh_port=self.ssh_port, ssh_options=None) 24 | 25 | def test_make_ssh_connection(self): 26 | self.assertTrue(self.ssh_client.make_ssh_connection()) 27 | 28 | def test_execute_ssh_command(self): 29 | # Setup the SSH connection 30 | self.ssh_client.make_ssh_connection() 31 | 32 | # Execute a known good command 33 | cmd = "/sbin/ip addr show" 34 | cmd_exec_status, cmd_exec_output = self.ssh_client.execute_ssh_command(cmd) 35 | self.assertTrue(cmd_exec_status) 36 | self.assertTrue(len(cmd_exec_output) > 0) 37 | 38 | # Execute a known misspelled command 39 | err_cmd = "/sbin/ifconfig -c" 40 | cmd_exec_status, cmd_exec_output = self.ssh_client.execute_ssh_command(err_cmd) 41 | self.assertFalse(cmd_exec_status) 42 | 43 | if __name__ == '__main__': 44 | unittest.main() 45 | -------------------------------------------------------------------------------- /scripts/mysql_failover: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # (c) 2013, Ovais Tariq 4 | # 5 | # This file is part of mha_helper 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | 20 | dead_master_host="" 21 | conf="" 22 | 23 | OPTIND=1 24 | while getopts "h?c:d:" opt; do 25 | case "$opt" in 26 | h|\?) 27 | #show_help 28 | exit 0 29 | ;; 30 | c) conf=$OPTARG 31 | ;; 32 | d) dead_master_host=$OPTARG 33 | ;; 34 | esac 35 | done 36 | shift $((OPTIND-1)) # Shift off the options and optional --. 37 | 38 | if [[ -z ${dead_master_host} ]] 39 | then 40 | echo "ERROR: option '-d HOST' (hostname of dead master) not given. See -h" >&2 41 | exit 1 42 | fi 43 | 44 | if [[ -z "$conf" ]] 45 | then 46 | echo "ERROR: option '-c CONF' not given. See --help" >&2 47 | exit 1 48 | fi 49 | 50 | if [[ ! -e "$conf" ]] 51 | then 52 | echo "ERROR: $conf does not exist" 53 | exit 1 54 | fi 55 | 56 | /usr/bin/masterha_master_switch \ 57 | --conf=${conf} \ 58 | --master_state=dead \ 59 | --interactive=0 \ 60 | --ignore_last_failover \ 61 | --dead_master_host=${dead_master_host} 62 | -------------------------------------------------------------------------------- /tests/integration/Vagrantfile.erb: -------------------------------------------------------------------------------- 1 | Vagrant.configure("2") do |c| 2 | c.vm.box = "<%= config[:box] %>" 3 | c.vm.box_url = "<%= config[:box_url] %>" 4 | 5 | if Vagrant.has_plugin?("vagrant-cachier") 6 | c.cache.auto_detect = true 7 | c.cache.scope = :box 8 | end 9 | 10 | if Vagrant.has_plugin?("vagrant-omnibus") 11 | c.omnibus.cache_packages = true 12 | end 13 | 14 | if Vagrant.has_plugin?("vagrant-vbguest") 15 | c.vbguest.auto_update = false 16 | end 17 | 18 | <% if config[:vm_hostname] %> 19 | c.vm.hostname = "<%= config[:vm_hostname] %>" 20 | <% end %> 21 | <% if config[:guest] %> 22 | c.vm.guest = <%= config[:guest] %> 23 | <% end %> 24 | <% if config[:username] %> 25 | c.ssh.username = "<%= config[:username] %>" 26 | <% end %> 27 | <% if config[:ssh_key] %> 28 | c.ssh.private_key_path = "<%= config[:ssh_key] %>" 29 | <% end %> 30 | 31 | <% Array(config[:network]).each do |opts| %> 32 | c.vm.network(:<%= opts[0] %>, <%= opts[1..-1].join(", ") %>) 33 | <% end %> 34 | 35 | c.vm.synced_folder ".", "/vagrant", disabled: true 36 | <% config[:synced_folders].each do |source, destination, options| %> 37 | c.vm.synced_folder "<%= source %>", "<%= destination %>", <%= options %> 38 | <% end %> 39 | 40 | c.vm.provider :<%= config[:provider] %> do |p| 41 | <% config[:customize].each do |key, value| %> 42 | <% case config[:provider] 43 | when "virtualbox" %> 44 | p.customize ["modifyvm", :id, "--<%= key %>", "<%= value %>"] 45 | <% when "rackspace", "softlayer" %> 46 | p.<%= key %> = "<%= value%>" 47 | <% when /^vmware_/ %> 48 | <% if key == :memory %> 49 | <% unless config[:customize].include?(:memsize) %> 50 | p.vmx["memsize"] = "<%= value %>" 51 | <% end %> 52 | <% else %> 53 | p.vmx["<%= key %>"] = "<%= value %>" 54 | <% end %> 55 | <% end %> 56 | <% end %> 57 | end 58 | 59 | end 60 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/recipes/slave.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: mysql_replication 3 | # Recipe:: slave 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Set attributes that need to be set differently for the master 21 | node.default["mysql"]["server"]["read_only"] = 1 22 | 23 | # Include the base recipe that sets up and configures the MySQL server 24 | include_recipe "mysql_replication::_server" 25 | 26 | # During the first converge of slave we setup replication 27 | master_host = node['mysql_replication']['master_host'] 28 | repl_user = node["mysql"]["replication"]["user"] 29 | repl_pass = node["mysql"]["replication"]["password"] 30 | log_bin_file = "#{node['mysql']['server']['log_bin_filename']}.000002" 31 | 32 | bash 'slave replication :change_master during first run' do 33 | user 'root' 34 | code <<-EOH 35 | /usr/bin/mysql -e 'CHANGE MASTER TO MASTER_HOST="#{master_host}", MASTER_USER="#{repl_user}", MASTER_PASSWORD="#{repl_pass}", MASTER_LOG_FILE="#{log_bin_file}", MASTER_LOG_POS=0; START SLAVE;' 36 | EOH 37 | notifies :create, 'file[/tmp/slave_replication_done]', :immediately 38 | not_if { File.exist?('/tmp/slave_replication_done') } 39 | end 40 | 41 | file '/tmp/slave_replication_done' do 42 | owner 'root' 43 | group 'root' 44 | mode 0755 45 | action :nothing 46 | end 47 | -------------------------------------------------------------------------------- /mha_helper/email_helper.py: -------------------------------------------------------------------------------- 1 | # (c) 2015, Ovais Tariq 2 | # 3 | # This file is part of mha_helper 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | from __future__ import print_function 19 | import socket 20 | import smtplib 21 | from email.mime.text import MIMEText 22 | from config_helper import ConfigHelper 23 | 24 | 25 | class EmailHelper(object): 26 | def __init__(self, host): 27 | config_helper = ConfigHelper(host) 28 | self._smtp_host = config_helper.get_smtp_host() 29 | self._sender = "mha_helper@%s" % socket.getfqdn() 30 | self._receiver = config_helper.get_report_email() 31 | 32 | def send_email(self, subject, msg): 33 | try: 34 | smtp = smtplib.SMTP(self._smtp_host) 35 | 36 | email_msg = MIMEText(msg) 37 | email_msg['Subject'] = subject 38 | email_msg['From'] = self._sender 39 | email_msg['To'] = self._receiver 40 | 41 | print("Sending email to %s via %s with the subject '%s'" % (self._receiver, self._smtp_host, subject)) 42 | smtp.sendmail(self._sender, self._receiver, email_msg.as_string()) 43 | smtp.quit() 44 | except Exception as e: 45 | print("Failed to send email From: %s, To: %s" % (self._sender, self._receiver)) 46 | print(str(e)) 47 | return False 48 | 49 | return True 50 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/recipes/_server.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: mysql_replication 3 | # Recipe:: server 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Setup the MySQL service 21 | mysql_service "default" do 22 | port node["mysql"]["server"]["port"] 23 | data_dir node["mysql"]["server"]["datadir"] 24 | version node["mysql"]["version"] 25 | initial_root_password node["mysql"]["root_password"] 26 | socket node["mysql"]["server"]["socket"] 27 | action [:create, :start] 28 | end 29 | 30 | # Setup the main MySQL configuration 31 | mysql_config "default" do 32 | source "my.cnf.erb" 33 | action :create 34 | notifies :restart, "mysql_service[default]", :immediately 35 | end 36 | 37 | # Setup the .my.cnf file for the root user 38 | template "/root/.my.cnf" do 39 | variables( 40 | :root_password => node["mysql"]["root_password"], 41 | :mysql_socket => node["mysql"]["server"]["socket"] 42 | ) 43 | source "my.cnf.root.erb" 44 | owner "root" 45 | group "root" 46 | mode "0600" 47 | action :create 48 | end 49 | 50 | # Include the recipe that sets up the replication user 51 | include_recipe "mysql_replication::replication_user" 52 | 53 | # Include the recipe that sets up the MHA user 54 | include_recipe "mysql_replication::mha_user" 55 | 56 | # Include the recipe that creates the test database used during testing 57 | include_recipe "mysql_replication::test_db" 58 | -------------------------------------------------------------------------------- /TESTING.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | MHA Helper Integration Test Suite 3 | ================================= 4 | 5 | Bundler 6 | ------- 7 | A ruby environment with Bundler installed is a prerequisite for using 8 | this testing harness. At the time of this writing, it works with 9 | Ruby >= 1.9.3 and Bundler >= 1.5.3. All programs involved, with the 10 | exception of Vagrant, can be installed by cd'ing into the "tests/integration" 11 | directory of this repository and running "bundle install" as shown 12 | below. 13 | 14 | :: 15 | 16 | $ sudo gem install bundler 17 | $ cd tests/integration 18 | $ bundle install --binstubs --path .bundle 19 | 20 | 21 | Rakefile 22 | -------- 23 | The Rakefile ships with a number of tasks, each of which can be ran 24 | individually, or in groups. Typing "rake" by itself will perform 25 | integration with Test Kitchen using the Vagrant driver by 26 | default. Alternatively, integration tests can be ran with Test Kitchen 27 | EC2 driver. 28 | 29 | :: 30 | 31 | $ ./bin/rake -T 32 | rake cleanup # Clean up generated files 33 | rake cleanup:kitchen_destroy # Destroy test-kitchen instances 34 | rake integration # Run full integration 35 | rake integration:vagrant_setup # Setup the test-kitchen vagrant instances 36 | rake integration:vagrant_verify # Verify the test-kitchen vagrant instances 37 | rake setup # Generate the setup 38 | 39 | Integration Testing 40 | ------------------- 41 | Integration testing is performed by Test Kitchen. Test Kitchen will 42 | use the Vagrant driver to instantiate machines and apply Chef cookbooks. 43 | After a successful converge, tests are uploaded and ran out of band of Chef. 44 | 45 | Integration Testing using Vagrant 46 | --------------------------------- 47 | Integration tests can be performed on a local workstation using 48 | Virtualbox or VMWare. Detailed instructions for setting this up can be 49 | found at the [Bento](https://github.com/chef/bento) project web site. 50 | 51 | Integration tests using Vagrant can be performed with 52 | 53 | :: 54 | 55 | ./bin/rake integration 56 | -------------------------------------------------------------------------------- /tests/unit/test_failover_report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import os 6 | import subprocess 7 | from mha_helper.config_helper import ConfigHelper 8 | 9 | __author__ = 'ovais.tariq' 10 | __project__ = 'mha_helper' 11 | 12 | 13 | class TestFailoverReport(unittest.TestCase): 14 | def setUp(self): 15 | self.root_directory = os.path.dirname(os.path.realpath(__file__)) 16 | self.failover_report_script_path = os.path.realpath( 17 | "{0}/../scripts/master_failover_report".format(self.root_directory)) 18 | 19 | self.mha_helper_config_dir = os.path.join(self.root_directory, 'conf', 'good') 20 | if not self.mha_helper_config_dir: 21 | self.fail(msg='mha_helper configuration dir not set') 22 | 23 | ConfigHelper.MHA_HELPER_CONFIG_DIR = self.mha_helper_config_dir 24 | if not ConfigHelper.load_config(): 25 | self.fail(msg='Could not load mha_helper configuration from %s' % self.mha_helper_config_dir) 26 | 27 | self.orig_master_host = os.getenv('ORIG_MASTER_HOST') 28 | self.new_master_host = os.getenv('NEW_MASTER_HOST') 29 | 30 | def test_main(self): 31 | mha_config_path = os.path.join(self.mha_helper_config_dir, 'test_cluster.conf') 32 | email_subject = "Testing Emails via %s" % self.__class__.__name__ 33 | email_body = "Test message sent through %s" % self.__class__.__name__ 34 | 35 | print("\n- Testing sending the failover report") 36 | cmd = """{0} --conf={1} --orig_master_host={2} --new_master_host={3} --subject="{4}" --body="{5}" \ 37 | --test_config_path={6}""".format(self.failover_report_script_path, mha_config_path, self.orig_master_host, 38 | self.new_master_host, email_subject, email_body, self.mha_helper_config_dir) 39 | 40 | proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 41 | stdout, stderr = proc.communicate() 42 | print("STDOUT: \n%s" % stdout) 43 | print("STDERR: \n%s" % stderr) 44 | 45 | self.assertEqual(proc.returncode, 0) 46 | 47 | if __name__ == '__main__': 48 | unittest.main() 49 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/recipes/master.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: mysql_replication 3 | # Recipe:: master 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # Set attributes that need to be set differently for the master 21 | node.default["mysql"]["server"]["read_only"] = 0 22 | 23 | # Include the base recipe that sets up and configures the MySQL server 24 | include_recipe "mysql_replication::_server" 25 | 26 | # During the first converge of master we run 'FLUSH LOGS' to start a fresh 27 | # binlog file that the slaves then can use 28 | bash 'binary logs :flush during first run' do 29 | user 'root' 30 | code <<-EOH 31 | /usr/bin/mysqladmin flush-logs 32 | EOH 33 | notifies :create, 'file[/tmp/master_flush_logs_done]', :immediately 34 | not_if { File.exist?('/tmp/master_flush_logs_done') } 35 | end 36 | 37 | file '/tmp/master_flush_logs_done' do 38 | owner 'root' 39 | group 'root' 40 | mode 0755 41 | action :nothing 42 | end 43 | 44 | # During the first converge of master we assign the master VIP, afterwards 45 | # its controlled through MHA 46 | bash 'writer VIP :assign during first run' do 47 | user 'root' 48 | code <<-EOH 49 | ip addr add #{node["mysql"]["mha"]["writer_vip_cidr"]} dev #{node["mysql"]["mha"]["cluster_interface"]} 50 | arping -q -c 3 -A -I #{node["mysql"]["mha"]["cluster_interface"]} #{node["mysql"]["mha"]["writer_vip"]} 51 | EOH 52 | notifies :create, 'file[/tmp/master_writer_vip_done]', :immediately 53 | not_if { File.exist?('/tmp/master_writer_vip_done') } 54 | end 55 | 56 | file '/tmp/master_writer_vip_done' do 57 | owner 'root' 58 | group 'root' 59 | mode 0755 60 | action :nothing 61 | end 62 | -------------------------------------------------------------------------------- /support-files/mha_manager_daemon-test_cluster-init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # (c) 2015, Ovais Tariq 4 | # 5 | # This file is part of mha-helper 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | 20 | # 21 | ### BEGIN INIT INFO 22 | # Provides: mha_manager_daemon-test_cluster 23 | # Default-Start: 2 3 4 5 24 | # Default-Stop: 0 1 6 25 | # Short-Description: This is mha_manager_daemon for test_cluster 26 | # Description: There can be multiple instances of mha_manager_daemon 27 | # running each monitoring a different MySQL master-slave 28 | # cluster. The purpose of this init-script is to provide 29 | # a controlling the daemon specific to the cluster named 30 | # test_cluster. 31 | ### END INIT INFO 32 | 33 | # Get function from functions library 34 | . /etc/init.d/functions 35 | 36 | CONF=/usr/local/mha-helper/conf/test_cluster.conf 37 | MHA_DAEMON=/usr/local/mha-helper/support-files/mha_manager_daemon 38 | SLAVE_CHECK_DAEMON=/usr/local/mha-helper/support-files/slave_health_check_daemon 39 | 40 | case "$1" in 41 | start) 42 | ${MHA_DAEMON} --conf=${CONF} start 43 | ${SLAVE_CHECK_DAEMON} start 44 | ;; 45 | stop) 46 | ${MHA_DAEMON} --conf=${CONF} stop 47 | ${SLAVE_CHECK_DAEMON} stop 48 | ;; 49 | status) 50 | ${MHA_DAEMON} --conf=${CONF} status 51 | ${SLAVE_CHECK_DAEMON} status 52 | ;; 53 | restart) 54 | ${MHA_DAEMON} --conf=${CONF} restart 55 | ${SLAVE_CHECK_DAEMON} restart 56 | ;; 57 | *) 58 | echo "Usage: $0 {start|stop|restart|status}" 59 | exit 1 60 | ;; 61 | esac 62 | -------------------------------------------------------------------------------- /scripts/master_failover_report: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # (c) 2015, Ovais Tariq 4 | # 5 | # This file is part of mha_helper 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | 20 | import sys 21 | import os 22 | from optparse import OptionParser 23 | from mha_helper.config_helper import ConfigHelper 24 | from mha_helper.email_helper import EmailHelper 25 | 26 | # Define exit codes 27 | ERR_CODE_FAILOVER_REPORT_COMMAND_ARG = 10 28 | ERR_CODE_FAILOVER_REPORT_CONFIG_ERR = 20 29 | ERR_CODE_FAILOVER_REPORT_EMAIL_ERR = 20 30 | SUCCESS_CODE_FAILOVER_REPORT = 0 31 | 32 | 33 | def main(): 34 | # parse comand line arguments 35 | parser = OptionParser() 36 | parser.add_option('--conf', type='string') 37 | parser.add_option('--orig_master_host', type='string') 38 | parser.add_option('--new_master_host', type='string') 39 | parser.add_option('--new_slave_hosts', type='string') 40 | parser.add_option('--subject', type='string') 41 | parser.add_option('--body', type='string') 42 | parser.add_option('--test_config_path', type='string') 43 | 44 | (options, args) = parser.parse_args() 45 | 46 | # This is purely for testing purposes 47 | if options.test_config_path is not None and os.path.exists(options.test_config_path): 48 | ConfigHelper.MHA_HELPER_CONFIG_DIR = options.test_config_path 49 | 50 | # Bail out if the required options are not provided 51 | if options.orig_master_host is None or options.subject is None or options.body is None: 52 | sys.exit(ERR_CODE_FAILOVER_REPORT_COMMAND_ARG) 53 | 54 | if not ConfigHelper.load_config(): 55 | sys.exit(ERR_CODE_FAILOVER_REPORT_CONFIG_ERR) 56 | 57 | email_sender = EmailHelper(options.orig_master_host) 58 | if not email_sender.send_email(options.subject, options.body): 59 | sys.exit(ERR_CODE_FAILOVER_REPORT_EMAIL_ERR) 60 | 61 | # exit the script with the appropriate code 62 | sys.exit(SUCCESS_CODE_FAILOVER_REPORT) 63 | 64 | if __name__ == "__main__": 65 | main() 66 | -------------------------------------------------------------------------------- /tests/unit/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | script_root=$(dirname $(readlink -f $0)) 5 | 6 | # Export the variables to be used by the MySQLHelper tests 7 | export MYSQL_TEST_VERSION="5.6.19-log" 8 | export MYSQL_TEST_IP="192.168.30.11" 9 | export MYSQL_TEST_USER="msandbox" 10 | export MYSQL_TEST_PASSWORD="msandbox" 11 | export MYSQL_TEST_PORT="3306" 12 | 13 | # Export the variables that are needed by the SSHHelper test 14 | export SSH_TEST_HOST="master" 15 | export SSH_TEST_IP="192.168.30.11" 16 | export SSH_TEST_USER="root" 17 | export SSH_TEST_PORT="22" 18 | 19 | # Export the variables that are needed by the MHAHelper test 20 | export ORIG_MASTER_HOST="master" 21 | export ORIG_MASTER_IP="192.168.30.11" 22 | export ORIG_MASTER_PORT=3306 23 | export ORIG_MASTER_USER="msandbox" 24 | export ORIG_MASTER_PASSWORD="msandbox" 25 | export ORIG_MASTER_SSH_HOST="manager" 26 | export ORIG_MASTER_SSH_IP="192.168.30.11" 27 | export ORIG_MASTER_SSH_PORT=22 28 | export ORIG_MASTER_SSH_USER="root" 29 | export NEW_MASTER_HOST="node1" 30 | export NEW_MASTER_IP="192.168.30.12" 31 | export NEW_MASTER_PORT=3306 32 | export NEW_MASTER_USER="msandbox" 33 | export NEW_MASTER_PASSWORD="msandbox" 34 | export NEW_MASTER_SSH_USER="root" 35 | export NEW_MASTER_SSH_HOST="node1" 36 | export NEW_MASTER_SSH_IP="192.168.30.12" 37 | export NEW_MASTER_SSH_PORT=22 38 | 39 | function unit_test_python_classes() { 40 | # Run the Unit tests for the Python classes 41 | echo "-- Running ${script_root}/test_mysql_helper.py" 42 | python ${script_root}/test_mysql_helper.py -v 43 | echo 44 | echo "-- Running ${script_root}/test_config_helper.py" 45 | python ${script_root}/test_config_helper.py -v 46 | echo 47 | echo "-- Running ${script_root}/test_ssh_helper.py" 48 | python ${script_root}/test_ssh_helper.py -v 49 | echo 50 | echo "-- Running ${script_root}/test_vip_metal_helper.py" 51 | python ${script_root}/test_vip_metal_helper.py -v 52 | echo 53 | echo "-- Running ${script_root}/test_email_helper.py" 54 | python ${script_root}/test_email_helper.py -v 55 | echo 56 | echo "-- Running ${script_root}/test_mha_helper.py" 57 | python ${script_root}/test_mha_helper.py -v 58 | } 59 | 60 | function test_python_scripts() { 61 | # Run the unit tests for the Python scripts 62 | echo 63 | echo "-- Running ${script_root}/test_master_ip_online_failover_helper.py" 64 | python ${script_root}/test_master_ip_online_failover_helper.py -v 65 | echo 66 | echo "-- Running ${script_root}/test_master_ip_hard_failover_helper.py" 67 | python ${script_root}/test_master_ip_hard_failover_helper.py -v 68 | } 69 | 70 | unit_test_python_classes 71 | test_python_scripts 72 | -------------------------------------------------------------------------------- /tests/unit/test_vip_metal_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import os 6 | from mha_helper.config_helper import ConfigHelper 7 | from mha_helper.vip_metal_helper import VIPMetalHelper 8 | 9 | __author__ = 'ovais.tariq' 10 | __project__ = 'mha_helper' 11 | 12 | 13 | class TestVIPMetalHelper(unittest.TestCase): 14 | def setUp(self): 15 | self.root_directory = os.path.dirname(os.path.realpath(__file__)) 16 | 17 | mha_helper_config_dir = os.path.join(self.root_directory, 'conf', 'good') 18 | if not mha_helper_config_dir: 19 | self.fail(msg='mha_helper configuration dir not set') 20 | 21 | ConfigHelper.MHA_HELPER_CONFIG_DIR = mha_helper_config_dir 22 | if not ConfigHelper.load_config(): 23 | self.fail(msg='Could not load mha_helper configuration from %s' % mha_helper_config_dir) 24 | 25 | self.ssh_host = os.getenv('SSH_TEST_HOST') 26 | self.ssh_host_ip = os.getenv('SSH_TEST_IP') 27 | self.ssh_user = os.getenv('SSH_TEST_USER') 28 | self.ssh_port = os.getenv('SSH_TEST_PORT') 29 | 30 | if not self.ssh_host or not self.ssh_host_ip or not self.ssh_user or not self.ssh_port: 31 | self.fail(msg='SSH connection information not set') 32 | 33 | self.vip_helper = VIPMetalHelper(host=self.ssh_host, host_ip=self.ssh_host_ip, ssh_user=self.ssh_user, 34 | ssh_port=self.ssh_port, ssh_options=None) 35 | 36 | def tearDown(self): 37 | self.vip_helper.remove_vip() 38 | 39 | def test_assign_vip(self): 40 | # We test assigning a VIP to a host that already does not have the VIP 41 | self.assertTrue(self.vip_helper.assign_vip()) 42 | 43 | # We then test assigning a VIP to a host that already has the the VIP assigned 44 | self.assertFalse(self.vip_helper.assign_vip()) 45 | 46 | def test_remove_vip(self): 47 | # We test removing the VIP from the host that already does not have the VIP 48 | self.assertFalse(self.vip_helper.remove_vip()) 49 | 50 | # We then test removing the VIP from the host that already has the the VIP assigned 51 | self.vip_helper.assign_vip() 52 | self.assertTrue(self.vip_helper.remove_vip()) 53 | 54 | def test_has_vip(self): 55 | # We test to see that we are able to validate the function against a host without the VIP 56 | self.assertFalse(self.vip_helper.has_vip()) 57 | 58 | # We now test to see that we are able to validate the function against a host that does have the VIP 59 | self.vip_helper.assign_vip() 60 | self.assertTrue(self.vip_helper.has_vip()) 61 | 62 | if __name__ == '__main__': 63 | unittest.main() 64 | 65 | -------------------------------------------------------------------------------- /tests/unit/test_mysql_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import os 6 | from mha_helper.mysql_helper import MySQLHelper 7 | 8 | __author__ = 'ovais.tariq' 9 | __project__ = 'mha_helper' 10 | 11 | 12 | class TestMySQLHelper(unittest.TestCase): 13 | def setUp(self): 14 | mysql_host = os.getenv('MYSQL_TEST_IP') 15 | mysql_user = os.getenv('MYSQL_TEST_USER') 16 | mysql_password = os.getenv('MYSQL_TEST_PASSWORD') 17 | mysql_port = os.getenv('MYSQL_TEST_PORT') 18 | 19 | if not mysql_host or not mysql_user or not mysql_password or not mysql_port: 20 | self.fail(msg='MySQL database connection information not set') 21 | 22 | self.mysql_conn = MySQLHelper(host=mysql_host, port=mysql_port, user=mysql_user, password=mysql_password) 23 | if not self.mysql_conn.connect(): 24 | self.fail(msg='Could not connect to the MySQL database') 25 | 26 | self.mysql_version = os.getenv('MYSQL_TEST_VERSION') 27 | 28 | def tearDown(self): 29 | self.mysql_conn.disconnect() 30 | 31 | def test_is_read_only_query(self): 32 | self.assertEqual(MySQLHelper.is_read_only_query('SELECT 1'), True) 33 | self.assertEqual(MySQLHelper.is_read_only_query('INSERT INTO foo VALUES ("bar")'), False) 34 | 35 | def test_get_version(self): 36 | self.assertEqual(self.mysql_conn.get_version(), self.mysql_version) 37 | 38 | def test_get_connection_id(self): 39 | self.assertNotEqual(self.mysql_conn.get_connection_id(), None) 40 | 41 | def test_get_current_user(self): 42 | self.assertNotEqual(self.mysql_conn.get_current_user(), None) 43 | 44 | def test_set_read_only(self): 45 | self.mysql_conn.set_read_only() 46 | self.assertEqual(self.mysql_conn.is_read_only(), True) 47 | 48 | def test_unset_read_only(self): 49 | self.mysql_conn.unset_read_only() 50 | self.assertEqual(self.mysql_conn.is_read_only(), False) 51 | 52 | def test_get_processlist(self): 53 | processlist = self.mysql_conn.get_processlist() 54 | self.assertTrue(len(processlist) >= 1) 55 | 56 | def test_kill_connection(self): 57 | mysql_host = os.getenv('MYSQL_TEST_IP') 58 | mysql_user = os.getenv('MYSQL_TEST_USER') 59 | mysql_password = os.getenv('MYSQL_TEST_PASSWORD') 60 | mysql_port = int(os.getenv('MYSQL_TEST_PORT')) 61 | 62 | mysql_conn = MySQLHelper(host=mysql_host, port=mysql_port, user=mysql_user, password=mysql_password) 63 | if not mysql_conn.connect(): 64 | self.fail(msg='Could not connect to the MySQL database') 65 | 66 | mysql_conn_id = mysql_conn.get_connection_id() 67 | self.assertTrue(self.mysql_conn.kill_connection(mysql_conn_id)) 68 | 69 | del mysql_conn 70 | 71 | if __name__ == '__main__': 72 | unittest.main() 73 | -------------------------------------------------------------------------------- /tests/integration/test/fixtures/cookbooks/mysql_replication/attributes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: mysql_replication 3 | # Attributes:: default 4 | # 5 | # Copyright 2015, Ovais Tariq 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # MySQL version 21 | default["mysql"]["version"] = "5.6" 22 | 23 | # MySQL root user 24 | default["mysql"]["root_password"] = "root" 25 | 26 | # MySQL replication credentials 27 | default["mysql"]["replication"]["user"] = "repl" 28 | default["mysql"]["replication"]["password"] = "repl" 29 | 30 | # MySQL MHA access 31 | default["mysql"]["mha"]["user"] = "mha" 32 | default["mysql"]["mha"]["password"] = "mha" 33 | 34 | # Setup the barebone attribute used within the cookbook 35 | default['mysql_replication'] = Hash.new 36 | 37 | # This is needed temporarily for the first converge 38 | default["mysql"]["mha"]["writer_vip_cidr"] = "192.168.30.100/24" 39 | default["mysql"]["mha"]["writer_vip"] = "192.168.30.100" 40 | default["mysql"]["mha"]["cluster_interface"] = "eth1" 41 | 42 | # MySQL configuration 43 | # GENERAL # 44 | default["mysql"]["server"]["socket"] = "/var/lib/mysql/mysql.sock" 45 | default["mysql"]["server"]["port"] = "3306" 46 | 47 | # DATA STORAGE # 48 | default["mysql"]["server"]["datadir"] = "/var/lib/mysql" 49 | default["mysql"]["server"]["logdir"] = "/var/lib/mysql" 50 | 51 | # BINARY LOGGING # 52 | default["mysql"]["server"]["log_bin_filename"] = "mysql-bin" 53 | default["mysql"]["server"]["log_bin"] = "#{node["mysql"]["server"]["logdir"]}/#{node["mysql"]["server"]["log_bin_filename"]}" 54 | default["mysql"]["server"]["sync_binlog"] = 0 55 | 56 | # REPLICATION # 57 | default["mysql"]["server"]["read_only"] = 1 58 | default["mysql"]["server"]["log_slave_updates"] = 1 59 | default["mysql"]["server"]["relay_log"] = "#{node["mysql"]["server"]["logdir"]}/relay-bin" 60 | default["mysql"]["server"]["slave_net_timeout"] = 60 61 | 62 | # InnoDB # 63 | default["mysql"]["server"]["innodb_buffer_pool_size"] = "16M" 64 | 65 | # LOGGING # 66 | default["mysql"]["server"]["log_error"] = "#{node["mysql"]["server"]["logdir"]}/mysql-error.log" 67 | default["mysql"]["server"]["log_warnings"] = 2 68 | -------------------------------------------------------------------------------- /tests/integration/Rakefile: -------------------------------------------------------------------------------- 1 | # Copyright 2015, Ovais Tariq 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | require 'rspec/core/rake_task' 17 | require 'rubocop/rake_task' 18 | require 'foodcritic' 19 | require 'kitchen' 20 | require 'mixlib/shellout' 21 | require 'kitchen/rake_tasks' 22 | 23 | # Integration tests. Kitchen.ci 24 | namespace :integration do 25 | desc 'Setup the test-kitchen vagrant instances' 26 | task :vagrant_setup do 27 | Kitchen.logger = Kitchen.default_file_logger 28 | Kitchen::Config.new.instances.each do |instance| 29 | # this happens serially because virualbox/vagrant can't handle 30 | # parallel vm creation 31 | instance.create() 32 | 33 | # Initial converge 34 | instance.converge() 35 | end 36 | end 37 | 38 | desc 'Verify the test-kitchen vagrant instances' 39 | task :vagrant_verify do 40 | Kitchen.logger = Kitchen.default_file_logger 41 | Kitchen::Config.new.instances.each do |instance| 42 | # Run the integration tests now 43 | instance.verify() 44 | end 45 | end 46 | end 47 | 48 | # TODO: implement sysbench based tests 49 | # sysbench --test=/tmp/kitchen/cache/sysbench/sysbench/tests/db/oltp.lua --db-driver=mysql --oltp-tables-count=8 --oltp-table-size=10000 --mysql-table-engine=innodb --mysql-user=mha --mysql-password=mha --mysql-host=192.168.30.100 --mysql-db=test prepare 50 | # 51 | # sysbench --test=/tmp/kitchen/cache/sysbench/sysbench/tests/db/oltp.lua --db-driver=mysql --oltp-tables-count=8 --oltp-table-size=10000 --mysql-table-engine=innodb --mysql-user=mha --mysql-password=mha --mysql-host=192.168.30.100 --mysql-db=test --max-time=930 --num-threads=8 --max-requests=0 --oltp-reconnect --mysql-ignore-errors=2013 --report-interval=10 run 52 | 53 | # Clean up 54 | namespace :cleanup do 55 | desc 'Destroy test-kitchen instances' 56 | task :kitchen_destroy do 57 | destroy = Kitchen::RakeTasks.new do |obj| 58 | def obj.destroy 59 | config.instances.each(&:destroy) 60 | end 61 | end 62 | destroy.destroy 63 | end 64 | end 65 | 66 | desc 'Generate the setup' 67 | task setup: ['integration:vagrant_setup'] 68 | 69 | desc 'Clean up generated files' 70 | task cleanup: ['cleanup:kitchen_destroy'] 71 | 72 | desc 'Run full integration' 73 | task integration: ['cleanup:kitchen_destroy', 'integration:vagrant_setup', 'integration:vagrant_verify', 'cleanup:kitchen_destroy'] 74 | 75 | # Default 76 | task default: ['style', 'spec', 'integration', 'cleanup'] 77 | -------------------------------------------------------------------------------- /tests/integration/test/integration/data_bags/mysql_mha_secrets/test_pod.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "test_pod", 3 | "comment": { 4 | "encrypted_data": "Gznt6PjQS/d1h78y8IsTAWM5A4Ulh9dxZyWO3lCe9qrp5Wu5ckBDJiPxbYbT\nvbL0Qi/YEYjGhy46xXWfV+2Ouw==\n", 5 | "iv": "4xE4Sk55kVg8fCC8TFdXQw==\n", 6 | "version": 1, 7 | "cipher": "aes-256-cbc" 8 | }, 9 | "mysql": { 10 | "encrypted_data": "axZshEcbk17kRL6DftRhzG1N5a6Vd+gkFFBrKRHQU1OoSauyf4rCzQbOOZXP\noyztueaaYQ1DX7TDcTpHAvoI8A==\n", 11 | "iv": "wyU8J7Tp/Ev+//s0gCeNyw==\n", 12 | "version": 1, 13 | "cipher": "aes-256-cbc" 14 | }, 15 | "remote_user": { 16 | "encrypted_data": "v7Ewv/Cn+pZ3Hbv/VUzZEuaMZ3VlC9/6Fqe8ZUTDgBOlEJoFgAmJKc/ckgKR\nRhYWqjD/Og7S22iFmqZ6DzA/ldyG3lnd2v3BjwYvjJ4DFQKMq87eL8CyywFj\nUeo7tasHSvfpMBB1V56y/I7wP0dEbb+MIIf44l1hc4AKtHGOYmRoBS3m+B7y\n7GFxKEipq1ulEOw8DTYbjSd2UzK/5RcRgG6yL0m84+hwFF0CEFHbcevmRcjL\nAgdJ1sWe21UDjpCeSIedbQFiKyUjZOFAaOkTF60iLgOb/2MhwKgqhQXWYYil\nQaqMtux0+Fss39Guj3A8Em/I2z+AG3WJOhRpDcS7By3Jv5ULx//mc4ddB2oa\nL+9eZ2zbH6cb34UxqY9JFpxy+KrvBb0gc7+2n/DkGKudYs/Q5SYpe+mWfCLL\n9Kp9y+v5dgJBXB/9QZt7YIDvccsPGi0iijoyc8TSIPU7z+5II1dq5DStygEn\nA7z0CRuAiWf7HHc3mAKukD+wunndAyFwEVigKRuJRp7NIj9/t4gvMdfqmnxE\nvcfe1q2b6plKPvWIs9MvdC/ZOZxmUNXWBv3kuuzudvfhSneafU/rX1CdKJBx\nd7fLq/ufQKQDDlDLbgS/gPkJFKNGDznW8aWXCVbm58EFxdwTSSYOdAxtIK7V\nwg+F9Qm7ZsGpCo9nKxkVNSJf1HNmsKR8GgI2QgJ0eFPyk9X3lIhuSM5H771o\nMh5INN5Q+lcbXbwIvoGOHj+HsR2JzY8rPStpuTCed7aCjnNn4zf+iJusX3KI\nEr3sow3BmupKQceaRSdWlWyjGQypY8WuEvPt8DeU/wRg4QlVthZpbcf9aP+c\n2jO7FKpiJ5M/wwQZMHet2TIh86rds3hXNvo/H9L3oO9Cg6ik7Q47UGQGd4DL\nOJCWqPoS2IlJtkupbUq1JRN9OdKG9MeWOdNfdtBs5vhbkrTLslljuxIrEjCV\nOiwbQnl6FF9XSbEwj5U5DDw4goMpaJ79VHKMOcFP26Lm4omKakk2oKnN0pIm\nPtSOvQHKO6IR3/mEnyoAQcrwLLQBqRTkxYRMsiEfqFy+oEDde083iF7UyVg7\nr0fX5NKTC1Ctsdsrvi0REmG3a8KGgiUX91UcenCPeB1zACppwjrNxsvjvCpF\nQbapFSIRS4BUlnMSgP2iAwE5iW8FevTicZjNFKODwh4PXnzr3cXid0jPMgSW\nvo8MnXoYidVG5csk3trPJBH50OTrcksug54spsHHLjRDsrzGFTBDMJ+kUGWm\n0U/3EiPDbEFCp+jaCxHrIfk2Tt8VaVJW3IWVhMZBSTVMpMvGDFXhPS3EDTg5\nke+WA15Hvhu+6GRksRnXzSOVnHezfKDNyhhOcXPQO4cRR+jaEckdvN6voNg9\n3X4croabqsAgomQjl2TjX8j11JpRT6SnFarXKMOK6QkCn/XAMW3K0ZvXXXNA\nKrx9ai0M4EQ10uP2SBfbuQO2WA06LHkZB62T3hQjFZ5TnJT9/nvj3PjYV3ha\n2c73o16Kf0MqBcN/Rm7UYdqK3T3mOYEYWi4WSw3F2rNBWicwgKSHNZ4+e5fT\nYpMHCUVqYTlyT9fhpZZCwy6gJDiimDOHp7Z4Lhf8ruSng+qm9tooak53FBVy\n36puRyIYh+4PljtPMr7tWkx23hHf1x7tLoSZR2AKoXfVlv9C97JjLzkRxwpm\n04C3l3XNmb4eqRff598isrT5EHYom60tx7GTKmHK52U0Jjjg+jv8LQpVaEgd\nGSedjHRYWkqtuSqeJWNhL4xgU1L0A2cuItUp3VfDtsO+FO9HDV0gs4sIr2O4\nf9D8cM/CJQRxv1oK+ux/4w70piifnm+L4YX4XKKq6mO1k8leh7COBWPWumDg\n0f9tLrkoo4r3M9C5xgBqjq1HqIvXOv7sNvCZnuZPENyodwnksK0RCumcnDSm\n4pnk8s5+Z92cMguwBFV8BIAd6xrCnqPXfjItWoI2az5+lApNI/b+xm9oEUQ6\n/1HVnEHXZSCN9crHcCTkXiCUWnnaOExaJ2WdQinJY6GUmLU27sAnhRftW+VJ\nfQLwmXIM/gvCcx0C6LqSe3klWl8JatTrHi3PMLR9yeZcZnCBUxF/0SyoKunS\nk7ZNJp/xqirTsOAOGzfRZePF1Ov/pWvG+q2OeEkJFha9jU+CYpPDT2zZ/xo9\nefqX2Do9wAmQWUa4xSxTu9kLARlQIRF7w2G5dHMK2u6r0oHLLbsaRiFxZe+4\nh3qRlTrzfTEYYGCQ9Csk5OjRNzk3ak++q23E6d0JFFeLgZd0/h6S1BsjjQP9\nLip1GBxIqFLeJE+uobpvLbai8iTS1jIOyXJNJxFBJlEJTw==\n", 17 | "iv": "prK1TaxAYW68osi2bzoh7Q==\n", 18 | "version": 1, 19 | "cipher": "aes-256-cbc" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /scripts/master_ip_hard_failover_helper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # (c) 2015, Ovais Tariq 4 | # 5 | # This file is part of mha_helper 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | 20 | from __future__ import print_function 21 | import sys 22 | import os 23 | from optparse import OptionParser 24 | from mha_helper.config_helper import ConfigHelper 25 | from mha_helper.mha_helper import MHAHelper 26 | 27 | # Define exit codes 28 | ERR_CODE_HARD_FAILOVER_COMMAND_ARG = 10 29 | ERR_CODE_HARD_FAILOVER_FAILURE = 20 30 | SUCCESS_CODE_HARD_FAILOVER = 0 31 | 32 | 33 | def main(): 34 | # parse command line arguments 35 | parser = OptionParser() 36 | parser.add_option('--command', type='string') 37 | parser.add_option('--orig_master_host', type='string') 38 | parser.add_option('--orig_master_ip', type='string') 39 | parser.add_option('--orig_master_port', type='string') 40 | parser.add_option('--orig_master_ssh_host', type='string') 41 | parser.add_option('--orig_master_ssh_ip', type='string') 42 | parser.add_option('--orig_master_ssh_port', type='string') 43 | parser.add_option('--new_master_host', type='string') 44 | parser.add_option('--new_master_ip', type='string') 45 | parser.add_option('--new_master_port', type='string') 46 | parser.add_option('--new_master_ssh_host', type='string') 47 | parser.add_option('--new_master_ssh_ip', type='string') 48 | parser.add_option('--new_master_ssh_port', type='string') 49 | parser.add_option('--new_master_user', type='string') 50 | parser.add_option('--new_master_password', type='string') 51 | parser.add_option('--ssh_user', type='string') 52 | parser.add_option('--ssh_options', type='string') 53 | parser.add_option('--test_config_path', type='string') 54 | 55 | (options, args) = parser.parse_args() 56 | 57 | # This is purely for testing purposes 58 | if options.test_config_path is not None and os.path.exists(options.test_config_path): 59 | ConfigHelper.MHA_HELPER_CONFIG_DIR = options.test_config_path 60 | 61 | mha_helper = MHAHelper(MHAHelper.FAILOVER_TYPE_HARD) 62 | 63 | failover_arguments = dict() 64 | for opt, value in options.__dict__.items(): 65 | if value is not None and opt != "test_config_path": 66 | failover_arguments[opt] = value 67 | 68 | if not mha_helper.execute_command(**failover_arguments): 69 | sys.exit(ERR_CODE_HARD_FAILOVER_FAILURE) 70 | 71 | # exit the script with the appropriate code 72 | # if script exits with a 0 status code, MHA continues with the failover 73 | sys.exit(SUCCESS_CODE_HARD_FAILOVER) 74 | 75 | 76 | if __name__ == "__main__": 77 | main() 78 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # (c) 2015, Ovais Tariq 2 | # 3 | # This file is part of mha_helper 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | # To workaround hard link error 19 | # http://stackoverflow.com/questions/7719380/python-setup-py-sdist-error-operation-not-permitted 20 | # http://bugs.python.org/issue8876#msg208792 21 | import os 22 | del os.link 23 | 24 | try: 25 | from setuptools import setup 26 | except ImportError: 27 | from distutils.core import setup 28 | 29 | import mha_helper 30 | 31 | 32 | with open('README.rst') as readme_file: 33 | readme = readme_file.read() 34 | 35 | with open('HISTORY.rst') as history_file: 36 | history = history_file.read().replace('.. :changelog:', '') 37 | 38 | longdesc = ''' 39 | MHA helper is a Python module that supplements in doing proper failover using MHA_. 40 | MHA is responsible for executing the important failover steps such as finding the 41 | most recent slave to failover to, applying differential logs, monitoring master for 42 | failure, etc. But it does not deal with additional steps that need to be taken 43 | before and after failover. These would include steps such as setting the read-only 44 | flag, killing connections, moving writer virtual IP, etc. 45 | 46 | Required packages: 47 | PyMySQL 48 | paramiko 49 | 50 | .. _MHA: https://code.google.com/p/mysql-master-ha/ 51 | ''' 52 | 53 | setup( 54 | name=mha_helper.__name__, 55 | version=mha_helper.__version__, 56 | description="A Python module that supplements MHA in doing proper MySQL master failover", 57 | long_description=longdesc, 58 | author=mha_helper.__author__, 59 | author_email=mha_helper.__email__, 60 | url=mha_helper.__url__, 61 | license="GNU GENERAL PUBLIC LICENSE Version 3", 62 | packages=['mha_helper'], 63 | scripts=['scripts/master_failover_report', 64 | 'scripts/master_ip_hard_failover_helper', 65 | 'scripts/master_ip_online_failover_helper', 66 | 'scripts/mysql_failover', 67 | 'scripts/mysql_online_failover'], 68 | install_requires=['PyMySQL>=0.6.3', 69 | 'paramiko>=1.10.0'], 70 | keywords='mha_helper, mha, mysql, failover, high availability', 71 | classifiers=[ 72 | "Programming Language :: Python :: 2", 73 | 'Programming Language :: Python :: 2.6', 74 | 'Programming Language :: Python :: 2.7', 75 | 'Development Status :: 4 - Beta', 76 | 'Intended Audience :: Developers', 77 | 'Intended Audience :: System Administrators', 78 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 79 | 'Environment :: Console', 80 | 'Topic :: Database', 81 | 'Natural Language :: English' 82 | ] 83 | ) 84 | -------------------------------------------------------------------------------- /scripts/master_ip_online_failover_helper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # (c) 2015, Ovais Tariq 4 | # 5 | # This file is part of mha_helper 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | 20 | from __future__ import print_function 21 | import sys 22 | import os 23 | from optparse import OptionParser 24 | from mha_helper.config_helper import ConfigHelper 25 | from mha_helper.mha_helper import MHAHelper 26 | 27 | # Define exit codes 28 | ERR_CODE_ONLINE_FAILOVER_COMMAND_ARG = 10 29 | ERR_CODE_ONLINE_FAILOVER_FAILURE = 20 30 | SUCCESS_CODE_ONLINE_FAILOVER = 0 31 | 32 | 33 | def main(): 34 | # parse command line arguments 35 | parser = OptionParser() 36 | parser.add_option('--command', type='string') 37 | parser.add_option('--orig_master_host', type='string') 38 | parser.add_option('--orig_master_ip', type='string') 39 | parser.add_option('--orig_master_port', type='string') 40 | parser.add_option('--orig_master_user', type='string') 41 | parser.add_option('--orig_master_password', type='string') 42 | parser.add_option('--orig_master_ssh_host', type='string') 43 | parser.add_option('--orig_master_ssh_ip', type='string') 44 | parser.add_option('--orig_master_ssh_port', type='string') 45 | parser.add_option('--orig_master_ssh_user', type='string') 46 | parser.add_option('--new_master_host', type='string') 47 | parser.add_option('--new_master_ip', type='string') 48 | parser.add_option('--new_master_port', type='string') 49 | parser.add_option('--new_master_user', type='string') 50 | parser.add_option('--new_master_password', type='string') 51 | parser.add_option('--new_master_ssh_user', type='string') 52 | parser.add_option('--new_master_ssh_host', type='string') 53 | parser.add_option('--new_master_ssh_ip', type='string') 54 | parser.add_option('--new_master_ssh_port', type='string') 55 | parser.add_option('--ssh_options', type='string') 56 | parser.add_option('--orig_master_is_new_slave', action="store_true") 57 | parser.add_option('--test_config_path', type='string') 58 | 59 | (options, args) = parser.parse_args() 60 | 61 | # This is purely for testing purposes 62 | if options.test_config_path is not None and os.path.exists(options.test_config_path): 63 | ConfigHelper.MHA_HELPER_CONFIG_DIR = options.test_config_path 64 | 65 | mha_helper = MHAHelper(MHAHelper.FAILOVER_TYPE_ONLINE) 66 | 67 | failover_arguments = dict() 68 | for opt, value in options.__dict__.items(): 69 | if value is not None and opt != "test_config_path": 70 | failover_arguments[opt] = value 71 | 72 | if not mha_helper.execute_command(**failover_arguments): 73 | sys.exit(ERR_CODE_ONLINE_FAILOVER_FAILURE) 74 | 75 | # exit the script with the appropriate code 76 | # if script exits with a 0 status code, MHA continues with the failover 77 | sys.exit(SUCCESS_CODE_ONLINE_FAILOVER) 78 | 79 | 80 | if __name__ == "__main__": 81 | main() 82 | -------------------------------------------------------------------------------- /tests/integration/.kitchen.yml: -------------------------------------------------------------------------------- 1 | --- 2 | driver: 3 | name: vagrant 4 | require_chef_omnibus: 12.4.1 5 | vagrantfile_erb: Vagrantfile.erb 6 | synced_folders: 7 | - ['../../', '/tmp/mha-helper'] 8 | 9 | provisioner: 10 | name: chef_zero 11 | encrypted_data_bag_secret_key_path: test/integration/encrypted_data_bag_secret 12 | attributes: 13 | hosts_list: 14 | - 15 | hostname: "manager" 16 | ip: "192.168.30.10" 17 | - 18 | hostname: "master" 19 | ip: "192.168.30.11" 20 | - 21 | hostname: "slave01" 22 | ip: "192.168.30.12" 23 | - 24 | hostname: "slave02" 25 | ip: "192.168.30.13" 26 | 27 | platforms: 28 | - name: centos-6.6 29 | 30 | suites: 31 | - name: manager 32 | driver: 33 | network: 34 | - ["private_network", {ip: "192.168.30.10"}] 35 | vm_hostname: manager.localhost 36 | run_list: 37 | - recipe[base_test_setup] 38 | - recipe[mysql_replication::client] 39 | - recipe[mysql-mha::manager] 40 | - recipe[base_test_setup::dev_install] 41 | attributes: 42 | provisioner: 43 | client_rb: 44 | environment: prod 45 | 46 | - name: master 47 | driver: 48 | network: 49 | - ["private_network", {ip: "192.168.30.11"}] 50 | vm_hostname: master.localhost 51 | run_list: 52 | - recipe[base_test_setup] 53 | - recipe[mysql_replication::master] 54 | - recipe[mysql-mha::node] 55 | attributes: 56 | mysql_mha: 57 | pod_name: "test_pod" 58 | node: 59 | candidate_master: 1 60 | check_repl_delay: 0 61 | mysql_port: 3306 62 | mysql_binlog_dir: /var/lib/mysql 63 | ssh_port: 22 64 | mysql: 65 | server: 66 | port: 3306 67 | logdir: /var/lib/mysql 68 | server_id: 1 69 | provisioner: 70 | client_rb: 71 | environment: prod 72 | 73 | - name: slave01 74 | driver: 75 | network: 76 | - ["private_network", {ip: "192.168.30.12"}] 77 | vm_hostname: slave01.localhost 78 | run_list: 79 | - recipe[base_test_setup] 80 | - recipe[mysql_replication::slave] 81 | - recipe[mysql-mha::node] 82 | attributes: 83 | mysql_mha: 84 | pod_name: "test_pod" 85 | node: 86 | candidate_master: 1 87 | check_repl_delay: 0 88 | mysql_port: 3306 89 | mysql_binlog_dir: /var/log/mysql-default 90 | ssh_port: 22 91 | mysql: 92 | server: 93 | port: 3306 94 | logdir: /var/log/mysql-default 95 | server_id: 2 96 | mysql_replication: 97 | master_host: "192.168.30.11" 98 | provisioner: 99 | client_rb: 100 | environment: prod 101 | 102 | - name: slave02 103 | driver: 104 | network: 105 | - ["private_network", {ip: "192.168.30.13"}] 106 | vm_hostname: slave02.localhost 107 | run_list: 108 | - recipe[base_test_setup] 109 | - recipe[mysql_replication::slave] 110 | - recipe[mysql-mha::node] 111 | attributes: 112 | mysql_mha: 113 | pod_name: "test_pod" 114 | node: 115 | no_master: 1 116 | mysql_port: 3307 117 | mysql_binlog_dir: /var/log/mysql-default 118 | ssh_port: 22 119 | mysql: 120 | server: 121 | port: 3307 122 | logdir: /var/log/mysql-default 123 | server_id: 3 124 | mysql_replication: 125 | master_host: "192.168.30.11" 126 | provisioner: 127 | client_rb: 128 | environment: prod 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | Contributions are welcome, and they are greatly appreciated! Every 6 | little bit helps, and credit will always be given. 7 | 8 | You can contribute in many ways: 9 | 10 | Types of Contributions 11 | ---------------------- 12 | 13 | Report Bugs 14 | ~~~~~~~~~~~ 15 | 16 | Report bugs at https://github.com/ovaistariq/mha-helper/issues. 17 | 18 | If you are reporting a bug, please include: 19 | 20 | * Your operating system name and version. 21 | * Any details about your local setup that might be helpful in troubleshooting. 22 | * Detailed steps to reproduce the bug. 23 | 24 | Fix Bugs 25 | ~~~~~~~~ 26 | 27 | Look through the GitHub issues for bugs. Anything tagged with "bug" 28 | is open to whoever wants to implement it. 29 | 30 | Implement Features 31 | ~~~~~~~~~~~~~~~~~~ 32 | 33 | Look through the GitHub issues for features. Anything tagged with "feature" 34 | is open to whoever wants to implement it. 35 | 36 | Write Documentation 37 | ~~~~~~~~~~~~~~~~~~~ 38 | 39 | MHA Helper could always use more documentation, whether as part of the 40 | official docs, in docstrings, or even on the web in blog posts, articles, and such. 41 | 42 | Submit Feedback 43 | ~~~~~~~~~~~~~~~ 44 | 45 | The best way to send feedback is to file an issue at https://github.com/ovaistariq/mha-helper/issues. 46 | 47 | If you are proposing a feature: 48 | 49 | * Explain in detail how it would work. 50 | * Keep the scope as narrow as possible, to make it easier to implement. 51 | * Remember that this is a volunteer-driven project, and that contributions 52 | are welcome :) 53 | 54 | Get Started! 55 | ------------ 56 | 57 | Ready to contribute? Here's how to set up `mha-helper` for local development. 58 | 59 | 1. Fork the `mha-helper` repo on GitHub. 60 | 2. Clone your fork locally:: 61 | 62 | $ git clone git@github.com:your_name_here/mha-helper.git 63 | 64 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 65 | 66 | $ mkvirtualenv mha_helper 67 | $ cd mha_helper/ 68 | $ python setup.py develop 69 | 70 | 4. Create a branch for local development:: 71 | 72 | $ git checkout -b name-of-your-bugfix-or-feature 73 | 74 | Now you can make your changes locally. 75 | 76 | 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: 77 | 78 | $ flake8 mha_helper tests 79 | $ python setup.py test 80 | $ tox 81 | 82 | To get flake8 and tox, just pip install them into your virtualenv. 83 | 84 | 6. Commit your changes and push your branch to GitHub:: 85 | 86 | $ git add . 87 | $ git commit -m "Your detailed description of your changes." 88 | $ git push origin name-of-your-bugfix-or-feature 89 | 90 | 7. Submit a pull request through the GitHub website. 91 | 92 | Pull Request Guidelines 93 | ----------------------- 94 | 95 | Before you submit a pull request, check that it meets these guidelines: 96 | 97 | 1. The pull request should include tests. 98 | 2. If the pull request adds functionality, the docs should be updated. Put 99 | your new functionality into a function with a docstring, and add the 100 | feature to the list in README.rst. 101 | 3. The pull request should work for Python 2.6, 2.7 and for PyPy. Check 102 | https://travis-ci.org/ovaistariq/mha_helper/pull_requests 103 | and make sure that the tests pass for all supported Python versions. 104 | 105 | Tips 106 | ---- 107 | 108 | To run a subset of tests:: 109 | 110 | $ python -m unittest tests.test_mha_helper 111 | -------------------------------------------------------------------------------- /mha_helper/vip_metal_helper.py: -------------------------------------------------------------------------------- 1 | # (c) 2015, Ovais Tariq 2 | # 3 | # This file is part of mha_helper 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | from __future__ import print_function 19 | from ssh_helper import SSHHelper 20 | from config_helper import ConfigHelper 21 | import re 22 | 23 | 24 | class VIPMetalHelper(object): 25 | IP_CMD = "/sbin/ip" 26 | ARPING_CMD = "/usr/sbin/arping" 27 | 28 | def __init__(self, host, host_ip=None, ssh_user=None, ssh_port=None, ssh_options=None): 29 | config_helper = ConfigHelper(host) 30 | self._cluster_interface = config_helper.get_cluster_interface() 31 | self._writer_vip_cidr = config_helper.get_writer_vip_cidr() 32 | self._writer_vip = config_helper.get_writer_vip() 33 | self._requires_sudo = config_helper.get_requires_sudo() 34 | self._requires_arping = config_helper.get_requires_arping() 35 | 36 | self._ssh_client = SSHHelper(host, host_ip, ssh_user, ssh_port, ssh_options) 37 | 38 | def assign_vip(self): 39 | ip_cmd = "%s addr add %s dev %s" % (VIPMetalHelper.IP_CMD, self._writer_vip_cidr, self._cluster_interface) 40 | arping_cmd = "%s -q -c 3 -A -I %s %s" % (VIPMetalHelper.ARPING_CMD, self._cluster_interface, self._writer_vip) 41 | 42 | if self._requires_sudo: 43 | ip_cmd = "sudo %s" % ip_cmd 44 | arping_cmd = "sudo %s" % arping_cmd 45 | 46 | # Connect to the host over SSH 47 | if not self._ssh_client.make_ssh_connection(): 48 | return False 49 | 50 | # Assign the VIP to the host 51 | ret_code, stdout_lines = self._ssh_client.execute_ssh_command(ip_cmd) 52 | if not ret_code: 53 | if len(stdout_lines) > 0: 54 | print("Command output: %s" % "\n".join(stdout_lines)) 55 | return False 56 | 57 | # Send ARP update requests to all the listening hosts 58 | if self._requires_arping: 59 | ret_code, stdout_lines = self._ssh_client.execute_ssh_command(arping_cmd) 60 | if not ret_code: 61 | if len(stdout_lines) > 0: 62 | print("Command output: %s" % "\n".join(stdout_lines)) 63 | return False 64 | 65 | return True 66 | 67 | def remove_vip(self): 68 | ip_cmd = "%s addr delete %s dev %s" % (VIPMetalHelper.IP_CMD, self._writer_vip_cidr, self._cluster_interface) 69 | if self._requires_sudo: 70 | ip_cmd = "sudo %s" % ip_cmd 71 | 72 | # Connect to the host over SSH 73 | if not self._ssh_client.make_ssh_connection(): 74 | return False 75 | 76 | # Remove the VIP from the host 77 | ret_code, stdout_lines = self._ssh_client.execute_ssh_command(ip_cmd) 78 | if not ret_code: 79 | if len(stdout_lines) > 0: 80 | print("Command output: %s" % "\n".join(stdout_lines)) 81 | return False 82 | 83 | return True 84 | 85 | def has_vip(self): 86 | ip_cmd = "%s addr show dev %s" % (VIPMetalHelper.IP_CMD, self._cluster_interface) 87 | if self._requires_sudo: 88 | ip_cmd = "sudo %s" % ip_cmd 89 | 90 | # Connect to the host over SSH 91 | if not self._ssh_client.make_ssh_connection(): 92 | return False 93 | 94 | # Fetch the output of the command `ip addr show dev eth` and parse it to list the IP addresses 95 | # If the VIP is in that list then that means the VIP is assigned to the host 96 | ret_code, stdout_lines = self._ssh_client.execute_ssh_command(ip_cmd) 97 | if not ret_code: 98 | if len(stdout_lines) > 0: 99 | print("Command output: %s" % "\n".join(stdout_lines)) 100 | return False 101 | 102 | vip_found = False 103 | for line in stdout_lines: 104 | # We want to match a line similar to the following: 105 | # inet 192.168.30.11/24 brd 192.168.30.255 scope global eth1 106 | # or 107 | # inet6 fe80::a00:27ff:fed8:f757/64 scope link 108 | if re.search(r'\b(inet|inet6)\b', line): 109 | # The second element of the matching line is the IP address in CIDR format 110 | if line.split()[1] == self._writer_vip_cidr: 111 | vip_found = True 112 | break 113 | 114 | return vip_found 115 | -------------------------------------------------------------------------------- /mha_helper/unix_daemon.py: -------------------------------------------------------------------------------- 1 | # (c) 2015, Ovais Tariq 2 | # 3 | # This file is part of mha_helper 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | import sys 19 | import os 20 | import time 21 | import atexit 22 | import signal 23 | 24 | 25 | class UnixDaemon(object): 26 | """ 27 | A generic daemon class. 28 | 29 | Usage: subclass the Unix_daemon class and override the run() method 30 | """ 31 | def __init__(self, pid_file, stdin=os.devnull, stdout=os.devnull, stderr=os.devnull, home_dir='/', umask=0): 32 | self.pid_file = pid_file 33 | self.stdin = stdin 34 | self.stdout = stdout 35 | self.stderr = stderr 36 | self.home_dir = home_dir 37 | self.umask = umask 38 | 39 | def daemonize(self): 40 | """ 41 | do the UNIX double-fork magic, see Stevens' "Advanced 42 | Programming in the UNIX Environment" for details (ISBN 0201563177) 43 | http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 44 | """ 45 | try: 46 | pid = os.fork() 47 | if pid > 0: 48 | # exit first parent 49 | sys.exit(0) 50 | except OSError, e: 51 | sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror)) 52 | sys.exit(1) 53 | 54 | # decouple from parent environment 55 | os.chdir(self.home_dir) 56 | os.setsid() 57 | os.umask(self.umask) 58 | 59 | # do second fork 60 | try: 61 | pid = os.fork() 62 | if pid > 0: 63 | # exit from second parent 64 | sys.exit(0) 65 | except OSError, e: 66 | sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror)) 67 | sys.exit(1) 68 | 69 | # redirect standard file descriptors 70 | sys.stdout.flush() 71 | sys.stderr.flush() 72 | si = file(self.stdin, 'r') 73 | so = file(self.stdout, 'a+') 74 | se = file(self.stderr, 'a+', 0) 75 | os.dup2(si.fileno(), sys.stdin.fileno()) 76 | os.dup2(so.fileno(), sys.stdout.fileno()) 77 | os.dup2(se.fileno(), sys.stderr.fileno()) 78 | 79 | # write pidfile 80 | atexit.register(self.delete_pid) 81 | pid = str(os.getpid()) 82 | file(self.pid_file, 'w+').write("%s\n" % pid) 83 | 84 | def delete_pid(self): 85 | os.remove(self.pid_file) 86 | 87 | def start(self): 88 | """ 89 | Start the daemon 90 | """ 91 | # Check for a pidfile to see if the daemon already runs 92 | try: 93 | pf = file(self.pid_file, 'r') 94 | pid = int(pf.read().strip()) 95 | pf.close() 96 | except IOError: 97 | pid = None 98 | 99 | if pid: 100 | message = "PID file %s already exist. Daemon already running?\n" 101 | sys.stderr.write(message % self.pid_file) 102 | sys.exit(1) 103 | 104 | # Start the daemon 105 | self.daemonize() 106 | self.run() 107 | 108 | def stop(self): 109 | """ 110 | Stop the daemon 111 | """ 112 | # Get the pid from the pidfile 113 | try: 114 | pf = file(self.pid_file, 'r') 115 | pid = int(pf.read().strip()) 116 | pf.close() 117 | except IOError: 118 | pid = None 119 | 120 | if not pid: 121 | message = "PID file %s does not exist. Daemon not running?\n" 122 | sys.stderr.write(message % self.pid_file) 123 | return # not an error in a restart 124 | 125 | # Try killing the daemon process 126 | try: 127 | while 1: 128 | os.kill(pid, signal.SIGTERM) 129 | time.sleep(0.1) 130 | except OSError, err: 131 | err = str(err) 132 | if err.find("No such process") > 0: 133 | if os.path.exists(self.pid_file): 134 | os.remove(self.pid_file) 135 | else: 136 | print str(err) 137 | sys.exit(1) 138 | 139 | def restart(self): 140 | """ 141 | Restart the daemon 142 | """ 143 | self.stop() 144 | self.start() 145 | 146 | def run(self): 147 | """ 148 | You should override this method when you subclass Daemon. It will be called after the process has been 149 | daemonized by start() or restart(). 150 | """ 151 | 152 | def status(self): 153 | """ 154 | You should override this method when you subclass Daemon. 155 | """ 156 | 157 | -------------------------------------------------------------------------------- /tests/unit/test_config_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import os 6 | from mha_helper.config_helper import ConfigHelper 7 | 8 | __author__ = 'ovais.tariq' 9 | __project__ = 'mha_helper' 10 | 11 | 12 | class TestConfigHelper(unittest.TestCase): 13 | def setUp(self): 14 | self.root_directory = os.path.dirname(os.path.realpath(__file__)) 15 | 16 | def test_load_config_with_good_config(self): 17 | # Test with the correct config 18 | mha_helper_config_dir = os.path.join(self.root_directory, 'conf', 'good') 19 | if not mha_helper_config_dir: 20 | self.fail(msg='mha_helper configuration dir not set') 21 | 22 | ConfigHelper.MHA_HELPER_CONFIG_DIR = mha_helper_config_dir 23 | self.assertTrue(ConfigHelper.load_config()) 24 | 25 | def test_load_config_with_bad_config(self): 26 | # Test with bad config 27 | mha_helper_config_dir = os.path.join(self.root_directory, 'conf', 'bad') 28 | if not mha_helper_config_dir: 29 | self.fail(msg='mha_helper configuration dir not set') 30 | 31 | ConfigHelper.MHA_HELPER_CONFIG_DIR = mha_helper_config_dir 32 | self.assertFalse(ConfigHelper.load_config()) 33 | 34 | def test_validate_config_value(self): 35 | # Validate valid values 36 | self.assertTrue(ConfigHelper.validate_config_value('writer_vip_cidr', '192.168.10.1/22')) 37 | self.assertTrue(ConfigHelper.validate_config_value('vip_type', 'aws')) 38 | self.assertTrue(ConfigHelper.validate_config_value('report_email', 'me@ovaistariq.net')) 39 | self.assertTrue(ConfigHelper.validate_config_value('smtp_host', 'smtp.sj.lithium.com')) 40 | self.assertTrue(ConfigHelper.validate_config_value('requires_sudo', 'no')) 41 | self.assertTrue(ConfigHelper.validate_config_value('cluster_interface', 'eth0')) 42 | 43 | # Validate invalid values 44 | self.assertFalse(ConfigHelper.validate_config_value('writer_vip_cidr', '192.168.1/22')) 45 | self.assertFalse(ConfigHelper.validate_config_value('vip_type', 'foo')) 46 | self.assertFalse(ConfigHelper.validate_config_value('report_email', 'foo@bar')) 47 | self.assertFalse(ConfigHelper.validate_config_value('smtp_host', 'smtp')) 48 | self.assertFalse(ConfigHelper.validate_config_value('requires_sudo', 'bar')) 49 | self.assertFalse(ConfigHelper.validate_config_value('cluster_interface', '')) 50 | 51 | def test_get_writer_vip(self): 52 | self.test_load_config_with_good_config() 53 | 54 | host_config = ConfigHelper('master') 55 | self.assertEqual(host_config.get_writer_vip(), '192.168.30.100') 56 | 57 | host_config = ConfigHelper('db13') 58 | self.assertEqual(host_config.get_writer_vip(), '192.168.10.155') 59 | 60 | def test_get_writer_vip_cidr(self): 61 | self.test_load_config_with_good_config() 62 | 63 | host_config = ConfigHelper('master') 64 | self.assertEqual(host_config.get_writer_vip_cidr(), '192.168.30.100/24') 65 | 66 | host_config = ConfigHelper('db13') 67 | self.assertEqual(host_config.get_writer_vip_cidr(), '192.168.10.155/24') 68 | 69 | def test_get_vip_type(self): 70 | self.test_load_config_with_good_config() 71 | 72 | host_config = ConfigHelper('node1') 73 | self.assertEqual(host_config.get_vip_type(), 'metal') 74 | 75 | host_config = ConfigHelper('db11') 76 | self.assertEqual(host_config.get_vip_type(), 'aws') 77 | 78 | def test_get_manage_vip(self): 79 | self.test_load_config_with_good_config() 80 | 81 | host_config = ConfigHelper('node1') 82 | self.assertTrue(host_config.get_manage_vip()) 83 | 84 | host_config = ConfigHelper('db12') 85 | self.assertFalse(host_config.get_manage_vip()) 86 | 87 | def test_get_report_email(self): 88 | self.test_load_config_with_good_config() 89 | 90 | host_config = ConfigHelper('node2') 91 | self.assertEqual(host_config.get_report_email(), 'ovaistariq@gmail.com') 92 | 93 | host_config = ConfigHelper('db12') 94 | self.assertEqual(host_config.get_report_email(), 'notify@host-db12.com') 95 | 96 | def test_get_smtp_host(self): 97 | self.test_load_config_with_good_config() 98 | 99 | host_config = ConfigHelper('node2') 100 | self.assertEqual(host_config.get_smtp_host(), 'localhost') 101 | 102 | host_config = ConfigHelper('db12') 103 | self.assertEqual(host_config.get_smtp_host(), 'smtp.sj.lithium.com') 104 | 105 | def test_get_requires_sudo(self): 106 | self.test_load_config_with_good_config() 107 | 108 | host_config = ConfigHelper('master') 109 | self.assertEqual(host_config.get_requires_sudo(), True) 110 | 111 | host_config = ConfigHelper('db12') 112 | self.assertEqual(host_config.get_requires_sudo(), False) 113 | 114 | def test_get_cluster_interface(self): 115 | self.test_load_config_with_good_config() 116 | 117 | host_config = ConfigHelper('node2') 118 | self.assertEqual(host_config.get_cluster_interface(), 'eth1') 119 | 120 | host_config = ConfigHelper('db10') 121 | self.assertEqual(host_config.get_cluster_interface(), 'eth10') 122 | 123 | if __name__ == '__main__': 124 | unittest.main() 125 | -------------------------------------------------------------------------------- /mha_helper/ssh_helper.py: -------------------------------------------------------------------------------- 1 | # (c) 2015, Ovais Tariq 2 | # 3 | # This file is part of mha_helper 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | from __future__ import print_function 19 | import paramiko 20 | import socket 21 | import os 22 | import pwd 23 | import optparse 24 | import shlex 25 | 26 | 27 | class SSHHelper(object): 28 | SSH_CMD_TIMEOUT = 30 29 | 30 | def __init__(self, host, host_ip=None, ssh_user=None, ssh_port=None, ssh_options=None): 31 | if host_ip is None: 32 | host_ip = host 33 | 34 | self._host = host 35 | self._host_ip = host_ip 36 | self._ssh_user = ssh_user 37 | self._ssh_port = ssh_port 38 | self._ssh_options = ssh_options 39 | 40 | # Disable keep alive for SSH connection by default 41 | self._keep_alive_interval_seconds = 0 42 | 43 | # Timeout for initiating the connection, as well as the number of retries 44 | self._ssh_connect_timeout_seconds = 30 45 | self._ssh_connect_retries = 1 46 | 47 | self._ssh_client = None 48 | 49 | def make_ssh_connection(self): 50 | if self._ssh_client is not None: 51 | return True 52 | 53 | self._ssh_client = paramiko.SSHClient() 54 | 55 | # We first load the SSH options from user's SSH config file 56 | ssh_options = self._get_options_from_ssh_config() 57 | 58 | # Load the host keys and set the default policy, later on we may do strict host key checking 59 | self._ssh_client.load_system_host_keys() 60 | self._ssh_client.set_missing_host_key_policy(paramiko.WarningPolicy()) 61 | 62 | # Parse SSH options passed in by MHA to the Helper 63 | # Any options passed over the command-line overrides the options that are set in the user's SSH config file 64 | if self._ssh_options: 65 | parser = optparse.OptionParser() 66 | parser.add_option('-o', '--additional_options', action='append', type='string') 67 | parser.add_option('-i', '--key_file_path', type='string') 68 | 69 | (options, args) = parser.parse_args(shlex.split(self._ssh_options)) 70 | 71 | if options.key_file_path is not None: 72 | ssh_options['key_filename'] = options.key_file_path 73 | 74 | # Handle additional options that are passed to ssh via the '-o' flag 75 | # Some of the common options that are ignored are 'PasswordAuthentication' and 'BatchMode' 76 | for ssh_opt in options.additional_options: 77 | (opt_name, opt_value) = ssh_opt.split('=') 78 | 79 | if opt_name == 'StrictHostKeyChecking': 80 | if opt_value == 'no': 81 | self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 82 | else: 83 | self._ssh_client.set_missing_host_key_policy(paramiko.RejectPolicy()) 84 | 85 | if opt_name == 'ServerAliveInterval': 86 | self._keep_alive_interval_seconds = int(opt_value) 87 | 88 | if opt_name == 'ConnectionAttempts': 89 | self._ssh_connect_retries = int(opt_value) 90 | 91 | # Calculate the SSH connection timeout 92 | ssh_options['timeout'] = self._ssh_connect_timeout_seconds * self._ssh_connect_retries 93 | 94 | # Make the SSH connection 95 | try: 96 | print("Connecting to '%s'@'%s'" % (self._ssh_user, self._host)) 97 | self._ssh_client.connect(**ssh_options) 98 | except paramiko.SSHException as e: 99 | print("Error connecting to '%s': %s" % (self._host, repr(e))) 100 | return False 101 | except socket.error as e: 102 | print("Failed to connect to '%s': %s" % (self._host, repr(e))) 103 | return False 104 | 105 | # If the user asked for keep alive then we configure it here after the SSH connection has been made 106 | transport = self._ssh_client.get_transport() 107 | transport.set_keepalive(self._keep_alive_interval_seconds) 108 | 109 | return True 110 | 111 | def execute_ssh_command(self, cmd): 112 | cmd_exec_status = True 113 | stdout_lines = [] 114 | stderr_lines = [] 115 | 116 | try: 117 | print("Executing command on '%s': %s" % (self._host, cmd)) 118 | stdin, stdout, stderr = self._ssh_client.exec_command(cmd, get_pty=True, 119 | timeout=SSHHelper.SSH_CMD_TIMEOUT) 120 | 121 | stdout_lines = stdout.readlines() 122 | stderr_lines = stderr.readlines() 123 | 124 | if stdout.channel.recv_exit_status() != 0: 125 | raise paramiko.SSHException() 126 | 127 | except paramiko.SSHException as e: 128 | print("Failed to execute the command on '%s': %s" % (self._host, str(e))) 129 | if len(stderr_lines) > 0: 130 | print("Error reported by %s: %s" % (self._host, "\n".join(stderr_lines))) 131 | 132 | cmd_exec_status = False 133 | 134 | return cmd_exec_status, stdout_lines 135 | 136 | def _get_options_from_ssh_config(self): 137 | # Merge any configuration present in ssh-config with the ones passed on the command-line 138 | ssh_config = paramiko.SSHConfig() 139 | user_config_file = os.path.expanduser("~/.ssh/config") 140 | if os.path.exists(user_config_file): 141 | f = open(user_config_file) 142 | ssh_config.parse(f) 143 | f.close() 144 | 145 | user_config = ssh_config.lookup(self._host_ip) 146 | 147 | # If SSH port is not passed by the user then lookup in ssh-config, if port is configured there then use it, 148 | # otherwise use the default ssh port '22' 149 | if self._ssh_port is None or self._ssh_port < 1: 150 | if 'port' in user_config: 151 | self._ssh_port = user_config['port'] 152 | else: 153 | self._ssh_port = 22 154 | else: 155 | self._ssh_port = int(self._ssh_port) 156 | 157 | # If SSH username is not passed by the user then lookup in ssh-config, if username is configured there then 158 | # use it, otherwise use the current system user 159 | if self._ssh_user is None: 160 | if 'username' in user_config: 161 | self._ssh_user = user_config['username'] 162 | else: 163 | self._ssh_user = pwd.getpwuid(os.getuid())[0] 164 | 165 | cfg = dict(hostname=self._host_ip, username=self._ssh_user, port=self._ssh_port) 166 | 167 | if 'identityfile' in user_config: 168 | cfg['key_filename'] = user_config['identityfile'] 169 | 170 | return cfg 171 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/mha_helper.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/mha_helper.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/mha_helper" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/mha_helper" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\mha_helper.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\mha_helper.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /support-files/mha_manager_daemon: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # (c) 2015, Ovais Tariq 4 | # 5 | # This file is part of mha_helper 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | 20 | import sys 21 | import time 22 | import subprocess 23 | import ConfigParser 24 | import os.path 25 | from optparse import OptionParser 26 | 27 | sys.path.append("/usr/local/mha-helper/scripts/") 28 | from lib.unix_daemon import Unix_daemon 29 | 30 | class MHA_manager_daemon(Unix_daemon): 31 | MASTERHA_CHECK_STATUS = "/usr/bin/masterha_check_status" 32 | MASTERHA_MANAGER = "/usr/bin/masterha_manager" 33 | MASTERHA_STOP = "/usr/bin/masterha_stop" 34 | 35 | # masterha_manager states 36 | STATE_PING_OK = 0 37 | STATE_UNKONWN = 1 38 | STATE_NOT_RUNNING = 2 39 | STATE_PARTIALLY_RUNNING = 3 40 | STATE_INITIALIZING_MONITOR = 10 41 | STATE_CONFIG_ERROR = 31 42 | STATE_FAILOVER_RUNNING = 50 43 | STATE_FAILOVER_ERROR = 51 44 | 45 | def __init__(self, conf_path): 46 | self._conf_path = conf_path 47 | 48 | if os.path.exists(self._conf_path) == False: 49 | raise ValueError("Configuration file could not be found") 50 | 51 | self._config = ConfigParser.RawConfigParser() 52 | self._config.read(conf_path) 53 | 54 | self._pidfile = self.get_manager_pidfile() 55 | if self._pidfile == False: 56 | raise ValueError("Problems generating pid path") 57 | 58 | self._workdir = self.get_manager_workdir() 59 | if self._workdir == False: 60 | raise ValueError("Error determining manager_workdir from config") 61 | 62 | self._log = self.get_manager_log() 63 | if self._log == False: 64 | raise ValueError("Error determining manager_log from config") 65 | 66 | super(MHA_manager_daemon, self).__init__(pidfile=self._pidfile, 67 | stdout=self._log, stderr=self._log, home_dir=self._workdir) 68 | 69 | def run(self): 70 | while True: 71 | manager_state = self.check_manager_state() 72 | sleep_seconds = 1 73 | 74 | if manager_state == MHA_manager_daemon.STATE_NOT_RUNNING: 75 | self.start_manager() 76 | 77 | if manager_state == MHA_manager_daemon.STATE_PARTIALLY_RUNNING: 78 | self.restart_manager() 79 | 80 | if manager_state == MHA_manager_daemon.STATE_INITIALIZING_MONITOR: 81 | sleep_seconds = 5 82 | 83 | time.sleep(sleep_seconds) 84 | 85 | def stop(self): 86 | super(MHA_manager_daemon, self).stop() 87 | self.stop_manager() 88 | 89 | def status(self): 90 | cmd = [MHA_manager_daemon.MASTERHA_CHECK_STATUS, 91 | "--conf=%s" % self._conf_path] 92 | proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, 93 | stderr=subprocess.STDOUT) 94 | (stdout_data, stderr_data) = proc.communicate() 95 | 96 | print stdout_data 97 | return True 98 | 99 | def check_manager_state(self): 100 | cmd = [MHA_manager_daemon.MASTERHA_CHECK_STATUS, 101 | "--conf=%s" % self._conf_path] 102 | proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, 103 | stderr=subprocess.STDOUT) 104 | (stdout, stderr) = proc.communicate() 105 | 106 | return proc.returncode 107 | 108 | def start_manager(self): 109 | cmd = [MHA_manager_daemon.MASTERHA_MANAGER, 110 | "--conf=%s" % self._conf_path, 111 | "--wait_on_monitor_error=60", 112 | "--ignore_fail_on_start", 113 | "--last_failover_minute=60", 114 | "--wait_on_failover_error=60"] 115 | 116 | print "[INFO] Starting masterha_manager: %s" % " ".join(cmd) 117 | 118 | subprocess.Popen(cmd) 119 | return True 120 | 121 | def stop_manager(self): 122 | while True: 123 | manager_state = self.check_manager_state() 124 | 125 | # We try to stop the manager only when its not doing a failover 126 | # Otherwise we wait for the failover to finish 127 | if manager_state != MHA_manager_daemon.STATE_FAILOVER_RUNNING: 128 | cmd = [MHA_manager_daemon.MASTERHA_STOP, 129 | "--conf=%s" % self._conf_path] 130 | 131 | print "[INFO] Stopping masterha_manager: %s" % " ".join(cmd) 132 | proc = subprocess.Popen(cmd) 133 | 134 | # Give manager 30 seconds to terminate 135 | check_seconds=1 136 | while check_seconds <= 30: 137 | return_code = proc.poll() 138 | if return_code is not None: # masterha_stop finished 139 | print "[INFO] masterha_manager stopped" 140 | return return_code 141 | 142 | check_seconds += 1 143 | time.sleep(1) 144 | 145 | # Manager could not be stopped in 30 seconds so we abort 146 | cmd = [MHA_manager_daemon.MASTERHA_STOP, 147 | "--conf=%s" % self._conf_path, "--abort"] 148 | 149 | print "[ERROR] masterha_manager could not be stopped" 150 | print "[ERROR] Sending abort command: %s" % " ".join(cmd) 151 | 152 | return subprocess.call(cmd) 153 | 154 | # We came here meaning that manager is doing a failover 155 | print "[INFO] Waiting for masterha_manager to complete failover" 156 | time.sleep(1) 157 | 158 | def restart_manager(self): 159 | self.stop_manager() 160 | self.start_manager() 161 | 162 | def get_manager_pidfile(self): 163 | paths = os.path.splitext(self._conf_path) 164 | if len(paths) < 2: 165 | return False 166 | 167 | cluster_name = os.path.basename(paths[0]) 168 | return "/var/run/mha_manager_daemon_" + cluster_name + ".pid" 169 | 170 | def get_manager_workdir(self): 171 | return self.get_param_value(param_name='manager_workdir') 172 | 173 | def get_manager_log(self): 174 | return self.get_param_value(param_name='manager_log') 175 | 176 | def get_param_value(self, param_name): 177 | if self._config.has_section('server default') == False: 178 | return False 179 | 180 | param_value = False 181 | 182 | if self._config.has_option('server default', param_name): 183 | param_value = self._config.get('server default', param_name) 184 | 185 | return param_value 186 | 187 | def main(): 188 | usage = "usage: %prog --conf=CONF start|stop|restart|status" 189 | parser = OptionParser(usage) 190 | parser.add_option('--conf', type='string') 191 | (options, args) = parser.parse_args() 192 | 193 | if len(args) == 1 and options.conf is not None: 194 | try: 195 | manager_daemon = MHA_manager_daemon(conf_path=options.conf) 196 | except ValueError as e: 197 | parser.error(str(e)) 198 | sys.exit(3) 199 | 200 | if args[0] == "start": 201 | print "Starting daemon" 202 | manager_daemon.start() 203 | elif args[0] == "stop": 204 | print "Stopping daemon" 205 | manager_daemon.stop() 206 | elif args[0] == "restart": 207 | print "Restarting daemon" 208 | manager_daemon.restart() 209 | elif args[0] == "status": 210 | manager_daemon.status() 211 | else: 212 | parser.error("Unknown command") 213 | sys.exit(2) 214 | 215 | sys.exit(0) 216 | else: 217 | parser.error("Incorrect number of arguments") 218 | sys.exit(2) 219 | 220 | if __name__ == "__main__": 221 | main() 222 | -------------------------------------------------------------------------------- /tests/unit/test_master_ip_online_failover_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import os 6 | import subprocess 7 | from mha_helper.config_helper import ConfigHelper 8 | from mha_helper.mysql_helper import MySQLHelper 9 | from mha_helper.vip_metal_helper import VIPMetalHelper 10 | 11 | __author__ = 'ovais.tariq' 12 | __project__ = 'mha_helper' 13 | 14 | 15 | class TestMasterIPOnlineFailoverHelper(unittest.TestCase): 16 | def setUp(self): 17 | self.root_directory = os.path.dirname(os.path.realpath(__file__)) 18 | self.failover_script_path = os.path.realpath( 19 | "{0}/../scripts/master_ip_online_failover_helper".format(self.root_directory)) 20 | 21 | self.mha_helper_config_dir = os.path.join(self.root_directory, 'conf', 'good') 22 | if not self.mha_helper_config_dir: 23 | self.fail(msg='mha_helper configuration dir not set') 24 | 25 | ConfigHelper.MHA_HELPER_CONFIG_DIR = self.mha_helper_config_dir 26 | if not ConfigHelper.load_config(): 27 | self.fail(msg='Could not load mha_helper configuration from %s' % self.mha_helper_config_dir) 28 | 29 | self.orig_master_host = os.getenv('ORIG_MASTER_HOST') 30 | self.orig_master_ip = os.getenv('ORIG_MASTER_IP') 31 | self.orig_master_port = os.getenv('ORIG_MASTER_PORT') 32 | self.orig_master_user = os.getenv('ORIG_MASTER_USER') 33 | self.orig_master_password = os.getenv('ORIG_MASTER_PASSWORD') 34 | self.orig_master_ssh_host = os.getenv('ORIG_MASTER_SSH_HOST') 35 | self.orig_master_ssh_ip = os.getenv('ORIG_MASTER_SSH_IP') 36 | self.orig_master_ssh_port = os.getenv('ORIG_MASTER_SSH_PORT') 37 | self.orig_master_ssh_user = os.getenv('ORIG_MASTER_SSH_USER') 38 | self.new_master_host = os.getenv('NEW_MASTER_HOST') 39 | self.new_master_ip = os.getenv('NEW_MASTER_IP') 40 | self.new_master_port = os.getenv('NEW_MASTER_PORT') 41 | self.new_master_user = os.getenv('NEW_MASTER_USER') 42 | self.new_master_password = os.getenv('NEW_MASTER_PASSWORD') 43 | self.new_master_ssh_user = os.getenv('NEW_MASTER_SSH_USER') 44 | self.new_master_ssh_host = os.getenv('NEW_MASTER_SSH_HOST') 45 | self.new_master_ssh_ip = os.getenv('NEW_MASTER_SSH_IP') 46 | self.new_master_ssh_port = os.getenv('NEW_MASTER_SSH_PORT') 47 | 48 | def tearDown(self): 49 | # We remove the VIP just to have a clean slate at the end of the test 50 | VIPMetalHelper(self.orig_master_host, self.orig_master_ip, self.orig_master_ssh_user, 51 | self.orig_master_ssh_port).remove_vip() 52 | VIPMetalHelper(self.new_master_host, self.new_master_ip, self.new_master_ssh_user, 53 | self.new_master_ssh_port).remove_vip() 54 | 55 | # We unset read only on original master to have a clean slate at the end of the test 56 | orig_mysql = MySQLHelper(self.orig_master_ip, self.orig_master_port, self.orig_master_user, 57 | self.orig_master_password) 58 | orig_mysql.connect() 59 | orig_mysql.unset_read_only() 60 | 61 | # We set read only on new master to have a clean slate at the end of the test 62 | new_mysql = MySQLHelper(self.new_master_ip, self.new_master_port, self.new_master_user, 63 | self.new_master_password) 64 | new_mysql.connect() 65 | new_mysql.set_read_only() 66 | 67 | def test_main(self): 68 | # We setup the VIP first on the original master as it is assumed that the master already has the VIP attached 69 | # to it before we enter the stop command 70 | VIPMetalHelper(self.orig_master_host, self.orig_master_ip, self.orig_master_ssh_user, 71 | self.orig_master_ssh_port).assign_vip() 72 | 73 | print("\n- Testing rejects_update stage by executing stop command") 74 | cmd = """{0} --command=stop --orig_master_host={1} --orig_master_ip={2} --orig_master_port={3} \ 75 | --orig_master_user={4} --orig_master_password={5} --orig_master_ssh_host={6} --orig_master_ssh_ip={7} \ 76 | --orig_master_ssh_port={8} --orig_master_ssh_user={9} --new_master_host={10} --new_master_ip={11} \ 77 | --new_master_port={12} --new_master_user={13} --new_master_password={14} --new_master_ssh_user={15} \ 78 | --new_master_ssh_host={16} --new_master_ssh_ip={17} --new_master_ssh_port={18} \ 79 | --test_config_path={19}""".format(self.failover_script_path, self.orig_master_host, self.orig_master_ip, 80 | self.orig_master_port, self.orig_master_user, self.orig_master_password, 81 | self.orig_master_ssh_host, self.orig_master_ssh_ip, 82 | self.orig_master_ssh_port, self.orig_master_ssh_user, self.new_master_host, 83 | self.new_master_ip, self.new_master_port, self.new_master_user, 84 | self.new_master_password, self.new_master_ssh_user, 85 | self.new_master_ssh_host, self.new_master_ssh_ip, self.new_master_ssh_port, 86 | self.mha_helper_config_dir) 87 | 88 | proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 89 | stdout, stderr = proc.communicate() 90 | print("STDOUT: \n%s" % stdout) 91 | print("STDERR: \n%s" % stderr) 92 | 93 | self.assertEqual(proc.returncode, 0) 94 | 95 | # Once the STOP command completes successfully, we would have read_only enabled on original master and we 96 | # would have the VIP removed from the original master, so we are going to confirm that separately here 97 | orig_mysql = MySQLHelper(self.orig_master_ip, self.orig_master_port, self.orig_master_user, 98 | self.orig_master_password) 99 | orig_mysql.connect() 100 | self.assertTrue(orig_mysql.is_read_only()) 101 | 102 | self.assertFalse(VIPMetalHelper(self.orig_master_host, self.orig_master_ip, self.orig_master_ssh_user, 103 | self.orig_master_ssh_port).has_vip()) 104 | 105 | print("\n- Testing 'Allow write access on the new master' stage by executing start command") 106 | cmd = """{0} --command=start --orig_master_host={1} --orig_master_ip={2} --orig_master_port={3} \ 107 | --orig_master_user={4} --orig_master_password={5} --orig_master_ssh_host={6} --orig_master_ssh_ip={7} \ 108 | --orig_master_ssh_port={8} --orig_master_ssh_user={9} --new_master_host={10} --new_master_ip={11} \ 109 | --new_master_port={12} --new_master_user={13} --new_master_password={14} --new_master_ssh_user={15} \ 110 | --new_master_ssh_host={16} --new_master_ssh_ip={17} --new_master_ssh_port={18} \ 111 | --test_config_path={19}""".format(self.failover_script_path, self.orig_master_host, self.orig_master_ip, 112 | self.orig_master_port, self.orig_master_user, self.orig_master_password, 113 | self.orig_master_ssh_host, self.orig_master_ssh_ip, 114 | self.orig_master_ssh_port, self.orig_master_ssh_user, self.new_master_host, 115 | self.new_master_ip, self.new_master_port, self.new_master_user, 116 | self.new_master_password, self.new_master_ssh_user, 117 | self.new_master_ssh_host, self.new_master_ssh_ip, self.new_master_ssh_port, 118 | self.mha_helper_config_dir) 119 | 120 | proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 121 | stdout, stderr = proc.communicate() 122 | print("STDOUT: \n%s" % stdout) 123 | print("STDERR: \n%s" % stderr) 124 | 125 | self.assertEqual(proc.returncode, 0) 126 | 127 | # Once the START command completes successfully, we would have read_only disabled on the new master and we 128 | # would have the VIP assigned to the new master, so we are going to confirm that separately here 129 | new_mysql = MySQLHelper(self.new_master_ip, self.new_master_port, self.new_master_user, 130 | self.new_master_password) 131 | 132 | new_mysql.connect() 133 | self.assertFalse(new_mysql.is_read_only()) 134 | 135 | self.assertTrue(VIPMetalHelper(self.new_master_host, self.new_master_ip, self.new_master_ssh_user, 136 | self.new_master_ssh_port).has_vip()) 137 | 138 | if __name__ == '__main__': 139 | unittest.main() 140 | -------------------------------------------------------------------------------- /mha_helper/mysql_helper.py: -------------------------------------------------------------------------------- 1 | # (c) 2015, Ovais Tariq 2 | # 3 | # This file is part of mha_helper 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | from __future__ import print_function 19 | import pymysql 20 | import re 21 | 22 | 23 | class MySQLHelper(object): 24 | @staticmethod 25 | def is_read_only_query(sql): 26 | if re.match('^select', sql, re.I) is not None: 27 | return True 28 | 29 | if re.match('^show', sql, re.I) is not None: 30 | return True 31 | 32 | return False 33 | 34 | def __init__(self, host, port, user, password): 35 | if port is None or port < 1: 36 | port = 3306 37 | 38 | self._host = host 39 | self._port = int(port) 40 | self._user = user 41 | self._password = password 42 | self._connection = None 43 | 44 | def connect(self): 45 | try: 46 | self._connection = pymysql.connect(host=self._host, port=self._port, user=self._user, passwd=self._password) 47 | except pymysql.Error as e: 48 | print("Error %d: %s" % (e.args[0], e.args[1])) 49 | return False 50 | 51 | return True 52 | 53 | def disconnect(self): 54 | if self._connection is not None: 55 | self._connection.close() 56 | 57 | def get_version(self): 58 | cursor = self._connection.cursor() 59 | try: 60 | cursor.execute("SELECT VERSION()") 61 | row = cursor.fetchone() 62 | except pymysql.Error as e: 63 | print("Error %d: %s" % (e.args[0], e.args[1])) 64 | return False 65 | finally: 66 | cursor.close() 67 | 68 | return row[0] 69 | 70 | def get_connection_id(self): 71 | return self._connection.thread_id() 72 | 73 | def get_current_user(self): 74 | return self._user 75 | 76 | def set_read_only(self): 77 | cursor = self._connection.cursor() 78 | try: 79 | cursor.execute("SET GLOBAL read_only = 1") 80 | except pymysql.Error as e: 81 | print("Error %d: %s" % (e.args[0], e.args[1])) 82 | return False 83 | finally: 84 | cursor.close() 85 | 86 | return True 87 | 88 | def unset_read_only(self): 89 | cursor = self._connection.cursor() 90 | try: 91 | cursor.execute("SET GLOBAL read_only = 0") 92 | except pymysql.Error as e: 93 | print("Error %d: %s" % (e.args[0], e.args[1])) 94 | return False 95 | finally: 96 | cursor.close() 97 | 98 | return True 99 | 100 | def is_read_only(self): 101 | cursor = self._connection.cursor() 102 | try: 103 | cursor.execute("SELECT @@read_only") 104 | row = cursor.fetchone() 105 | except pymysql.Error as e: 106 | print("Error %d: %s" % (e.args[0], e.args[1])) 107 | return False 108 | finally: 109 | cursor.close() 110 | 111 | if row[0] == 1: 112 | return True 113 | 114 | return False 115 | 116 | def super_read_only_supported(self): 117 | cursor = self._connection.cursor() 118 | try: 119 | cursor.execute("SHOW VARIABLES LIKE 'super_read_only'") 120 | num_rows = cursor.rowcount 121 | except pymysql.Error as e: 122 | print("Error %d: %s" % (e.args[0], e.args[1])) 123 | return False 124 | finally: 125 | cursor.close() 126 | 127 | if num_rows > 0: 128 | return True 129 | 130 | return False 131 | 132 | def set_super_read_only(self): 133 | cursor = self._connection.cursor() 134 | try: 135 | cursor.execute("SET GLOBAL super_read_only = 1") 136 | except pymysql.Error as e: 137 | print("Error %d: %s" % (e.args[0], e.args[1])) 138 | return False 139 | finally: 140 | cursor.close() 141 | 142 | return True 143 | 144 | def unset_super_read_only(self): 145 | cursor = self._connection.cursor() 146 | try: 147 | cursor.execute("SET GLOBAL super_read_only = 0") 148 | except pymysql.Error as e: 149 | print("Error %d: %s" % (e.args[0], e.args[1])) 150 | return False 151 | finally: 152 | cursor.close() 153 | 154 | return True 155 | 156 | def is_super_read_only(self): 157 | cursor = self._connection.cursor() 158 | try: 159 | cursor.execute("SELECT @@super_read_only") 160 | row = cursor.fetchone() 161 | except pymysql.Error as e: 162 | print("Error %d: %s" % (e.args[0], e.args[1])) 163 | return False 164 | finally: 165 | cursor.close() 166 | 167 | if row[0] == 1: 168 | return True 169 | 170 | return False 171 | 172 | def disable_log_bin(self): 173 | cursor = self._connection.cursor() 174 | try: 175 | cursor.execute("SET SQL_LOG_BIN = 0") 176 | except pymysql.Error as e: 177 | print("Error %d: %s" % (e.args[0], e.args[1])) 178 | return False 179 | finally: 180 | cursor.close() 181 | 182 | return True 183 | 184 | def enable_log_bin(self): 185 | cursor = self._connection.cursor() 186 | try: 187 | cursor.execute("SET SQL_LOG_BIN = 1") 188 | except pymysql.Error as e: 189 | print("Error %d: %s" % (e.args[0], e.args[1])) 190 | return False 191 | finally: 192 | cursor.close() 193 | 194 | return True 195 | 196 | def get_processlist(self): 197 | cursor = self._connection.cursor(pymysql.cursors.DictCursor) 198 | try: 199 | cursor.execute("SHOW PROCESSLIST") 200 | threads = cursor.fetchall() 201 | except pymysql.Error as e: 202 | print("Error %d: %s" % (e.args[0], e.args[1])) 203 | return False 204 | finally: 205 | cursor.close() 206 | 207 | return threads 208 | 209 | def get_all_users(self): 210 | cursor = self._connection.cursor(pymysql.cursors.DictCursor) 211 | try: 212 | cursor.execute("SELECT * from mysql.user") 213 | users = cursor.fetchall() 214 | except pymysql.Error as e: 215 | print("Error %d: %s" % (e.args[0], e.args[1])) 216 | return False 217 | finally: 218 | cursor.close() 219 | 220 | return users 221 | 222 | def get_user_grants(self, username, hostname): 223 | cursor = self._connection.cursor() 224 | grants = [] 225 | try: 226 | cursor.execute("SHOW GRANTS FOR '%s'@'%s'" % (username, hostname)) 227 | for row in cursor.fetchall(): 228 | grants.append(row[0]) 229 | except pymysql.Error as e: 230 | print("Error %d: %s" % (e.args[0], e.args[1])) 231 | return False 232 | finally: 233 | cursor.close() 234 | 235 | return grants 236 | 237 | def revoke_all_privileges(self, user, hostname): 238 | cursor = self._connection.cursor() 239 | try: 240 | cursor.execute(print("REVOKE ALL PRIVILEGES, GRANT OPTION FROM '%s'@'%s'" % (user, hostname))) 241 | except pymysql.Error as e: 242 | print("Error %d: %s" % (e.args[0], e.args[1])) 243 | return False 244 | finally: 245 | cursor.close() 246 | 247 | return True 248 | 249 | def get_slave_status(self): 250 | cursor = self._connection.cursor(pymysql.cursors.DictCursor) 251 | try: 252 | cursor.execute("SHOW SLAVE STATUS") 253 | row = cursor.fetchone() 254 | except pymysql.Error as e: 255 | print("Error %d: %s" % (e.args[0], e.args[1])) 256 | return False 257 | finally: 258 | cursor.close() 259 | 260 | return row 261 | 262 | def kill_connection(self, connection_id): 263 | cursor = self._connection.cursor() 264 | try: 265 | cursor.execute("KILL CONNECTION %d" % connection_id) 266 | except pymysql.Error as e: 267 | print("Error %d: %s" % (e.args[0], e.args[1])) 268 | return False 269 | finally: 270 | cursor.close() 271 | 272 | return True 273 | 274 | def execute_admin_query(self, sql): 275 | cursor = self._connection.cursor() 276 | try: 277 | cursor.execute(sql) 278 | except pymysql.Error as e: 279 | print("Error %d: %s" % (e.args[0], e.args[1])) 280 | return False 281 | finally: 282 | cursor.close() 283 | 284 | return True 285 | -------------------------------------------------------------------------------- /mha_helper/config_helper.py: -------------------------------------------------------------------------------- 1 | # (c) 2015, Ovais Tariq 2 | # 3 | # This file is part of mha_helper 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | from __future__ import print_function 19 | import fnmatch 20 | import os 21 | import socket 22 | import re 23 | import ConfigParser 24 | 25 | 26 | class ConfigHelper(object): 27 | MHA_HELPER_CONFIG_DIR = '/etc/mha-helper' 28 | MHA_HELPER_CONFIG_OPTIONS = ['writer_vip_cidr', 'vip_type', 'report_email', 'smtp_host', 'requires_sudo', 29 | 'super_read_only', 'requires_arping', 'cluster_interface', 'kill_after_timeout'] 30 | VIP_PROVIDER_TYPE_NONE = 'none' 31 | VIP_PROVIDER_TYPE_METAL = 'metal' 32 | VIP_PROVIDER_TYPE_AWS = 'aws' 33 | VIP_PROVIDER_TYPE_OS = 'openstack' 34 | VIP_PROVIDER_TYPES = [VIP_PROVIDER_TYPE_NONE, VIP_PROVIDER_TYPE_METAL, VIP_PROVIDER_TYPE_AWS, VIP_PROVIDER_TYPE_OS] 35 | 36 | # This stores the configuration for every host 37 | host_config = dict() 38 | 39 | @staticmethod 40 | def load_config(): 41 | pattern = '*.conf' 42 | 43 | if not os.path.exists(ConfigHelper.MHA_HELPER_CONFIG_DIR): 44 | return False 45 | 46 | for root, dirs, files in os.walk(ConfigHelper.MHA_HELPER_CONFIG_DIR): 47 | for filename in fnmatch.filter(files, pattern): 48 | config_file_path = os.path.join(ConfigHelper.MHA_HELPER_CONFIG_DIR, filename) 49 | print("Reading config file: %s" % config_file_path) 50 | 51 | config_parser = ConfigParser.RawConfigParser() 52 | config_parser.read(config_file_path) 53 | 54 | # Read the default config values first. The default config values are used when a config option is not 55 | # defined for the specific host 56 | if not config_parser.has_section('default'): 57 | return False 58 | 59 | default_config = dict() 60 | for opt in ConfigHelper.MHA_HELPER_CONFIG_OPTIONS: 61 | opt_value = config_parser.get('default', opt) 62 | if not ConfigHelper.validate_config_value(opt, opt_value): 63 | print("Parsing the option '%s' with value '%s' failed" % (opt, opt_value)) 64 | return False 65 | 66 | default_config[opt] = opt_value 67 | 68 | # Setup host based configs. Initially hosts inherit config from the default section but override them 69 | # within their own sections 70 | for hostname in config_parser.sections(): 71 | ConfigHelper.host_config[hostname] = dict() 72 | 73 | # We read the options from the host section of the config 74 | for opt in ConfigHelper.MHA_HELPER_CONFIG_OPTIONS: 75 | if config_parser.has_option(hostname, opt) and opt != 'writer_vip_cidr' and opt != 'smtp_host': 76 | ConfigHelper.host_config[hostname][opt] = config_parser.get(hostname, opt) 77 | 78 | # We now read the options from the default section and if any option has not been set by the host 79 | # section we set that to what is defined in the default section, writer_vip_cidr is always read from 80 | # the default section because it has to be global for the entire replication cluster 81 | # If the option is not defined in both default and host section, we throw an error 82 | for opt in ConfigHelper.MHA_HELPER_CONFIG_OPTIONS: 83 | if (opt not in ConfigHelper.host_config[hostname] or opt == 'writer_vip_cidr' or 84 | opt == 'smtp_host'): 85 | # If the host section did not define the config option and the default config also does 86 | # not define the config option then we bail out 87 | if opt not in default_config: 88 | print("Missing required option '%s'. The option should either be set in default " 89 | "section or the host section of the config" % opt) 90 | return False 91 | 92 | ConfigHelper.host_config[hostname][opt] = default_config[opt] 93 | 94 | # If no host configuration was found it is still an error as we may be analyzing empty files 95 | if len(ConfigHelper.host_config) < 1: 96 | return False 97 | 98 | return True 99 | 100 | @staticmethod 101 | def validate_config_value(config_key, config_value): 102 | if config_key == 'writer_vip_cidr': 103 | return ConfigHelper.validate_ip_address(config_value) 104 | 105 | if config_key == 'vip_type': 106 | return config_value in ConfigHelper.VIP_PROVIDER_TYPES 107 | 108 | if config_key == 'report_email': 109 | return ConfigHelper.validate_email_address(config_value) 110 | 111 | if config_key == 'smtp_host': 112 | return ConfigHelper.validate_hostname(config_value) 113 | 114 | if config_key == 'kill_after_timeout': 115 | return ConfigHelper.validate_integer(config_value) 116 | 117 | if config_key == 'requires_sudo': 118 | return config_value in ['yes', 'no'] 119 | 120 | if config_key == 'requires_arping': 121 | return config_value in ['yes', 'no'] 122 | 123 | if config_key == 'cluster_interface': 124 | return config_value is not None and len(config_value) > 0 125 | 126 | if config_key == 'super_read_only': 127 | return config_value in ['yes', 'no'] 128 | 129 | @staticmethod 130 | def validate_ip_address(ip_address): 131 | try: 132 | socket.inet_pton(socket.AF_INET, ip_address.split('/')[0]) 133 | except socket.error: 134 | try: 135 | socket.inet_pton(socket.AF_INET6, ip_address.split('/')[0]) 136 | except socket.error: 137 | return False 138 | return True 139 | return True 140 | 141 | @staticmethod 142 | def validate_email_address(email_address): 143 | pattern = '^.+\\@(\\[?)[a-zA-Z0-9\\-\\.]+\\.([a-zA-Z]{2,3}|[0-9]{1,3})(\\]?)$' 144 | return bool(re.match(pattern, email_address)) 145 | 146 | @staticmethod 147 | def validate_integer(potential_integer): 148 | try: 149 | int(potential_integer) 150 | except ValueError: 151 | return False 152 | 153 | return True 154 | 155 | @staticmethod 156 | def validate_hostname(hostname): 157 | if len(hostname) > 255: 158 | return False 159 | 160 | if hostname[-1] == ".": 161 | hostname = hostname[:-1] # strip exactly one dot from the right, if present 162 | 163 | allowed = re.compile("(?!-)[A-Z\d-]{1,63}(? v documentation". 126 | #html_title = None 127 | 128 | # A shorter title for the navigation bar. Default is the same as 129 | # html_title. 130 | #html_short_title = None 131 | 132 | # The name of an image file (relative to this directory) to place at the 133 | # top of the sidebar. 134 | #html_logo = None 135 | 136 | # The name of an image file (within the static path) to use as favicon 137 | # of the docs. This file should be a Windows icon file (.ico) being 138 | # 16x16 or 32x32 pixels large. 139 | #html_favicon = None 140 | 141 | # Add any paths that contain custom static files (such as style sheets) 142 | # here, relative to this directory. They are copied after the builtin 143 | # static files, so a file named "default.css" will overwrite the builtin 144 | # "default.css". 145 | html_static_path = ['_static'] 146 | 147 | # If not '', a 'Last updated on:' timestamp is inserted at every page 148 | # bottom, using the given strftime format. 149 | #html_last_updated_fmt = '%b %d, %Y' 150 | 151 | # If true, SmartyPants will be used to convert quotes and dashes to 152 | # typographically correct entities. 153 | #html_use_smartypants = True 154 | 155 | # Custom sidebar templates, maps document names to template names. 156 | #html_sidebars = {} 157 | 158 | # Additional templates that should be rendered to pages, maps page names 159 | # to template names. 160 | #html_additional_pages = {} 161 | 162 | # If false, no module index is generated. 163 | #html_domain_indices = True 164 | 165 | # If false, no index is generated. 166 | #html_use_index = True 167 | 168 | # If true, the index is split into individual pages for each letter. 169 | #html_split_index = False 170 | 171 | # If true, links to the reST sources are added to the pages. 172 | #html_show_sourcelink = True 173 | 174 | # If true, "Created using Sphinx" is shown in the HTML footer. 175 | # Default is True. 176 | #html_show_sphinx = True 177 | 178 | # If true, "(C) Copyright ..." is shown in the HTML footer. 179 | # Default is True. 180 | #html_show_copyright = True 181 | 182 | # If true, an OpenSearch description file will be output, and all pages 183 | # will contain a tag referring to it. The value of this option 184 | # must be the base URL from which the finished HTML is served. 185 | #html_use_opensearch = '' 186 | 187 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 188 | #html_file_suffix = None 189 | 190 | # Output file base name for HTML help builder. 191 | htmlhelp_basename = 'mha_helperdoc' 192 | 193 | 194 | # -- Options for LaTeX output ------------------------------------------ 195 | 196 | latex_elements = { 197 | # The paper size ('letterpaper' or 'a4paper'). 198 | #'papersize': 'letterpaper', 199 | 200 | # The font size ('10pt', '11pt' or '12pt'). 201 | #'pointsize': '10pt', 202 | 203 | # Additional stuff for the LaTeX preamble. 204 | #'preamble': '', 205 | } 206 | 207 | # Grouping the document tree into LaTeX files. List of tuples 208 | # (source start file, target name, title, author, documentclass 209 | # [howto/manual]). 210 | latex_documents = [ 211 | ('index', 'mha_helper.tex', 212 | u'mha_helper Documentation', 213 | u'Ovais Tariq', 'manual'), 214 | ] 215 | 216 | # The name of an image file (relative to this directory) to place at 217 | # the top of the title page. 218 | #latex_logo = None 219 | 220 | # For "manual" documents, if this is true, then toplevel headings 221 | # are parts, not chapters. 222 | #latex_use_parts = False 223 | 224 | # If true, show page references after internal links. 225 | #latex_show_pagerefs = False 226 | 227 | # If true, show URL addresses after external links. 228 | #latex_show_urls = False 229 | 230 | # Documents to append as an appendix to all manuals. 231 | #latex_appendices = [] 232 | 233 | # If false, no module index is generated. 234 | #latex_domain_indices = True 235 | 236 | 237 | # -- Options for manual page output ------------------------------------ 238 | 239 | # One entry per manual page. List of tuples 240 | # (source start file, name, description, authors, manual section). 241 | man_pages = [ 242 | ('index', 'mha_helper', 243 | u'mha_helper Documentation', 244 | [u'Ovais Tariq'], 1) 245 | ] 246 | 247 | # If true, show URL addresses after external links. 248 | #man_show_urls = False 249 | 250 | 251 | # -- Options for Texinfo output ---------------------------------------- 252 | 253 | # Grouping the document tree into Texinfo files. List of tuples 254 | # (source start file, target name, title, author, 255 | # dir menu entry, description, category) 256 | texinfo_documents = [ 257 | ('index', 'mha_helper', 258 | u'mha_helper Documentation', 259 | u'Ovais Tariq', 260 | 'mha_helper', 261 | 'One line description of project.', 262 | 'Miscellaneous'), 263 | ] 264 | 265 | # Documents to append as an appendix to all manuals. 266 | #texinfo_appendices = [] 267 | 268 | # If false, no module index is generated. 269 | #texinfo_domain_indices = True 270 | 271 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 272 | #texinfo_show_urls = 'footnote' 273 | 274 | # If true, do not generate a @detailmenu in the "Top" node's menu. 275 | #texinfo_no_detailmenu = False 276 | -------------------------------------------------------------------------------- /tests/unit/test_master_ip_hard_failover_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import os 6 | import subprocess 7 | from mha_helper.config_helper import ConfigHelper 8 | from mha_helper.mysql_helper import MySQLHelper 9 | from mha_helper.vip_metal_helper import VIPMetalHelper 10 | 11 | __author__ = 'ovais.tariq' 12 | __project__ = 'mha_helper' 13 | 14 | 15 | class TestMasterIPHardFailoverHelper(unittest.TestCase): 16 | def setUp(self): 17 | self.root_directory = os.path.dirname(os.path.realpath(__file__)) 18 | self.failover_script_path = os.path.realpath( 19 | "{0}/../scripts/master_ip_hard_failover_helper".format(self.root_directory)) 20 | 21 | self.mha_helper_config_dir = os.path.join(self.root_directory, 'conf', 'good') 22 | if not self.mha_helper_config_dir: 23 | self.fail(msg='mha_helper configuration dir not set') 24 | 25 | ConfigHelper.MHA_HELPER_CONFIG_DIR = self.mha_helper_config_dir 26 | if not ConfigHelper.load_config(): 27 | self.fail(msg='Could not load mha_helper configuration from %s' % self.mha_helper_config_dir) 28 | 29 | self.orig_master_host = os.getenv('ORIG_MASTER_HOST') 30 | self.orig_master_ip = os.getenv('ORIG_MASTER_IP') 31 | self.orig_master_port = os.getenv('ORIG_MASTER_PORT') 32 | self.orig_master_user = os.getenv('ORIG_MASTER_USER') 33 | self.orig_master_password = os.getenv('ORIG_MASTER_PASSWORD') 34 | self.orig_master_ssh_host = os.getenv('ORIG_MASTER_SSH_HOST') 35 | self.orig_master_ssh_ip = os.getenv('ORIG_MASTER_SSH_IP') 36 | self.orig_master_ssh_port = os.getenv('ORIG_MASTER_SSH_PORT') 37 | self.orig_master_ssh_user = os.getenv('ORIG_MASTER_SSH_USER') 38 | self.new_master_host = os.getenv('NEW_MASTER_HOST') 39 | self.new_master_ip = os.getenv('NEW_MASTER_IP') 40 | self.new_master_port = os.getenv('NEW_MASTER_PORT') 41 | self.new_master_user = os.getenv('NEW_MASTER_USER') 42 | self.new_master_password = os.getenv('NEW_MASTER_PASSWORD') 43 | self.new_master_ssh_user = os.getenv('NEW_MASTER_SSH_USER') 44 | self.new_master_ssh_host = os.getenv('NEW_MASTER_SSH_HOST') 45 | self.new_master_ssh_ip = os.getenv('NEW_MASTER_SSH_IP') 46 | self.new_master_ssh_port = os.getenv('NEW_MASTER_SSH_PORT') 47 | 48 | def tearDown(self): 49 | # We remove the VIP just to have a clean slate at the end of the test 50 | VIPMetalHelper(self.orig_master_host, self.orig_master_ip, self.orig_master_ssh_user, 51 | self.orig_master_ssh_port).remove_vip() 52 | VIPMetalHelper(self.new_master_host, self.new_master_ip, self.new_master_ssh_user, 53 | self.new_master_ssh_port).remove_vip() 54 | 55 | # We unset read only on original master to have a clean slate at the end of the test 56 | orig_mysql = MySQLHelper(self.orig_master_ip, self.orig_master_port, self.orig_master_user, 57 | self.orig_master_password) 58 | orig_mysql.connect() 59 | orig_mysql.unset_read_only() 60 | 61 | # We set read only on new master to have a clean slate at the end of the test 62 | new_mysql = MySQLHelper(self.new_master_ip, self.new_master_port, self.new_master_user, 63 | self.new_master_password) 64 | new_mysql.connect() 65 | new_mysql.set_read_only() 66 | 67 | def test_no_ssh(self): 68 | print("\n- Testing 'disable write on the current master' stage by executing stop command") 69 | cmd = """{0} --command=stop --orig_master_host={1} --orig_master_ip={2} --orig_master_port={3} \ 70 | --orig_master_ssh_host={4} --orig_master_ssh_ip={5} --orig_master_ssh_port={6} \ 71 | --test_config_path={7}""".format(self.failover_script_path, self.orig_master_host, self.orig_master_ip, 72 | self.orig_master_port, self.orig_master_ssh_host, self.orig_master_ssh_ip, 73 | self.orig_master_ssh_port, self.mha_helper_config_dir) 74 | 75 | proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 76 | stdout, stderr = proc.communicate() 77 | print("STDOUT: \n%s" % stdout) 78 | print("STDERR: \n%s" % stderr) 79 | 80 | self.assertEqual(proc.returncode, 0) 81 | 82 | # Once the STOP command completes successfully, we would have the VIP removed from the original master, 83 | # so we are going to confirm that separately here 84 | self.assertFalse(VIPMetalHelper(self.orig_master_host, self.orig_master_ip, self.orig_master_ssh_user, 85 | self.orig_master_ssh_port).has_vip()) 86 | 87 | print("\n- Testing 'enable writes on the active master' stage by executing start command") 88 | cmd = """{0} --command=start --orig_master_host={1} --orig_master_ip={2} --orig_master_port={3} \ 89 | --orig_master_ssh_host={4} --orig_master_ssh_ip={5} --orig_master_ssh_port={6} --new_master_host={7} \ 90 | --new_master_ip={8} --new_master_port={9} --new_master_user={10} --new_master_password={11} --ssh_user={12} \ 91 | --new_master_ssh_host={13} --new_master_ssh_ip={14} --new_master_ssh_port={15} \ 92 | --test_config_path={16}""".format(self.failover_script_path, self.orig_master_host, self.orig_master_ip, 93 | self.orig_master_port, self.orig_master_ssh_host, self.orig_master_ssh_ip, 94 | self.orig_master_ssh_port, self.new_master_host, self.new_master_ip, 95 | self.new_master_port, self.new_master_user, self.new_master_password, 96 | self.new_master_ssh_user, self.new_master_ssh_host, self.new_master_ssh_ip, 97 | self.new_master_ssh_port, self.mha_helper_config_dir) 98 | 99 | proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 100 | stdout, stderr = proc.communicate() 101 | print("STDOUT: \n%s" % stdout) 102 | print("STDERR: \n%s" % stderr) 103 | 104 | self.assertEqual(proc.returncode, 0) 105 | 106 | # Once the START command completes successfully, we would have read_only disabled on the new master and we 107 | # would have the VIP assigned to the new master, so we are going to confirm that separately here 108 | new_mysql = MySQLHelper(self.new_master_ip, self.new_master_port, self.new_master_user, 109 | self.new_master_password) 110 | 111 | new_mysql.connect() 112 | self.assertFalse(new_mysql.is_read_only()) 113 | 114 | self.assertTrue(VIPMetalHelper(self.new_master_host, self.new_master_ip, self.new_master_ssh_user, 115 | self.new_master_ssh_port).has_vip()) 116 | 117 | def test_has_ssh(self): 118 | # We setup the VIP first on the original master as it is assumed that the master already has the VIP attached 119 | # to it before we enter the stop command 120 | VIPMetalHelper(self.orig_master_host, self.orig_master_ip, self.orig_master_ssh_user, 121 | self.orig_master_ssh_port).assign_vip() 122 | 123 | print("\n- Testing 'disable write on the current master' stage by executing stopssh command") 124 | cmd = """{0} --command=stopssh --orig_master_host={1} --orig_master_ip={2} --orig_master_port={3} \ 125 | --orig_master_ssh_host={4} --orig_master_ssh_ip={5} --orig_master_ssh_port={6} --ssh_user={7} \ 126 | --test_config_path={8}""".format(self.failover_script_path, self.orig_master_host, self.orig_master_ip, 127 | self.orig_master_port, self.orig_master_ssh_host, self.orig_master_ssh_ip, 128 | self.orig_master_ssh_port, self.orig_master_ssh_user, 129 | self.mha_helper_config_dir) 130 | 131 | proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 132 | stdout, stderr = proc.communicate() 133 | print("STDOUT: \n%s" % stdout) 134 | print("STDERR: \n%s" % stderr) 135 | 136 | self.assertEqual(proc.returncode, 0) 137 | 138 | # Once the STOP command completes successfully, we would have the VIP removed from the original master, 139 | # so we are going to confirm that separately here 140 | self.assertFalse(VIPMetalHelper(self.orig_master_host, self.orig_master_ip, self.orig_master_ssh_user, 141 | self.orig_master_ssh_port).has_vip()) 142 | 143 | print("\n- Testing 'enable writes on the active master' stage by executing start command") 144 | cmd = """{0} --command=start --orig_master_host={1} --orig_master_ip={2} --orig_master_port={3} \ 145 | --orig_master_ssh_host={4} --orig_master_ssh_ip={5} --orig_master_ssh_port={6} --new_master_host={7} \ 146 | --new_master_ip={8} --new_master_port={9} --new_master_user={10} --new_master_password={11} --ssh_user={12} \ 147 | --new_master_ssh_host={13} --new_master_ssh_ip={14} --new_master_ssh_port={15} \ 148 | --test_config_path={16}""".format(self.failover_script_path, self.orig_master_host, self.orig_master_ip, 149 | self.orig_master_port, self.orig_master_ssh_host, self.orig_master_ssh_ip, 150 | self.orig_master_ssh_port, self.new_master_host, self.new_master_ip, 151 | self.new_master_port, self.new_master_user, self.new_master_password, 152 | self.new_master_ssh_user, self.new_master_ssh_host, self.new_master_ssh_ip, 153 | self.new_master_ssh_port, self.mha_helper_config_dir) 154 | 155 | proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 156 | stdout, stderr = proc.communicate() 157 | print("STDOUT: \n%s" % stdout) 158 | print("STDERR: \n%s" % stderr) 159 | 160 | self.assertEqual(proc.returncode, 0) 161 | 162 | # Once the START command completes successfully, we would have read_only disabled on the new master and we 163 | # would have the VIP assigned to the new master, so we are going to confirm that separately here 164 | new_mysql = MySQLHelper(self.new_master_ip, self.new_master_port, self.new_master_user, 165 | self.new_master_password) 166 | 167 | new_mysql.connect() 168 | self.assertFalse(new_mysql.is_read_only()) 169 | 170 | self.assertTrue(VIPMetalHelper(self.new_master_host, self.new_master_ip, self.new_master_ssh_user, 171 | self.new_master_ssh_port).has_vip()) 172 | 173 | def test_status(self): 174 | # We setup the VIP first on the original master as it is assumed that the master already has the VIP attached 175 | # to it before we enter the status command 176 | VIPMetalHelper(self.orig_master_host, self.orig_master_ip, self.orig_master_ssh_user, 177 | self.orig_master_ssh_port).assign_vip() 178 | 179 | print("\n- Testing the status command") 180 | cmd = """{0} --command=status --orig_master_host={1} --orig_master_ip={2} --orig_master_port={3} \ 181 | --orig_master_ssh_host={4} --orig_master_ssh_ip={5} --orig_master_ssh_port={6} --ssh_user={7} \ 182 | --test_config_path={8}""".format(self.failover_script_path, self.orig_master_host, self.orig_master_ip, 183 | self.orig_master_port, self.orig_master_ssh_host, self.orig_master_ssh_ip, 184 | self.orig_master_ssh_port, self.orig_master_ssh_user, 185 | self.mha_helper_config_dir) 186 | 187 | proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 188 | stdout, stderr = proc.communicate() 189 | print("STDOUT: \n%s" % stdout) 190 | print("STDERR: \n%s" % stderr) 191 | 192 | self.assertEqual(proc.returncode, 0) 193 | 194 | # Once the STOP command completes successfully, we would have the VIP still attached on the original master, 195 | # so we are going to confirm that separately here 196 | self.assertTrue(VIPMetalHelper(self.orig_master_host, self.orig_master_ip, self.orig_master_ssh_user, 197 | self.orig_master_ssh_port).has_vip()) 198 | 199 | if __name__ == '__main__': 200 | unittest.main() 201 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | MHA Helper 3 | ========== 4 | 5 | .. image:: https://badges.gitter.im/Join%20Chat.svg 6 | :alt: Join the chat at https://gitter.im/ovaistariq/mha-helper 7 | :target: https://gitter.im/ovaistariq/mha-helper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge 8 | 9 | .. image:: https://img.shields.io/pypi/v/mha_helper.svg?style=flat-square 10 | :alt: Latest version released on PyPi 11 | :target: https://pypi.python.org/pypi/mha_helper 12 | 13 | 14 | MHA helper is a Python module that supplements in doing proper failover using MHA_. MHA is responsible for executing the important failover steps such as finding the most recent slave to failover to, applying differential logs, monitoring master for failure, etc. But it does not deal with additional steps that need to be taken before and after failover. These would include steps such as setting the read-only flag, killing connections, moving writer virtual IP, etc. 15 | 16 | .. _MHA: https://code.google.com/p/mysql-master-ha/ 17 | 18 | Introduction 19 | ------------ 20 | There are three functions of MHA Helper: 21 | 22 | 1. Execute pre-failover and post-failover steps during an online failover. An online failover is one in which the original master is not dead and the failover is performed, for example, for maintenance purposes. 23 | 2. Execute pre-failover and post-failover steps during a hard master failover. In this case the original master is dead, meaning either the host is dead or the MySQL server process has died. 24 | 3. Daemonize the monitor that monitors the masters for failure *(Currently not implemented)* 25 | 26 | Package Requirements and Dependencies 27 | ------------------------------------- 28 | First and foremost MHA itself needs to be installed. You need the MHA manager and node packages installed. You can read more about installing MHA and its dependencies here: http://code.google.com/p/mysql-master-ha/wiki/Installation 29 | 30 | MHA Helper has been developed and tested against Python 2.6 and 2.7. Versions < 2.6 are not supported. 31 | 32 | In addition to Python 2.6, the following Python modules are needed: 33 | 34 | - paramiko 35 | - PyMySQL 36 | 37 | There are other MHA specific requirements, please go through the link below to read about them: https://code.google.com/p/mysql-master-ha/wiki/Requirements 38 | 39 | Installation 40 | ------------ 41 | MHA Helper packages are available in RPM format. Currently packages are only available for CentOS/RHEL 6.x 42 | 43 | Before installing the package you will have to configure the TwinDB package repository:: 44 | 45 | curl -s https://packagecloud.io/install/repositories/twindb/main/script.rpm.sh | sudo bash 46 | 47 | Once the repository has been configured you can install the package as follows:: 48 | 49 | yum install python-mha_helper 50 | 51 | MHA Helper is also available as a python module mha_helper_ and can be installed using pip as follows:: 52 | 53 | pip install mha_helper 54 | 55 | .. _mha_helper: https://pypi.python.org/pypi/mha_helper 56 | 57 | Installation using Opscode Chef 58 | ------------------------------- 59 | The most easiest way to install and configure both MHA and MHA Helper is to use the Chef cookbook mysql-mha_. 60 | 61 | Below are some of the benefits of using the cookbook: 62 | 63 | - Installation of MHA and MHA Helper 64 | - Automatic discovery of MySQL replication clusters and automatic configuration of MHA as well as MHA Helper 65 | - Ability to discover new nodes joining existing MySQL replication clusters and automatic reconfiguration of MHA and MHA Helper 66 | - System configuration related to other MHA prerequisites such as system user creation, SSH setup to allow password-less login to MySQL replication nodes 67 | 68 | .. _mysql-mha: https://supermarket.chef.io/cookbooks/mysql-mha 69 | 70 | Configuration 71 | ------------- 72 | MHA Helper uses ini-style configuration files. 73 | 74 | The Helper expects one configuration file per MySQL replication cluster present in the directory */etc/mha-helper*. 75 | 76 | The configuration file has a *'default'* section and then one section per host for every host in the MySQL replication cluster. 77 | The following configuration options are supported: 78 | 79 | writer_vip_cidr 80 | The virtual IP that is assigned to the MySQL master. This must be in CIDR format. 81 | vip_type 82 | The type of VIP which can be anyone of these: 83 | - none : When this is set then MHA Helper does not do VIP management 84 | - metal : When this is set then traditional baremetal-style VIP management is done using the standard *ip* command 85 | - aws : When this is set then VIP management is done in a way relevant to AWS *(Currently not implemented)* 86 | - openstack : When this is set then VIP management is done in a way relevant to OpenStack *(Currently not implemented)* 87 | super_read_only 88 | Certain MySQL flavors (such as Percona Server: https://www.percona.com/doc/percona-server/5.6/management/super_read_only.html) have super_read_only which also disallows users with SUPER privileges to perform any writes when MySQL read_only flag is enabled. Set this to *yes* to use this feature. 89 | report_email 90 | The email address which receives the email notification when a MySQL failover is performed 91 | smtp_host 92 | The SMTP host that is used to send the failover report email 93 | requires_sudo 94 | Some of the system commands executed as part of the failover process require either the use of a privileged user or a user with sudo privileges. Set this to *no* when the system user does not need to execute commands using sudo, set to *yes* otherwise 95 | requires_arping 96 | Some environments do not need to do anything with ARP caches. Set this to *no* in order to not send those arping commands. 97 | cluster_interface 98 | The ethernet interface on the machine that gets the Virtual IP assigned or removed 99 | kill_after_timeout 100 | How many seconds do we want to give the application to close MySQL connections gracefully before killing still active connections on the old master. Set this to *0* to disable waiting and kill all connections immediately. 101 | 102 | All the options above can be specified either in the default section or in the host specific sections. Values specified in host specific sections override the values specified in the *default* section. 103 | 104 | Let me show you an example configuration file: 105 | 106 | :: 107 | 108 | [default] 109 | requires_sudo = yes 110 | requires_arping = yes 111 | vip_type = metal 112 | writer_vip_cidr = 192.168.10.155/24 113 | cluster_interface = eth1 114 | super_read_only = no 115 | report_email = me@ovaistariq.net 116 | smtp_host = localhost 117 | kill_after_timeout = 5 118 | 119 | [db10] 120 | cluster_interface = eth10 121 | 122 | [db11] 123 | 124 | [db12] 125 | report_email = notify@host-db12.com 126 | smtp_host = localhost2 127 | requires_sudo = no 128 | 129 | Apart from the configuration file needed by MHA Helper, you also need to setup the MHA specific application configuration file which defines the master-slave hosts. You can find details on how the application configuration file should be written here: https://code.google.com/p/mysql-master-ha/wiki/Configuration#Writing_an_application_configuration_file 130 | 131 | I would also suggest that you go through this link to see all the available MHA configuration options: https://code.google.com/p/mysql-master-ha/wiki/Parameters 132 | 133 | Following are the important options that must be specified in the MHA application configuration file: 134 | 135 | - user 136 | - password 137 | - ssh_user 138 | - manager_workdir 139 | - manager_log 140 | - master_ip_failover_script 141 | - master_ip_online_change_script 142 | - report_script 143 | 144 | 145 | Below is an example application configuration file: 146 | 147 | :: 148 | 149 | [server default] 150 | user = mha_helper 151 | password = helper 152 | ssh_user = mha_helper 153 | ssh_port = 2202 154 | repl_user = replicator 155 | repl_password = replicator 156 | master_binlog_dir = /var/log/mysql 157 | manager_workdir = /var/log/mha/test_cluster 158 | manager_log = /var/log/mha/test_cluster/test_cluster.log 159 | remote_workdir = /var/log/mha/test_cluster 160 | master_ip_failover_script = /usr/bin/master_ip_hard_failover_helper 161 | master_ip_online_change_script = /usr/bin/master_ip_online_failover_helper 162 | report_script = /usr/bin/master_failover_report 163 | 164 | [server1] 165 | hostname = db10 166 | candidate_master = 1 167 | check_repl_delay = 0 168 | 169 | [server2] 170 | hostname = db11 171 | candidate_master = 1 172 | check_repl_delay = 0 173 | 174 | [server3] 175 | hostname = db12 176 | no_master = 1 177 | 178 | Pre-failover Steps During Online Failover 179 | ----------------------------------------- 180 | To make sure that the failover is safe and does not cause any data inconsistencies, MHA Helper takes the following steps before the failover: 181 | 182 | 1. Set read_only on the new master to avoid any data inconsistencies 183 | 2. Remove the writer VIP from the original master if vip_type != none 184 | 3. Set read_only=1 on the original master 185 | 4. Wait up to 5 seconds for all connected threads to disconnect on the original master 186 | 5. Terminate all the connections except those that are replication-related, the connection made by MHA Helper and the connections opened by the *'system user'* 187 | 6. Disconnect from the original master 188 | 189 | 190 | If any of the above steps fail, any changes made during pre-failover are rolled back. 191 | 192 | Post-failover Steps During Online Failover 193 | ------------------------------------------ 194 | Once MHA has switched the masters and reconfigured replication, the MHA Helper takes the following steps: 195 | 196 | 1. Remove the read_only flag from the new master 197 | 2. Assign the writer VIP to the new master if vip_type != none 198 | 199 | 200 | Pre-failover Steps During Hard Failover 201 | --------------------------------------- 202 | If the original master is accessible via SSH, i.e. in cases where MySQL crashed and stopped but the host is still up, then MHA Helper takes the following step: 203 | 204 | 1. Remove the writer VIP from the original master if vip_type != none 205 | 206 | 207 | Post-failover Steps During Hard Failover 208 | ---------------------------------------- 209 | Once MHA has switched the masters and reconfigured replication, the MHA Helper takes the following steps: 210 | 211 | 1. Remove the read_only flag from the new master 212 | 2. Assign the writer VIP to the new master if vip_type != none 213 | 214 | 215 | Automated Failover and Monitoring via MHA Manager Daemon 216 | -------------------------------------------------------- 217 | **TODO** 218 | 219 | 220 | Manual Failover Examples 221 | ------------------------ 222 | Once everything is configured and running, doing the failover is pretty simple. 223 | 224 | Do a failover when the master db1 goes down:: 225 | 226 | /usr/bin/mysql_failover -d db1 -c /etc/mha/test_cluster.conf 227 | 228 | Do an online failover:: 229 | 230 | /usr/bin/mysql_online_failover -c /etc/mha/test_cluster.conf 231 | 232 | Using Non-root User 233 | ------------------- 234 | If you are using non-root user to connect to master-slave hosts via ssh (the user that you use for this purpose is taken from the *ssh_user* option) then you need to make sure that the user can execute the following commands: 235 | - /sbin/ip 236 | - /sbin/arping 237 | 238 | The user should be able to execute the above commands using sudo, and should not have to provide a password. This can accomplished by editing the file /etc/sudoers using visudo and adding the following lines:: 239 | 240 | mha_helper ALL=NOPASSWD: /sbin/ip, /sbin/arping 241 | 242 | In the example above I am assuming that ssh_user=mha_helper. 243 | 244 | Some General Recommendations 245 | ---------------------------- 246 | There are some general recommendations that I want to make, to prevent race-condition that can cause data inconsistencies: 247 | 248 | 1. Do not persist interface with writer VIP in the network scripts. This is important for example in cases where both the candidate masters go down i.e. hosts go down and then come back online. In which case we should need to manually intervene because there is no automated way to find out which MySQL server should be the source of truth 249 | 2. Persist read_only in the MySQL configuration file of all the candidate masters as well. This is again important for example in cases where both the candidate masters go down. 250 | -------------------------------------------------------------------------------- /mha_helper/mha_helper.py: -------------------------------------------------------------------------------- 1 | # (c) 2015, Ovais Tariq 2 | # 3 | # This file is part of mha_helper 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | from __future__ import print_function 19 | import time 20 | import datetime 21 | import re 22 | from mysql_helper import MySQLHelper 23 | from config_helper import ConfigHelper 24 | from vip_metal_helper import VIPMetalHelper 25 | 26 | 27 | class MHAHelper(object): 28 | FAILOVER_TYPE_ONLINE = 'online_failover' 29 | FAILOVER_TYPE_HARD = 'hard_failover' 30 | FAILOVER_STOP_CMD = 'stop' 31 | FAILOVER_STOPSSH_CMD = 'stopssh' 32 | FAILOVER_START_CMD = 'start' 33 | FAILOVER_STATUS_CMD = 'status' 34 | 35 | def __init__(self, failover_type): 36 | self.failover_type = failover_type 37 | 38 | if not self.__validate_failover_type(): 39 | raise ValueError 40 | 41 | # Setup configuration 42 | if not ConfigHelper.load_config(): 43 | raise ValueError 44 | 45 | self.orig_master_host = None 46 | self.new_master_host = None 47 | 48 | self.orig_master_config = None 49 | self.new_master_config = None 50 | 51 | def execute_command(self, **kwargs): 52 | for key, value in kwargs.iteritems(): 53 | setattr(self, key, value) 54 | 55 | try: 56 | command = getattr(self, "command") 57 | except Exception as e: 58 | print("No command supplied: %s" % str(e)) 59 | return False 60 | 61 | # Delegate the work to other functions 62 | if command == self.FAILOVER_STOP_CMD: 63 | if self.failover_type == self.FAILOVER_TYPE_ONLINE: 64 | if not self.__stop_command(): 65 | self.__rollback_stop_command() 66 | return False 67 | elif self.failover_type == self.FAILOVER_TYPE_HARD: 68 | return self.__stop_hard_command() 69 | 70 | return True 71 | elif command == self.FAILOVER_STOPSSH_CMD: 72 | return self.__stop_ssh_command() 73 | elif command == self.FAILOVER_START_CMD: 74 | return self.__start_command() 75 | elif command == self.FAILOVER_STATUS_CMD: 76 | return self.__status_command() 77 | 78 | # If we reach here that means no valid command was provided so we return an error here 79 | return False 80 | 81 | def __validate_failover_type(self): 82 | return (self.failover_type == MHAHelper.FAILOVER_TYPE_ONLINE or 83 | self.failover_type == MHAHelper.FAILOVER_TYPE_HARD) 84 | 85 | def __stop_command(self): 86 | try: 87 | self.orig_master_host = getattr(self, "orig_master_host") 88 | self.orig_master_config = ConfigHelper(self.orig_master_host) 89 | except Exception as e: 90 | print("Failed to read configuration for original master: %s" % str(e)) 91 | return False 92 | 93 | # Original master 94 | try: 95 | orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host) 96 | orig_master_mysql_port = getattr(self, "orig_master_port", None) 97 | orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip) 98 | orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None) 99 | orig_master_ssh_user = getattr(self, "orig_master_ssh_user", None) 100 | orig_master_ssh_options = getattr(self, "ssh_options", None) 101 | orig_master_mysql_user = self.__unescape_from_shell(getattr(self, "orig_master_user")) 102 | orig_master_mysql_pass = self.__unescape_from_shell(getattr(self, "orig_master_password")) 103 | except AttributeError as e: 104 | print("Failed to read one or more required original master parameter(s): %s" % str(e)) 105 | return False 106 | 107 | # Setup MySQL connections 108 | mysql_orig_master = MySQLHelper(orig_master_ip, orig_master_mysql_port, orig_master_mysql_user, 109 | orig_master_mysql_pass) 110 | 111 | try: 112 | print("Connecting to mysql on the original master '%s'" % self.orig_master_host) 113 | if not mysql_orig_master.connect(): 114 | return False 115 | 116 | if self.orig_master_config.get_manage_vip(): 117 | vip_type = self.orig_master_config.get_vip_type() 118 | print("Removing the vip using the '%s' provider from the original master '%s'" % 119 | (vip_type, self.orig_master_host)) 120 | 121 | if not self.__remove_vip_from_host(vip_type, self.orig_master_host, orig_master_ssh_ip, 122 | orig_master_ssh_user, orig_master_ssh_port, orig_master_ssh_options, 123 | self.FAILOVER_TYPE_ONLINE): 124 | return False 125 | 126 | if self.orig_master_config.get_super_read_only() and mysql_orig_master.super_read_only_supported(): 127 | print("Setting super_read_only to '1' on the original master '%s'" % self.orig_master_host) 128 | if not mysql_orig_master.set_super_read_only() or not mysql_orig_master.is_super_read_only(): 129 | return False 130 | else: 131 | print("Setting read_only to '1' on the original master '%s'" % self.orig_master_host) 132 | if not mysql_orig_master.set_read_only() or not mysql_orig_master.is_read_only(): 133 | return False 134 | 135 | if not self.__mysql_kill_threads(self.orig_master_host, mysql_orig_master, 136 | self.orig_master_config.get_kill_after_timeout()): 137 | return False 138 | except Exception as e: 139 | print("Unexpected error: %s" % str(e)) 140 | return False 141 | finally: 142 | print("Disconnecting from mysql on the original master '%s'" % self.orig_master_host) 143 | mysql_orig_master.disconnect() 144 | 145 | return True 146 | 147 | def __stop_hard_command(self): 148 | try: 149 | self.orig_master_host = getattr(self, "orig_master_host") 150 | self.orig_master_config = ConfigHelper(self.orig_master_host) 151 | except Exception as e: 152 | print("Failed to read configuration for original master: %s" % str(e)) 153 | return False 154 | 155 | # Original master 156 | try: 157 | orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host) 158 | orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip) 159 | orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None) 160 | orig_master_ssh_options = getattr(self, "ssh_options", None) 161 | except AttributeError as e: 162 | print("Failed to read one or more required original master parameter(s): %s" % str(e)) 163 | return False 164 | 165 | try: 166 | if self.orig_master_config.get_manage_vip(): 167 | vip_type = self.orig_master_config.get_vip_type() 168 | print("Removing the vip using the '%s' provider from the original master '%s'" % 169 | (vip_type, self.orig_master_host)) 170 | 171 | if not self.__remove_vip_from_host(vip_type, self.orig_master_host, orig_master_ssh_ip, None, 172 | orig_master_ssh_port, orig_master_ssh_options, 173 | self.FAILOVER_TYPE_HARD): 174 | return False 175 | except Exception as e: 176 | print("Unexpected error: %s" % str(e)) 177 | return False 178 | 179 | return True 180 | 181 | def __stop_ssh_command(self): 182 | try: 183 | self.orig_master_host = getattr(self, "orig_master_host") 184 | self.orig_master_config = ConfigHelper(self.orig_master_host) 185 | except Exception as e: 186 | print("Failed to read configuration for original master: %s" % str(e)) 187 | return False 188 | 189 | # Original master 190 | try: 191 | orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host) 192 | orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip) 193 | orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None) 194 | orig_master_ssh_user = getattr(self, "ssh_user", None) 195 | orig_master_ssh_options = getattr(self, "ssh_options", None) 196 | except AttributeError as e: 197 | print("Failed to read one or more required original master parameter(s): %s" % str(e)) 198 | return False 199 | 200 | try: 201 | if self.orig_master_config.get_manage_vip(): 202 | vip_type = self.orig_master_config.get_vip_type() 203 | print("Removing the vip using the '%s' provider from the original master '%s'" % 204 | (vip_type, self.orig_master_host)) 205 | 206 | if not self.__remove_vip_from_host(vip_type, self.orig_master_host, orig_master_ssh_ip, 207 | orig_master_ssh_user, orig_master_ssh_port, orig_master_ssh_options, 208 | self.FAILOVER_TYPE_ONLINE): 209 | return False 210 | except Exception as e: 211 | print("Unexpected error: %s" % str(e)) 212 | return False 213 | 214 | return True 215 | 216 | def __start_command(self): 217 | try: 218 | self.orig_master_host = getattr(self, "orig_master_host") 219 | self.orig_master_config = ConfigHelper(self.orig_master_host) 220 | except Exception as e: 221 | print("Failed to read configuration for original master: %s" % str(e)) 222 | return False 223 | 224 | try: 225 | self.new_master_host = getattr(self, "new_master_host") 226 | self.new_master_config = ConfigHelper(self.new_master_host) 227 | except Exception as e: 228 | print("Failed to read configuration for new master: %s" % str(e)) 229 | return False 230 | 231 | # New master 232 | try: 233 | new_master_ip = getattr(self, "new_master_ip", self.new_master_host) 234 | new_master_mysql_port = getattr(self, "new_master_port", None) 235 | new_master_mysql_user = self.__unescape_from_shell(getattr(self, "new_master_user")) 236 | new_master_mysql_pass = self.__unescape_from_shell(getattr(self, "new_master_password")) 237 | new_master_ssh_ip = getattr(self, "new_master_ssh_ip", new_master_ip) 238 | new_master_ssh_port = getattr(self, "new_master_ssh_port", None) 239 | new_master_ssh_options = getattr(self, "ssh_options", None) 240 | 241 | if self.failover_type == self.FAILOVER_TYPE_HARD: 242 | new_master_ssh_user = getattr(self, "ssh_user", None) 243 | else: 244 | new_master_ssh_user = getattr(self, "new_master_ssh_user", None) 245 | except AttributeError as e: 246 | print("Failed to read one or more required new master parameter(s): %s" % str(e)) 247 | return False 248 | 249 | # Setup MySQL connection 250 | mysql_new_master = MySQLHelper(new_master_ip, new_master_mysql_port, new_master_mysql_user, 251 | new_master_mysql_pass) 252 | 253 | try: 254 | print("Connecting to mysql on the new master '%s'" % self.new_master_host) 255 | if not mysql_new_master.connect(): 256 | return False 257 | 258 | print("Setting read_only to '0' on the new master '%s'" % self.new_master_host) 259 | if not mysql_new_master.unset_read_only() or mysql_new_master.is_read_only(): 260 | return False 261 | 262 | if self.new_master_config.get_manage_vip(): 263 | vip_type = self.new_master_config.get_vip_type() 264 | print("Adding the vip using the '%s' provider to the new master '%s'" % 265 | (vip_type, self.new_master_host)) 266 | 267 | if not self.__add_vip_to_host(vip_type, self.new_master_host, new_master_ssh_ip, new_master_ssh_user, 268 | new_master_ssh_port, new_master_ssh_options): 269 | return False 270 | except Exception as e: 271 | print("Unexpected error: %s" % str(e)) 272 | return False 273 | finally: 274 | print("Disconnecting from mysql on the new master '%s'" % self.new_master_host) 275 | mysql_new_master.disconnect() 276 | 277 | return True 278 | 279 | def __status_command(self): 280 | try: 281 | self.orig_master_host = getattr(self, "orig_master_host") 282 | self.orig_master_config = ConfigHelper(self.orig_master_host) 283 | except Exception as e: 284 | print("Failed to read configuration for original master: %s" % str(e)) 285 | return False 286 | 287 | # Original master 288 | try: 289 | orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host) 290 | orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip) 291 | orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None) 292 | orig_master_ssh_user = getattr(self, "ssh_user", None) 293 | orig_master_ssh_options = getattr(self, "ssh_options", None) 294 | except AttributeError as e: 295 | print("Failed to read one or more required original master parameter(s): %s" % str(e)) 296 | return False 297 | 298 | try: 299 | if self.orig_master_config.get_manage_vip(): 300 | vip_type = self.orig_master_config.get_vip_type() 301 | print("Checking the vip using the '%s' provider on the original master '%s'" % 302 | (vip_type, self.orig_master_host)) 303 | 304 | if not self.__check_vip_on_host(vip_type, self.orig_master_host, orig_master_ssh_ip, 305 | orig_master_ssh_user, orig_master_ssh_port, orig_master_ssh_options): 306 | print("The VIP was not found on host %s" % self.orig_master_host) 307 | return False 308 | except Exception as e: 309 | print("Unexpected error: %s" % str(e)) 310 | return False 311 | 312 | return True 313 | 314 | def __rollback_stop_command(self): 315 | try: 316 | self.orig_master_host = getattr(self, "orig_master_host") 317 | self.orig_master_config = ConfigHelper(self.orig_master_host) 318 | except Exception as e: 319 | print("Failed to read configuration for original master: %s" % str(e)) 320 | return False 321 | 322 | # Original master 323 | try: 324 | orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host) 325 | orig_master_mysql_port = getattr(self, "orig_master_port", None) 326 | orig_master_mysql_user = self.__unescape_from_shell(getattr(self, "orig_master_user")) 327 | orig_master_mysql_pass = self.__unescape_from_shell(getattr(self, "orig_master_password")) 328 | orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip) 329 | orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None) 330 | orig_master_ssh_user = getattr(self, "orig_master_ssh_user", None) 331 | orig_master_ssh_options = getattr(self, "ssh_options", None) 332 | except AttributeError as e: 333 | print("Failed to read one or more required original master parameter(s): %s" % str(e)) 334 | return False 335 | 336 | # Setup MySQL connections 337 | mysql_orig_master = MySQLHelper(orig_master_ip, orig_master_mysql_port, orig_master_mysql_user, 338 | orig_master_mysql_pass) 339 | 340 | print("Rolling back the failover changes on the original master '%s'" % self.orig_master_host) 341 | try: 342 | if not mysql_orig_master.connect(): 343 | print("Failed to connect to mysql on the original master '%s'" % self.orig_master_host) 344 | return False 345 | 346 | if not mysql_orig_master.unset_read_only() or mysql_orig_master.is_read_only(): 347 | print("Failed to reset read_only to '0' on the original master '%s'" % self.orig_master_host) 348 | return False 349 | 350 | print("Set read_only back to '0' on the original master '%s'" % self.orig_master_host) 351 | 352 | if self.orig_master_config.get_manage_vip(): 353 | vip_type = self.orig_master_config.get_vip_type() 354 | if not self.__add_vip_to_host(vip_type, self.orig_master_host, orig_master_ssh_ip, orig_master_ssh_user, 355 | orig_master_ssh_port, orig_master_ssh_options): 356 | print("Failed to add back the vip using the '%s' provider to the original master '%s'" % 357 | (vip_type, self.orig_master_host)) 358 | return False 359 | 360 | print("Added back the vip to the original master '%s'" % self.orig_master_host) 361 | except Exception as e: 362 | print("Unexpected error: %s" % str(e)) 363 | return False 364 | finally: 365 | mysql_orig_master.disconnect() 366 | 367 | return True 368 | 369 | @classmethod 370 | def __remove_vip_from_host(cls, vip_type, host, host_ip, ssh_user, ssh_port, ssh_options, failover_type): 371 | if vip_type == ConfigHelper.VIP_PROVIDER_TYPE_METAL: 372 | vip_helper = VIPMetalHelper(host, host_ip, ssh_user, ssh_port, ssh_options) 373 | 374 | # If this is a hard failover and we cannot connect to the original master over SSH then we cannot really do 375 | # anything here at the moment. 376 | # TODO: At the moment we are not doing anything here but we would probably want to do something here 377 | if failover_type == cls.FAILOVER_TYPE_HARD: 378 | return True 379 | 380 | if not vip_helper.remove_vip(): 381 | return False 382 | elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_AWS: 383 | pass 384 | elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_OS: 385 | pass 386 | else: 387 | # There are no other vip providers apart from what we are testing for above. Hence we throw an 388 | # error here 389 | return False 390 | 391 | return True 392 | 393 | @classmethod 394 | def __add_vip_to_host(cls, vip_type, host, host_ip, ssh_user, ssh_port, ssh_options): 395 | if vip_type == ConfigHelper.VIP_PROVIDER_TYPE_METAL: 396 | vip_helper = VIPMetalHelper(host, host_ip, ssh_user, ssh_port, ssh_options) 397 | if not vip_helper.assign_vip(): 398 | return False 399 | elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_AWS: 400 | pass 401 | elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_OS: 402 | pass 403 | else: 404 | # There are no other vip providers apart from what we are testing for above. Hence we throw an 405 | # error here 406 | return False 407 | 408 | return True 409 | 410 | @classmethod 411 | def __check_vip_on_host(cls, vip_type, host, host_ip, ssh_user, ssh_port, ssh_options): 412 | if vip_type == ConfigHelper.VIP_PROVIDER_TYPE_METAL: 413 | return VIPMetalHelper(host, host_ip, ssh_user, ssh_port, ssh_options).has_vip() 414 | elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_AWS: 415 | pass 416 | elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_OS: 417 | pass 418 | else: 419 | # There are no other vip providers apart from what we are testing for above. Hence we throw an 420 | # error here 421 | return False 422 | 423 | return True 424 | 425 | @classmethod 426 | def __mysql_kill_threads(cls, host, mysql_connection, timeout): 427 | sleep_interval = 0.1 428 | start = datetime.datetime.now() 429 | 430 | print("Waiting %d seconds for application threads to disconnect from the MySQL server '%s'" % (timeout, host)) 431 | while True: 432 | try: 433 | mysql_threads = cls.__get_mysql_threads_list(mysql_connection) 434 | if len(mysql_threads) < 1: 435 | break 436 | except Exception as e: 437 | print("Unexpected error: %s" % str(e)) 438 | return False 439 | 440 | time.sleep(sleep_interval) 441 | now = datetime.datetime.now() 442 | if (now - start).seconds > timeout: 443 | break 444 | 445 | print("Terminating all application threads connected to the MySQL server '%s'" % host) 446 | try: 447 | for thread in iter(cls.__get_mysql_threads_list(mysql_connection)): 448 | print("Terminating thread Id => %s, User => %s, Host => %s" % 449 | (thread['Id'], thread['User'], thread['Host'])) 450 | mysql_connection.kill_connection(thread['Id']) 451 | except Exception as e: 452 | print("Unexpected error: %s" % str(e)) 453 | return False 454 | 455 | return True 456 | 457 | @classmethod 458 | def __get_mysql_threads_list(cls, mysql_connection): 459 | threads = list() 460 | try: 461 | for row in mysql_connection.get_processlist(): 462 | if (mysql_connection.get_connection_id() == row['Id'] or row['Command'] == "Binlog Dump" or 463 | row['Command'] == "Binlog Dump GTID" or row['User'] == "system user"): 464 | continue 465 | threads.append(row) 466 | except Exception as e: 467 | print("Failed to get list of processes from MySQL: %s" % str(e)) 468 | return False 469 | 470 | return threads 471 | 472 | @classmethod 473 | def __unescape_from_shell(cls, escaped): 474 | # This matches with mha4mysql-node::NodeUtil.pm::@shell_escape_chars 475 | # username and password provided by MHA are escaped like this 476 | unescaped = re.sub(r'\\(?!\\)', '', escaped) 477 | 478 | return unescaped 479 | --------------------------------------------------------------------------------