├── masterha ├── mha_sudoer ├── app_default.cnf ├── app_56.conf ├── masterha-script.cnf └── masterha-script.pm ├── bin ├── init_conf_loads ├── master_ip_failover └── master_ip_online_change ├── LICENSE ├── README.md └── log └── switch.log /masterha/mha_sudoer: -------------------------------------------------------------------------------- 1 | # sudoer for mha user 2 | Cmnd_Alias MHA = /sbin/ip, /sbin/arping, /bin/ping 3 | 4 | mha ALL=(root) NOPASSWD: MHA 5 | -------------------------------------------------------------------------------- /bin/init_conf_loads: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | use MIME::Base64; 5 | 6 | my $pstring = decode_base64('bW9uaXRvcg=='); 7 | 8 | print "password=$pstring\n"; 9 | -------------------------------------------------------------------------------- /masterha/app_default.cnf: -------------------------------------------------------------------------------- 1 | [server default] 2 | user=root 3 | init_conf_load_script=/usr/bin/init_conf_loads 4 | ssh_user=mha 5 | ssh_options="-i /home/mha/.ssh/id_rsa -o ConnectTimeout=3" 6 | remote_workdir=/web/masterhalog 7 | ping_interval=3 8 | master_ip_failover_script=/usr/bin/master_ip_failover 9 | master_ip_online_change_script=/usr/bin/master_ip_online_change 10 | report_script=/usr/bin/send_report 11 | multi_tier_slave=1 12 | -------------------------------------------------------------------------------- /masterha/app_56.conf: -------------------------------------------------------------------------------- 1 | [server default] 2 | manager_workdir=/web/masterhalog 3 | manager_log=/web/masterhalog/app_56.log 4 | port=3308 5 | client_bindir=/opt/Percona-Server-5.6.21-rel69.0-675.Linux.x86_64/bin 6 | client_libdir=/opt/Percona-Server-5.6.21-rel69.0-675.Linux.x86_64/lib/mysql 7 | repl_user=user_replica 8 | repl_password=xxxxxxxx 9 | 10 | [server1] 11 | hostname=10.0.21.7 12 | master_binlog_dir=/web/mysql/node3308/data/ 13 | candidate_master=1 14 | 15 | [server2] 16 | hostname=10.0.21.17 17 | master_binlog_dir=/web/mysql/node3308/data/ 18 | candidate_master=1 19 | -------------------------------------------------------------------------------- /masterha/masterha-script.cnf: -------------------------------------------------------------------------------- 1 | # sharp-sign starts a comment 2 | # ip:port are not indented 3 | # any other non-indented lines are default directives 4 | # settings are indented 5 | # 6 | # directives: vip, block_user, block_host, vip_on_loopback 7 | # vip and proxysql are optional option, you can ignore if you do not use vip or proxysql 8 | 9 | #127.0.0.1:3308 127.0.0.1:3309 10 | # vip 127.0.0.3 11 | # block_user ^abc$ 12 | # block_host ^localhost$ 13 | 14 | 10.0.21.7:3308 10.0.21.17:3308 15 | vip 10.0.21.97 16 | block_user ^percona$|^proxysqlmon$ 17 | block_host ^10\.0\.21\.%$ 18 | proxysql admin2:admin2@10.0.21.5:6032:w1:r2,admin2:admin2@10.0.21.7:6032:w1:r2 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 arstercz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Note 2 | 3 | this project is no longer maintained, all of the features are merged to https://github.com/arstercz/mha4mysql-manager. read more from [wiki](https://github.com/arstercz/mha4mysql-manager/wiki). 4 | 5 | ## mha_switch 6 | 7 | Switch MySQL replication with custom scripts by use [MHA](https://github.com/yoshinorim/mha4mysql-manager), read more from [blog](https://blog.arstercz.com/mha_switch-%E7%BB%93%E5%90%88-proxysql-%E5%92%8C-mha-%E5%88%87%E6%8D%A2-mysql-%E4%B8%BB%E4%BB%8E/). 8 | 9 | *note:* 10 | 11 | - all dependecy Perl modules are the same with `MHA`, and all of the scripts are based on `MHA 0.56` version. 12 | - the `event_scheduler` feature can be used in `MHA 0.58` version, otherwise you must give a big enough value to the `--running_updates_limit` and `--running_seconds_limit` option. read more from [pull-44](https://github.com/yoshinorim/mha4mysql-manager/pull/44) and you can apply this patch into your low version `MHA`. 13 | 14 | 15 | ## Introduction 16 | 17 | the full code structure: 18 | ``` 19 | mha_switch 20 | ├── bin 21 | │   ├── init_conf_loads 22 | │   ├── master_ip_failover 23 | │   └── master_ip_online_change 24 | ├── LICENSE 25 | ├── log 26 | │   └── switch.log 27 | ├── masterha 28 | │   ├── app_56.conf 29 | │   ├── app_default.cnf 30 | │   ├── masterha-script.cnf 31 | │   ├── masterha-script.pm 32 | │   └── mha_sudoer 33 | └── README.md 34 | 35 | ``` 36 | 37 | ### scripts 38 | 39 | 1. the `master_ip_failover` and `master_ip_online_change` are refer to `mha4mysql-manager/samples/scripts/`. these path should be the same as the option values in the global default file `app_default.cnf`. 40 | 41 | 2. `init_conf_loads` contains the MySQl `root user` password which in `base64` format. 42 | 43 | 3. `masterha-script.pm` is our custum module that support the following features: 44 | ``` 45 | parse masterha-script.cnf file 46 | virtual ip switch, 47 | block/release mysql user 48 | proxysql switch 49 | ``` 50 | read more from [proxysql](https://github.com/sysown/proxysql). 51 | 52 | 4. `mha_sudoer` should be copy to all of the MySQL Host's `/etc/sudoers.d`, so that `mha_switch` can switch virtual ip address with normal user(default is `mha` user). **note:** you must comment the `Defaults requiretty` so that avoid sudo error `sorry, you must have a tty to run sudo`. 53 | 54 | ### configure file 55 | 56 | 1. `app_56.conf` is one MySQL replication instances. 57 | 58 | 2. `masterha-script.cnf` is the MySQL replication info which refers to `app_56.conf`, you can specify multigroup if you have multiple MySQL replications. the `proxysql` is optional option, you can set multiple proxysqls(master/backup) which split by comma symbol. eg: 59 | ``` 60 | # vip and proxysql are optional option, you can ignore if you do not use vip or proxysql 61 | 10.0.21.7:3308 10.0.21.17:3308 62 | vip 10.0.21.97 63 | block_user ^percona$|^proxysqlmon$ 64 | block_host ^10\.0\.21\.%$ 65 | proxysql admin2:admin2@10.0.21.5:6032:w1:r2,admin2:admin2@10.0.21.7:6032:w1:r2 66 | ``` 67 | the `10.0.21.7:3308 10.0.21.17:3308` is the master, slave ip address and port, you must specify multi `ip:port` if you have many slaves; `vip 10.0.21.97` means that master services to application by the virtual ip address, MHA will switch the vip address when you use MHA switch the replication, this means the application will do nothing to connect new master only when it has the retry mechanism, `vip` is optional option, you can ignore this if you do not use vip address; `block_user` and `block_host` means MHA will block the user which in old master instance, read more from [blocking-user-accounts](http://code.openark.org/blog/mysql/blocking-user-accounts); the last line `proxysql` option is optional, you can setup if you use proxysql, the value means there are two proxysqls, include the following proxysql administration info: 68 | ``` 69 | proxysql1: 70 | username: admin2 71 | password: admin2 72 | ip: 10.0.21.5 73 | port: 6032 # admin port 74 | write group: 1 # begin with w 75 | read group: 2 # begin with r 76 | 77 | proxysql2: 78 | username: admin2 79 | password: admin2 80 | ip: 10.0.21.7 81 | port: 6032 # admin port 82 | write group: 1 # begin with w 83 | read group: 2 # begin with r 84 | ``` 85 | 86 | ## How to use 87 | 88 | the `master_ip_failover` and `master_ip_online_change` refer the path `/etc/masterha/masterha-script.pm` and `/etc/masterha/masterha-script.cnf` by default, if your `masterha` dirs not in `/etc`, then you should change the value in the two scripts. 89 | 90 | ### structure 91 | 92 | if we have the following structure: 93 | ``` 94 | +------------+ +------------+ 95 | | proxysql 1 | 10.0.21.5 | proxysql 2 | 10.0.21.7 96 | +------------+ +------------+ 97 | | | 98 | | | 99 | | | 100 | | | 101 | +----------+-------------------------+----------------------+ 102 | | | 103 | | +--------+ +-------+ | 104 | | | master | 10.0.21.17:3308 | slave | 10.0.21.7:3308 | 105 | | +--------+ +-------+ | 106 | | | 107 | +-----------------------------------------------------------+ 108 | 109 | ``` 110 | 111 | ### proxysql mysql servers 112 | 113 | before `mha` the proxysql `runtime_mysql_servers` is: 114 | ``` 115 | +--------------+------------+------+--------+--------+-------------+-----------------+---------------------+---------+----------------+---------+ 116 | | hostgroup_id | hostname | port | status | weight | compression | max_connections | max_replication_lag | use_ssl | max_latency_ms | comment | 117 | +--------------+------------+------+--------+--------+-------------+-----------------+---------------------+---------+----------------+---------+ 118 | | 1 | 10.0.21.17 | 3308 | ONLINE | 1000 | 0 | 2000 | 0 | 0 | 0 | | 119 | | 2 | 10.0.21.17 | 3308 | ONLINE | 1000 | 0 | 2000 | 0 | 0 | 0 | | 120 | | 2 | 10.0.21.7 | 3308 | ONLINE | 1000 | 0 | 2000 | 30 | 0 | 0 | | 121 | +--------------+------------+------+--------+--------+-------------+-----------------+---------------------+---------+----------------+---------+ 122 | ``` 123 | 124 | Execute the following command to switch MySQL recplication and change the proxysql setting: 125 | ``` 126 | # masterha_master_switch --master_state=alive --global_conf=/etc/masterha/app_default.cnf --conf=/etc/masterha/app_56.conf --orig_master_is_new_slave 127 | ... 128 | ... 129 | Thu Nov 16 15:52:00 2017.490641 delete proxysql repl group on 10.0.21.5:6032 ok! 130 | Thu Nov 16 15:52:00 2017.497134 set read_only on proxysql 10.0.21.5:6032 ok! 131 | ... 132 | ... 133 | Thu Nov 16 15:52:06 2017.306978 Delete old proxysql write group 10.0.21.17:3308 with group 1 ok! 134 | Thu Nov 16 15:52:06 2017.307767 Insert new proxysql write group 10.0.21.7:3308 with group 1 ok! 135 | Thu Nov 16 15:52:06 2017.308202 Insert new proxysql read group 10.0.21.7:3308 with group 2 ok! 136 | Thu Nov 16 15:52:06 2017.312171 Insert orig master as new proxysql read group 10.0.21.17:3308 with group 2 ok! 137 | Thu Nov 16 15:52:06 2017.312908 insert proxysql repl group on 10.0.21.5:6032 ok! 138 | Thu Nov 16 15:52:06 2017.321330 proxysql load server to runtime ok! 139 | Thu Nov 16 15:52:06 2017.348377 proxysql save server to disk ok! 140 | Thu Nov 16 15:52:06 2017.352720 set proxysql 10.0.21.7:6032 readwrite ok! 141 | ``` 142 | you can read the `log/switch.log` to get more message. 143 | 144 | ### proxysql mysql servers 145 | 146 | after `mha` the proxysql `runtime_mysql_servers` is: 147 | ``` 148 | +--------------+------------+------+--------+--------+-------------+-----------------+---------------------+---------+----------------+---------+ 149 | | hostgroup_id | hostname | port | status | weight | compression | max_connections | max_replication_lag | use_ssl | max_latency_ms | comment | 150 | +--------------+------------+------+--------+--------+-------------+-----------------+---------------------+---------+----------------+---------+ 151 | | 1 | 10.0.21.7 | 3308 | ONLINE | 1000 | 0 | 2000 | 0 | 0 | 0 | | 152 | | 2 | 10.0.21.7 | 3308 | ONLINE | 1000 | 0 | 2000 | 0 | 0 | 0 | | 153 | | 2 | 10.0.21.17 | 3308 | ONLINE | 1000 | 0 | 2000 | 30 | 0 | 0 | | 154 | +--------------+------------+------+--------+--------+-------------+-----------------+---------------------+---------+----------------+---------+ 155 | ``` 156 | 157 | there is no `10.0.21.17` entries if no `--orig_master_is_new_slave` option in MHA execute. 158 | 159 | ## License 160 | 161 | MIT / BSD 162 | -------------------------------------------------------------------------------- /log/switch.log: -------------------------------------------------------------------------------- 1 | Thu Nov 16 15:51:55 2017 - [info] MHA::MasterRotate version 0.56. 2 | Thu Nov 16 15:51:55 2017 - [info] Starting online master switch.. 3 | Thu Nov 16 15:51:55 2017 - [info] 4 | Thu Nov 16 15:51:55 2017 - [info] * Phase 1: Configuration Check Phase.. 5 | Thu Nov 16 15:51:55 2017 - [info] 6 | Thu Nov 16 15:51:55 2017 - [info] Reading default configuration from /etc/masterha/app_default.cnf.. 7 | Thu Nov 16 15:51:55 2017 - [info] Reading application default configuration from /etc/masterha/app_56.conf.. 8 | Thu Nov 16 15:51:55 2017 - [info] Updating application default configuration from /usr/bin/init_conf_loads.. 9 | Thu Nov 16 15:51:55 2017 - [info] Reading server configuration from /etc/masterha/app_56.conf.. 10 | Thu Nov 16 15:51:56 2017 - [info] GTID failover mode = 1 11 | Thu Nov 16 15:51:56 2017 - [info] Current Alive Master: 10.0.21.17(10.0.21.17:3308) 12 | Thu Nov 16 15:51:56 2017 - [info] Alive Slaves: 13 | Thu Nov 16 15:51:56 2017 - [info] 10.0.21.7(10.0.21.7:3308) Version=5.6.21-69.0-log (oldest major version between slaves) log-bin:enabled 14 | Thu Nov 16 15:51:56 2017 - [info] GTID ON 15 | Thu Nov 16 15:51:56 2017 - [info] Replicating from 10.0.21.17(10.0.21.17:3308) 16 | Thu Nov 16 15:51:56 2017 - [info] Primary candidate for the new Master (candidate_master is set) 17 | 18 | It is better to execute FLUSH NO_WRITE_TO_BINLOG TABLES on the master before switching. Is it ok to execute on 10.0.21.17(10.0.21.17:3308)? (YES/no): yes 19 | Thu Nov 16 15:51:58 2017 - [info] Executing FLUSH NO_WRITE_TO_BINLOG TABLES. This may take long time.. 20 | Thu Nov 16 15:51:58 2017 - [info] ok. 21 | Thu Nov 16 15:51:58 2017 - [info] Checking MHA is not monitoring or doing failover.. 22 | Thu Nov 16 15:51:58 2017 - [info] Checking replication health on 10.0.21.7.. 23 | Thu Nov 16 15:51:58 2017 - [info] ok. 24 | Thu Nov 16 15:51:58 2017 - [info] Searching new master from slaves.. 25 | Thu Nov 16 15:51:58 2017 - [info] Candidate masters from the configuration file: 26 | Thu Nov 16 15:51:58 2017 - [info] 10.0.21.7(10.0.21.7:3308) Version=5.6.21-69.0-log (oldest major version between slaves) log-bin:enabled 27 | Thu Nov 16 15:51:58 2017 - [info] GTID ON 28 | Thu Nov 16 15:51:58 2017 - [info] Replicating from 10.0.21.17(10.0.21.17:3308) 29 | Thu Nov 16 15:51:58 2017 - [info] Primary candidate for the new Master (candidate_master is set) 30 | Thu Nov 16 15:51:58 2017 - [info] 10.0.21.17(10.0.21.17:3308) Version=5.6.21-69.0-log log-bin:enabled 31 | Thu Nov 16 15:51:58 2017 - [info] GTID ON 32 | Thu Nov 16 15:51:58 2017 - [info] Non-candidate masters: 33 | Thu Nov 16 15:51:58 2017 - [info] Searching from candidate_master slaves which have received the latest relay log events.. 34 | Thu Nov 16 15:51:58 2017 - [info] 35 | From: 36 | 10.0.21.17(10.0.21.17:3308) (current master) 37 | +--10.0.21.7(10.0.21.7:3308) 38 | 39 | To: 40 | 10.0.21.7(10.0.21.7:3308) (new master) 41 | +--10.0.21.17(10.0.21.17:3308) 42 | 43 | Starting master switch from 10.0.21.17(10.0.21.17:3308) to 10.0.21.7(10.0.21.7:3308)? (yes/NO): yes 44 | Thu Nov 16 15:52:00 2017 - [info] Checking whether 10.0.21.7(10.0.21.7:3308) is ok for the new master.. 45 | Thu Nov 16 15:52:00 2017 - [info] ok. 46 | Thu Nov 16 15:52:00 2017 - [info] 10.0.21.17(10.0.21.17:3308): SHOW SLAVE STATUS returned empty result. To check replication filtering rules, temporarily executing CHANGE MASTER to a dummy host. 47 | Thu Nov 16 15:52:00 2017 - [info] 10.0.21.17(10.0.21.17:3308): Resetting slave pointing to the dummy host. 48 | Thu Nov 16 15:52:00 2017 - [info] ** Phase 1: Configuration Check Phase completed. 49 | Thu Nov 16 15:52:00 2017 - [info] 50 | Thu Nov 16 15:52:00 2017 - [info] * Phase 2: Rejecting updates Phase.. 51 | Thu Nov 16 15:52:00 2017 - [info] 52 | Thu Nov 16 15:52:00 2017 - [info] Executing master ip online change script to disable write on the current master: 53 | Thu Nov 16 15:52:00 2017 - [info] /usr/bin/master_ip_online_change --command=stop --orig_master_host=10.0.21.17 --orig_master_ip=10.0.21.17 --orig_master_port=3308 --orig_master_user='root' --orig_master_password='xxxxxx' --new_master_host=10.0.21.7 --new_master_ip=10.0.21.7 --new_master_port=3308 --new_master_user='root' --new_master_password='xxxxxx' --orig_master_ssh_user=mha --new_master_ssh_user=mha --ssh_options='-o ServerAliveInterval=60 -o ServerAliveCountMax=20 -o StrictHostKeyChecking=no -o ConnectionAttempts=5 -o PasswordAuthentication=no -o BatchMode=yes -i /home/mha/.ssh/id_rsa' --orig_master_is_new_slave 54 | Thu Nov 16 15:52:00 2017.490641 delete proxysql repl group on 10.0.21.5:6032 ok! 55 | Thu Nov 16 15:52:00 2017.497134 set read_only on proxysql 10.0.21.5:6032 ok! 56 | Thu Nov 16 15:52:00 2017.500391 delete proxysql repl group on 10.0.21.7:6032 ok! 57 | Thu Nov 16 15:52:00 2017.508106 set read_only on proxysql 10.0.21.7:6032 ok! 58 | Thu Nov 16 15:52:00 2017.511853 Set read_only on the new master.. ok. 59 | Thu Nov 16 15:52:00 2017.517152 Blocking app user on the orig master.. 60 | Thu Nov 16 15:52:00 2017.521610 Waiting all running 2 threads are disconnected.. (max 1500 milliseconds) 61 | {'Time' => '91','db' => undef,'Id' => '16487','User' => 'user_replica','State' => 'Master has sent all binlog to slave; waiting for binlog to be updated','Command' => 'Binlog Dump GTID','Rows_examined' => '0','Info' => undef,'Host' => '10.0.21.7:55573','Rows_sent' => '0'} 62 | {'Time' => '0','db' => undef,'Id' => '16489','User' => 'proxysqlmon','State' => '','Command' => 'Sleep','Rows_examined' => '1','Info' => undef,'Host' => '10.0.21.5:56059','Rows_sent' => '1'} 63 | Thu Nov 16 15:52:01 2017.023535 Waiting all running 3 threads are disconnected.. (max 1000 milliseconds) 64 | {'Time' => '0','db' => undef,'Id' => '15960','User' => 'proxysqlmon','State' => '','Command' => 'Sleep','Rows_examined' => '0','Info' => undef,'Host' => '10.0.21.7:54111','Rows_sent' => '0'} 65 | {'Time' => '91','db' => undef,'Id' => '16487','User' => 'user_replica','State' => 'Master has sent all binlog to slave; waiting for binlog to be updated','Command' => 'Binlog Dump GTID','Rows_examined' => '0','Info' => undef,'Host' => '10.0.21.7:55573','Rows_sent' => '0'} 66 | {'Time' => '0','db' => undef,'Id' => '16489','User' => 'proxysqlmon','State' => '','Command' => 'Sleep','Rows_examined' => '0','Info' => undef,'Host' => '10.0.21.5:56059','Rows_sent' => '0'} 67 | Thu Nov 16 15:52:01 2017.524518 Waiting all running 1 threads are disconnected.. (max 500 milliseconds) 68 | {'Time' => '92','db' => undef,'Id' => '16487','User' => 'user_replica','State' => 'Master has sent all binlog to slave; waiting for binlog to be updated','Command' => 'Binlog Dump GTID','Rows_examined' => '0','Info' => undef,'Host' => '10.0.21.7:55573','Rows_sent' => '0'} 69 | Thu Nov 16 15:52:02 2017.025009 Set read_only=1 on the orig master.. ok. 70 | Thu Nov 16 15:52:02 2017.028706 Waiting all running 1 queries are disconnected.. (max 500 milliseconds) 71 | {'Time' => '92','db' => undef,'Id' => '16487','User' => 'user_replica','State' => 'Master has sent all binlog to slave; waiting for binlog to be updated','Command' => 'Binlog Dump GTID','Rows_examined' => '0','Info' => undef,'Host' => '10.0.21.7:55573','Rows_sent' => '0'} 72 | Thu Nov 16 15:52:02 2017.526151 Killing all application threads.. 73 | Thu Nov 16 15:52:02 2017.527516 done. 74 | Thu Nov 16 15:52:02 2017.528644 Release app user on the orig master.. 75 | Thu Nov 16 15:52:02 2017.533862 Stopping VIP .. 76 | Thu Nov 16 15:52:03 2017 - [info] ok. 77 | Thu Nov 16 15:52:03 2017 - [info] Locking all tables on the orig master to reject updates from everybody (including root): 78 | Thu Nov 16 15:52:03 2017 - [info] Executing FLUSH TABLES WITH READ LOCK.. 79 | Thu Nov 16 15:52:03 2017 - [info] ok. 80 | Thu Nov 16 15:52:03 2017 - [info] Orig master binlog:pos is mysql-bin.000010:4115. 81 | Thu Nov 16 15:52:03 2017 - [info] Waiting to execute all relay logs on 10.0.21.7(10.0.21.7:3308).. 82 | Thu Nov 16 15:52:03 2017 - [info] master_pos_wait(mysql-bin.000010:4115) completed on 10.0.21.7(10.0.21.7:3308). Executed 0 events. 83 | Thu Nov 16 15:52:03 2017 - [info] done. 84 | Thu Nov 16 15:52:03 2017 - [info] Getting new master's binlog name and position.. 85 | Thu Nov 16 15:52:03 2017 - [info] mysql-bin.000001:4115 86 | Thu Nov 16 15:52:03 2017 - [info] All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='10.0.21.7', MASTER_PORT=3308, MASTER_AUTO_POSITION=1, MASTER_USER='user_replica', MASTER_PASSWORD='xxx'; 87 | Thu Nov 16 15:52:03 2017 - [info] Executing master ip online change script to allow write on the new master: 88 | Thu Nov 16 15:52:03 2017 - [info] /usr/bin/master_ip_online_change --command=start --orig_master_host=10.0.21.17 --orig_master_ip=10.0.21.17 --orig_master_port=3308 --orig_master_user='root' --orig_master_password='xxxxxx' --new_master_host=10.0.21.7 --new_master_ip=10.0.21.7 --new_master_port=3308 --new_master_user='root' --new_master_password='xxxxxx' --orig_master_ssh_user=mha --new_master_ssh_user=mha --ssh_options='-o ServerAliveInterval=60 -o ServerAliveCountMax=20 -o StrictHostKeyChecking=no -o ConnectionAttempts=5 -o PasswordAuthentication=no -o BatchMode=yes -i /home/mha/.ssh/id_rsa' --orig_master_is_new_slave 89 | Thu Nov 16 15:52:04 2017.071473 Set read_only=0 on the new master. 90 | Thu Nov 16 15:52:04 2017.073304 Releasing app user on the new master.. 91 | Thu Nov 16 15:52:04 2017.075662 Starting VIP on the new master.. 92 | Thu Nov 16 15:52:06 2017.306421 set proxysql 10.0.21.5:6032 readwrite ok! 93 | Thu Nov 16 15:52:06 2017.306978 Delete old proxysql write group 10.0.21.17:3308 with group 1 ok! 94 | Thu Nov 16 15:52:06 2017.307767 Insert new proxysql write group 10.0.21.7:3308 with group 1 ok! 95 | Thu Nov 16 15:52:06 2017.308202 Insert new proxysql read group 10.0.21.7:3308 with group 2 ok! 96 | Thu Nov 16 15:52:06 2017.312171 Insert orig master as new proxysql read group 10.0.21.17:3308 with group 2 ok! 97 | Thu Nov 16 15:52:06 2017.312908 insert proxysql repl group on 10.0.21.5:6032 ok! 98 | Thu Nov 16 15:52:06 2017.321330 proxysql load server to runtime ok! 99 | Thu Nov 16 15:52:06 2017.348377 proxysql save server to disk ok! 100 | Thu Nov 16 15:52:06 2017.352720 set proxysql 10.0.21.7:6032 readwrite ok! 101 | Thu Nov 16 15:52:06 2017.353430 Delete old proxysql write group 10.0.21.17:3308 with group 1 ok! 102 | Thu Nov 16 15:52:06 2017.354135 Insert new proxysql write group 10.0.21.7:3308 with group 1 ok! 103 | Thu Nov 16 15:52:06 2017.354895 Insert new proxysql read group 10.0.21.7:3308 with group 2 ok! 104 | Thu Nov 16 15:52:06 2017.359129 Insert orig master as new proxysql read group 10.0.21.17:3308 with group 2 ok! 105 | Thu Nov 16 15:52:06 2017.360009 insert proxysql repl group on 10.0.21.7:6032 ok! 106 | Thu Nov 16 15:52:06 2017.367143 proxysql load server to runtime ok! 107 | Thu Nov 16 15:52:06 2017.388661 proxysql save server to disk ok! 108 | Thu Nov 16 15:52:06 2017 - [info] ok. 109 | Thu Nov 16 15:52:06 2017 - [info] 110 | Thu Nov 16 15:52:06 2017 - [info] * Switching slaves in parallel.. 111 | Thu Nov 16 15:52:06 2017 - [info] 112 | Thu Nov 16 15:52:06 2017 - [info] Unlocking all tables on the orig master: 113 | Thu Nov 16 15:52:06 2017 - [info] Executing UNLOCK TABLES.. 114 | Thu Nov 16 15:52:06 2017 - [info] ok. 115 | Thu Nov 16 15:52:06 2017 - [info] Starting orig master as a new slave.. 116 | Thu Nov 16 15:52:06 2017 - [info] Resetting slave 10.0.21.17(10.0.21.17:3308) and starting replication from the new master 10.0.21.7(10.0.21.7:3308).. 117 | Thu Nov 16 15:52:06 2017 - [info] Executed CHANGE MASTER. 118 | Thu Nov 16 15:52:07 2017 - [info] Slave started. 119 | Thu Nov 16 15:52:07 2017 - [info] All new slave servers switched successfully. 120 | Thu Nov 16 15:52:07 2017 - [info] 121 | Thu Nov 16 15:52:07 2017 - [info] * Phase 5: New master cleanup phase.. 122 | Thu Nov 16 15:52:07 2017 - [info] 123 | Thu Nov 16 15:52:07 2017 - [info] 10.0.21.7: Resetting slave info succeeded. 124 | Thu Nov 16 15:52:07 2017 - [info] Switching master to 10.0.21.7(10.0.21.7:3308) completed successfully. 125 | -------------------------------------------------------------------------------- /bin/master_ip_failover: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) 2011 DeNA Co.,Ltd. 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | 20 | # refer to mha4mysql-manager/samples/scripts/master_ip_failover 21 | 22 | use strict; 23 | use warnings FATAL => 'all'; 24 | 25 | use Getopt::Long; 26 | use MHA::DBHelper; 27 | use Data::Dumper; 28 | use Time::HiRes qw( gettimeofday ); 29 | use Carp; 30 | 31 | use constant SCRIPT_CONF => '/etc/masterha/masterha-script.cnf'; 32 | use constant SCRIPT_MODULE => '/etc/masterha/masterha-script.pm'; 33 | require(SCRIPT_MODULE) or die 'Fail to load ' . SCRIPT_MODULE; 34 | my $config = Extra::MHA::Config->new(SCRIPT_CONF); 35 | die "Need meaningful configuration in " . SCRIPT_CONF if $config->is_empty; 36 | 37 | my ( 38 | $command, $ssh_user, $ssh_options , $orig_master_host, 39 | $orig_master_ip, $orig_master_port, $orig_master_ssh_host, 40 | $orig_master_ssh_port, $orig_master_ssh_ip, $orig_master_ssh_user, 41 | $new_master_host, $new_master_ip, $new_master_port, 42 | $new_master_user, $new_master_password, $new_master_ssh_host, 43 | $new_master_ssh_ip, $new_master_ssh_port, $new_master_ssh_user 44 | ); 45 | GetOptions( 46 | 'command=s' => \$command, 47 | 'ssh_user=s' => \$ssh_user, 48 | 'ssh_options=s' => \$ssh_options, 49 | 'orig_master_host=s' => \$orig_master_host, 50 | 'orig_master_ip=s' => \$orig_master_ip, 51 | 'orig_master_port=i' => \$orig_master_port, 52 | 'orig_master_ssh_host=s' => \$orig_master_ssh_host, 53 | 'orig_master_ssh_ip=s' => \$orig_master_ssh_ip, 54 | 'orig_master_ssh_port=i' => \$orig_master_ssh_port, 55 | 'orig_master_ssh_user=s' => \$orig_master_ssh_user, 56 | 'new_master_host=s' => \$new_master_host, 57 | 'new_master_ip=s' => \$new_master_ip, 58 | 'new_master_port=i' => \$new_master_port, 59 | 'new_master_user=s' => \$new_master_user, 60 | 'new_master_password=s' => \$new_master_password, 61 | 'new_master_ssh_host=s' => \$new_master_ssh_host, 62 | 'new_master_ssh_ip=s' => \$new_master_ssh_ip, 63 | 'new_master_ssh_port=i' => \$new_master_ssh_port, 64 | 'new_master_ssh_user=s' => \$new_master_ssh_user, 65 | ); 66 | 67 | sub current_time_us { 68 | my ( $sec, $microsec ) = gettimeofday(); 69 | my $curdate = localtime($sec); 70 | return $curdate . "." . sprintf( "%06d", $microsec ); 71 | } 72 | 73 | $orig_master_ssh_ip ||= $orig_master_ssh_host 74 | || $orig_master_ip 75 | || $orig_master_host; 76 | $new_master_ssh_ip ||= $new_master_ssh_host 77 | || $new_master_ip 78 | || $new_master_host; 79 | 80 | 81 | 82 | exit &main(); 83 | 84 | sub main { 85 | if ( $command eq "stop" || $command eq "stopssh" ) { 86 | 87 | # $orig_master_host, $orig_master_ip, $orig_master_port are passed. 88 | # If you manage master ip address at global catalog database, 89 | # invalidate orig_master_ip here. 90 | my $exit_code = 1; 91 | eval { 92 | my $cfg = $config->lookup( $orig_master_ip, $orig_master_port ) 93 | or die "lookup failed with $orig_master_ip:$orig_master_port"; 94 | 95 | # proxysql setting 96 | my $proxysql_list = $cfg->{'proxysql'}; 97 | if ( $proxysql_list ) { 98 | foreach my $list (@$proxysql_list) { 99 | my $proxysql_handler = new Extra::MHA::Proxysql(); 100 | $proxysql_handler->connect( $list->{'ip'}, $list->{'port'}, 101 | $list->{'user'}, $list->{'pass'}, "", 1); 102 | next unless $proxysql_handler->{dbh}; 103 | 104 | # delete repl group 105 | if ($proxysql_handler->proxysql_delete_repl_group($list->{'wgroup'}, 106 | $list->{'rgroup'})) { 107 | carp " delete proxysql repl group error!"; 108 | } 109 | else { 110 | print current_time_us() 111 | . " delete proxysql repl group on " 112 | . $list->{'ip'} . ":" . $list->{'port'} 113 | . " ok!\n" 114 | } 115 | if ($proxysql_handler->proxysql_load_server_to_runtime()) { 116 | carp " load servers to runtime error!"; 117 | } 118 | 119 | # set readonly to disable proxysql update configure in runtime; 120 | if ($proxysql_handler->proxysql_readonly()) { 121 | carp " set proxysql readonly failed!"; 122 | } 123 | else { 124 | print current_time_us() 125 | . " set read_only on proxysql " 126 | . $list->{'ip'} . ":" . $list->{'port'} 127 | . " ok!\n"; 128 | } 129 | $proxysql_handler->disconnect(); 130 | } 131 | $exit_code = 0; 132 | } 133 | 134 | if ($cfg->{vip}) { 135 | print "Stopping VIP ..\n"; 136 | my $iphelper = Extra::MHA::IpHelper->new( 137 | host => $orig_master_ssh_ip, 138 | port => $orig_master_ssh_port, 139 | user => $ssh_user, 140 | option => $ssh_options, 141 | ); 142 | eval { 143 | $iphelper->stop_vip( $cfg->{vip} ); 144 | }; 145 | if ($@ && $@ =~ /No route to host/) { 146 | $exit_code = 0; 147 | } 148 | else { 149 | $exit_code = 0; 150 | } 151 | } 152 | }; 153 | if ($@) { 154 | warn "Got Error: $@\n"; 155 | exit $exit_code; 156 | } 157 | exit $exit_code; 158 | } 159 | elsif ( $command eq "start" ) { 160 | 161 | # all arguments are passed. 162 | # If you manage master ip address at global catalog database, 163 | # activate new_master_ip here. 164 | # You can also grant write access (create user, set read_only=0, etc) here. 165 | my $exit_code = 10; 166 | eval { 167 | my $new_master_handler = new Extra::MHA::DBHelper(); 168 | my $cfg = $config->lookup( $new_master_ip, $new_master_port ) 169 | or die "lookup failed with $new_master_ip:$new_master_port"; 170 | 171 | # args: hostname, port, user, password, raise_error_or_not 172 | $new_master_handler->connect( $new_master_ip, $new_master_port, 173 | $new_master_user, $new_master_password, 1 ); 174 | 175 | ## Set read_only=0 on the new master 176 | $new_master_handler->disable_log_bin_local(); 177 | print "Set read_only=0 on the new master.\n"; 178 | $new_master_handler->disable_read_only(); 179 | 180 | ## change rpl_semi_sync variables if enable semi replication 181 | print current_time_us() 182 | . " reset new master rpl_semi_sync variables if enable semi replication\n"; 183 | $new_master_handler->rpl_semi_new_master_set(); 184 | 185 | $new_master_handler->enable_log_bin_local(); 186 | # set event scheduler 187 | print current_time_us() 188 | . " set on the scheduler if new master enable the event scheduler.\n"; 189 | $new_master_handler->set_event_scheduler_on(); 190 | 191 | $new_master_handler->disconnect(); 192 | 193 | if ($cfg->{vip}) { 194 | print "Starting VIP on the new master..\n"; 195 | my $iphelper = Extra::MHA::IpHelper->new( 196 | host => $new_master_ssh_ip, 197 | port => $new_master_ssh_port, 198 | user => $ssh_user, 199 | option => $ssh_options, 200 | ); 201 | $iphelper->start_vip( $cfg->{vip}, 202 | ( $cfg->{vip_on_loopback} ? ('lo') : () ) ); 203 | } 204 | 205 | # proxysql setting 206 | my $proxysql_list = $cfg->{'proxysql'}; 207 | if ($proxysql_list) { 208 | foreach my $list (@$proxysql_list) { 209 | my $proxysql_handler = new Extra::MHA::Proxysql(); 210 | $proxysql_handler->connect($list->{'ip'}, $list->{'port'}, 211 | $list->{'user'}, $list->{'pass'}, 1); 212 | next unless $proxysql_handler->{dbh}; 213 | 214 | # enable proxysql readwrite 215 | if ($proxysql_handler->proxysql_readwrite()) { 216 | carp "set proxysql readwrite error!"; 217 | next; 218 | } 219 | else { 220 | print current_time_us() . " set proxysql " 221 | . $list->{ip} . ":" . $list->{port} 222 | . " readwrite ok!\n" 223 | } 224 | # delete old read/write group with original master 225 | if ($proxysql_handler->proxysql_delete_group($list->{wgroup}, 226 | $list->{rgroup}, $orig_master_host, $orig_master_port)) { 227 | carp "delete old proxysql write group error!"; 228 | } 229 | else { 230 | print current_time_us() 231 | . " Delete old proxysql write group " 232 | . "$orig_master_host:$orig_master_port" 233 | . " with group " . $list->{'wgroup'} . " ok!\n"; 234 | 235 | } 236 | # insert new write group 237 | if ($proxysql_handler->proxysql_insert_new_server($list->{'wgroup'}, 238 | $new_master_host, $new_master_port, 0)) { 239 | carp "insert new write group error!"; 240 | } 241 | else { 242 | print current_time_us() 243 | . " Insert new proxysql write group " 244 | . "$new_master_host:$new_master_port" 245 | . " with group " . $list->{'wgroup'} . " ok!\n"; 246 | } 247 | 248 | # insert new read group 249 | if ($proxysql_handler->proxysql_insert_new_server($list->{'rgroup'}, 250 | $new_master_host, $new_master_port, 0)) { 251 | carp "insert new read group error!"; 252 | } 253 | else { 254 | print current_time_us() 255 | . " Insert new proxysql read group " 256 | . "$new_master_host:$new_master_port" 257 | . " with group " . $list->{'rgroup'} . " ok!\n"; 258 | } 259 | # insert new repl group 260 | if ($proxysql_handler->proxysql_insert_repl_group($list->{'wgroup'}, 261 | $list->{'rgroup'})) { 262 | carp " insert proxysql repl group error!"; 263 | } 264 | else { 265 | print current_time_us() 266 | . " insert proxysql repl group on " 267 | . $list->{'ip'} . ":" . $list->{'port'} 268 | . " ok!\n" 269 | } 270 | # load mysql servers to runtime. 271 | if ($proxysql_handler->proxysql_load_server_to_runtime()) { 272 | carp "proxysql load mysql server to runtime error!"; 273 | } 274 | else { 275 | print current_time_us() . " proxysql load server to runtime ok!\n"; 276 | } 277 | 278 | if ($proxysql_handler->proxysql_save_server_to_disk()) { 279 | carp "proxysql save mysql server to disk error!"; 280 | } 281 | else { 282 | print current_time_us() . " proxysql save server to disk ok!\n"; 283 | } 284 | } 285 | } 286 | 287 | $exit_code = 0; 288 | }; 289 | if ($@) { 290 | warn $@; 291 | 292 | # If you want to continue failover, exit 10. 293 | exit $exit_code; 294 | } 295 | exit $exit_code; 296 | } 297 | elsif ( $command eq "status" ) { 298 | my $cfg = $config->lookup( $orig_master_ip, $orig_master_port ) 299 | or die "lookup failed with $orig_master_ip:$orig_master_port"; 300 | 301 | my $iphelper = Extra::MHA::IpHelper->new( 302 | host => $orig_master_ssh_ip, 303 | port => $orig_master_ssh_port, 304 | user => $ssh_user, 305 | option => $ssh_options, 306 | ); 307 | if (!$cfg->{vip} && $cfg->{'proxysql'} ) { 308 | print current_time_us . " use proxysql without vip configured\n"; 309 | exit 0; 310 | } 311 | elsif ( $iphelper->check_node_vip( $cfg->{vip} ) ) { 312 | print current_time_us() . " vip $cfg->{vip} is configured\n"; 313 | exit 0; 314 | } 315 | else { 316 | print current_time_us() . " neither vip $cfg->{vip} is not configured on host $orig_master_ssh_host nor does not use proxysql\n"; 317 | exit 1; 318 | } 319 | } 320 | else { 321 | &usage(); 322 | exit 1; 323 | } 324 | } 325 | 326 | sub usage { 327 | print 328 | "Usage: master_ip_failover --command=start|stop|stopssh|status --orig_master_host=host --orig_master_ip=ip --orig_master_port=port --new_master_host=host --new_master_ip=ip --new_master_port=port\n"; 329 | } 330 | 331 | -------------------------------------------------------------------------------- /bin/master_ip_online_change: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) 2011 DeNA Co.,Ltd. 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | 20 | ## refer to mha4mysql-manager/samples/scripts/master_ip_online_change 21 | 22 | use strict; 23 | use warnings FATAL => 'all'; 24 | 25 | use Getopt::Long; 26 | use Time::HiRes qw( sleep gettimeofday tv_interval ); 27 | use Data::Dumper; 28 | use Carp; 29 | 30 | use constant SCRIPT_CONF => '/etc/masterha/masterha-script.cnf'; 31 | use constant SCRIPT_MODULE => '/etc/masterha/masterha-script.pm'; 32 | require(SCRIPT_MODULE) or die 'Fail to load ' . SCRIPT_MODULE; 33 | my $config = Extra::MHA::Config->new(SCRIPT_CONF); 34 | die "Need meaningful configuration in " . SCRIPT_CONF if $config->is_empty; 35 | 36 | my $_tstart; 37 | my $_running_interval = 0.1; 38 | my ( 39 | $command, $orig_master_host, $orig_master_ip, 40 | $orig_master_port, $orig_master_user, $orig_master_password, 41 | $orig_master_ssh_user, $orig_master_ssh_host, $orig_master_ssh_ip, 42 | $orig_master_ssh_port, $new_master_host, $new_master_ip, 43 | $new_master_port, $new_master_user, $new_master_password, 44 | $new_master_ssh_user, $new_master_ssh_host, $new_master_ssh_ip, 45 | $new_master_ssh_port, $ssh_options, $orig_master_is_new_slave, 46 | ); 47 | GetOptions( 48 | 'command=s' => \$command, 49 | 'orig_master_host=s' => \$orig_master_host, 50 | 'orig_master_ip=s' => \$orig_master_ip, 51 | 'orig_master_port=i' => \$orig_master_port, 52 | 'orig_master_user=s' => \$orig_master_user, 53 | 'orig_master_password=s' => \$orig_master_password, 54 | 'orig_master_ssh_host=s' => \$orig_master_ssh_host, 55 | 'orig_master_ssh_ip=s' => \$orig_master_ssh_ip, 56 | 'orig_master_ssh_port=i' => \$orig_master_ssh_port, 57 | 'orig_master_ssh_user=s' => \$orig_master_ssh_user, 58 | 'orig_master_is_new_slave' => \$orig_master_is_new_slave, 59 | 'new_master_host=s' => \$new_master_host, 60 | 'new_master_ip=s' => \$new_master_ip, 61 | 'new_master_port=i' => \$new_master_port, 62 | 'new_master_user=s' => \$new_master_user, 63 | 'new_master_password=s' => \$new_master_password, 64 | 'new_master_ssh_host=s' => \$new_master_ssh_host, 65 | 'new_master_ssh_ip=s' => \$new_master_ssh_ip, 66 | 'new_master_ssh_port=i' => \$new_master_ssh_port, 67 | 'new_master_ssh_user=s' => \$new_master_ssh_user, 68 | 'ssh_options=s' => \$ssh_options, 69 | ); 70 | 71 | my $ssh_user = $new_master_ssh_user || $orig_master_ssh_user || 'root'; 72 | 73 | $orig_master_ssh_ip ||= $orig_master_ssh_host 74 | || $orig_master_ip 75 | || $orig_master_host; 76 | $new_master_ssh_ip ||= $new_master_ssh_host 77 | || $new_master_ip 78 | || $new_master_host; 79 | 80 | exit &main(); 81 | 82 | sub current_time_us { 83 | my ( $sec, $microsec ) = gettimeofday(); 84 | my $curdate = localtime($sec); 85 | return $curdate . "." . sprintf( "%06d", $microsec ); 86 | } 87 | 88 | sub sleep_until { 89 | my $elapsed = tv_interval($_tstart); 90 | if ( $_running_interval > $elapsed ) { 91 | sleep( $_running_interval - $elapsed ); 92 | } 93 | } 94 | 95 | sub get_threads_util { 96 | my $dbh = shift; 97 | my $my_connection_id = shift; 98 | my $running_time_threshold = shift; 99 | my $type = shift; 100 | $running_time_threshold = 0 unless ($running_time_threshold); 101 | $type = 0 unless ($type); 102 | my @threads; 103 | 104 | my $sth = $dbh->prepare("SHOW PROCESSLIST"); 105 | $sth->execute(); 106 | 107 | while ( my $ref = $sth->fetchrow_hashref() ) { 108 | my $id = $ref->{Id}; 109 | my $user = $ref->{User}; 110 | my $host = $ref->{Host}; 111 | my $command = $ref->{Command}; 112 | my $state = $ref->{State}; 113 | my $query_time = $ref->{Time}; 114 | my $info = $ref->{Info}; 115 | $info =~ s/^\s*(.*?)\s*$/$1/ if defined($info); 116 | next if ( $my_connection_id == $id ); 117 | next if ( defined($query_time) && $query_time < $running_time_threshold ); 118 | next if ( defined($command) && $command eq "Binlog Dump" ); 119 | next if ( defined($user) && $user eq "system user" ); 120 | next if ( defined($user) && $user eq "event_scheduler" ); 121 | next 122 | if defined($query_time) 123 | && $query_time < $running_time_threshold + 1; 124 | 125 | if ( $type >= 1 ) { 126 | next if ( defined($command) && $command eq "Sleep" ); 127 | next if ( defined($command) && $command eq "Connect" ); 128 | } 129 | 130 | if ( $type >= 2 ) { 131 | next if ( defined($info) && $info =~ m/^select/i ); 132 | next if ( defined($info) && $info =~ m/^show/i ); 133 | } 134 | 135 | push @threads, $ref; 136 | } 137 | return @threads; 138 | } 139 | 140 | sub main { 141 | if ( $command eq "stop" ) { 142 | ## Gracefully killing connections on the current master 143 | # 1. Set read_only= 1 on the new master 144 | # 2. Disable app user so that no app user can establish new connections 145 | # 3. Set read_only= 1 on the current master 146 | # 4. Kill current queries 147 | # * Any database access failure will result in script die. 148 | my $exit_code = 1; 149 | # check new master config before execute switch 150 | my $cfg = $config->lookup( $new_master_ip, $new_master_port ) 151 | or die "lookup failed with $new_master_ip:$new_master_port"; 152 | 153 | eval { 154 | ## Setting read_only=1 on the new master (to avoid accident) 155 | my $new_master_handler = new Extra::MHA::DBHelper(); 156 | my $cfg = $config->lookup( $orig_master_ip, $orig_master_port ) 157 | or die "lookup failed with $orig_master_ip:$orig_master_port"; 158 | 159 | # proxysql setting 160 | my $proxysql_list = $cfg->{'proxysql'}; 161 | if ( $proxysql_list ) { 162 | foreach my $list (@$proxysql_list) { 163 | my $proxysql_handler = new Extra::MHA::Proxysql(); 164 | $proxysql_handler->connect( $list->{'ip'}, $list->{'port'}, 165 | $list->{'user'}, $list->{'pass'}, "", 1); 166 | next unless $proxysql_handler->{dbh}; 167 | 168 | # delete repl group 169 | if ($proxysql_handler->proxysql_delete_repl_group($list->{'wgroup'}, 170 | $list->{'rgroup'})) { 171 | carp " delete proxysql repl group error!"; 172 | } 173 | else { 174 | print current_time_us() 175 | . " delete proxysql repl group on " 176 | . $list->{'ip'} . ":" . $list->{'port'} 177 | . " ok!\n" 178 | } 179 | if ($proxysql_handler->proxysql_load_server_to_runtime()) { 180 | carp " load servers to runtime error!"; 181 | } 182 | 183 | # set readonly to disable proxysql update configure in runtime; 184 | if ($proxysql_handler->proxysql_readonly()) { 185 | carp " set proxysql readonly failed!"; 186 | } 187 | else { 188 | print current_time_us() 189 | . " set read_only on proxysql " 190 | . $list->{'ip'} . ":" . $list->{'port'} 191 | . " ok!\n"; 192 | } 193 | $proxysql_handler->disconnect(); 194 | } 195 | } 196 | 197 | # args: hostname, port, user, password, raise_error(die_on_error)_or_not 198 | $new_master_handler->connect( $new_master_ip, $new_master_port, 199 | $new_master_user, $new_master_password, 1 ); 200 | print current_time_us() . " Set read_only on the new master.. "; 201 | $new_master_handler->enable_read_only(); 202 | if ( $new_master_handler->is_read_only() ) { 203 | print "ok.\n"; 204 | } 205 | else { 206 | die "Failed!\n"; 207 | } 208 | $new_master_handler->disconnect(); 209 | 210 | # Connecting to the orig master, die if any database error happens 211 | my $orig_master_handler = new Extra::MHA::DBHelper(); 212 | $orig_master_handler->connect( $orig_master_ip, $orig_master_port, 213 | $orig_master_user, $orig_master_password, 1 ); 214 | 215 | # set event scheduler 216 | print current_time_us() 217 | . " set off the scheduler if orig master enable the event scheduler.\n"; 218 | $orig_master_handler->set_event_scheduler_off(); 219 | 220 | ## Drop application user so that nobody can connect. Disabling per-session binlog beforehand 221 | $orig_master_handler->disable_log_bin_local(); 222 | print current_time_us() . " Blocking app user on the orig master..\n"; 223 | $orig_master_handler->block_user_regexp( $cfg->{'block_user'}, 224 | $cfg->{'block_host'} ); 225 | 226 | ## Waiting for N * 100 milliseconds so that current connections can exit 227 | my $time_until_read_only = 15; 228 | $_tstart = [gettimeofday]; 229 | my @threads = get_threads_util( $orig_master_handler->{dbh}, 230 | $orig_master_handler->{connection_id} ); 231 | while ( $time_until_read_only > 0 && $#threads >= 0 ) { 232 | if ( $time_until_read_only % 5 == 0 ) { 233 | printf 234 | "%s Waiting all running %d threads are disconnected.. (max %d milliseconds)\n", 235 | current_time_us(), $#threads + 1, $time_until_read_only * 100; 236 | if ( $#threads < 5 ) { 237 | print Data::Dumper->new( [$_] )->Indent(0)->Terse(1)->Dump . "\n" 238 | foreach (@threads); 239 | } 240 | } 241 | sleep_until(); 242 | $_tstart = [gettimeofday]; 243 | $time_until_read_only--; 244 | @threads = get_threads_util( $orig_master_handler->{dbh}, 245 | $orig_master_handler->{connection_id} ); 246 | } 247 | 248 | ## Setting read_only=1 on the current master so that nobody(except SUPER) can write 249 | print current_time_us() . " Set read_only=1 on the orig master.. "; 250 | $orig_master_handler->enable_read_only(); 251 | if ( $orig_master_handler->is_read_only() ) { 252 | print "ok.\n"; 253 | } 254 | else { 255 | die "Failed!\n"; 256 | } 257 | 258 | ## Waiting for M * 100 milliseconds so that current update queries can complete 259 | my $time_until_kill_threads = 5; 260 | @threads = get_threads_util( $orig_master_handler->{dbh}, 261 | $orig_master_handler->{connection_id} ); 262 | while ( $time_until_kill_threads > 0 && $#threads >= 0 ) { 263 | if ( $time_until_kill_threads % 5 == 0 ) { 264 | printf 265 | "%s Waiting all running %d queries are disconnected.. (max %d milliseconds)\n", 266 | current_time_us(), $#threads + 1, $time_until_kill_threads * 100; 267 | if ( $#threads < 5 ) { 268 | print Data::Dumper->new( [$_] )->Indent(0)->Terse(1)->Dump . "\n" 269 | foreach (@threads); 270 | } 271 | } 272 | sleep_until(); 273 | $_tstart = [gettimeofday]; 274 | $time_until_kill_threads--; 275 | @threads = get_threads_util( $orig_master_handler->{dbh}, 276 | $orig_master_handler->{connection_id} ); 277 | } 278 | 279 | ## Terminating all threads 280 | print current_time_us() . " Killing all application threads..\n"; 281 | $orig_master_handler->kill_threads(@threads) if ( $#threads >= 0 ); 282 | print current_time_us() . " done.\n"; 283 | 284 | ## release app user to enable MHA auto change master. 285 | if ($orig_master_handler->is_read_only() 286 | && ($cfg->{vip} || $cfg->{proxysql}) ) { 287 | print current_time_us() . " Release app user on the orig master..\n"; 288 | $orig_master_handler->release_user_regexp( $cfg->{'block_user'}, 289 | $cfg->{'block_host'}); 290 | } 291 | ## change rpl_semi_sync variables if enable semi replication 292 | print current_time_us() 293 | . "reset orig master rpl_semi_sync variables if enable semi replication\n"; 294 | $orig_master_handler->rpl_semi_orig_master_set(); 295 | 296 | $orig_master_handler->enable_log_bin_local(); 297 | $orig_master_handler->disconnect(); 298 | 299 | if ($cfg->{vip}) { 300 | print current_time_us() . " Stopping VIP ..\n"; 301 | my $iphelper = Extra::MHA::IpHelper->new( 302 | host => $orig_master_ssh_ip, 303 | port => $orig_master_ssh_port, 304 | user => $ssh_user, 305 | option => $ssh_options, 306 | ); 307 | $iphelper->stop_vip( $cfg->{vip} ); 308 | } 309 | 310 | ## After finishing the script, MHA executes FLUSH TABLES WITH READ LOCK 311 | $exit_code = 0; 312 | }; 313 | if ($@) { 314 | warn "Got Error: $@\n"; 315 | exit $exit_code; 316 | } 317 | exit $exit_code; 318 | } 319 | elsif ( $command eq "start" ) { 320 | ## Activating master ip on the new master 321 | # 1. Create app user with write privileges 322 | # 2. Moving backup script if needed 323 | # 3. Register new master's ip to the catalog database 324 | 325 | # We don't return error even though activating updatable accounts/ip failed so that we don't interrupt slaves' recovery. 326 | # If exit code is 0 or 10, MHA does not abort 327 | my $exit_code = 10; 328 | eval { 329 | my $new_master_handler = new Extra::MHA::DBHelper(); 330 | my $cfg = $config->lookup( $new_master_ip, $new_master_port ) 331 | or die "lookup failed with $new_master_ip:$new_master_port"; 332 | 333 | # args: hostname, port, user, password, raise_error_or_not 334 | $new_master_handler->connect( $new_master_ip, $new_master_port, 335 | $new_master_user, $new_master_password, 1 ); 336 | 337 | ## Set read_only=0 on the new master 338 | $new_master_handler->disable_log_bin_local(); 339 | print current_time_us() . " Set read_only=0 on the new master.\n"; 340 | $new_master_handler->disable_read_only(); 341 | 342 | ## Creating an app user on the new master 343 | print current_time_us() . " Releasing app user on the new master..\n"; 344 | $new_master_handler->release_user_regexp( $cfg->{'block_user'}, 345 | $cfg->{'block_host'} ); 346 | 347 | ## change rpl_semi_sync variables if enable semi replication 348 | print current_time_us() 349 | . " reset new master rpl_semi_sync variables if enable semi replication\n"; 350 | $new_master_handler->rpl_semi_new_master_set(); 351 | 352 | $new_master_handler->enable_log_bin_local(); 353 | 354 | # set event scheduler 355 | print current_time_us() 356 | . " set on the scheduler if new master enable the event scheduler.\n"; 357 | $new_master_handler->set_event_scheduler_on(); 358 | 359 | $new_master_handler->disconnect(); 360 | 361 | if ($cfg->{vip}) { 362 | print current_time_us() . " Starting VIP on the new master..\n"; 363 | my $iphelper = Extra::MHA::IpHelper->new( 364 | host => $new_master_ssh_ip, 365 | port => $new_master_ssh_port, 366 | user => $ssh_user, 367 | option => $ssh_options, 368 | ); 369 | $iphelper->start_vip( $cfg->{vip}, 370 | ( $cfg->{vip_on_loopback} ? ('lo') : () ) ); 371 | } 372 | 373 | # proxysql setting 374 | my $proxysql_list = $cfg->{'proxysql'}; 375 | if ($proxysql_list) { 376 | foreach my $list (@$proxysql_list) { 377 | my $proxysql_handler = new Extra::MHA::Proxysql(); 378 | $proxysql_handler->connect($list->{'ip'}, $list->{'port'}, 379 | $list->{'user'}, $list->{'pass'}, 1); 380 | next unless $proxysql_handler->{dbh}; 381 | 382 | # enable proxysql readwrite 383 | if ($proxysql_handler->proxysql_readwrite()) { 384 | carp "set proxysql readwrite error!"; 385 | next; 386 | } 387 | else { 388 | print current_time_us() . " set proxysql " 389 | . $list->{ip} . ":" . $list->{port} 390 | . " readwrite ok!\n" 391 | } 392 | # delete old read/write group with original master 393 | if ($proxysql_handler->proxysql_delete_group($list->{wgroup}, 394 | $list->{rgroup}, $orig_master_host, $orig_master_port)) { 395 | carp "delete old proxysql write group error!"; 396 | } 397 | else { 398 | print current_time_us() 399 | . " Delete old proxysql write group " 400 | . "$orig_master_host:$orig_master_port" 401 | . " with group " . $list->{'wgroup'} . " ok!\n"; 402 | 403 | } 404 | # insert new write group 405 | if ($proxysql_handler->proxysql_insert_new_server($list->{'wgroup'}, 406 | $new_master_host, $new_master_port, 0)) { 407 | carp "insert new write group error!"; 408 | } 409 | else { 410 | print current_time_us() 411 | . " Insert new proxysql write group " 412 | . "$new_master_host:$new_master_port" 413 | . " with group " . $list->{'wgroup'} . " ok!\n"; 414 | } 415 | 416 | # insert new read group 417 | if ($proxysql_handler->proxysql_insert_new_server($list->{'rgroup'}, 418 | $new_master_host, $new_master_port, 0)) { 419 | carp "insert new read group error!"; 420 | } 421 | else { 422 | print current_time_us() 423 | . " Insert new proxysql read group " 424 | . "$new_master_host:$new_master_port" 425 | . " with group " . $list->{'rgroup'} . " ok!\n"; 426 | } 427 | # insert orig master as new slave 428 | my $orig_master_handler = new Extra::MHA::DBHelper(); 429 | $orig_master_handler->connect( $orig_master_ip, $orig_master_port, 430 | $orig_master_user, $orig_master_password, 1 ); 431 | if ($orig_master_handler->is_read_only()) { 432 | if ($proxysql_handler->proxysql_insert_new_server($list->{'rgroup'}, 433 | $orig_master_host, $orig_master_port, 30)) { 434 | carp "insert orig master as read group error!"; 435 | } 436 | else { 437 | print current_time_us() 438 | . " Insert orig master as new proxysql read group " 439 | . "$orig_master_host:$orig_master_port" 440 | . " with group " . $list->{'rgroup'} . " ok!\n"; 441 | } 442 | } 443 | # insert new repl group 444 | if ($proxysql_handler->proxysql_insert_repl_group($list->{'wgroup'}, 445 | $list->{'rgroup'})) { 446 | carp " insert proxysql repl group error!"; 447 | } 448 | else { 449 | print current_time_us() 450 | . " insert proxysql repl group on " 451 | . $list->{'ip'} . ":" . $list->{'port'} 452 | . " ok!\n" 453 | } 454 | # load mysql servers to runtime. 455 | if ($proxysql_handler->proxysql_load_server_to_runtime()) { 456 | carp "proxysql load mysql server to runtime error!"; 457 | } 458 | else { 459 | print current_time_us() . " proxysql load server to runtime ok!\n"; 460 | } 461 | 462 | if ($proxysql_handler->proxysql_save_server_to_disk()) { 463 | carp "proxysql save mysql server to disk error!"; 464 | } 465 | else { 466 | print current_time_us() . " proxysql save server to disk ok!\n"; 467 | } 468 | } 469 | } 470 | 471 | ## Update master ip on the catalog database, etc 472 | $exit_code = 0; 473 | }; 474 | if ($@) { 475 | warn "Got Error: $@\n"; 476 | exit $exit_code; 477 | } 478 | exit $exit_code; 479 | } 480 | elsif ( $command eq "status" ) { 481 | my $cfg = $config->lookup( $orig_master_ip, $orig_master_port ) 482 | or die "lookup failed with $orig_master_ip:$orig_master_port"; 483 | 484 | my $iphelper = Extra::MHA::IpHelper->new( 485 | host => $orig_master_ssh_ip, 486 | port => $orig_master_ssh_port, 487 | user => $ssh_user, 488 | option => $ssh_options, 489 | ); 490 | if ( !$cfg->{vip} && $cfg->{'proxysql'} ) { 491 | print "use proxysql without vip configured\n"; 492 | exit 0; 493 | } 494 | elsif ( $iphelper->check_node_vip( $cfg->{vip} ) ) { 495 | print "vip $cfg->{vip} is configured\n"; 496 | exit 0; 497 | } 498 | else { 499 | warn "neither vip $cfg->{vip} is not configured on host $orig_master_ssh_host nor does not use proxysql\n"; 500 | exit 1; 501 | } 502 | } 503 | else { 504 | &usage(); 505 | exit 1; 506 | } 507 | } 508 | 509 | sub usage { 510 | print 511 | "Usage: master_ip_online_change --command=start|stop|status --orig_master_host=host --orig_master_ip=ip --orig_master_port=port --new_master_host=host --new_master_ip=ip --new_master_port=port\n"; 512 | die; 513 | } 514 | 515 | -------------------------------------------------------------------------------- /masterha/masterha-script.pm: -------------------------------------------------------------------------------- 1 | package Extra::MHA; 2 | 3 | # kc20130626 to suppport MHA hooks 4 | 5 | our $VERSION = '0.1'; 6 | 7 | 1; 8 | 9 | package Extra::MHA::Config; 10 | 11 | use strict; 12 | use warnings; 13 | use Carp; 14 | use Data::Dumper; 15 | 16 | sub new { 17 | my ( $class, $filename ) = @_; 18 | bless { 19 | file => $filename, 20 | servers => _load_config($filename), 21 | }, 22 | __PACKAGE__; 23 | } 24 | 25 | sub lookup { 26 | my ( $self, $ip, $port ) = @_; 27 | return _lookup( $self->{servers}, $ip, $port ); 28 | } 29 | 30 | sub is_empty { 31 | my ($self) = @_; 32 | return @{ $self->{servers} } == 0; 33 | } 34 | 35 | sub _lookup { 36 | my ( $servers, $ip, $port ) = @_; 37 | for my $s ( @{$servers} ) { 38 | if ( $s->{ip} eq $ip && $s->{port} eq $port ) { 39 | return $s; 40 | } 41 | } 42 | } 43 | 44 | sub _load_config { 45 | my ($file) = @_; 46 | my @lines = (); 47 | if ( open my $fh, '<', $file ) { 48 | @lines = <$fh>; 49 | chomp(@lines); 50 | close $fh; 51 | } 52 | return _parse( \@lines ); 53 | } 54 | 55 | sub _parse { 56 | my ($lines) = @_; 57 | 58 | my %global = (); 59 | my @lookup = (); 60 | 61 | my @block = (); 62 | my %setting = (); 63 | my $ctx = ""; 64 | my $i = 0; 65 | my %seen_server = (); 66 | my %seen_vip = (); 67 | my %seen_rip = (); 68 | for ( @{$lines} ) { 69 | $i++; 70 | next if /^\s*(#.*)?$/; # blank line 71 | s/#.*$|\s+$//; # trim comment or postfix space 72 | 73 | # default setting 74 | if (/^(\w+)\s+([^#]*)/) { 75 | $global{$1} = $2; 76 | $ctx = 'global'; 77 | } 78 | 79 | # servers line 80 | elsif (/^\d/) { 81 | if ( $ctx && $ctx eq "setting" ) { # finish a group 82 | for (@block) { 83 | $_ = { %setting, %{$_} }; # ip port won't be overwritten 84 | } 85 | push @lookup, @block; 86 | @block = (); 87 | %setting = (); 88 | } 89 | 90 | $ctx = 'server'; 91 | my @servers = /(\d+\.\d+\.\d+\.\d+:\d+)/g; 92 | for (@servers) { 93 | my ( $ip, $port ) = split /:/; 94 | push @{ $seen_server{"$ip:$port"} }, $i; 95 | push @{ $seen_rip{$ip} }, $i; 96 | push @block, { ip => $ip, port => $port }; 97 | } 98 | } 99 | 100 | # server settings 101 | elsif (/^\s+\w/) { 102 | $ctx = 'setting'; 103 | if ( my ( $k, $v ) = /^\s+(\w+)\s+([^#]+)/ ) { 104 | push( @{ $seen_vip{$v} }, $i ) if $k eq "vip"; 105 | if ( $k eq "proxysql" ) { 106 | $setting{$k} = _proxysql_parse($v); 107 | } 108 | else { 109 | $setting{$k} = $v; 110 | } 111 | } 112 | } 113 | } 114 | 115 | # wrap up 116 | if (@block) { 117 | for (@block) { 118 | $_ = { %setting, %{$_} }; # ip port won't be overwritten 119 | } 120 | push @lookup, @block; 121 | } 122 | 123 | for my $server (@lookup) { 124 | for my $k ( keys %global ) { 125 | $server->{$k} = $global{$k} unless defined $server->{$k}; 126 | } 127 | } 128 | 129 | while ( my ( $k, $v ) = each %seen_server ) { 130 | croak "detected server duplication $k at lines @{$v}" if @{$v} > 1; 131 | } 132 | while ( my ( $k, $v ) = each %seen_vip ) { 133 | croak "detected vip duplication $k at lines @{$v}" if @{$v} > 1; 134 | } 135 | for my $vip ( keys %seen_vip ) { 136 | if ( $seen_rip{$vip} ) { 137 | croak 138 | "detected vip $vip at line @{$seen_vip{$vip}} duplicates rip at line @{$seen_rip{$vip}}"; 139 | } 140 | } 141 | for my $s (@lookup) { 142 | my $server = "$s->{ip}:$s->{port}"; 143 | carp "vip is not configured for $server at line @{$seen_server{$server}}" 144 | unless $s->{vip}; 145 | croak 146 | "block_user and block_host must be both or none for $server at line @{$seen_server{$server}}" 147 | unless ( $s->{block_user} && $s->{block_host} ) 148 | || ( !$s->{block_user} && !$s->{block_host} ); 149 | } 150 | 151 | return \@lookup; 152 | } 153 | 154 | sub _proxysql_parse { 155 | my $list = shift; 156 | my @proxysql = (); 157 | my $n = 0; 158 | # such as: admin:admin@10.0.21.5:6032:r1:w2,admin:admin@10.0.21.7:6032:r1:w2 159 | foreach (split(/,/, $list)) { 160 | if (/(\S+?):(\S+?)\@(\S+?):(\d+):w(\d+):r(\d+)/) { 161 | $proxysql[$n]->{user} = $1; 162 | $proxysql[$n]->{pass} = $2; 163 | $proxysql[$n]->{ip} = $3; 164 | $proxysql[$n]->{port} = $4; 165 | $proxysql[$n]->{wgroup} = $5; 166 | $proxysql[$n]->{rgroup} = $6; 167 | $n++; 168 | } 169 | } 170 | return \@proxysql; 171 | } 172 | 173 | 1; 174 | 175 | package Extra::MHA::DBHelper; 176 | 177 | use base 'MHA::DBHelper'; 178 | 179 | use strict; 180 | use warnings; 181 | use Carp; 182 | 183 | use constant Get_Privileges_SQL => "SHOW GRANTS"; 184 | use constant Select_User_Regexp_SQL => 185 | "SELECT user, host, password FROM mysql.user WHERE user REGEXP ? AND host REGEXP ?"; 186 | 187 | # for MySQL 5.7 or above version 188 | use constant Select_User_Regexp_New_SQL => 189 | "SELECT user, host, authentication_string AS password FROM mysql.user WHERE user REGEXP ? AND host REGEXP ?"; 190 | use constant Get_Version_SQL => "SELECT LEFT(VERSION(), 3) AS Value"; 191 | 192 | use constant Set_Password_SQL => "SET PASSWORD FOR ?\@? = ?"; 193 | use constant Granted_Privileges => 194 | '^GRANT ([A-Z, ]+) ON (`\\w+`|\\*)\\.\\* TO'; # poor match on db 195 | use constant Old_Password_Length => 16; 196 | use constant Blocked_Empty_Password => '?' x 41; 197 | use constant Blocked_Old_Password_Head => '~' x 25; 198 | use constant Blocked_New_Password_Regexp => qr/^[0-9a-fA-F]{40}\*$/o; 199 | use constant Released_New_Password_Regexp => qr/^\*[0-9a-fA-F]{40}$/o; 200 | use constant Set_Rpl_Semi_Sync_Master_OFF => "SET GLOBAL rpl_semi_sync_master_enabled = OFF"; 201 | use constant Set_Rpl_Semi_Sync_Master_ON => "SET GLOBAL rpl_semi_sync_master_enabled = ON"; 202 | use constant Set_Rpl_Semi_Sync_Master_Timeout => "SET GLOBAL rpl_semi_sync_master_timeout = 2000"; 203 | use constant Set_Rpl_Semi_Sync_Slave_OFF => "SET GLOBAL rpl_semi_sync_slave_enabled = OFF"; 204 | use constant Set_Rpl_Semi_Sync_Slave_On => "SET GLOBAL rpl_semi_sync_slave_enabled = ON"; 205 | use constant Get_Event_Scheduler_SQL => 206 | "SELECT EVENT_NAME, EVENT_SCHEMA, STATUS FROM information_schema.EVENTS"; 207 | use constant Set_Event_Scheduler_ON => "SET GLOBAL event_scheduler = ON"; 208 | use constant Set_Event_Scheduler_OFF => "SET GLOBAL event_scheduler = OFF"; 209 | use constant Enable_Event_Scheduler => "ALTER EVENT %s ENABLE"; 210 | use constant Disable_Event_Scheduler => "ALTER EVENT %s DISABLE"; 211 | 212 | sub new { 213 | my ($class) = @_; 214 | bless {}, __PACKAGE__; 215 | } 216 | 217 | sub _get_variable { 218 | my $dbh = shift; 219 | my $query = shift; 220 | my $sth = $dbh->prepare($query); 221 | $sth->execute(); 222 | my $href = $sth->fetchrow_hashref; 223 | return $href->{Value} || 5.5; 224 | } 225 | 226 | sub _get_version { 227 | my $dbh = shift; 228 | my $value = _get_variable($dbh, Get_Version_SQL); 229 | return $value; 230 | } 231 | 232 | # see http://code.openark.org/blog/mysql/blocking-user-accounts 233 | sub _blocked_password { 234 | my $password = shift; 235 | if ( $password eq '' ) { 236 | return Blocked_Empty_Password; 237 | } 238 | elsif ( length($password) == Old_Password_Length ) { 239 | return Blocked_Old_Password_Head . $password; 240 | } 241 | elsif ( $password =~ Released_New_Password_Regexp ) { 242 | return join( "", reverse( split //, $password ) ); 243 | } 244 | else { 245 | return; 246 | } 247 | } 248 | 249 | sub _released_password { 250 | my $password = shift; 251 | if ( $password eq Blocked_Empty_Password ) { 252 | return ''; 253 | } 254 | elsif ( index( $password, Blocked_Old_Password_Head ) == 0 ) { 255 | return substr( $password, length(Blocked_Old_Password_Head) ); 256 | } 257 | elsif ( $password =~ Blocked_New_Password_Regexp ) { 258 | return join( "", reverse( split //, $password ) ); 259 | } 260 | else { 261 | return; 262 | } 263 | } 264 | 265 | sub _block_release_user_by_regexp { 266 | my ( $dbh, $user, $host, $block ) = @_; 267 | #my $users_to_block = $dbh->selectall_arrayref( Select_User_Regexp_SQL, { Slice => {} }, 268 | #$user, $host ); 269 | my $users_to_block = do { 270 | if ( _get_version($dbh) >= 5.7 ) { 271 | $dbh->selectall_arrayref( Select_User_Regexp_New_SQL, { Slice => {} }, 272 | $user, $host ); 273 | } 274 | else { 275 | $dbh->selectall_arrayref( Select_User_Regexp_SQL, { Slice => {} }, 276 | $user, $host ); 277 | } 278 | }; 279 | my $failure = 0; 280 | for my $u ( @{$users_to_block} ) { 281 | my $password = 282 | $block 283 | ? _blocked_password( $u->{password} ) 284 | : _released_password( $u->{password} ); 285 | if ( defined $password ) { 286 | my $ret = 287 | $dbh->do( Set_Password_SQL, undef, $u->{user}, $u->{host}, $password ); 288 | unless ( $ret eq "0E0" ) { 289 | $failure++; 290 | } 291 | } 292 | } 293 | return $failure; 294 | } 295 | 296 | sub block_user_regexp { 297 | my ( $self, $user, $host ) = @_; 298 | return _block_release_user_by_regexp( $self->{dbh}, $user, $host, 1 ); 299 | } 300 | 301 | sub release_user_regexp { 302 | my ( $self, $user, $host ) = @_; 303 | return _block_release_user_by_regexp( $self->{dbh}, $user, $host, 0 ); 304 | } 305 | 306 | sub rpl_semi_orig_master_set { 307 | my $self = shift; 308 | my $status = $self->show_variable("rpl_semi_sync_master_enabled") || ''; 309 | if ($status eq "ON") { 310 | $self->execute(Set_Rpl_Semi_Sync_Master_OFF); 311 | $self->execute(Set_Rpl_Semi_Sync_Slave_On); 312 | } 313 | } 314 | 315 | sub rpl_semi_new_master_set { 316 | my $self = shift; 317 | my $status = $self->show_variable("rpl_semi_sync_slave_enabled") || ''; 318 | if ($status eq "ON") { 319 | $self->execute(Set_Rpl_Semi_Sync_Slave_OFF); 320 | $self->execute(Set_Rpl_Semi_Sync_Master_ON); 321 | $self->execute(Set_Rpl_Semi_Sync_Master_Timeout); 322 | } 323 | } 324 | 325 | sub _get_event_info { 326 | my $dbh = shift; 327 | my %events; 328 | my $sth = $dbh->selectall_arrayref(Get_Event_Scheduler_SQL); 329 | foreach my $k (@$sth) { 330 | my ($name, $schema, $status) = @$k; 331 | if ($name && $schema) { 332 | $events{"$schema.$name"} = $status; 333 | } 334 | } 335 | return %events; 336 | } 337 | 338 | sub set_event_scheduler_on { 339 | my $self = shift; 340 | my $status = $self->show_variable("event_scheduler") || ''; 341 | if ($status eq "OFF") { 342 | $self->execute(Set_Event_Scheduler_ON); 343 | my %events = _get_event_info($self->{dbh}); 344 | foreach my $k (keys %events) { 345 | if ($events{$k} =~ /DISABLED/i) { 346 | my $event_sql = sprintf(Enable_Event_Scheduler, $k); 347 | $self->execute($event_sql); 348 | } 349 | } 350 | } 351 | } 352 | 353 | sub set_event_scheduler_off { 354 | my $self = shift; 355 | my $status = $self->show_variable("event_scheduler") || ''; 356 | if ($status eq "ON") { 357 | $self->execute(Set_Event_Scheduler_OFF); 358 | my %events = _get_event_info($self->{dbh}); 359 | foreach my $k (keys %events) { 360 | if ($events{$k} =~ /ENABLED/i) { 361 | my $event_sql = sprintf(Disable_Event_Scheduler, $k); 362 | $self->execute($event_sql); 363 | } 364 | } 365 | } 366 | } 367 | 368 | 1; 369 | 370 | package Extra::MHA::IpHelper; 371 | 372 | # helps to manipulate VIP on target host 373 | 374 | use strict; 375 | use warnings; 376 | use Carp; 377 | 378 | sub new { 379 | my ( $class, %args ) = @_; 380 | croak "missing host to check against" unless $args{host}; 381 | my $self = {}; 382 | bless $self, $class; 383 | $self->{host} = $args{host}; 384 | $self->{port} = $args{port} || 22; 385 | $self->{user} = $args{user} || 'root'; 386 | $self->{option}= $args{option}; 387 | 388 | return $self; 389 | } 390 | 391 | # see perlsec 392 | sub _safe_qx { 393 | my (@cmd) = @_; 394 | use English '-no_match_vars'; 395 | my $pid; 396 | croak "Can't fork: $!" unless defined( $pid = open( KID, "-|" ) ); 397 | if ($pid) { # parent 398 | if (wantarray) { 399 | my @output = ; 400 | close KID; 401 | return @output; 402 | } 403 | else { 404 | local $/; # slurp mode 405 | my $output = ; 406 | close KID; 407 | return $output; 408 | } 409 | } 410 | else { 411 | my @temp = ( $EUID, $EGID ); 412 | my $orig_uid = $UID; 413 | my $orig_gid = $GID; 414 | $EUID = $UID; 415 | $EGID = $GID; 416 | 417 | # Drop privileges 418 | $UID = $orig_uid; 419 | $GID = $orig_gid; 420 | 421 | # Make sure privs are really gone 422 | ( $EUID, $EGID ) = @temp; 423 | die "Can’t drop privileges" 424 | unless $UID == $EUID && $GID eq $EGID; 425 | $ENV{PATH} = "/bin:/usr/bin"; # Minimal PATH. 426 | # Consider sanitizing the environment even more. 427 | exec @cmd 428 | or die "can’t exec m$cmd[0]: $!"; 429 | } 430 | } 431 | 432 | sub ssh_cmd { 433 | my ( $self, $cmd ) = @_; 434 | $self->{user} ||= 'root'; 435 | $self->{port} ||= 22; 436 | my @cmd = (); 437 | push @cmd, 'ssh'; 438 | push @cmd, split(/\s/, $self->{option}) if defined $self->{option}; 439 | push @cmd, '-p', $self->{port}; 440 | push @cmd, '-l', $self->{user}; 441 | push @cmd, $self->{host}; 442 | push @cmd, $cmd; 443 | return @cmd; 444 | } 445 | 446 | sub run_ssh_cmd { 447 | my ( $self, $cmd ) = @_; 448 | my @cmd = $self->ssh_cmd($cmd); 449 | return _safe_qx(@cmd); 450 | } 451 | 452 | sub assert_status { 453 | my ( $high, $low ) = get_run_status(); 454 | if ( $high || $low ) { 455 | croak "command error $high:$low"; 456 | } 457 | } 458 | 459 | sub get_run_status { 460 | return ( ( $? >> 8 ), ( $? & 0xff ) ); # high, low 461 | } 462 | 463 | sub get_ipaddr { 464 | my ($self) = @_; 465 | my $sudo = $self->{user} ne 'root' ? 'sudo' : ''; 466 | chomp( my @ipaddr = $self->run_ssh_cmd("$sudo /sbin/ip addr") ); 467 | assert_status(); 468 | return \@ipaddr; 469 | } 470 | 471 | sub parse_ipaddr { 472 | my $output = shift; 473 | my %intf = (); 474 | my $name; 475 | for ( @{$output} ) { 476 | if (/^\d+: (\S+): <[^,]+(?:,[^,]+)*> mtu \d+ qdisc \w+/) { 477 | $name = $1; 478 | $name =~ s/\@.*//g if $name =~ /\@/; 479 | } 480 | elsif (/^\s+link\/(\w+) (\S+) brd (\S+)/) { 481 | $intf{$name}{'link'} = { type => $1, mac => $2, brd => $3 }; 482 | } 483 | elsif (/^\s+inet ([\d.]+)\/(\d+) (?:brd ([\d.]+))?/) { 484 | push @{ $intf{$name}{inet} }, 485 | { ip => $1, bits => $2, ( $3 ? ( brd => $3 ) : () ) }; 486 | } 487 | elsif (/^\w+inet6 ([\d:]+)\/(\d+)/) { 488 | push @{ $intf{$name}{inet6} }, { ip => $1, bits => $2 }; 489 | } 490 | } 491 | return \%intf; 492 | } 493 | 494 | sub _get_numeric_ipv4 { 495 | my @parts = split /\./, shift; 496 | return ( $parts[0] << 24 ) + ( $parts[1] << 16 ) + ( $parts[2] << 8 ) + 497 | $parts[3]; 498 | } 499 | 500 | sub _find_dev { 501 | my ( $intf, $vip ) = @_; 502 | for my $dev ( keys %{$intf} ) { 503 | my $inet = $intf->{$dev}{inet} or next; 504 | for my $addr ( @{$inet} ) { 505 | my $m = ~( ( 1 << ( 32 - $addr->{bits} ) ) - 1 ); 506 | my $ip1 = _get_numeric_ipv4( $addr->{ip} ); 507 | my $ip2 = _get_numeric_ipv4($vip); 508 | if ( ( $ip1 & $m ) == ( $ip2 & $m ) ) { 509 | return ( $vip, $addr->{bits}, $dev ); 510 | } 511 | } 512 | } 513 | return; 514 | } 515 | 516 | # is vip configured? 517 | sub _check_vip { 518 | my ( $intf, $vip ) = @_; 519 | for my $dev ( keys %{$intf} ) { 520 | my $inet = $intf->{$dev}{inet} or next; 521 | my $i = 0; 522 | for my $addr ( @{$inet} ) { 523 | if ( $addr->{ip} eq $vip ) { 524 | return ( $vip, $addr->{bits}, $dev ) 525 | if $i > 0; # 1st entry is RIP rather than VIP 526 | } 527 | $i++; 528 | } 529 | } 530 | return; 531 | } 532 | 533 | sub find_dev { 534 | my ( $self, $vip ) = @_; 535 | 536 | my $output = $self->get_ipaddr() or return; 537 | my $intf = parse_ipaddr($output); 538 | 539 | return _find_dev( $intf, $vip ); 540 | } 541 | 542 | sub find_dev_with_check { 543 | my ( $self, $vip ) = @_; 544 | 545 | my $output = $self->get_ipaddr() or return; 546 | my $intf = parse_ipaddr($output); 547 | 548 | return 1 if _check_vip( $intf, $vip ); 549 | return _find_dev( $intf, $vip ); 550 | } 551 | 552 | sub check_node_vip { 553 | my ( $self, $vip ) = @_; 554 | 555 | my $output = $self->get_ipaddr() or return; 556 | my $intf = parse_ipaddr($output); 557 | 558 | return _check_vip( $intf, $vip ); 559 | } 560 | 561 | sub stop_vip { 562 | my ( $self, $vip ) = @_; 563 | my ( $ip, $bits, $dev ) = $self->check_node_vip($vip) 564 | or croak "vip $vip is not configured on the node"; 565 | 566 | my $sudo = $self->{user} ne 'root' ? 'sudo' : ''; 567 | $self->run_ssh_cmd("$sudo /sbin/ip addr del $ip/$bits dev $dev"); 568 | assert_status(); 569 | return ( $ip, $bits, $dev ); 570 | } 571 | 572 | sub start_vip { 573 | my ( $self, $vip, $dev ) = @_; 574 | my @vip = $self->find_dev_with_check($vip); 575 | croak "vip $vip is already configured on the node" 576 | if @vip == 1; # some suck trick 577 | $dev ||= $vip[2]; # third component 578 | croak "vip $vip does not match any device" unless defined $dev; 579 | 580 | my $sudo = $self->{user} ne 'root' ? 'sudo' : ''; 581 | $self->run_ssh_cmd( "$sudo /sbin/ip addr add $vip dev $dev" 582 | . ( $dev =~ /^lo/ ? "" : "; $sudo /sbin/arping -U -I $dev -c 3 $vip" ) ); 583 | assert_status(); 584 | } 585 | 586 | 1; 587 | 588 | 589 | package Extra::MHA::Proxysql; 590 | 591 | use strict; 592 | use warnings; 593 | use Carp; 594 | 595 | use constant Proxysql_Read_Only => "PROXYSQL READONLY"; 596 | use constant Proxysql_Read_Write => "PROXYSQL READWRITE"; 597 | use constant Proxysql_Load_Variable_To_Runtime => "LOAD MYSQL VARIABLES TO RUNTIME"; 598 | use constant Proxysql_Load_Servers_To_Runtime => "LOAD MYSQL SERVERS TO RUNTIME"; 599 | use constant Proxysql_Save_Variable_To_Disk => "SAVE MYSQL SERVERS TO DISK"; 600 | 601 | use constant Proxysql_Delete_Repl_Group => 602 | "DELETE FROM mysql_replication_hostgroups WHERE writer_hostgroup = ? AND reader_hostgroup = ?"; 603 | use constant Proxysql_Insert_Repl_Group => 604 | "REPLACE INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, comment) " 605 | . "VALUES (?, ?, ?)"; 606 | use constant Proxysql_Delete_Hostgroup => 607 | "DELETE FROM mysql_servers WHERE hostgroup_id in (?, ?) AND hostname = ? AND port = ?"; 608 | use constant Proxysql_Insert_New_Server => 609 | "REPLACE INTO mysql_servers " . 610 | "(hostgroup_id, hostname, port, status, weight, max_connections, max_replication_lag) " . 611 | "VALUES (?, ?, ?, ?, ?, ?, ?)"; 612 | 613 | sub new { 614 | my ($class) = @_; 615 | bless {}, __PACKAGE__; 616 | } 617 | 618 | sub connect { 619 | my $self = shift; 620 | my $host = shift; 621 | my $port = shift; 622 | my $user = shift; 623 | my $password = shift; 624 | my $database = shift; 625 | my $raise_error = shift; 626 | $raise_error = 0 if ( !defined($raise_error) ); 627 | my $defaults = { 628 | PrintError => 0, 629 | RaiseError => ( $raise_error ? 1 : 0 ), 630 | }; 631 | 632 | $database ||= ""; 633 | my $dbh = eval { 634 | DBI->connect("DBI:mysql:database=$database;host=$host;port=$port", 635 | $user, $password, $defaults); 636 | }; 637 | if (!$dbh && $@) { 638 | carp "get proxysql connect for $host:$port error:$@"; 639 | $self->{dbh}->undef; 640 | } 641 | $self->{dbh} = $dbh; 642 | } 643 | 644 | sub disconnect { 645 | my $self = shift; 646 | $self->{dbh}->disconnect(); 647 | } 648 | 649 | 650 | sub proxysql_readonly { 651 | my $self = shift; 652 | my $failure = 0; 653 | my $ret = $self->{dbh}->do(Proxysql_Read_Only); 654 | unless ($ret eq "0E0") { 655 | $failure++; 656 | } 657 | else { 658 | $self->{dbh}->do(Proxysql_Load_Variable_To_Runtime); 659 | } 660 | return $failure; 661 | } 662 | 663 | sub proxysql_readwrite { 664 | my $self = shift; 665 | my $failure = 0; 666 | my $ret = $self->{dbh}->do(Proxysql_Read_Write); 667 | unless ($ret eq "0E0") { 668 | $failure++; 669 | } 670 | else { 671 | $self->{dbh}->do(Proxysql_Load_Variable_To_Runtime); 672 | } 673 | return $failure; 674 | } 675 | 676 | sub proxysql_delete_repl_group { 677 | my $self = shift; 678 | my $wgroup = shift; 679 | my $rgroup = shift; 680 | my $failure = 0; 681 | if ($wgroup && $rgroup) { 682 | eval { 683 | $self->{dbh}->do(Proxysql_Delete_Repl_Group, undef, $wgroup, $rgroup); 684 | }; 685 | if ($@) { 686 | $failure++; 687 | } 688 | } 689 | else { 690 | $failure++; 691 | } 692 | return $failure; 693 | } 694 | 695 | sub proxysql_insert_repl_group { 696 | my $self = shift; 697 | my $wgroup = shift; 698 | my $rgroup = shift; 699 | my $failure = 0; 700 | if ($wgroup && $rgroup) { 701 | eval { 702 | $self->{dbh}->do(Proxysql_Insert_Repl_Group, 703 | undef, $wgroup, $rgroup, "MHA switch proxysql"); 704 | }; 705 | if ($@) { 706 | $failure++; 707 | } 708 | } 709 | else { 710 | $failure++ 711 | } 712 | return $failure; 713 | } 714 | 715 | sub proxysql_delete_group { 716 | my ($self, $wgroup, $rgroup, $host, $port) = @_; 717 | my $failure = 0; 718 | if ($wgroup && $rgroup && $host && $port) { 719 | eval { 720 | $self->{dbh}->do(Proxysql_Delete_Hostgroup, undef, $wgroup, $rgroup, $host, $port); 721 | }; 722 | if ($@) { 723 | $failure++; 724 | } 725 | } 726 | else { 727 | $failure++; 728 | } 729 | return $failure; 730 | } 731 | 732 | sub proxysql_insert_new_server { 733 | my ($self, $group, $host, $port, $lag) = @_; 734 | my $failure = 0; 735 | if ($group && $host && $port) { 736 | eval{ 737 | $self->{dbh}->do(Proxysql_Insert_New_Server, undef, $group, $host, $port, 738 | 'ONLINE', 1000, 2000, $lag); 739 | }; 740 | if ($@) { 741 | $failure++; 742 | } 743 | } 744 | else { 745 | $failure++; 746 | } 747 | return $failure; 748 | } 749 | 750 | sub proxysql_load_server_to_runtime { 751 | my $self = shift; 752 | my $failure = 0; 753 | my $ret = $self->{dbh}->do(Proxysql_Load_Servers_To_Runtime); 754 | unless ($ret eq "0E0") { 755 | $failure++; 756 | } 757 | else { 758 | $self->{dbh}->do(Proxysql_Load_Variable_To_Runtime); 759 | } 760 | return $failure; 761 | } 762 | 763 | sub proxysql_save_server_to_disk { 764 | my $self = shift; 765 | my $dbh = shift; 766 | my $failure = 0; 767 | my $ret = $self->{dbh}->do(Proxysql_Save_Variable_To_Disk); 768 | unless ($ret eq "0E0") { 769 | $failure++; 770 | } 771 | else { 772 | $self->{dbh}->do(Proxysql_Load_Variable_To_Runtime); 773 | } 774 | return $failure; 775 | } 776 | 777 | 1; 778 | --------------------------------------------------------------------------------