├── CHANGELOG ├── COPYING ├── INSTALL ├── Makefile ├── README ├── UPGRADE ├── VERSION ├── bin ├── agent │ ├── check_ip │ ├── clear_ip │ ├── configure_ip │ ├── get_master_log_file │ ├── get_master_log_pos │ ├── kill_process │ ├── mysql_allow_write │ ├── mysql_deny_write │ ├── mysql_may_write │ ├── set_active_master │ ├── sync_with_master │ ├── turn_off_slave │ └── turn_on_slave ├── monitor │ └── checker └── tools │ ├── create_snapshot │ └── remove_snapshot ├── etc ├── init.d │ ├── mysql-mmm-agent │ └── mysql-mmm-monitor └── mysql-mmm │ ├── mmm_agent.conf │ ├── mmm_common.conf │ ├── mmm_mon.conf │ └── mmm_tools.conf ├── lib ├── Agent │ ├── Agent.pm │ ├── Agent.pm.orig │ ├── Helpers.pm │ ├── Helpers │ │ ├── Actions.pm │ │ └── Network.pm │ └── Role.pm ├── Common │ ├── Angel.pm │ ├── Config.pm │ ├── Log.pm │ ├── PidFile.pm │ ├── Role.pm │ ├── Socket.pm │ └── Uptime.pm ├── Monitor │ ├── Agent.pm │ ├── Agents.pm │ ├── CheckResult.pm │ ├── Checker.pm │ ├── Checker │ │ └── Checks.pm │ ├── ChecksStatus.pm │ ├── Commands.pm │ ├── Monitor.pm │ ├── NetworkChecker.pm │ ├── Role.pm │ ├── Roles.pm │ └── StartupStatus.pm └── Tools │ ├── MySQL.pm │ ├── Snapshot │ ├── LVM.pm │ └── MySQL.pm │ └── Tools.pm └── sbin ├── mmm_agentd ├── mmm_backup ├── mmm_clone ├── mmm_control ├── mmm_mond └── mmm_restore /CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 2.2.2 2 | fixed: Issue #1 Slow slave cause the whole swith process hang 3 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | 2 | 1. Install dependencies 3 | See .pdf-Documentation. 4 | 5 | 2. Install mysql-mmm 6 | make install 7 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | ifndef INSTALLDIR 3 | INSTALLDIR = installvendorlib 4 | endif 5 | 6 | MODULEDIR = $(DESTDIR)$(shell eval "`perl -V:${INSTALLDIR}`"; echo "$$${INSTALLDIR}")/MMM 7 | BINDIR = $(DESTDIR)/usr/lib/mysql-mmm 8 | SBINDIR = $(DESTDIR)/usr/sbin 9 | LOGDIR = $(DESTDIR)/var/log/mysql-mmm 10 | ETCDIR = $(DESTDIR)/etc 11 | CONFDIR = $(ETCDIR)/mysql-mmm 12 | 13 | install_common: 14 | mkdir -p $(DESTDIR) $(MODULEDIR) $(BINDIR) $(SBINDIR) $(LOGDIR) $(ETCDIR) $(CONFDIR) $(ETCDIR)/init.d/ 15 | cp -r lib/Common/ $(MODULEDIR) 16 | [ -f $(CONFDIR)/mmm_common.conf ] || cp etc/mysql-mmm/mmm_common.conf $(ETCDIR)/mysql-mmm/ 17 | 18 | install_agent: install_common 19 | mkdir -p $(BINDIR)/agent/ 20 | cp -r lib/Agent/ $(MODULEDIR) 21 | cp -r bin/agent/* $(BINDIR)/agent/ 22 | cp -r etc/init.d/mysql-mmm-agent $(ETCDIR)/init.d/ 23 | cp sbin/mmm_agentd $(SBINDIR) 24 | [ -f $(CONFDIR)/mmm_agent.conf ] || cp etc/mysql-mmm/mmm_agent.conf $(ETCDIR)/mysql-mmm/ 25 | 26 | install_monitor: install_common 27 | mkdir -p $(BINDIR)/monitor/ 28 | cp -r lib/Monitor/ $(MODULEDIR) 29 | cp -r bin/monitor/* $(BINDIR)/monitor/ 30 | cp -r etc/init.d/mysql-mmm-monitor $(ETCDIR)/init.d/ 31 | cp sbin/mmm_control sbin/mmm_mond $(SBINDIR) 32 | [ -f $(CONFDIR)/mmm_mon.conf ] || cp etc/mysql-mmm/mmm_mon.conf $(ETCDIR)/mysql-mmm/ 33 | 34 | install_tools: install_common 35 | mkdir -p $(BINDIR)/tools/ 36 | cp -r lib/Tools/ $(MODULEDIR) 37 | cp -r bin/tools/* $(BINDIR)/tools/ 38 | cp sbin/mmm_backup sbin/mmm_clone sbin/mmm_restore $(SBINDIR) 39 | [ -f $(CONFDIR)/mmm_tools.conf ] || cp etc/mysql-mmm/mmm_tools.conf $(ETCDIR)/mysql-mmm/ 40 | 41 | install: install_agent install_monitor install_tools 42 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | This is mysql-mmm-2.3.1 2 | 3 | Change Log For 2.3.1 4 | #change: merge get_master_log_fil & get_master_log_pos into one command call 5 | #change: remove vip before deny_write, when doing set_offline writer 6 | -------------------------------------------------------------------------------- /UPGRADE: -------------------------------------------------------------------------------- 1 | ===== Upgrading from 2.0.x to 2.1.0 ===== 2 | 3 | MMM 2.1.0 is backwards compatible with MMM 2.0.x, so a monitoring host running 4 | version 2.1.0 can talk to agents running 2.0.x and the other way around 5 | 6 | 7 | ==== On the monitoring host ==== 8 | 1. Stop mmmd_mon (if you have multiple clusters running, be sure to stop all 9 | instances of mmmd_mon): 10 | /etc/init.d/mysql-mmm-monitor stop 11 | 2. When using the .debs edit /etc/default/mysql-mmm-monitor and set ENABLED 12 | to 0. 13 | 3. Install the new version of MMM. 14 | 4. mmmd_mon was renamed to mmm_mond. So you can delete the old binary: 15 | rm -f /usr/sbin/mmmd_mon 16 | 5. If you have multiple clusters running, recreate your init-scripts for these 17 | by cloning /etc/init.d/mysql-mmm-monitor and adjusting the variable CLUSTER 18 | 6. (Optional) Edit mmm_mon.conf and change status_path from 19 | /var/lib/misc/mmmd_mon.status to /var/lib/misc/mmm_mond.status - afterwards 20 | copy the existing status file to its new location: 21 | cp /var/lib/misc/mmmd_mon.status /var/lib/misc/mmm_mond.status 22 | 7. Edit mmm_mon.conf and change pid_path from /var/run/mmmd_mon.pid to 23 | /var/run/mmm_mond.pid 24 | 8. The helper binaries have been moved so edit mmm_*.conf and change bin_path 25 | from /usr/bin/mysql-mmm/ to /usr/lib/mysql-mmm/ - then delete the old 26 | binaries: 27 | rm -rf /usr/bin/mysql-mmm 28 | 9. When using the .debs edit /etc/default/mysql-mmm-monitor and set ENABLED 29 | to 1. 30 | 10. Start mmm_mond (if you use more than one cluster, don't forget to start them 31 | too): 32 | /etc/init.d/mysql-mmm-monitor start 33 | 34 | ==== On agent hosts ==== 35 | 1. Stop mmmd_agent: 36 | /etc/init.d/mysql-mmm-agent stop 37 | 2. When using the .debs edit /etc/default/mysql-mmm-agent and set ENABLED to 0 38 | 3. Install the new version of MMM. 39 | 4. mmmd_agent was renamed to mmm_agentd. So you can delete the old binary: 40 | rm -f /usr/sbin/mmmd_agent 41 | 5. The helper binaries have been moved so edit mmm_*.conf and change bin_path 42 | from /usr/bin/mysql-mmm/ to /usr/lib/mysql-mmm/ - then delete the old 43 | binaries: 44 | rm -rf /usr/bin/mysql-mmm 45 | 6. Edit mmm_*.conf and change pid_path in the sections from 46 | /var/run/mmmd_agent.pid to /var/run/mmm_agentd.pid 47 | 7. When using the .debs edit /etc/default/mysql-mmm-agent and set ENABLED to 1 48 | 8. Start mmm_agentd: 49 | /etc/init.d/mysql-mmm-agent start 50 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 2.2.1 2 | -------------------------------------------------------------------------------- /bin/agent/check_ip: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Agent::Helpers::Actions; 7 | 8 | # Check arguments 9 | if (scalar(@ARGV) != 3) { 10 | print "Usage: $0 \n\n"; 11 | exit(1); 12 | } 13 | 14 | # Fetch arguments 15 | my $config_file = shift; 16 | my $if = shift; 17 | my $ip = shift; 18 | 19 | # Finally do the work 20 | MMM::Agent::Helpers::Actions::check_ip($if, $ip); 21 | 22 | __END__ 23 | 24 | =head1 NAME 25 | 26 | check_ip 27 | 28 | =head1 DESCRIPTION 29 | 30 | check_ip is a helper binary for B. It checks if the given ip is configured. 31 | 32 | =head1 USAGE 33 | 34 | check_ip 35 | 36 | =head1 EXAMPLE 37 | 38 | check_ip eth0 192.168.0.200 39 | -------------------------------------------------------------------------------- /bin/agent/clear_ip: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Agent::Helpers::Actions; 7 | 8 | # Check arguments 9 | if (scalar(@ARGV) != 3) { 10 | print "Usage: $0 \n\n"; 11 | exit(1); 12 | } 13 | 14 | # Fetch arguments 15 | my $config_file = shift; 16 | my $if = shift; 17 | my $ip = shift; 18 | 19 | # Finally do the work 20 | MMM::Agent::Helpers::Actions::clear_ip($if, $ip); 21 | 22 | __END__ 23 | 24 | =head1 NAME 25 | 26 | clear_ip 27 | 28 | =head1 DESCRIPTION 29 | 30 | clear_ip is a helper binary for B. It removes the given ip from the given interface. 31 | 32 | =head1 USAGE 33 | 34 | clear_ip 35 | 36 | =head1 EXAMPLE 37 | 38 | clear_ip eth0 192.168.0.200 39 | -------------------------------------------------------------------------------- /bin/agent/configure_ip: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Agent::Helpers::Actions; 7 | 8 | # Check arguments 9 | if (scalar(@ARGV) != 3) { 10 | print "Usage: $0 \n\n"; 11 | exit(1); 12 | } 13 | 14 | # Fetch arguments 15 | my $config_file = shift; 16 | my $if = shift; 17 | my $ip = shift; 18 | 19 | # Finally do the work 20 | MMM::Agent::Helpers::Actions::configure_ip($if, $ip); 21 | 22 | __END__ 23 | 24 | =head1 NAME 25 | 26 | check_ip 27 | 28 | =head1 DESCRIPTION 29 | 30 | configure_ip is a helper binary for B. It checks if the given ip is configured. If not, it configures it and sends arp requests to notify other hosts. 31 | 32 | =head1 USAGE 33 | 34 | configure_ip 35 | 36 | =head1 EXAMPLE 37 | 38 | configure_ip eth0 192.168.0.200 39 | -------------------------------------------------------------------------------- /bin/agent/get_master_log_file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Common::Config; 7 | use MMM::Agent::Helpers::Actions; 8 | 9 | # Check arguments 10 | if (scalar(@ARGV) != 1) { 11 | print "Usage: $0 \n\n"; 12 | exit(1); 13 | } 14 | 15 | my $config_file = shift; 16 | 17 | # Read config file 18 | our $config = new MMM::Common::Config::; 19 | $config->read($config_file); 20 | $config->check('AGENT'); 21 | 22 | # Finally do the work 23 | my $output = MMM::Agent::Helpers::Actions::get_master_log_file(); 24 | 25 | print $output, "\n"; 26 | exit(0); 27 | 28 | __END__ 29 | 30 | =head1 NAME 31 | 32 | set_active_master 33 | 34 | =head1 DESCRIPTION 35 | 36 | set_active_master is a helper binary for B. It tries to catch up with the old master as far as possible and changes the master to the new host. 37 | (Syncs to the master log if the old master is reachable. Otherwise syncs to the relay log.) 38 | 39 | =head1 USAGE 40 | 41 | set_active_master 42 | 43 | =head1 EXAMPLE 44 | 45 | set_active_master db2 46 | -------------------------------------------------------------------------------- /bin/agent/get_master_log_pos: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Common::Config; 7 | use MMM::Agent::Helpers::Actions; 8 | 9 | # Check arguments 10 | if (scalar(@ARGV) != 1) { 11 | print "Usage: $0 \n\n"; 12 | exit(1); 13 | } 14 | 15 | my $config_file = shift; 16 | 17 | # Read config file 18 | our $config = new MMM::Common::Config::; 19 | $config->read($config_file); 20 | $config->check('AGENT'); 21 | 22 | # Finally do the work 23 | my $output = MMM::Agent::Helpers::Actions::get_master_log_pos(); 24 | 25 | print $output, "\n"; 26 | exit(0); 27 | 28 | __END__ 29 | 30 | =head1 NAME 31 | 32 | set_active_master 33 | 34 | =head1 DESCRIPTION 35 | 36 | set_active_master is a helper binary for B. It tries to catch up with the old master as far as possible and changes the master to the new host. 37 | (Syncs to the master log if the old master is reachable. Otherwise syncs to the relay log.) 38 | 39 | =head1 USAGE 40 | 41 | set_active_master 42 | 43 | =head1 EXAMPLE 44 | 45 | set_active_master db2 46 | -------------------------------------------------------------------------------- /bin/agent/kill_process: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Agent::Helpers::Actions; 7 | 8 | # Check arguments 9 | if (scalar(@ARGV) != 1) { 10 | print "Usage: $0 \n\n"; 11 | exit(1); 12 | } 13 | 14 | # Fetch arguments 15 | my $config_file = shift; 16 | 17 | # Finally do the work 18 | MMM::Agent::Helpers::Actions::kill_process(); 19 | 20 | __END__ 21 | 22 | =head1 NAME 23 | 24 | kill_process 25 | 26 | =head1 DESCRIPTION 27 | 28 | kill_process is a helper binary for B. It kills all processes in MySQL 29 | 30 | =head1 USAGE 31 | 32 | kill_process 33 | 34 | =head1 EXAMPLE 35 | 36 | kill_process 37 | -------------------------------------------------------------------------------- /bin/agent/mysql_allow_write: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Common::Config; 7 | use MMM::Agent::Helpers::Actions; 8 | 9 | # Check arguments 10 | if (scalar(@ARGV) != 1) { 11 | print "Usage: $0 \n\n"; 12 | exit(1); 13 | } 14 | 15 | # Read config file 16 | my $config_file = shift; 17 | our $config = new MMM::Common::Config::; 18 | $config->read($config_file); 19 | $config->check('AGENT'); 20 | 21 | # Finally do the work 22 | MMM::Agent::Helpers::Actions::mysql_allow_write(); 23 | 24 | __END__ 25 | 26 | =head1 NAME 27 | 28 | mysql_allow_write 29 | 30 | =head1 DESCRIPTION 31 | 32 | mysql_allow_write is a helper binary for B. It allowes writes on the local MySQL server by setting global read_only to 0. 33 | 34 | =head1 USAGE 35 | 36 | mysql_allow_write 37 | -------------------------------------------------------------------------------- /bin/agent/mysql_deny_write: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Common::Config; 7 | use MMM::Agent::Helpers::Actions; 8 | 9 | # Check arguments 10 | if (scalar(@ARGV) != 1) { 11 | print "Usage: $0 \n\n"; 12 | exit(1); 13 | } 14 | 15 | # Read config file 16 | my $config_file = shift; 17 | our $config = new MMM::Common::Config::; 18 | $config->read($config_file); 19 | $config->check('AGENT'); 20 | 21 | # Finally do the work 22 | MMM::Agent::Helpers::Actions::mysql_deny_write(); 23 | 24 | __END__ 25 | 26 | =head1 NAME 27 | 28 | mysql_deny_write 29 | 30 | =head1 DESCRIPTION 31 | 32 | mysql_deny_write is a helper binary for B. It denies writes on the local MySQL server by setting global read_only to 1. 33 | 34 | =head1 USAGE 35 | 36 | mysql_deny_write 37 | -------------------------------------------------------------------------------- /bin/agent/mysql_may_write: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Common::Config; 7 | use MMM::Agent::Helpers::Actions; 8 | 9 | # Check arguments 10 | if (scalar(@ARGV) != 1) { 11 | print "Usage: $0 \n\n"; 12 | exit(1); 13 | } 14 | 15 | # Read config file 16 | my $config_file = shift; 17 | our $config = new MMM::Common::Config::; 18 | $config->read($config_file); 19 | $config->check('AGENT'); 20 | 21 | # Finally do the work 22 | MMM::Agent::Helpers::Actions::mysql_may_write(); 23 | 24 | __END__ 25 | 26 | =head1 NAME 27 | 28 | mysql_may_write 29 | 30 | =head1 DESCRIPTION 31 | 32 | mysql_may_write is a helper binary for B. It checks if writes on the local MySQL server are allowed. 33 | 34 | =head1 USAGE 35 | 36 | mysql_may_write 37 | -------------------------------------------------------------------------------- /bin/agent/set_active_master: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Common::Config; 7 | use MMM::Agent::Helpers::Actions; 8 | 9 | # Check arguments 10 | if (scalar(@ARGV) != 2) { 11 | print "Usage: $0 \n\n"; 12 | exit(1); 13 | } 14 | 15 | my $config_file = shift; 16 | my $new_master_str = shift; 17 | my ($new_master, $new_master_log, $new_master_pos) = split(',', $new_master_str); 18 | 19 | 20 | # Read config file 21 | our $config = new MMM::Common::Config::; 22 | $config->read($config_file); 23 | $config->check('AGENT'); 24 | 25 | # Finally do the work 26 | my $output = MMM::Agent::Helpers::Actions::set_active_master($new_master_str); 27 | 28 | print $output, "\n"; 29 | exit(0); 30 | 31 | __END__ 32 | 33 | =head1 NAME 34 | 35 | set_active_master 36 | 37 | =head1 DESCRIPTION 38 | 39 | set_active_master is a helper binary for B. It tries to catch up with the old master as far as possible and changes the master to the new host. 40 | (Syncs to the master log if the old master is reachable. Otherwise syncs to the relay log.) 41 | 42 | =head1 USAGE 43 | 44 | set_active_master 45 | 46 | =head1 EXAMPLE 47 | 48 | set_active_master db2 49 | -------------------------------------------------------------------------------- /bin/agent/sync_with_master: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Common::Config; 7 | use MMM::Agent::Helpers::Actions; 8 | 9 | # Check arguments 10 | if (scalar(@ARGV) != 1) { 11 | print "Usage: $0 \n\n"; 12 | exit(1); 13 | } 14 | 15 | # Read config file 16 | my $config_file = shift; 17 | our $config = new MMM::Common::Config::; 18 | $config->read($config_file); 19 | $config->check('AGENT'); 20 | 21 | # Finally do the work 22 | my $output = MMM::Agent::Helpers::Actions::sync_with_master(); 23 | 24 | print $output, "\n"; 25 | exit(0); 26 | 27 | __END__ 28 | 29 | =head1 NAME 30 | 31 | sync_with_master 32 | 33 | =head1 DESCRIPTION 34 | 35 | sync_with_master is a helper binary for B. It tries to sync up a (soon active) master with his peer (old active master) when the I is moved. If the peer is reachable it syncs with the master log. If not reachable, syncs with the relay log. 36 | 37 | =head1 USAGE 38 | 39 | sync_with_master 40 | -------------------------------------------------------------------------------- /bin/agent/turn_off_slave: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Common::Config; 7 | use MMM::Agent::Helpers::Actions; 8 | 9 | # Check arguments 10 | if (scalar(@ARGV) != 1) { 11 | print "Usage: $0 \n\n"; 12 | exit(1); 13 | } 14 | 15 | # Read config file 16 | my $config_file = shift; 17 | our $config = new MMM::Common::Config::; 18 | $config->read($config_file); 19 | $config->check('AGENT'); 20 | 21 | # Finally do the work 22 | my $output = MMM::Agent::Helpers::Actions::toggle_slave(0); 23 | 24 | print $output, "\n"; 25 | exit(0); 26 | 27 | __END__ 28 | 29 | =head1 NAME 30 | 31 | turn_off_slave 32 | 33 | =head1 DESCRIPTION 34 | 35 | turn_off_slave is a helper binary for B. It stops the slave threads on the local MySQL server. 36 | 37 | =head1 USAGE 38 | 39 | turn_off_slave 40 | -------------------------------------------------------------------------------- /bin/agent/turn_on_slave: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use MMM::Common::Config; 7 | use MMM::Agent::Helpers::Actions; 8 | 9 | # Check arguments 10 | if (scalar(@ARGV) != 1) { 11 | print "Usage: $0 \n\n"; 12 | exit(1); 13 | } 14 | 15 | # Read config file 16 | my $config_file = shift; 17 | our $config = new MMM::Common::Config::; 18 | $config->read($config_file); 19 | $config->check('AGENT'); 20 | 21 | # Finally do the work 22 | my $output = MMM::Agent::Helpers::Actions::toggle_slave(1); 23 | 24 | print $output, "\n"; 25 | exit(0); 26 | 27 | __END__ 28 | 29 | =head1 NAME 30 | 31 | turn_on_slave 32 | 33 | =head1 DESCRIPTION 34 | 35 | turn_on_slave is a helper binary for B. It starts the slave threads on the local MySQL server. 36 | 37 | =head1 USAGE 38 | 39 | turn_on_slave 40 | -------------------------------------------------------------------------------- /bin/monitor/checker: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Use mandatory external modules 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use English qw( OUTPUT_AUTOFLUSH ); 7 | use Log::Log4perl qw(:easy); 8 | 9 | # TODO configurable logging via MMM::Common::Log 10 | 11 | # Include parts of the system 12 | use MMM::Common::Config; 13 | use MMM::Monitor::Checker::Checks; 14 | 15 | # Disable output buffering 16 | $OUTPUT_AUTOFLUSH = 1; 17 | 18 | # Check if cluster was passed 19 | my $postfix = ""; 20 | if (scalar(@ARGV) && $ARGV[0] =~ /^@(.*)/) { 21 | shift(@ARGV); 22 | $postfix = "_$1"; 23 | } 24 | 25 | # Check arguments 26 | if (scalar(@ARGV) != 1) { 27 | print "Usage: $0 \n\n"; 28 | exit(1); 29 | } 30 | 31 | # Fetch arguments 32 | my $check_name = shift; 33 | 34 | 35 | # Read config file 36 | our $config = new MMM::Common::Config::; 37 | $config->read("mmm_mon$postfix"); 38 | $config->check('MONITOR'); 39 | 40 | our $check; 41 | 42 | # NOTE: the check "ping_ip" is not a host check. Its a checker for pinging single IPs to test if the monitor network connection is still working. 43 | if ($check_name eq 'ping_ip') { 44 | $check = { 45 | restart_after => 0, 46 | timeout => 1 47 | } 48 | } 49 | else { 50 | LOGDIE "checker: Unknown check $check_name" unless (defined($config->{check}->{$check_name})); 51 | $check = \%{$config->{check}->{$check_name}}; 52 | } 53 | 54 | my $check_function; 55 | if ($check_name eq 'ping_ip' ) { $check_function = \&MMM::Monitor::Checker::Checks::ping_ip; } 56 | elsif ($check_name eq 'ping' ) { $check_function = \&MMM::Monitor::Checker::Checks::ping; } 57 | elsif ($check_name eq 'mysql' ) { $check_function = \&MMM::Monitor::Checker::Checks::mysql; } 58 | elsif ($check_name eq 'rep_backlog') { $check_function = \&MMM::Monitor::Checker::Checks::rep_backlog; } 59 | elsif ($check_name eq 'rep_threads') { $check_function = \&MMM::Monitor::Checker::Checks::rep_threads; } 60 | else { LOGDIE "checker: Unknown check $check_name"; } 61 | 62 | my $max_checks = $check->{restart_after}; 63 | my $timeout = $check->{timeout}; 64 | 65 | # Workaround to prevent checker from hanging in case of unnoticed broken pipe errors 66 | my $max_empty_commands = 100; 67 | my $empty_commands = 0; 68 | 69 | INFO "$check_name: Start"; 70 | 71 | # Process loop 72 | while (!eof(STDIN)) { 73 | 74 | # Check if it is time to die 75 | if ($max_checks && $max_checks < 1) { 76 | INFO "$check_name: Max checks performed, restarting..."; 77 | last; 78 | } 79 | 80 | # Read command 81 | chomp(my $cmd = ); 82 | my @command = split(/\s+/, $cmd); 83 | my $params = scalar(@command) - 1; 84 | 85 | # Workaround to prevent checker from hanging in case of unnoticed broken pipe errors 86 | if ($params < 0) { 87 | if (++$empty_commands > $max_empty_commands) { 88 | WARN "$check_name: Too many empty commands ($empty_commands) in a row - looks like pipe is broken! Exiting!"; 89 | last; 90 | } 91 | next; 92 | } 93 | $empty_commands = 0; 94 | 95 | 96 | last if ($command[0] eq 'quit' && $params == 0); 97 | 98 | if ($command[0] eq 'ping' && $params == 0) { 99 | print "OK: Pong!\n"; 100 | next; 101 | } 102 | if ($command[0] eq 'check' && $params == 1) { 103 | print $check_function->($timeout, $command[1]), "\n"; 104 | next; 105 | } 106 | 107 | print "ERROR: Invalid command '$cmd'\n"; 108 | } 109 | INFO "$check_name: Exit"; 110 | 111 | print "OK: Finished\n"; 112 | exit(0); 113 | 114 | __END__ 115 | 116 | =head1 NAME 117 | 118 | checker 119 | 120 | =head1 DESCRIPTION 121 | 122 | B is a helper binary for B. It is called from, and communicates with B. B processes commands from STDIN and writes the results of this commands to STDOUT: 123 | 124 | =open 4 125 | 126 | =item ping 127 | 128 | Check if B is still alive. 129 | 130 | =back 131 | 132 | =head1 USAGE 133 | 134 | checker 135 | 136 | 137 | =head1 EXAMPLES 138 | 139 | checker ping 140 | checker mysql 141 | checker rep_backlog 142 | checker rep_threads 143 | -------------------------------------------------------------------------------- /bin/tools/create_snapshot: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use DBI; 6 | use MMM::Common::Config; 7 | use MMM::Tools::Snapshot::LVM; 8 | use MMM::Tools::Snapshot::MySQL; 9 | 10 | our $config = new MMM::Common::Config::; 11 | $config->read("mmm_tools"); 12 | $config->check('TOOLS'); 13 | 14 | 15 | print create_snapshot(), "\n"; 16 | exit(0); 17 | 18 | 19 | sub create_snapshot { 20 | my $this = $config->{this}; 21 | 22 | unless (defined($config->{host}->{$this})) { 23 | return "ERROR: Invalid 'this' value: '$this'!"; 24 | } 25 | 26 | my $host = $config->{host}->{$this}; 27 | my $dump_dir = $host->{lvm_mount_dir}; 28 | 29 | system ('mkdir', '-p', $dump_dir); 30 | unless (-d $dump_dir && -w _ && -r _ && -x _) { 31 | return "ERROR: Directory '$dump_dir' has invalid permissions (it must be readable/writable/executable)"; 32 | } 33 | # Check mount dir 34 | if (scalar(glob("$dump_dir/*"))) { 35 | return "ERROR: LVM mount dir is not empty!"; 36 | } 37 | 38 | my $dbh = MMM::Tools::Snapshot::MySQL::connect($this); 39 | 40 | return "ERROR: Can't connect to database! Error = " . DBI::errstr unless ($dbh); 41 | 42 | my $res; 43 | 44 | # Lock tables 45 | $res = MMM::Tools::Snapshot::MySQL::lock_tables($dbh); 46 | return "ERROR: Can't lock tables! Error = " . $dbh->errstr unless ($res); 47 | 48 | # Get position info 49 | my $pos_info = {}; 50 | $pos_info->{host} = $config->{this}; 51 | $res = MMM::Tools::Snapshot::MySQL::get_pos_info($dbh, $pos_info); 52 | return "ERROR: Can't get position info: $res" unless ($res =~ /^OK/); 53 | 54 | # Create and mount snapshot 55 | $res = MMM::Tools::Snapshot::LVM::create(); 56 | return "ERROR: Can't create or mount snapshot: $res" unless ($res =~ /^OK/); 57 | 58 | # Unlock tables 59 | MMM::Tools::Snapshot::MySQL::unlock_tables($dbh); 60 | 61 | # Change dir to snapshot and create _mmm directory 62 | chdir($dump_dir); 63 | system('mkdir -p _mmm'); 64 | 65 | MMM::Tools::Snapshot::MySQL::save_pos_info($pos_info, '_mmm/status.txt'); 66 | $res = system('cp', $host->{mysql_cnf}, '_mmm/'); 67 | return "ERROR: Can't copy mysql config file to backup!" if ($res); 68 | 69 | return 'OK: Snapshot created!'; 70 | } 71 | 72 | __END__ 73 | 74 | =head1 NAME 75 | 76 | remove_snapshot 77 | 78 | =head1 DESCRIPTION 79 | 80 | remove_snapshot is a helper binary for the mmm tools. It removes a snapshot created by B. 81 | 82 | =head1 USAGE 83 | 84 | remove_snapshot 85 | 86 | -------------------------------------------------------------------------------- /bin/tools/remove_snapshot: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use MMM::Common::Config; 6 | use MMM::Tools::Snapshot::LVM; 7 | 8 | our $config = new MMM::Common::Config::; 9 | $config->read("mmm_tools"); 10 | $config->check('TOOLS'); 11 | 12 | 13 | print MMM::Tools::Snapshot::LVM::remove(), "\n"; 14 | exit(0); 15 | 16 | __END__ 17 | 18 | =head1 NAME 19 | 20 | remove_snapshot 21 | 22 | =head1 DESCRIPTION 23 | 24 | remove_snapshot is a helper binary for the mmm tools. It removes a snapshot created by B. 25 | 26 | =head1 USAGE 27 | 28 | remove_snapshot 29 | 30 | -------------------------------------------------------------------------------- /etc/init.d/mysql-mmm-agent: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # mysql-mmm-agent This shell script takes care of starting and stopping 4 | # the mmm agent daemon. 5 | # 6 | # chkconfig: - 64 36 7 | # description: MMM Agent. 8 | # processname: mmm_agentd 9 | # config: /etc/mmm_agent.conf 10 | # pidfile: /var/run/mmm_agentd.pid 11 | 12 | # Cluster name (it can be empty for default cases) 13 | CLUSTER='' 14 | 15 | 16 | #----------------------------------------------------------------------- 17 | # Paths 18 | if [ "$CLUSTER" != "" ]; then 19 | MMM_AGENTD_BIN="/usr/sbin/mmm_agentd @$CLUSTER" 20 | MMM_AGENTD_PIDFILE="/var/run/mmm_agentd-$CLUSTER.pid" 21 | else 22 | MMM_AGENTD_BIN="/usr/sbin/mmm_agentd" 23 | MMM_AGENTD_PIDFILE="/var/run/mmm_agentd.pid" 24 | fi 25 | 26 | echo "Daemon bin: '$MMM_AGENTD_BIN'" 27 | echo "Daemon pid: '$MMM_AGENTD_PIDFILE'" 28 | 29 | #----------------------------------------------------------------------- 30 | # See how we were called. 31 | case "$1" in 32 | start) 33 | # Start daemon. 34 | echo -n "Starting MMM Agent daemon... " 35 | if [ -s $MMM_AGENTD_PIDFILE ] && kill -0 `cat $MMM_AGENTD_PIDFILE` 2> /dev/null; then 36 | echo " already running." 37 | exit 0 38 | fi 39 | $MMM_AGENTD_BIN 40 | if [ "$?" -ne 0 ]; then 41 | echo "failed" 42 | exit 1 43 | fi 44 | echo "Ok" 45 | exit 0 46 | ;; 47 | 48 | stop) 49 | # Stop daemon. 50 | echo -n "Shutting down MMM Agent daemon" 51 | if [ -s $MMM_AGENTD_PIDFILE ]; then 52 | pid="$(cat $MMM_AGENTD_PIDFILE)" 53 | cnt=0 54 | kill "$pid" 55 | while kill -0 "$pid" 2>/dev/null; do 56 | cnt=`expr "$cnt" + 1` 57 | if [ "$cnt" -gt 15 ]; then 58 | kill -9 "$pid" 59 | break 60 | fi 61 | sleep 2 62 | echo -n "." 63 | done 64 | echo " Ok" 65 | exit 0 66 | fi 67 | echo " not running." 68 | exit 0 69 | ;; 70 | 71 | status) 72 | echo -n "Checking MMM Agent process:" 73 | if [ ! -s $MMM_AGENTD_PIDFILE ]; then 74 | echo " not running." 75 | exit 3 76 | fi 77 | pid="$(cat $MMM_AGENTD_PIDFILE)" 78 | if ! kill -0 "$pid" 2> /dev/null; then 79 | echo " not running." 80 | exit 1 81 | fi 82 | echo " running." 83 | exit 0 84 | ;; 85 | 86 | restart|reload) 87 | $0 stop 88 | $0 start 89 | exit $? 90 | ;; 91 | 92 | *) 93 | echo "Usage: $0 {start|stop|restart|status}" 94 | ;; 95 | esac 96 | 97 | exit 1 98 | -------------------------------------------------------------------------------- /etc/init.d/mysql-mmm-monitor: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # mysql-mmm-monitor This shell script takes care of starting and stopping 4 | # the mmm monitoring daemon. 5 | # 6 | # chkconfig: - 64 36 7 | # description: MMM Monitor. 8 | # processname: mmm_mond 9 | # config: /etc/mmm_mon.conf 10 | # pidfile: /var/run/mmm_mond.pid 11 | 12 | # Cluster name (it can be empty for default cases) 13 | CLUSTER='' 14 | 15 | 16 | #----------------------------------------------------------------------- 17 | # Paths 18 | if [ "$CLUSTER" != "" ]; then 19 | MMM_MOND_BIN="/usr/sbin/mmm_mond @$CLUSTER" 20 | MMM_MOND_PIDFILE="/var/run/mmm_mond-$CLUSTER.pid" 21 | else 22 | MMM_MOND_BIN="/usr/sbin/mmm_mond" 23 | MMM_MOND_PIDFILE="/var/run/mmm_mond.pid" 24 | fi 25 | 26 | echo "Daemon bin: '$MMM_MOND_BIN'" 27 | echo "Daemon pid: '$MMM_MOND_PIDFILE'" 28 | 29 | #----------------------------------------------------------------------- 30 | # See how we were called. 31 | case "$1" in 32 | start) 33 | # Start daemon. 34 | echo -n "Starting MMM Monitor daemon: " 35 | if [ -s $MMM_MOND_PIDFILE ] && kill -0 `cat $MMM_MOND_PIDFILE` 2> /dev/null; then 36 | echo " already running." 37 | exit 0 38 | fi 39 | $MMM_MOND_BIN 40 | if [ "$?" -ne 0 ]; then 41 | echo "failed" 42 | exit 1 43 | fi 44 | echo "Ok" 45 | exit 0 46 | ;; 47 | 48 | stop) 49 | # Stop daemon. 50 | echo -n "Shutting down MMM Monitor daemon: " 51 | if [ -s $MMM_MOND_PIDFILE ]; then 52 | pid="$(cat $MMM_MOND_PIDFILE)" 53 | cnt=0 54 | kill "$pid" 55 | while kill -0 "$pid" 2>/dev/null; do 56 | cnt=`expr "$cnt" + 1` 57 | if [ "$cnt" -gt 15 ]; then 58 | kill -9 "$pid" 59 | break 60 | fi 61 | sleep 2 62 | echo -n "." 63 | done 64 | echo " Ok" 65 | exit 0 66 | fi 67 | echo " not running." 68 | exit 0 69 | ;; 70 | 71 | status) 72 | echo -n "Checking MMM Monitor process:" 73 | if [ ! -s $MMM_MOND_PIDFILE ]; then 74 | echo " not running." 75 | exit 3 76 | fi 77 | pid="$(cat $MMM_MOND_PIDFILE)" 78 | if ! kill -0 "$pid" 2> /dev/null; then 79 | echo " not running." 80 | exit 1 81 | fi 82 | echo " running." 83 | exit 0 84 | ;; 85 | 86 | restart|reload) 87 | $0 stop 88 | $0 start 89 | exit $? 90 | ;; 91 | 92 | *) 93 | echo "Usage: $0 {start|stop|restart|status}" 94 | ;; 95 | esac 96 | 97 | exit 1 98 | -------------------------------------------------------------------------------- /etc/mysql-mmm/mmm_agent.conf: -------------------------------------------------------------------------------- 1 | include mmm_common.conf 2 | this db1 3 | -------------------------------------------------------------------------------- /etc/mysql-mmm/mmm_common.conf: -------------------------------------------------------------------------------- 1 | active_master_role writer 2 | 3 | 4 | 5 | cluster_interface eth0 6 | 7 | pid_path /var/run/mmm_agentd.pid 8 | bin_path /usr/lib/mysql-mmm/ 9 | 10 | replication_user replication 11 | replication_password slave 12 | 13 | agent_user mmm_agent 14 | agent_password RepAgent 15 | 16 | 17 | 18 | ip 192.168.0.31 19 | mode master 20 | peer db2 21 | 22 | 23 | 24 | ip 192.168.0.32 25 | mode master 26 | peer db1 27 | 28 | 29 | 30 | ip 192.168.0.33 31 | mode slave 32 | 33 | 34 | 35 | 36 | hosts db1, db2 37 | ips 192.168.0.50 38 | mode exclusive 39 | 40 | 41 | 42 | hosts db1, db2, db3 43 | ips 192.168.0.51, 192.168.0.52, 192.168.0.53 44 | mode balanced 45 | 46 | -------------------------------------------------------------------------------- /etc/mysql-mmm/mmm_mon.conf: -------------------------------------------------------------------------------- 1 | include mmm_common.conf 2 | 3 | 4 | ip 127.0.0.1 5 | pid_path /var/run/mmm_mond.pid 6 | bin_path /usr/lib/mysql-mmm/ 7 | status_path /var/lib/misc/mmm_mond.status 8 | ping_ips 192.168.0.1, 192.168.0.2, 192.168.0.3 9 | 10 | 11 | 12 | monitor_user mmm_monitor 13 | monitor_password RepMonitor 14 | 15 | 16 | debug 0 17 | -------------------------------------------------------------------------------- /etc/mysql-mmm/mmm_tools.conf: -------------------------------------------------------------------------------- 1 | include mmm_agent.conf 2 | 3 | default_copy_method scp 4 | clone_dirs data, logs 5 | 6 | 7 | ssh_user root 8 | 9 | lvm_snapshot_size 1G 10 | lvm_logical_volume mysql 11 | lvm_volume_group storage 12 | lvm_mount_dir /mmm_snapshot 13 | lvm_mount_opts -orw,nouuid 14 | 15 | tools_user mmm_tools 16 | tools_password RepTools 17 | 18 | backup_dir /mmm_backup 19 | restore_dir /mysql 20 | 21 | 22 | 23 | 24 | backup_command scp -c blowfish -r %SSH_USER%@%IP%:%SNAPSHOT%/%CLONE_DIR% %DEST_DIR%/ 25 | restore_command cp -axv %BACKUP_DIR%/* %DEST_DIR%/ 26 | true_copy 1 27 | 28 | 29 | 30 | backup_command rdiff-backup --ssh-no-compression -v 5 !--include %SNAPSHOT%/%CLONE_DIR%! --exclude %SNAPSHOT% %SSH_USER%@%IP%::%SNAPSHOT%/ %DEST_DIR%/ 31 | restore_command rdiff-backup --force -v 5 -r %VERSION% %BACKUP_DIR% %DEST_DIR%/.mmm_restore; cp -axvl --remove-destination %DEST_DIR%/.mmm_restore/* %DEST_DIR%/; rm -r %DEST_DIR%/.mmm_restore/ 32 | incremental_command rdiff-backup --parsable-output -l %BACKUP_DIR% 33 | single_run 1 34 | incremental 1 35 | 36 | 37 | 38 | backup_command ssh -c blowfish %SSH_USER%@%IP% "cd '%SNAPSHOT%'; tar cv !'%CLONE_DIR%'!" | gzip > %DEST_DIR%/backup.tar.gz 39 | restore_command cd %DEST_DIR%; tar xzfv %BACKUP_DIR%/backup.tar.gz 40 | single_run 1 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /lib/Agent/Agent.pm: -------------------------------------------------------------------------------- 1 | package MMM::Agent::Agent; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw(EVAL_ERROR); 6 | use Algorithm::Diff; 7 | use DBI; 8 | use Class::Struct; 9 | use Errno qw(EINTR); 10 | use Log::Log4perl qw(:easy); 11 | use MMM::Common::Role; 12 | use MMM::Common::Socket; 13 | use MMM::Agent::Helpers; 14 | use MMM::Agent::Role; 15 | 16 | eval { 17 | no warnings 'once'; 18 | require Unix::Uptime; 19 | *uptime = *Unix::Uptime->uptime; 20 | }; 21 | if ($EVAL_ERROR) { 22 | require MMM::Common::Uptime; 23 | MMM::Common::Uptime->import(qw(uptime)); 24 | } 25 | 26 | our $VERSION = '0.01'; 27 | 28 | 29 | struct 'MMM::Agent::Agent' => { 30 | name => '$', 31 | ip => '$', 32 | port => '$', 33 | interface => '$', 34 | mode => '$', 35 | mysql_port => '$', 36 | mysql_user => '$', 37 | mysql_password => '$', 38 | writer_role => '$', 39 | bin_path => '$', 40 | 41 | active_master => '$', 42 | state => '$', 43 | roles => '@', 44 | config_file => '$' 45 | }; 46 | 47 | sub main($) { 48 | my $self = shift; 49 | my $socket = MMM::Common::Socket::create_listener($self->ip, $self->port); 50 | $self->roles([]); 51 | $self->active_master(''); 52 | 53 | while (!$main::shutdown) { 54 | 55 | DEBUG 'Listener: Waiting for connection...'; 56 | my $client = $socket->accept(); 57 | next unless ($client); 58 | 59 | DEBUG 'Listener: Connect!'; 60 | while (my $cmd = <$client>) { 61 | chomp($cmd); 62 | DEBUG "Daemon: Command = '$cmd'"; 63 | 64 | my $res = $self->handle_command($cmd); 65 | my $uptime = uptime(); 66 | 67 | print $client "$res|UP:$uptime\n"; 68 | DEBUG "Daemon: Answer = '$res'"; 69 | 70 | return 0 if ($main::shutdown); 71 | } 72 | 73 | close($client); 74 | DEBUG 'Listener: Disconnect!'; 75 | 76 | $self->check_roles(); 77 | } 78 | } 79 | 80 | sub handle_command($$) { 81 | my $self = shift; 82 | my $cmd = shift; 83 | 84 | DEBUG "Received Command $cmd"; 85 | my ($cmd_name, $version, $host, @params) = split('\|', $cmd, -1); 86 | 87 | return "ERROR: Invalid command '$cmd'!" unless (defined($host)); 88 | return "ERROR: Invalid hostname in command ($host)! My name is '" . $self->name . "'" if ($host ne $self->name); 89 | 90 | if ($version > main::MMM_PROTOCOL_VERSION) { 91 | WARN "Version in command '$cmd_name' ($version) is greater than mine (", main::MMM_PROTOCOL_VERSION, ")" 92 | } 93 | 94 | if ($cmd_name eq 'PING') { return cmd_ping (); } 95 | elsif ($cmd_name eq 'SET_STATUS') { return $self->cmd_set_status (@params); } 96 | elsif ($cmd_name eq 'GET_AGENT_STATUS') { return $self->cmd_get_agent_status (); } 97 | elsif ($cmd_name eq 'GET_SYSTEM_STATUS') { return $self->cmd_get_system_status (); } 98 | elsif ($cmd_name eq 'CLEAR_BAD_ROLES') { return $self->cmd_clear_bad_roles (); } 99 | elsif ($cmd_name eq 'GET_MASTER_LOG') { return $self->cmd_get_master_log_file(); } 100 | elsif ($cmd_name eq 'GET_MASTER_POS') { return $self->cmd_get_master_log_pos(); } 101 | 102 | return "ERROR: Invalid command '$cmd_name'!"; 103 | } 104 | 105 | sub cmd_ping() { 106 | return 'OK: Pinged!'; 107 | } 108 | 109 | sub cmd_get_agent_status($) { 110 | my $self = shift; 111 | 112 | my $answer = join ('|', ( 113 | $self->state, 114 | join(',', @{$self->roles}), 115 | $self->active_master 116 | )); 117 | return "OK: Returning status!|$answer"; 118 | } 119 | 120 | sub cmd_get_system_status($) { 121 | my $self = shift; 122 | 123 | # determine master info 124 | my $dsn = sprintf("DBI:mysql:host=%s;port=%s;mysql_connect_timeout=3", $self->ip, $self->mysql_port); 125 | my $eintr = EINTR; 126 | my $master_ip = ''; 127 | 128 | my $dbh; 129 | CONNECT: { 130 | DEBUG "Connecting to mysql"; 131 | $dbh = DBI->connect($dsn, $self->mysql_user, $self->mysql_password, { PrintError => 0 }); 132 | unless ($dbh) { 133 | redo CONNECT if ($DBI::err == 2003 && $DBI::errstr =~ /\($eintr\)/); 134 | WARN "Couldn't connect to mysql. Can't determine current master host." . $DBI::err . " " . $DBI::errstr; 135 | } 136 | } 137 | 138 | my $slave_status = $dbh->selectrow_hashref('SHOW SLAVE STATUS'); 139 | $master_ip = $slave_status->{Master_Host} if (defined($slave_status)); 140 | 141 | my @roles; 142 | foreach my $role (keys(%{$main::config->{role}})) { 143 | my $role_info = $main::config->{role}->{$role}; 144 | foreach my $ip (@{$role_info->{ips}}) { 145 | my $res = MMM::Agent::Helpers::check_ip($self->interface, $ip); 146 | my $ret = $? >> 8; 147 | return "ERROR: Could not check if IP is configured: $res" if ($ret == 255); 148 | next unless ($ret == 0); 149 | # IP is configured... 150 | push @roles, new MMM::Common::Role::(name => $role, ip => $ip); 151 | } 152 | } 153 | my $res = MMM::Agent::Helpers::may_write(); 154 | my $ret = $? >> 8; 155 | return "ERROR: Could not check if MySQL is writable: $res" if ($ret == 255); 156 | my $writable = ($ret == 1); 157 | 158 | my $answer = join('|', ($writable, join(',', @roles), $master_ip)); 159 | return "OK: Returning status!|$answer"; 160 | } 161 | 162 | sub cmd_clear_bad_roles($) { 163 | my $self = shift; 164 | my $count = 0; 165 | foreach my $role (keys(%{$main::config->{role}})) { 166 | my $role_info = $main::config->{role}->{$role}; 167 | foreach my $ip (@{$role_info->{ips}}) { 168 | my $role_valid = 0; 169 | foreach my $agentrole (@{$self->roles}) { 170 | next unless ($agentrole->name eq $role); 171 | next unless ($agentrole->ip eq $ip); 172 | $role_valid = 1; 173 | last; 174 | } 175 | next if ($role_valid); 176 | my $res = MMM::Agent::Helpers::check_ip($self->interface, $ip); 177 | my $ret = $? >> 8; 178 | return "ERROR: Could not check if IP is configured: $res" if ($ret == 255); 179 | next if ($ret == 1); 180 | # IP is configured... 181 | my $roleobj = new MMM::Agent::Role::(name => $role, ip => $ip); 182 | $roleobj->del(); 183 | $count++; 184 | } 185 | } 186 | return "OK: Removed $count roles"; 187 | } 188 | 189 | sub cmd_set_status($$) { 190 | my $self = shift; 191 | my ($new_state, $new_roles_str, $new_master_str) = @_; 192 | my ($new_master, $new_master_log, $new_master_pos) = split(',', $new_master_str); 193 | 194 | $new_master='' unless(defined($new_master)); 195 | # Change master if we are a slave 196 | if ($new_master ne $self->active_master && $self->mode eq 'slave' && $new_state eq 'ONLINE' && $new_master ne '') { 197 | INFO "Changing active master to '$new_master_str'"; 198 | my $res = MMM::Agent::Helpers::set_active_master($new_master_str); 199 | DEBUG sprintf("Result: %s", defined($res) ? $res : 'undef'); 200 | if (defined($res) && $res =~ /^OK/) { 201 | $self->active_master($new_master); 202 | } 203 | else { 204 | FATAL sprintf("Failed to change master to '%s': %s", $new_master, defined($res) ? $res : 'undef'); 205 | } 206 | } 207 | 208 | # Parse roles 209 | my @new_roles_arr = sort(split(/\,/, $new_roles_str)); 210 | my @new_roles; 211 | foreach my $role_str (@new_roles_arr) { 212 | my $role = MMM::Agent::Role->from_string($role_str); 213 | if (defined($role)) { 214 | push @new_roles, $role; 215 | } 216 | } 217 | 218 | # Process roles 219 | my @added_roles = (); 220 | my @deleted_roles = (); 221 | my $changes_count = 0; 222 | 223 | # Determine changes 224 | my $diff = new Algorithm::Diff:: ($self->roles, \@new_roles, { keyGen => \&MMM::Common::Role::to_string }); 225 | while ($diff->Next) { 226 | next if ($diff->Same); 227 | 228 | $changes_count++; 229 | push (@deleted_roles, $diff->Items(1)) if ($diff->Items(1)); 230 | push (@added_roles, $diff->Items(2)) if ($diff->Items(2)); 231 | } 232 | 233 | # Apply changes 234 | if ($changes_count) { 235 | INFO 'We have some new roles added or old rules deleted!'; 236 | INFO 'Deleted: ', join(', ', sort(@deleted_roles)) if (scalar(@deleted_roles)); 237 | INFO 'Added: ', join(', ', sort(@added_roles)) if (scalar(@added_roles)); 238 | 239 | foreach my $role (@deleted_roles) { $role->del(); } 240 | foreach my $role (@added_roles) { $role->add(); } 241 | 242 | $self->roles(\@new_roles); 243 | } 244 | 245 | # Process state change 246 | if ($new_state ne $self->state) { 247 | if ($new_state eq 'ADMIN_OFFLINE') { MMM::Agent::Helpers::turn_off_slave(); } 248 | if ($self->state eq 'ADMIN_OFFLINE') { MMM::Agent::Helpers::turn_on_slave(); } 249 | $self->state($new_state); 250 | } 251 | 252 | return 'OK: Status applied successfully!'; 253 | } 254 | 255 | sub cmd_get_master_log_pos(){ 256 | my $res = MMM::Agent::Helpers::get_master_log_pos(); 257 | return $res; 258 | } 259 | 260 | sub check_roles($) { 261 | my $self = shift; 262 | 263 | foreach my $role (@{$self->roles}) { 264 | $role->check(); 265 | } 266 | } 267 | 268 | sub from_config($%) { 269 | my $self = shift; 270 | my $config = shift; 271 | 272 | my $host = $config->{host}->{$config->{this}}; 273 | 274 | $self->name ($config->{this}); 275 | $self->ip ($host->{ip}); 276 | $self->port ($host->{agent_port}); 277 | $self->interface ($host->{cluster_interface}); 278 | $self->mode ($host->{mode}); 279 | $self->mysql_port ($host->{mysql_port}); 280 | $self->mysql_user ($host->{agent_user}); 281 | $self->mysql_password ($host->{agent_password}); 282 | $self->writer_role ($config->{active_master_role}); 283 | $self->bin_path ($host->{bin_path}); 284 | } 285 | 286 | 1; 287 | -------------------------------------------------------------------------------- /lib/Agent/Agent.pm.orig: -------------------------------------------------------------------------------- 1 | package MMM::Agent::Agent; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw(EVAL_ERROR); 6 | use Algorithm::Diff; 7 | use DBI; 8 | use Class::Struct; 9 | use Errno qw(EINTR); 10 | use Log::Log4perl qw(:easy); 11 | use MMM::Common::Role; 12 | use MMM::Common::Socket; 13 | use MMM::Agent::Helpers; 14 | use MMM::Agent::Role; 15 | 16 | eval { 17 | no warnings 'once'; 18 | require Unix::Uptime; 19 | *uptime = *Unix::Uptime->uptime; 20 | }; 21 | if ($EVAL_ERROR) { 22 | require MMM::Common::Uptime; 23 | MMM::Common::Uptime->import(qw(uptime)); 24 | } 25 | 26 | our $VERSION = '0.01'; 27 | 28 | 29 | struct 'MMM::Agent::Agent' => { 30 | name => '$', 31 | ip => '$', 32 | port => '$', 33 | interface => '$', 34 | mode => '$', 35 | mysql_port => '$', 36 | mysql_user => '$', 37 | mysql_password => '$', 38 | writer_role => '$', 39 | bin_path => '$', 40 | 41 | active_master => '$', 42 | state => '$', 43 | roles => '@' 44 | }; 45 | 46 | sub main($) { 47 | my $self = shift; 48 | my $socket = MMM::Common::Socket::create_listener($self->ip, $self->port); 49 | $self->roles([]); 50 | $self->active_master(''); 51 | 52 | while (!$main::shutdown) { 53 | 54 | DEBUG 'Listener: Waiting for connection...'; 55 | my $client = $socket->accept(); 56 | next unless ($client); 57 | 58 | DEBUG 'Listener: Connect!'; 59 | while (my $cmd = <$client>) { 60 | chomp($cmd); 61 | DEBUG "Daemon: Command = '$cmd'"; 62 | 63 | my $res = $self->handle_command($cmd); 64 | my $uptime = uptime(); 65 | 66 | print $client "$res|UP:$uptime\n"; 67 | DEBUG "Daemon: Answer = '$res'"; 68 | 69 | return 0 if ($main::shutdown); 70 | } 71 | 72 | close($client); 73 | DEBUG 'Listener: Disconnect!'; 74 | 75 | $self->check_roles(); 76 | } 77 | } 78 | 79 | sub handle_command($$) { 80 | my $self = shift; 81 | my $cmd = shift; 82 | 83 | DEBUG "Received Command $cmd"; 84 | my ($cmd_name, $version, $host, @params) = split('\|', $cmd, -1); 85 | 86 | return "ERROR: Invalid command '$cmd'!" unless (defined($host)); 87 | return "ERROR: Invalid hostname in command ($host)! My name is '" . $self->name . "'" if ($host ne $self->name); 88 | 89 | if ($version > main::MMM_PROTOCOL_VERSION) { 90 | WARN "Version in command '$cmd_name' ($version) is greater than mine (", main::MMM_PROTOCOL_VERSION, ")" 91 | } 92 | 93 | if ($cmd_name eq 'PING') { return cmd_ping (); } 94 | elsif ($cmd_name eq 'SET_STATUS') { return $self->cmd_set_status (@params); } 95 | elsif ($cmd_name eq 'GET_AGENT_STATUS') { return $self->cmd_get_agent_status (); } 96 | elsif ($cmd_name eq 'GET_SYSTEM_STATUS') { return $self->cmd_get_system_status (); } 97 | elsif ($cmd_name eq 'CLEAR_BAD_ROLES') { return $self->cmd_clear_bad_roles (); } 98 | 99 | return "ERROR: Invalid command '$cmd_name'!"; 100 | } 101 | 102 | sub cmd_ping() { 103 | return 'OK: Pinged!'; 104 | } 105 | 106 | sub cmd_get_agent_status($) { 107 | my $self = shift; 108 | 109 | my $answer = join ('|', ( 110 | $self->state, 111 | join(',', @{$self->roles}), 112 | $self->active_master 113 | )); 114 | return "OK: Returning status!|$answer"; 115 | } 116 | 117 | sub cmd_get_system_status($) { 118 | my $self = shift; 119 | 120 | # determine master info 121 | my $dsn = sprintf("DBI:mysql:host=%s;port=%s;mysql_connect_timeout=3", $self->ip, $self->mysql_port); 122 | my $eintr = EINTR; 123 | my $master_ip = ''; 124 | 125 | my $dbh; 126 | CONNECT: { 127 | DEBUG "Connecting to mysql"; 128 | $dbh = DBI->connect($dsn, $self->mysql_user, $self->mysql_password, { PrintError => 0 }); 129 | unless ($dbh) { 130 | redo CONNECT if ($DBI::err == 2003 && $DBI::errstr =~ /\($eintr\)/); 131 | WARN "Couldn't connect to mysql. Can't determine current master host." . $DBI::err . " " . $DBI::errstr; 132 | } 133 | } 134 | 135 | my $slave_status = $dbh->selectrow_hashref('SHOW SLAVE STATUS'); 136 | $master_ip = $slave_status->{Master_Host} if (defined($slave_status)); 137 | 138 | my @roles; 139 | foreach my $role (keys(%{$main::config->{role}})) { 140 | my $role_info = $main::config->{role}->{$role}; 141 | foreach my $ip (@{$role_info->{ips}}) { 142 | my $res = MMM::Agent::Helpers::check_ip($self->interface, $ip); 143 | my $ret = $? >> 8; 144 | return "ERROR: Could not check if IP is configured: $res" if ($ret == 255); 145 | next unless ($ret == 0); 146 | # IP is configured... 147 | push @roles, new MMM::Common::Role::(name => $role, ip => $ip); 148 | } 149 | } 150 | my $res = MMM::Agent::Helpers::may_write(); 151 | my $ret = $? >> 8; 152 | return "ERROR: Could not check if MySQL is writable: $res" if ($ret == 255); 153 | my $writable = ($ret == 1); 154 | 155 | my $answer = join('|', ($writable, join(',', @roles), $master_ip)); 156 | return "OK: Returning status!|$answer"; 157 | } 158 | 159 | sub cmd_clear_bad_roles($) { 160 | my $self = shift; 161 | my $count = 0; 162 | foreach my $role (keys(%{$main::config->{role}})) { 163 | my $role_info = $main::config->{role}->{$role}; 164 | foreach my $ip (@{$role_info->{ips}}) { 165 | my $role_valid = 0; 166 | foreach my $agentrole (@{$self->roles}) { 167 | next unless ($agentrole->name eq $role); 168 | next unless ($agentrole->ip eq $ip); 169 | $role_valid = 1; 170 | last; 171 | } 172 | next if ($role_valid); 173 | my $res = MMM::Agent::Helpers::check_ip($self->interface, $ip); 174 | my $ret = $? >> 8; 175 | return "ERROR: Could not check if IP is configured: $res" if ($ret == 255); 176 | next if ($ret == 1); 177 | # IP is configured... 178 | my $roleobj = new MMM::Agent::Role::(name => $role, ip => $ip); 179 | $roleobj->del(); 180 | $count++; 181 | } 182 | } 183 | return "OK: Removed $count roles"; 184 | } 185 | 186 | sub cmd_set_status($$) { 187 | my $self = shift; 188 | my ($new_state, $new_roles_str, $new_master) = @_; 189 | 190 | # Change master if we are a slave 191 | if ($new_master ne $self->active_master && $self->mode eq 'slave' && $new_state eq 'ONLINE' && $new_master ne '') { 192 | INFO "Changing active master to '$new_master'"; 193 | my $res = MMM::Agent::Helpers::set_active_master($new_master); 194 | DEBUG sprintf("Result: %s", defined($res) ? $res : 'undef'); 195 | if (defined($res) && $res =~ /^OK/) { 196 | $self->active_master($new_master); 197 | } 198 | else { 199 | FATAL sprintf("Failed to change master to '%s': %s", $new_master, defined($res) ? $res : 'undef'); 200 | } 201 | } 202 | 203 | # Parse roles 204 | my @new_roles_arr = sort(split(/\,/, $new_roles_str)); 205 | my @new_roles; 206 | foreach my $role_str (@new_roles_arr) { 207 | my $role = MMM::Agent::Role->from_string($role_str); 208 | if (defined($role)) { 209 | push @new_roles, $role; 210 | } 211 | } 212 | 213 | # Process roles 214 | my @added_roles = (); 215 | my @deleted_roles = (); 216 | my $changes_count = 0; 217 | 218 | # Determine changes 219 | my $diff = new Algorithm::Diff:: ($self->roles, \@new_roles, { keyGen => \&MMM::Common::Role::to_string }); 220 | while ($diff->Next) { 221 | next if ($diff->Same); 222 | 223 | $changes_count++; 224 | push (@deleted_roles, $diff->Items(1)) if ($diff->Items(1)); 225 | push (@added_roles, $diff->Items(2)) if ($diff->Items(2)); 226 | } 227 | 228 | # Apply changes 229 | if ($changes_count) { 230 | INFO 'We have some new roles added or old rules deleted!'; 231 | INFO 'Deleted: ', join(', ', sort(@deleted_roles)) if (scalar(@deleted_roles)); 232 | INFO 'Added: ', join(', ', sort(@added_roles)) if (scalar(@added_roles)); 233 | 234 | foreach my $role (@deleted_roles) { $role->del(); } 235 | foreach my $role (@added_roles) { $role->add(); } 236 | 237 | $self->roles(\@new_roles); 238 | } 239 | 240 | # Process state change 241 | if ($new_state ne $self->state) { 242 | if ($new_state eq 'ADMIN_OFFLINE') { MMM::Agent::Helpers::turn_off_slave(); } 243 | if ($self->state eq 'ADMIN_OFFLINE') { MMM::Agent::Helpers::turn_on_slave(); } 244 | $self->state($new_state); 245 | } 246 | 247 | return 'OK: Status applied successfully!'; 248 | } 249 | 250 | sub check_roles($) { 251 | my $self = shift; 252 | 253 | foreach my $role (@{$self->roles}) { 254 | $role->check(); 255 | } 256 | } 257 | 258 | sub from_config($%) { 259 | my $self = shift; 260 | my $config = shift; 261 | 262 | my $host = $config->{host}->{$config->{this}}; 263 | 264 | $self->name ($config->{this}); 265 | $self->ip ($host->{ip}); 266 | $self->port ($host->{agent_port}); 267 | $self->interface ($host->{cluster_interface}); 268 | $self->mode ($host->{mode}); 269 | $self->mysql_port ($host->{mysql_port}); 270 | $self->mysql_user ($host->{agent_user}); 271 | $self->mysql_password ($host->{agent_password}); 272 | $self->writer_role ($config->{active_master_role}); 273 | $self->bin_path ($host->{bin_path}); 274 | } 275 | 276 | 1; 277 | -------------------------------------------------------------------------------- /lib/Agent/Helpers.pm: -------------------------------------------------------------------------------- 1 | package MMM::Agent::Helpers; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use Log::Log4perl qw(:easy); 6 | 7 | our $VERSION = '0.01'; 8 | 9 | 10 | =head1 NAME 11 | 12 | MMM::Agent::Helpers - an interface to helper programs for B 13 | 14 | =cut 15 | 16 | 17 | =head1 FUNCTIONS 18 | 19 | =over 4 20 | 21 | =item check_ip($if, $ip) 22 | 23 | Check if the IP $ip is configured on interface $if. 24 | 25 | Calls B. 26 | 27 | =cut 28 | 29 | sub check_ip($$) { 30 | my $if = shift; 31 | my $ip = shift; 32 | return _execute('check_ip', "$if $ip"); 33 | } 34 | 35 | 36 | =item configure_ip($if, $ip) 37 | 38 | Check if the IP $ip is configured on interface $if. If not, configure it and 39 | send arp requests to notify other hosts. 40 | 41 | Calls B. 42 | 43 | =cut 44 | 45 | sub configure_ip($$) { 46 | my $if = shift; 47 | my $ip = shift; 48 | return _execute('configure_ip', "$if $ip"); 49 | } 50 | 51 | 52 | =item clear_ip($if, $ip) 53 | 54 | Remove the IP address $ip from interface $if. 55 | 56 | Calls B. 57 | 58 | =cut 59 | 60 | sub clear_ip($$) { 61 | my $if = shift; 62 | my $ip = shift; 63 | return _execute('clear_ip', "$if $ip"); 64 | } 65 | 66 | 67 | =item mysql_may_write( ) 68 | 69 | Determine wheter writes on local MySQL server are allowes. 70 | 71 | Calls B, which reads the config file. 72 | 73 | =cut 74 | 75 | sub may_write() { 76 | return _execute('mysql_may_write'); 77 | } 78 | 79 | =item mysql_allow_write( ) 80 | 81 | Allow writes on local MySQL server. Sets global read_only to 0. 82 | 83 | Calls B, which reads the config file. 84 | 85 | =cut 86 | 87 | sub allow_write() { 88 | return _execute('mysql_allow_write'); 89 | } 90 | 91 | 92 | =item mysql_kill_process( ) 93 | 94 | Deny writes on local MySQL server. Sets global read_only to 1. 95 | 96 | Calls B, which reads the config file. 97 | 98 | =cut 99 | 100 | sub kill_process() { 101 | return _execute('kill_process'); 102 | } 103 | 104 | 105 | =item mysql_deny_write( ) 106 | 107 | Deny writes on local MySQL server. Sets global read_only to 1. 108 | 109 | Calls B, which reads the config file. 110 | 111 | =cut 112 | 113 | sub deny_write() { 114 | return _execute('mysql_deny_write'); 115 | } 116 | 117 | 118 | =item turn_on_slave( ) 119 | 120 | Start slave on local MySQL server. 121 | 122 | Calls B, which reads the config file. 123 | 124 | =cut 125 | 126 | sub turn_on_slave() { 127 | return _execute('turn_on_slave'); 128 | } 129 | 130 | 131 | =item turn_off_slave( ) 132 | 133 | Stop slave on local MySQL server. 134 | 135 | Calls B, which reads the config file. 136 | 137 | =cut 138 | 139 | sub turn_off_slave() { 140 | return _execute('turn_off_slave'); 141 | } 142 | 143 | 144 | =item sync_with_master( ) 145 | 146 | Try to sync a (soon active) master up with his peer (old active master) when the 147 | I is moved. If peer is reachable sync with master log. If 148 | not reachable, sync with relay log. 149 | 150 | Calls B, which reads the config file. 151 | 152 | =cut 153 | 154 | sub sync_with_master() { 155 | return _execute('sync_with_master'); 156 | } 157 | 158 | =item get_master_log_pos( ) 159 | 160 | Try to get current new master's binlog filename and position. 161 | The information will be used for slave to issue "change master to" 162 | 163 | Calls B, which reads the config file. 164 | 165 | =cut 166 | 167 | sub get_master_log_pos() { 168 | sync_with_master(); 169 | return _execute('get_master_log_pos'); 170 | } 171 | 172 | 173 | =item set_active_master($new_master) 174 | 175 | Try to catch up with the old master as far as possible and change the master to the new host. 176 | (Syncs to the master log if the old master is reachable. Otherwise syncs to the relay log.) 177 | 178 | Calls B, which reads the config file. 179 | 180 | =cut 181 | 182 | sub set_active_master($) { 183 | my $new_master_str = shift; 184 | my ($new_master, $new_master_log, $new_master_pos) = split(',', $new_master_str); 185 | return "ERROR: Unknown host $new_master" unless (defined($main::config->{host}->{$new_master})); 186 | return _execute('set_active_master', "$new_master_str"); 187 | } 188 | 189 | #------------------------------------------------------------------------------- 190 | sub _execute($$$) { 191 | my $command = shift; 192 | my $params = shift; 193 | my $return_all = shift; 194 | 195 | my $path = $main::agent->bin_path . "/agent/$command"; 196 | my $config_file = $main::agent->config_file; 197 | $params = '' unless defined($params); 198 | 199 | DEBUG "Executing $path $params"; 200 | my $res = `$path $config_file $params`; 201 | 202 | unless ($return_all) { 203 | my @lines = split /\n/, $res; 204 | return pop(@lines); 205 | } 206 | 207 | return $res; 208 | } 209 | 210 | 1; 211 | 212 | =back 213 | =cut 214 | 215 | -------------------------------------------------------------------------------- /lib/Agent/Helpers/Actions.pm: -------------------------------------------------------------------------------- 1 | package MMM::Agent::Helpers::Actions; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use MMM::Agent::Helpers::Network; 6 | 7 | our $VERSION = '0.01'; 8 | 9 | =head1 NAME 10 | 11 | MMM::Agent::Helpers::Actions - functions for the B helper programs 12 | 13 | =cut 14 | 15 | use DBI; 16 | 17 | 18 | =head1 FUNCTIONS 19 | 20 | =over 4 21 | 22 | =item check_ip($if, $ip) 23 | 24 | Check if the IP $ip is configured on interface $if. 25 | 26 | =cut 27 | 28 | sub check_ip($$) { 29 | my $if = shift; 30 | my $ip = shift; 31 | 32 | if (MMM::Agent::Helpers::Network::check_ip($if, $ip)) { 33 | _exit_ok('IP address is configured'); 34 | } 35 | 36 | _exit_ok('IP address is not configured', 1); 37 | } 38 | 39 | 40 | =item configure_ip($if, $ip) 41 | 42 | Check if the IP $ip is configured on interface $if. If not, configure it and 43 | send arp requests to notify other hosts. 44 | 45 | =cut 46 | 47 | sub configure_ip($$) { 48 | my $if = shift; 49 | my $ip = shift; 50 | 51 | if (MMM::Agent::Helpers::Network::check_ip($if, $ip)) { 52 | _exit_ok('IP address is configured'); 53 | } 54 | 55 | if (!MMM::Agent::Helpers::Network::add_ip($if, $ip)) { 56 | _exit_error("Could not configure ip adress $ip on interface $if!"); 57 | } 58 | MMM::Agent::Helpers::Network::send_arp($if, $ip); 59 | _exit_ok(); 60 | } 61 | 62 | 63 | =item clear_ip($if, $ip) 64 | 65 | Remove the IP address $ip from interface $if. 66 | 67 | =cut 68 | 69 | sub clear_ip($$) { 70 | my $if = shift; 71 | my $ip = shift; 72 | 73 | if (!MMM::Agent::Helpers::Network::check_ip($if, $ip)) { 74 | _exit_ok('IP address is not configured'); 75 | } 76 | 77 | MMM::Agent::Helpers::Network::clear_ip($if, $ip); 78 | _exit_ok(); 79 | } 80 | 81 | 82 | =item mysql_may_write( ) 83 | 84 | Check if writes on local MySQL server are allowed. 85 | 86 | =cut 87 | 88 | sub mysql_may_write() { 89 | my ($host, $port, $user, $password) = _get_connection_info(); 90 | _exit_error('No connection info') unless defined($host); 91 | 92 | # connect to server 93 | my $dbh = _mysql_connect($host, $port, $user, $password); 94 | _exit_error("Can't connect to MySQL (host = $host:$port, user = $user)! " . $DBI::errstr) unless ($dbh); 95 | 96 | # check old read_only state 97 | (my $read_only) = $dbh->selectrow_array('select @@read_only'); 98 | _exit_error('SQL Query Error: ' . $dbh->errstr) unless (defined $read_only); 99 | 100 | _exit_ok('Not allowed') if ($read_only); 101 | _exit_ok('Allowed', 1); 102 | } 103 | 104 | 105 | =item mysql_allow_write( ) 106 | 107 | Allow writes on local MySQL server. Sets global read_only to 0. 108 | 109 | =cut 110 | 111 | sub mysql_allow_write() { 112 | _mysql_set_read_only(0); 113 | _exit_ok(); 114 | } 115 | 116 | 117 | =item mysql_kill_process( ) 118 | 119 | kill all user process on local MySQL 120 | 121 | =cut 122 | 123 | sub mysql_kill_process() { 124 | killall_sql(); 125 | _exit_ok(); 126 | } 127 | 128 | =item mysql_deny_write( ) 129 | 130 | Deny writes on local MySQL server. Sets global read_only to 1. 131 | 132 | =cut 133 | 134 | sub mysql_deny_write() { 135 | _mysql_set_read_only(1); 136 | kill_sql(); 137 | _exit_ok(); 138 | } 139 | 140 | 141 | sub _mysql_set_read_only($) { 142 | my $read_only_new = shift; 143 | my ($host, $port, $user, $password) = _get_connection_info(); 144 | _exit_error('No connection info') unless defined($host); 145 | 146 | # connect to server 147 | my $dbh = _mysql_connect($host, $port, $user, $password); 148 | _exit_error("Can't connect to MySQL (host = $host:$port, user = $user)! " . $DBI::errstr) unless ($dbh); 149 | 150 | # check old read_only state 151 | (my $read_only_old) = $dbh->selectrow_array('select @@read_only'); 152 | _exit_error('SQL Query Error: ' . $dbh->errstr) unless (defined $read_only_old); 153 | return 1 if ($read_only_old == $read_only_new); 154 | 155 | my $res = $dbh->do("set global read_only=$read_only_new"); 156 | _exit_error('SQL Query Error: ' . $dbh->errstr) unless($res); 157 | 158 | $dbh->disconnect(); 159 | $dbh = undef; 160 | 161 | return 1; 162 | } 163 | 164 | =item killall_sql 165 | 166 | kill all user threads to prevent further writes 167 | 168 | =cut 169 | 170 | sub killall_sql() { 171 | 172 | my ($host, $port, $user, $password) = _get_connection_info(); 173 | _exit_error('No connection info') unless defined($host); 174 | 175 | # Connect to server 176 | my $dbh = _mysql_connect($host, $port, $user, $password); 177 | _exit_error("Can't connect to MySQL (host = $host:$port, user = $user)! " . $DBI::errstr) unless ($dbh); 178 | 179 | my $my_id = $dbh->{'mysql_thread_id'}; 180 | 181 | my $max_retries = 2; 182 | my $elapsed_retries = 0; 183 | my $retry = 1; 184 | 185 | while ($elapsed_retries < $max_retries && $retry) { 186 | $retry = 0; 187 | 188 | # Fetch process list 189 | my $processlist = $dbh->selectall_hashref('SHOW PROCESSLIST', 'Id'); 190 | 191 | # Kill processes 192 | foreach my $id (keys(%{$processlist})) { 193 | # Skip ourselves 194 | next if ($id == $my_id); 195 | 196 | # Skip non-client threads (i.e. I/O or SQL threads used on replication slaves, ...) 197 | next if ($processlist->{$id}->{User} eq 'system user'); 198 | 199 | # skip threads of replication clients 200 | next if ($processlist->{$id}->{Command} eq 'Binlog Dump'); 201 | 202 | # Give threads a chance to finish if we're not on our last retry 203 | if ($elapsed_retries < $max_retries 204 | && defined ($processlist->{$id}->{Info}) 205 | && $processlist->{$id}->{Info} =~ /^\s*(\/\*.*?\*\/)?\s*(INSERT|UPDATE|DELETE|REPLACE|CREATE|DROP|ALTER|REPAIR|OPTIMIZE|ANALYZE|CHECK)/si 206 | ) { 207 | # Kill process 208 | $dbh->do("KILL $id"); 209 | next; 210 | } 211 | 212 | } 213 | 214 | $elapsed_retries++; 215 | } 216 | } 217 | 218 | =item kill_sql 219 | 220 | kill all user threads to prevent further writes 221 | 222 | =cut 223 | 224 | sub kill_sql() { 225 | 226 | my ($host, $port, $user, $password) = _get_connection_info(); 227 | _exit_error('No connection info') unless defined($host); 228 | 229 | # Connect to server 230 | my $dbh = _mysql_connect($host, $port, $user, $password); 231 | _exit_error("Can't connect to MySQL (host = $host:$port, user = $user)! " . $DBI::errstr) unless ($dbh); 232 | 233 | my $my_id = $dbh->{'mysql_thread_id'}; 234 | 235 | my $max_retries = $main::config->{max_kill_retries}; 236 | my $elapsed_retries = 0; 237 | my $retry = 1; 238 | 239 | while ($elapsed_retries <= $max_retries && $retry) { 240 | $retry = 0; 241 | 242 | # Fetch process list 243 | my $processlist = $dbh->selectall_hashref('SHOW PROCESSLIST', 'Id'); 244 | 245 | # Kill processes 246 | foreach my $id (keys(%{$processlist})) { 247 | # Skip ourselves 248 | next if ($id == $my_id); 249 | 250 | # Skip non-client threads (i.e. I/O or SQL threads used on replication slaves, ...) 251 | next if ($processlist->{$id}->{User} eq 'system user'); 252 | 253 | # skip threads of replication clients 254 | next if ($processlist->{$id}->{Command} eq 'Binlog Dump'); 255 | 256 | # Give threads a chance to finish if we're not on our last retry 257 | if ($elapsed_retries < $max_retries 258 | && defined ($processlist->{$id}->{Info}) 259 | && $processlist->{$id}->{Info} =~ /^\s*(\/\*.*?\*\/)?\s*(INSERT|UPDATE|DELETE|REPLACE|CREATE|DROP|ALTER|REPAIR|OPTIMIZE|ANALYZE|CHECK)/si 260 | ) { 261 | $retry = 1; 262 | next; 263 | } 264 | 265 | # Kill process 266 | $dbh->do("KILL $id"); 267 | } 268 | 269 | sleep(1) if ($elapsed_retries < $max_retries && $retry); 270 | $elapsed_retries++; 271 | } 272 | } 273 | 274 | 275 | =item toggle_slave($state) 276 | 277 | Toggle slave state. Starts slave if $state != 0. Stops it otherwise. 278 | 279 | =cut 280 | 281 | sub toggle_slave($) { 282 | my $state = shift; 283 | 284 | my ($host, $port, $user, $password) = _get_connection_info(); 285 | _exit_error('No connection info') unless defined($host); 286 | 287 | my $query = $state ? 'START SLAVE' : 'STOP SLAVE'; 288 | 289 | # connect to server 290 | my $dbh = _mysql_connect($host, $port, $user, $password); 291 | _exit_error("Can't connect to MySQL (host = $host:$port, user = $user)! " . $DBI::errstr) unless ($dbh); 292 | 293 | # execute query 294 | my $res = $dbh->do($query); 295 | _exit_error('SQL Query Error: ' . $dbh->errstr) unless($res); 296 | _exit_ok(); 297 | } 298 | 299 | 300 | =item sync_with_master( ) 301 | 302 | Try to sync up a (soon active) master with his peer (old active master) when the I is moved. If the peer is reachable it syncs with the master log. If not reachable, syncs with the relay log. 303 | 304 | =cut 305 | 306 | sub sync_with_master() { 307 | 308 | my $this = _get_this(); 309 | 310 | my ($this_host, $this_port, $this_user, $this_password) = _get_connection_info($this); 311 | _exit_error('No local connection info') unless defined($this_host); 312 | 313 | my $peer = $main::config->{host}->{$this}->{peer}; 314 | _exit_error('No peer defined') unless defined($peer); 315 | 316 | my ($peer_host, $peer_port, $peer_user, $peer_password) = _get_connection_info($peer); 317 | _exit_error('No peer connection info') unless defined($peer_host); 318 | 319 | # Connect to local server 320 | my $this_dbh = _mysql_connect($this_host, $this_port, $this_user, $this_password); 321 | _exit_error("Can't connect to MySQL (host = $this_host:$this_port, user = $this_user)! " . $DBI::errstr) unless ($this_dbh); 322 | 323 | # Connect to peer 324 | my $peer_dbh = _mysql_connect($peer_host, $peer_port, $peer_user, $peer_password); 325 | 326 | # Determine wait log and wait pos 327 | my $wait_log; 328 | my $wait_pos; 329 | if ($peer_dbh) { 330 | my $master_status = $peer_dbh->selectrow_hashref('SHOW MASTER STATUS'); 331 | if (defined($master_status)) { 332 | $wait_log = $master_status->{File}; 333 | $wait_pos = $master_status->{Position}; 334 | } 335 | $peer_dbh->disconnect; 336 | } 337 | unless (defined($wait_log)) { 338 | my $slave_status = $this_dbh->selectrow_hashref('SHOW SLAVE STATUS'); 339 | _exit_error('SQL Query Error: ' . $this_dbh->errstr) unless defined($slave_status); 340 | $wait_log = $slave_status->{Master_Log_File}; 341 | $wait_pos = $slave_status->{Read_Master_Log_Pos}; 342 | } 343 | 344 | # Sync with logs 345 | my $res = $this_dbh->do("SELECT MASTER_POS_WAIT('$wait_log', $wait_pos)"); 346 | _exit_error('SQL Query Error: ' . $this_dbh->errstr) unless($res); 347 | 348 | _exit_ok(); 349 | 350 | } 351 | 352 | sub get_master_log_pos(){ 353 | my $this = _get_this(); 354 | 355 | # Get local connection info 356 | my ($this_host, $this_port, $this_user, $this_password) = _get_connection_info($this); 357 | _exit_error("No connection info for local host '$this_host'") unless defined($this_host); 358 | 359 | # Connect to local server 360 | my $this_dbh = _mysql_connect($this_host, $this_port, $this_user, $this_password); 361 | _exit_error("Can't connect to MySQL (host = $this_host:$this_port, user = $this_user)! " . $DBI::errstr) unless ($this_dbh); 362 | 363 | my $master_log_file; 364 | my $master_log_pos; 365 | if ($this_dbh) { 366 | my $old_master_status = $this_dbh->selectrow_hashref('SHOW MASTER STATUS'); 367 | if (defined($old_master_status)) { 368 | $master_log_file= $old_master_status->{File}; 369 | $master_log_pos= $old_master_status->{Position}; 370 | } 371 | $this_dbh->disconnect; 372 | } 373 | return "$master_log_file:$master_log_pos"; 374 | } 375 | 376 | =item set_active_master($new_master) 377 | 378 | Try to catch up with the old master as far as possible and change the master to the new host. 379 | (Syncs to the master log if the old master is reachable. Otherwise syncs to the relay log.) 380 | 381 | =cut 382 | 383 | sub set_active_master($) { 384 | my $master_str = shift; 385 | my ($new_peer, $new_peer_log, $new_peer_pos) = split( ',', $master_str); 386 | 387 | _exit_error('Name of new master is missing') unless (defined($new_peer)); 388 | 389 | my $this = _get_this(); 390 | 391 | _exit_error('New master is equal to local host!?') if ($this eq $new_peer); 392 | 393 | # Get local connection info 394 | my ($this_host, $this_port, $this_user, $this_password) = _get_connection_info($this); 395 | _exit_error("No connection info for local host '$this_host'") unless defined($this_host); 396 | 397 | # Get connection info for new peer 398 | my ($new_peer_host, $new_peer_port, $new_peer_user, $new_peer_password) = _get_connection_info($new_peer); 399 | _exit_error("No connection info for new peer '$new_peer'") unless defined($new_peer_host); 400 | 401 | # Connect to local server 402 | my $this_dbh = _mysql_connect($this_host, $this_port, $this_user, $this_password); 403 | _exit_error("Can't connect to MySQL (host = $this_host:$this_port, user = $this_user)! " . $DBI::errstr) unless ($this_dbh); 404 | 405 | # Get slave info 406 | my $slave_status = $this_dbh->selectrow_hashref('SHOW SLAVE STATUS'); 407 | _exit_error('SQL Query Error: ' . $this_dbh->errstr) unless defined($slave_status); 408 | 409 | my $wait_log = $slave_status->{Master_Log_File}; 410 | my $wait_pos = $slave_status->{Read_Master_Log_Pos}; 411 | 412 | my $old_peer_ip = $slave_status->{Master_Host}; 413 | _exit_error('No ip for old peer') unless ($old_peer_ip); 414 | 415 | # Get connection info for old peer 416 | my $old_peer = _find_host_by_ip($old_peer_ip); 417 | _exit_error('Invalid master host in show slave status') unless ($old_peer); 418 | 419 | _exit_ok('We are already a slave of the new master') if ($old_peer eq $new_peer); 420 | 421 | my ($old_peer_host, $old_peer_port, $old_peer_user, $old_peer_password) = _get_connection_info($old_peer); 422 | _exit_error("No connection info for new peer '$old_peer'") unless defined($old_peer_host); 423 | 424 | my $old_peer_dbh = _mysql_connect($old_peer_host, $old_peer_port, $old_peer_user, $old_peer_password); 425 | if ($old_peer_dbh) { 426 | my $old_master_status = $old_peer_dbh->selectrow_hashref('SHOW MASTER STATUS'); 427 | if (defined($old_master_status)) { 428 | $wait_log = $old_master_status->{File}; 429 | $wait_pos = $old_master_status->{Position}; 430 | } 431 | $old_peer_dbh->disconnect; 432 | } 433 | 434 | # Sync with logs 435 | my $res = $this_dbh->do("SELECT MASTER_POS_WAIT('$wait_log', $wait_pos)"); 436 | _exit_error('SQL Query Error: ' . $this_dbh->errstr) unless($res); 437 | 438 | # Stop slave 439 | $res = $this_dbh->do('STOP SLAVE'); 440 | _exit_error('SQL Query Error: ' . $this_dbh->errstr) unless($res); 441 | 442 | # Connect to new peer 443 | my $new_peer_dbh = _mysql_connect($new_peer_host, $new_peer_port, $new_peer_user, $new_peer_password); 444 | _exit_error("Can't connect to MySQL (host = $new_peer_host:$new_peer_port, user = $new_peer_user)! " . $DBI::errstr) unless ($new_peer_dbh); 445 | 446 | if( defined($new_peer_log) && defined($new_peer_pos) ){ 447 | $new_peer_log=$new_peer_log; 448 | }else{ 449 | # Get log position of new master 450 | my $new_master_status = $new_peer_dbh->selectrow_hashref('SHOW MASTER STATUS'); 451 | _exit_error('SQL Query Error: ' . $new_peer_dbh->errstr) unless($new_master_status); 452 | 453 | $new_peer_log = $new_master_status->{File}; 454 | $new_peer_pos = $new_master_status->{Position}; 455 | } 456 | $new_peer_dbh->disconnect; 457 | 458 | # Get replication credentials 459 | my ($repl_user, $repl_password) = _get_replication_credentials($new_peer); 460 | 461 | # Change master 462 | my $sql = 'CHANGE MASTER TO' 463 | . " MASTER_HOST='$new_peer_host'," 464 | . " MASTER_PORT=$new_peer_port," 465 | . " MASTER_USER='$repl_user'," 466 | . " MASTER_PASSWORD='$repl_password'," 467 | . " MASTER_LOG_FILE='$new_peer_log'," 468 | . " MASTER_LOG_POS=$new_peer_pos"; 469 | $res = $this_dbh->do($sql); 470 | _exit_error('SQL Query Error: ' . $this_dbh->errstr) unless($res); 471 | 472 | # Start slave 473 | $res = $this_dbh->do('START SLAVE'); 474 | _exit_error('SQL Query Error: ' . $this_dbh->errstr) unless($res); 475 | 476 | return 'OK'; 477 | } 478 | 479 | 480 | =item _get_connection_info([$host]) 481 | 482 | Get connection info for host $host || local host. 483 | 484 | =cut 485 | 486 | sub _get_connection_info($) { 487 | my $host = shift; 488 | 489 | _exit_error('No config present') unless (defined($main::config)); 490 | 491 | $host = $main::config->{this} unless defined($host); 492 | _exit_error('No config present') unless (defined($main::config->{host}->{$host})); 493 | 494 | return ( 495 | $main::config->{host}->{$host}->{ip}, 496 | $main::config->{host}->{$host}->{mysql_port}, 497 | $main::config->{host}->{$host}->{agent_user}, 498 | $main::config->{host}->{$host}->{agent_password} 499 | ); 500 | } 501 | 502 | sub _get_this() { 503 | _exit_error('No config present') unless (defined($main::config)); 504 | return $main::config->{this}; 505 | } 506 | 507 | sub _mysql_connect($$$$) { 508 | my ($host, $port, $user, $password) = @_; 509 | my $dsn = "DBI:mysql:host=$host;port=$port;mysql_connect_timeout=3"; 510 | return DBI->connect($dsn, $user, $password, { PrintError => 0 }); 511 | } 512 | 513 | sub _find_host_by_ip($) { 514 | my $ip = shift; 515 | return undef unless ($ip); 516 | 517 | _exit_error('No config present') unless (defined($main::config)); 518 | 519 | my $hosts = $main::config->{host}; 520 | foreach my $host (keys(%$hosts)) { 521 | return $host if ($hosts->{$host}->{ip} eq $ip); 522 | } 523 | 524 | return undef; 525 | } 526 | 527 | sub _get_replication_credentials($) { 528 | my $host = shift; 529 | return undef unless ($host); 530 | 531 | _exit_error('No config present') unless (defined($main::config)); 532 | _exit_error('No config present') unless (defined($main::config->{host}->{$host})); 533 | 534 | return ( 535 | $main::config->{host}->{$host}->{replication_user}, 536 | $main::config->{host}->{$host}->{replication_password}, 537 | ); 538 | } 539 | 540 | sub _exit_error { 541 | my $msg = shift; 542 | 543 | print "ERROR: $msg\n" if ($msg); 544 | print "ERROR\n" unless ($msg); 545 | 546 | exit(255); 547 | } 548 | 549 | sub _exit_ok { 550 | my $msg = shift; 551 | my $ret = shift || 0; 552 | 553 | print "OK: $msg\n" if ($msg); 554 | print "OK\n" unless ($msg); 555 | 556 | exit($ret); 557 | } 558 | 559 | sub _verbose_exit($$) { 560 | my $ret = shift; 561 | my $msg = shift; 562 | 563 | print $msg, "\n"; 564 | exit($ret); 565 | } 566 | 567 | 1; 568 | -------------------------------------------------------------------------------- /lib/Agent/Helpers/Network.pm: -------------------------------------------------------------------------------- 1 | package MMM::Agent::Helpers::Network; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw( OSNAME ); 6 | 7 | our $VERSION = '0.01'; 8 | 9 | if ($OSNAME eq 'linux' || $OSNAME eq 'freebsd') { 10 | # these libs will always be loaded, use require and then import to avoid that 11 | use Net::ARP; 12 | use Time::HiRes qw( usleep ); 13 | } 14 | 15 | =head1 NAME 16 | 17 | MMM::Agent::Helpers::Network - network related functions for the B helper programs 18 | 19 | =cut 20 | 21 | 22 | =head1 FUNCTIONS 23 | 24 | =over 4 25 | 26 | =item check_ip($if, $ip) 27 | 28 | Check if the IP $ip is configured on interface $if. Returns 0 if not, 1 otherwise. 29 | 30 | =cut 31 | 32 | sub check_ip($$) { 33 | my $if = shift; 34 | my $ip = shift; 35 | 36 | my $output; 37 | if ($OSNAME eq 'linux') { 38 | $output = `/sbin/ip addr show dev $if`; 39 | _exit_error("Could not check if ip $ip is configured on $if: $output") if ($? >> 8 == 255); 40 | } 41 | elsif ($OSNAME eq 'solaris') { 42 | # FIXME $if is not used here 43 | $output = `/usr/sbin/ifconfig -a | grep inet`; 44 | _exit_error("Could not check if ip $ip is configured on $if: $output") if ($? >> 8 == 255); 45 | } 46 | elsif ($OSNAME eq 'freebsd') { 47 | $output = `/sbin/ifconfig $if | grep inet`; 48 | _exit_error("Could not check if ip $ip is configured on $if: $output") if ($? >> 8 == 255); 49 | } 50 | else { 51 | _exit_error("ERROR: Unsupported platform!"); 52 | } 53 | 54 | return ($output =~ /\D+$ip\D+/) ? 1 : 0; 55 | } 56 | 57 | 58 | =item add_ip($if, $ip) 59 | 60 | Add IP $ip to the interface $if. 61 | 62 | =cut 63 | 64 | sub add_ip($$) { 65 | my $if = shift; 66 | my $ip = shift; 67 | 68 | my $output; 69 | if ($OSNAME eq 'linux') { 70 | $output = `/sbin/ip addr add $ip/32 dev $if`; 71 | _exit_error("Could not configure ip $ip on interface $if: $output") if ($? >> 8 == 255); 72 | } 73 | elsif ($OSNAME eq 'solaris') { 74 | $output = `/usr/sbin/ifconfig $if addif $ip`; 75 | _exit_error("Could not configure ip $ip on interface $if: $output") if ($? >> 8 == 255); 76 | my $logical_if = _solaris_find_logical_if($ip); 77 | unless ($logical_if) { 78 | _exit_error("ERROR: Can't find logical interface with IP = $ip"); 79 | } 80 | $output = `/usr/sbin/ifconfig $logical_if up`; 81 | _exit_error("Could not activate logical interface $logical_if with ip $ip on interface: $output") if ($? >> 8 == 255); 82 | } 83 | elsif ($OSNAME eq 'freebsd') { 84 | $output = `/sbin/ifconfig $if inet $ip netmask 255.255.255.255 alias`; 85 | _exit_error("Could not configure ip $ip on interface $if: $output") if ($? >> 8 == 255); 86 | } 87 | else { 88 | _exit_error("ERROR: Unsupported platform!"); 89 | } 90 | return check_ip($if, $ip); 91 | } 92 | 93 | 94 | =item clear_ip($if, $ip) 95 | 96 | Remove the IP $ip from the interface $if. 97 | 98 | =cut 99 | 100 | sub clear_ip($$) { 101 | my $if = shift; 102 | my $ip = shift; 103 | 104 | my $output; 105 | if ($OSNAME eq 'linux') { 106 | $output = `/sbin/ip addr del $ip/32 dev $if`; 107 | _exit_error("Could not remove ip $ip from interface $if: $output") if ($? >> 8 == 255); 108 | } 109 | elsif ($OSNAME eq 'solaris') { 110 | $output = `/usr/sbin/ifconfig $if removeif $ip`; 111 | _exit_error("Could not remove ip $ip from interface $if: $output") if ($? >> 8 == 255); 112 | } 113 | elsif ($OSNAME eq 'freebsd') { 114 | $output = `/sbin/ifconfig $if inet $ip -alias`; 115 | _exit_error("Could not remove ip $ip from interface $if: $output") if ($? >> 8 == 255); 116 | } 117 | else { 118 | exit(1); 119 | } 120 | } 121 | 122 | 123 | =item send_arp($if, $ip) 124 | 125 | Send arp requests for the IP $ip to the broadcast address on network interface $if. 126 | 127 | =cut 128 | 129 | sub send_arp($$) { 130 | my $if = shift; 131 | my $ip = shift; 132 | 133 | 134 | if ($OSNAME eq 'linux' || $OSNAME eq 'freebsd') { 135 | my $mac = ''; 136 | if ($Net::ARP::VERSION < 1.0) { 137 | Net::ARP::get_mac($if, $mac); 138 | } 139 | else { 140 | $mac = Net::ARP::get_mac($if); 141 | } 142 | return "ERROR: Couldn't get mac adress of interface $if" unless ($mac); 143 | 144 | for (my $i = 0; $i < 5; $i++) { 145 | Net::ARP::send_packet($if, $ip, $ip, $mac, 'ff:ff:ff:ff:ff:ff', 'request'); 146 | usleep(50); 147 | Net::ARP::send_packet($if, $ip, $ip, $mac, 'ff:ff:ff:ff:ff:ff', 'reply'); 148 | usleep(50) if ($i < 4); 149 | } 150 | } 151 | elsif ($OSNAME eq 'solaris') { 152 | # Get params for send_arp 153 | my $ipaddr = `/usr/sbin/ifconfig $if`; 154 | 155 | # Get broadcast address and netmask 156 | $ipaddr =~ /netmask\s*([0-9a-f]+)\s*broadcast\s*([\d\.]+)/i; 157 | my $if_bcast = $1; 158 | my $if_mask = $2; 159 | `/bin/send_arp -i 100 -r 5 -p /tmp/send_arp $if $ip auto $if_bcast $if_mask`; 160 | } 161 | else { 162 | _exit_error("ERROR: Unsupported platform!"); 163 | } 164 | } 165 | 166 | sub _exit_error { 167 | my $msg = shift; 168 | 169 | print "ERROR: $msg\n" if ($msg); 170 | print "ERROR\n" unless ($msg); 171 | 172 | exit(255); 173 | } 174 | 175 | #------------------------------------------------------------------------------- 176 | sub _solaris_find_logical_if($) { 177 | my $ip = shift; 178 | my $ifconfig = `/usr/sbin/ifconfig -a`; 179 | $ifconfig =~ s/\n/ /g; 180 | 181 | while ($ifconfig =~ s/([a-z0-9\:]+)(\:\s+.*?)inet\s*([0-9\.]+)//) { 182 | return $1 if ($3 eq $ip); 183 | } 184 | return undef; 185 | } 186 | 187 | 1; 188 | -------------------------------------------------------------------------------- /lib/Agent/Role.pm: -------------------------------------------------------------------------------- 1 | package MMM::Agent::Role; 2 | use base 'MMM::Common::Role'; 3 | 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use Log::Log4perl qw(:easy); 7 | use MMM::Agent::Helpers; 8 | 9 | our $VERSION = '0.01'; 10 | 11 | =head1 NAME 12 | 13 | MMM::Agent::Role - role class (agent) 14 | 15 | =cut 16 | 17 | 18 | =head1 METHODS 19 | 20 | =over 4 21 | 22 | =item check() 23 | 24 | Check (=assure) that the role is configured on the local host. 25 | 26 | =cut 27 | sub check($) { 28 | my $self = shift; 29 | 30 | my $res; 31 | 32 | if ($self->name eq $main::agent->writer_role) { 33 | $res = MMM::Agent::Helpers::allow_write(); 34 | if (!defined($res) || $res !~ /^OK/) { 35 | FATAL sprintf("Couldn't allow writes: %s", defined($res) ? $res : 'undef'); 36 | return; 37 | } 38 | } 39 | 40 | $res = MMM::Agent::Helpers::configure_ip($main::agent->interface, $self->ip); 41 | if (!defined($res) || $res !~ /^OK/) { 42 | FATAL sprintf("Couldn't configure IP '%s' on interface '%s': %s", $self->ip, $main::agent->interface, defined($res) ? $res : 'undef'); 43 | return; 44 | } 45 | } 46 | 47 | =item add() 48 | 49 | Add a role to the local host. 50 | 51 | =cut 52 | sub add($) { 53 | my $self = shift; 54 | 55 | my $res; 56 | 57 | if ($self->name eq $main::agent->writer_role) { 58 | $res = MMM::Agent::Helpers::sync_with_master(); 59 | if (!defined($res) || $res !~ /^OK/) { 60 | FATAL sprintf("Couldn't sync with master: %s", defined($res) ? $res : 'undef'); 61 | return; 62 | } 63 | $res = MMM::Agent::Helpers::allow_write(); 64 | if (!defined($res) || $res !~ /^OK/) { 65 | FATAL sprintf("Couldn't allow writes: %s", defined($res) ? $res : 'undef'); 66 | return; 67 | } 68 | } 69 | 70 | $res = MMM::Agent::Helpers::configure_ip($main::agent->interface, $self->ip); 71 | if (!defined($res) || $res !~ /^OK/) { 72 | FATAL sprintf("Couldn't configure IP '%s' on interface '%s': %s", $self->ip, $main::agent->interface, defined($res) ? $res : 'undef'); 73 | return; 74 | } 75 | } 76 | 77 | =item del() 78 | 79 | Delete a role from the local host. 80 | 81 | =cut 82 | sub del($) { 83 | my $self = shift; 84 | 85 | my $res; 86 | 87 | $res = MMM::Agent::Helpers::clear_ip($main::agent->interface, $self->ip); 88 | if (!defined($res) || $res !~ /^OK/) { 89 | FATAL sprintf("Couldn't clear IP '%s' from interface '%s': %s", $self->ip, $main::agent->interface, defined($res) ? $res : 'undef'); 90 | } 91 | 92 | #kill all active process 93 | #let all uncommited transactions to rollback 94 | $res = MMM::Agent::Helpers::kill_process(); 95 | if (!defined($res) || $res !~ /^OK/) { 96 | FATAL sprintf("Couldn't kill active process in MySQL: %s", defined($res) ? $res : 'undef'); 97 | } 98 | 99 | if ($self->name eq $main::agent->writer_role) { 100 | $res = MMM::Agent::Helpers::deny_write(); 101 | if (!defined($res) || $res !~ /^OK/) { 102 | FATAL sprintf("Couldn't deny writes: %s", defined($res) ? $res : 'undef'); 103 | } 104 | } 105 | 106 | } 107 | 108 | 1; 109 | -------------------------------------------------------------------------------- /lib/Common/Angel.pm: -------------------------------------------------------------------------------- 1 | package MMM::Common::Angel; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw( CHILD_ERROR ERRNO ); 6 | use Log::Log4perl qw(:easy); 7 | use Errno qw( EINTR ); 8 | use POSIX qw( WIFEXITED WIFSIGNALED WEXITSTATUS WTERMSIG WNOHANG ); 9 | 10 | 11 | our $start_process; 12 | our $pid; 13 | our $attempts; 14 | our $starttime; 15 | 16 | 17 | sub Init($) { 18 | my $pidfile = shift; 19 | 20 | 21 | $MMM::Common::Angel::start_process = 1; 22 | $MMM::Common::Angel::attempts = 0; 23 | $MMM::Common::Angel::starttime = time(); 24 | my $is_shutdown = 0; 25 | 26 | $pidfile->create() if (defined($pidfile)); 27 | 28 | local $SIG{INT} = \&MMM::Common::Angel::SignalHandler; 29 | local $SIG{TERM} = \&MMM::Common::Angel::SignalHandler; 30 | local $SIG{QUIT} = \&MMM::Common::Angel::SignalHandler; 31 | 32 | do { 33 | $MMM::Common::Angel::attempts++; 34 | 35 | if ($MMM::Common::Angel::start_process) { 36 | $MMM::Common::Angel::start_process = 0; 37 | 38 | # Create a new child 39 | $MMM::Common::Angel::pid = fork(); 40 | 41 | # Die if we couldn't fork 42 | LOGDIE "Couldn't fork child process." unless (defined($MMM::Common::Angel::pid)); 43 | 44 | # Return if we are the child 45 | return if ($MMM::Common::Angel::pid == 0); 46 | } 47 | 48 | # Wait for child to exit 49 | if (waitpid($MMM::Common::Angel::pid, 0) == -1) { 50 | if ($ERRNO{ECHLD}) { 51 | $is_shutdown = 1 unless ($MMM::Common::Angel::start_process); 52 | } 53 | } 54 | else { 55 | if (WIFEXITED($CHILD_ERROR)) { 56 | if (WEXITSTATUS($?) == 0) { 57 | INFO "Child exited normally (with exitcode 0), shutting down"; 58 | $is_shutdown = 1; 59 | } 60 | else { 61 | my $now = time(); 62 | my $diff = $now - $MMM::Common::Angel::starttime; 63 | if ($MMM::Common::Angel::attempts >= 10 && $diff < 300) { 64 | FATAL sprintf("Child exited with exitcode %s and has failed more than 10 times consecutively in the last 5 minutes, not restarting", WEXITSTATUS($?)); 65 | $MMM::Common::Angel::start_process = 0; 66 | $is_shutdown = 1; 67 | } 68 | else { 69 | FATAL sprintf("Child exited with exitcode %s, restarting after 10 second sleep", WEXITSTATUS($?)); 70 | if ($diff >= 300 ) { 71 | # reset attempts and starttime 72 | $MMM::Common::Angel::attempts = 0; 73 | $MMM::Common::Angel::starttime = time(); 74 | } 75 | sleep(10); 76 | $MMM::Common::Angel::start_process = 1; 77 | } 78 | } 79 | } 80 | if (WIFSIGNALED($CHILD_ERROR)) { 81 | FATAL sprintf("Child exited with signal %s, restarting", WTERMSIG($?)); 82 | $MMM::Common::Angel::start_process = 1; 83 | } 84 | } 85 | 86 | } while (!$is_shutdown); 87 | 88 | 89 | $pidfile->remove() if (defined($pidfile)); 90 | exit(0); 91 | } 92 | 93 | sub SignalHandler { 94 | my $signame = shift; 95 | kill ($signame, $MMM::Common::Angel::pid); 96 | } 97 | 98 | 1; 99 | -------------------------------------------------------------------------------- /lib/Common/Log.pm: -------------------------------------------------------------------------------- 1 | package MMM::Common::Log; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use Log::Log4perl qw(:easy); 6 | use English qw( PROGRAM_NAME ); 7 | 8 | our $VERSION = '0.01'; 9 | 10 | 11 | sub init($$) { 12 | my $file = shift; 13 | my $progam = shift; 14 | 15 | my @paths = qw(/etc /etc/mmm /etc/mysql-mmm); 16 | 17 | # Determine filename 18 | my $fullname; 19 | foreach my $path (@paths) { 20 | if (-r "$path/$file") { 21 | $fullname = "$path/$file"; 22 | last; 23 | } 24 | } 25 | 26 | # Read configuration from file 27 | if ($fullname) { 28 | Log::Log4perl->init($fullname); 29 | return; 30 | } 31 | 32 | # Use default configuration 33 | my $conf = " 34 | log4perl.logger = INFO, LogFile 35 | 36 | log4perl.appender.LogFile = Log::Log4perl::Appender::File 37 | log4perl.appender.LogFile.Threshold = INFO 38 | log4perl.appender.LogFile.filename = /var/log/mysql-mmm/$progam.log 39 | log4perl.appender.LogFile.recreate = 1 40 | log4perl.appender.LogFile.layout = PatternLayout 41 | log4perl.appender.LogFile.layout.ConversionPattern = %d %5p %m%n 42 | "; 43 | Log::Log4perl->init(\$conf); 44 | 45 | } 46 | 47 | sub debug() { 48 | my $stdout_appender = Log::Log4perl::Appender->new( 49 | 'Log::Log4perl::Appender::Screen', 50 | name => 'ScreenLog', 51 | stderr => 0 52 | ); 53 | my $layout = Log::Log4perl::Layout::PatternLayout->new('%d %5p %m%n'); 54 | $stdout_appender->layout($layout); 55 | Log::Log4perl::Logger->get_root_logger()->add_appender($stdout_appender); 56 | Log::Log4perl::Logger->get_root_logger()->level($DEBUG); 57 | } 58 | 59 | 1; 60 | -------------------------------------------------------------------------------- /lib/Common/PidFile.pm: -------------------------------------------------------------------------------- 1 | package MMM::Common::PidFile; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw( PROCESS_ID ); 6 | use Log::Log4perl qw(:easy); 7 | 8 | our $VERSION = '0.01'; 9 | 10 | sub new($$) { 11 | my $self = shift; 12 | my $path = shift; 13 | 14 | return bless { 'path' => $path }, $self; 15 | } 16 | 17 | sub exists($) { 18 | my $self = shift; 19 | return -f $self->{path}; 20 | } 21 | 22 | sub is_running($) { 23 | my $self = shift; 24 | 25 | return 0 unless $self->exists(); 26 | 27 | open(PID, $self->{path}) || LOGDIE "Can't open pid file '$self->{path}' for reading!\n"; 28 | chomp(my $pid = ); 29 | close(PID); 30 | 31 | return kill(0, $pid); 32 | } 33 | 34 | sub create($) { 35 | my $self = shift; 36 | 37 | open(PID, ">" . $self->{path}) || LOGDIE "Can't open pid file '$self->{path}' for writing!\n"; 38 | print PID $PROCESS_ID; 39 | close(PID); 40 | 41 | DEBUG "Created pid file '$self->{path}' with pid $PROCESS_ID"; 42 | } 43 | 44 | sub remove($) { 45 | my $self = shift; 46 | 47 | unlink $self->{path}; 48 | } 49 | 50 | 1; 51 | 52 | __END__ 53 | 54 | =head1 NAME 55 | 56 | MMM::Common::Pidfile - Manage process id files 57 | 58 | =cut 59 | 60 | 61 | =head1 SYNOPSIS 62 | 63 | my $pidfile = new MMM::Common::PidFile:: '/path/to/your.pid'; 64 | 65 | # create pidfile with current process id 66 | $pidfile->create(); 67 | 68 | # check if pidfile exists 69 | $pidfile->exists(); 70 | 71 | # check if the process with the process id from the pidfile is still running 72 | $pidfile->is_running(); 73 | 74 | # remove pidfile 75 | $pidfile->remove(); 76 | 77 | =cut 78 | 79 | -------------------------------------------------------------------------------- /lib/Common/Role.pm: -------------------------------------------------------------------------------- 1 | package MMM::Common::Role; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | 6 | our $VERSION = '0.01'; 7 | 8 | use Class::Struct; 9 | 10 | use overload 11 | '==' => \&is_equal_full, 12 | 'eq' => \&is_equal_name, 13 | '!=' => sub { return !MMM::Common::Role::is_equal_full($_[0], $_[1]); }, 14 | 'ne' => sub { return !MMM::Common::Role::is_equal_name($_[0], $_[1]); }, 15 | 'cmp' => \&cmp, 16 | '""' => \&to_string; 17 | 18 | 19 | struct 'MMM::Common::Role' => { 20 | name => '$', 21 | ip => '$', 22 | }; 23 | 24 | 25 | #------------------------------------------------------------------------------- 26 | # NOTE: takes a role object as param 27 | sub is_equal_full($$) { 28 | my $self = shift; 29 | my $other = shift; 30 | 31 | return 0 if ($self->name ne $other->name); 32 | return 0 if ($self->ip ne $other->ip); 33 | return 1; 34 | } 35 | 36 | #------------------------------------------------------------------------------- 37 | # NOTE: takes a role object as param 38 | sub is_equal_name($$) { 39 | my $self = shift; 40 | my $other = shift; 41 | 42 | return ($self->name eq $other->name); 43 | } 44 | 45 | sub cmp($$) { 46 | my $self = shift; 47 | my $other = shift; 48 | 49 | return ($self->name cmp $other->name) if ($self->name ne $other->name); 50 | return ($self->ip cmp $other->ip); 51 | } 52 | 53 | sub to_string($) { 54 | my $self = shift; 55 | return sprintf('%s(%s)', $self->name, $self->ip); 56 | } 57 | 58 | sub from_string($$) { 59 | my $class = shift; 60 | my $string = shift; 61 | 62 | if (my ($name, $ip) = $string =~ /(.*)\((.*)\)/) { 63 | return $class->new(name => $name, ip => $ip); 64 | } 65 | return undef; 66 | } 67 | 68 | 1; 69 | -------------------------------------------------------------------------------- /lib/Common/Socket.pm: -------------------------------------------------------------------------------- 1 | package MMM::Common::Socket; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use Log::Log4perl qw(:easy); 6 | use IO::Socket::INET; 7 | 8 | our $VERSION = '0.01'; 9 | 10 | 11 | =head1 NAME 12 | 13 | MMM::Common::Socket - functions for socket creation 14 | 15 | =cut 16 | 17 | 18 | =head1 FUNCTIONS 19 | 20 | =over 4 21 | 22 | =item create_listener($host, $port) 23 | 24 | Create a listening (ssl) socket on $host:$port. 25 | 26 | =cut 27 | 28 | sub create_listener($$) { 29 | my $host = shift; 30 | my $port = shift; 31 | 32 | my $socket_class = 'IO::Socket::INET'; 33 | my %socket_opts; 34 | my $err = sub {''}; 35 | if (defined($main::config->{'socket'}) && $main::config->{'socket'}->{type} eq 'ssl') { 36 | require IO::Socket::SSL; 37 | $socket_class = 'IO::Socket::SSL'; 38 | %socket_opts = ( 39 | SSL_cert_file => $main::config->{'socket'}->{cert_file}, 40 | SSL_key_file => $main::config->{'socket'}->{key_file}, 41 | SSL_ca_file => $main::config->{'socket'}->{ca_file}, 42 | SSL_verify_mode => 0x03 43 | ); 44 | $err = sub {"\n ", IO::Socket::SSL::errstr()}; 45 | } 46 | 47 | my $sock = $socket_class->new( 48 | LocalHost => $host, 49 | LocalPort => $port, 50 | Proto => 'tcp', 51 | Listen => 10, 52 | Reuse => 1, 53 | %socket_opts, 54 | ) or LOGDIE "Listener: Can't create socket!", $err->(); 55 | 56 | 57 | $sock->timeout(3); 58 | return($sock); 59 | } 60 | 61 | =item create_sender($host, $port, $timeout) 62 | 63 | Create a (ssl) client socket on $host:$port with timeout $timeout. 64 | 65 | =cut 66 | 67 | sub create_sender($$$) { 68 | my $host = shift; 69 | my $port = shift; 70 | my $timeout = shift; 71 | 72 | my $socket_class = 'IO::Socket::INET'; 73 | my %socket_opts; 74 | 75 | if (defined($main::config->{'socket'}) && $main::config->{'socket'}->{type} eq "ssl") { 76 | require IO::Socket::SSL; 77 | $socket_class = 'IO::Socket::SSL'; 78 | %socket_opts = ( 79 | SSL_use_cert => 1, 80 | SSL_cert_file => $main::config->{'socket'}->{cert_file}, 81 | SSL_key_file => $main::config->{'socket'}->{key_file}, 82 | SSL_ca_file => $main::config->{'socket'}->{ca_file}, 83 | ); 84 | } 85 | 86 | return $socket_class->new( 87 | PeerAddr => $host, 88 | PeerPort => $port, 89 | Proto => 'tcp', 90 | ($timeout ? (Timeout => $timeout) : ()), 91 | %socket_opts, 92 | ); 93 | } 94 | 95 | 1; 96 | -------------------------------------------------------------------------------- /lib/Common/Uptime.pm: -------------------------------------------------------------------------------- 1 | package MMM::Common::Uptime; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw( OSNAME ); 6 | use Log::Log4perl qw(:easy); 7 | 8 | require Exporter; 9 | our @ISA = qw( Exporter ); 10 | our @EXPORT_OK = qw( uptime ); 11 | 12 | our $VERSION = '0.01'; 13 | 14 | # FIXME Solaris 15 | 16 | if ($OSNAME eq 'linux') { 17 | use constant UPTIME => "/proc/uptime"; 18 | } 19 | else { 20 | LOGDIE "Unsupported platform - can't get uptime!"; 21 | } 22 | 23 | sub uptime { 24 | if ($OSNAME eq 'linux') { 25 | DEBUG "Fetching uptime from ", UPTIME; 26 | open(FILE, UPTIME) || LOGDIE "Unable to get uptime from ", UPTIME; 27 | my $line = ; 28 | my ($uptime, $idle) = split(/\s+/, $line); 29 | close(FILE); 30 | 31 | DEBUG "Uptime is ", $uptime; 32 | return $uptime; 33 | } 34 | 35 | LOGDIE "Unsupported platform - can't get uptime!"; 36 | } 37 | 38 | 1; 39 | -------------------------------------------------------------------------------- /lib/Monitor/Agent.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::Agent; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use Log::Log4perl qw(:easy); 6 | use MMM::Common::Socket; 7 | use MMM::Monitor::ChecksStatus; 8 | 9 | our $VERSION = '0.01'; 10 | 11 | use Class::Struct; 12 | no warnings qw(Class::Struct); 13 | 14 | 15 | struct 'MMM::Monitor::Agent' => { 16 | host => '$', 17 | mode => '$', 18 | ip => '$', 19 | port => '$', 20 | 21 | state => '$', 22 | roles => '@', 23 | uptime => '$', 24 | last_uptime => '$', 25 | last_state_change => '$', 26 | 27 | online_since => '$', 28 | 29 | flapping => '$', 30 | flapstart => '$', 31 | flapcount => '$', 32 | 33 | agent_down => '$' 34 | }; 35 | 36 | sub state { 37 | my $self = shift; 38 | if (@_) { 39 | my $new_state = shift; 40 | my $old_state = $self->{'MMM::Monitor::Agent::state'}; 41 | 42 | $self->last_state_change(time()) if ($old_state ne $new_state); 43 | $self->online_since(time()) if ($old_state ne $new_state && $new_state eq 'ONLINE'); 44 | 45 | if ($old_state ne $new_state && $main::config->{monitor}->{flap_count}) { 46 | if ($old_state eq 'ONLINE' and $new_state ne 'ADMIN_OFFLINE') { 47 | if (!$self->flapstart || $self->flapstart < time() - $main::config->{monitor}->{flap_duration}) { 48 | $self->flapstart(time()); 49 | $self->flapcount(1); 50 | } 51 | else { 52 | $self->{'MMM::Monitor::Agent::flapcount'}++; 53 | if ($self->flapcount > $main::config->{monitor}->{flap_count}) { 54 | $self->flapping(1); 55 | $self->flapstart(0); 56 | FATAL sprintf('Host %s is flapping!', $self->host); 57 | } 58 | } 59 | } 60 | } 61 | $self->{'MMM::Monitor::Agent::state'} = $new_state; 62 | warn 'Too many args to state' if @_; 63 | } 64 | return $self->{'MMM::Monitor::Agent::state'}; 65 | } 66 | 67 | sub _send_command { 68 | my $self = shift; 69 | my $cmd = shift; 70 | my @params = @_; 71 | 72 | 73 | my $checks_status = MMM::Monitor::ChecksStatus->instance(); 74 | unless ($checks_status->ping($self->host)) { 75 | return 0; 76 | } 77 | 78 | DEBUG sprintf("Sending command '%s(%s)' to %s (%s:%s)", $cmd, join(', ', @params), $self->host, $self->ip, $self->port); 79 | 80 | my $socket; 81 | CONNECT: { 82 | $socket = MMM::Common::Socket::create_sender($self->ip, $self->port, 10); 83 | unless ($socket && $socket->connected) { 84 | redo CONNECT if ($!{EINTR}); 85 | return 0; 86 | } 87 | } 88 | 89 | 90 | print $socket join('|', $cmd, main::MMM_PROTOCOL_VERSION, $self->host, @params), "\n"; 91 | 92 | my $res; 93 | READ: { 94 | $res = <$socket>; 95 | redo READ if !$res && $!{EINTR}; 96 | } 97 | close($socket); 98 | 99 | unless (defined($res)) { 100 | WARN sprintf('Received undefined answer from host %s. $!: %s', $self->host, $!); 101 | return 0; 102 | } 103 | 104 | DEBUG "Received Answer: $res"; 105 | 106 | if ($res =~ /(.*)\|UP:(.*)/) { 107 | $res = $1; 108 | my $uptime = $2; 109 | $self->uptime($uptime); 110 | $self->last_uptime($uptime) if ($self->state eq 'ONLINE'); 111 | } 112 | else { 113 | WARN sprintf('Received bad answer \'%s\' from host %s. $!: %s', $res, $self->host, $!); 114 | } 115 | 116 | return $res; 117 | } 118 | 119 | sub _send_command_retry { 120 | my $self = shift; 121 | my $retries = shift; 122 | my $cmd = shift; 123 | my @params = @_; 124 | 125 | my $res; 126 | 127 | do { 128 | $res = $self->_send_command($cmd, @params); 129 | if ($res) { return $res; } 130 | $retries--; 131 | if ($retries >= 0) { DEBUG "Retrying to send command"; } 132 | } while ($retries >= 0); 133 | return $res; 134 | } 135 | 136 | sub cmd_ping($) { 137 | my $self = shift; 138 | my $retries = shift || 0; 139 | return $self->_send_command_retry($retries, 'PING'); 140 | } 141 | 142 | sub cmd_get_master_log_pos($$) { 143 | my $self = shift; 144 | my $master = shift; 145 | my $retries = shift || 0; 146 | 147 | return $self->_send_command_retry($retries, 'GET_MASTER_POS', $self->state, join(',', sort(@{$self->roles})), $master); 148 | } 149 | 150 | sub cmd_set_status($$) { 151 | my $self = shift; 152 | my $master_str = shift; 153 | my $retries = shift || 0; 154 | 155 | return $self->_send_command_retry($retries, 'SET_STATUS', $self->state, join(',', sort(@{$self->roles})), $master_str); 156 | } 157 | 158 | sub cmd_get_agent_status($) { 159 | my $self = shift; 160 | my $retries = shift || 0; 161 | return $self->_send_command_retry($retries, 'GET_AGENT_STATUS'); 162 | } 163 | 164 | sub cmd_get_system_status($) { 165 | my $self = shift; 166 | my $retries = shift || 0; 167 | return $self->_send_command_retry($retries, 'GET_SYSTEM_STATUS'); 168 | } 169 | 170 | sub cmd_clear_bad_roles($) { 171 | my $self = shift; 172 | my $retries = shift || 0; 173 | return $self->_send_command_retry($retries, 'CLEAR_BAD_ROLES'); 174 | } 175 | 176 | 1; 177 | -------------------------------------------------------------------------------- /lib/Monitor/Agents.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::Agents; 2 | use base 'Class::Singleton'; 3 | 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use Log::Log4perl qw(:easy); 7 | use IO::Handle; 8 | use File::Temp; 9 | use File::Basename; 10 | use MMM::Monitor::Agent; 11 | use MMM::Monitor::Role; 12 | 13 | 14 | 15 | 16 | =head1 NAME 17 | 18 | MMM::Monitor::Agents - single instance class holding status information for all agent hosts 19 | 20 | =head1 SYNOPSIS 21 | 22 | # Get the instance 23 | my $agents = MMM::Monitor::Agents->instance(); 24 | 25 | =cut 26 | 27 | sub _new_instance($) { 28 | my $class = shift; 29 | my $data = {}; 30 | 31 | my @hosts = keys(%{$main::config->{host}}); 32 | 33 | foreach my $host (@hosts) { 34 | $data->{$host} = new MMM::Monitor::Agent:: ( 35 | host => $host, 36 | mode => $main::config->{host}->{$host}->{mode}, 37 | ip => $main::config->{host}->{$host}->{ip}, 38 | port => $main::config->{host}->{$host}->{agent_port}, 39 | state => 'UNKNOWN', 40 | roles => [], 41 | uptime => 0, 42 | last_uptime => 0 43 | ); 44 | } 45 | return bless $data, $class; 46 | } 47 | 48 | 49 | =head1 FUNCTIONS 50 | 51 | =over 4 52 | 53 | =item exists($host) 54 | 55 | Check if host $host exists. 56 | 57 | =cut 58 | 59 | sub exists($$) { 60 | my $self = shift; 61 | my $host = shift; 62 | return defined($self->{$host}); 63 | } 64 | 65 | 66 | =item get($host) 67 | 68 | Get agent for host $host. 69 | 70 | =cut 71 | 72 | sub get($$) { 73 | my $self = shift; 74 | my $host = shift; 75 | return $self->{$host}; 76 | } 77 | 78 | 79 | =item state($host) 80 | 81 | Get state of host $host. 82 | 83 | =cut 84 | 85 | sub state($$) { 86 | my $self = shift; 87 | my $host = shift; 88 | LOGDIE "Can't get state of invalid host '$host'" if (!defined($self->{$host})); 89 | return $self->{$host}->state; 90 | } 91 | 92 | 93 | =item online_since($host) 94 | 95 | Get time since host $host is online. 96 | 97 | =cut 98 | 99 | sub online_since($$) { 100 | my $self = shift; 101 | my $host = shift; 102 | LOGDIE "Can't get time since invalid host '$host' is online" if (!defined($self->{$host})); 103 | return $self->{$host}->online_since; 104 | } 105 | 106 | 107 | =item set_state($host, $state) 108 | 109 | Set state of host $host to $state. 110 | 111 | =cut 112 | 113 | sub set_state($$$) { 114 | my $self = shift; 115 | my $host = shift; 116 | my $state = shift; 117 | 118 | LOGDIE "Can't set state of invalid host '$host'" if (!defined($self->{$host})); 119 | $self->{$host}->state($state); 120 | } 121 | 122 | 123 | =item get_status_info 124 | 125 | Get string containing status information. 126 | 127 | =cut 128 | 129 | sub get_status_info($) { 130 | my $self = shift; 131 | my $detailed= shift || 0; 132 | my $res = ''; 133 | my $agent_res = ''; 134 | 135 | keys (%$self); # reset iterator 136 | foreach my $host (sort(keys(%$self))) { 137 | my $agent = $self->{$host}; 138 | next unless $agent; 139 | $agent_res .= "# Warning: agent on host $host is not reachable\n" if ($agent->agent_down()); 140 | $res .= sprintf(" %s(%s) %s/%s. Roles: %s\n", $host, $agent->ip, $agent->mode, $agent->state, join(', ', sort(@{$agent->roles}))); 141 | } 142 | $res = $agent_res . $res if ($detailed); 143 | return $res; 144 | } 145 | 146 | 147 | =item save_status 148 | 149 | Save status information into status file. 150 | 151 | =cut 152 | 153 | sub save_status($) { 154 | my $self = shift; 155 | 156 | my $filename = $main::config->{monitor}->{status_path}; 157 | 158 | my ($fh, $tempname) = File::Temp::tempfile(basename($filename) . ('X' x 10), UNLINK => 0, DIR => dirname($filename)); 159 | 160 | keys (%$self); # reset iterator 161 | while (my ($host, $agent) = each(%$self)) { 162 | next unless $agent; 163 | printf($fh "%s|%s|%s\n", $host, $agent->state, join(',', sort(@{$agent->roles}))); 164 | } 165 | IO::Handle::flush($fh); 166 | IO::Handle::sync($fh); 167 | close($fh); 168 | rename($tempname, $filename) || LOGDIE "Can't savely overwrite status file '$filename'!"; 169 | return; 170 | } 171 | 172 | 173 | =item load_status 174 | 175 | Load status information from status file 176 | 177 | =cut 178 | 179 | sub load_status($) { 180 | my $self = shift; 181 | 182 | my $filename = $main::config->{monitor}->{status_path}; 183 | 184 | # Open status file 185 | unless (open(STATUS, '<', $filename)) { 186 | FATAL "Couldn't open status file '$filename': Starting up without status information."; 187 | return; 188 | } 189 | 190 | while (my $line = ) { 191 | chomp($line); 192 | my ($host, $state, $roles) = split(/\|/, $line); 193 | unless (defined($self->{$host})) { 194 | WARN "Ignoring saved status information for unknown host '$host'"; 195 | next; 196 | } 197 | 198 | # Parse roles 199 | my @saved_roles_str = sort(split(/\,/, $roles)); 200 | my @saved_roles = (); 201 | foreach my $role_str (@saved_roles_str) { 202 | my $role = MMM::Monitor::Role->from_string($role_str); 203 | push (@saved_roles, $role) if defined($role); 204 | } 205 | 206 | $self->{$host}->state($state); 207 | $self->{$host}->roles(\@saved_roles); 208 | } 209 | close(STATUS); 210 | return; 211 | } 212 | 213 | 1; 214 | -------------------------------------------------------------------------------- /lib/Monitor/CheckResult.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::CheckResult; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use Log::Log4perl qw(:easy); 6 | use threads; 7 | use threads::shared; 8 | 9 | our $VERSION = '0.01'; 10 | 11 | sub new($$$$$) { 12 | my $class = shift; 13 | my $host = shift; 14 | my $check = shift; 15 | my $result = shift; 16 | my $message = shift; 17 | 18 | my %self :shared; 19 | $self{host} = $host; 20 | $self{check} = $check; 21 | $self{result} = $result; 22 | $self{message} = $message; 23 | return bless \%self, $class; 24 | } 25 | 26 | 1; 27 | -------------------------------------------------------------------------------- /lib/Monitor/Checker.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::Checker; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use Log::Log4perl qw(:easy); 6 | use IPC::Open2; 7 | use MMM::Monitor::CheckResult; 8 | 9 | our $VERSION = '0.01'; 10 | 11 | =head1 NAME 12 | 13 | MMM::Monitor::Checker - Checker class and main function for the checker threads 14 | 15 | =head1 SYNOPSIS 16 | 17 | # Spawn a checker thread which will create and poll a checker and push its status to $queue 18 | our $shutdown :shared = 0; 19 | $SIG{INT} = sub { $shutdown = 1; }; 20 | my $queue = checker_queue(new Thread::Queue::) 21 | my $ping_thread = new threads(\&MMM::Monitor::Checker::main, 'ping', $queue) 22 | my $mysql_thread = new threads(\&MMM::Monitor::Checker::main, 'mysql', $queue) 23 | ... 24 | 25 | =cut 26 | 27 | sub main($$) { 28 | my $check_name = shift; 29 | my $queue = shift; 30 | 31 | # Some shortcuts 32 | my @checks = keys(%{$main::config->{check}}); 33 | my @hosts = keys(%{$main::config->{host}}); 34 | my $options = $main::config->{check}->{$check_name}; 35 | 36 | # Create checker 37 | my $checker = new MMM::Monitor::Checker::($check_name); 38 | 39 | # Initialize failure counters 40 | my $failures = {}; 41 | foreach my $host_name (@hosts) { 42 | $failures->{$host_name} = { 43 | state => -1, # -1 undefined; 1 - ok; -2 untrapped error; 0 trapped error 44 | time => 0, 45 | } 46 | } 47 | 48 | # Perform checks until shutdown 49 | while (!$main::shutdown) { 50 | foreach my $host_name (@hosts) { 51 | last if ($main::shutdown); 52 | last unless ($main::have_net); 53 | 54 | # Ping checker 55 | $checker->spawn() unless $checker->ping(); 56 | 57 | # Check service ... 58 | my $res = $checker->check($host_name); 59 | 60 | # If success 61 | if ($res =~ /^OK/) { 62 | next if ($failures->{$host_name}->{state} == 1); 63 | if ($failures->{$host_name}->{state} != -2) { 64 | INFO "Check '$check_name' on '$host_name' is ok!"; 65 | $queue->enqueue(new MMM::Monitor::CheckResult::($host_name, $check_name, 1, $res)); 66 | } 67 | $failures->{$host_name}->{time} = 0; 68 | $failures->{$host_name}->{state} = 1; 69 | next; 70 | } 71 | 72 | # If unknown 73 | if ($res =~ /^UNKNOWN/) { 74 | next if ($failures->{$host_name}->{state} == -3); 75 | $failures->{$host_name}->{time} = time(); 76 | $failures->{$host_name}->{state}= -3; 77 | WARN "Check '$check_name' on '$host_name' is in unknown state! Message: $res"; 78 | next; 79 | } 80 | 81 | # If failed 82 | if ($res =~ /^ERROR/) { 83 | last unless ($main::have_net); 84 | next if ($failures->{$host_name}->{state} == 0); 85 | if ($failures->{$host_name}->{state} != 0 && $failures->{$host_name}->{state} != -2) { 86 | $failures->{$host_name}->{time} = time(); 87 | $failures->{$host_name}->{state}= -2; 88 | } 89 | my $failure_age = time() - $failures->{$host_name}->{time}; 90 | 91 | next if ($failure_age < $options->{trap_period}); 92 | 93 | ERROR "Check '$check_name' on '$host_name' has failed for $failure_age seconds! Message: $res"; 94 | $queue->enqueue(new MMM::Monitor::CheckResult::($host_name, $check_name, 0, $res)); 95 | $failures->{$host_name}->{state} = 0; 96 | next; 97 | } 98 | } 99 | 100 | sleep($options->{check_period}); 101 | } 102 | $checker->shutdown(); 103 | } 104 | 105 | 106 | =pod 107 | 108 | # Create checker - will spawn a checker process 109 | my $checker = new MMM::Monitor::Checker::('ping'); 110 | 111 | =cut 112 | 113 | sub new($$) { 114 | my $class = shift; 115 | my $name = shift; 116 | 117 | my $self = {}; 118 | 119 | $self->{name} = $name; 120 | bless $self, $class; 121 | $self->spawn(); 122 | return $self; 123 | } 124 | 125 | 126 | =pod 127 | 128 | # Respawn checker if it doesn't respond 129 | $checker->spawn() unless $checker->ping(); 130 | 131 | =cut 132 | 133 | sub spawn($) { 134 | my $self = shift; 135 | my $name = $self->{name}; 136 | 137 | my $reader; # STDOUT of checker 138 | my $writer; # STDIN of checker 139 | 140 | INFO "Spawning checker '$name'..."; 141 | 142 | my $cluster = ($main::cluster_name ? '@' . $main::cluster_name : ''); 143 | my $pid = open2($reader, $writer, $main::config->{monitor}->{bin_path} . "/monitor/checker $cluster $name"); 144 | if (!$pid) { 145 | LOGDIE "Can't spawn checker! Error: $!"; 146 | } 147 | 148 | $self->{pid} = $pid; 149 | $self->{reader} = $reader; 150 | $self->{writer} = $writer; 151 | } 152 | 153 | 154 | =pod 155 | 156 | # Shutdown checker process 157 | $checker->shutdown(); 158 | 159 | =cut 160 | 161 | sub shutdown($) { 162 | my $self = shift; 163 | my $name = $self->{name}; 164 | 165 | INFO "Shutting down checker '$name'..."; 166 | 167 | my $reader = $self->{reader}; 168 | my $writer = $self->{writer}; 169 | 170 | my $send_res = print $writer "quit\n"; 171 | my $recv_res = <$reader>; 172 | chomp($recv_res) if defined($recv_res); 173 | } 174 | 175 | 176 | =pod 177 | 178 | # Check if checker process is still alive 179 | $checker->ping(); 180 | 181 | =cut 182 | 183 | sub ping($) { 184 | my $self = shift; 185 | my $name = $self->{name}; 186 | 187 | # DEBUG "Pinging checker '$name'..."; 188 | 189 | my $reader = $self->{reader}; 190 | my $writer = $self->{writer}; 191 | 192 | my $send_res = print $writer "ping\n"; 193 | my $recv_res; 194 | READ: { 195 | $recv_res = <$reader>; 196 | redo READ if !$recv_res && $!{EINTR}; 197 | } 198 | chomp($recv_res) if defined($recv_res); 199 | 200 | if (!$send_res || !$recv_res || !($recv_res =~ /^OK/)) { 201 | WARN "Checker '$name' is dead!"; 202 | return 0; 203 | } 204 | 205 | # DEBUG "Checker '$name' is OK ($recv_res)"; 206 | return 1; 207 | } 208 | 209 | 210 | =pod 211 | 212 | # Tell the checker to check host 'db2' 213 | $checker->check('db2'); 214 | 215 | =cut 216 | 217 | sub check($$) { 218 | my $self = shift; 219 | my $host = shift; 220 | 221 | my $name = $self->{name}; 222 | 223 | my $reader = $self->{reader}; 224 | my $writer = $self->{writer}; 225 | 226 | my $send_res = print $writer "check $host\n"; 227 | my $recv_res; 228 | READ: { 229 | $recv_res = <$reader>; 230 | redo READ if !$recv_res && $!{EINTR}; 231 | } 232 | chomp($recv_res) if defined($recv_res); 233 | 234 | return "UNKNOWN: Checker '$name' is dead!" unless ($send_res && $recv_res); 235 | return $recv_res; 236 | } 237 | 238 | 1; 239 | -------------------------------------------------------------------------------- /lib/Monitor/Checker/Checks.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::Checker::Checks; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw(OSNAME EFFECTIVE_USER_ID); 6 | use DBI; 7 | use Net::Ping; 8 | use POSIX ':signal_h'; 9 | 10 | our $VERSION = '0.01'; 11 | 12 | =head1 NAME 13 | 14 | MMM::Monitor::Checker::Checks - functions for the B helper program B 15 | 16 | =cut 17 | 18 | 19 | my $fping_path; 20 | 21 | =head1 FUNCTIONS 22 | 23 | =over 4 24 | 25 | =item ping($timeout, $host) 26 | 27 | Check if the host $host is reachable. 28 | 29 | =cut 30 | 31 | sub ping($$) { 32 | my $timeout = shift; 33 | my $host = shift; 34 | 35 | my $ip = $main::config->{host}->{$host}->{ip}; 36 | return "ERROR: Invalid host '$host'" unless ($ip); 37 | 38 | # if super user, use Net::Ping - it's faster 39 | if ($EFFECTIVE_USER_ID == 0) { 40 | my $p = Net::Ping->new('icmp'); 41 | $p->hires(); 42 | if ($p->ping($ip, 0.5)) { 43 | return 'OK'; 44 | } 45 | return "ERROR: Could not ping $ip"; 46 | } 47 | 48 | # Find appropriate fping version 49 | _determine_fping_path() unless defined($fping_path); 50 | unless (defined($fping_path)) { 51 | return "ERROR: fping is not functional - please, install your own version of fping on this server!"; 52 | } 53 | 54 | my $res = `$fping_path -q -u -t 500 -C 1 $ip 2>&1`; 55 | return "ERROR: fping could not reach $ip" if ($res =~ /$ip.*\-$/); 56 | return 'OK'; 57 | } 58 | 59 | =item ping_ip($timeout, $ip) 60 | 61 | Check if the IP $ip is reachable. 62 | 63 | =cut 64 | 65 | sub ping_ip($$) { 66 | my $timeout = shift; 67 | my $ip = shift; 68 | 69 | # if super user, use Net::Ping - it's faster 70 | if ($EFFECTIVE_USER_ID == 0) { 71 | my $p = Net::Ping->new('icmp'); 72 | $p->hires(); 73 | if ($p->ping($ip, 0.5)) { 74 | return 'OK'; 75 | } 76 | return "ERROR: Could not ping $ip"; 77 | } 78 | 79 | # Find appropriate fping version 80 | _determine_fping_path() unless defined($fping_path); 81 | unless (defined($fping_path)) { 82 | return "ERROR: fping is not functional - please, install your own version of fping on this server!"; 83 | } 84 | 85 | my $res = `$fping_path -q -u -t 500 -C 1 $ip 2>&1`; 86 | return "ERROR: fping could not reach $ip" if ($res =~ /$ip.*\-$/); 87 | return 'OK'; 88 | } 89 | 90 | 91 | =item mysql($timeout, $host) 92 | 93 | Check if the mysql server on host $host is reachable. 94 | 95 | =cut 96 | 97 | sub mysql($$) { 98 | my $timeout = shift; 99 | my $host = shift; 100 | 101 | my ($peer_host, $peer_port, $peer_user, $peer_password) = _get_connection_info($host); 102 | return "ERROR: Invalid host '$host'" unless ($peer_host); 103 | 104 | my $mask = POSIX::SigSet->new( SIGALRM ); 105 | my $action = POSIX::SigAction->new( 106 | sub { die 'TIMEOUT'; }, 107 | $mask, 108 | ); 109 | my $oldaction = POSIX::SigAction->new(); 110 | sigaction( SIGALRM, $action, $oldaction ); 111 | 112 | my $res = eval { 113 | alarm($timeout + 1); 114 | 115 | # connect to server 116 | my $dsn = "DBI:mysql:host=$peer_host;port=$peer_port;mysql_connect_timeout=$timeout"; 117 | my $dbh = DBI->connect($dsn, $peer_user, $peer_password, { PrintError => 0 }); 118 | 119 | unless ($dbh) { 120 | alarm(0); 121 | # We don't want to trigger any action because of a simple 'too many connections' error 122 | return "UNKNOWN: Too many connections! " . $DBI::errstr if ($DBI::err == 1040); 123 | return "ERROR: Connect error (host = $peer_host:$peer_port, user = $peer_user)! " . $DBI::errstr; 124 | } 125 | 126 | # Check server (simple) 127 | my $res = $dbh->do('SELECT NOW()'); 128 | unless ($res) { 129 | alarm(0); 130 | return 'ERROR: SQL Query Error: ' . $dbh->errstr; 131 | } 132 | 133 | alarm(1); 134 | $dbh->disconnect(); 135 | $dbh = undef; 136 | 137 | alarm(0); 138 | return 0; 139 | }; 140 | alarm(0); 141 | 142 | return $res if ($res); 143 | return 'ERROR: Timeout' if ($@ =~ /^TIMEOUT/); 144 | return "UNKNOWN: Error occurred: $@" if $@; 145 | return 'OK'; 146 | 147 | } 148 | 149 | 150 | =item rep_backlog($timeout, $host) 151 | 152 | Check the replication backlog on host $host. 153 | 154 | =cut 155 | 156 | sub rep_backlog($$) { 157 | my $timeout = shift; 158 | my $host = shift; 159 | 160 | my ($peer_host, $peer_port, $peer_user, $peer_password) = _get_connection_info($host); 161 | return "ERROR: Invalid host '$host'" unless ($peer_host); 162 | 163 | my $mask = POSIX::SigSet->new( SIGALRM ); 164 | my $action = POSIX::SigAction->new( 165 | sub { die 'TIMEOUT'; }, 166 | $mask, 167 | ); 168 | my $oldaction = POSIX::SigAction->new(); 169 | sigaction( SIGALRM, $action, $oldaction ); 170 | 171 | my $res = eval { 172 | alarm($timeout + 1); 173 | 174 | # connect to server 175 | my $dsn = "DBI:mysql:host=$peer_host;port=$peer_port;mysql_connect_timeout=$timeout"; 176 | my $dbh = DBI->connect($dsn, $peer_user, $peer_password, { PrintError => 0 }); 177 | unless ($dbh) { 178 | alarm(0); 179 | return "UNKNOWN: Connect error (host = $peer_host:$peer_port, user = $peer_user)! " . $DBI::errstr; 180 | } 181 | 182 | # Check server (replication backlog) 183 | my $sth = $dbh->prepare('SHOW SLAVE STATUS'); 184 | my $res = $sth->execute; 185 | 186 | if ($dbh->err) { 187 | alarm(1); 188 | my $ret = 'UNKNOWN: Unknown state. Execute error: ' . $dbh->errstr; 189 | $ret = "ERROR: The monitor user '$peer_user' doesn't have the required REPLICATION CLIENT privilege! " . $dbh->errstr if ($dbh->err == 1227); 190 | $sth->finish(); 191 | $dbh->disconnect(); 192 | $dbh = undef; 193 | alarm(0); 194 | return $ret; 195 | } 196 | 197 | unless ($res) { 198 | alarm(1); 199 | $sth->finish(); 200 | $dbh->disconnect(); 201 | $dbh = undef; 202 | alarm(0); 203 | return 'ERROR: Replication is not running'; 204 | } 205 | 206 | my $status = $sth->fetchrow_hashref; 207 | alarm(1); 208 | $sth->finish(); 209 | $dbh->disconnect(); 210 | $dbh = undef; 211 | alarm(0); 212 | 213 | return 'ERROR: Replication is not set up' unless defined($status); 214 | 215 | 216 | # Check backlog size 217 | my $backlog = $status->{Seconds_Behind_Master}; 218 | $backlog = 0 unless ($backlog); 219 | 220 | return 'OK: Backlog is null' if ($backlog == 0); 221 | return 'ERROR: Backlog is too big' if ($backlog > $main::check->{max_backlog}); 222 | return 0; 223 | }; 224 | alarm(0); 225 | 226 | return $res if ($res); 227 | return 'ERROR: Timeout' if ($@ =~ /^TIMEOUT/); 228 | return "UNKNOWN: Error occurred: $@" if $@; 229 | return 'OK'; 230 | } 231 | 232 | 233 | =item rep_threads($timeout, $host) 234 | 235 | Check if the mysql slave threads on host $host are running. 236 | 237 | =cut 238 | 239 | sub rep_threads($$) { 240 | my $timeout = shift; 241 | my $host = shift; 242 | 243 | my ($peer_host, $peer_port, $peer_user, $peer_password) = _get_connection_info($host); 244 | return "ERROR: Invalid host '$host'" unless ($peer_host); 245 | 246 | my $mask = POSIX::SigSet->new( SIGALRM ); 247 | my $action = POSIX::SigAction->new( 248 | sub { die 'TIMEOUT'; }, 249 | $mask, 250 | ); 251 | my $oldaction = POSIX::SigAction->new(); 252 | sigaction( SIGALRM, $action, $oldaction ); 253 | 254 | my $res = eval { 255 | alarm($timeout + 1); 256 | 257 | # connect to server 258 | my $dsn = "DBI:mysql:host=$peer_host;port=$peer_port;mysql_connect_timeout=$timeout"; 259 | my $dbh = DBI->connect($dsn, $peer_user, $peer_password, { PrintError => 0 }); 260 | return "UNKNOWN: Connect error (host = $peer_host:$peer_port, user = $peer_user)! " . $DBI::errstr unless ($dbh); 261 | 262 | # Check server (replication backlog) 263 | my $sth = $dbh->prepare('SHOW SLAVE STATUS'); 264 | my $res = $sth->execute; 265 | 266 | if ($dbh->err) { 267 | alarm(1); 268 | my $ret = 'UNKNOWN: Unknown state. Execute error: ' . $dbh->errstr; 269 | $ret = "ERROR: The monitor user '$peer_user' doesn't have the required REPLICATION CLIENT privilege! " . $dbh->errstr if ($dbh->err == 1227); 270 | $sth->finish(); 271 | $dbh->disconnect(); 272 | $dbh = undef; 273 | alarm(0); 274 | return $ret; 275 | } 276 | 277 | unless ($res) { 278 | alarm(1); 279 | $sth->finish(); 280 | $dbh->disconnect(); 281 | $dbh = undef; 282 | alarm(0); 283 | return 'ERROR: Replication is not running'; 284 | } 285 | 286 | my $status = $sth->fetchrow_hashref; 287 | 288 | alarm(1); 289 | $sth->finish(); 290 | $dbh->disconnect(); 291 | $dbh = undef; 292 | alarm(0); 293 | 294 | return 'ERROR: Replication is not set up' unless defined($status); 295 | 296 | # Check peer replication state 297 | if ($status->{Slave_IO_Running} eq 'No' || $status->{Slave_SQL_Running} eq 'No') { 298 | return 'ERROR: Replication is broken'; 299 | } 300 | return 0; 301 | }; 302 | alarm(0); 303 | 304 | return $res if ($res); 305 | return 'ERROR: Timeout' if ($@ =~ /^TIMEOUT/); 306 | return "UNKNOWN: Error occurred: $@" if $@; 307 | return 'OK'; 308 | } 309 | 310 | 311 | =item _get_connection_info($host) 312 | 313 | Get connection info for host $host. 314 | 315 | =cut 316 | 317 | sub _get_connection_info($) { 318 | my $host = shift; 319 | return ( 320 | $main::config->{host}->{$host}->{ip}, 321 | $main::config->{host}->{$host}->{mysql_port}, 322 | $main::config->{host}->{$host}->{monitor_user}, 323 | $main::config->{host}->{$host}->{monitor_password} 324 | ); 325 | } 326 | 327 | 328 | =item _determine_fping_path() 329 | 330 | Determine path of fping binary 331 | 332 | =cut 333 | 334 | sub _determine_fping_path() { 335 | $fping_path = _determine_path('fping'); 336 | } 337 | 338 | sub _determine_path($) { 339 | my $program = shift; 340 | my @paths = qw(/usr/sbin /sbin /usr/bin /bin); 341 | 342 | foreach my $path (@paths) { 343 | my $fullpath = "$path/$program"; 344 | if (-f $fullpath && -x $fullpath) { 345 | return $fullpath; 346 | } 347 | } 348 | return undef; 349 | } 350 | 351 | 1; 352 | -------------------------------------------------------------------------------- /lib/Monitor/ChecksStatus.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::ChecksStatus; 2 | use base 'Class::Singleton'; 3 | 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use Log::Log4perl qw(:easy); 7 | use MMM::Monitor::Checker 8 | 9 | our $VERSION = '0.01'; 10 | 11 | sub _new_instance($) { 12 | my $class = shift; 13 | 14 | my $data = {}; 15 | 16 | my @checks = keys(%{$main::config->{check}}); 17 | my @hosts = keys(%{$main::config->{host}}); 18 | 19 | foreach my $host_name (@hosts) { 20 | $data->{$host_name} = {}; 21 | } 22 | 23 | my $time = time(); 24 | 25 | # Perform initial checks 26 | INFO 'Performing initial checks...'; 27 | foreach my $check_name (@checks) { 28 | 29 | # Spawn checker 30 | my $checker = new MMM::Monitor::Checker::($check_name); 31 | 32 | # Check all hosts 33 | foreach my $host_name (@hosts) { 34 | DEBUG "Trying initial check '$check_name' on host '$host_name'"; 35 | my $res = $checker->check($host_name); 36 | DEBUG "$check_name($host_name) = '$res'"; 37 | $data->{$host_name}->{$check_name} = {}; 38 | $data->{$host_name}->{$check_name}->{status} = ($res =~ /^OK/)? 1 : 0; 39 | $data->{$host_name}->{$check_name}->{last_change} = $time; 40 | $data->{$host_name}->{$check_name}->{message} = $res; 41 | } 42 | 43 | # Shutdown checker 44 | $checker->shutdown(); 45 | } 46 | return bless $data, $class; 47 | } 48 | 49 | 50 | =item handle_result(MMM::Monitor::CheckResult $result) 51 | 52 | handle the results of a check and change state accordingly 53 | 54 | =cut 55 | 56 | sub handle_result($$) { 57 | my $self = shift; 58 | my $result = shift; 59 | 60 | # always save the latest message, but don't override time of last change 61 | $self->{$result->{host}}->{$result->{check}}->{message} = $result->{message}; 62 | return if ($result->{result} == $self->{$result->{host}}->{$result->{check}}->{status}); 63 | 64 | $self->{$result->{host}}->{$result->{check}}->{status} = $result->{result}; 65 | $self->{$result->{host}}->{$result->{check}}->{last_change} = time(); 66 | } 67 | 68 | =item ping($host) 69 | 70 | Get state of check "ping" on host $host. 71 | 72 | =cut 73 | 74 | sub ping($$) { 75 | my $self = shift; 76 | my $host = shift; 77 | return $self->{$host}->{ping}->{status}; 78 | } 79 | 80 | 81 | =item ping($host) 82 | 83 | Get state of check "mysql" on host $host. 84 | 85 | =cut 86 | 87 | sub mysql($$) { 88 | my $self = shift; 89 | my $host = shift; 90 | return $self->{$host}->{mysql}->{status}; 91 | } 92 | 93 | 94 | =item rep_threads($host) 95 | 96 | Get state of check "rep_threads" on host $host. 97 | 98 | =cut 99 | 100 | sub rep_threads($$) { 101 | my $self = shift; 102 | my $host = shift; 103 | return $self->{$host}->{rep_threads}->{status}; 104 | } 105 | 106 | 107 | =item rep_backlog($host) 108 | 109 | Get state of check "rep_backlog" on host $host. 110 | 111 | =cut 112 | 113 | sub rep_backlog($$) { 114 | my $self = shift; 115 | my $host = shift; 116 | return $self->{$host}->{rep_backlog}->{status}; 117 | } 118 | 119 | 120 | =item last_change($host, [$check]) 121 | 122 | Get time of last state change 123 | 124 | =cut 125 | 126 | sub last_change { 127 | my $self = shift; 128 | my $host = shift; 129 | my $check = shift || undef; 130 | 131 | return $self->{$host}->{$check}->{last_change} if (defined($check)); 132 | 133 | my $time = $self->{$host}->{ping}->{last_change}; 134 | $time = $self->{$host}->{mysql}->{last_change} if ($self->{$host}->{mysql}->{last_change} > $time); 135 | $time = $self->{$host}->{rep_threads}->{last_change} if ($self->{$host}->{rep_threads}->{last_change} > $time); 136 | $time = $self->{$host}->{rep_backlog}->{last_change} if ($self->{$host}->{rep_backlog}->{last_change} > $time); 137 | return $time; 138 | } 139 | 140 | 141 | =item message($host, $check) 142 | 143 | Get time of last state change 144 | 145 | =cut 146 | 147 | sub message($$$) { 148 | my $self = shift; 149 | my $host = shift; 150 | my $check = shift; 151 | return $self->{$host}->{$check}->{message}; 152 | } 153 | 154 | 1; 155 | -------------------------------------------------------------------------------- /lib/Monitor/Commands.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::Commands; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use Log::Log4perl qw(:easy); 6 | use List::Util qw(max); 7 | use threads; 8 | use threads::shared; 9 | use Log::Log4perl::DateFormat; 10 | use MMM::Common::Socket; 11 | use MMM::Monitor::Agents; 12 | use MMM::Monitor::ChecksStatus; 13 | use MMM::Monitor::Monitor; 14 | use MMM::Monitor::Roles; 15 | 16 | our $VERSION = '0.01'; 17 | 18 | 19 | sub main($$) { 20 | my $queue_in = shift; 21 | my $queue_out = shift; 22 | 23 | my $socket = MMM::Common::Socket::create_listener($main::config->{monitor}->{ip}, $main::config->{monitor}->{port}); 24 | 25 | while (!$main::shutdown) { 26 | DEBUG 'Listener: Waiting for connection...'; 27 | my $client = $socket->accept(); 28 | next unless ($client); 29 | 30 | DEBUG 'Listener: Connect!'; 31 | while (my $cmd = <$client>) { 32 | chomp($cmd); 33 | last if ($cmd eq 'quit'); 34 | 35 | $queue_out->enqueue($cmd); 36 | my $res; 37 | until ($res) { 38 | lock($queue_in); 39 | cond_timedwait($queue_in, time() + 1); 40 | $res = $queue_in->dequeue_nb(); 41 | return 0 if ($main::shutdown); 42 | } 43 | print $client $res; 44 | return 0 if ($main::shutdown); 45 | } 46 | 47 | close($client); 48 | DEBUG 'Listener: Disconnect!'; 49 | 50 | } 51 | } 52 | 53 | sub ping() { 54 | return 'OK: Pinged successfully!'; 55 | } 56 | 57 | 58 | sub show() { 59 | my $agents = MMM::Monitor::Agents->instance(); 60 | my $monitor = MMM::Monitor::Monitor->instance(); 61 | my $roles = MMM::Monitor::Roles->instance(); 62 | 63 | my $ret = ''; 64 | if ($monitor->is_passive) { 65 | $ret .= "--- Monitor is in PASSIVE MODE ---\n"; 66 | $ret .= sprintf("Cause: %s\n", $monitor->passive_info); 67 | $ret =~ s/^/# /mg; 68 | } 69 | $ret .= $agents->get_status_info(1); 70 | $ret .= $roles->get_preference_info(); 71 | return $ret; 72 | } 73 | 74 | sub checks { 75 | my $host = shift || 'all'; 76 | my $check = shift || 'all'; 77 | 78 | my $checks = MMM::Monitor::ChecksStatus->instance(); 79 | my $ret = ''; 80 | 81 | my $dateformat = Log::Log4perl::DateFormat->new('yyyy/MM/dd HH:mm:ss'); 82 | 83 | my @valid_checks = qw(ping mysql rep_threads rep_backlog); 84 | return "ERROR: Unknown check '$check'!" unless ($check eq 'all' || grep(/^$check$/, @valid_checks)); 85 | 86 | if ($host ne 'all') { 87 | return "ERROR: Unknown host name '$host'!" unless (defined($main::config->{host}->{$host})); 88 | if ($check ne 'all') { 89 | return sprintf("%s %s [last change: %s] %s", 90 | $host, 91 | $check, 92 | $dateformat->format($checks->last_change($host, $check)), 93 | $checks->message($host, $check) 94 | ); 95 | } 96 | foreach $check (@valid_checks) { 97 | $ret .= sprintf("%s %-11s [last change: %s] %s\n", 98 | $host, 99 | $check, 100 | $dateformat->format($checks->last_change($host, $check)), 101 | $checks->message($host, $check) 102 | ); 103 | } 104 | return $ret; 105 | } 106 | 107 | my $len = 0; 108 | foreach my $host (keys(%{$main::config->{host}})) { $len = max($len, length $host) } 109 | 110 | if ($check ne 'all') { 111 | foreach my $host (keys(%{$main::config->{host}})) { 112 | $ret .= sprintf("%*s %s [last change: %s] %s\n", 113 | $len * -1, 114 | $host, 115 | $check, 116 | $dateformat->format($checks->last_change($host, $check)), 117 | $checks->message($host, $check) 118 | ); 119 | } 120 | return $ret; 121 | } 122 | foreach my $host (keys(%{$main::config->{host}})) { 123 | foreach $check (@valid_checks) { 124 | $ret .= sprintf("%*s %-11s [last change: %s] %s\n", 125 | $len * -1, 126 | $host, 127 | $check, 128 | $dateformat->format($checks->last_change($host, $check)), 129 | $checks->message($host, $check) 130 | ); 131 | } 132 | } 133 | return $ret; 134 | } 135 | 136 | sub set_online($) { 137 | my $host = shift; 138 | 139 | my $agents = MMM::Monitor::Agents->instance(); 140 | 141 | return "ERROR: Unknown host name '$host'!" unless (defined($main::config->{host}->{$host})); 142 | 143 | my $host_state = $agents->state($host); 144 | return "OK: This host is already ONLINE. Skipping command." if ($host_state eq 'ONLINE'); 145 | 146 | unless ($host_state eq 'ADMIN_OFFLINE' || $host_state eq 'AWAITING_RECOVERY') { 147 | return "ERROR: Host '$host' is '$host_state' at the moment. It can't be switched to ONLINE."; 148 | } 149 | 150 | my $checks = MMM::Monitor::ChecksStatus->instance(); 151 | 152 | if ((!$checks->ping($host) || !$checks->mysql($host))) { 153 | return "ERROR: Checks ping and/or mysql are not ok for host '$host'. It can't be switched to ONLINE."; 154 | } 155 | 156 | # Check peer replication state 157 | if ($main::config->{host}->{$host}->{peer}) { 158 | my $peer = $main::config->{host}->{$host}->{peer}; 159 | if ($agents->state($peer) eq 'ONLINE' && (!$checks->rep_threads($peer) || !$checks->rep_backlog($peer))) { 160 | return "ERROR: Some replication checks failed on peer '$peer'. We can't set '$host' online now. Please, wait some time."; 161 | } 162 | } 163 | 164 | my $agent = MMM::Monitor::Agents->instance()->get($host); 165 | if (!$agent->cmd_ping()) { 166 | return "ERROR: Can't reach agent daemon on '$host'! Can't switch its state!"; 167 | } 168 | 169 | FATAL "Admin changed state of '$host' from $host_state to ONLINE"; 170 | $agents->set_state($host, 'ONLINE'); 171 | $agent->flapping(0); 172 | MMM::Monitor::Monitor->instance()->send_agent_status($host); 173 | 174 | return "OK: State of '$host' changed to ONLINE. Now you can wait some time and check its new roles!"; 175 | } 176 | 177 | sub set_offline($) { 178 | my $host = shift; 179 | 180 | my $agents = MMM::Monitor::Agents->instance(); 181 | 182 | return "ERROR: Unknown host name '$host'!" unless (defined($main::config->{host}->{$host})); 183 | 184 | my $host_state = $agents->state($host); 185 | return "OK: This host is already ADMIN_OFFLINE. Skipping command." if ($host_state eq 'ADMIN_OFFLINE'); 186 | 187 | unless ($host_state eq 'ONLINE' || $host_state eq 'REPLICATION_FAIL' || $host_state eq 'REPLICATION_DELAY') { 188 | return "ERROR: Host '$host' is '$host_state' at the moment. It can't be switched to ADMIN_OFFLINE."; 189 | } 190 | 191 | my $agent = MMM::Monitor::Agents->instance()->get($host); 192 | return "ERROR: Can't reach agent daemon on '$host'! Can't switch its state!" unless ($agent->cmd_ping()); 193 | 194 | FATAL "Admin changed state of '$host' from $host_state to ADMIN_OFFLINE"; 195 | $agents->set_state($host, 'ADMIN_OFFLINE'); 196 | MMM::Monitor::Roles->instance()->clear_roles($host); 197 | MMM::Monitor::Monitor->instance()->send_agent_status($host); 198 | 199 | return "OK: State of '$host' changed to ADMIN_OFFLINE. Now you can wait some time and check all roles!"; 200 | } 201 | 202 | sub set_ip($$) { 203 | my $ip = shift; 204 | my $host = shift; 205 | 206 | return "ERROR: This command is only allowed in passive mode" unless (MMM::Monitor::Monitor->instance()->is_passive); 207 | 208 | my $agents = MMM::Monitor::Agents->instance(); 209 | my $roles = MMM::Monitor::Roles->instance(); 210 | 211 | my $role = $roles->find_by_ip($ip); 212 | 213 | return "ERROR: Unknown ip '$ip'!" unless (defined($role)); 214 | return "ERROR: Unknown host name '$host'!" unless ($agents->exists($host)); 215 | 216 | unless ($roles->can_handle($role, $host)) { 217 | return "ERROR: Host '$host' can't handle role '$role'. Following hosts could: " . join(', ', @{ $roles->get_valid_hosts($role) }); 218 | } 219 | 220 | my $host_state = $agents->state($host); 221 | unless ($host_state eq 'ONLINE') { 222 | return "ERROR: Host '$host' is '$host_state' at the moment. Can't move role with ip '$ip' there."; 223 | } 224 | 225 | FATAL "Admin set role '$role($ip)' to host '$host'"; 226 | 227 | $roles->set_role($role, $ip, $host); 228 | 229 | # Determine all roles and propagate them to agent objects. 230 | foreach my $one_host (@{ $roles->get_valid_hosts($role) }) { 231 | my $agent = $agents->get($one_host); 232 | my @agent_roles = sort($roles->get_host_roles($one_host)); 233 | $agent->roles(\@agent_roles); 234 | } 235 | return "OK: Set role '$role($ip)' to host '$host'."; 236 | } 237 | 238 | sub move_role($$) { 239 | my $role = shift; 240 | my $host = shift; 241 | 242 | my $monitor = MMM::Monitor::Monitor->instance(); 243 | return "ERROR: This command is not allowed in passive mode" if ($monitor->is_passive); 244 | 245 | my $agents = MMM::Monitor::Agents->instance(); 246 | my $roles = MMM::Monitor::Roles->instance(); 247 | 248 | return "ERROR: Unknown role name '$role'!" unless ($roles->exists($role)); 249 | return "ERROR: Unknown host name '$host'!" unless ($agents->exists($host)); 250 | 251 | unless ($roles->is_exclusive($role)) { 252 | $roles->clear_balanced_role($host, $role); 253 | return "OK: Balanced role $role has been removed from host '$host'. Now you can wait some time and check new roles info!"; 254 | } 255 | 256 | my $host_state = $agents->state($host); 257 | return "ERROR: Can't move role to host with state $host_state." unless ($host_state eq 'ONLINE'); 258 | 259 | unless ($roles->can_handle($role, $host)) { 260 | return "ERROR: Host '$host' can't handle role '$role'. Only following hosts could: " . join(', ', @{ $roles->get_valid_hosts($role) }); 261 | } 262 | 263 | my $old_owner = $roles->get_exclusive_role_owner($role); 264 | return "OK: Role is on '$host' already. Skipping command." if ($old_owner eq $host); 265 | 266 | my $agent = MMM::Monitor::Agents->instance()->get($host); 267 | return "ERROR: Can't reach agent daemon on '$host'! Can't move roles there!" unless ($agent->cmd_ping()); 268 | 269 | if ($monitor->is_active && $roles->assigned_to_preferred_host($role)) { 270 | return "ERROR: Role '$role' is assigned to preferred host '$old_owner'. Can't move it!"; 271 | } 272 | 273 | my $ip = $roles->get_exclusive_role_ip($role); 274 | return "Error: Role $role has no IP." unless ($ip); 275 | 276 | FATAL "Admin moved role '$role' from '$old_owner' to '$host'"; 277 | 278 | # Assign role to new host 279 | $roles->set_role($role, $ip, $host); 280 | 281 | # Notify old host (if is_active_master_role($role) this will make the host non writable) 282 | $monitor->send_agent_status($old_owner); 283 | 284 | # Notify slaves (this will make them switch the master) 285 | $monitor->notify_slaves($host) if ($roles->is_active_master_role($role)); 286 | 287 | # Notify new host (if is_active_master_role($role) this will make the host writable) 288 | $monitor->send_agent_status($host); 289 | 290 | return "OK: Role '$role' has been moved from '$old_owner' to '$host'. Now you can wait some time and check new roles info!"; 291 | 292 | } 293 | 294 | sub forced_move_role($$) { 295 | my $role = shift; 296 | my $host = shift; 297 | 298 | my $monitor = MMM::Monitor::Monitor->instance(); 299 | return "ERROR: This command is not allowed in passive mode" if (MMM::Monitor::Monitor->instance()->is_passive); 300 | 301 | my $agents = MMM::Monitor::Agents->instance(); 302 | my $roles = MMM::Monitor::Roles->instance(); 303 | my $checks = MMM::Monitor::ChecksStatus->instance(); 304 | 305 | 306 | return "ERROR: Unknown role name '$role'!" unless ($roles->exists($role)); 307 | return "ERROR: Unknown host name '$host'!" unless ($agents->exists($host)); 308 | return "ERROR: move_role --forced may be used for the active master role only!" unless ($roles->is_active_master_role($role)); 309 | 310 | my $host_state = $agents->state($host); 311 | unless ($host_state eq 'REPLICATION_FAIL' || $host_state eq 'REPLICATION_DELAY') { 312 | return "ERROR: Can't force move of role to host with state $host_state."; 313 | } 314 | 315 | unless ($roles->can_handle($role, $host)) { 316 | return "ERROR: Host '$host' can't handle role '$role'. Only following hosts could: " . join(', ', @{ $roles->get_valid_hosts($role) }); 317 | } 318 | 319 | my $old_owner = $roles->get_exclusive_role_owner($role); 320 | return "OK: Role is on '$host' already. Skipping command." if ($old_owner eq $host); 321 | 322 | my $agent = $agents->get($host); 323 | my $old_agent = $agents->get($old_owner); 324 | return "ERROR: Can't reach agent daemon on '$host'! Can't move roles there!" unless ($agent->cmd_ping()); 325 | 326 | my $ip = $roles->get_exclusive_role_ip($role); 327 | return "Error: Role $role has no IP." unless ($ip); 328 | 329 | FATAL "Admin forced move of role '$role' from '$old_owner' to '$host'"; 330 | 331 | # Assign role to new host 332 | $roles->set_role($role, $ip, $host); 333 | FATAL "State of host '$host' changed from $host_state to ONLINE (because of move_role --force)"; 334 | $agent->state('ONLINE'); 335 | 336 | if (!$checks->rep_threads($old_owner)) { 337 | FATAL "State of host '$old_owner' changed from ONLINE to REPLICATION_FAIL (because of move_role --force)"; 338 | $old_agent->state('REPLICATION_FAIL'); 339 | $roles->clear_roles($old_owner) if ($monitor->is_active); 340 | } 341 | elsif (!$checks->rep_backlog($old_owner)) { 342 | FATAL "State of host '$old_owner' changed from ONLINE to REPLICATION_DELAY (because of move_role --force)"; 343 | $old_agent->state('REPLICATION_DELAY'); 344 | $roles->clear_roles($old_owner) if ($monitor->is_active); 345 | } 346 | 347 | # Notify old host (this will make the host non writable) 348 | MMM::Monitor::Monitor->instance()->send_agent_status($old_owner); 349 | 350 | # Notify slaves (this will make them switch the master) 351 | MMM::Monitor::Monitor->instance()->notify_slaves($host); 352 | 353 | # Notify new host (this will make the host writable) 354 | MMM::Monitor::Monitor->instance()->send_agent_status($host); 355 | 356 | return "OK: Role '$role' has been moved from '$old_owner' to '$host' enforcedly. Now you can wait some time and check new roles info!"; 357 | 358 | } 359 | 360 | 361 | =item mode 362 | 363 | Get information about current mode (active, manual or passive) 364 | 365 | =cut 366 | 367 | sub mode() { 368 | my $monitor = MMM::Monitor::Monitor->instance(); 369 | return $monitor->get_mode_string(); 370 | } 371 | 372 | 373 | =item set_active 374 | 375 | Switch to active mode. 376 | 377 | =cut 378 | 379 | sub set_active() { 380 | my $monitor = MMM::Monitor::Monitor->instance(); 381 | 382 | return 'OK: Already in active mode.' if ($monitor->is_active); 383 | 384 | my $old_mode = $monitor->get_mode_string(); 385 | INFO "Admin changed mode from '$old_mode' to 'ACTIVE'"; 386 | 387 | if ($monitor->is_passive) { 388 | $monitor->set_active(); # so that we can send status to agents 389 | $monitor->cleanup_and_send_status(); 390 | $monitor->passive_info(''); 391 | } 392 | elsif ($monitor->is_manual) { 393 | # remove all roles from hosts which are not ONLINE 394 | my $roles = MMM::Monitor::Roles->instance(); 395 | my $agents = MMM::Monitor::Agents->instance(); 396 | my $checks = MMM::Monitor::ChecksStatus->instance(); 397 | foreach my $host (keys(%{$main::config->{host}})) { 398 | my $host_state = $agents->state($host); 399 | next if ($host_state eq 'ONLINE' || $roles->get_host_roles($host) == 0); 400 | my $agent = $agents->get($host); 401 | $roles->clear_roles($host); 402 | my $ret = $monitor->send_agent_status($host); 403 | # next if ($host_state eq 'REPLICATION_FAIL'); 404 | # next if ($host_state eq 'REPLICATION_DELAY'); 405 | # NOTE host_state should never be ADMIN_OFFLINE at this point 406 | if (!$ret) { 407 | ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host); 408 | $monitor->_kill_host($host, $checks->ping($host)); 409 | } 410 | } 411 | } 412 | 413 | $monitor->set_active(); 414 | return 'OK: Switched into active mode.'; 415 | } 416 | 417 | 418 | =item set_manual 419 | 420 | Switch to manual mode. 421 | 422 | =cut 423 | 424 | sub set_manual() { 425 | my $monitor = MMM::Monitor::Monitor->instance(); 426 | 427 | return 'OK: Already in manual mode.' if ($monitor->is_manual); 428 | 429 | my $old_mode = $monitor->get_mode_string(); 430 | INFO "Admin changed mode from '$old_mode' to 'MANUAL'"; 431 | 432 | if ($monitor->is_passive) { 433 | $monitor->set_manual(); # so that we can send status to agents 434 | $monitor->cleanup_and_send_status(); 435 | $monitor->passive_info(''); 436 | } 437 | 438 | $monitor->set_manual(); 439 | return 'OK: Switched into manual mode.'; 440 | } 441 | 442 | 443 | =item set_passive 444 | 445 | Switch to passive mode. 446 | 447 | =cut 448 | 449 | sub set_passive() { 450 | my $monitor = MMM::Monitor::Monitor->instance(); 451 | 452 | return 'OK: Already in passive mode.' if ($monitor->is_passive); 453 | 454 | my $old_mode = $monitor->get_mode_string(); 455 | INFO "Admin changed mode from '$old_mode' to 'PASSIVE'"; 456 | 457 | $monitor->set_passive(); 458 | $monitor->passive_info('Admin switched to passive mode.'); 459 | return 'OK: Switched into passive mode.'; 460 | } 461 | 462 | sub help() { 463 | return: "Valid commands are: 464 | help - show this message 465 | ping - ping monitor 466 | show - show status 467 | checks [|all [|all]] - show checks status 468 | set_online - set host online 469 | set_offline - set host offline 470 | mode - print current mode. 471 | set_active - switch into active mode. 472 | set_manual - switch into manual mode. 473 | set_passive - switch into passive mode. 474 | move_role [--force] - move exclusive role to host 475 | (Only use --force if you know what you are doing!) 476 | set_ip - set role with ip to host 477 | "; 478 | } 479 | 480 | 1; 481 | -------------------------------------------------------------------------------- /lib/Monitor/NetworkChecker.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::NetworkChecker; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use Log::Log4perl qw(:easy); 6 | use MMM::Monitor::Checker; 7 | 8 | our $VERSION = '0.01'; 9 | 10 | =head1 NAME 11 | 12 | MMM::Monitor::NetworkChecker - Function for checking state of network 13 | 14 | =head1 SYNOPSIS 15 | 16 | our $shutdown :shared = 0; 17 | our $have_net :shared = 0; 18 | $SIG{INT} = sub { $shutdown = 1; }; 19 | my $thread = new threads(\&MMM::Monitor::NetworkChecker::main) 20 | 21 | =cut 22 | sub main() { 23 | my @ips = @{$main::config->{monitor}->{ping_ips}}; 24 | 25 | # Create checker 26 | my $checker = new MMM::Monitor::Checker::('ping_ip'); 27 | 28 | # Perform checks until shutdown 29 | while (!$main::shutdown) { 30 | my $state = 0; 31 | 32 | foreach my $ip (@ips) { 33 | last if ($main::shutdown); 34 | 35 | # Ping checker 36 | $checker->spawn() unless $checker->ping(); 37 | 38 | my $res = $checker->check($ip); 39 | if ($res =~ /^OK/) { 40 | $state = 1; 41 | last; 42 | } 43 | } 44 | 45 | if ($main::have_net != $state) { 46 | FATAL "Network is reachable" if ($state); 47 | FATAL "Network is unreachable" unless ($state); 48 | $main::have_net = $state; 49 | } 50 | 51 | # Sleep a while before checking every ip again 52 | sleep($main::config->{monitor}->{ping_interval}); 53 | } 54 | $checker->shutdown(); 55 | } 56 | 57 | sub wait_for_network() { 58 | my @ips = @{$main::config->{monitor}->{ping_ips}}; 59 | 60 | # Create checker 61 | my $checker = new MMM::Monitor::Checker::('ping_ip'); 62 | 63 | while (!$main::shutdown) { 64 | # Ping all ips 65 | foreach my $ip (@ips) { 66 | last if ($main::shutdown); 67 | # Ping checker 68 | $checker->spawn() unless $checker->ping(); 69 | 70 | my $res = $checker->check($ip); 71 | if ($res =~ /^OK/) { 72 | DEBUG "IP '$ip' is reachable: $res"; 73 | $checker->shutdown(); 74 | return 1; 75 | } 76 | } 77 | 78 | # Sleep a while before checking every ip again 79 | sleep($main::config->{monitor}->{ping_interval}); 80 | } 81 | $checker->shutdown(); 82 | return 0; 83 | } 84 | 85 | 1; 86 | -------------------------------------------------------------------------------- /lib/Monitor/Role.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::Role; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use MMM::Common::Role; 6 | 7 | our $VERSION = '0.01'; 8 | our @ISA = qw(MMM::Common::Role); 9 | 10 | 11 | 1; 12 | -------------------------------------------------------------------------------- /lib/Monitor/Roles.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::Roles; 2 | use base 'Class::Singleton'; 3 | 4 | use strict; 5 | use warnings FATAL => 'all'; 6 | use Log::Log4perl qw(:easy); 7 | use MMM::Monitor::Agents; 8 | use MMM::Monitor::Role; 9 | 10 | our $VERSION = '0.01'; 11 | 12 | =head1 NAME 13 | 14 | MMM::Monitor::Roles - holds information for all roles 15 | 16 | =cut 17 | 18 | sub _new_instance($) { 19 | my $class = shift; 20 | 21 | my $self = {}; 22 | 23 | # create list of roles - each role will be orphaned by default 24 | foreach my $role (keys(%{$main::config->{role}})) { 25 | my $role_info = $main::config->{role}->{$role}; 26 | my $ips = {}; 27 | foreach my $ip (@{$role_info->{ips}}) { 28 | $ips->{$ip} = { 'assigned_to' => '' } 29 | } 30 | $self->{$role} = { 31 | mode => $role_info->{mode}, 32 | hosts => $role_info->{hosts}, 33 | ips => $ips 34 | }; 35 | if ($role_info->{mode} eq 'exclusive' && $role_info->{prefer}) { 36 | $self->{$role}->{prefer} = $role_info->{prefer}; 37 | } 38 | } 39 | 40 | return bless $self, $class; 41 | } 42 | 43 | 44 | =head1 FUNCTIONS 45 | 46 | =over 4 47 | 48 | =item assign($role, $host) 49 | 50 | Assign role $role to host $host 51 | 52 | =cut 53 | 54 | sub assign($$$) { 55 | my $self = shift; 56 | my $role = shift; 57 | my $host = shift; 58 | 59 | LOGDIE "Can't assign role '$role' - no host given" unless (defined($host)); 60 | 61 | # Check if the ip is still configured for this role 62 | unless (defined($self->{$role->name}->{ips}->{$role->ip})) { 63 | WARN sprintf("Detected configuration change: ip '%s' was removed from role '%s'", $role->ip, $role->name); 64 | return; 65 | } 66 | INFO sprintf("Adding role '%s' with ip '%s' to host '%s'", $role->name, $role->ip, $host); 67 | 68 | $self->{$role->name}->{ips}->{$role->ip}->{assigned_to} = $host; 69 | } 70 | 71 | =item get_role_hosts($role) 72 | 73 | Get all hosts which may handle role $role 74 | 75 | =cut 76 | 77 | sub get_role_hosts($$) { 78 | my $self = shift; 79 | my $role = shift; 80 | 81 | return () unless (defined($role)); 82 | 83 | my $role_info = $self->{$role}; 84 | return () unless $role_info; 85 | 86 | return @{$role_info->{hosts}}; 87 | } 88 | 89 | 90 | =item get_host_roles($host) 91 | 92 | Get all roles assigned to host $host 93 | 94 | =cut 95 | 96 | sub get_host_roles($$) { 97 | my $self = shift; 98 | my $host = shift; 99 | 100 | return () unless (defined($host)); 101 | 102 | my @roles = (); 103 | foreach my $role (keys(%$self)) { 104 | my $role_info = $self->{$role}; 105 | foreach my $ip (keys(%{$role_info->{ips}})) { 106 | my $ip_info = $role_info->{ips}->{$ip}; 107 | next unless ($ip_info->{assigned_to} eq $host); 108 | push(@roles, new MMM::Monitor::Role::(name => $role, ip => $ip)); 109 | } 110 | } 111 | return @roles; 112 | } 113 | 114 | 115 | =item host_has_roles($host) 116 | 117 | Check whether there are roles assigned to host $host 118 | 119 | =cut 120 | 121 | sub host_has_roles($$) { 122 | my $self = shift; 123 | my $host = shift; 124 | 125 | return 0 unless (defined($host)); 126 | 127 | foreach my $role (keys(%$self)) { 128 | my $role_info = $self->{$role}; 129 | foreach my $ip (keys(%{$role_info->{ips}})) { 130 | my $ip_info = $role_info->{ips}->{$ip}; 131 | return 1 if ($ip_info->{assigned_to} eq $host); 132 | } 133 | } 134 | return 0; 135 | } 136 | 137 | 138 | =item count_host_roles($host) 139 | 140 | Count all roles assigned to host $host 141 | 142 | =cut 143 | 144 | sub count_host_roles($$) { 145 | my $self = shift; 146 | my $host = shift; 147 | 148 | return 0 unless (defined($host)); 149 | 150 | my $cnt = 0; 151 | foreach my $role (keys(%$self)) { 152 | my $role_info = $self->{$role}; 153 | foreach my $ip (keys(%{$role_info->{ips}})) { 154 | my $ip_info = $role_info->{ips}->{$ip}; 155 | next if ($ip_info->{assigned_to} ne $host); 156 | $cnt++; 157 | $cnt -= 0.5 if ($role eq $main::config->{active_master_role}); 158 | } 159 | } 160 | return $cnt; 161 | } 162 | 163 | 164 | =item get_active_master 165 | 166 | Get the host with the active master-role 167 | 168 | =cut 169 | 170 | sub get_active_master($) { 171 | my $self = shift; 172 | 173 | my $role = $self->{$main::config->{active_master_role}}; 174 | return '' unless $role; 175 | 176 | my @ips = keys( %{ $role->{ips} } ); 177 | return $role->{ips}->{$ips[0]}->{assigned_to}; 178 | } 179 | 180 | 181 | =item get_passive_master 182 | 183 | Get the passive master 184 | 185 | =cut 186 | 187 | sub get_passive_master($) { 188 | my $self = shift; 189 | 190 | my $role = $self->{$main::config->{active_master_role}}; 191 | my $active_master = $self->get_active_master(); 192 | return '' unless $role; 193 | return '' unless $active_master; 194 | 195 | foreach my $host ( @{ $role->{hosts} } ) { 196 | return $host if ($host ne $active_master); 197 | } 198 | return ''; 199 | } 200 | 201 | 202 | =item get_first_master 203 | 204 | Get the first master 205 | 206 | =cut 207 | 208 | sub get_first_master($) { 209 | my $self = shift; 210 | 211 | my $role = $self->{$main::config->{active_master_role}}; 212 | return '' unless $role; 213 | return '' unless $role->{hosts}[0]; 214 | return $role->{hosts}[0]; 215 | } 216 | 217 | 218 | =item get_second_master 219 | 220 | Get the second master 221 | 222 | =cut 223 | 224 | sub get_second_master($) { 225 | my $self = shift; 226 | 227 | my $role = $self->{$main::config->{active_master_role}}; 228 | return '' unless $role; 229 | return '' unless $role->{hosts}[1]; 230 | return $role->{hosts}[1]; 231 | } 232 | 233 | 234 | =item get_master_hosts 235 | 236 | Get the hosts which can handle the active master-role 237 | 238 | =cut 239 | 240 | sub get_master_hosts($) { 241 | my $self = shift; 242 | 243 | my $role = $self->{$main::config->{active_master_role}}; 244 | return '' unless $role; 245 | return $self->{$role}->{hosts}; 246 | } 247 | 248 | 249 | =item get_exclusive_role_owner($role) 250 | 251 | Get the host which has the exclusive role $role assigned 252 | 253 | =cut 254 | 255 | sub get_exclusive_role_owner($$) { 256 | my $self = shift; 257 | my $role = shift; 258 | 259 | my $role_info = $self->{$role}; 260 | return '' unless $role_info; 261 | 262 | my @ips = keys( %{ $role_info->{ips} } ); 263 | return $role_info->{ips}->{$ips[0]}->{assigned_to}; 264 | } 265 | 266 | 267 | =item get_exclusive_role_ip($role) 268 | 269 | Get the ip of an exclusive role $role 270 | 271 | =cut 272 | 273 | sub get_exclusive_role_ip($$) { 274 | my $self = shift; 275 | my $role = shift; 276 | 277 | my $role_info = $self->{$role}; 278 | return undef unless $role_info; 279 | 280 | my @ips = keys( %{ $role_info->{ips} } ); 281 | return $ips[0]; 282 | } 283 | 284 | 285 | =item assigned_to_preferred_host($role) 286 | 287 | Check if role is assigned to preferred host 288 | 289 | =cut 290 | 291 | sub assigned_to_preferred_host($$) { 292 | my $self = shift; 293 | my $role = shift; 294 | 295 | my $role_info = $self->{$role}; 296 | return undef unless $role_info; 297 | return undef unless ($role_info->{prefer}); 298 | 299 | my @ips = keys( %{ $role_info->{ips} } ); 300 | return ($role_info->{ips}->{$ips[0]}->{assigned_to} eq $role_info->{prefer}); 301 | 302 | } 303 | 304 | 305 | =item clear_roles($host) 306 | 307 | Remove all roles from host $host. 308 | 309 | =cut 310 | 311 | sub clear_roles($$) { 312 | my $self = shift; 313 | my $host = shift; 314 | 315 | INFO "Removing all roles from host '$host':"; 316 | 317 | my $orphaned_master_role = 0; 318 | foreach my $role (keys(%$self)) { 319 | my $role_info = $self->{$role}; 320 | foreach my $ip (keys(%{$role_info->{ips}})) { 321 | my $ip_info = $role_info->{ips}->{$ip}; 322 | next unless ($ip_info->{assigned_to} eq $host); 323 | INFO " Removed role '$role($ip)' from host '$host'"; 324 | $ip_info->{assigned_to} = ''; 325 | $orphaned_master_role = 1 if ($role eq $main::config->{active_master_role}); 326 | } 327 | } 328 | return $orphaned_master_role; 329 | } 330 | 331 | 332 | =item clear_balanced_role($host, $role) 333 | 334 | Remove balanced role $role from host $host. 335 | 336 | =cut 337 | 338 | sub clear_balanced_role($$$) { 339 | my $self = shift; 340 | my $host = shift; 341 | my $role = shift; 342 | 343 | INFO "Removing balanced role $role from host '$host':"; 344 | 345 | my $role_info = $self->{$role}; 346 | return 0 unless $role_info; 347 | my $cnt = 0; 348 | next unless ($role_info->{mode} eq 'balanced'); 349 | foreach my $ip (keys(%{$role_info->{ips}})) { 350 | my $ip_info = $role_info->{ips}->{$ip}; 351 | next unless ($ip_info->{assigned_to} eq $host); 352 | $cnt++; 353 | INFO " Removed role '$role($ip)' from host '$host'"; 354 | $ip_info->{assigned_to} = ''; 355 | } 356 | return $cnt; 357 | } 358 | 359 | 360 | =item find_eligible_host($role) 361 | 362 | find host which can take over the role $role 363 | 364 | =cut 365 | 366 | sub find_eligible_host($$) { 367 | my $self = shift; 368 | my $role = shift; 369 | 370 | my $min_host = ''; 371 | my $min_count = 0; 372 | 373 | my $agents = MMM::Monitor::Agents->instance(); 374 | 375 | # Maybe role has a preferred hosts 376 | if ($self->{$role}->{prefer}) { 377 | my $host = $self->{$role}->{prefer}; 378 | if ($agents->{$host}->state eq 'ONLINE' && !$agents->{$host}->agent_down) { 379 | return $host; 380 | } 381 | } 382 | 383 | # Use host with fewest roles 384 | foreach my $host ( @{ $self->{$role}->{hosts} } ) { 385 | next unless ($agents->{$host}->state eq 'ONLINE'); 386 | next if ($agents->{$host}->agent_down); 387 | my $cnt = $self->count_host_roles($host); 388 | next unless ($cnt < $min_count || $min_host eq ''); 389 | $min_host = $host; 390 | $min_count = $cnt; 391 | } 392 | 393 | return $min_host; 394 | } 395 | 396 | 397 | =item find_eligible_hosts($role) 398 | 399 | find all hosts which can take over the role $role. 400 | 401 | =cut 402 | 403 | sub find_eligible_hosts($$) { 404 | my $self = shift; 405 | my $role = shift; 406 | 407 | my $hosts = {}; 408 | 409 | my $agents = MMM::Monitor::Agents->instance(); 410 | 411 | foreach my $host ( @{ $self->{$role}->{hosts} } ) { 412 | next unless ($agents->{$host}->state eq 'ONLINE'); 413 | next if ($agents->{$host}->agent_down); 414 | my $cnt = $self->count_host_roles($host); 415 | $hosts->{$host} = $cnt; 416 | } 417 | 418 | return $hosts; 419 | } 420 | 421 | 422 | =item process_orphans 423 | 424 | Find orphaned roles and assign them to a host if possible. 425 | 426 | =cut 427 | 428 | sub process_orphans($$) { 429 | my $self = shift; 430 | my $mode = shift; 431 | 432 | foreach my $role (keys(%$self)) { 433 | my $role_info = $self->{$role}; 434 | next if ($mode && $role_info->{mode} ne $mode); 435 | 436 | foreach my $ip (keys(%{$role_info->{ips}})) { 437 | my $ip_info = $role_info->{ips}->{$ip}; 438 | next unless ($ip_info->{assigned_to} eq ''); 439 | 440 | # Find host which can take over the role - skip if none found 441 | my $host = $self->find_eligible_host($role); 442 | last unless ($host); 443 | 444 | # Assign this ip to host 445 | $ip_info->{assigned_to} = $host; 446 | INFO "Orphaned role '$role($ip)' has been assigned to '$host'"; 447 | } 448 | } 449 | } 450 | 451 | 452 | =item obey_preferences 453 | 454 | Obey preferences by moving roles to preferred hosts 455 | 456 | =cut 457 | sub obey_preferences($) { 458 | my $self = shift; 459 | 460 | my $agents = MMM::Monitor::Agents->instance(); 461 | 462 | foreach my $role (keys(%$self)) { 463 | my $role_info = $self->{$role}; 464 | 465 | next unless ($role_info->{prefer}); 466 | 467 | my $host = $role_info->{prefer}; 468 | 469 | next unless ($agents->{$host}->state eq 'ONLINE'); 470 | next if ($agents->{$host}->agent_down); 471 | 472 | my @ips = keys( %{ $role_info->{ips} } ); 473 | my $ip = $ips[0]; 474 | my $ip_info = $role_info->{ips}->{$ip}; 475 | my $old_host = $ip_info->{assigned_to}; 476 | 477 | next if ($old_host eq $host); 478 | 479 | $ip_info->{assigned_to} = $host; 480 | INFO "Moving role '$role($ip)' from host '$old_host' to preferred host '$host'"; 481 | } 482 | } 483 | 484 | 485 | =item get_preference_info 486 | 487 | Get information about roles with preferred hosts 488 | 489 | =cut 490 | sub get_preference_info($) { 491 | my $self = shift; 492 | 493 | my $ret = ''; 494 | 495 | foreach my $role (keys(%$self)) { 496 | my $role_info = $self->{$role}; 497 | next unless ($role_info->{prefer}); 498 | 499 | my $host = $role_info->{prefer}; 500 | my $other_host = $self->get_exclusive_role_owner($role); 501 | if ($host eq $other_host) { 502 | $ret .= "# Role $role is assigned to it's preferred host $host.\n"; 503 | } 504 | elsif($other_host ne '') { 505 | $ret .= "# Role $role has $host configured as it's preferred host but is assigned to $other_host at the moment.\n"; 506 | } 507 | else { 508 | $ret .= "# Role $role has $host configured as it's preferred host.\n"; 509 | } 510 | } 511 | return $ret; 512 | } 513 | 514 | 515 | =item balance 516 | 517 | Balance roles with mode 'balanced' 518 | 519 | =cut 520 | 521 | sub balance($) { 522 | my $self = shift; 523 | 524 | foreach my $role (keys(%$self)) { 525 | my $role_info = $self->{$role}; 526 | 527 | next unless ($role_info->{mode} eq 'balanced'); 528 | 529 | my $hosts = $self->find_eligible_hosts($role); 530 | next if (scalar(keys(%$hosts)) < 2); 531 | 532 | while (1) { 533 | my $max_host = ''; 534 | my $min_host = ''; 535 | foreach my $host (keys(%$hosts)) { 536 | $max_host = $host if ($max_host eq '' || $hosts->{$host} > $hosts->{$max_host}); 537 | $min_host = $host if ($min_host eq '' || $hosts->{$host} < $hosts->{$min_host}); 538 | } 539 | 540 | if ($hosts->{$max_host} - $hosts->{$min_host} <= 1) { 541 | last; 542 | } 543 | 544 | $self->move_one_ip($role, $max_host, $min_host); 545 | $hosts->{$max_host}--; 546 | $hosts->{$min_host}++; 547 | } 548 | } 549 | } 550 | 551 | 552 | =item move_one_ip($role, $host1, $host2) 553 | 554 | Move one IP of role $role from $host1 to $host2. 555 | 556 | =cut 557 | 558 | sub move_one_ip($$$$) { 559 | my $self = shift; 560 | my $role = shift; 561 | my $host1 = shift; 562 | my $host2 = shift; 563 | 564 | foreach my $ip (keys(%{$self->{$role}->{ips}})) { 565 | my $ip_info = $self->{$role}->{ips}->{$ip}; 566 | next unless ($ip_info->{assigned_to} eq $host1); 567 | 568 | INFO "Moving role '$role($ip)' from host '$host1' to host '$host2'"; 569 | $ip_info->{assigned_to} = $host2; 570 | return 1; 571 | } 572 | 573 | # No ip was moved 574 | return 0; 575 | } 576 | 577 | 578 | =item find_by_ip($ip) 579 | 580 | Find name of role with IP $ip. 581 | 582 | =cut 583 | 584 | sub find_by_ip($$) { 585 | my $self = shift; 586 | my $ip = shift; 587 | 588 | foreach my $role (keys(%$self)) { 589 | return $role if (defined($self->{$role}->{ips}->{$ip})); 590 | } 591 | 592 | return undef; 593 | } 594 | 595 | 596 | =item set_role($role, $ip, $host) 597 | 598 | Set role $role with IP $ip to host $host. 599 | 600 | NOTE: No checks are done. Caller should assure that: 601 | Role is valid, IP is valid, Host is valid, Host can handle role 602 | 603 | =cut 604 | sub set_role($$$$) { 605 | my $self = shift; 606 | my $role = shift; 607 | my $ip = shift; 608 | my $host = shift; 609 | 610 | $self->{$role}->{ips}->{$ip}->{assigned_to} = $host; 611 | } 612 | 613 | 614 | =item exists($role) 615 | 616 | Check if role $role exists. 617 | 618 | =cut 619 | sub exists($$) { 620 | my $self = shift; 621 | my $role = shift; 622 | return defined($self->{$role}); 623 | } 624 | 625 | 626 | =item exists_ip($role, $ip) 627 | 628 | Check if role $role with IP $ip exists. 629 | 630 | =cut 631 | 632 | sub exists_ip($$$) { 633 | my $self = shift; 634 | my $role = shift; 635 | my $ip = shift; 636 | return 0 unless defined($self->{$role}); 637 | return defined($self->{$role}->{ips}->{$ip}); 638 | } 639 | 640 | 641 | =item is_exclusive($role) 642 | 643 | Determine whether given role is an exclusive role. 644 | 645 | =cut 646 | 647 | sub is_exclusive($$) { 648 | my $self = shift; 649 | my $role = shift; 650 | return 0 unless defined($self->{$role}); 651 | return ($self->{$role}->{mode} eq 'exclusive'); 652 | } 653 | 654 | 655 | =item get_valid_hosts($role) 656 | 657 | Get all valid hosts for role $role. 658 | 659 | =cut 660 | 661 | sub get_valid_hosts($$) { 662 | my $self = shift; 663 | my $role = shift; 664 | return () unless defined($self->{$role}); 665 | return $self->{$role}->{hosts}; 666 | } 667 | 668 | 669 | =item can_handle($role, $host) 670 | 671 | Check if host $host can handle role $role. 672 | 673 | =cut 674 | 675 | sub can_handle($$$) { 676 | my $self = shift; 677 | my $role = shift; 678 | my $host = shift; 679 | return 0 unless defined($self->{$role}); 680 | return grep({$_ eq $host} @{$self->{$role}->{hosts}}); 681 | } 682 | 683 | 684 | =item is_master($host) 685 | 686 | Check if host $host can handle role $role. 687 | 688 | =cut 689 | 690 | sub is_master($$) { 691 | my $self = shift; 692 | my $host = shift; 693 | my $role = $self->{$main::config->{active_master_role}}; 694 | return 0 unless defined($role); 695 | return grep({$_ eq $host} @{$role->{hosts}}); 696 | } 697 | 698 | 699 | =item is_active_master_role($role) 700 | 701 | Check whether $role is the active master role. 702 | 703 | =cut 704 | 705 | sub is_active_master_role($$) { 706 | my $self = shift; 707 | my $role = shift; 708 | 709 | return ($role eq $main::config->{active_master_role}); 710 | } 711 | 712 | 1; 713 | -------------------------------------------------------------------------------- /lib/Monitor/StartupStatus.pm: -------------------------------------------------------------------------------- 1 | package MMM::Monitor::StartupStatus; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use List::Util qw(max); 6 | use Log::Log4perl qw(:easy); 7 | use MMM::Common::Role; 8 | use MMM::Monitor::Role; 9 | use MMM::Monitor::Roles; 10 | 11 | our $VERSION = '0.01'; 12 | 13 | =head1 NAME 14 | 15 | MMM::Monitor::StartupStatus - holds information about agent/system/stored status during startup 16 | 17 | =cut 18 | 19 | sub new($) { 20 | my $class = shift; 21 | 22 | my $self = { 23 | roles => {}, 24 | hosts => {}, 25 | result=> {} 26 | }; 27 | return bless $self, $class; 28 | } 29 | 30 | 31 | =head1 FUNCTIONS 32 | 33 | =over 4 34 | 35 | =item set_agent_status($host, $state, $roles, $master) 36 | 37 | Set agent status 38 | 39 | =cut 40 | 41 | sub set_agent_status($$\@$) { 42 | my $self = shift; 43 | my $host = shift; 44 | my $state = shift; 45 | my $roles = shift; 46 | my $master = shift; 47 | 48 | $self->{hosts}->{$host} = {} unless (defined($self->{hosts}->{$host})); 49 | $self->{hosts}->{$host}->{agent} = { 50 | state => $state, 51 | master => $master 52 | }; 53 | foreach my $role (@{$roles}) { 54 | unless (MMM::Monitor::Roles->instance()->exists_ip($role->name, $role->ip)) { 55 | WARN "Detected change in role definitions: Role '$role' was removed."; 56 | next; 57 | } 58 | unless (MMM::Monitor::Roles->instance()->can_handle($role->name, $host)) { 59 | WARN "Detected change in role definitions: Host '$host' can't handle role '$role' anymore."; 60 | next; 61 | } 62 | my $role_str = $role->to_string(); 63 | $self->{roles}->{$role_str} = {} unless (defined($self->{roles}->{$role_str})); 64 | $self->{roles}->{$role_str}->{$host} = {} unless (defined($self->{roles}->{$role_str}->{$host})); 65 | $self->{roles}->{$role_str}->{$host}->{agent} = 1; 66 | } 67 | } 68 | 69 | 70 | =item set_stored_status($host, $state, $roles) 71 | 72 | Set stored status 73 | 74 | =cut 75 | 76 | sub set_stored_status($$\@$) { 77 | my $self = shift; 78 | my $host = shift; 79 | my $state = shift; 80 | my $roles = shift; 81 | 82 | $self->{hosts}->{$host} = {} unless (defined($self->{hosts}->{$host})); 83 | $self->{hosts}->{$host}->{stored} = { 84 | state => $state, 85 | }; 86 | foreach my $role (@{$roles}) { 87 | unless (MMM::Monitor::Roles->instance()->exists_ip($role->name, $role->ip)) { 88 | WARN "Detected change in role definitions: Role '$role' was removed."; 89 | next; 90 | } 91 | unless (MMM::Monitor::Roles->instance()->can_handle($role->name, $host)) { 92 | WARN "Detected change in role definitions: Host '$host' can't handle role '$role' anymore."; 93 | next; 94 | } 95 | my $role_str = $role->to_string(); 96 | $self->{roles}->{$role_str} = {} unless (defined($self->{roles}->{$role_str})); 97 | $self->{roles}->{$role_str}->{$host} = {} unless (defined($self->{roles}->{$role_str}->{$host})); 98 | $self->{roles}->{$role_str}->{$host}->{stored} = 1; 99 | } 100 | } 101 | 102 | 103 | =item set_system_status($host, $writable, $roles, $master) 104 | 105 | Set system status 106 | 107 | =cut 108 | 109 | sub set_system_status($$\@$) { 110 | my $self = shift; 111 | my $host = shift; 112 | my $writable= shift; 113 | my $roles = shift; 114 | my $master = shift; 115 | 116 | $self->{hosts}->{$host} = {} unless (defined($self->{hosts}->{$host})); 117 | $self->{hosts}->{$host}->{system} = { 118 | writable=> $writable, 119 | master => $master 120 | }; 121 | foreach my $role (@{$roles}) { 122 | unless (MMM::Monitor::Roles->instance()->exists_ip($role->name, $role->ip)) { 123 | WARN "Detected change in role definitions: Role '$role' was removed."; 124 | next; 125 | } 126 | unless (MMM::Monitor::Roles->instance()->can_handle($role->name, $host)) { 127 | WARN "Detected change in role definitions: Host '$host' can't handle role '$role' anymore."; 128 | next; 129 | } 130 | my $role_str = $role->to_string(); 131 | $self->{roles}->{$role_str} = {} unless (defined($self->{roles}->{$role_str})); 132 | $self->{roles}->{$role_str}->{$host} = {} unless (defined($self->{roles}->{$role_str}->{$host})); 133 | $self->{roles}->{$role_str}->{$host}->{system} = 1; 134 | } 135 | } 136 | 137 | sub determine_status() { 138 | my $self = shift; 139 | my $roles = MMM::Monitor::Roles->instance(); 140 | 141 | my $is_manual = MMM::Monitor::Monitor->instance()->is_manual(); 142 | 143 | my $conflict = 0; 144 | 145 | foreach my $host (keys(%{$main::config->{host}})) { 146 | 147 | # Figure out host state 148 | 149 | my $stored_state = 'UNKNOWN'; 150 | my $agent_state = 'UNKNOWN'; 151 | my $state; 152 | 153 | $stored_state = $self->{hosts}->{$host}->{stored}->{state} if (defined($self->{hosts}->{$host}->{stored}->{state})); 154 | $agent_state = $self->{hosts}->{$host}->{agent}->{state} if (defined($self->{hosts}->{$host}->{agent}->{state} )); 155 | 156 | if ( $stored_state eq 'ADMIN_OFFLINE' || $agent_state eq 'ADMIN_OFFLINE' ) { $state = 'ADMIN_OFFLINE'; } 157 | elsif ($stored_state eq 'HARD_OFFLINE' || $agent_state eq 'HARD_OFFLINE' ) { $state = 'HARD_OFFLINE'; } 158 | elsif ($stored_state eq 'REPLICATION_FAIL' || $agent_state eq 'REPLICATION_FAIL' ) { $state = 'REPLICATION_FAIL'; } 159 | elsif ($stored_state eq 'REPLICATION_DELAY' || $agent_state eq 'REPLICATION_DELAY') { $state = 'REPLICATION_DELAY'; } 160 | elsif ($stored_state eq 'ONLINE' || $agent_state eq 'ONLINE' ) { $state = 'ONLINE'; } 161 | else { $state = 'AWAITING_RECOVERY'; } 162 | 163 | $self->{result}->{$host} = { state => $state, roles => [] }; 164 | } 165 | 166 | foreach my $role_str (keys(%{$self->{roles}})) { 167 | my $role = MMM::Monitor::Role->from_string($role_str); 168 | next unless(defined($role)); 169 | 170 | if ($roles->is_active_master_role($role->name)) { 171 | # active master role 172 | my $max = 0; 173 | my $target = undef; 174 | my $system_cnt = 0; 175 | foreach my $host (keys(%{$self->{roles}->{$role_str}})) { 176 | my $votes = 0; 177 | my $info = $self->{roles}->{$role_str}->{$host}; 178 | my $host_info = $self->{hosts}->{$host}; 179 | 180 | # host is writable 181 | $votes += 4 if (defined($host_info->{system}->{writable}) && $host_info->{system}->{writable}); 182 | 183 | # IP is configured 184 | if (defined($info->{system})) { 185 | $votes += 2; 186 | $system_cnt++; 187 | } 188 | 189 | $votes += 1 if (defined($info->{stored})); 190 | $votes += 1 if (defined($info->{agent})); 191 | 192 | foreach my $slave_host (keys(%{$self->{hosts}})) { 193 | my $slave_info = $self->{hosts}->{$slave_host}; 194 | next if MMM::Monitor::Roles->instance()->is_master($slave_host); 195 | $votes++ if (defined($slave_info->{system}->{master}) && $slave_info->{system}->{master} eq $host); 196 | } 197 | 198 | 199 | my $state = $self->{result}->{$host}->{state}; 200 | $votes = 0 if ($state eq 'ADMIN_OFFLINE'); 201 | $votes = 0 if ($state eq 'HARD_OFFLINE' && !$is_manual); 202 | 203 | if ($votes > $max) { 204 | $target = $host; 205 | $max = $votes; 206 | } 207 | } 208 | if ($system_cnt > 1) { 209 | WARN "Role '$role_str' was configured on $system_cnt hosts during monitor startup."; 210 | $conflict = 1; 211 | } 212 | if (defined($target)) { 213 | push (@{$self->{result}->{$target}->{roles}}, $role); 214 | my $state = $self->{result}->{$target}->{state}; 215 | $self->{result}->{$target}->{state} = 'ONLINE' if (!$is_manual || $state eq 'REPLICATION_FAIL' || $state eq 'REPLICATION_DELAY'); 216 | } 217 | next; 218 | } 219 | 220 | # Handle non-writer roles 221 | my $max = 0; 222 | my $target = undef; 223 | my $system_cnt = 0; 224 | foreach my $host (keys(%{$self->{roles}->{$role_str}})) { 225 | my $votes = 0; 226 | my $info = $self->{roles}->{$role_str}->{$host}; 227 | 228 | # IP is configured 229 | if (defined($info->{system})) { 230 | $votes += 4; 231 | $system_cnt++; 232 | } 233 | 234 | $votes += 2 if (defined($info->{stored})); 235 | $votes += 1 if (defined($info->{agent})); 236 | 237 | 238 | my $state = $self->{result}->{$host}->{state}; 239 | if ($state eq 'ADMIN_OFFLINE' || (!$is_manual && $state ne 'ONLINE' && $state ne 'AWAITING_RECOVERY')) { 240 | $votes = 0; 241 | } 242 | if ($votes > $max) { 243 | $target = $host; 244 | $max = $votes; 245 | } 246 | } 247 | if ($system_cnt > 1) { 248 | WARN "Role '$role_str' was configured on $system_cnt hosts during monitor startup."; 249 | } 250 | if (defined($target)) { 251 | push (@{$self->{result}->{$target}->{roles}}, $role); 252 | $self->{result}->{$target}->{state} = 'ONLINE' if ($self->{result}->{$target}->{state} eq 'AWAITING_RECOVERY'); 253 | } 254 | } 255 | return $conflict; 256 | } 257 | 258 | 259 | sub to_string($) { 260 | my $self = shift; 261 | my $ret = "Startup status:\n"; 262 | $ret .= "\nRoles:\n"; 263 | 264 | my $role_len = 4; # "Role" 265 | my $host_len = 6; # "Master" 266 | 267 | foreach my $role (keys(%{$main::config->{role}})) { $role_len = max($role_len, length $role) } 268 | foreach my $host (keys(%{$main::config->{host}})) { $host_len = max($host_len, length $host) } 269 | $role_len += 17; # "(999.999.999.999)" 270 | 271 | $ret .= sprintf(" %-*s %-*s %-6s %-6s %-5s\n", $role_len, 'Role', $host_len, 'Host', 'Stored', 'System', 'Agent'); 272 | foreach my $role (keys(%{$self->{roles}})) { 273 | foreach my $host (keys(%{$self->{roles}->{$role}})) { 274 | my $info = $self->{roles}->{$role}->{$host}; 275 | $ret .= sprintf(" %-*s %-*s %-6s %-6s %-5s\n", $role_len, $role, $host_len, $host, 276 | defined($info->{stored}) ? 'Yes' : '-', 277 | defined($info->{system}) ? 'Yes' : '-', 278 | defined($info->{agent}) ? 'Yes' : '-' 279 | ); 280 | } 281 | } 282 | 283 | $ret .= "\nHosts:\n"; 284 | $ret .= sprintf(" %-*s %-*s %-8s %-16s %-16s\n", $host_len, 'Host', $host_len, 'Master', 'Writable', 'Stored state', 'Agent state'); 285 | foreach my $host (keys(%{$self->{hosts}})) { 286 | my $info = $self->{hosts}->{$host}; 287 | my $is_master = MMM::Monitor::Roles->instance()->is_master($host); 288 | $ret .= sprintf(" %-*s %-*s %-8s %-16s %-16s\n", $host_len, $host, $host_len, 289 | $is_master ? '-' : (defined($info->{system}->{master}) ? $info->{system}->{master} : '?'), 290 | defined($info->{system}->{writable}) ? ($info->{system}->{writable} ? 'Yes' : 'No') : '?', 291 | defined($info->{stored}->{state}) ? $info->{stored}->{state} : '?', 292 | defined($info->{agent}->{state}) ? $info->{agent}->{state} : '?', 293 | ); 294 | } 295 | return $ret; 296 | } 297 | 298 | 1; 299 | -------------------------------------------------------------------------------- /lib/Tools/MySQL.pm: -------------------------------------------------------------------------------- 1 | package MMM::Tools::MySQL; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use DBI; 6 | use Log::Log4perl qw(:easy); 7 | 8 | 9 | our $VERSION = '0.01'; 10 | 11 | =head1 NAME 12 | 13 | MMM::Tools::MySQL - MySQL related functions for the mmm-tools. 14 | 15 | =cut 16 | 17 | 18 | =over 4 19 | 20 | =item is_running([$pid]) 21 | 22 | Check if mysqld is running 23 | 24 | =cut 25 | 26 | sub is_running { 27 | 28 | my $pid = shift; 29 | 30 | unless ($pid) { 31 | my $pidfile = $main::config->{host}->{$main::config->{this}}->{mysql_pidfile}; 32 | return 0 unless (-f $pidfile); 33 | open(PID, $pidfile) || LOGDIE "ERROR: Can't read MySQL pid file '$pidfile'"; 34 | chomp($pid = ); 35 | close(PID); 36 | } 37 | 38 | my $cnt = kill(0, $pid); 39 | return $pid if ($cnt); 40 | return 0; 41 | } 42 | 43 | 44 | =item start 45 | 46 | Start mysqld 47 | 48 | =cut 49 | 50 | sub start() { 51 | my $rcscript = $main::config->{host}->{$main::config->{this}}->{mysql_rcscript}; 52 | 53 | my $pid = is_running(); 54 | if ($pid) { 55 | ERROR "Error: Local MySQL server is running with pid $pid"; 56 | return 0; 57 | } 58 | 59 | INFO 'MySQL is not running. Going to start it...'; 60 | 61 | my $res = system($rcscript, 'start'); 62 | if ($res) { 63 | ERROR "ERROR: Can't start local MySQL server!"; 64 | return 0; 65 | } 66 | 67 | INFO 'MySQL has been started!'; 68 | return 1; 69 | } 70 | 71 | =item stop 72 | 73 | Stop mysqld 74 | 75 | =cut 76 | 77 | sub stop() { 78 | my $rcscript = $main::config->{host}->{$main::config->{this}}->{mysql_rcscript}; 79 | 80 | my $pid = is_running(); 81 | unless ($pid) { 82 | WARN 'MySQL is not running now, skipping shutdown ...'; 83 | return 1; 84 | } 85 | 86 | my $res = system($rcscript, 'stop'); 87 | if ($res) { 88 | ERROR "ERROR: Can't stop local MySQL server!"; 89 | return 0; 90 | } 91 | 92 | my $wait = 15; 93 | DEBUG ("Waiting MySQL process with $pid to shutdown: "); 94 | while ($wait--) { 95 | $pid = is_running($pid); 96 | last if ($pid == 0); 97 | DEBUG '.'; 98 | sleep(1); 99 | } 100 | 101 | if ($pid != 0) { 102 | ERROR "ERROR: MySQL is running with PID $pid after shutdown request!"; 103 | return 0; 104 | } 105 | 106 | INFO 'MySQL has been stopped!'; 107 | return 1; 108 | } 109 | 110 | =item change_master_to(named_params) 111 | 112 | Required params 113 | master_host 114 | master_port 115 | master_user 116 | master_pass 117 | 118 | Optional params 119 | master_log 120 | master_pos 121 | 122 | =cut 123 | sub change_master_to { 124 | my $args = shift; 125 | 126 | $args->{host} ||= $main::config->{this}; 127 | 128 | 129 | LOGDIE 'Bad call of change_master_to()' unless ( 130 | defined($args->{master_host}) && defined($args->{master_port}) 131 | && defined($args->{master_user}) && defined($args->{master_pass}) 132 | ); 133 | 134 | INFO "Changing master of host $args->{host} to $args->{master_host} ..."; 135 | 136 | # Get connection information 137 | my ($host, $port, $user, $password) = _get_connection_info($args->{host}); 138 | unless (defined($host)) { 139 | ERROR "No connection info for host '$args->{host}'"; 140 | return 0; 141 | } 142 | 143 | # Connect to server 144 | my $dbh = _connect($host, $port, $user, $password); 145 | unless ($dbh) { 146 | ERROR "Can't connect to MySQL (host = $host:$port, user = $user)!"; 147 | return 0; 148 | } 149 | 150 | 151 | my $res; 152 | 153 | # Stop slave 154 | $res = $dbh->do('STOP SLAVE'); 155 | unless ($res) { 156 | ERROR 'SQL Query Error: ', $dbh->errstr; 157 | return 0; 158 | } 159 | 160 | # Force deletion of obsolete master.info, relay-log.info and relay logs. 161 | $res = $dbh->do('RESET SLAVE'); 162 | unless ($res) { 163 | ERROR 'SQL Query Error: ', $dbh->errstr; 164 | return 0; 165 | } 166 | 167 | # Change master 168 | my $sql = sprintf( 169 | "CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%s, MASTER_USER='%s', MASTER_PASSWORD='%s'", 170 | $args->{master_host}, $args->{master_port}, $args->{master_user}, $args->{master_pass} 171 | ); 172 | 173 | if ($args->{master_log} && $args->{master_pos}) { 174 | $sql .= sprintf(", MASTER_LOG_FILE='%s', MASTER_LOG_POS=%s", $args->{master_log}, $args->{master_pos}); 175 | } 176 | 177 | $res = $dbh->do($sql); 178 | unless ($res) { 179 | ERROR 'SQL Query Error: ', $dbh->errstr; 180 | return 0; 181 | } 182 | 183 | # Start slave 184 | $res = $dbh->do('START SLAVE'); 185 | unless ($res) { 186 | ERROR 'SQL Query Error: ', $dbh->errstr; 187 | return 0; 188 | } 189 | 190 | # Disconnect 191 | $dbh->disconnect; 192 | 193 | INFO "Successfully changed master."; 194 | 195 | return 1; 196 | } 197 | 198 | =item _get_connection_info($host) 199 | 200 | Get connection info for host $host 201 | 202 | =cut 203 | 204 | sub _get_connection_info($) { 205 | my $host = shift; 206 | 207 | # TODO maybe check $host 208 | 209 | return ( 210 | $main::config->{host}->{$host}->{ip}, 211 | $main::config->{host}->{$host}->{mysql_port}, 212 | $main::config->{host}->{$host}->{tools_user}, 213 | $main::config->{host}->{$host}->{tools_password} 214 | ); 215 | } 216 | 217 | sub _connect($$$$) { 218 | my ($host, $port, $user, $password) = @_; 219 | my $dsn = "DBI:mysql:host=$host;port=$port;mysql_connect_timeout=3"; 220 | return DBI->connect($dsn, $user, $password, { PrintError => 0 }); 221 | } 222 | 223 | sub get_master_host($) { 224 | my $host_name = shift; 225 | 226 | # Get connection information 227 | my ($host, $port, $user, $password) = _get_connection_info($host_name); 228 | unless (defined($host)) { 229 | ERROR "No connection info for host '$host_name'"; 230 | return undef; 231 | } 232 | 233 | # Connect to server 234 | my $dbh = _connect($host, $port, $user, $password); 235 | unless ($dbh) { 236 | ERROR "Can't connect to MySQL (host = $host:$port, user = $user)!"; 237 | return undef; 238 | } 239 | 240 | # Get slave status 241 | my $res = $dbh->selectrow_hashref('SHOW SLAVE STATUS'); 242 | return "ERROR: Can't get slave status for host '$host_name'! Error: " . $dbh->errstr unless ($res); 243 | 244 | # Disconnect 245 | $dbh->disconnect(); 246 | 247 | return $res->{Master_Host}; 248 | } 249 | 250 | 1; 251 | -------------------------------------------------------------------------------- /lib/Tools/Snapshot/LVM.pm: -------------------------------------------------------------------------------- 1 | package MMM::Tools::Snapshot::LVM; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use Log::Log4perl qw(:easy); 6 | 7 | 8 | sub create() { 9 | my $this = $main::config->{this}; 10 | my $host = $main::config->{host}->{$this}; 11 | 12 | my @command = ( 13 | $host->{lvm_bin_lvcreate}, 14 | '--snapshot', 15 | '--size', $host->{lvm_snapshot_size}, 16 | '--name', 'mmm_snapshot', 17 | join('/', '/dev' , $host->{lvm_volume_group}, $host->{lvm_logical_volume}) 18 | ); 19 | 20 | my $lvm_res = system(@command); 21 | INFO "lvcreate output: '$lvm_res'"; 22 | 23 | return "ERROR: Can't create snapshot: $!" if ($lvm_res); 24 | 25 | my $mount_opts = $host->{lvm_mount_opts}; 26 | $mount_opts = '-o rw' unless ($mount_opts); 27 | 28 | my $res = system(sprintf('mount %s /dev/%s/mmm_snapshot %s', $mount_opts, $host->{lvm_volume_group}, $host->{lvm_mount_dir})); 29 | 30 | return "ERROR: Can't mount snapshot: $!" if ($res); 31 | return 'OK: Snapshot createg!'; 32 | } 33 | 34 | 35 | sub remove() { 36 | my $this = $main::config->{this}; 37 | my $host = $main::config->{host}->{$this}; 38 | if (!$host) { 39 | return "ERROR: Invalid 'this' value: '$this'!"; 40 | } 41 | 42 | # Unmount snapshot 43 | my $res = system('umount', $host->{lvm_mount_dir}); 44 | return "ERROR: Can't umount snapshot: $!" if ($res); 45 | 46 | my @command = ( 47 | $host->{lvm_bin_lvremove}, 48 | '-f', 49 | join('/', '/dev', $host->{lvm_volume_group}, 'mmm_snapshot') 50 | ); 51 | my $lvm_res = system(@command); 52 | INFO "lvremove output: '$lvm_res'"; 53 | 54 | return "ERROR: Can't remove snapshot: $!" if ($lvm_res); 55 | return 'OK: Snapshot removed!'; 56 | } 57 | 58 | 1; 59 | -------------------------------------------------------------------------------- /lib/Tools/Snapshot/MySQL.pm: -------------------------------------------------------------------------------- 1 | package MMM::Tools::Snapshot::MySQL; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use Data::Dumper; 6 | use DBI; 7 | use Log::Log4perl qw(:easy); 8 | 9 | 10 | sub lock_tables($) { 11 | my $dbh = shift; 12 | 13 | INFO 'Locking tables...'; 14 | my $res = $dbh->do('FLUSH TABLES WITH READ LOCK'); 15 | INFO "Result: '$res'"; 16 | 17 | system('sync'); 18 | sleep(1); 19 | system('sync'); 20 | 21 | return $res; 22 | 23 | } 24 | 25 | sub unlock_tables($) { 26 | my $dbh = shift; 27 | return $dbh->do('UNLOCK TABLES'); 28 | } 29 | 30 | sub get_pos_info($$) { 31 | my $dbh = shift; 32 | my $pos_info = shift; 33 | 34 | # Get master status info 35 | my $res = $dbh->selectrow_hashref('SHOW MASTER STATUS'); 36 | return "ERROR: Can't get master status information! Error: " . $dbh->errstr unless ($res); 37 | $pos_info->{master} = $res; 38 | 39 | # Get slave status info 40 | $res = $dbh->selectrow_hashref('SHOW SLAVE STATUS'); 41 | return "ERROR: Can't get slave status information! Error: " . $dbh->errstr if (defined($dbh->err)); 42 | $res = {} unless ($res); 43 | $pos_info->{slave} = $res; 44 | 45 | return 'OK: Got status info!'; 46 | } 47 | 48 | 49 | sub save_pos_info($$) { 50 | my $pos_info = shift; 51 | my $file = shift; 52 | 53 | open(POSFILE, '>', $file) || return "ERROR: Can't create pos file: $!"; 54 | print POSFILE Dumper($pos_info); 55 | close(POSFILE); 56 | 57 | return 'OK: Saved position info'; 58 | } 59 | 60 | =item _get_connection_info($host) 61 | 62 | Get connection info for host $host 63 | 64 | =cut 65 | 66 | sub get_connection_info($) { 67 | my $host = shift; 68 | 69 | # TODO maybe check $host 70 | 71 | return ( 72 | $main::config->{host}->{$host}->{ip}, 73 | $main::config->{host}->{$host}->{mysql_port}, 74 | $main::config->{host}->{$host}->{tools_user}, 75 | $main::config->{host}->{$host}->{tools_password} 76 | ); 77 | } 78 | 79 | sub connect($) { 80 | my $host_name = shift; 81 | my ($host, $port, $user, $password) = get_connection_info($host_name); 82 | my $dsn = "DBI:mysql:host=$host;port=$port;mysql_connect_timeout=3"; 83 | return DBI->connect($dsn, $user, $password, { PrintError => 0 }); 84 | } 85 | 86 | 1; 87 | -------------------------------------------------------------------------------- /lib/Tools/Tools.pm: -------------------------------------------------------------------------------- 1 | package MMM::Tools::Tools; 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw(FORMAT_NAME); 6 | use Path::Class qw(dir); 7 | use Log::Log4perl qw(:easy); 8 | 9 | 10 | our $VERSION = '0.01'; 11 | 12 | =head1 NAME 13 | 14 | MMM::Tools::Tools - functions for the mmm-tools. 15 | 16 | =cut 17 | 18 | =over 4 19 | 20 | =item check_backup_destination($path, [$should_be_empty]) 21 | 22 | Check backup destination 23 | 24 | =cut 25 | 26 | sub check_backup_destination { 27 | my $dir = shift; 28 | my $should_be_empty = shift || 0; 29 | 30 | INFO "Checking local destination directory '$dir'..."; 31 | 32 | system("mkdir -p $dir"); 33 | unless (-d $dir && -x _ && -r _ && -w _) { 34 | ERROR "Destination dir '$dir' has invalid permissions (should be readable/writeable/executable)!"; 35 | return 0; 36 | } 37 | 38 | if ($should_be_empty && scalar(glob("$dir/*"))) { 39 | ERROR "Destination dir '$dir' is not empty!"; 40 | return 0; 41 | } 42 | 43 | INFO 'Directory is ok'; 44 | return 1; 45 | } 46 | 47 | sub check_restore_source($) { 48 | my $dir = shift; 49 | 50 | INFO "Checking restore source directory '$dir'..."; 51 | 52 | unless (-d $dir && -x _ && -r _) { 53 | ERROR "Source dir '$dir' has invalid permissions (should be readable/executable)!"; 54 | return 0; 55 | } 56 | unless (scalar(glob("$dir/*"))) { 57 | ERROR "Source dir '$dir' is empty!"; 58 | return 0; 59 | } 60 | unless (-d "$dir/_mmm" && -r _ && -x _) { 61 | ERROR "Source dir doesn't contain _mmm sub-directory!"; 62 | return 0; 63 | } 64 | unless (-f "$dir/_mmm/status.txt" && -r _) { 65 | ERROR "$dir/_mmm/status.txt doesn't exist or isn't readable!"; 66 | return 0; 67 | } 68 | unless (-f "$dir/_mmm/copy_method.txt" && -r _) { 69 | ERROR "$dir/_mmm/copy_method.txt doesn't exist or isn't readable!"; 70 | return 0; 71 | } 72 | 73 | INFO 'Directory is ok'; 74 | return 1; 75 | } 76 | 77 | sub check_restore_destination($) { 78 | my $dir = shift; 79 | 80 | INFO "Checking destination directory '$dir'..."; 81 | 82 | system("mkdir -p $dir"); 83 | unless (-d $dir && -x _ && -r _ && -w _) { 84 | ERROR "Destination dir '$dir' has invalid permissions (should be readable/writeable/executable)!"; 85 | return 0; 86 | } 87 | 88 | INFO 'Directory is ok'; 89 | return 1; 90 | } 91 | =item check_ssh_connection($host) 92 | 93 | Check SSH connection to host $host. 94 | 95 | =cut 96 | 97 | sub check_ssh_connection($) { 98 | my $host = shift; 99 | 100 | my $ssh_host = $main::config->{host}->{$host}->{ssh_user} . '@' . $main::config->{host}->{$host}->{ip}; 101 | my $ssh_port = $main::config->{host}->{$host}->{ssh_port}; 102 | my $ssh_params = $main::config->{host}->{$host}->{ssh_parameters}; 103 | 104 | my $check_cmd = "ssh $ssh_params -p $ssh_port $ssh_host date"; 105 | 106 | INFO "Verifying ssh connection to remote host '$ssh_host' (command: $check_cmd)..."; 107 | 108 | my $res = system($check_cmd); 109 | if ($res) { 110 | ERROR "Can't execute remote commands on host '$host' ($ssh_host): $!"; 111 | return 0; 112 | } 113 | 114 | INFO "OK: SSH connection works fine!"; 115 | return 1; 116 | } 117 | 118 | 119 | =item execute_remote($host, $program) 120 | 121 | Execute command $program on remote host $host. 122 | 123 | =cut 124 | 125 | sub execute_remote($$) { 126 | my $host = shift; 127 | my $program = shift; 128 | 129 | my $ssh_host = $main::config->{host}->{$host}->{ssh_user} . '@' . $main::config->{host}->{$host}->{ip}; 130 | my $ssh_port = $main::config->{host}->{$host}->{ssh_port}; 131 | my $ssh_params = $main::config->{host}->{$host}->{ssh_parameters}; 132 | 133 | my $command = $main::config->{host}->{$host}->{bin_path} . '/tools/' . $program; 134 | 135 | DEBUG "Executing $program on host '$host'..."; 136 | INFO "ssh $ssh_params -p $ssh_port $ssh_host $command"; 137 | 138 | chomp(my $res = `ssh $ssh_params -p $ssh_port $ssh_host $command`); 139 | print "$res\n"; 140 | my @res_lines = split(/\n/, $res); 141 | my $last_line = pop(@res_lines); 142 | 143 | unless ($last_line =~ /^OK/) { 144 | ERROR $res; 145 | return 0; 146 | } 147 | return 1; 148 | } 149 | 150 | 151 | =item create_remote_snapshot($host) 152 | 153 | Create snapshot on host $host. 154 | 155 | =cut 156 | 157 | sub create_remote_snapshot($) { 158 | my $host = shift; 159 | return execute_remote($host, 'create_snapshot'); 160 | } 161 | 162 | 163 | =item remove_remote_snapshot($host) 164 | 165 | Remove snapshot on host $host. 166 | 167 | =cut 168 | 169 | sub remove_remote_snapshot($) { 170 | my $host = shift; 171 | return execute_remote($host, 'remove_snapshot'); 172 | } 173 | 174 | 175 | sub save_copy_method($$) { 176 | my $dir = shift; 177 | my $copy_method = shift; 178 | 179 | $dir .= '/_mmm'; 180 | 181 | # Check config option 182 | if (! -d $dir) { 183 | ERROR "Directory _mmm doesn't exist!"; 184 | return 0; 185 | } 186 | 187 | unless (open(F, ">$dir/copy_method.txt")) { 188 | ERROR "I/O Error while saving copy method!"; 189 | return 0; 190 | } 191 | print F $copy_method; 192 | close(F); 193 | 194 | DEBUG "Saved copy method"; 195 | return 1; 196 | } 197 | 198 | 199 | sub load_status($) { 200 | my $dir = shift; 201 | my $status; 202 | 203 | DEBUG 'Loading status info...'; 204 | 205 | my $status_file = $dir . '/_mmm/status.txt'; 206 | unless (-f $status_file && -r _) { 207 | ERROR "Status file '$status_file' doesn't exist or isn't readable!"; 208 | return undef; 209 | } 210 | 211 | my $status_data = `cat $status_file`; 212 | 213 | my $VAR1; 214 | eval($status_data); 215 | if ($@) { 216 | ERROR "Can't parse status info: $@"; 217 | return undef; 218 | } 219 | 220 | $status = $VAR1; 221 | 222 | my $method_file = $dir . '/_mmm/copy_method.txt'; 223 | chomp(my $copy_method = `cat $method_file`); 224 | $status->{copy_method} = $copy_method; 225 | 226 | return \%{$status}; 227 | } 228 | 229 | 230 | sub copy_clone_dirs($$$) { 231 | my $host = shift; 232 | my $copy_method = shift; 233 | my $dest_dir = shift; 234 | 235 | my @clone_dirs = @{$main::config->{clone_dirs}}; 236 | 237 | if ($main::config->{copy_method}->{$copy_method}->{single_run}) { 238 | return copy_from_remote_single_run($host, $copy_method, $dest_dir, \@clone_dirs); 239 | } 240 | 241 | foreach my $sub_dir (@clone_dirs) { 242 | return 0 unless (copy_from_remote($host, $copy_method, $dest_dir, $sub_dir)); 243 | } 244 | return 1; 245 | } 246 | 247 | sub copy_from_remote($$$$) { 248 | my $host = shift; 249 | my $copy_method = shift; 250 | my $dest_dir = shift; 251 | my $sub_dir = shift; 252 | 253 | my $host_info = $main::config->{host}->{$host}; 254 | my $ssh_host = $host_info->{ssh_user} . '@' . $host_info->{ip}; 255 | 256 | INFO "Copying '$sub_dir' from snapshot on host '$host' with copy method '$copy_method'"; 257 | 258 | my $command = $main::config->{copy_method}->{$copy_method}->{backup_command}; 259 | 260 | my $dir = dir('/', $sub_dir); 261 | unless ($dir->parent() eq '/') { 262 | $dest_dir .= '/' . $dir->parent(); 263 | system("mkdir -p $dest_dir"); 264 | } 265 | 266 | $command =~ s/%SSH_USER%/$host_info->{ssh_user}/ig; 267 | $command =~ s/%IP%/$host_info->{ip}/ig; 268 | $command =~ s/%SNAPSHOT%/$host_info->{lvm_mount_dir}/ig; 269 | $command =~ s/%DEST_DIR%/$dest_dir/ig; 270 | $command =~ s/%BACKUP_DIR%/$dest_dir/ig; 271 | $command =~ s/%CLONE_DIR%/$sub_dir/ig; 272 | 273 | INFO "Executing command $command"; 274 | if (system($command)) { 275 | ERROR "Can't copy $sub_dir: $!"; 276 | return 0; 277 | } 278 | 279 | INFO "Copied directory $sub_dir!"; 280 | return 1; 281 | } 282 | 283 | sub copy_from_remote_single_run($$$$) { 284 | my $host = shift; 285 | my $copy_method = shift; 286 | my $dest_dir = shift; 287 | my $clone_dirs = shift; 288 | 289 | my $host_info = $main::config->{host}->{$host}; 290 | my $ssh_host = $host_info->{ssh_user} . '@' . $host_info->{ip}; 291 | 292 | INFO "Copying files from snapshot on host '$host' with copy method '$copy_method'"; 293 | 294 | my $command = $main::config->{copy_method}->{$copy_method}->{backup_command}; 295 | 296 | $command =~ s/%SSH_USER%/$host_info->{ssh_user}/ig; 297 | $command =~ s/%IP%/$host_info->{ip}/ig; 298 | $command =~ s/%SNAPSHOT%/$host_info->{lvm_mount_dir}/ig; 299 | $command =~ s/%DEST_DIR%/$dest_dir/ig; 300 | $command =~ s/%BACKUP_DIR%/$dest_dir/ig; 301 | 302 | if ($command =~ /!(.*)!/) { 303 | my $sub_tmpl = $1; 304 | my $sub_cmd = ""; 305 | for my $sub_dir (@$clone_dirs) { 306 | my $partial = $sub_tmpl; 307 | $partial =~ s/%CLONE_DIR%/$sub_dir/ig; 308 | $sub_cmd .= " $partial"; 309 | } 310 | 311 | $command =~ s/!.*!/$sub_cmd/; 312 | } 313 | 314 | INFO "Executing command $command"; 315 | 316 | if (system($command)) { 317 | # TODO New config entry "check command"? 318 | # system("rdiff-backup --check-destination-dir '$config->{dest_dir}'"); 319 | ERROR "Can't copy from remote host: $!"; 320 | return 0; 321 | } 322 | 323 | INFO sprintf("Copied directories '%s' from host '$host'!", join("', '", sort(@$clone_dirs))); 324 | return 1; 325 | } 326 | 327 | 328 | =item restore($copy_method, $src_dir, $dest_dir) 329 | 330 | restore non-incremental backup 331 | 332 | =cut 333 | 334 | sub restore($$$) { 335 | my $copy_method = shift; 336 | my $src_dir = shift; 337 | my $dest_dir = shift; 338 | 339 | # TODO check copy method 340 | 341 | if ($main::config->{copy_method}->{$copy_method}->{incremental}) { 342 | ERROR 'The backup directory contains an incremental backup! Use --version option to restore a specific version.'; 343 | return 0; 344 | } 345 | 346 | my $command = $main::config->{copy_method}->{$copy_method}->{restore_command}; 347 | 348 | $command =~ s/%SRC_DIR%/$src_dir/ig; 349 | $command =~ s/%BACKUP_DIR%/$src_dir/ig; 350 | $command =~ s/%DEST_DIR%/$dest_dir/ig; 351 | $command =~ s/%DATA_DIR%/$dest_dir/ig; 352 | INFO "Executing command $command"; 353 | 354 | if (system($command)) { 355 | ERROR "Can't restore data: $!"; 356 | return 0; 357 | } 358 | INFO "Restored backup from '$src_dir' to '$dest_dir'"; 359 | return 1; 360 | } 361 | 362 | 363 | =item restore_incremental($copy_method, $src_dir, $dest_dir, $version) 364 | 365 | restore incremental backup 366 | 367 | =cut 368 | 369 | sub restore_incremental($$$$) { 370 | my $copy_method = shift; 371 | my $src_dir = shift; 372 | my $dest_dir = shift; 373 | my $version = shift; 374 | 375 | # TODO check copy method 376 | 377 | unless ($main::config->{copy_method}->{$copy_method}->{incremental}) { 378 | ERROR 'The backup directory contains an non-incremental backup!'; 379 | return 0; 380 | } 381 | 382 | my $command = $main::config->{copy_method}->{$copy_method}->{restore_command}; 383 | 384 | $command =~ s/%SRC_DIR%/$src_dir/ig; 385 | $command =~ s/%BACKUP_DIR%/$src_dir/ig; 386 | $command =~ s/%DEST_DIR%/$dest_dir/ig; 387 | $command =~ s/%DATA_DIR%/$dest_dir/ig; 388 | $command =~ s/%VERSION%/$version/ig; 389 | INFO "Executing command $command"; 390 | 391 | if (system($command)) { 392 | ERROR "Can't restore data: $!"; 393 | return 0; 394 | } 395 | INFO "Restored backup version '$version' from '$src_dir' to '$dest_dir'"; 396 | return 1; 397 | } 398 | 399 | 400 | =item list_increments($backup_dir, $copy_method) 401 | 402 | list available backup increments 403 | 404 | =cut 405 | 406 | sub list_increments($$) { 407 | my $backup_dir = shift; 408 | my $copy_method = shift; 409 | 410 | my $command = $main::config->{copy_method}->{$copy_method}->{incremental_command}; 411 | 412 | unless ($main::config->{copy_method}->{$copy_method}->{incremental}) { 413 | ERROR 'Invalid backup directory for incremental operations'; 414 | exit(0); 415 | } 416 | 417 | $command =~ s/%BACKUP_DIR%/$backup_dir/ig; 418 | 419 | # List versions 420 | my $res = open(COMMAND, "$command|"); 421 | unless ($res) { 422 | LogError("Can't read version info from backup!"); 423 | exit(1); 424 | } 425 | 426 | my $line; 427 | if ($command =~ /rdiff-backup/ && $command =~ 'parsable-output') { 428 | # Beautify rdiff-backup output 429 | print "Following backup versions are available:\n"; 430 | print " Version | Date\n"; 431 | print "-------------|---------------------------\n"; 432 | 433 | my $timestamp; 434 | format VERSION_LINE = 435 | @>>>>>>>>>> | @<<<<<<<<<<<<<<<<<<<<<<<< 436 | $timestamp, scalar(localtime($timestamp)) 437 | . 438 | $FORMAT_NAME = 'VERSION_LINE'; 439 | while ($line = ) { 440 | chomp $line; 441 | ($timestamp,) = split(/\s+/, $line); 442 | write; 443 | } 444 | } 445 | else { 446 | while ($line = ) { 447 | print $line; 448 | } 449 | } 450 | close(COMMAND); 451 | } 452 | 453 | 454 | =item cleanup($status, $dir) 455 | 456 | clean up restore directory 457 | 458 | =cut 459 | sub cleanup($$$) { 460 | 461 | my $status = shift; 462 | my $dir = shift; 463 | my $clone_dirs = shift; 464 | 465 | INFO 'Cleaning dump from master.info and binary logs...'; 466 | 467 | my $master_log = $status->{master}->{File}; 468 | unless ($master_log =~ /^(.*)\.(\d+)$/) { 469 | ERROR "Unknown master binary log file name format '$master_log'!"; 470 | return 0; 471 | } 472 | 473 | INFO "Deleting master binary logs: $1.*"; 474 | system("find $dir -name '$1.*' | xargs rm -vf"); 475 | 476 | if ($status->{slave} && $status->{slave}->{Relay_Log_File}) { 477 | my $slave_log = $status->{slave}->{Relay_Log_File}; 478 | unless ($slave_log =~ /^(.*)\.(\d+)$/) { 479 | ERROR "Unknown relay binary log file name format '$slave_log'!"; 480 | return 0; 481 | } 482 | INFO "Deleting relay binary logs: $1.*"; 483 | system("find $dir -name '$1.*' | xargs rm -vf"); 484 | } 485 | 486 | 487 | INFO 'Deleting .info and .pid files...'; 488 | system("find $dir -name master.info | xargs rm -vf"); 489 | system("find $dir -name relay-log.info | xargs rm -vf"); 490 | system("find $dir -name '*.pid' | xargs rm -vf"); 491 | 492 | INFO 'Changing permissions on mysql data dir...'; 493 | foreach my $sub_dir (@$clone_dirs) { 494 | system("chown -R mysql:mysql $dir/$sub_dir"); 495 | } 496 | 497 | return 1; 498 | } 499 | 500 | 501 | =item get_host_by_ip($ip) 502 | 503 | get hostname of host with ip $ip. 504 | 505 | =cut 506 | 507 | sub get_host_by_ip($) { 508 | my $ip = shift; 509 | foreach my $host (keys(%{$main::config->{host}})) { 510 | return $host if ($main::config->{host}->{$host}->{ip} eq $ip); 511 | } 512 | return ''; 513 | } 514 | 515 | 516 | 1; 517 | -------------------------------------------------------------------------------- /sbin/mmm_agentd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw( PROGRAM_NAME ); 6 | use File::Basename; 7 | use Proc::Daemon; 8 | use Log::Log4perl qw(:easy); 9 | Log::Log4perl->easy_init($INFO); 10 | 11 | # Define version and protocol version 12 | use constant MMM_VERSION => '2.2.1'; 13 | use constant MMM_PROTOCOL_VERSION => 1; 14 | 15 | 16 | # Include parts of the system 17 | use MMM::Common::Angel; 18 | use MMM::Common::Config; 19 | use MMM::Common::Log; 20 | use MMM::Common::PidFile; 21 | use MMM::Agent::Agent; 22 | 23 | 24 | # Maybe we were just asked for our version 25 | if (scalar(@ARGV) && $ARGV[0] eq "--version") { 26 | printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION; 27 | exit(0); 28 | } 29 | 30 | 31 | chdir('/'); 32 | umask(0022); 33 | 34 | our $cluster_name = ''; 35 | my $postfix = ''; 36 | if (scalar(@ARGV) && $ARGV[0] =~ /^@(.*)/) { 37 | shift(@ARGV); 38 | $cluster_name = $1; 39 | $postfix = "_$cluster_name"; 40 | $PROGRAM_NAME = basename($PROGRAM_NAME) . '-' . $cluster_name; 41 | } 42 | else { 43 | $PROGRAM_NAME = basename($PROGRAM_NAME); 44 | } 45 | 46 | 47 | 48 | 49 | MMM::Common::Log::init("mmm_agent_log$postfix.conf", "mmm_agentd$postfix"); 50 | 51 | # Read configuration 52 | our $config = new MMM::Common::Config::; 53 | $config->read("mmm_agent$postfix"); 54 | $config->check('AGENT'); 55 | 56 | 57 | 58 | 59 | my $debug = $config->{debug}; 60 | 61 | MMM::Common::Log::debug() if ($debug); 62 | 63 | our $agent = new MMM::Agent::Agent::( 64 | protocol_version => 1, 65 | active_master => '', 66 | state => 'UNKNOWN', 67 | config_file => "mmm_agent$postfix" 68 | ); 69 | $agent->from_config($config); 70 | 71 | my $pidfilename = $config->{host}->{ $config->{this} }->{pid_path}; 72 | my $pidfile = new MMM::Common::PidFile:: $pidfilename; 73 | 74 | # Check pid file 75 | LOGDIE "Can't run second copy of ", $PROGRAM_NAME if ($pidfile->is_running()); 76 | WARN "Unclean start - found stale pid file!" if ($pidfile->exists()); 77 | 78 | unless ($debug) { 79 | # Go to background 80 | Proc::Daemon::Init(); 81 | # Set umask again 82 | umask(0022); 83 | # Init logging again to re-open fds 84 | MMM::Common::Log::init("mmm_agent_log$postfix.conf", "mmm_agentd$postfix"); 85 | } 86 | 87 | # Init angel magic, which will restart us if we die unexpected 88 | MMM::Common::Angel::Init($pidfile); 89 | 90 | # Shutdown flag 91 | our $shutdown = 0; 92 | 93 | # Set signal handlers 94 | $SIG{INT} = \&ShutdownHandler; 95 | $SIG{TERM} = \&ShutdownHandler; 96 | $SIG{QUIT} = \&ShutdownHandler; 97 | 98 | $agent->main(); 99 | 100 | INFO 'END'; 101 | exit(0); 102 | 103 | #----------------------------------------------------------------- 104 | sub ShutdownHandler() { 105 | INFO "Signal received: exiting..."; 106 | $shutdown = 1; 107 | } 108 | -------------------------------------------------------------------------------- /sbin/mmm_backup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw( PROGRAM_NAME ); 6 | use File::Basename; 7 | use Log::Log4perl qw(:easy); 8 | use Getopt::Long; 9 | Log::Log4perl->easy_init( { level => $INFO, layout => '%p: %m%n' }); 10 | 11 | # Define version 12 | use constant MMM_VERSION => '2.2.1'; 13 | 14 | # Include parts of the system 15 | use MMM::Common::Config; 16 | use MMM::Tools::Tools; 17 | 18 | # Maybe we were just asked for our version 19 | if (scalar(@ARGV) == 1 && $ARGV[0] eq "--version") { 20 | printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION; 21 | exit(0); 22 | } 23 | 24 | 25 | my $config_file = ''; 26 | my $host_name = ''; 27 | my $copy_method = ''; 28 | my $dest_dir = ''; 29 | 30 | print_usage() unless ( 31 | GetOptions( 32 | 'config=s' => \$config_file, 33 | 'host=s' => \$host_name, 34 | 'copy-method=s' => \$copy_method, 35 | 'dest-dir=s' => \$dest_dir 36 | ) 37 | ); 38 | 39 | $config_file = 'mmm_tools' if ($config_file eq ''); 40 | 41 | # Read configuration 42 | our $config = new MMM::Common::Config::; 43 | $config->read($config_file); 44 | $config->check('TOOLS'); 45 | 46 | print_usage("Invalid host name '$host_name'") unless (defined($config->{host}->{$host_name})); 47 | 48 | my $host = $config->{host}->{$host_name}; 49 | 50 | $copy_method = $config->{default_copy_method} if ($copy_method eq ''); 51 | $dest_dir = $host->{backup_dir} if ($dest_dir eq ''); 52 | 53 | print_usage("Invalid copy method '$copy_method'") unless (defined($config->{copy_method}->{$copy_method})); 54 | my $should_be_empty = (!$config->{copy_method}->{$copy_method}->{incremental}); 55 | print_usage("Invalid backup directory '$dest_dir'") unless (MMM::Tools::Tools::check_backup_destination($dest_dir, $should_be_empty)); 56 | 57 | die unless (MMM::Tools::Tools::check_ssh_connection($host_name)); 58 | die unless (MMM::Tools::Tools::create_remote_snapshot($host_name)); 59 | die unless (MMM::Tools::Tools::copy_clone_dirs($host_name, $copy_method, $dest_dir)); 60 | die unless (MMM::Tools::Tools::copy_from_remote($host_name, 'scp', $dest_dir, '_mmm')); 61 | die unless (MMM::Tools::Tools::save_copy_method($dest_dir, $copy_method)); 62 | die unless (MMM::Tools::Tools::remove_remote_snapshot($host_name)); 63 | 64 | exit 0; 65 | 66 | sub print_usage { 67 | my $msg = shift; 68 | 69 | print "$msg\n\n" if ($msg); 70 | print "Usage: $0 [--config ] --host [--copy-method ] [--dest-dir ]\n"; 71 | if ($main::config) { 72 | print "Where:\n"; 73 | printf(" host : %s\n", join(' | ', sort(keys(%{$main::config->{host}})))); 74 | printf(" copy-method: %s (default: %s)\n", join(' | ', sort(keys(%{$main::config->{copy_method}}))), $config->{default_copy_method}); 75 | print " dest-dir : directory where data should be backed up to\n\n"; 76 | } 77 | exit(1); 78 | } 79 | -------------------------------------------------------------------------------- /sbin/mmm_clone: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw( PROGRAM_NAME ); 6 | use File::Basename; 7 | use Log::Log4perl qw(:easy); 8 | use Getopt::Long; 9 | Log::Log4perl->easy_init( { level => $INFO, layout => '%p: %m%n' }); 10 | 11 | # Define version 12 | use constant MMM_VERSION => '2.2.1'; 13 | 14 | # Include parts of the system 15 | use MMM::Common::Config; 16 | use MMM::Tools::Tools; 17 | use MMM::Tools::MySQL; 18 | 19 | # Maybe we were just asked for our version 20 | if (scalar(@ARGV) == 1 && $ARGV[0] eq "--version") { 21 | printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION; 22 | exit(0); 23 | } 24 | 25 | our @CLONE_MODES = qw( 26 | master-master 27 | master-slave 28 | slave-slave 29 | ); 30 | 31 | my $config_file = ''; 32 | my $host_name = ''; 33 | my $clone_mode = ''; 34 | my $copy_method = ''; 35 | my $dest_dir = ''; 36 | my $dry_run = 0; 37 | 38 | print_usage() unless ( 39 | GetOptions( 40 | 'config=s' => \$config_file, 41 | 'host=s' => \$host_name, 42 | 'clone-mode=s' => \$clone_mode, 43 | 'copy-method=s' => \$copy_method, 44 | 'dest-dir=s' => \$dest_dir, 45 | 'dry-run' => \$dry_run 46 | ) 47 | ); 48 | 49 | $config_file = 'mmm_tools' if ($config_file eq ''); 50 | 51 | # Read configuration 52 | our $config = new MMM::Common::Config::; 53 | $config->read($config_file); 54 | $config->check('TOOLS'); 55 | 56 | # Check params and set defaults 57 | print_usage("Invalid host name '$host_name'") unless (defined($config->{host}->{$host_name})); 58 | print_usage("We can't clone ourselves") unless ($host_name ne $config->{this}); 59 | 60 | my $dest_host_info = $config->{host}->{$config->{this}}; 61 | 62 | $copy_method = $config->{default_copy_method} if ($copy_method eq ''); 63 | $dest_dir = $dest_host_info->{restore_dir} if ($dest_dir eq ''); 64 | 65 | print_usage("Unknown clone mode '$clone_mode'") unless (grep(/^$clone_mode$/, @CLONE_MODES)); 66 | print_usage("Invalid copy method '$copy_method'") unless (defined($config->{copy_method}->{$copy_method})); 67 | print_usage("Only copy methods which create an exact copy can be used for cloning") 68 | unless ($config->{copy_method}->{$copy_method}->{true_copy}); 69 | print_usage("Invalid destination directory '$dest_dir'") unless (MMM::Tools::Tools::check_restore_destination($dest_dir)); 70 | 71 | 72 | # Determine replication peer 73 | my $peer_host = $host_name; 74 | if ($clone_mode eq 'slave-slave') { 75 | my $master_ip = MMM::Tools::MySQL::get_master_host($host_name); 76 | die unless ($master_ip); 77 | $peer_host = MMM::Tools::Tools::get_host_by_ip($master_ip); 78 | } 79 | 80 | unless (defined($config->{host}->{$peer_host})) { 81 | LOGDIE "Unknown peer host '$peer_host'"; 82 | } 83 | 84 | my $peer_host_info = $config->{host}->{$peer_host}; 85 | 86 | 87 | 88 | # Print info 89 | my $label; 90 | my $value; 91 | format CLONE_INFO = 92 | @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<: @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 93 | $label, $value 94 | . 95 | $FORMAT_NAME = 'CLONE_INFO'; 96 | 97 | $label = 'Source host'; 98 | $value = $host_name; 99 | write; 100 | 101 | $label = 'Destination dir'; 102 | $value = $dest_dir; 103 | write; 104 | 105 | $label = 'Dirs to clone'; 106 | $value = join(', ', sort(@{$config->{clone_dirs}})); 107 | write; 108 | 109 | $label = 'Clone mode'; 110 | $value = $clone_mode; 111 | write; 112 | 113 | $label = 'Copy method'; 114 | $value = $copy_method; 115 | write; 116 | 117 | $label = 'Replication peer'; 118 | $value = $peer_host; 119 | write; 120 | 121 | $label = 'Setup master-master replication'; 122 | $value = ($clone_mode eq 'master-master'? 'yes' : 'no'); 123 | write; 124 | 125 | $label = 'Dry run'; 126 | $value = $dry_run ? 'yes' : 'no'; 127 | write; 128 | 129 | # Stop here if this is a dry-run 130 | exit 0 if ($dry_run); 131 | 132 | 133 | # Stop mysql 134 | die unless (MMM::Tools::MySQL::stop()); 135 | 136 | 137 | # Fetch snapshot from remote host 138 | die unless (MMM::Tools::Tools::check_ssh_connection($host_name)); 139 | die unless (MMM::Tools::Tools::create_remote_snapshot($host_name)); 140 | die unless (MMM::Tools::Tools::copy_clone_dirs($host_name, $copy_method, $dest_dir)); 141 | die unless (MMM::Tools::Tools::copy_from_remote($host_name, 'scp', $dest_dir, '_mmm')); 142 | die unless (MMM::Tools::Tools::save_copy_method($dest_dir, $copy_method)); 143 | die unless (MMM::Tools::Tools::remove_remote_snapshot($host_name)); 144 | 145 | # Load information from status file 146 | my $status = MMM::Tools::Tools::load_status($dest_dir); 147 | die unless (defined($status)); 148 | 149 | # Cleanup 150 | die unless (MMM::Tools::Tools::cleanup($status, $dest_dir, $config->{clone_dirs})); 151 | 152 | # Start MySQL server 153 | die unless (MMM::Tools::MySQL::start()); 154 | 155 | 156 | # Setup replication 157 | my %dest_replication_info = ( 158 | host => $config->{this}, 159 | master_host => $peer_host_info->{ip}, 160 | master_port => $peer_host_info->{mysql_port}, 161 | master_user => $peer_host_info->{replication_user}, 162 | master_pass => $peer_host_info->{replication_password}, 163 | ); 164 | 165 | if ($clone_mode eq 'master-slave' || $clone_mode eq 'master-master') { 166 | $dest_replication_info{master_log} = $status->{master}->{'File'}; 167 | $dest_replication_info{master_pos} = $status->{master}->{'Position'}; 168 | die unless MMM::Tools::MySQL::change_master_to(\%dest_replication_info); 169 | } 170 | elsif ($clone_mode eq 'slave-slave') { 171 | $dest_replication_info{master_log} = $status->{slave}->{'Relay_Master_Log_File'}; 172 | $dest_replication_info{master_pos} = $status->{slave}->{'Exec_Master_Log_Pos'}; 173 | die unless MMM::Tools::MySQL::change_master_to(\%dest_replication_info); 174 | } 175 | 176 | # Setup peer replication 177 | if ($clone_mode eq 'master-master') { 178 | my %peer_replication_info = ( 179 | host => $peer_host, 180 | master_host => $dest_host_info->{ip}, 181 | master_port => $dest_host_info->{mysql_port}, 182 | master_user => $dest_host_info->{replication_user}, 183 | master_pass => $dest_host_info->{replication_password}, 184 | ); 185 | die unless MMM::Tools::MySQL::change_master_to(\%peer_replication_info); 186 | } 187 | 188 | INFO 'Clone operation finished!'; 189 | 190 | exit 0; 191 | 192 | sub print_usage { 193 | my $msg = shift; 194 | 195 | print "$msg\n\n" if ($msg); 196 | print "Usage: $0 [--config ] --host --clone-mode [--copy-method ] [--dest-dir ]\n"; 197 | if ($main::config) { 198 | my @allowed_methods = grep { $main::config->{copy_method}->{$_}->{true_copy}} keys(%{$main::config->{copy_method}}); 199 | print "Where:\n"; 200 | printf(" host : %s\n", join(' | ', sort(keys(%{$main::config->{host}})))); 201 | printf(" clone-mode : %s\n", join(' | ', sort(@CLONE_MODES))); 202 | printf(" copy-method: %s (default: %s)\n", join(' | ', sort(@allowed_methods)), $config->{default_copy_method}); 203 | print " dest-dir : directory where data should be cloned to\n\n"; 204 | } 205 | exit(1); 206 | } 207 | -------------------------------------------------------------------------------- /sbin/mmm_control: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw( PROGRAM_NAME ); 6 | use File::Basename; 7 | use Log::Log4perl qw(:easy); 8 | Log::Log4perl->easy_init($INFO); 9 | 10 | # Define version 11 | use constant MMM_VERSION => '2.2.1'; 12 | 13 | # Include parts of the system 14 | use MMM::Common::Config; 15 | use MMM::Common::Socket; 16 | 17 | # Maybe we were just asked for our version 18 | if (scalar(@ARGV) && $ARGV[0] eq "--version") { 19 | printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION; 20 | exit(0); 21 | } 22 | 23 | my $postfix = ''; 24 | if (scalar(@ARGV) && $ARGV[0] =~ /^@(.*)/) { 25 | shift(@ARGV); 26 | $postfix = "_$1"; 27 | } 28 | 29 | # Read configuration 30 | our $config = new MMM::Common::Config::; 31 | $config->read("mmm_mon$postfix"); 32 | $config->check('CONTROL'); 33 | 34 | die "See '$0 help' for usage information" if (scalar(@ARGV) < 1); 35 | 36 | my $socket = MMM::Common::Socket::create_sender($config->{monitor}->{ip}, $config->{monitor}->{port}, 10); 37 | unless ($socket && $socket->connected) { 38 | print "ERROR: Can't connect to monitor daemon!\n"; 39 | exit(1); 40 | } 41 | 42 | print $socket join(' ', @ARGV), "\nquit\n"; 43 | my $res = ''; 44 | my $line; 45 | $res .= $line while ($line = <$socket>); 46 | print $res, "\n"; 47 | 48 | exit(0); 49 | 50 | -------------------------------------------------------------------------------- /sbin/mmm_mond: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use threads; 6 | use threads::shared; 7 | use Config; 8 | use English qw( PROGRAM_NAME ); 9 | use File::Basename; 10 | use POSIX ':sys_wait_h'; 11 | use Proc::Daemon; 12 | use Log::Log4perl qw(:easy); 13 | Log::Log4perl->easy_init($INFO); 14 | 15 | # Define version and protocol version 16 | use constant MMM_VERSION => '2.2.1'; 17 | use constant MMM_PROTOCOL_VERSION => 1; 18 | 19 | # Check perl for threads support 20 | $Config{useithreads} or die "Recompile Perl with threads to run this program."; 21 | 22 | 23 | # Include parts of the system 24 | use MMM::Common::Angel; 25 | use MMM::Common::Config; 26 | use MMM::Common::Log; 27 | use MMM::Common::PidFile; 28 | use MMM::Monitor::Monitor; 29 | use MMM::Monitor::NetworkChecker; 30 | 31 | 32 | # Maybe we were just asked for our version 33 | if (scalar(@ARGV) && $ARGV[0] eq "--version") { 34 | printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION; 35 | exit(0); 36 | } 37 | 38 | 39 | chdir('/'); 40 | umask(0022); 41 | 42 | our $cluster_name = ''; 43 | my $postfix = ''; 44 | if (scalar(@ARGV) && $ARGV[0] =~ /^@(.*)/) { 45 | shift(@ARGV); 46 | $cluster_name = $1; 47 | $postfix = "_$cluster_name"; 48 | $PROGRAM_NAME = basename($PROGRAM_NAME) . '-' . $cluster_name; 49 | } 50 | else { 51 | $PROGRAM_NAME = basename($PROGRAM_NAME); 52 | } 53 | 54 | 55 | 56 | 57 | MMM::Common::Log::init("mmm_mon_log$postfix.conf", "mmm_mond$postfix"); 58 | 59 | # Read configuration 60 | our $config = new MMM::Common::Config::; 61 | $config->read("mmm_mon$postfix"); 62 | $config->check('MONITOR'); 63 | 64 | 65 | 66 | 67 | my $debug = $config->{debug}; 68 | 69 | MMM::Common::Log::debug() if ($debug); 70 | 71 | INFO 'STARTING...'; 72 | 73 | our $monitor = new MMM::Monitor::Monitor::(); 74 | 75 | my $pidfilename = $config->{monitor}->{pid_path}; 76 | my $pidfile = new MMM::Common::PidFile:: $pidfilename; 77 | 78 | # Check pid file 79 | LOGDIE "Can't run second copy of ", $PROGRAM_NAME if ($pidfile->is_running()); 80 | WARN "Unclean start - found stale pid file!" if ($pidfile->exists()); 81 | 82 | unless ($debug) { 83 | # Go to background 84 | Proc::Daemon::Init(); 85 | # Set umask again 86 | umask(0022); 87 | # Init logging again to re-open fds 88 | MMM::Common::Log::init("mmm_mon_log$postfix.conf", "mmm_mond$postfix"); 89 | } 90 | 91 | # Init angel magic, which will restart us if we die unexpected 92 | MMM::Common::Angel::Init($pidfile); 93 | 94 | our $shutdown :shared = 0; # Shutdown flag 95 | our $have_net :shared = 1; # Network status flag 96 | 97 | # Set signal handlers 98 | $SIG{INT} = \&ShutdownHandler; 99 | $SIG{TERM} = \&ShutdownHandler; 100 | $SIG{QUIT} = \&ShutdownHandler; 101 | $SIG{PIPE} = 'IGNORE'; 102 | $SIG{CHLD} = \&ChildHandler; 103 | 104 | if ($monitor->init()) { 105 | $monitor->main(); 106 | } 107 | 108 | INFO 'END'; 109 | exit(0); 110 | 111 | #----------------------------------------------------------------- 112 | sub ShutdownHandler() { 113 | INFO "Signal received: exiting..."; 114 | $shutdown = 1; 115 | } 116 | 117 | #----------------------------------------------------------------- 118 | sub ChildHandler { 119 | local $!; # don't let waitpid() overwrite current error 120 | while ((my $pid = waitpid(-1, WNOHANG)) > 0 && WIFEXITED($?)) { 121 | DEBUG "Core: reaped child $pid" . ($? ? " with exit $?" : ''); 122 | } 123 | $SIG{CHLD} = \&ChildHandler; # loathe sysV 124 | } 125 | -------------------------------------------------------------------------------- /sbin/mmm_restore: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings FATAL => 'all'; 5 | use English qw( PROGRAM_NAME FORMAT_NAME ); 6 | use File::Basename; 7 | use Log::Log4perl qw(:easy); 8 | use Getopt::Long; 9 | Log::Log4perl->easy_init( { level => $INFO, layout => '%p: %m%n' }); 10 | 11 | # Define version 12 | use constant MMM_VERSION => '2.2.1'; 13 | 14 | # Include parts of the system 15 | use MMM::Common::Config; 16 | use MMM::Tools::Tools; 17 | use MMM::Tools::MySQL; 18 | 19 | # Maybe we were just asked for our version 20 | if (scalar(@ARGV) == 1 && $ARGV[0] eq "--version") { 21 | printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION; 22 | exit(0); 23 | } 24 | 25 | my @RESTORE_MODES = qw( 26 | data-only 27 | 28 | single-single 29 | slave-single 30 | master-single 31 | 32 | master-slave 33 | slave-slave 34 | ); 35 | 36 | my $config_file = ''; 37 | my $src_dir = ''; 38 | my $dest_dir = ''; 39 | my $restore_mode= ''; 40 | my $version = ''; 41 | my $skip_mysqld = 0; 42 | my $dry_run = 0; 43 | 44 | print_usage() unless ( 45 | GetOptions( 46 | 'config=s' => \$config_file, 47 | 'src-dir=s' => \$src_dir, 48 | 'dest-dir=s' => \$dest_dir, 49 | 'mode=s' => \$restore_mode, 50 | 'version=s' => \$version, 51 | 'dry-run' => \$dry_run 52 | ) 53 | ); 54 | 55 | # Set defaults (part 1) 56 | $config_file = 'mmm_tools' if ($config_file eq ''); 57 | $restore_mode = 'single-single' if ($restore_mode eq ''); 58 | $skip_mysqld = 1 if ($restore_mode eq 'data-only'); 59 | 60 | # Read configuration 61 | our $config = new MMM::Common::Config::; 62 | $config->read($config_file); 63 | $config->check('TOOLS'); 64 | 65 | my $host = $config->{host}->{$config->{this}}; 66 | 67 | # Set defaults (part 2) 68 | $src_dir = $host->{backup_dir} if ($src_dir eq ''); 69 | $dest_dir = $host->{restore_dir} if ($dest_dir eq ''); 70 | 71 | # Check params 72 | print_usage("Invalid restore mode '$restore_mode'") unless (grep(/^$restore_mode$/, @RESTORE_MODES)); 73 | print_usage("Invalid backup directory '$src_dir'") unless (MMM::Tools::Tools::check_restore_source($src_dir)); 74 | 75 | if ($version eq 'list') { 76 | # Print list of increments 77 | die unless (MMM::Tools::Tools::list_increments($src_dir, 'rdiff')); 78 | exit 0; 79 | } 80 | 81 | # Load information from status file 82 | my $status = MMM::Tools::Tools::load_status($src_dir); 83 | die unless (defined($status)); 84 | 85 | 86 | # Check copy method of backup 87 | unless (defined($config->{copy_method}->{$status->{copy_method}})) { 88 | ERROR "Backup was created with unknown copy-method '$status->{copy_method}' - can't restore it"; 89 | die; 90 | } 91 | 92 | # Check if backup is incremental and we were called with version 93 | if ($config->{copy_method}->{$status->{copy_method}}->{incremental} && $version eq '') { 94 | ERROR "'$src_dir' contains an incremental backup, but no version was specified"; 95 | MMM::Tools::Tools::list_increments($src_dir, 'rdiff'); 96 | die; 97 | } 98 | 99 | 100 | # Determine replication peer 101 | $restore_mode =~ /(\w+)\-(\w+)/; 102 | my $src_mode = $1; 103 | my $dest_mode = $2; 104 | 105 | my $peer_host = $status->{host}; 106 | my $peer_host_info; 107 | if ($src_mode eq 'slave') { 108 | $peer_host = MMM::Tools::Tools::get_host_by_ip($status->{slave}->{'Master_Host'}); 109 | } 110 | unless ($dest_mode eq 'single' || $skip_mysqld) { 111 | unless (defined($config->{host}->{$peer_host})) { 112 | LOGDIE "Unknown peer host '$peer_host'"; 113 | } 114 | $peer_host_info = $config->{host}->{$peer_host}; 115 | } 116 | 117 | 118 | # Print info 119 | my $label; 120 | my $value; 121 | format RESTORE_INFO = 122 | @<<<<<<<<<<<<<<<<<<<<<: @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< 123 | $label, $value 124 | . 125 | $FORMAT_NAME = 'RESTORE_INFO'; 126 | 127 | $label = 'Source dir'; 128 | $value = $src_dir; 129 | write; 130 | 131 | $label = 'Destination dir'; 132 | $value = $dest_dir; 133 | write; 134 | 135 | $label = 'Dirs to restore'; 136 | $value = join(', ', sort(@{$config->{clone_dirs}})); 137 | write; 138 | 139 | $label = 'Restore mode'; 140 | $value = $restore_mode; 141 | write; 142 | 143 | if ($version ne '') { 144 | $label = 'Incremental version'; 145 | $value = $version; 146 | write; 147 | } 148 | 149 | if (defined($peer_host_info)) { 150 | $label = 'Replication peer'; 151 | $value = $peer_host; 152 | write; 153 | } 154 | 155 | $label = 'Skip mysqld operations'; 156 | $value = $skip_mysqld ? 'yes' : 'no'; 157 | write; 158 | 159 | $label = 'Dry run'; 160 | $value = $dry_run ? 'yes' : 'no'; 161 | write; 162 | 163 | 164 | 165 | # Stop here if this is a dry-run 166 | exit 0 if ($dry_run); 167 | 168 | 169 | 170 | unless ($skip_mysqld) { 171 | # Stop MySQL server 172 | die unless (MMM::Tools::MySQL::stop()); 173 | } 174 | 175 | 176 | # Check/Create destination directory 177 | die unless (MMM::Tools::Tools::check_restore_destination($dest_dir)); 178 | 179 | 180 | # Restore backup 181 | if ($version eq '') { 182 | die unless (MMM::Tools::Tools::restore($status->{copy_method}, $src_dir, $dest_dir)); 183 | } 184 | else { 185 | die unless (MMM::Tools::Tools::restore_incremental($status->{copy_method}, $src_dir, $dest_dir, $version)); 186 | } 187 | 188 | 189 | # Cleanup 190 | die unless (MMM::Tools::Tools::cleanup($status, $dest_dir, $config->{clone_dirs})); 191 | 192 | 193 | unless ($skip_mysqld) { 194 | 195 | # Start MySQL server 196 | die unless (MMM::Tools::MySQL::start()); 197 | 198 | 199 | # Setup replication 200 | if ($dest_mode eq 'single') { 201 | INFO "Skipping replication setup because destination configuration is '$dest_mode'"; 202 | } 203 | else { 204 | my %dest_replication_info = ( 205 | host => $config->{this}, 206 | master_host => $peer_host_info->{ip}, 207 | master_port => $peer_host_info->{mysql_port}, 208 | master_user => $peer_host_info->{replication_user}, 209 | master_pass => $peer_host_info->{replication_password}, 210 | ); 211 | 212 | if ($src_mode eq 'master' && $dest_mode eq 'slave') { 213 | $dest_replication_info{master_log} = $status->{master}->{'File'}; 214 | $dest_replication_info{master_pos} = $status->{master}->{'Position'}; 215 | die unless MMM::Tools::MySQL::change_master_to(\%dest_replication_info); 216 | } 217 | elsif ($src_mode eq 'slave' && $dest_mode eq 'slave') { 218 | $dest_replication_info{master_log} = $status->{slave}->{'Relay_Master_Log_File'}; 219 | $dest_replication_info{master_pos} = $status->{slave}->{'Exec_Master_Log_Pos'}; 220 | die unless MMM::Tools::MySQL::change_master_to(\%dest_replication_info); 221 | } 222 | else { 223 | WARN "Bad restore mode '$src_mode-$dest_mode', skipping replication setup"; 224 | } 225 | } 226 | } 227 | 228 | INFO 'Restore operation finished!'; 229 | 230 | exit 0; 231 | 232 | sub print_usage { 233 | my $msg = shift; 234 | 235 | print "$msg\n\n" if ($msg); 236 | print "Usage: $0 [--config ] [--mode ] [--version ] [--src-dir ] [--dest-dir ] [--dry-run]\n"; 237 | print "Where:\n"; 238 | print " src-dir : directory where backup resides\n"; 239 | print " dest-dir: directory where backup should be restored to\n"; 240 | print " mode : " . join(', ', @RESTORE_MODES) . "\n"; 241 | print " version : \n"; 242 | print " - when run with 'list' parameter, displays available versions of incremental backups\n"; 243 | print " - if version is specified, tries to restore backup for specified version\n"; 244 | print " dry-run : check everything and exit without any changes\n\n"; 245 | exit(1); 246 | } 247 | --------------------------------------------------------------------------------