├── .gitattributes ├── .idea ├── .name ├── bashsupport_project.xml ├── codeStyleSettings.xml ├── encodings.xml ├── inspectionProfiles │ ├── Project_Default.xml │ └── profiles_settings.xml ├── misc.xml ├── modules.xml ├── mysql_web.iml ├── sqldialects.xml └── vcs.xml ├── README.md ├── backup ├── __init__.py ├── backup.py ├── backup_base.py ├── backup_server.py ├── mydumper.py ├── mysqldump.py └── xtrabackup.py ├── install.sh ├── mha ├── __init__.py └── mha_check.py ├── monitor ├── __init__.py ├── binlog_util.py ├── cache.py ├── chart.py ├── common.py ├── custom_algorithm.py ├── db_util.py ├── entitys.py ├── mail_util.py ├── mysql_enum.py ├── mysql_manager.py ├── new_slow_log.py ├── report.py ├── server.py ├── tablespace.py ├── tablespace_new.py └── user_login.py ├── mysql_web.py ├── python_upgrade.sh ├── settings.py ├── sql └── table.sql ├── startup.sh ├── static ├── css │ ├── bootstrap-datetimepicker.min.css │ ├── bootstrap-select.css │ ├── bootstrap-theme.css │ ├── bootstrap-theme.css.map │ ├── bootstrap-theme.min.css │ ├── bootstrap-theme.min.css.map │ ├── bootstrap.css │ ├── bootstrap.css.map │ ├── bootstrap.min.css │ └── bootstrap.min.css.map ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.svg │ ├── glyphicons-halflings-regular.ttf │ ├── glyphicons-halflings-regular.woff │ └── glyphicons-halflings-regular.woff2 ├── img │ ├── 111.png │ ├── 112.png │ ├── 113.png │ ├── 114.png │ ├── 115.png │ ├── 116.png │ ├── 117.png │ ├── 118.png │ ├── 119.png │ ├── 120.png │ ├── 121.png │ ├── 123.png │ └── 124.png ├── jquery │ ├── jquery-form.js │ └── jquery.min.js └── js │ ├── Chart.js │ ├── Chart.min.js │ ├── bootstrap-datetimepicker.min.js │ ├── bootstrap-select.js │ ├── bootstrap-treeview.min.js │ ├── bootstrap.js │ ├── bootstrap.min.js │ ├── echarts.min.js │ ├── jscharts.js │ ├── md5.min.js │ ├── my.js │ ├── npm.js │ └── pako.js ├── templates ├── alarm_config.html ├── alarm_log.html ├── alarm_report.html ├── backup.html ├── binlog.html ├── binlog_display.html ├── chart.html ├── chart_auto.html ├── chart_mysql.html ├── chart_new.html ├── config.html ├── execute_sql.html ├── general_log.html ├── general_log_detail.html ├── home.html ├── host_search.html ├── login.html ├── mha.html ├── monitor.html ├── mysql_exception_log.html ├── mysql_exception_log_display.html ├── mysql_host.html ├── mysql_host_view.html ├── mysqls.html ├── new_slow_log_detail.html ├── new_slow_log_home.html ├── report.html ├── show_processlist.html ├── slow_log_detail.html ├── slow_log_display.html ├── table.html ├── tablespace.html ├── tablespace_dispaly.html ├── thread.html ├── thread_display.html ├── user.html └── user_display.html ├── tools ├── backup │ └── bk_xtrabackup.py ├── binlog │ ├── __init__.py │ ├── binlog_backup.py │ ├── binlog_util.py │ └── binlog_util_new.py ├── check_os.sh ├── check_tablespace.py ├── collect_mysql_status.sh ├── dns_ha │ └── __init__.py ├── host.conf ├── mha │ ├── app.conf │ ├── master_ip_failover │ ├── masterha_master_switch │ ├── mha_auto_install.py │ ├── mha_manual │ ├── mha线上故障恢复 │ ├── package │ │ ├── mha-0.56.zip │ │ └── mha-0.57.zip │ ├── purge_relay_log.sh │ └── start_mha.sh ├── mysql │ ├── __init__.py │ ├── common_sql.py │ ├── mysql_auto_install.py │ ├── mysql_create_slave_for_mysqldump.py │ └── mysql_create_slave_for_xtrabackup.py ├── mysql_backup.py ├── mysql_general_log.sh ├── mysql_monitor.py └── mysql_slow_log.sh └── tools_new ├── binlog_bk.py ├── binlog_sync.py ├── binlog_util.py ├── binlog_util_new.py ├── bk_recovery_xtrabackup.py ├── bk_recovery_xtrabackup_remote.py ├── bk_xtrabackup.py ├── bk_xtrabackup_remote.py ├── collect_mysql_status_log.sh ├── mha ├── app.conf ├── master_ip_failover ├── masterha_master_switch ├── mha_auto_install.py ├── mha_manual ├── mha线上故障恢复 ├── package │ ├── mha-0.56.zip │ └── mha-0.57.zip ├── purge_relay_log.sh ├── send_report.sh └── start_mha.sh ├── mysql_auto_install.py ├── mysql_processlist_monitor.py ├── mysql_replication_repair.py ├── mysql_skip_rpl_error.py ├── mysql_slow_log.sh └── mysql_tablespace.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.js linguist-language=python 2 | *.css linguist-language=python 3 | *.html linguist-language=python 4 | -------------------------------------------------------------------------------- /.idea/.name: -------------------------------------------------------------------------------- 1 | mysql_web -------------------------------------------------------------------------------- /.idea/bashsupport_project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 10 | -------------------------------------------------------------------------------- /.idea/codeStyleSettings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 14 | 16 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 35 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /.idea/mysql_web.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 17 | 18 | -------------------------------------------------------------------------------- /.idea/sqldialects.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mysql monitor web - MySQL实时监控 2 | 3 | # 安装环境： 4 | 1. 基于python2.7.11开发的 5 | 2. 安装MySQL数据库 6 | 3. 安装python第三方包 7 | ``` shell 8 | #更新setuptools 9 | wget http://pypi.python.org/packages/source/s/setuptools/setuptools-0.6c11.tar.gz 10 | tar -zxvf setuptools-0.6c11.tar.gz 11 | cd setuptools-0.6c11 12 | python setup.py build 13 | python setup.py install 14 | 15 | #更新pip 16 | wget https://pypi.python.org/packages/11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/pip-9.0.1.tar.gz#md5=35f01da33009719497f01a4ba69d63c9 17 | tar -zxvf pip-9.0.1.tar.gz 18 | cd pip-9.0.1 19 | python setup.py build 20 | python setup.py install 21 | 22 | #安装python包 23 | pip install flask flask-login gevent threadpool pymysql DBUtils six packaging appdirs mysql-replication sqlparse paramiko 24 | ``` 25 | 4. 在setting.py设置MySQL_Host相关账户信息 26 | ``` python 27 | MySQL_Host = host_info.HoseInfo(host="192.168.11.128", port=3306, user="yangcg", password="yangcaogui", remark="Monitor") 28 | ``` 29 | 5. 导入sql/table.sql的SQL脚本 30 | 6. 添加系统登录账号 31 | ``` mysql 32 | insert into mysql_web.mysql_web_user_info (user_name, user_password)values("yangcaogui", md5("123456")); 33 | ``` 34 | 7. 启动mysql web系统 35 | ``` shell 36 | #前台启动： 37 | python mysql_web.py runserver 38 | 39 | #后台启动 40 | nohup python mysql_web.py runserver & 41 | ``` 42 | 9. 如果要监控慢查询还要进行几步配置 43 | 44 | # 支持的功能: 45 | 1. mysql tps qps table_cache handler监控 46 | 2. 支持对innodb各种status进行监控 47 | 3. 支持对show engine innodb status分析 48 | 4. 支持对复制进行监控 49 | 5. 支持对表空间进行分析 50 | 7. 支持对os基本监控 51 | 8. 支持收集慢查询监控 52 | 9. 支持对thread进行完整分析 53 | 10. 支持实时的图表显示 54 | 11. 支持对数据库用户账号的查询 55 | 12. 支持登录验证，未登录不允许查看其它任何界面 56 | 13. 支持半同步复制的实时监控 57 | 58 | # 完成的脚本: 59 | 1. binlog_bk.py - 实现使用mysqlbinlog对binlog日志进行备份 60 | 2. binlog_sync.py - 实现对binlog进行分析，可以把数据同步到另一个实例中 61 | 3. binlog_util.py - 基于mysql-replication的binlog分析，可生成回滚SQL，实现误操作的闪回 62 | 4. binlog_util_new.py - 实现对binlog文件的分析，可生成回滚SQL 63 | 5. bk_xtrabackup.py - 实现对xtrabackup的备份封装，可以增量和全备 64 | 6. bk_recovery_xtrbackup.py - 是基于bk_xtrabackup.py实现的备份恢复，可以远程和本地恢复 65 | 7. collect_mysql_status_log.sh - 实现对mysql指定时间段的日志收集，有助于排除问题 66 | 8. mysql_auto_install.py - 实现mysql的远程自动安装 67 | 9. mysql_replication_repair.py - 实现对slave出现1032和1062错误的自动修复功能 68 | 10. mysql_slow_log.sh - 基于pt工具的慢查询收集脚本，需要和mysql_web一起使用 69 | 11. bk_xtrabackup_remote.py - 支持远程备份，比较强大 70 | 71 | # 联系方式 72 | 1. QQ: 779647966 73 | 2. Email: ycg166911@163.com 74 | 75 | # 界面展示: 76 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/111.png) 77 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/112.png) 78 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/113.png) 79 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/114.png) 80 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/115.png) 81 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/116.png) 82 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/117.png) 83 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/118.png) 84 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/119.png) 85 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/120.png) 86 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/121.png) 87 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/123.png) 88 | ![image](https://github.com/ycg/mysql_web/blob/master/static/img/124.png) 89 | -------------------------------------------------------------------------------- /backup/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Administrator' 2 | -------------------------------------------------------------------------------- /backup/backup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import json 4 | 5 | import settings 6 | from monitor import base_class, cache, common 7 | 8 | #备份 9 | #备份工具mysqldump dumper xtrabackup 10 | #备份方式增量还是全量或者全量+增量 11 | #备份时间每日几点 12 | #备份周期几号全量几号增量 13 | #备份完整性检查|检查备份是否完成是否出现错误 14 | 15 | #xtrabackup可以重定向日志，然后去检查日志，可以判断备份是否完成 16 | #备份是否可用，需要进行恢复才行，可以手动 17 | 18 | #独特的binlog备份 19 | #binlog备份可以使用mysqlbinlog进行 20 | 21 | #指定备份计划 22 | 23 | #1.备份名称 24 | #2.选择备份机器 25 | #3.选择备份工具 26 | #4.选择备份方式全量还是增量 27 | #5.选择备份时间-也就是备份周期，根据上面的来，如果是全量那就只有时间，全量+增量会选择周期 28 | #6.填写备份目录-会对这个目录进行检查 29 | #7.备份路径，备份文件 30 | #8.是否需要压缩-只有mysqldump才需要压缩，xtrabackup不需要 31 | 32 | #创建备份的时候要对备份进行检查 33 | #路径检查是否合法 34 | #备份存储的机器磁盘是否充足 35 | #是否已经有备份任务 36 | #还有检测是否已经安装过备份工具，三种工具要进行收集，然后统一上传到服务器上 37 | #检查时候已经有了备份计划了 38 | #发送成功和失败邮件 39 | 40 | #创建的备份计划也需要建个表 41 | #查看备份任务历史数据-需要建个表 42 | 43 | backup_infos = {} 44 | backup_tools = {1: "mydumper", 2: "mysqldump", 3: "xtrabackup"} 45 | 46 | 47 | def add_backup(info): 48 | result_info = check_backup_parameters(info) 49 | if(result_info.flag == 1): 50 | info.host_name = cache.Cache().get_host_info(int(info.backup_host_id)).remark 51 | backup_infos[info.backup_host_id] = info 52 | result_info.message = "add backup task success." 53 | return json.loads(result_info) 54 | 55 | def check_backup_parameters(info): 56 | result_info = base_class.BaseClass(None) 57 | result_info.flag = 1 58 | if(backup_infos.has_key(info.backup_host_id)): 59 | result_info.flag = 0 60 | result_info.message = "此主机的备份任务已经存在" 61 | elif(len(info.backup_name) <= 0): 62 | result_info.flag = 0 63 | result_info.message = "备份名称不能为空" 64 | elif(len(info.backup_time) <= 0): 65 | result_info.flag = 0 66 | result_info.message = "备份时间不能为空" 67 | return result_info 68 | 69 | 70 | def add_backup_task(obj): 71 | # 1.监测数据源是否正常 72 | 73 | # 2.如果是xtrabackup备份需要检测有没有安装 74 | if (obj.backup_tool == settings.BACKUP_TOOL_XTRABACKUP): 75 | stdin, stdout, stderr = common.execute_remote_command(obj.host_info, "xtrabackup --version") 76 | print(stdout.readlines(), stderr) 77 | 78 | # 3.创建备份目录 79 | pass 80 | 81 | -------------------------------------------------------------------------------- /backup/backup_base.py: -------------------------------------------------------------------------------- 1 | 2 | import time 3 | 4 | import settings 5 | from monitor import db_util, common 6 | 7 | class BackupBase(): 8 | def __init__(self): 9 | pass 10 | 11 | def backup(self, backup_info): 12 | pass 13 | 14 | def restore(self, restor_info): 15 | pass 16 | 17 | def get_current_time(self): 18 | return time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime(time.time())) 19 | 20 | def get_backup_info(self, backup_task_id): 21 | return common.get_object(db_util.DBUtil().fetchone(settings.MySQL_Host, 22 | "select * from mysql_web.backup_task where id = {0} and is_deleted = 0;".format(backup_task_id))) 23 | 24 | def insert_backup_log(self, backup_info): 25 | sql = """insert into mysql_web.backup_log 26 | (task_id, file, `size`, start_datetime, stop_datetime, status, result) 27 | VALUES 28 | ({0}, '{1}', {2}, '{3}', '{4}', {5}, '{6}')"""\ 29 | .format(backup_info.task_id, backup_info.file_name, backup_info.file_size, 30 | backup_info.start_datetime, backup_info.stop_datetime, backup_info.status, backup_info.result) 31 | db_util.DBUtil().execute(settings.MySQL_Host, sql) 32 | 33 | -------------------------------------------------------------------------------- /backup/backup_server.py: -------------------------------------------------------------------------------- 1 | 2 | import threading, time 3 | 4 | class BackupServer(threading.Thread): 5 | def __init__(self): 6 | threading.Thread.__init__(self) 7 | self.setDaemon(True) 8 | 9 | def run(self): 10 | while(True): 11 | time.sleep(1) 12 | 13 | 14 | -------------------------------------------------------------------------------- /backup/mydumper.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Administrator' 2 | -------------------------------------------------------------------------------- /backup/mysqldump.py: -------------------------------------------------------------------------------- 1 | 2 | import time, os 3 | import traceback 4 | from backup_base import BackupBase 5 | from monitor import cache, common 6 | import settings 7 | 8 | class MySQLDump(BackupBase): 9 | def __init__(self, task_id): 10 | self.task_id = task_id 11 | 12 | def backup(self, backup_info): 13 | try: 14 | backup_info.start_datetime = self.get_current_time() 15 | backup_info.file_path = os.path.join(backup_info.backup_directory, "{0}_{1}.gz".format(backup_info.host, self.get_current_time())) 16 | command = self.backup_command.format(backup_info.host_info.host, backup_info.host_info.user, 17 | backup_info.host_info.password, backup_info.host_info.port, backup_info.file_path) 18 | stdin, stdout, stderr = common.execute_remote_command(backup_info.backup_remote_host, command) 19 | backup_info.backup_status = settings.BACKUP_SUCCESS 20 | backup_info.backup_result = stdout.readline() 21 | backup_info.stop_datetime = self.get_current_time() 22 | self.insert_backup_log(backup_info) 23 | except: 24 | backup_info.stop_datetime = self.get_current_time() 25 | backup_info.backup_status = settings.BACKUP_ERROR 26 | self.insert_backup_log(backup_info) 27 | traceback.print_exc() 28 | 29 | def restore(self, restor_info): 30 | pass 31 | 32 | backup_command = "mysqldump -h{0} -u{1} -p{2} -P{3} " \ 33 | "--max-allowed-packet=1G --single-transaction --master-data=2 " \ 34 | "--default-character-set=utf8mb4 --triggers --routines --events -B --all-databases | gzip > {4}" 35 | 36 | restore_command = "mysql -h{0} -u{1} -p{2} -P{3} --default-character-set=utf8mb4 --max-allowed-packet=1G < {4}" 37 | 38 | 39 | 40 | 41 | print(os.path.join("/opt/", "aaa.txt")) 42 | -------------------------------------------------------------------------------- /backup/xtrabackup.py: -------------------------------------------------------------------------------- 1 | 2 | from backup_base import BackupBase 3 | from monitor import common 4 | 5 | 6 | class Xtrabackup(BackupBase): 7 | def __init__(self): 8 | pass 9 | 10 | 11 | def backup(self, backup_info): 12 | pass 13 | 14 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | yum install -y zip unzip python-devel libffi-devel wget gcc sysstat 4 | 5 | function install_package() 6 | { 7 | python setup.py build 8 | python setup.py install 9 | } 10 | 11 | #old setuptools 12 | cd /tmp 13 | file_name="setuptools-0.6c11.tar.gz" 14 | `rm -rf ${file_name}` 15 | wget http://pypi.python.org/packages/source/s/setuptools/setuptools-0.6c11.tar.gz 16 | `tar -zxvf ${file_name}` 17 | cd setuptools-0.6c11 18 | install_package 19 | 20 | #pip 21 | cd /tmp 22 | file_name="pip-9.0.1.tar.gz" 23 | `rm -rf ${file_name}` 24 | wget https://pypi.python.org/packages/11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/pip-9.0.1.tar.gz#md5=35f01da33009719497f01a4ba69d63c9 25 | `tar -zxvf ${file_name}` 26 | cd pip-9.0.1 27 | install_package 28 | 29 | #pip install package 30 | pip install flask flask-login gevent threadpool pymysql DBUtils six packaging appdirs mysql-replication sqlparse 31 | 32 | #new setuptools 33 | cd /tmp 34 | file_name="setuptools-35.0.2.zip" 35 | `rm -rf ${file_name}` 36 | wget https://pypi.python.org/packages/88/13/7d560b75334a8e4b4903f537b7e5a1ad9f1a2f1216e2587aaaf91b38c991/setuptools-35.0.2.zip#md5=c368b4970d3ad3eab5afe4ef4dbe2437 37 | `unzip -o ${file_name}` 38 | cd setuptools-35.0.2 39 | install_package 40 | pip install paramiko 41 | 42 | -------------------------------------------------------------------------------- /mha/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Administrator' 2 | -------------------------------------------------------------------------------- /mha/mha_check.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | 4 | def stop_mha(mha_config): 5 | command = "masterha_stop --conf={0}".format(mha_config.conf_path) 6 | os.system(command) 7 | 8 | def start_mha(mha_config): 9 | command = "masterha_manager --conf={0} --ignore_last_failover".format(mha_config.conf_path) 10 | os.system(command) 11 | 12 | def check_ssh(mha_config): 13 | command = "masterha_check_ssh --conf={0}".format(mha_config.conf_path) 14 | os.system(command) 15 | 16 | def check_repl(mha_config): 17 | command = "masterha_check_repl --conf={0}".format(mha_config.conf_path) 18 | os.system(command) 19 | 20 | def check_status(mha_config): 21 | command = "masterha_check_status --conf={0}".format(mha_config.conf_path) 22 | os.system(command) 23 | 24 | -------------------------------------------------------------------------------- /monitor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/monitor/__init__.py -------------------------------------------------------------------------------- /monitor/binlog_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import cache 4 | import datetime, random, time, traceback 5 | from entitys import Entity 6 | from pymysqlreplication import BinLogStreamReader 7 | from pymysqlreplication.row_event import WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent 8 | 9 | connection_settings = {} 10 | insert_sql = "INSERT INTO `{0}`.`{1}` ({2}) VALUES ({3});" 11 | update_sql = "UPDATE `{0}`.`{1}` set {2} WHERE {3};" 12 | delete_sql = "DELETE FROM `{0}`.`{1}` WHERE {2};" 13 | 14 | 15 | def sql_format(dic, split_value): 16 | list = [] 17 | for key, value in dic.items(): 18 | if (value == None): 19 | list.append("`%s`=NULL" % key) 20 | continue 21 | if (isinstance(value, int) or isinstance(value, float) or isinstance(value, long)): 22 | list.append("`%s`=%d" % (key, value)) 23 | else: 24 | list.append("`%s`='%s'" % (key, value)) 25 | return split_value.join(list) 26 | 27 | 28 | def sql_format_for_insert(values): 29 | list = [] 30 | for value in values: 31 | if (value == None): 32 | list.append("NULL") 33 | continue 34 | if (isinstance(value, int) or isinstance(value, float) or isinstance(value, long)): 35 | list.append('%d' % value) 36 | else: 37 | list.append('\'%s\'' % value) 38 | return ', '.join(list) 39 | 40 | 41 | def get_binlog(obj): 42 | args = Entity() 43 | args.flashback = False 44 | args.log_file = obj.log_file 45 | args.server_id = random.randrange(66666, 99999) 46 | args.start_pos = obj.start_pos if obj.start_pos else 4 47 | args.stop_pos = obj.stop_pos if obj.stop_pos else None 48 | args.stop_datetime = datetime.datetime.strptime(obj.stop_datetime, "%Y-%m-%d %H:%M:%S") if obj.stop_datetime else None 49 | args.start_datetime = datetime.datetime.strptime(obj.start_datetime, "%Y-%m-%d %H:%M:%S") if obj.start_datetime else None 50 | args.stop_datetime_timestamp = time.mktime(args.stop_datetime.timetuple()) if obj.stop_datetime else 0 51 | args.start_datetime_timestamp = time.mktime(args.start_datetime.timetuple()) if obj.start_datetime else 0 52 | host_info = cache.Cache().get_host_info(obj.host_id) 53 | args.connection_settings = {"host": host_info.host, "port": host_info.port, "user": host_info.user, "passwd": host_info.password} 54 | 55 | message = check_args(args) 56 | if (len(message) > 0): 57 | return message 58 | 59 | return binlog_process(args) 60 | 61 | 62 | def check_args(args): 63 | message = "" 64 | if (args.start_datetime == None and args.stop_datetime == None): 65 | if (args.stop_pos == None): 66 | message = "stop position不允许为空值" 67 | else: 68 | if (args.stop_pos < args.start_pos): 69 | message = "stop position不允许比start position值小" 70 | elif (args.start_pos > args.stop_pos): 71 | message = "start position不允许比stop position值大" 72 | else: 73 | if (args.start_datetime == None and args.stop_datetime != None): 74 | message = "start datetime不允许为空值" 75 | elif (args.start_datetime != None and args.stop_datetime == None): 76 | message = "stop datetime不允许为空值" 77 | else: 78 | if (args.stop_datetime_timestamp < args.start_datetime_timestamp): 79 | message = "stop datetime不允许比start datetime值小" 80 | elif (args.start_datetime_timestamp > args.stop_datetime_timestamp): 81 | message = "start datetime不允许比stop datetime值大" 82 | return message 83 | 84 | 85 | def binlog_process(args): 86 | stream = None 87 | sql_list = [] 88 | try: 89 | stream = BinLogStreamReader(connection_settings=args.connection_settings, log_file=args.log_file, log_pos=args.start_pos, server_id=args.server_id) 90 | 91 | for binlogevent in stream: 92 | if (args.log_file != stream.log_file): 93 | break 94 | 95 | if (args.stop_pos != None): 96 | if (int(binlogevent.packet.log_pos) > int(args.stop_pos)): 97 | break 98 | 99 | if (args.start_datetime != None): 100 | if (binlogevent.timestamp < args.start_datetime_timestamp): 101 | continue 102 | 103 | if (args.stop_datetime != None): 104 | if (binlogevent.timestamp > args.stop_datetime_timestamp): 105 | break 106 | 107 | if (isinstance(binlogevent, WriteRowsEvent)): 108 | for row in binlogevent.rows: 109 | if (args.flashback): 110 | sql_list.append(delete_to_sql(row, binlogevent) + "\n") 111 | else: 112 | sql_list.append(insert_to_sql(row, binlogevent) + "\n") 113 | elif (isinstance(binlogevent, DeleteRowsEvent)): 114 | for row in binlogevent.rows: 115 | if (args.flashback): 116 | sql_list.append(insert_to_sql(row, binlogevent) + "\n") 117 | else: 118 | sql_list.append(delete_to_sql(row, binlogevent) + "\n") 119 | elif (isinstance(binlogevent, UpdateRowsEvent)): 120 | for row in binlogevent.rows: 121 | sql_list.append(update_to_sql(row, binlogevent, args.flashback) + "\n") 122 | except Exception, e: 123 | traceback.print_exc() 124 | raise e 125 | finally: 126 | if (stream != None): 127 | stream.close() 128 | return "".join(sql_list) 129 | 130 | 131 | def insert_to_sql(row, binlogevent): 132 | return insert_sql.format(binlogevent.schema, binlogevent.table, ', '.join(map(lambda k: '`%s`' % k, row['values'].keys())), sql_format_for_insert(row["values"].values())) 133 | 134 | 135 | def delete_to_sql(row, binlogevent): 136 | return delete_sql.format(binlogevent.schema, binlogevent.table, sql_format(row['values'], " AND ")) 137 | 138 | 139 | def update_to_sql(row, binlogevent, flashback): 140 | if (flashback): 141 | return update_sql.format(binlogevent.schema, binlogevent.table, sql_format(row['before_values'], ", "), sql_format(row['after_values'], " AND ")) 142 | else: 143 | return update_sql.format(binlogevent.schema, binlogevent.table, sql_format(row['after_values'], ", "), sql_format(row['before_values'], " AND ")) 144 | -------------------------------------------------------------------------------- /monitor/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import paramiko, subprocess, traceback, json, datetime, time 4 | from entitys import BaseClass 5 | import db_util 6 | 7 | 8 | def convert_object_json(obj): 9 | if isinstance(obj, datetime): 10 | return obj.strftime('%Y-%m-%d %H:%M:%S') 11 | elif isinstance(obj, time.date): 12 | return obj.strftime('%Y-%m-%d') 13 | else: 14 | return obj.__dict__ 15 | 16 | 17 | # 把数据库返回数据转换为对象 18 | def get_object(row): 19 | info = BaseClass(None) 20 | for key, value in row.items(): 21 | if (value == "None"): 22 | value = None 23 | setattr(info, key.lower(), value) 24 | return info 25 | 26 | 27 | # 把数据库返回数据转换为对象集合 28 | def get_object_list(rows): 29 | info_list = [] 30 | for row in rows: 31 | info = BaseClass(None) 32 | for key, value in row.items(): 33 | if (value == "None"): 34 | value = None 35 | setattr(info, key.lower(), value) 36 | info_list.append(info) 37 | return info_list 38 | 39 | 40 | # 转换对象为json数据 41 | def convert_obj_to_json_str(obj): 42 | return json.dumps(obj, default=lambda o: o.__dict__) 43 | #return json.dumps(obj, default=convert_obj_to_json_str(xxx)) 44 | #return json.dumps(obj, default=lambda o: o.strftime("%Y-%m-%d %H:%M:%S") if (isinstance(o, datetime)) else o.__dict__) 45 | 46 | 47 | # 执行本地命令 48 | def execute_localhost_command(command): 49 | result = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) 50 | result.wait() 51 | return result.stdin, result.stdout, result.stderr 52 | 53 | 54 | # ssh执行远程命令 55 | def execute_remote_command(host_info, command): 56 | host_client = None 57 | try: 58 | host_client = paramiko.SSHClient() 59 | host_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 60 | host_client.connect(host_info.host, port=host_info.ssh_port, username=host_info.ssh_user, password=host_info.ssh_password, timeout=1) 61 | stdin, stdout, stderr = host_client.exec_command(command) 62 | return stdin, stdout, stderr 63 | finally: 64 | if (host_client == None): 65 | host_client.close() 66 | 67 | 68 | # 测试SSH连接是否OK 69 | def test_ssh_connection_is_ok(obj): 70 | try: 71 | host_info = BaseClass(None) 72 | host_info.host = obj.host_ip 73 | host_info.ssh_port = obj.host_ssh_port 74 | host_info.ssh_user = obj.host_ssh_user 75 | host_info.ssh_password = obj.host_ssh_password if(len(obj.host_ssh_password) > 0) else None 76 | execute_remote_command(host_info, "df -h") 77 | except: 78 | traceback.print_exc() 79 | return False 80 | return True 81 | 82 | 83 | # 测试MySQL连接是否OK 84 | def test_mysql_connection_is_ok(obj): 85 | try: 86 | db_util.DBUtil().execute_sql(obj.host_ip, obj.host_port, obj.host_user, obj.host_password, "select 1;") 87 | except Exception: 88 | traceback.print_exc() 89 | return False 90 | return True 91 | 92 | 93 | -------------------------------------------------------------------------------- /monitor/custom_algorithm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | def encrypt(key, value): 4 | if (len(str(value)) <= 0): 5 | return str(value) 6 | b = bytearray(str(value).encode("utf8")) 7 | n = len(b) 8 | c = bytearray(n * 2) 9 | j = 0 10 | for i in range(0, n): 11 | b1 = b[i] 12 | b2 = b1 ^ key 13 | c1 = b2 % 16 14 | c2 = b2 // 16 15 | c1 += 65 16 | c2 += 65 17 | c[j] = c1 18 | c[j + 1] = c2 19 | j += 2 20 | return c.decode("utf8") 21 | 22 | 23 | def decrypt(key, value): 24 | c = bytearray(str(value).encode("utf8")) 25 | n = len(c) 26 | if n % 2 != 0: 27 | return "" 28 | n //= 2 29 | b = bytearray(n) 30 | j = 0 31 | for i in range(0, n): 32 | c1 = c[j] 33 | c2 = c[j + 1] 34 | j += 2 35 | c1 -= 65 36 | c2 -= 65 37 | b2 = c2 * 16 + c1 38 | b1 = b2 ^ key 39 | b[i] = b1 40 | return b.decode("utf8") 41 | 42 | 43 | if (__name__ == "__main__"): 44 | key = 20 45 | s1 = encrypt(key, "") 46 | s2 = decrypt(key, s1) 47 | print s1, '\n', s2 48 | -------------------------------------------------------------------------------- /monitor/db_util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pymysql 4 | from entitys import BaseClass 5 | from DBUtils.PooledDB import PooledDB 6 | 7 | 8 | class DBUtil(object): 9 | __instance = None 10 | __connection_pools = {} 11 | 12 | def __init__(self): 13 | pass 14 | 15 | def __new__(cls, *args, **kwargs): 16 | if (DBUtil.__instance is None): 17 | DBUtil.__instance = object.__new__(cls, *args, **kwargs) 18 | return DBUtil.__instance 19 | 20 | def execute(self, host_info, sql): 21 | connection, cursor = None, None 22 | try: 23 | connection, cursor = self.execute_for_db(host_info, sql) 24 | finally: 25 | self.close(connection, cursor) 26 | 27 | def fetchone(self, host_info, sql): 28 | connection, cursor = None, None 29 | try: 30 | connection, cursor = self.execute_for_db(host_info, sql) 31 | return cursor.fetchone() 32 | finally: 33 | self.close(connection, cursor) 34 | 35 | def fetchall(self, host_info, sql): 36 | connection, cursor = None, None 37 | try: 38 | connection, cursor = self.execute_for_db(host_info, sql) 39 | return cursor.fetchall() 40 | finally: 41 | self.close(connection, cursor) 42 | 43 | def execute_for_cursor(self, sql, connection=None, cursor=None): 44 | return self.cursor_execute(connection, cursor, sql) 45 | 46 | def fetchone_for_cursor(self, sql, connection=None, cursor=None): 47 | cursor = self.cursor_execute(connection, cursor, sql) 48 | return cursor.fetchone() 49 | 50 | def fetchall_for_cursor(self, sql, connection=None, cursor=None): 51 | cursor = self.cursor_execute(connection, cursor, sql) 52 | return cursor.fetchall() 53 | 54 | def cursor_execute(self, connection, cursor, sql): 55 | if (cursor == None): 56 | cursor = connection.cursor() 57 | cursor.execute(sql) 58 | return cursor 59 | 60 | def close(self, connection, cursor): 61 | if (cursor != None): 62 | cursor.close() 63 | if (connection != None): 64 | connection.commit() 65 | connection.close() 66 | 67 | def execute_for_db(self, host_info, sql): 68 | connection, cursor = self.get_conn_and_cur(host_info) 69 | cursor.execute(sql) 70 | return connection, cursor 71 | 72 | def get_conn_and_cur(self, host_info): 73 | connection = self.get_mysql_connection(host_info) 74 | cursor = connection.cursor() 75 | return connection, cursor 76 | 77 | def get_mysql_connection(self, host_info): 78 | if (self.__connection_pools.get(host_info.key) == None): 79 | pool = PooledDB(creator=pymysql, mincached=4, maxcached=8, maxconnections=10, 80 | host=host_info.host, port=host_info.port, user=host_info.user, passwd=host_info.password, 81 | use_unicode=False, charset="utf8", cursorclass=pymysql.cursors.DictCursor, reset=False, autocommit=True, 82 | connect_timeout=1, read_timeout=10, write_timeout=10) 83 | self.__connection_pools[host_info.key] = pool 84 | return self.__connection_pools[host_info.key].connection() 85 | 86 | def get_list_infos(self, host_info, sql): 87 | result = [] 88 | for row in self.fetchall(host_info, sql): 89 | info = BaseClass(None) 90 | for key, value in row.items(): 91 | setattr(info, key, value) 92 | result.append(info) 93 | return result 94 | 95 | def get_list_infos_to_lower(self, host_info, sql): 96 | result = [] 97 | for row in self.fetchall(host_info, sql): 98 | info = BaseClass(None) 99 | for key, value in row.items(): 100 | setattr(info, key.lower(), value) 101 | result.append(info) 102 | return result 103 | 104 | def escape(self, string): 105 | return pymysql.escape_string(string) 106 | 107 | def execute_sql(self, ip, port, user, password, sql): 108 | connection, cursor = None, None 109 | try: 110 | connection = pymysql.connect(host=ip, user=user, passwd=password, port=port, use_unicode=True, charset="utf8", connect_timeout=1) 111 | cursor = connection.cursor() 112 | cursor.execute(sql) 113 | finally: 114 | if (cursor != None): 115 | cursor.close() 116 | if (connection != None): 117 | connection.close() 118 | -------------------------------------------------------------------------------- /monitor/entitys.py: -------------------------------------------------------------------------------- 1 | class BaseClass(object): 2 | trx_count = 0 3 | old_trx_count = 0 4 | new_trx_count = 0 5 | history_list_length = 0 6 | net_send_new = 0 7 | net_receive_new = 0 8 | thread_waits_count = 0 9 | 10 | def __init__(self, host_info): 11 | self.host_info = host_info 12 | 13 | class HoseInfo(): 14 | def __init__(self, host="", port=3306, user="", password="", remark=""): 15 | self.host = host 16 | self.port = port 17 | self.user = user 18 | self.remark = remark 19 | self.password = password 20 | self.key = host + ":" + str(port) 21 | 22 | 23 | class Entity(): 24 | def __init__(self): 25 | pass 26 | 27 | 28 | -------------------------------------------------------------------------------- /monitor/mail_util.py: -------------------------------------------------------------------------------- 1 | import smtplib, settings 2 | from email.mime.text import MIMEText 3 | 4 | def send_text(subject, to_list, content): 5 | send_mail(subject, to_list, content, "plain") 6 | 7 | def send_html(subject, to_list, content): 8 | send_mail(subject, to_list, content, "html") 9 | 10 | def send_mail(subject, to_list, content, mail_type): 11 | list_t = [] 12 | server = None 13 | if(isinstance(to_list, list) == False): 14 | list_t.append(to_list) 15 | try: 16 | message = MIMEText(content, _subtype=mail_type, _charset="utf8") 17 | message['Subject'] = subject 18 | message['To'] = ";".join(list_t) 19 | message['From'] = settings.EMAIL_USER 20 | 21 | server = smtplib.SMTP() 22 | server.connect(settings.EMAIL_HOST) 23 | server.login(settings.EMAIL_USER, settings.EMAIL_PASSWORD) 24 | server.sendmail(settings.EMAIL_USER, list_t, message.as_string()) 25 | finally: 26 | if(server != None): 27 | server.close() -------------------------------------------------------------------------------- /monitor/mysql_enum.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | class MySQLBranch(enum.Enum): 4 | MySQL = 0 5 | Percona = 1 6 | Mariadb = 2 7 | 8 | class MonitorEnum(enum.Enum): 9 | mysql = 4 10 | host = 3 11 | status = 0 12 | innodb = 1 13 | replication = 2 -------------------------------------------------------------------------------- /monitor/report.py: -------------------------------------------------------------------------------- 1 | import time 2 | from flask import render_template 3 | 4 | import cache, settings, mail_util 5 | 6 | def send_report_everyday(): 7 | html_str = render_template("report.html", tablespace_infos=cache.Cache().get_all_tablespace_infos()) 8 | subject = "MySQL Report - {0}".format(time.strftime("%Y-%m-%d", time.localtime(time.time()))) 9 | mail_util.send_html(subject, settings.EMAIL_SEND_USERS, html_str) 10 | print("send report ok") 11 | 12 | -------------------------------------------------------------------------------- /monitor/tablespace_new.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import time 4 | import db_util, settings, cache, common, entitys 5 | 6 | 7 | def get_table_infos(host_info): 8 | table_infos = {} 9 | db_names = db_util.DBUtil().fetchall(host_info, "show databases;") 10 | for db_name in db_names: 11 | sql = """select table_schema, table_name, DATA_LENGTH, INDEX_LENGTH, TABLE_ROWS, AUTO_INCREMENT, create_time, engine, update_time 12 | from information_schema.tables 13 | where table_schema = '{0}' 14 | and table_schema != 'mysql' and table_schema != 'information_schema' and table_schema != 'performance_schema' and table_schema != 'sys'""".format(db_name["Database"]) 15 | for row in db_util.DBUtil().fetchall(host_info, sql): 16 | table_info = entitys.Entity() 17 | table_info.schema = row["table_schema"] 18 | table_info.t_name = row["table_name"] 19 | table_info.rows = row["TABLE_ROWS"] 20 | table_info.data_size = row["DATA_LENGTH"] if row["DATA_LENGTH"] else 0 21 | table_info.index_size = row["INDEX_LENGTH"] if row["INDEX_LENGTH"] else 0 22 | table_info.auto_increment = row["AUTO_INCREMENT"] if row["AUTO_INCREMENT"] else 0 23 | table_info.total_size = long(table_info.data_size) + long(table_info.index_size) 24 | table_info.create_time = row["create_time"] 25 | table_info.update_time = row["update_time"] if row["update_time"] else '' 26 | table_info.engine = row["engine"] 27 | table_name = row["table_schema"] + "." + row["table_name"] 28 | table_infos[table_name] = table_info 29 | return table_infos 30 | -------------------------------------------------------------------------------- /monitor/user_login.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import hashlib 4 | from flask_login import UserMixin 5 | 6 | import cache 7 | 8 | class User(UserMixin): 9 | def __init__(self, username): 10 | self.username = username 11 | self.id = self.get_id() 12 | self.password = self.get_password() 13 | 14 | def verify_password(self, password, result): 15 | if self.id is None or self.password is None: 16 | result.error = "用户名不正确!" 17 | return False 18 | if(self.password == self.get_value_for_md5(password)): 19 | result.success = "ok" 20 | return True 21 | result.error = "密码不正确!" 22 | return False 23 | 24 | def get_password(self): 25 | return self.get_user_info_by_user_name(self.username, "user_password") 26 | 27 | def get_id(self): 28 | return self.get_user_info_by_user_name(self.username, "user_id") 29 | 30 | def get_user_info_by_user_name(self, user_name, attr_name): 31 | list_tmp = cache.Cache().get_mysql_web_user_infos() 32 | for info in list_tmp: 33 | if(info.user_name == user_name): 34 | value = getattr(info, attr_name) 35 | return value 36 | return None 37 | 38 | @staticmethod 39 | def get(user_id): 40 | if not user_id: 41 | return None 42 | user_info = cache.Cache().get_mysql_web_user_infos(user_id) 43 | if(user_info != None): 44 | return User(user_info.user_name) 45 | return None 46 | 47 | def get_value_for_md5(self, password): 48 | obj = hashlib.md5() 49 | obj.update(password) 50 | return obj.hexdigest() 51 | -------------------------------------------------------------------------------- /python_upgrade.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | #python2.6升级到2.7一键安装脚本 4 | 5 | echo "---------------1.download python2.7 package-----------------------" 6 | cd /tmp 7 | file_name="Python-2.7.12.tgz" 8 | `rm -rf ${file_name}` 9 | wget https://www.python.org/ftp/python/2.7.12/Python-2.7.12.tgz 10 | `tar -zxvf ${file_name}` 11 | 12 | echo "--------------------2.build and install---------------------------" 13 | cd Python-2.7.12 14 | ./configure 15 | make all 16 | make install 17 | make clean 18 | make distclean 19 | 20 | echo "----------------------3.python version---------------------------" 21 | /usr/local/bin/python2.7 -V 22 | 23 | echo "----------------------4.modify link------------------------------" 24 | mv /usr/bin/python /usr/bin/python2.6.6 25 | ln -s /usr/local/bin/python2.7 /usr/bin/python 26 | 27 | echo "---------------------5.python new version------------------------" 28 | python -V 29 | 30 | echo "--------------------6.modify yum config--------------------------" 31 | sed '1/\!\/usr\/bin\/python/\!\/usr\/bin\/python2.6/g' /usr/bin/yum 32 | 33 | echo "----------------------success------------------------------------" 34 | 35 | #blog 36 | #http://www.cnblogs.com/emanlee/p/6111613.html 37 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import platform 4 | from monitor.entitys import HoseInfo 5 | 6 | MySQL_Host = HoseInfo(host="192.168.11.101", port=3306, user="yangcg", password="yangcaogui", remark="Monitor") 7 | 8 | ONE_DAY = 24 * 60 * 60 9 | UPDATE_INTERVAL = 4 10 | THREAD_POOL_SIZE = 50 11 | REPORT_INTERVAL = ONE_DAY 12 | TABLE_CHECK_INTERVAL = ONE_DAY 13 | 14 | EMAIL_HOST = "" 15 | EMAIL_PORT = 25 16 | EMAIL_USER = "" 17 | EMAIL_PASSWORD = "" 18 | EMAIL_SEND_USERS = "" 19 | 20 | LINUX_OS = 'Linux' in platform.system() 21 | WINDOWS_OS = 'Windows' in platform.system() 22 | 23 | BACKUP_ING = 1 24 | BACKUP_SUCCESS = 2 25 | BACKUP_ERROR = 3 26 | 27 | BACKUP_TOOL_MYDUMPER = 1 28 | BACKUP_TOOL_MYSQLDUMP = 2 29 | BACKUP_TOOL_XTRABACKUP = 3 30 | 31 | BACKUP_MODE_FULL = 1 32 | BACKUP_MODE_INCREMENT = 2 33 | 34 | IS_INSERT_MONITOR_LOG = False 35 | 36 | MY_KEY = 20 37 | 38 | -------------------------------------------------------------------------------- /sql/table.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/sql/table.sql -------------------------------------------------------------------------------- /startup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | python mysql_web.py runserver -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /static/img/111.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/111.png -------------------------------------------------------------------------------- /static/img/112.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/112.png -------------------------------------------------------------------------------- /static/img/113.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/113.png -------------------------------------------------------------------------------- /static/img/114.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/114.png -------------------------------------------------------------------------------- /static/img/115.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/115.png -------------------------------------------------------------------------------- /static/img/116.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/116.png -------------------------------------------------------------------------------- /static/img/117.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/117.png -------------------------------------------------------------------------------- /static/img/118.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/118.png -------------------------------------------------------------------------------- /static/img/119.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/119.png -------------------------------------------------------------------------------- /static/img/120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/120.png -------------------------------------------------------------------------------- /static/img/121.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/121.png -------------------------------------------------------------------------------- /static/img/123.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/123.png -------------------------------------------------------------------------------- /static/img/124.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/img/124.png -------------------------------------------------------------------------------- /static/js/jscharts.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/static/js/jscharts.js -------------------------------------------------------------------------------- /static/js/md5.min.js: -------------------------------------------------------------------------------- 1 | !function(a){"use strict";function b(a,b){var c=(65535&a)+(65535&b),d=(a>>16)+(b>>16)+(c>>16);return d<<16|65535&c}function c(a,b){return a<>>32-b}function d(a,d,e,f,g,h){return b(c(b(b(d,a),b(f,h)),g),e)}function e(a,b,c,e,f,g,h){return d(b&c|~b&e,a,b,f,g,h)}function f(a,b,c,e,f,g,h){return d(b&e|c&~e,a,b,f,g,h)}function g(a,b,c,e,f,g,h){return d(b^c^e,a,b,f,g,h)}function h(a,b,c,e,f,g,h){return d(c^(b|~e),a,b,f,g,h)}function i(a,c){a[c>>5]|=128<>>9<<4)+14]=c;var d,i,j,k,l,m=1732584193,n=-271733879,o=-1732584194,p=271733878;for(d=0;d>5]>>>b%32&255);return c}function k(a){var b,c=[];for(c[(a.length>>2)-1]=void 0,b=0;b>5]|=(255&a.charCodeAt(b/8))<16&&(e=i(e,8*a.length)),c=0;16>c;c+=1)f[c]=909522486^e[c],g[c]=1549556828^e[c];return d=i(f.concat(k(b)),512+8*b.length),j(i(g.concat(d),640))}function n(a){var b,c,d="0123456789abcdef",e="";for(c=0;c>>4&15)+d.charAt(15&b);return e}function o(a){return unescape(encodeURIComponent(a))}function p(a){return l(o(a))}function q(a){return n(p(a))}function r(a,b){return m(o(a),o(b))}function s(a,b){return n(r(a,b))}function t(a,b,c){return b?c?r(b,a):s(b,a):c?p(a):q(a)}"function"==typeof define&&define.amd?define(function(){return t}):a.md5=t}(this); -------------------------------------------------------------------------------- /static/js/my.js: -------------------------------------------------------------------------------- 1 | var timer = ""; 2 | var host_ids = new Array(); 3 | var my_url = "/mysql"; 4 | 5 | var post_url = ["/mysql", "/status", "/innodb", "/replication", "/os"] 6 | 7 | function myrefresh() { 8 | if ($.inArray(my_url, post_url) >= 0) { 9 | $.post(my_url, {"keys": JSON.stringify(host_ids)}, function (data) { 10 | var login_flag = data.substring(0, 25); 11 | if (login_flag == "") { 12 | window.location.href = 'login'; 13 | } 14 | else { 15 | $("#data").html(ungzip_new(data)); 16 | } 17 | }); 18 | 19 | /*$.post(my_url, {"keys": JSON.stringify(host_ids)}).success(function (data, status, xhr) { 20 | console.log(xhr); 21 | $("#data").html(ungzip_new(data)); 22 | }).success(function (data, status, xhr) { 23 | console.log(xhr); 24 | });*/ 25 | } 26 | else { 27 | $.get(my_url, "", function (data) { 28 | $("#data").html(data); 29 | }); 30 | } 31 | } 32 | 33 | function get_data(url) { 34 | var result_data = ""; 35 | $.get(url, "", function (data) { 36 | result_data = data; 37 | }); 38 | return result_data 39 | } 40 | 41 | function post_data(url, input_data) { 42 | var result_data = ""; 43 | $.post(url, {"keys": input_data}, function (data) { 44 | result_data = data 45 | }); 46 | return result_data 47 | } 48 | 49 | function show_modal_dialog(id_name) { 50 | $(id_name).modal("show") 51 | } 52 | 53 | function hide_modal_dialog(id_name) { 54 | $(id_name).modal("hide") 55 | } 56 | 57 | function stop(parameter, id_name) { 58 | search_div(id_name) 59 | myActive(id_name) 60 | my_url = parameter 61 | myrefresh() 62 | stop_timer() 63 | } 64 | 65 | function start(parameter, id_name) { 66 | search_div(id_name) 67 | myActive(id_name) 68 | my_url = parameter 69 | myrefresh() 70 | start_timer() 71 | } 72 | 73 | function start_timer() { 74 | stop_timer() 75 | timer = setInterval(myrefresh, interval_refresh) 76 | } 77 | 78 | function stop_timer() { 79 | clearInterval(timer) 80 | } 81 | 82 | myrefresh() 83 | start_timer() 84 | 85 | function myActive(id_name) { 86 | if (id_name == "") { 87 | return false 88 | } 89 | $("#myTab").find("a").each(function () { 90 | if ($(this).attr("id") == id_name) { 91 | $(this).addClass("active"); 92 | } else { 93 | $(this).removeClass("active"); 94 | } 95 | }); 96 | } 97 | 98 | var id_names = ["sql", "slowlog", "tablespace", "general", "user", "thread", "chart", "config", "backup", "mysql_log", "chart_new", "host", "binlog", "alarm"] 99 | 100 | function search_div(id_name) { 101 | if ($.inArray(id_name, id_names) >= 0) { 102 | $("#host_search_div").hide(); 103 | } 104 | else { 105 | $("#host_search_div").show(); 106 | } 107 | //$("#host_search_div").hide() 108 | } 109 | 110 | function set_select_ids() { 111 | host_ids = $("#host_search").val(); 112 | if (host_ids == null || host_ids == 0) { 113 | host_ids = new Array(); 114 | } 115 | } 116 | 117 | function reset_select_ids() { 118 | $("#host_search").each(function () { 119 | $(this).selectpicker('val', $(this).find('option:first').val()); 120 | $(this).find("option").attr("selected", false); 121 | $(this).find("option:first").attr("selected", true); 122 | }); 123 | host_ids = new Array() 124 | } 125 | 126 | function gzip_new(string) { 127 | var charData = string.split('').map(function (x) { 128 | return x.charCodeAt(0); 129 | }); 130 | var binData = new Uint8Array(charData); 131 | var data = pako.gzip(binData); 132 | var strData = String.fromCharCode.apply(null, new Uint16Array(data)); 133 | return btoa(strData); 134 | } 135 | 136 | function ungzip_new(string) { 137 | var strData = atob(string); 138 | var charData = strData.split('').map(function (x) { 139 | return x.charCodeAt(0); 140 | }); 141 | var binData = new Uint8Array(charData); 142 | var data = pako.ungzip(binData); 143 | var strData = String.fromCharCode.apply(null, new Uint16Array(data)); 144 | return strData; 145 | } 146 | 147 | function ajaxSubmit(frm, fn, efn) { 148 | var dataPara = get_form_json(frm); 149 | $.ajax({ 150 | url: frm.action, 151 | type: frm.method, 152 | data: dataPara, 153 | success: fn, 154 | error: efn 155 | }); 156 | } 157 | 158 | function get_form_json(frm) { 159 | var o = {}; 160 | var a = $(frm).serializeArray(); 161 | $.each(a, function () { 162 | if (o[this.name] !== undefined) { 163 | if (!o[this.name].push) { 164 | o[this.name] = [o[this.name]]; 165 | } 166 | o[this.name].push(this.value || ''); 167 | } else { 168 | o[this.name] = this.value || ''; 169 | } 170 | }); 171 | return o; 172 | } 173 | 174 | $(function () { 175 | $(document).on('click', '.accordion-toggle', function (event) { 176 | event.stopPropagation(); 177 | var $this = $(this); 178 | var parent = $this.data('parent'); 179 | var actives = parent && $(parent).find('.collapse.in'); 180 | 181 | // From bootstrap itself 182 | if (actives && actives.length) { 183 | actives.data('collapse'); 184 | actives.collapse('hide'); 185 | } 186 | 187 | var target = $this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, ''); //strip for ie7 188 | $(target).collapse('toggle'); 189 | }); 190 | }) 191 | 192 | function changeFeedback(id) { 193 | var str = document.getElementById(id).className; 194 | var tag = str.substring(25, str.length); 195 | if (tag == "right") { 196 | document.getElementById(id).className = "glyphicon glyphicon-menu-down"; 197 | } else { 198 | document.getElementById(id).className = "glyphicon glyphicon-menu-right"; 199 | } 200 | } 201 | 202 | $("button[type='reset']").click(function () { 203 | $('input').attr("value", ''); 204 | $("textarea").val(""); 205 | $("select.selectpicker").each(function () { 206 | $(this).selectpicker('val', $(this).find('option:first').val()); 207 | $(this).find("option").attr("selected", false); 208 | $(this).find("option:first").attr("selected", true); 209 | $(this).val(0) 210 | }); 211 | }); 212 | 213 | function skip_slave_error(host_id) { 214 | 215 | } 216 | 217 | function kill_mysql_thread_id(host_id, thread_id) { 218 | if (window.confirm("确认kill掉?")) { 219 | $.post("/mysql/kill/" + host_id + "/" + thread_id, "", function (data) { 220 | alert(data); 221 | myrefresh(); 222 | }); 223 | } 224 | } 225 | 226 | function post_request(url, json_data) { 227 | $.post(url, json_data, function (data) { 228 | alert(data) 229 | }); 230 | } 231 | 232 | function logout() { 233 | if (window.confirm("是否确认退出?")) { 234 | $.post("/logout", "", function (data) { 235 | alert("logout ok!") 236 | }); 237 | } 238 | } 239 | 240 | function input_data_for_post(url, json_data, div_id) { 241 | $.post(url, json_data, function (data) { 242 | $(div_id).html(data) 243 | }); 244 | } 245 | 246 | -------------------------------------------------------------------------------- /static/js/npm.js: -------------------------------------------------------------------------------- 1 | // This file is autogenerated via the `commonjs` Grunt task. You can require() this file in a CommonJS environment. 2 | require('../../js/transition.js') 3 | require('../../js/alert.js') 4 | require('../../js/button.js') 5 | require('../../js/carousel.js') 6 | require('../../js/collapse.js') 7 | require('../../js/dropdown.js') 8 | require('../../js/modal.js') 9 | require('../../js/tooltip.js') 10 | require('../../js/popover.js') 11 | require('../../js/scrollspy.js') 12 | require('../../js/tab.js') 13 | require('../../js/affix.js') -------------------------------------------------------------------------------- /templates/alarm_config.html: -------------------------------------------------------------------------------- 1 | 9 | 10 |
11 |
12 |
13 |
14 | 全局配置 15 |
16 |
17 | 23 |
24 | -------------------------------------------------------------------------------- /templates/alarm_log.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
9 |
10 |
11 | 12 | 13 | 14 | 17 | 20 | 23 | 26 | 29 | 32 | 35 | 38 | 41 | 42 | 43 | 44 | {% for data in mysql_infos %} 45 | 46 | 49 | 52 | 55 | 58 | 61 | 64 | 67 | 70 | 73 | 74 | {% endfor %} 75 | 76 |
15 | Remark 16 | 18 | QPS 19 | 21 | TPS 22 | 24 | Trxs 25 | 27 | Uptime(Day) 28 | 30 | Version 31 | 33 | Port 34 | 36 | User 37 | 39 | 40 |
47 | {{ data.remark }} 48 | 50 | {{ data.qps }} 51 | 53 | {{ data.tps }} 54 | 56 | {{ data.trxs }} 57 | 59 | {{ data.uptime }} 60 | 62 | {{ data.version }}-{{ data.branch.name }} 63 | 65 | {{ data.port }} 66 | 68 | {{ data.user }} 69 | 71 | Details 72 |
77 |
78 |
79 |
80 | 81 | -------------------------------------------------------------------------------- /templates/alarm_report.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /templates/binlog_display.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | 9 |
10 |    11 |
12 | 18 |
19 |    20 |
21 | 26 |
27 |    28 |
29 | 30 | - 31 | 32 |
33 |
34 |    35 | 36 |    37 | 38 |
39 |
40 |
-------------------------------------------------------------------------------- /templates/chart_auto.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{ host_info.remark }}监控图表 6 | 7 | 8 | 9 | 10 | 11 | 12 |
13 |
14 |
15 | 16 | 17 | 18 | 133 | -------------------------------------------------------------------------------- /templates/chart_new.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 8 |
9 |
10 | 15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 | 24 | -------------------------------------------------------------------------------- /templates/execute_sql.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | 7 |
8 | 9 |
10 |
11 | 13 |
14 |
15 | 25 |
26 |
27 | 34 |
35 |
36 | 37 | 38 |
39 |
40 |
41 |
42 | 43 | 44 | 45 | 61 | 62 | 83 | -------------------------------------------------------------------------------- /templates/general_log.html: -------------------------------------------------------------------------------- 1 | {% if(general_logs != None) %} 2 |
3 |
4 |
5 |
    6 |
  • «
    • 7 | {% for number in page_list %} 8 |
  • 9 | {{ number }} 10 |
    • 11 | {% endfor %} 12 |
13 |
14 | 15 | 16 | 17 | 18 | 21 | 24 | 27 | 30 | 33 | 36 | 39 | 40 | 41 | 42 | {% for status in general_logs %} 43 | 100) %} class="text-danger" {% endif %}> 44 | 47 | 50 | 53 | 56 | 59 | 66 | 69 | 70 | {% endfor %} 71 | 72 |
19 | Checksum 20 | 22 | Fingerprint 23 | 25 | First_Seen 26 | 28 | Last_Seen 29 | 31 | Count 32 | 34 | State 35 | 37 | Detail 38 |
45 | {{ status.checksum }} 46 | 48 | {{ status.fingerprint }} 49 | 51 | {{ status.first_seen }} 52 | 54 | {{ status.last_seen }} 55 | 57 | {{ status.ts_cnt }} 58 | 60 | {% if(status.is_reviewed == 0) %} 61 | 62 | {% else %} 63 | 64 | {% endif %} 65 | 67 | 68 |
73 | 74 |
75 |
    76 |
  • «
    • 77 | {% for number in page_list %} 78 |
  • 79 | {{ number }} 80 |
    • 81 | {% endfor %} 82 |
83 |
84 |
85 |
86 | {% endif %} 87 | 88 | 89 | 90 | 106 | 107 | 124 | 125 | 152 | -------------------------------------------------------------------------------- /templates/general_log_detail.html: -------------------------------------------------------------------------------- 1 | {% if(general_log_detail != None) %} 2 | 3 | 4 | 5 | 8 | 9 | 10 | 11 | 12 | 15 | 18 | 19 | 20 | 23 | 26 | 27 | 28 | 31 | 34 | 35 | 36 | 39 | 42 | 43 | 44 | 47 | 50 | 51 | 52 | 55 | 58 | 59 | 60 |
6 |

General Log Detail

 7 |
13 | Checksum 14 | 16 | {{ general_log_detail.checksum }} 17 |
21 | Fingerprint 22 | 24 | {{ general_log_detail.fingerprint }} 25 |
29 | First Seen 30 | 32 | {{ general_log_detail.first_seen }} 33 |
37 | Last Seen 38 | 40 | {{ general_log_detail.last_seen }} 41 |
45 | Count 46 | 48 | {{ general_log_detail.ts_cnt }} 49 |
53 | Sample 54 | 56 | {{ general_log_detail.sample }} 57 |
61 | {% endif %} 62 | -------------------------------------------------------------------------------- /templates/host_search.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |

16 |
17 | 25 | 26 |
27 | 28 | 29 | 30 | 37 | 38 | -------------------------------------------------------------------------------- /templates/login.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 登录 6 | 7 | 8 | 9 | 10 | 11 | 19 | 20 | 21 |
22 |
23 | @ 24 | 25 |
26 |
27 | 28 |
29 | @ 30 | 31 |
32 |
33 | 34 | 35 |
36 | 37 | 38 | 39 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /templates/mha.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /templates/mysql_exception_log.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | 9 |
10 |    11 |
12 | 13 | - 14 | 15 |
16 |
17 |    18 | 19 |    20 | 21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 | 29 | -------------------------------------------------------------------------------- /templates/mysql_exception_log_display.html: -------------------------------------------------------------------------------- 1 | {% if(mysql_logs != None) %} 2 | 3 | 4 | 5 | 8 | 11 | 12 | 13 | 16 | 19 | 22 | 25 | 28 | 31 | 32 | 33 | 34 | {% for status in mysql_logs %} 35 | 36 | 39 | 42 | 45 | 48 | 51 | 54 | 55 | {% endfor %} 56 | 57 |
6 | MySQL Log 7 | 9 | 10 |
14 | Host 15 | 17 | Problem 18 | 20 | Type 21 | 23 | Level 24 | 26 | Created_Time 27 | 29 | 30 |
37 | {{ status.remark }} 38 | 40 | {{ status.exception_type }} 41 | 43 | {{ status.log_type }} 44 | 46 | {{ status.level }} 47 | 49 | {{ status.created_time }} 50 | 52 | 53 |
58 | {% endif %} -------------------------------------------------------------------------------- /templates/mysql_host_view.html: -------------------------------------------------------------------------------- 1 | {% if(mysql_host_infos != None) %} 2 | 3 | 4 | 5 | 8 | 11 | 14 | 17 | 18 | 19 | 22 | 25 | 28 | 31 | 34 | 37 | 40 | 43 | 46 | 47 | 48 | 49 | {% for info in mysql_host_infos %} 50 | 51 | 54 | 57 | 60 | 63 | 66 | 69 | 72 | 75 | 79 | 80 | {% endfor %} 81 | 82 |
6 | 7 | 9 | MySQL 10 | 12 | SSH 13 | 15 | 16 |
20 | ID 21 | 23 | Remark 24 | 26 | IP 27 | 29 | Port 30 | 32 | User 33 | 35 | User 36 | 38 | Port 39 | 41 | Created_Time 42 | 44 | 45 |
52 | {{ info.host_id }} 53 | 55 | {{ info.remark }} 56 | 58 | {{ info.host }} 59 | 61 | {{ info.port }} 62 | 64 | {{ info.user }} 65 | 67 | {{ info.ssh_user }} 68 | 70 | {{ info.ssh_port }} 71 | 73 | {{ info.created_time }} 74 | 76 | 77 | 78 |
83 | {% endif %} 84 | -------------------------------------------------------------------------------- /templates/mysqls.html: -------------------------------------------------------------------------------- 1 | {% if(mysql_infos != None) %} 2 | 3 | 4 | 5 | 8 | 11 | 14 | 17 | 20 | 23 | 25 | 26 | 27 | 30 | 33 | 36 | 39 | 42 | 45 | 48 | 51 | 54 | 57 | 60 | 63 | 66 | 69 | 72 | 75 | 78 | 81 | 84 | 85 | 86 | 87 | {% for data in mysql_infos %} 88 | 89 | 92 | 95 | 104 | 107 | 112 | 117 | 124 | 127 | 130 | 133 | 136 | 139 | 142 | 145 | 148 | 151 | 154 | 157 | 163 | 164 | {% endfor %} 165 | 166 |
6 | Host 7 | 9 | Slave Status 10 | 12 | Status 13 | 15 | Thread 16 | 18 | Net 19 | 21 | MySQL 22 | 24 |
28 | Host 29 | 31 | IP:Port 32 | 34 | Role 35 | 37 | Master_Name 38 | 40 | IO 41 | 43 | SQL 44 | 46 | Run 47 | 49 | QPS 50 | 52 | TPS 53 | 55 | Trxs 56 | 58 | B_P_Z 59 | 61 | Count 62 | 64 | Run 65 | 67 | Send 68 | 70 | Rece 71 | 73 | Uptime 74 | 76 | Branch 77 | 79 | Version 80 | 82 | 83 |
90 | {{ data.remark }} 91 | 93 | {{ data.host }}:{{ data.port }} 94 | 96 | {% if(data.is_slave and data.is_master) %} 97 | & 98 | {% elif(data.is_master) %} 99 | 100 | {% elif(data.is_slave) %} 101 | 102 | {% endif %} 103 | 105 | {{ data.master_name }} 106 | 108 | {% if(data.is_slave) %} 109 | {{ data.io_status }} 110 | {% endif %} 111 | 113 | {% if(data.is_slave) %} 114 | {{ data.sql_status }} 115 | {% endif %} 116 | 118 | {% if(data.is_running == 1) %} 119 | Yes 120 | {% else %} 121 | No 122 | {% endif %} 123 | 125 | {{ data.qps }} 126 | 128 | {{ data.tps }} 129 | 131 | {{ data.trxs }} 132 | 134 | {{ data.innodb_buffer_pool_size }} 135 | 1000) %} class="text-danger" {% endif %}> 137 | {{ data.threads }} 138 | 5) %} class="text-danger" {% endif %}> 140 | {{ data.threads_running }} 141 | 143 | {{ data.send_bytes }} 144 | 146 | {{ data.receive_bytes }} 147 | 149 | {{ data.uptime }} 150 | 152 | {{ data.branch.name }} 153 | 155 | {{ data.version }} 156 | 158 | 159 | 160 | 161 | 162 |
167 | {% endif %} 168 | -------------------------------------------------------------------------------- /templates/report.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | MySQL Report 6 | 7 | 33 | 34 |
35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | {% for info in tablespace_infos %} 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | {% endfor %} 63 |
MySQL Tablesspace Report
HostRowsData_TIndex_TTotalFile_TFree_TTable_CLast Update Time
{{ info.host_info.remark }}{{ info.rows_total }}{{ info.data_total }}{{ info.index_total }}{{ info.total }}{{ info.file_total }}{{ info.free_total }}{{ info.table_count }}{{ info.last_update_time }}
64 |
65 |
66 |
67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 |
MySQL Status Report
QPSTPSThreadTmp TableTmp Disk Table
HostMinMaxAvgMinMaxAvgMinMaxAvgMinMaxAvgMinMaxAvg
101 |
102 |
103 |
104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 |
OS Report
CPU1 LoadCPU5 LoadCPU15 LoadCPU UserCPU SysIO Util
HostMinMaxAvgMinMaxAvgMinMaxAvgMinMaxAvgMinMaxAvgMinMaxAvg
139 |
140 |
141 |
142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 |
MySQL Slow Log Report
Host
150 |
151 |
152 | 153 | -------------------------------------------------------------------------------- /templates/show_processlist.html: -------------------------------------------------------------------------------- 1 | {% if(processlist_infos != None) %} 2 | 3 | 4 | 5 | 8 | 9 | 10 | 13 | 16 | 19 | 22 | 25 | 28 | 31 | 34 | 37 | 38 | 39 | 40 | {% for status in processlist_infos %} 41 | = 1) %} class="bg-danger" {% endif %}> 42 | 45 | 48 | 51 | 54 | 57 | 60 | 63 | 66 | 69 | 70 | {% endfor %} 71 | 72 |
6 | {{ host_info.remark }} Show Processlist 7 |
11 | ID 12 | 14 | User 15 | 17 | Host 18 | 20 | DB 21 | 23 | Command 24 | 26 | Time 27 | 29 | State 30 | 32 | Info 33 | 35 | Opertion 36 |
43 | {{ status.id }} 44 | 46 | {{ status.user }} 47 | 49 | {{ status.host }} 50 | 52 | {{ status.db }} 53 | 55 | {{ status.command }} 56 | 58 | {{ status.time }} 59 | 61 | {{ status.state }} 62 | 64 | {{ status.info }} 65 | 67 | 68 |
73 | {% endif %} -------------------------------------------------------------------------------- /templates/slow_log_display.html: -------------------------------------------------------------------------------- 1 | {% if(slow_logs != None) %} 2 | 3 | 4 | 5 | 8 | 11 | 14 | 17 | 18 | 19 | 22 | 25 | 28 | 31 | 34 | 37 | 40 | 43 | 46 | 49 | 52 | 55 | 58 | 61 | 62 | 63 | 64 | {% for status in slow_logs %} 65 | 66 | 71 | 74 | 77 | 80 | 83 | 86 | 89 | 92 | 95 | 98 | 101 | 104 | 107 | 116 | 117 | 118 | 121 | 122 | {% endfor %} 123 | 124 |
6 | SQL 7 | 9 | Query Time 10 | 12 | Lock Time 13 | 15 | 16 |
20 | Fingerprint 21 | 23 | DB 24 | 26 | User 27 | 29 | Last_Seen 30 | 32 | Count 33 | 35 | Max 36 | 38 | Min 39 | 41 | Avg 42 | 44 | Total 45 | 47 | Max 48 | 50 | Min 51 | 53 | Avg 54 | 56 | Total 57 | 59 | 60 |
67 | 68 | {{ status.fingerprint_tmp }} 69 | 70 | 72 | {{ status.db_max }} 73 | 75 | {{ status.user_max }} 76 | 78 | {{ status.last_seen }} 79 | 10) %} class="text-danger" {% endif %}> 81 | {{ status.ts_cnt }} 82 | 84 | {{ status.Query_time_max }} 85 | 87 | {{ status.Query_time_min }} 88 | 90 | {{ status.Query_time_avg }} 91 | 93 | {{ status.Query_time_sum }} 94 | 96 | {{ status.Lock_time_max }} 97 | 99 | {{ status.Lock_time_min }} 100 | 102 | {{ status.Lock_time_avg }} 103 | 105 | {{ status.Lock_time_sum }} 106 | 108 | 109 | 110 | {% if(status.is_reviewed == 0) %} 111 | 112 | {% else %} 113 | 114 | {% endif %} 115 |
125 | 126 |
127 |
    128 |
  • «
    • 129 | {% for number in page_list %} 130 |
  • 131 | {{ number }} 132 |
    • 133 | {% endfor %} 134 |
135 |
136 | {% endif %} 137 | 138 | -------------------------------------------------------------------------------- /templates/table.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 数据库表查询 13 | 14 | 查询界面 15 | 16 | 17 | 图标 详情 优化 18 | 19 | 20 | 21 | 比如最近一周表数据增长量，比如一周 22 | 23 | 24 | 25 | 总览 冗余索引检查 表空间碎片检查 - 支持导出文件 26 | 27 | 表详情 支持简单的查询 - 有查看详情的功能 包括表结构 表数据增量量图标 表索引 表基本信息 -------------------------------------------------------------------------------- /templates/tablespace.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 9 |
10 |
11 | 20 |
21 |
22 | 28 |
29 |
30 | 31 |
32 |
33 | 34 |
35 |
36 |
37 |
38 |
39 |
40 | 41 | 58 | 59 | 60 | 75 | 76 | 77 | 128 | -------------------------------------------------------------------------------- /templates/thread.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 9 |
10 |
11 | 16 |
17 |
18 | 19 |    20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 | 28 | -------------------------------------------------------------------------------- /templates/thread_display.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 10 | 13 | 16 | 19 | 22 | 23 | 24 | 25 | {% for status in thread_infos %} 26 | 27 | 30 | 33 | 36 | 39 | 42 | 45 | 46 | {% endfor %} 47 | 48 |
5 | User 6 | 8 | Host 9 | 11 | Command 12 | 14 | Count 15 | 17 | State 18 | 20 | SQL 21 |
28 | {{ status.user }} 29 | 31 | {{ status.host }} 32 | 34 | {{ status.command }} 35 | 37 | {{ status.count }} 38 | 40 | {{ status.state }} 41 | 43 | {{ status.sql }} 44 |
49 | -------------------------------------------------------------------------------- /templates/user.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 | 14 |
15 |
16 | 17 |
18 |
19 | 20 |
21 |
22 | 23 |    24 | 25 |
26 |
27 | 28 |
29 |
30 | 31 | 36 | 37 | 38 |
39 |
40 |
41 |
42 |
43 | 44 | 45 | 46 | 62 | 63 | 64 | 105 |
106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /templates/user_display.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 10 | 13 | 16 | 19 | 22 | 25 | 28 | 29 | 30 | 31 | {% for status in user_infos %} 32 | 33 | 36 | 39 | 42 | 45 | 48 | 51 | 54 | 58 | 59 | {% endfor %} 60 | 61 |
5 | User 6 | 8 | Host 9 | 11 | Select 12 | 14 | Insert 15 | 17 | Update 18 | 20 | Delete 21 | 23 | 24 | 26 | 27 |
34 | {{ status.user }} 35 | 37 | {{ status.host }} 38 | 40 | {{ status.select }} 41 | 43 | {{ status.insert }} 44 | 46 | {{ status.update }} 47 | 49 | {{ status.delete }} 50 | 52 | Detail 53 | 55 | 56 | 57 |
62 | -------------------------------------------------------------------------------- /tools/binlog/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Administrator' 2 | -------------------------------------------------------------------------------- /tools/binlog/binlog_backup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import random, argparse, sys, os, commands, time 4 | 5 | #1.封装mysqlbinlog远程备份操作 6 | #参数如下： 7 | #file-path：指定备份目录，不指定则备份在当前执行脚本的目录 8 | #server-id：用户复制的server_id，不指定则自动生成 9 | #log-file：指定日志输出文件，不指定默认在/tmp/binlog.log 10 | #最简单的例子 11 | #python binlog_backup.py --host=192.168.11.128 --user=yancg --password=123456 --port=3310 & 12 | #下面是建议的备份命令 13 | #python binlog_backup.py --host=192.168.11.128 --user=yancg --password=123456 --port=3310 --file-path=/data/binlog_backup/ & 14 | 15 | path = "/opt/mysql-5.7/bin/" 16 | mysql = path + "mysql -h{0} -u{1} -p\'{2}\' -P{3} -e\"{4}\" | head -n2 | tail -n 1 | awk \'{5}\'" 17 | mysqlbinlog = path + "mysqlbinlog -h{0} -u{1} -p{2} -P{3} --stop-never --raw " \ 18 | "--read-from-remote-server --stop-never-slave-server-id={4} --result-file={5} {6} &>> {7}" 19 | 20 | def check_arguments(): 21 | parser = argparse.ArgumentParser() 22 | parser.add_argument("--host", type=str, dest="host", help="mysql host") 23 | parser.add_argument("--port", type=int, dest="port", help="mysql port", default=3306) 24 | parser.add_argument("--user", type=str, dest="user", help="mysql user") 25 | parser.add_argument("--password", type=str, dest="password", help="mysql password") 26 | parser.add_argument("--file-path", type=str, dest="file_path", help="save binlog directory", default="") 27 | parser.add_argument("--conf-file", type=str, dest="conf_file", help="configuration file", default="") 28 | parser.add_argument("--server-id", type=int, dest="server_id", help="server id") 29 | parser.add_argument("--log-file", type=str, dest="log_file", help="log file path", default="/tmp/binlog.log") 30 | args = parser.parse_args() 31 | 32 | if(not args.conf_file or args.conf_file == None): 33 | if(not args.host or not args.user or not args.password): 34 | print(parser.format_usage()) 35 | sys.exit(1) 36 | else: 37 | read_conf_file(args) 38 | if(not args.server_id or args.server_id <= 0): 39 | args.server_id = random.randint(777777,999999) 40 | if(len(args.file_path) <= 0): 41 | args.file_path = os.getcwd() + "/" 42 | return args 43 | 44 | def read_conf_file(args): 45 | if(os.path.exists(args.conf_file)): 46 | pass 47 | else: 48 | print("配置文件不存在...") 49 | sys.exit(1) 50 | 51 | def get_binlog_file_name(args): 52 | files = os.listdir(args.file_path) 53 | file_count = len(files) 54 | if(file_count == 0): 55 | binglog_name = get_first_binlog_file_name(args) 56 | elif(file_count == 1): 57 | if(files[0] == os.path.basename(args.conf_file) or files[0] == sys.argv[0]): 58 | binglog_name = get_first_binlog_file_name(args) 59 | else: 60 | binglog_name = files[0] 61 | else: 62 | (status, output) = commands.getstatusoutput("ls -l {0} |tail -n 1 |awk \'{1}\'".format(args.file_path, "{print $9}")) 63 | binglog_name = output 64 | return binglog_name.replace("\n", "").replace("\t", "").replace("\r", "") 65 | 66 | def get_first_binlog_file_name(args): 67 | mysql_shell = mysql.format(args.host, args.user, args.password, args.port, "show master logs;", "{print $1}") 68 | (status, output) = commands.getstatusoutput(mysql_shell) 69 | return output.split("\n")[1] 70 | 71 | if(__name__ == "__main__"): 72 | args = check_arguments() 73 | print("开始远程备份binlog...") 74 | while(True): 75 | binlog_backup = mysqlbinlog.format(args.host, args.user, args.password, args.port, 76 | args.server_id, args.file_path, get_binlog_file_name(args), args.log_file) 77 | os.system(binlog_backup) 78 | time.sleep(10) -------------------------------------------------------------------------------- /tools/check_os.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ############################################# 4 | # Program : Check linux before install MySQL 5 | # Author : yxli 2017/04/27 6 | # History : 7 | ############################################# 8 | 9 | function printstr { 10 | if [ $2 -eq 1 ] 11 | then 12 | echo -e "[warning] "$1 13 | else 14 | echo -e "[info] "$1 15 | fi 16 | } 17 | 18 | function check { 19 | echo "" 20 | printstr "Check start..." 0 21 | printstr "Linux Version : "`uname -r` 0 22 | printstr "IP : `ifconfig|grep "inet addr"|grep -v "127.0"|sed 's/inet addr://g'|awk '{print $1}'`" 0 23 | 24 | ## Hostname 25 | hostname=`hostname` 26 | if [ "localhost.localdomain" = "$hostname" ] || [ "" = "$hostname" ] 27 | then 28 | printstr "Hostname : Hostname is not set!" 1 29 | else 30 | printstr "Hostname : $hostname" 0 31 | fi 32 | 33 | ## Swap 34 | swap=`free -m|grep Swap|awk '{print $2}'` 35 | if [ 1024 -le $swap ] 36 | then 37 | printstr "Swap : Swap($swap"M") too large!" 1 38 | else 39 | printstr "Swap : $swap(M)" 0 40 | fi 41 | 42 | ## Firewall 43 | firewall=`chkconfig --list|grep iptables|sed 's/[0-9]://g'` 44 | if [ "echo $firewall|awk '{print $4}'" = "off" ] || [ "echo $firewall|awk '{print $5}'" = "on" ] || [ "echo $firewall|awk '{print $6}'" = "on" ] || [ "echo $firewall|awk '{print $7}'" = "on" ] 45 | then 46 | printstr "Firewall : Firewall is not closed!" 1 47 | else 48 | printstr "`chkconfig --list|grep iptables|sed 's/iptables/Iptables : /g'`" 0 49 | fi 50 | 51 | ## File system 52 | filesystem=0 53 | for i in `df -Thl|grep 'dev'|grep -v 'tmpfs'|awk '{print $2}'` 54 | do 55 | if [ "$i" != "ext4" ] && [ "$i" != "xfs" ] 56 | then 57 | filesystem=$((filesystem+1)) 58 | fi 59 | done 60 | if [ 0 -lt $filesystem ] 61 | then 62 | printstr "File System : The file system is not ext4 or XFS!" 1 63 | else 64 | printstr "File System : `df -Thl|grep 'dev'|grep -v 'tmpfs'|awk '{print $1"|\t"$2"|\n"}'`" 0 65 | fi 66 | 67 | ## Selinux 68 | if [ `getenforce` != "Disabled" ] 69 | then 70 | printstr "Selinux is not closed!" 1 71 | else 72 | printstr "Selinux : `getenforce`" 0 73 | fi 74 | printstr "Check end!" 0 75 | echo "" 76 | } 77 | 78 | check 79 | -------------------------------------------------------------------------------- /tools/check_tablespace.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | #检查数据库表空间报表 4 | #python check_tablespace.py --host=192.168.11.130 --user=yangcg --password='123456' 5 | #python check_tablespace.py --host=192.168.11.130 --port=3310 --user=yangcg --password='123456' 6 | #数据保存在tmp目录下，以host值为文件名 7 | #--convert：是否把大小进行可读性格式化，如-1024M 8 | 9 | import MySQLdb, paramiko, sys, argparse 10 | 11 | class TableInfo(): 12 | diff = 0 13 | 14 | def check_arguments(): 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument("--host", type=str, dest="host", help="mysql host") 17 | parser.add_argument("--port", type=int, dest="port", help="mysql port", default=3306) 18 | parser.add_argument("--user", type=str, dest="user", help="mysql user") 19 | parser.add_argument("--password", type=str, dest="password", help="mysql password") 20 | parser.add_argument("--convert", type=str, dest="convert", help="data size convert", default=True) 21 | args = parser.parse_args() 22 | 23 | if(not args.host or not args.port or not args.user or not args.password): 24 | sys.exit(1) 25 | 26 | return args 27 | 28 | def get_table_infos(args): 29 | table_infos = [] 30 | connection = MySQLdb.connect(host=args.host, port=args.port, user=args.user, passwd=args.password) 31 | cursor = connection.cursor(cursorclass = MySQLdb.cursors.DictCursor) 32 | cursor.execute("show global variables like '%datadir%';") 33 | args.data_dir = cursor.fetchone()["Value"] 34 | 35 | cursor.execute("select table_schema, table_name, DATA_LENGTH, INDEX_LENGTH from information_schema.tables " 36 | "where table_schema != 'mysql' and table_schema != 'information_schema' and table_schema != 'performance_schema'") 37 | for row in cursor.fetchall(): 38 | table_info = TableInfo() 39 | table_info.schema = row["table_schema"] 40 | table_info.t_name = row["table_name"] 41 | table_info.data_size = row["DATA_LENGTH"] if row["DATA_LENGTH"] else 0 42 | table_info.index_size = row["INDEX_LENGTH"] if row["INDEX_LENGTH"] else 0 43 | table_info.total_size = long(table_info.data_size) + long(table_info.index_size) 44 | table_infos.append(table_info) 45 | cursor.close() 46 | connection.close() 47 | return table_infos 48 | 49 | def get_data_length(data_length): 50 | value = float(1024) 51 | if(data_length > value): 52 | result = round(data_length / value, 0) 53 | if(result > value): 54 | return str(int(round(result / value, 0))) + "M" 55 | else: 56 | return str(int(result)) + "K" 57 | else: 58 | return str(data_length) + "KB" 59 | 60 | def save_file(args, table_infos): 61 | file = open("/tmp/{0}_tb.txt".format(args.host), "w") 62 | file.write("schema\ttable_name\ttable_fragment\tdata_size\tindex_size\ttotal_size\tfile_size\n") 63 | for table_info in table_infos: 64 | file.write(get_print_string(args, table_info) + "\n") 65 | file.close() 66 | return table_infos 67 | 68 | def get_print_string(args, table_info): 69 | str_format = "{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}" 70 | if(args.convert == True): 71 | return str_format.format(table_info.schema, table_info.t_name, table_info.value, get_data_length(table_info.data_size), 72 | get_data_length(table_info.index_size), get_data_length(table_info.total_size), get_data_length(table_info.file_size)) 73 | else: 74 | return str_format.format(table_info.schema, table_info.t_name, table_info.value, table_info.data_size, 75 | table_info.index_size, table_info.total_size, table_info.file_size) 76 | 77 | ''' 78 | def get_all_table_file_size(args): 79 | table_size = {} 80 | host_client = paramiko.SSHClient() 81 | host_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 82 | host_client.connect(args.host, port=22, username="root") 83 | connection = MySQLdb.connect(host=args.host, port=args.port, user=args.user, passwd=args.password) 84 | cursor = connection.cursor(cursorclass = MySQLdb.cursors.DictCursor) 85 | cursor.execute("show databases;") 86 | for row in cursor.fetchall(): 87 | db_name = row["Database"] 88 | if(db_name == "mysql" or db_name == "information_schema" or db_name == "performance_schema"): 89 | return 90 | shell = "ls -al {0}/{1}/ | grep -E '(ibd$)' | awk \'{2}\'".format(args.data_dir, db_name, "{print $5, $9}") 91 | stdin, stdout, stderr = host_client.exec_command(shell) 92 | result = stdout.readlines() 93 | if(len(result) > 0 and len(stderr) <= 0): 94 | for value in result: 95 | list_tmp = value.replace("\n", "").split(" ") 96 | cursor.close() 97 | connection.close() 98 | host_client.close()''' 99 | 100 | def check_table_space(args): 101 | list_tmp = [] 102 | table_infos = get_table_infos(args) 103 | host_client = paramiko.SSHClient() 104 | host_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 105 | host_client.connect(args.host, port=22, username="root") 106 | 107 | for table_info in table_infos: 108 | shell = "ls -al {0}{1}/{2}.ibd | awk \'{3}\'".format(args.data_dir, table_info.schema, table_info.t_name, "{print $5}") 109 | stdin, stdout, stderr = host_client.exec_command(shell) 110 | result = stdout.readlines() 111 | if(len(result) > 0): 112 | table_info.file_size = long(result[0].replace("\n", "")) 113 | table_info.diff = table_info.file_size - table_info.total_size 114 | table_info.value = get_data_length(table_info.diff) 115 | list_tmp.append(table_info) 116 | print(get_print_string(args, table_info)) 117 | 118 | host_client.close() 119 | save_file(args, sorted(list_tmp, cmp=lambda x,y:cmp(x.diff,y.diff), reverse=True)) 120 | 121 | if(__name__ == "__main__"): 122 | print("start...") 123 | check_table_space(check_arguments()) 124 | print("table space check is ok.") 125 | 126 | 127 | -------------------------------------------------------------------------------- /tools/collect_mysql_status.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | host_slave1="db-group2-slave1" 4 | host_slave2="db-group2-slave2" 5 | user="dba_read" 6 | password="abc123!.+" 7 | 8 | start_timestamp=`date +%s -d '2017-04-19 04:30:00'` 9 | end_timestamp=`date +%s -d '2017-04-19 05:20:00'` 10 | 11 | sql1="show engine innodb status\G" 12 | sql2="SELECT * FROM information_schema.processlist where COMMAND != 'Sleep'\G" 13 | 14 | function collect_status() 15 | { 16 | echo "------------------------`date`-------------------------" >> /tmp/collect.txt 17 | 18 | #show engine innodb status 19 | echo "--------------------$sql1------------------------------" >> /tmp/collect.txt 20 | echo "---------------------$host_slave1----------------------" >> /tmp/collect.txt 21 | mysql -h$host_slave1 -u$user -p$password -e"$sql1" >> /tmp/collect.txt 22 | echo "---------------------$host_slave2----------------------" >> /tmp/collect.txt 23 | mysql -h$host_slave2 -u$user -p$password -e"$sql1" >> /tmp/collect.txt 24 | 25 | #show full processlist 26 | echo "--------------------$sql2------------------------------" >> /tmp/collect.txt 27 | echo "---------------------$host_slave1----------------------" >> /tmp/collect.txt 28 | mysql -h$host_slave1 -u$user -p$password -e"$sql2" >> /tmp/collect.txt 29 | echo "---------------------$host_slave2----------------------" >> /tmp/collect.txt 30 | mysql -h$host_slave2 -u$user -p$password -e"$sql2" >> /tmp/collect.txt 31 | } 32 | 33 | now_timestamp=`date +%s` 34 | while [ $now_timestamp -le $end_timestamp ] 35 | do 36 | if [ $now_timestamp -ge $start_timestamp ] && [ $now_timestamp -le $end_timestamp ]; then 37 | collect_status 38 | echo "collect ok" 39 | else 40 | echo "no collect" 41 | fi 42 | sleep 1 43 | now_timestamp=`date +%s` 44 | done 45 | -------------------------------------------------------------------------------- /tools/dns_ha/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Administrator' 2 | -------------------------------------------------------------------------------- /tools/host.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | global_port = 3306 3 | global_user = yangcg 4 | global_passoword = yangcaogui123!.+ 5 | 6 | [server1] 7 | host = 192.168.11.128 8 | remark = Test 9 | -------------------------------------------------------------------------------- /tools/mha/app.conf: -------------------------------------------------------------------------------- 1 | [server default] 2 | manager_workdir=/masterha/ 3 | manager_log=/masterha/manager.log 4 | master_binlog_dir=/mysql/binlog 5 | remote_workdir=/opt/mysql/tmpbinlog 6 | 7 | ssh_user=root 8 | ping_interval=1 9 | 10 | #mysql管理员用户 11 | user=yangcg 12 | password=yangcaogui 13 | 14 | #mysql复制用户 15 | repl_user=sys_repl 16 | repl_password=yangcaogui 17 | 18 | #MHA切换之后发生邮件报告 19 | report_script=/masterha/send_report 20 | #shutdown_script=/masterha/master_ip_failover 21 | #从其它机器ping主库机器 22 | secondary_check_script=/masterha/masterha_secondary_check -s 192.168.11.104 -s 192.168.11.102 -s 192.168.11.103 23 | 24 | #VIP切换脚本 25 | #master_ip_failover_script=/masterha/master_ip_failover 26 | 27 | #DNS切换脚本 28 | #master_ip_failover_script=/root/mhascript/master_ip_failover 29 | 30 | #注意：5.7要使用0.57版本的MHA 31 | #注意下面server的先后顺序 32 | #第一个为master 33 | #第二个为master stanby 34 | [server1] 35 | port=3306 36 | candidate_master=1 37 | check_repl_delay=0 38 | hostname=192.168.11.100 39 | 40 | [server2] 41 | port=3306 42 | candidate_master=1 43 | check_repl_delay=0 44 | hostname=192.168.11.104 45 | 46 | [server3] 47 | port=3306 48 | no_master=1 49 | hostname=192.168.11.102 50 | 51 | [server4] 52 | port=3306 53 | no_master=1 54 | hostname=192.168.11.103 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /tools/mha/master_ip_failover: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) 2011 DeNA Co.,Ltd. 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | 20 | ## Note: This is a sample script and is not complete. Modify the script based on your environment. 21 | 22 | use strict; 23 | use warnings FATAL => 'all'; 24 | 25 | use Getopt::Long; 26 | use MHA::DBHelper; 27 | 28 | #要切换的IP地址 29 | my $vip = "192.168.11.120"; 30 | #要绑定的网卡名称 31 | my $devc = "eth0"; 32 | 33 | my ( 34 | $command, $ssh_user, $orig_master_host, 35 | $orig_master_ip, $orig_master_port, $new_master_host, 36 | $new_master_ip, $new_master_port, $new_master_user, 37 | $new_master_password 38 | ); 39 | 40 | GetOptions( 41 | 'command=s' => \$command, 42 | 'ssh_user=s' => \$ssh_user, 43 | 'orig_master_host=s' => \$orig_master_host, 44 | 'orig_master_ip=s' => \$orig_master_ip, 45 | 'orig_master_port=i' => \$orig_master_port, 46 | 'new_master_host=s' => \$new_master_host, 47 | 'new_master_ip=s' => \$new_master_ip, 48 | 'new_master_port=i' => \$new_master_port, 49 | 'new_master_user=s' => \$new_master_user, 50 | 'new_master_password=s' => \$new_master_password, 51 | ); 52 | 53 | sub add_vip { 54 | my $output1 = `ssh -o ConnectTimeout=15 -o ConnectionAttempts=3 $orig_master_host /sbin/ip addr del $vip dev $devc`; 55 | my $output2 = `ssh -o ConnectTimeout=15 -o ConnectionAttempts=3 $new_master_host /sbin/ip addr add $vip dev $devc`; 56 | } 57 | 58 | 59 | exit &main(); 60 | 61 | sub main { 62 | if ( $command eq "stop" || $command eq "stopssh" ) { 63 | 64 | # $orig_master_host, $orig_master_ip, $orig_master_port are passed. 65 | # If you manage master ip address at global catalog database, 66 | # invalidate orig_master_ip here. 67 | my $exit_code = 1; 68 | eval { 69 | 70 | # updating global catalog, etc 71 | $exit_code = 0; 72 | }; 73 | if ($@) { 74 | warn "Got Error: $@\n"; 75 | exit $exit_code; 76 | } 77 | exit $exit_code; 78 | } 79 | elsif ( $command eq "start" ) { 80 | 81 | # all arguments are passed. 82 | # If you manage master ip address at global catalog database, 83 | # activate new_master_ip here. 84 | # You can also grant write access (create user, set read_only=0, etc) here. 85 | my $exit_code = 10; 86 | eval { 87 | my $new_master_handler = new MHA::DBHelper(); 88 | 89 | # args: hostname, port, user, password, raise_error_or_not 90 | $new_master_handler->connect( $new_master_ip, $new_master_port, 91 | $new_master_user, $new_master_password, 1 ); 92 | 93 | ## Set read_only=0 on the new master 94 | $new_master_handler->disable_log_bin_local(); 95 | print "Set read_only=0 on the new master.\n"; 96 | $new_master_handler->disable_read_only(); 97 | 98 | ## Creating an app user on the new master 99 | #print "Creating app user on the new master..\n"; 100 | #FIXME_xxx_create_user( $new_master_handler->{dbh} ); 101 | $new_master_handler->enable_log_bin_local(); 102 | $new_master_handler->disconnect(); 103 | 104 | ## Update master ip on the catalog database, etc 105 | &add_vip(); 106 | $exit_code = 0; 107 | }; 108 | if ($@) { 109 | warn $@; 110 | 111 | # If you want to continue failover, exit 10. 112 | exit $exit_code; 113 | } 114 | exit $exit_code; 115 | } 116 | elsif ( $command eq "status" ) { 117 | 118 | # do nothing 119 | exit 0; 120 | } 121 | else { 122 | &usage(); 123 | exit 1; 124 | } 125 | } 126 | 127 | sub usage { 128 | print 129 | "Usage: master_ip_failover --command=start|stop|stopssh|status --orig_master_host=host --orig_master_ip=ip --orig_master_port=port --new_master_host=host --new_master_ip=ip --new_master_port=port\n"; 130 | } 131 | -------------------------------------------------------------------------------- /tools/mha/masterha_master_switch: -------------------------------------------------------------------------------- 1 | #masterha_master_switch 2 | 3 | #如果主出现故障，手动进行故障转移 4 | masterha_master_switch --master_state=dead --conf=/usr/local/masterha/conf/app1.cnf --dead_master_host=192.168.1.100 5 | 6 | #当在线的MHA进行主故障转移之后，主恢复之后，重新进行切换的命令 7 | masterha_master_switch --master_state=alive --conf=/usr/local/masterha/conf/app1.cnf --new_master_host=192.168.1.100 --orig_master_is_new_slave --running_updates_limit=3 8 | 9 | 10 | 参数： 11 | --running_updates_limit：在所有的slaves上Seconds_Behind_Master都要小于此参数值(秒数) 12 | 13 | --master_state=dead：强制的参数，参数值为"dead" 或者 "alive" . 如果 设置为 alive 模式，masterha_master_switch 开始在线主库切换操作。 14 | 15 | --dead_master_host=(hostname)：强制参数，宕机的主库所在的主机名称。--dead_master_ip 和 --dead_master_port 是可选参数，如果这些参数没有设置，--dead_master_ip 就是 --dead_master_host 解析的IP地址。--dead_master_port 为 3306 16 | 17 | --new_master_host=(hostname)：新主机地址，可选参数，这个参数在你明确新的主库的主机，非常有用。(这就意味着你不需要让MHA来决定新的主库)。如果不设置此参数，MHA 将会利用自动failover的规则来选择新的主库。如果设置--new_master_host，MHA选择此主机为新的主库，如果不能成为主库，MHA将会退出 18 | 19 | --interactive=(0|1)：如果设置为0，在masterha_master_switch，它自动执行故障转移(非交互式)。这实际上是和masterha_manager的内部运行机制一样，这种非交互式故障转移是有用的，如果你已经证实了master死了,但你想尽快做故障转移。非交互式故障转移也是有用的,如果你使用其他现有的主监控软件和要调用的非交互式故障转移命令软件。典型的例子是masterha_master_switch调用从集群软件像起搏器。 20 | 21 | --ssh_reachable=(0|1|2)：指定master 经过SSH是否可达。0:不可达、1:可达、2:未知(默认值)。 如果设置为了2，此命令内部将会检测通过SSH 是否可达master，并且跟新SSH 状态。如果可达，且设置master_ip_failover_script 或者 shutdown_script .将会执行"--command=stopssh"。否则，执行 "--command=stop"。另外，如果宕机的master通过SSH可达，failover脚本试图从宕机的master机器上拷贝没有没有发送的binlog。 22 | 23 | --skip_change_master：如果设置此参数，当发生failover的时候，MAH 在应用完不同的relay log退出，忽略CHANGE MASTER 和 START SLAVE 操作。所以 slaves 不会指向 新的master. 开启此参数，有利于手动的二次检查slave 恢复是否成功 24 | 25 | --skip_disable_read_only：设置此参数，MHA 将不会在新的主库上执行 SET GLOBAL read_only =0 操作，有利于手动操作 26 | 27 | --last_failover_minute=(minutes)：参考master_manager 28 | 29 | --ignore_last_failover：参考master_manager 30 | 31 | --wait_on_failover_error=(seconds)：类似于master_manager, 此参数只用于自动的/非交互式的failover。如果没有设置--interval=0，wait_on_failover_error 将会被忽略，在发生错误的时候不会sleep。 32 | 33 | --remove_dead_master_conf：参考masterha_manager 34 | 35 | --wait_until_gtid_in_sync(0|1)：此参数从0.56版本开始可用，如果设置成1，当基于GITD的failover时,MHA 会等待所有的从库追上新主库的GITD 36 | 37 | --skip_change_master：此参数从0.56版本开始可用，如果开启此选项，MHA 跳过 CHANGE MASTER 的操作 38 | 39 | --skip_disable_read_only：此参数从0.56版本开始可用，如果开启此选项，MHA 将会在新的master 跳过 SET GLOBAL read_only = 0; 40 | 41 | --ignore_binlog_server_error：此参数从0.56版本开始可用，如果开启此选项，当执行failover的时，MHA忽略binlog server上任何错误 42 | 43 | 主在线切换时相关参数： 44 | --new_master_host=(hostname)：新主机地址，可选参数，这个参数在你明确新的主库的主机，非常有用。(这就意味着你不需要让MHA来决定新的主库)。如果不设置此参数，MHA 将会利用自动failover的规则来选择新的主库。如果设置--new_master_host，MHA选择此主机为新的主库，如果不能成为主库，MHA将会退出 45 | 46 | --orig_master_is_new_slave：当完成主库切换后，原先的主库将作为现在主库的slave运行。默认:不开启(原先的主库不会加入到新的复制环境中)。如果开启此选项，需要在配置文件中设置repl_password参数，由于当期的Master并不知道新的Master的replication的密码 47 | 48 | --remove_orig_master_conf：如果设置此参数，当成功failover后，MHA manager将会自动删除配置文件中关于dead master的配置选项。 49 | 50 | --skip_lock_all_tables：当在做主库切换的时候，MHA会在原先的主库上执行FLUSH TABLES WITH READ LOCK 操作，确保没有跟新操作，但是FLUSH TABLES WITH READ LOCK 操作是非常耗费资源的，并且你可以在原先的主库确定没有跟新操作(通过master_ip_online_change_script 中kill all clients操作等)。可以利用此选项避免锁表。 51 | -------------------------------------------------------------------------------- /tools/mha/mha_auto_install.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import argparse, paramiko, os 4 | 5 | #自动安装mha前提是上传mha安装包 6 | 7 | error = "error" 8 | output = "output" 9 | dowload_path = "/tmp/" 10 | 11 | def check_arguments(): 12 | global dowload_path 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--host", type=str, dest="host", help="linux host") 15 | parser.add_argument("--package", type=str, dest="package", help="mha package path") 16 | args = parser.parse_args() 17 | return args 18 | 19 | def install_mha_package(args): 20 | yum_command = "yum localinstall -y {0}mha4mysql-*.rpm".format(dowload_path) 21 | rpm_command = "rpm -ivh {0}epel-release-latest-7.noarch.rpm".format(dowload_path) 22 | wget_command = "wget http://mirrors.aliyun.com/epel/epel-release-latest-7.noarch.rpm -P /tmp/" 23 | 24 | if(not args.host): 25 | os.system("cp {0} {1}".format(args.package, dowload_path)) 26 | os.system(wget_command) 27 | os.system(rpm_command) 28 | os.system(yum_command) 29 | else: 30 | os.system("scp {0} {1}:/tmp/".format(args.package, args.host)) 31 | 32 | host_client = paramiko.SSHClient() 33 | host_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 34 | host_client.connect(args.host, port=22, username="root") 35 | 36 | execute_remote_shell(host_client, wget_command) 37 | execute_remote_shell(host_client, rpm_command) 38 | execute_remote_shell(host_client, yum_command) 39 | host_client.close() 40 | 41 | def execute_remote_shell(host_client, shell): 42 | result = {} 43 | try: 44 | stdin, stdout, stderr = host_client.exec_command(shell) 45 | result[error] = stderr.readlines() 46 | result[output] = stdout.readlines() 47 | if(len(result[error]) > 0): 48 | print(result[error][0].replace("\n", "")) 49 | else: 50 | print(result[output][0].replace("\n", "")) 51 | except: 52 | host_client.close() 53 | return result 54 | 55 | if(__name__ == "__main__"): 56 | args = check_arguments() 57 | install_mha_package(args) 58 | print("install {0} ok.".format(args.package)) 59 | 60 | -------------------------------------------------------------------------------- /tools/mha/mha_manual: -------------------------------------------------------------------------------- 1 | 2 | MHA安装手册 3 | 4 | #1.安装MHA 5 | #本地安装 6 | python mha_auto_install.py --package=/opt/mha* 7 | #远程安装 8 | python mha_auto_install.py --host=192.168.1.100 --package=/opt/mha* 9 | 10 | #2.配置文件 11 | 配置文件要注意机器的顺序，以及一些脚本的路径等等 12 | 13 | #3.测试SSH和Repl是否正常 14 | masterha_check_ssh --conf=/app.conf 15 | masterha_check_repl --conf=/app.conf 16 | 17 | #4.启动MHA 18 | nohup masterha_manager --conf=/app.conf --ignore_last_failover & 19 | 20 | #5.进行故障测试，并注意VIP是否转移 21 | 22 | #6.在做为主的备库上增加purge_relay_log.sh脚本，定时清除relay log 23 | 24 | #7.如果MHA进行了故障转移了，需要在切换回来，可以使用masterha_master_switch 25 | masterha_master_switch --master_state=alive --conf=/app1.cnf --new_master_host=192.168.1.100 --orig_master_is_new_slave --running_updates_limit=3 -------------------------------------------------------------------------------- /tools/mha/mha线上故障恢复: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/tools/mha/mha线上故障恢复 -------------------------------------------------------------------------------- /tools/mha/package/mha-0.56.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/tools/mha/package/mha-0.56.zip -------------------------------------------------------------------------------- /tools/mha/package/mha-0.57.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/tools/mha/package/mha-0.57.zip -------------------------------------------------------------------------------- /tools/mha/purge_relay_log.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | host="192.168.1.100" 4 | port=3306 5 | user=root 6 | passwd="123456" 7 | 8 | log_dir="/opt/mysql/binlog" 9 | work_dir="/opt/mysql/binlog" 10 | purge_shell="/usr/local/bin/purge_relay_logs" 11 | 12 | if [ ! -d ${log_dir} ] 13 | then 14 | mkdir ${log_dir} -p 15 | fi 16 | 17 | ${purge_shell} --host=${host} --user=${user} --password=${passwd} --disable_relay_log_purge --port=${port} --workdir=${work_dir} >> ${log_dir}/purge_relay_logs.log 2>&1 18 | 19 | 20 | #把脚本添加到crontab中去 21 | #* */1 * * * ./purge_relay_log.sh 22 | -------------------------------------------------------------------------------- /tools/mha/start_mha.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | masterha_manager --conf=$1 --ignore_last_failover & -------------------------------------------------------------------------------- /tools/mysql/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Administrator' 2 | -------------------------------------------------------------------------------- /tools/mysql/common_sql.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | #查看没有主键的表 4 | no_primary_key_sql = """ 5 | SELECT DISTINCT 6 | CONCAT(t.table_schema,'.',t.table_name) as tbl, 7 | t.engine, 8 | IF(ISNULL(c.constraint_name),'NOPK','') AS nopk, 9 | IF(s.index_type = 'FULLTEXT','FULLTEXT','') as ftidx, 10 | IF(s.index_type = 'SPATIAL','SPATIAL','') as gisidx 11 | FROM information_schema.tables AS t 12 | LEFT JOIN information_schema.key_column_usage AS c ON (t.table_schema = c.constraint_schema AND t.table_name = c.table_name AND c.constraint_name = 'PRIMARY') 13 | LEFT JOIN information_schema.statistics AS s ON (t.table_schema = s.table_schema AND t.table_name = s.table_name AND s.index_type IN ('FULLTEXT','SPATIAL')) 14 | WHERE t.table_schema NOT IN ('information_schema','performance_schema','mysql') 15 | AND t.table_type = 'BASE TABLE' AND (t.engine <> 'InnoDB' OR c.constraint_name IS NULL OR s.index_type IN ('FULLTEXT','SPATIAL')) 16 | ORDER BY t.table_schema,t.table_name;""" 17 | -------------------------------------------------------------------------------- /tools/mysql/mysql_create_slave_for_mysqldump.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pymysql, time, argparse, sys, commands, os 4 | 5 | ''' 6 | grant all on *.* to 'yangcg'@'%' identified by 'yangcaogui'; 7 | grant replication slave on *.* to 'sys_repl'@'%' identified by 'yangcaogui'; 8 | 9 | python mysql_create_slave_for_mysqldump.py \ 10 | --host=slave1 --user=yangcg --password=yangcaogui --port=3306 \ 11 | --master-host=master --master-user=yangcg --master-password=yangcaogui --master-port=3306 \ 12 | --repl-user=sys_repl --repl-password=yangcaogui --repl-mode=2 13 | ''' 14 | 15 | #参数详解 16 | #--host：从库地址 17 | #--port：从库端口 18 | #--user：从库用于执行sql的用户 19 | #--password：从库用户密码 20 | 21 | #--master-host 22 | #--master-port 23 | #--master-user 24 | #--master-password 25 | 26 | #--base-dir：mysql命令路径 27 | #--charset：设置字符集 28 | #--repl-mode：binlog模式[1-POS] or [2-GTID] 29 | 30 | def check_arguments(): 31 | parser = argparse.ArgumentParser() 32 | parser.add_argument("--host", type=str, dest="host", help="mysql slave host") 33 | parser.add_argument("--port", type=int, dest="port", help="mysql slave port", default=3306) 34 | parser.add_argument("--user", type=str, dest="user", help="mysql slave user") 35 | parser.add_argument("--password", type=str, dest="password", help="mysql slave password") 36 | parser.add_argument("--master-host", type=str, dest="master_host", help="mysql master host") 37 | parser.add_argument("--master-port", type=int, dest="master_port", help="mysql master port", default=3306) 38 | parser.add_argument("--master-user", type=str, dest="master_user", help="mysql master user") 39 | parser.add_argument("--master-password", type=str, dest="master_password", help="mysql master password") 40 | 41 | parser.add_argument("--base-dir", type=str, dest="base_dir", help="mysql base dir", default="/usr/local/mysql/bin") 42 | parser.add_argument("--charset", type=str, dest="charset", help="mysql charset", default="utf8mb4") 43 | parser.add_argument("--mysqldump-path", type=str, dest="mysqldump_path", help="mysql dump path", default="/opt/master_data.sql") 44 | parser.add_argument("--repl-user", type=str, dest="repl_user", help="mysql replication user name") 45 | parser.add_argument("--repl-password", type=str, dest="repl_password", help="mysql replication user password") 46 | parser.add_argument("--repl-mode", type=int, dest="repl_mode", help="mysql replication mode [1-POS] or [2-GTID]", default=1) 47 | args = parser.parse_args() 48 | 49 | if not args.host or not args.port or not args.user or not args.password or \ 50 | not args.master_host or not args.master_port or not args.master_user or not args.master_password or \ 51 | not args.repl_user or not args.repl_password: 52 | sys.exit(1) 53 | return args 54 | 55 | def check_accout_is_ok(args): 56 | execute_sql_for_slave(args, "select 2;") 57 | execute_sql_for_master(args, "select 1;") 58 | 59 | def create_slave(args): 60 | check_accout_is_ok(args) 61 | os.system("PATH=$PATH:{0}".format(args.base_dir)) 62 | 63 | #1.创建备份 64 | print("\n-------------------------------1.create mysqldump data-------------------------------------") 65 | execute_sql_for_slave(args, "stop slave;") 66 | shell = "mysqldump -h{0} -u{1} -p{2} -P{3} " \ 67 | "--max-allowed-packet=1G --single-transaction --master-data=2 " \ 68 | "--default-character-set={4} --triggers --routines --events -B --all-databases > {5}"\ 69 | .format(args.master_host, args.master_user, args.master_password, args.master_port, args.charset, args.mysqldump_path) 70 | os.system(shell) 71 | 72 | #2.导入数据-对导入速度进行优化，不要设置双1，而且设置不写binlog 73 | print("\n-------------------------------2.import mysqldump data-------------------------------------") 74 | os.system("mysql -h{0} -u{1} -p{2} -P{3} --max-allowed-packet=1G --default-character-set={4} < {5}" 75 | .format(args.host, args.user, args.password, args.port, args.charset, args.mysqldump_path)) 76 | 77 | #4.进行change master操作 78 | print("\n-------------------------------3.change master operation-----------------------------------") 79 | change_master(args) 80 | 81 | #5.监测从库状态是否正确，如果没有异常，则创建从库成功 82 | print("\n-------------------------------4.check slave status is ok----------------------------------") 83 | check_slave_is_ok(args) 84 | 85 | print("\n-------------------------------5.create slave is ok----------------------------------------") 86 | 87 | def change_master(args): 88 | if(args.repl_mode == 1): 89 | #普通复制 90 | status, output = commands.getstatusoutput("head -n50 {0} | grep -i 'change master to'".format(args.mysqldump_path)) 91 | sql = output[2:len(output)-1] + "," 92 | elif(args.repl_mode == 2): 93 | #gtid复制 94 | sql = "change master to master_auto_position=1," 95 | sql = sql + " master_host='{0}', master_port={1}, master_user='{2}', master_password='{3}';"\ 96 | .format(args.master_host, args.master_port, args.repl_user, args.repl_password) 97 | sql += "flush privileges;" 98 | execute_sql_for_slave(args, sql) 99 | execute_sql_for_slave(args, "start slave;") 100 | 101 | def check_slave_is_ok(args): 102 | number = 1 103 | while(number <= 5): 104 | result = execute_sql_for_slave(args, "show slave status;") 105 | if(len(result) > 0): 106 | result = result[0] 107 | if(result["Slave_IO_Running"] != "Yes" or result["Slave_SQL_Running"] != "Yes"): 108 | print(result["Last_Error"]) 109 | else: 110 | print("IO Thread: Yes | SQL Thread: Yes") 111 | number = number + 1 112 | time.sleep(1) 113 | 114 | def execute_sql_for_slave(args, sql): 115 | return execute_sql(args.host, args.port, args.user, args.password, args.charset, sql) 116 | 117 | def execute_sql_for_master(args, sql): 118 | return execute_sql(args.master_host, args.master_port, args.master_user, args.master_password, args.charset, sql) 119 | 120 | def execute_sql(host, port, user, password, charset, sql): 121 | cursor = None 122 | connection = None 123 | try: 124 | connection = pymysql.connect(host=host, port=port, user=user, password=password, charset=charset, cursorclass=pymysql.cursors.DictCursor) 125 | cursor = connection.cursor() 126 | cursor.execute(sql) 127 | return cursor.fetchall() 128 | finally: 129 | if(cursor != None): 130 | cursor.close() 131 | if(connection != None): 132 | connection.close() 133 | 134 | if(__name__ == "__main__"): 135 | create_slave(check_arguments()) -------------------------------------------------------------------------------- /tools/mysql/mysql_create_slave_for_xtrabackup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pymysql, time, argparse, sys, random, string, commands, os 4 | 5 | 6 | def check_arguments(): 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument("--host", type=str, dest="host", help="mysql slave host") 9 | parser.add_argument("--port", type=int, dest="port", help="mysql slave port", default=3306) 10 | parser.add_argument("--user", type=str, dest="user", help="mysql slave user") 11 | parser.add_argument("--password", type=str, dest="password", help="mysql slave password") 12 | parser.add_argument("--master-host", type=str, dest="master_host", help="mysql master host") 13 | parser.add_argument("--master-port", type=int, dest="master_port", help="mysql master port", default=3306) 14 | parser.add_argument("--master-user", type=str, dest="master_user", help="mysql master user") 15 | parser.add_argument("--master-password", type=str, dest="master_password", help="mysql master password") 16 | 17 | parser.add_argument("--base-dir", type=str, dest="base_dir", help="mysql base dir", default="/usr/local/mysql/") 18 | parser.add_argument("--charset", type=str, dest="charset", help="mysql charset", default="utf8") 19 | parser.add_argument("--mysqldump-path", type=str, dest="mysqldump_path", help="mysql dump path", default="/opt/master_data.sql") 20 | parser.add_argument("--repl-user", type=str, dest="repl_user", help="mysql replication user name", default="sys_repl_"+get_password(2)) 21 | parser.add_argument("--repl-password", type=str, dest="repl_password", help="mysql replication user password", default=get_password(10)) 22 | parser.add_argument("--repl-mode", type=int, dest="repl_mode", help="mysql replication mode [1-POS] or [2-GTID]", default=1) 23 | args = parser.parse_args() 24 | 25 | if not args.host or not args.port or not args.user or not args.password or \ 26 | not args.master_host or not args.master_port or not args.master_user or not args.master_password: 27 | sys.exit(1) 28 | return args 29 | 30 | def check_accout_is_ok(args): 31 | execute_sql_for_slave(args, "select 2;") 32 | execute_sql_for_master(args, "select 1;") 33 | 34 | def create_slave_for_xtrabackup(args): 35 | #1.全量备份 36 | shell = "innobackupex --defaults-file={0} --no-timestamp " \ 37 | "--host={1} --user={2} --password='{3}' --port={4} /slave_bak/" \ 38 | .format("/etc/my.cnf", args.master_host, args.master_user, args.master_password, args.master_port) 39 | 40 | #2.scp数据拷贝到目标机器 41 | shell = "scp /slave_bak/ root@{0}:/slave_bak/".format(args.host) 42 | 43 | #3.在目标机器进行恢复操作 44 | shell = "innobackupex --defaults-file=/etc/my.cnf --apply-log --use-momery=2G /slave_bak/" 45 | 46 | #4.拷贝数据到目标目录，如果不想改，需要修改配置文件数据目录路径 47 | shell = "mv /slave_bak/ /mysql_data/" 48 | 49 | #5.启动数据库 50 | shell = "mysqld --defaults-file=/etc/my.cnf &" 51 | 52 | #6.change master操作，需要分为POS和GTID设置 53 | pass 54 | 55 | def change_master(args): 56 | if(args.repl_mode == 1): 57 | #普通复制 58 | status, output = commands.getstatusoutput("head -n50 {0} | grep -i 'change master to'".format(args.mysqldump_path)) 59 | sql = output[2:len(output)-1] + "," 60 | elif(args.repl_mode == 2): 61 | #gtid复制 62 | sql = "change master to master_auto_position=1," 63 | sql = sql + " master_host='{0}', master_port={1}, master_user='{2}', master_password='{3}';"\ 64 | .format(args.master_host, args.master_port, args.repl_user, args.repl_password) 65 | execute_sql_for_slave(args, sql) 66 | execute_sql_for_slave(args, "start slave;") 67 | 68 | def check_slave_is_ok(args): 69 | number = 1 70 | while(number <= 5): 71 | result = execute_sql_for_slave(args, "show slave status;") 72 | if(result[0]["Slave_IO_Running"] != "Yes" or result[0]["Slave_SQL_Running"] != "Yes"): 73 | print(result["Last_Error"]) 74 | else: 75 | print("IO Thread: Yes | SQL Thread: Yes") 76 | number = number + 1 77 | time.sleep(1) 78 | 79 | def execute_sql_for_slave(args, sql): 80 | return execute_sql(args.host, args.port, args.user, args.password, args.charset, sql) 81 | 82 | def execute_sql_for_master(args, sql): 83 | return execute_sql(args.master_host, args.master_port, args.master_user, args.master_password, args.charset, sql) 84 | 85 | def execute_sql(host, port, user, password, charset, sql): 86 | cursor = None 87 | connection = None 88 | try: 89 | print(host, port, user, password, charset, sql) 90 | connection = pymysql.connect(host=host, port=port, user=user, password=password, charset=charset, cursorclass=pymysql.cursors.DictCursor) 91 | cursor = connection.cursor() 92 | cursor.execute(sql) 93 | return cursor.fetchall() 94 | finally: 95 | if(cursor != None): 96 | cursor.close() 97 | if(connection != None): 98 | connection.close() 99 | 100 | 101 | -------------------------------------------------------------------------------- /tools/mysql_backup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | ''' 4 | 备份脚本： 5 | 1.满足mysqldump和xtrabackup的备份选择 6 | 2.支持增量和全量备份两种 7 | 3.支持备份时间选择 8 | 4.支持旧备份数据删除 9 | 5.支持配置文件和参数化选择 10 | 6.要考虑大数据的备份方案，小数据可以选择mysqldump 11 | 7.还可以考虑备份binlog方式 12 | 8.备份文件名采用日期+时间的方式 13 | 8.mysqldump是否需要压缩 14 | 15 | 主机id，备份类型=全量，全量加增量 16 | 17 | ''' 18 | 19 | #pip install python-crontab 20 | 21 | from crontab import CronTab 22 | 23 | host = "" 24 | port = 3306 25 | user = "" 26 | password = "" 27 | backup_path = "" 28 | 29 | is_gzip = 0 30 | 31 | mysqldump_is_gzip = 0 32 | mysqldump_file_name = "" 33 | mysqldump = "mysqldump -h{0} -u{1} -p{2} -P{3} " \ 34 | "--max-allowed-packet=1G --single-transaction --master-data=2 " \ 35 | "--default-character-set=utf8mb4 --triggers --routines --events -B --all-databases > {4}" 36 | 37 | 38 | #xtrabackup 39 | 40 | 41 | #1.检查是全备还是增量备份 42 | #2.增量只支持xtrabackup，通过配置文件 43 | 44 | 45 | #备份路径 46 | #是否是全量 47 | 48 | #如果是增量什么时候增量，什么时候全量 49 | #比如周日全量，其它增量 50 | # 51 | 52 | #比如15天备份一个周期，第一天全量，第二天增量 53 | #days=20 说名20备份一个周期 54 | 55 | #新建一个文件，专门存储备份目录，放在备份目录下 56 | #backup.txt 57 | #存上一次备份路径用于增量备份 latest_backup_dir=/backup/ 58 | #存距离第一次全量备份间隔了多少天 interval_days=1 59 | 60 | #定义任务设置，比如晚上几点开始备份 61 | #hour=3 minute=30 62 | 63 | #要不要删除备份的数据 64 | #del_days=20 删除20天前的备份 65 | 66 | backup_dir = "/backup/" 67 | backup_info = "/backup/backup.txt" 68 | backup_log_dir = "/backup/log/" 69 | 70 | interval_days=1 71 | latest_backup_dir = "" 72 | 73 | def full_backup(): 74 | pass 75 | 76 | def incremental_backup(): 77 | pass 78 | 79 | my_cron = CronTab(user=True) 80 | 81 | job = my_cron.new(command="echo `date` >> /tmp/time.log") 82 | 83 | 84 | 85 | job.hour.every(5) 86 | job.set_comment("ycg job") 87 | job.setall("*/2 * * * *") 88 | 89 | my_cron.write() 90 | 91 | print("----------------------------------") 92 | for value in my_cron.crons: 93 | print(value) 94 | 95 | print("ok") -------------------------------------------------------------------------------- /tools/mysql_general_log.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/tools/mysql_general_log.sh -------------------------------------------------------------------------------- /tools/mysql_slow_log.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | #MySQL慢查询监控脚本 4 | #需要安装pt-tools工具包，然后把脚本设置为后台执行 5 | #注意如果SQL比较长需要修改sample字段为mediumtext 6 | #如果执行set global slow_query_log_file='';报错，注意slow_log目录权限的问题 7 | 8 | #alter table table1 modify sample mediumtext not null; 9 | #alter table table1 add column created_time timestamp not null default now(), 10 | #add column updated_time timestamp not null default current_timestamp on update current_timestamp, 11 | #ADD COLUMN `is_reviewed` TINYINT UNSIGNED NOT NULL DEFAULT 0; 12 | 13 | #ALTER TABLE `db1`.`mysql_slow_query_review` 14 | #CHANGE COLUMN `id` `id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT , 15 | #DROP PRIMARY KEY, 16 | #ADD PRIMARY KEY (`id`), 17 | #ADD UNIQUE INDEX `checksum_UNIQUE` (`checksum` ASC); 18 | 19 | #目前只之前percona分支版本 20 | #set global log_slow_verbosity = 'query_plan,innodb,microtime'; 21 | 22 | #1.slow log存放的地址 23 | #第一处修改的地方 24 | #慢查询数据上传的MySQL机器 25 | db_host="" 26 | db_port=3306 27 | db_user="" 28 | db_password="" 29 | db_database="mysql_web" 30 | 31 | #2.slow log本身的地址 32 | #第二处修改的地方 33 | #监控的慢查询MySQL机器 34 | mysql_client="/usr/local/mysql/bin/mysql" 35 | mysql_host="" 36 | mysql_port=3306 37 | mysql_user="" 38 | mysql_password="" 39 | 40 | #3.读取有关慢日志配置信息 41 | #第三处修改的地方，路径不改也没事 42 | slow_log_dir="/slow_log/" 43 | slow_query_long_time=2 44 | pt_query_digest="/usr/bin/pt-query-digest" 45 | 46 | #4.mysql_host_id对应着mysql_web.host_infos中id字段值 47 | #第四处修改的地址,这个字段很重要 48 | mysql_host_id=14 49 | 50 | function check_slow_log() 51 | { 52 | #获取slow log文件路径 53 | slow_log_file=`${mysql_client} -h${mysql_host} -P${mysql_port} -u${mysql_user} -p${mysql_password} \ 54 | -e "show variables like 'slow_query_log_file';" | grep log | awk '{print $2}'` 55 | 56 | echo "$slow_log_file" 57 | #设置新的slow log文件路径 58 | slow_log_file_path=${slow_log_dir}${mysql_host_id}_`date "+%Y-%m-%d_%H-%M-%S.slow"` 59 | `${mysql_client} -h${mysql_host} -P${mysql_port} -u${mysql_user} -p${mysql_password} \ 60 | -e "set global slow_query_log = 1;set global long_query_time=$slow_query_long_time;set global log_output='FILE';set global slow_query_log_file='$slow_log_file_path'"` 61 | 62 | echo "=========================start check================================" 63 | #对旧的slow log进行解析并入库 64 | #\$event->{user} !~ m/^serverdev/i 不包含某个用户的过滤，包含就把!改成= 65 | ${pt_query_digest} \ 66 | --user=${db_user} --password=${db_password} --port=${db_port} --charset=utf8 \ 67 | --review h=${db_host},D=${db_database},t=mysql_slow_query_review \ 68 | --history h=${db_host},D=${db_database},t=mysql_slow_query_review_history \ 69 | --no-report --limit=100% --filter=" \$event->{add_column} = length(\$event->{arg}) and \$event->{serverid}=$mysql_host_id and \$event->{user} !~ m/^serverdev/i" ${slow_log_file} > /tmp/slowquery.log 70 | 71 | echo "================check slow log [$slow_log_file] ok====================" 72 | if [ -f "$slow_log_file" ]; then 73 | rm -f "$slow_log_file" 74 | echo "remove $slow_log_file ok." 75 | fi 76 | echo "" 77 | } 78 | 79 | if [ ! -d "$slow_log_dir" ]; then 80 | mkdir "$slow_log_dir" 81 | fi 82 | 83 | chown -R mysql:mysql ${slow_log_dir} 84 | 85 | while true 86 | do 87 | check_slow_log 88 | sleep 5m 89 | done 90 | -------------------------------------------------------------------------------- /tools_new/binlog_bk.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import random, argparse, sys, os, commands, time, traceback 4 | 5 | # 封装mysqlbinlog远程备份binlog脚本 6 | 7 | # 参数详解： 8 | # file-path：指定备份目录，不指定则备份在当前执行脚本的目录 9 | # server-id：用户复制的server_id，不指定则自动生成 10 | # log-file：指定日志输出文件，不指定默认在/tmp/binlog.log 11 | 12 | # 注意：需要指定mysql命令的路径，如果配置了环境变量，可以把path设置为字符串空 13 | 14 | # 最简单的例子 15 | # nohup python binlog_backup.py --host=192.168.11.128 --user=yancg --password=123456 --port=3310 & 16 | 17 | # 下面是建议的备份命令 18 | # nohup python binlog_backup.py --host=192.168.11.128 --user=yancg --password=123456 --port=3310 --file-path=/data/binlog_backup/ & 19 | 20 | path = "/opt/mysql-5.7/bin/" 21 | mysql = path + "mysql -h{0} -u{1} -p\'{2}\' -P{3} -e\"{4}\" | head -n2 | tail -n 1 | awk \'{5}\'" 22 | mysqlbinlog = path + "mysqlbinlog -h{0} -u{1} -p{2} -P{3} --stop-never --raw --read-from-remote-server --stop-never-slave-server-id={4} --result-file={5} {6} &>> {7}" 23 | 24 | 25 | def check_arguments(): 26 | parser = argparse.ArgumentParser() 27 | parser.add_argument("--host", type=str, dest="host", help="mysql host") 28 | parser.add_argument("--user", type=str, dest="user", help="mysql user") 29 | parser.add_argument("--password", type=str, dest="password", help="mysql password") 30 | parser.add_argument("--port", type=int, dest="port", help="mysql port", default=3306) 31 | parser.add_argument("--file-path", type=str, dest="file_path", help="save binlog directory", default="") 32 | parser.add_argument("--conf-file", type=str, dest="conf_file", help="configuration file", default="") 33 | parser.add_argument("--server-id", type=int, dest="server_id", help="server id") 34 | parser.add_argument("--log-file", type=str, dest="log_file", help="log file path", default="/tmp/binlog.log") 35 | args = parser.parse_args() 36 | 37 | if (not args.conf_file or args.conf_file == None): 38 | if (not args.host or not args.user or not args.password): 39 | print(parser.format_usage()) 40 | sys.exit(1) 41 | else: 42 | read_conf_file(args) 43 | if (not args.server_id or args.server_id <= 0): 44 | args.server_id = random.randint(777777, 999999) 45 | if (len(args.file_path) <= 0): 46 | args.file_path = os.getcwd() + "/" 47 | return args 48 | 49 | 50 | def read_conf_file(args): 51 | if (os.path.exists(args.conf_file) == False): 52 | print("the conf file not exists.") 53 | sys.exit(1) 54 | 55 | 56 | def get_binlog_file_name(args): 57 | files = os.listdir(args.file_path) 58 | file_count = len(files) 59 | if (file_count == 0): 60 | binglog_name = get_first_binlog_file_name(args) 61 | elif (file_count == 1): 62 | if (files[0] == os.path.basename(args.conf_file) or files[0] == sys.argv[0]): 63 | binglog_name = get_first_binlog_file_name(args) 64 | else: 65 | binglog_name = files[0] 66 | else: 67 | (status, output) = commands.getstatusoutput("ls -l {0} |tail -n 1 |awk \'{1}\'".format(args.file_path, "{print $9}")) 68 | binglog_name = output 69 | return binglog_name.replace("\n", "").replace("\t", "").replace("\r", "") 70 | 71 | 72 | def get_first_binlog_file_name(args): 73 | mysql_shell = mysql.format(args.host, args.user, args.password, args.port, "show master logs;", "{print $1}") 74 | (status, output) = commands.getstatusoutput(mysql_shell) 75 | return output.split("\n")[1] 76 | 77 | 78 | if (__name__ == "__main__"): 79 | args = check_arguments() 80 | while (True): 81 | try: 82 | print("start remote backup mysql binlog...") 83 | binlog_backup = mysqlbinlog.format(args.host, args.user, args.password, args.port, args.server_id, args.file_path, get_binlog_file_name(args), args.log_file) 84 | os.system(binlog_backup) 85 | time.sleep(10) 86 | print("retry remote backuo mysql binlog...") 87 | except: 88 | traceback.print_exc() 89 | 90 | -------------------------------------------------------------------------------- /tools_new/bk_recovery_xtrabackup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import argparse, sys, os, traceback, subprocess 4 | 5 | # xtrabackup备份恢复脚本 6 | # 一个重要细节 7 | # 是在原有上进行备份恢复，还是把数据拷贝到一个目录中去再进行恢复 8 | # 可想而知还是不要在原有的基础上恢复，还是拷贝一下吧 9 | 10 | # 参数详解 11 | # --ssh-user：备份机器用户名，默认为root 12 | # --ssh-host：备份机器host 13 | # --ssh-password：密码 14 | # --log-file：备份日志文件路径 15 | # --recovery-dir：备份恢复目录 16 | 17 | # backup.log各个分割字段含义 18 | # {0}:{1}:{2}:{3}:{4}:{5}:{6}:{7} 19 | # 备份模式:备份路径:备份日志路径:备份开始时间:备份结束时间:备份日期:备份是否正常:备份目录名称 20 | 21 | # 本地恢复调用： 22 | # python bk_recovery_xtrabackup.py --log-file=/backup_test/bk_dir/backup.log --recovery-dir=/backup_test/recovery_dir/ 23 | 24 | # 远程恢复调用： 25 | # python bk_recovery_xtrabackup.py --ssh-host=192.168.11.101 --log-file=/backup_test/bk_dir/backup.log --recovery-dir=/backup_test/recovery_dir/ 26 | 27 | BACKUP_OK = "1" 28 | FULL_BACKUP = "1" 29 | INCREMENT_BACKUP = "2" 30 | 31 | 32 | class BackupInfo(): 33 | def __init__(self): 34 | pass 35 | 36 | 37 | # 获取调用脚本参数 38 | def check_arguments(): 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument("--ssh-user", type=str, dest="ssh_user", help="ssh user", default="root") 41 | parser.add_argument("--ssh-host", type=str, dest="ssh_host", help="ssh host", default=None) 42 | parser.add_argument("--ssh-password", type=str, dest="ssh_password", help="ssh password", default=None) 43 | parser.add_argument("--log-file", type=str, dest="log_file", help="backup log file path") 44 | parser.add_argument("--recovery-dir", type=str, dest="recovery_dir", help="backup recovery dir") 45 | args = parser.parse_args() 46 | 47 | if not args.log_file or not args.recovery_dir: 48 | print("[error]:Please input log file path.") 49 | sys.exit(1) 50 | return args 51 | 52 | 53 | # 备份恢复总目录 54 | def backup_recover(args): 55 | log_lines = check_log_file_is_correct(args) 56 | backup_infos = get_backup_recover_dir_infos(args, log_lines) 57 | copy_backup_dir_to_recovery_dir(args, backup_infos) 58 | xtrbackup_recovery(backup_infos) 59 | 60 | 61 | # 监测备份日志数据是否正常 62 | def check_log_file_is_correct(args): 63 | if (args.ssh_host != None): 64 | log_file_save_path = "/tmp/" 65 | execute_linux_command("scp root@{0}:{1} {2}".format(args.ssh_host, args.log_file, log_file_save_path)) 66 | args.log_file = log_file_save_path 67 | 68 | if (os.path.exists(args.log_file) == False): 69 | print("[error]:backup log file path is error.") 70 | sys.exit(1) 71 | 72 | log_lines = read_file_lines(args.log_file) 73 | if (log_lines == None or len(log_lines) <= 0): 74 | print("[error]:the file value is error.") 75 | sys.exit(1) 76 | 77 | for value in log_lines: 78 | tmp_list = value.split(":") 79 | if (len(tmp_list) != 8): 80 | # 检测backup log文件字段的个数是否一致 81 | print("[error]:the backup log file column is error.") 82 | sys.exit(1) 83 | 84 | return log_lines 85 | 86 | 87 | # 根据备份日志获取备份目录地址 88 | def get_backup_recover_dir_infos(args, log_lines): 89 | backup_infos = [] 90 | length = len(log_lines) 91 | for index in range(length): 92 | values = log_lines[length - index - 1].replace("\n", "").split(":") 93 | # 检测备份是否正常 94 | if (values[6] == BACKUP_OK): 95 | info = BackupInfo() 96 | info.key = index 97 | info.mode = values[0] 98 | info.backup_dir = values[1] 99 | info.recovery_dir = os.path.join(args.recovery_dir, values[7]) 100 | backup_infos.append(info) 101 | # 检测当前备份是否是全量备份 102 | if (values[0] == FULL_BACKUP): 103 | break 104 | return backup_infos 105 | 106 | 107 | # 拷贝备份目录到指定的路径 108 | def copy_backup_dir_to_recovery_dir(args, backup_infos): 109 | for info in backup_infos: 110 | if (args.ssh_host != None): 111 | # 将远程的备份文件拷贝到本地 112 | print("[info]:remote {0} copy {0} to {1} start.".format(args.ssh_host, info.backup_dir, args.recovery_dir)) 113 | command = "scp -r root@{0}:{1} {2}".format(args.ssh_host, info.backup_dir, args.recovery_dir) 114 | result = subprocess.Popen(command, shell=True) 115 | result.wait() 116 | print("[info]:remote {0} copy {0} to {1} ok.".format(args.ssh_host, info.backup_dir, args.recovery_dir)) 117 | else: 118 | # 本地拷贝文件 119 | print("[info]:copy {0} to {1} start.".format(info.backup_dir, args.recovery_dir)) 120 | command = "cp -r {0} {1}".format(info.backup_dir, args.recovery_dir) 121 | result = subprocess.Popen(command, shell=True) 122 | result.wait() 123 | print("[info]:copy {0} to {1} ok.".format(info.backup_dir, args.recovery_dir)) 124 | 125 | 126 | # 使用xtrabackup进行恢复 127 | def xtrbackup_recovery(backup_infos): 128 | if (len(backup_infos) > 0): 129 | if (len(backup_infos) == 1): 130 | # 只有一个备份而且是全量备份 131 | execute_linux_command("innobackupex --apply-log --redo-only {0}".format(backup_infos[0].recovery_dir)) 132 | else: 133 | # backup_infos字段的key是从小到大排序的，也就是key最大值为全量备份，所以先倒序排列 134 | number = 1 135 | full_backup_dir = "" 136 | sort_list = sorted(backup_infos, cmp=lambda x, y: cmp(x.key, y.key), reverse=True) 137 | for info in sort_list: 138 | if (number == 1): 139 | # 全量备份恢复 140 | full_backup_dir = info.recovery_dir 141 | execute_linux_command("innobackupex --apply-log --redo-only {0}".format(info.recovery_dir)) 142 | print("[info]:recovery full backup {0} ok.".format(info.recovery_dir)) 143 | elif (number == len(sort_list)): 144 | # 最后一个增量备份恢复 145 | execute_linux_command("innobackupex --apply-log {0} --incremental-dir={1}".format(full_backup_dir, info.recovery_dir)) 146 | print("[info]:recovery latest increment backup {0} ok.".format(info.recovery_dir)) 147 | else: 148 | # 其余增量备份恢复 149 | execute_linux_command("innobackupex --apply-log --redo-only {0} --incremental-dir={1}".format(full_backup_dir, info.recovery_dir)) 150 | print("[info]:recovery increment backup {0} ok.".format(info.recovery_dir)) 151 | number += 1 152 | 153 | print("[info]:xtrabackup recovery ok.") 154 | else: 155 | print("[error]:backup log infos is empty.") 156 | 157 | 158 | # 执行linux命令 159 | def execute_linux_command(command): 160 | result = subprocess.Popen(command, shell=True) 161 | result.wait() 162 | return result 163 | 164 | 165 | # 获取文件所有的行数据 166 | def read_file_lines(file_path): 167 | file = None 168 | try: 169 | file = open(file_path, "r") 170 | return file.readlines() 171 | except: 172 | traceback.print_exc() 173 | return None 174 | finally: 175 | if (file != None): 176 | file.close() 177 | 178 | 179 | if (__name__ == "__main__"): 180 | backup_recover(check_arguments()) 181 | -------------------------------------------------------------------------------- /tools_new/collect_mysql_status_log.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 用于在固定时间段收集mysql运行状态信息的脚本 4 | # 参数详解 5 | # host_list 需要收集的mysql机器host列表，已空格分割 6 | # start_timestamp 收集日志的开始时间 7 | # end_timestamp 收集日志的结束时间 8 | # user 数据库用户名 9 | # password 数据库密码 10 | # 注意点：尽量数据库用同一个账号 11 | # 日志保存在/tmp目录下 12 | # 注意：如果默认不是3306端口，需要给host_post指定端口 13 | 14 | user="" 15 | password="" 16 | host_port=("3306" "3306") 17 | host_list=("192.168.1.101" "192.168.1.102") 18 | 19 | start_timestamp=`date +%s -d "2017-08-04 00:49:00"` 20 | stop_timestamp=`date +%s -d "2017-08-04 00:53:00"` 21 | echo ${start_timestamp}, ${stop_timestamp} 22 | 23 | innodb_engine_status="show engine innodb status\G" 24 | show_processlist="SELECT * FROM information_schema.processlist where COMMAND != 'Sleep' and LENGTH(info) > 0\G" 25 | innodb_lock="SELECT * FROM information_schema.INNODB_TRX\G SELECT * FROM information_schema.INNODB_LOCKS\G SELECT * FROM information_schema.INNODB_LOCK_WAITS\G" 26 | 27 | function collect_status_log() 28 | { 29 | index=0 30 | for host in ${host_list[*]} 31 | do 32 | mysql_port=${host_port[${index}]} 33 | innodb_lock_file="/tmp/${host}_innodb_lock.txt" 34 | processlist_file="/tmp/${host}_processlist.txt" 35 | innodb_status_file="/tmp/${host}_innodb_status.txt" 36 | 37 | echo "-----------------------------------------`date "+%Y-%m-%d %H:%M:%S"`-------------------------------------------" >> ${processlist_file} 38 | mysql -h${host} -u${user} -p${password} -P${mysql_port} --default-character-set=utf8 -e"$show_processlist" >> ${processlist_file} 39 | echo "" >> ${processlist_file} 40 | 41 | echo "-----------------------------------------`date "+%Y-%m-%d %H:%M:%S"`-------------------------------------------" >> ${innodb_status_file} 42 | mysql -h${host} -u${user} -p${password} -P${mysql_port} --default-character-set=utf8 -e"$innodb_engine_status" >> ${innodb_status_file} 43 | echo "" >> ${innodb_status_file} 44 | 45 | echo "-----------------------------------------`date "+%Y-%m-%d %H:%M:%S"`-------------------------------------------" >> ${innodb_lock_file} 46 | mysql -h${host} -u${user} -p${password} -P${mysql_port} --default-character-set=utf8 -e"$innodb_lock" >> ${innodb_lock_file} 47 | echo "" >> ${innodb_lock_file} 48 | 49 | let "index ++" 50 | done 51 | } 52 | 53 | if [ ${start_timestamp} -gt ${stop_timestamp} ]; then 54 | echo "start time must less then stop time" 55 | exit 56 | fi 57 | 58 | now_timestamp=`date +%s` 59 | while [ ${now_timestamp} -le ${stop_timestamp} ] 60 | do 61 | if [ ${now_timestamp} -ge ${start_timestamp} ] && [ ${now_timestamp} -le ${stop_timestamp} ]; then 62 | collect_status_log 63 | echo "[`date "+%Y-%m-%d %H:%M:%S"`] collect ok" 64 | else 65 | echo "[`date "+%Y-%m-%d %H:%M:%S"`] no collect" 66 | fi 67 | sleep 1 68 | now_timestamp=`date +%s` 69 | done 70 | 71 | echo "[`date "+%Y-%m-%d %H:%M:%S"`] collect mysql log stop" 72 | -------------------------------------------------------------------------------- /tools_new/mha/app.conf: -------------------------------------------------------------------------------- 1 | [server default] 2 | manager_workdir=/masterha/ 3 | manager_log=/masterha/manager.log 4 | master_binlog_dir=/mysql/binlog 5 | remote_workdir=/opt/mysql/tmpbinlog 6 | 7 | ssh_user=root 8 | ping_interval=1 9 | 10 | #mysql管理员用户 11 | user=yangcg 12 | password=yangcaogui 13 | 14 | #mysql复制用户 15 | repl_user=sys_repl 16 | repl_password=yangcaogui 17 | 18 | #MHA切换之后发生邮件报告 19 | report_script=/masterha/send_report 20 | #shutdown_script=/masterha/master_ip_failover 21 | #从其它机器ping主库机器，这个还是比较重要的，防止出现网络抖动导致误认为master当掉 22 | secondary_check_script=/masterha/masterha_secondary_check -s 192.168.11.104 -s 192.168.11.102 -s 192.168.11.103 23 | 24 | #VIP切换脚本 25 | #master_ip_failover_script=/masterha/master_ip_failover 26 | 27 | #DNS切换脚本 28 | #master_ip_failover_script=/root/mhascript/master_ip_failover 29 | 30 | #注意：5.7要使用0.57版本的MHA 31 | #注意下面server的先后顺序 32 | #第一个为master 33 | #第二个为master stanby 34 | [server1] 35 | port=3306 36 | candidate_master=1 37 | check_repl_delay=0 38 | hostname=192.168.11.100 39 | 40 | [server2] 41 | port=3306 42 | candidate_master=1 43 | check_repl_delay=0 44 | hostname=192.168.11.104 45 | 46 | [server3] 47 | port=3306 48 | no_master=1 49 | hostname=192.168.11.102 50 | 51 | [server4] 52 | port=3306 53 | no_master=1 54 | hostname=192.168.11.103 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /tools_new/mha/master_ip_failover: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) 2011 DeNA Co.,Ltd. 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program; if not, write to the Free Software 17 | # Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 | 20 | ## Note: This is a sample script and is not complete. Modify the script based on your environment. 21 | 22 | use strict; 23 | use warnings FATAL => 'all'; 24 | 25 | use Getopt::Long; 26 | use MHA::DBHelper; 27 | 28 | #要切换的IP地址 29 | my $vip = "192.168.11.120"; 30 | #要绑定的网卡名称 31 | my $devc = "eth0"; 32 | 33 | my ( 34 | $command, $ssh_user, $orig_master_host, 35 | $orig_master_ip, $orig_master_port, $new_master_host, 36 | $new_master_ip, $new_master_port, $new_master_user, 37 | $new_master_password 38 | ); 39 | 40 | GetOptions( 41 | 'command=s' => \$command, 42 | 'ssh_user=s' => \$ssh_user, 43 | 'orig_master_host=s' => \$orig_master_host, 44 | 'orig_master_ip=s' => \$orig_master_ip, 45 | 'orig_master_port=i' => \$orig_master_port, 46 | 'new_master_host=s' => \$new_master_host, 47 | 'new_master_ip=s' => \$new_master_ip, 48 | 'new_master_port=i' => \$new_master_port, 49 | 'new_master_user=s' => \$new_master_user, 50 | 'new_master_password=s' => \$new_master_password, 51 | ); 52 | 53 | sub add_vip { 54 | my $output1 = `ssh -o ConnectTimeout=15 -o ConnectionAttempts=3 $orig_master_host /sbin/ip addr del $vip dev $devc`; 55 | my $output2 = `ssh -o ConnectTimeout=15 -o ConnectionAttempts=3 $new_master_host /sbin/ip addr add $vip dev $devc`; 56 | } 57 | 58 | 59 | exit &main(); 60 | 61 | sub main { 62 | if ( $command eq "stop" || $command eq "stopssh" ) { 63 | 64 | # $orig_master_host, $orig_master_ip, $orig_master_port are passed. 65 | # If you manage master ip address at global catalog database, 66 | # invalidate orig_master_ip here. 67 | my $exit_code = 1; 68 | eval { 69 | 70 | # updating global catalog, etc 71 | $exit_code = 0; 72 | }; 73 | if ($@) { 74 | warn "Got Error: $@\n"; 75 | exit $exit_code; 76 | } 77 | exit $exit_code; 78 | } 79 | elsif ( $command eq "start" ) { 80 | 81 | # all arguments are passed. 82 | # If you manage master ip address at global catalog database, 83 | # activate new_master_ip here. 84 | # You can also grant write access (create user, set read_only=0, etc) here. 85 | my $exit_code = 10; 86 | eval { 87 | my $new_master_handler = new MHA::DBHelper(); 88 | 89 | # args: hostname, port, user, password, raise_error_or_not 90 | $new_master_handler->connect( $new_master_ip, $new_master_port, 91 | $new_master_user, $new_master_password, 1 ); 92 | 93 | ## Set read_only=0 on the new master 94 | $new_master_handler->disable_log_bin_local(); 95 | print "Set read_only=0 on the new master.\n"; 96 | $new_master_handler->disable_read_only(); 97 | 98 | ## Creating an app user on the new master 99 | #print "Creating app user on the new master..\n"; 100 | #FIXME_xxx_create_user( $new_master_handler->{dbh} ); 101 | $new_master_handler->enable_log_bin_local(); 102 | $new_master_handler->disconnect(); 103 | 104 | ## Update master ip on the catalog database, etc 105 | &add_vip(); 106 | $exit_code = 0; 107 | }; 108 | if ($@) { 109 | warn $@; 110 | 111 | # If you want to continue failover, exit 10. 112 | exit $exit_code; 113 | } 114 | exit $exit_code; 115 | } 116 | elsif ( $command eq "status" ) { 117 | 118 | # do nothing 119 | exit 0; 120 | } 121 | else { 122 | &usage(); 123 | exit 1; 124 | } 125 | } 126 | 127 | sub usage { 128 | print 129 | "Usage: master_ip_failover --command=start|stop|stopssh|status --orig_master_host=host --orig_master_ip=ip --orig_master_port=port --new_master_host=host --new_master_ip=ip --new_master_port=port\n"; 130 | } 131 | -------------------------------------------------------------------------------- /tools_new/mha/masterha_master_switch: -------------------------------------------------------------------------------- 1 | #masterha_master_switch 2 | 3 | #如果主出现故障，手动进行故障转移 4 | masterha_master_switch --master_state=dead --conf=/usr/local/masterha/conf/app1.cnf --dead_master_host=192.168.1.100 5 | 6 | #当在线的MHA进行主故障转移之后，主恢复之后，重新进行切换的命令 7 | masterha_master_switch --master_state=alive --conf=/usr/local/masterha/conf/app1.cnf --new_master_host=192.168.1.100 --orig_master_is_new_slave --running_updates_limit=3 8 | 9 | 10 | 参数： 11 | --running_updates_limit：在所有的slaves上Seconds_Behind_Master都要小于此参数值(秒数) 12 | 13 | --master_state=dead：强制的参数，参数值为"dead" 或者 "alive" . 如果 设置为 alive 模式，masterha_master_switch 开始在线主库切换操作。 14 | 15 | --dead_master_host=(hostname)：强制参数，宕机的主库所在的主机名称。--dead_master_ip 和 --dead_master_port 是可选参数，如果这些参数没有设置，--dead_master_ip 就是 --dead_master_host 解析的IP地址。--dead_master_port 为 3306 16 | 17 | --new_master_host=(hostname)：新主机地址，可选参数，这个参数在你明确新的主库的主机，非常有用。(这就意味着你不需要让MHA来决定新的主库)。如果不设置此参数，MHA 将会利用自动failover的规则来选择新的主库。如果设置--new_master_host，MHA选择此主机为新的主库，如果不能成为主库，MHA将会退出 18 | 19 | --interactive=(0|1)：如果设置为0，在masterha_master_switch，它自动执行故障转移(非交互式)。这实际上是和masterha_manager的内部运行机制一样，这种非交互式故障转移是有用的，如果你已经证实了master死了,但你想尽快做故障转移。非交互式故障转移也是有用的,如果你使用其他现有的主监控软件和要调用的非交互式故障转移命令软件。典型的例子是masterha_master_switch调用从集群软件像起搏器。 20 | 21 | --ssh_reachable=(0|1|2)：指定master 经过SSH是否可达。0:不可达、1:可达、2:未知(默认值)。 如果设置为了2，此命令内部将会检测通过SSH 是否可达master，并且跟新SSH 状态。如果可达，且设置master_ip_failover_script 或者 shutdown_script .将会执行"--command=stopssh"。否则，执行 "--command=stop"。另外，如果宕机的master通过SSH可达，failover脚本试图从宕机的master机器上拷贝没有没有发送的binlog。 22 | 23 | --skip_change_master：如果设置此参数，当发生failover的时候，MAH 在应用完不同的relay log退出，忽略CHANGE MASTER 和 START SLAVE 操作。所以 slaves 不会指向 新的master. 开启此参数，有利于手动的二次检查slave 恢复是否成功 24 | 25 | --skip_disable_read_only：设置此参数，MHA 将不会在新的主库上执行 SET GLOBAL read_only =0 操作，有利于手动操作 26 | 27 | --last_failover_minute=(minutes)：参考master_manager 28 | 29 | --ignore_last_failover：参考master_manager 30 | 31 | --wait_on_failover_error=(seconds)：类似于master_manager, 此参数只用于自动的/非交互式的failover。如果没有设置--interval=0，wait_on_failover_error 将会被忽略，在发生错误的时候不会sleep。 32 | 33 | --remove_dead_master_conf：参考masterha_manager 34 | 35 | --wait_until_gtid_in_sync(0|1)：此参数从0.56版本开始可用，如果设置成1，当基于GITD的failover时,MHA 会等待所有的从库追上新主库的GITD 36 | 37 | --skip_change_master：此参数从0.56版本开始可用，如果开启此选项，MHA 跳过 CHANGE MASTER 的操作 38 | 39 | --skip_disable_read_only：此参数从0.56版本开始可用，如果开启此选项，MHA 将会在新的master 跳过 SET GLOBAL read_only = 0; 40 | 41 | --ignore_binlog_server_error：此参数从0.56版本开始可用，如果开启此选项，当执行failover的时，MHA忽略binlog server上任何错误 42 | 43 | 主在线切换时相关参数： 44 | --new_master_host=(hostname)：新主机地址，可选参数，这个参数在你明确新的主库的主机，非常有用。(这就意味着你不需要让MHA来决定新的主库)。如果不设置此参数，MHA 将会利用自动failover的规则来选择新的主库。如果设置--new_master_host，MHA选择此主机为新的主库，如果不能成为主库，MHA将会退出 45 | 46 | --orig_master_is_new_slave：当完成主库切换后，原先的主库将作为现在主库的slave运行。默认:不开启(原先的主库不会加入到新的复制环境中)。如果开启此选项，需要在配置文件中设置repl_password参数，由于当期的Master并不知道新的Master的replication的密码 47 | 48 | --remove_orig_master_conf：如果设置此参数，当成功failover后，MHA manager将会自动删除配置文件中关于dead master的配置选项。 49 | 50 | --skip_lock_all_tables：当在做主库切换的时候，MHA会在原先的主库上执行FLUSH TABLES WITH READ LOCK 操作，确保没有跟新操作，但是FLUSH TABLES WITH READ LOCK 操作是非常耗费资源的，并且你可以在原先的主库确定没有跟新操作(通过master_ip_online_change_script 中kill all clients操作等)。可以利用此选项避免锁表。 51 | -------------------------------------------------------------------------------- /tools_new/mha/mha_auto_install.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import argparse, paramiko, os 4 | 5 | #自动安装mha前提是上传mha安装包 6 | 7 | error = "error" 8 | output = "output" 9 | dowload_path = "/tmp/" 10 | 11 | def check_arguments(): 12 | global dowload_path 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--host", type=str, dest="host", help="linux host") 15 | parser.add_argument("--package", type=str, dest="package", help="mha package path") 16 | args = parser.parse_args() 17 | return args 18 | 19 | def install_mha_package(args): 20 | yum_command = "yum localinstall -y {0}mha4mysql-*.rpm".format(dowload_path) 21 | rpm_command = "rpm -ivh {0}epel-release-latest-7.noarch.rpm".format(dowload_path) 22 | wget_command = "wget http://mirrors.aliyun.com/epel/epel-release-latest-7.noarch.rpm -P /tmp/" 23 | 24 | if(not args.host): 25 | os.system("cp {0} {1}".format(args.package, dowload_path)) 26 | os.system(wget_command) 27 | os.system(rpm_command) 28 | os.system(yum_command) 29 | else: 30 | os.system("scp {0} {1}:/tmp/".format(args.package, args.host)) 31 | 32 | host_client = paramiko.SSHClient() 33 | host_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 34 | host_client.connect(args.host, port=22, username="root") 35 | 36 | execute_remote_shell(host_client, wget_command) 37 | execute_remote_shell(host_client, rpm_command) 38 | execute_remote_shell(host_client, yum_command) 39 | host_client.close() 40 | 41 | def execute_remote_shell(host_client, shell): 42 | result = {} 43 | try: 44 | stdin, stdout, stderr = host_client.exec_command(shell) 45 | result[error] = stderr.readlines() 46 | result[output] = stdout.readlines() 47 | if(len(result[error]) > 0): 48 | print(result[error][0].replace("\n", "")) 49 | else: 50 | print(result[output][0].replace("\n", "")) 51 | except: 52 | host_client.close() 53 | return result 54 | 55 | if(__name__ == "__main__"): 56 | args = check_arguments() 57 | install_mha_package(args) 58 | print("install {0} ok.".format(args.package)) 59 | 60 | -------------------------------------------------------------------------------- /tools_new/mha/mha_manual: -------------------------------------------------------------------------------- 1 | 2 | MHA安装手册 3 | 4 | #1.安装MHA 5 | #本地安装 6 | python mha_auto_install.py --package=/opt/mha* 7 | #远程安装 8 | python mha_auto_install.py --host=192.168.1.100 --package=/opt/mha* 9 | 10 | #2.配置文件 11 | 配置文件要注意机器的顺序，以及一些脚本的路径等等 12 | 13 | #3.测试SSH和Repl是否正常 14 | masterha_check_ssh --conf=/app.conf 15 | masterha_check_repl --conf=/app.conf 16 | 17 | #4.启动MHA 18 | nohup masterha_manager --conf=/app.conf --ignore_last_failover & 19 | 20 | #5.进行故障测试，并注意VIP是否转移 21 | 22 | #6.在做为主的备库上增加purge_relay_log.sh脚本，定时清除relay log 23 | 24 | #7.如果MHA进行了故障转移了，需要在切换回来，可以使用masterha_master_switch 25 | masterha_master_switch --master_state=alive --conf=/app1.cnf --new_master_host=192.168.1.100 --orig_master_is_new_slave --running_updates_limit=3 -------------------------------------------------------------------------------- /tools_new/mha/mha线上故障恢复: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/tools_new/mha/mha线上故障恢复 -------------------------------------------------------------------------------- /tools_new/mha/package/mha-0.56.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/tools_new/mha/package/mha-0.56.zip -------------------------------------------------------------------------------- /tools_new/mha/package/mha-0.57.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ycg/mysql_web/11d95acda391d2c96e8ad44b5441e982b264f5d8/tools_new/mha/package/mha-0.57.zip -------------------------------------------------------------------------------- /tools_new/mha/purge_relay_log.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | host="192.168.1.100" 4 | port=3306 5 | user=root 6 | password="123456" 7 | 8 | #mysql relay log存放的目录 9 | work_dir="/opt/mysql/binlog" 10 | #MHA的purge relay log的脚本路径地址 11 | purge_shell="/usr/local/bin/purge_relay_logs" 12 | 13 | #执行MHA的purge脚本 14 | ${purge_shell} --host=${host} --user=${user} --password=${password} --disable_relay_log_purge --port=${port} --workdir=${work_dir} >> /tmp/purge_relay_logs.log 2>&1 15 | 16 | 17 | #把脚本添加到crontab中去 18 | #* */1 * * * ./purge_relay_log.sh 19 | -------------------------------------------------------------------------------- /tools_new/mha/send_report.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | user="root" 4 | password="" 5 | to_email="ycg166911@163.com" 6 | new_master=$(echo $2 | cut -d = -f 2) 7 | orig_master=$(echo $1 | cut -d = -f 2) 8 | text="${orig_master} is down, new master is ${new_master}" 9 | 10 | mysql -u${user} -p${password} -h${new_master} -e"select HOST from information_schema.processlist where USER='repl'" 2> /dev/null | cut -d : -f 1 | sed 1d > slave_host.txt 11 | 12 | while read host_name 13 | do 14 | rpl_master_ip=$(mysql -u${user} -p${password} -h${host_name} -e"show slave status\G" 2> /dev/null | grep -i Master_Host | awk '{print $2}') 15 | if [ "${rpl_master_ip}" != "${new_master}" ]; then 16 | message="${message}${host_name}replication source exception\n" 17 | else 18 | message="${message}${host_name}replication master ip is ${rpl_master_ip}\n" ; 19 | fi 20 | 21 | slave_status=$(mysql -u${user} -p${password} -h${host_name} -e"show global status like 'Slave_running'" 2>/dev/null | grep -i "Slave_running" | awk '{print $2}') 22 | if [ "${slave_status}" == "off" ]; then 23 | message="${message}slave status is OFF\n" 24 | else 25 | message="${message}slave status is ON\n" 26 | fi 27 | done < slave_host.txt 28 | 29 | echo -e "${text}\n${message}" | mail -s "mha failover report" "${to_email}" 30 | echo -e "${text}\n${message}" 31 | echo "successfull send mail!!" 32 | -------------------------------------------------------------------------------- /tools_new/mha/start_mha.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | masterha_manager --conf=$1 --ignore_last_failover & -------------------------------------------------------------------------------- /tools_new/mysql_processlist_monitor.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pymysql 4 | from DBUtils.PooledDB import PooledDB 5 | 6 | # 可以连接mysql_web系统的host_infos表读取要监控的数据库信息 7 | 8 | # mysql主机信息列表 9 | host_infos = [] 10 | 11 | # 一个字符串，联系人通过逗号分割 12 | mail_list = "" 13 | 14 | # 连接池对象 15 | connection_pools = {} 16 | 17 | # processlist的SQL语句 18 | processlist_sql = """SELECT * FROM information_schema.processlist where COMMAND != 'Sleep' and LENGTH(info) > 0;""" 19 | 20 | 21 | class Entity(): 22 | def __init__(self): 23 | pass 24 | 25 | 26 | def check_all_mysql_processlist(): 27 | pass 28 | 29 | 30 | def get_monitor_mysql_host_infos(): 31 | return get_list_infos(None, "select host_id, host, port, user, password, remark, is_master, is_slave, master_id, is_deleted, ssh_port, ssh_password from mysql_web.host_infos;") 32 | 33 | 34 | def get_list_infos(host_info, sql): 35 | result = [] 36 | for row in fetchall(host_info, sql): 37 | info = Entity() 38 | for key, value in row.items(): 39 | setattr(info, key.lower(), value) 40 | result.append(info) 41 | return result 42 | 43 | def fetchall(host_info, sql): 44 | connection, cursor = None, None 45 | try: 46 | connection = get_mysql_connection(host_info.host, user=host_info.user, password=host_info.password, port=host_info.port) 47 | cursor = connection.connection.cursor() 48 | cursor.execute(sql) 49 | return cursor.fetchall() 50 | finally: 51 | if (cursor != None): 52 | cursor.close() 53 | if (connection != None): 54 | connection.close() 55 | 56 | def get_mysql_connection(host_name, port=3306, user="", password=""): 57 | if (connection_pools.get(host_name) == None): 58 | pool = PooledDB(creator=pymysql, mincached=2, maxcached=4, maxconnections=5, 59 | host=host_name, port=port, user=user, passwd=password, 60 | use_unicode=False, charset="utf8", cursorclass=pymysql.cursors.DictCursor, reset=False, autocommit=True) 61 | connection_pools[host_name] = pool 62 | return connection_pools[host_name].connection() 63 | 64 | 65 | if (__name__ == "__main__"): 66 | while(True): 67 | host_infos = get_monitor_mysql_host_infos() 68 | check_all_mysql_processlist(host_infos) 69 | 70 | 71 | -------------------------------------------------------------------------------- /tools_new/mysql_skip_rpl_error.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import argparse, sys, pymysql, time, traceback 4 | 5 | class Entity(): 6 | def __init__(self): 7 | pass 8 | 9 | 10 | def check_arguments(): 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument("--host", type=str, dest="host", help="mysql host") 13 | parser.add_argument("--user", type=str, dest="user", help="mysql user") 14 | parser.add_argument("--password", type=str, dest="password", help="mysql password") 15 | parser.add_argument("--port", type=int, dest="port", help="mysql port", default=3306) 16 | parser.add_argument("--sleep", type=float, dest="sleep", help="while sleep time second", default=0.5) 17 | args = parser.parse_args() 18 | 19 | if not args.host or not args.user or not args.password or not args.port: 20 | print("[ERROR]:Please input host or user or password or port.") 21 | sys.exit(1) 22 | return args 23 | 24 | 25 | def main(args): 26 | slave_status = show_slave_status(args) 27 | if (slave_status.slave_sql_running == "No"): 28 | print_log("[Error]:SQL thread is error, error code is [{0}]".format(slave_status.last_sql_errno)) 29 | print_log("[Error]:SQL error info is {0}".format(slave_status.last_sql_error)) 30 | else: 31 | print_log("[INFO]:Slave SQL thread is ok!") 32 | time.sleep(args.sleep) 33 | 34 | 35 | # 获取show slave status数据 36 | def show_slave_status(args): 37 | return execute_sql(args, "show slave status;") 38 | 39 | 40 | # 跳过sql线程的错误 41 | def skip_sql_thread_error(args): 42 | execute_sql(args, "set global sql_slave_skip_counter=1; select sleep(0.1); start slave sql_thread;") 43 | 44 | 45 | # 执行sql 46 | def execute_sql(args, sql): 47 | connection, cursor = None, None 48 | try: 49 | connection = pymysql.connect(host=args.host, user=args.user, passwd=args.password, port=args.port, use_unicode=True, charset="utf8", connect_timeout=2) 50 | cursor = connection.cursor(cursor=pymysql.cursors.DictCursor) 51 | cursor.execute(sql) 52 | result = cursor.fetchone() 53 | info = Entity() 54 | if (result != None): 55 | for key, value in result.items(): 56 | setattr(info, key.lower(), value) 57 | return info 58 | except: 59 | connection.rollback() 60 | traceback.print_exc() 61 | finally: 62 | if (cursor != None): 63 | cursor.close() 64 | if (connection != None): 65 | connection.close() 66 | 67 | 68 | # 打印日志 69 | def print_log(log_value): 70 | print("{0} {1}".format(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), log_value)) 71 | 72 | 73 | if (__name__ == "__mian__"): 74 | main(check_arguments()) 75 | 76 | 77 | -------------------------------------------------------------------------------- /tools_new/mysql_slow_log.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | #MySQL慢查询监控脚本 4 | #需要安装pt-tools工具包，然后把脚本设置为后台执行 5 | #注意如果SQL比较长需要修改sample字段为mediumtext 6 | #如果执行set global slow_query_log_file='';报错，注意slow_log目录权限的问题 7 | 8 | #alter table table1 modify sample mediumtext not null; 9 | #alter table table1 add column created_time timestamp not null default now(), 10 | #add column updated_time timestamp not null default current_timestamp on update current_timestamp, 11 | #ADD COLUMN `is_reviewed` TINYINT UNSIGNED NOT NULL DEFAULT 0; 12 | 13 | #ALTER TABLE `db1`.`mysql_slow_query_review` 14 | #CHANGE COLUMN `id` `id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT , 15 | #DROP PRIMARY KEY, 16 | #ADD PRIMARY KEY (`id`), 17 | #ADD UNIQUE INDEX `checksum_UNIQUE` (`checksum` ASC); 18 | 19 | #目前只之前percona分支版本 20 | #set global log_slow_verbosity = 'query_plan,innodb,microtime'; 21 | 22 | #1.slow log存放的地址 23 | #第一处修改的地方 24 | #慢查询数据上传的MySQL机器 25 | db_host="" 26 | db_port=3306 27 | db_user="" 28 | db_password="" 29 | db_database="mysql_web" 30 | 31 | #2.slow log本身的地址 32 | #第二处修改的地方 33 | #监控的慢查询MySQL机器 34 | mysql_client="/usr/local/mysql/bin/mysql" 35 | mysql_host="" 36 | mysql_port=3306 37 | mysql_user="" 38 | mysql_password="" 39 | 40 | #3.读取有关慢日志配置信息 41 | #第三处修改的地方，路径不改也没事 42 | slow_log_dir="/slow_log/" 43 | slow_query_long_time=2 44 | pt_query_digest="/usr/bin/pt-query-digest" 45 | 46 | #4.mysql_host_id对应着mysql_web.host_infos中id字段值 47 | #第四处修改的地址,这个字段很重要 48 | mysql_host_id=14 49 | 50 | function check_slow_log() 51 | { 52 | #获取slow log文件路径 53 | slow_log_file=`${mysql_client} -h${mysql_host} -P${mysql_port} -u${mysql_user} -p${mysql_password} \ 54 | -e "show variables like 'slow_query_log_file';" | grep log | awk '{print $2}'` 55 | 56 | echo "$slow_log_file" 57 | #设置新的slow log文件路径 58 | slow_log_file_path=${slow_log_dir}${mysql_host_id}_`date "+%Y-%m-%d_%H-%M-%S.slow"` 59 | `${mysql_client} -h${mysql_host} -P${mysql_port} -u${mysql_user} -p${mysql_password} \ 60 | -e "set global slow_query_log = 1;set global long_query_time=$slow_query_long_time;set global log_output='FILE';set global slow_query_log_file='$slow_log_file_path'"` 61 | 62 | echo "=========================start check================================" 63 | #对旧的slow log进行解析并入库 64 | #\$event->{user} !~ m/^serverdev/i 不包含某个用户的过滤，包含就把!改成= 65 | ${pt_query_digest} \ 66 | --user=${db_user} --password=${db_password} --port=${db_port} --charset=utf8 \ 67 | --review h=${db_host},D=${db_database},t=mysql_slow_query_review \ 68 | --history h=${db_host},D=${db_database},t=mysql_slow_query_review_history \ 69 | --no-report --limit=100% \ 70 | --filter=" \$event->{add_column} = length(\$event->{arg}) and \$event->{serverid}=$mysql_host_id and \$event->{user} !~ m/^serverdev/i" ${slow_log_file} > /tmp/slowquery.log 71 | 72 | echo "================check slow log [$slow_log_file] ok====================" 73 | if [ -f "$slow_log_file" ]; then 74 | rm -f "$slow_log_file" 75 | echo "remove $slow_log_file ok." 76 | fi 77 | echo "" 78 | } 79 | 80 | if [ ! -d "$slow_log_dir" ]; then 81 | mkdir "$slow_log_dir" 82 | fi 83 | 84 | chown -R mysql:mysql ${slow_log_dir} 85 | 86 | while true 87 | do 88 | check_slow_log 89 | sleep 5m 90 | done 91 | -------------------------------------------------------------------------------- /tools_new/mysql_tablespace.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Administrator' 2 | --------------------------------------------------------------------------------