├── MySQL_Replication_Role.py ├── MySQL安装 ├── __init__.py ├── function_test.py ├── initialize_linux4mysql.py └── start_instance.py ├── README.md ├── RRD └── main.py ├── Replication_ParalleltoChain_Py2 └── new_main.py ├── __init__.py ├── _config.yml ├── bulk_insert_to_mysql ├── functions.py └── main.py ├── check_MySQL_slave.py ├── pmm ├── CrossServerGraphs.json ├── MySQL Overview V2.json └── prometheus.yml ├── scan ├── __init__.py └── scan_port.py ├── separateSQL ├── README.md ├── functions.py ├── separateDatabase.py └── separateSpecifiedDatabase.py ├── 压测 ├── Gnuplot_5.2.pdf ├── gnuplot_readonly.jpg ├── gnuplot绘图示例.jpg ├── gp522-win64-mingw.exe └── 收集与分析的脚本 │ ├── README.txt │ ├── hi_analyze.sh │ └── hi_performance.sh ├── 性能优化 ├── qps_thread.sh └── 高性能MySQL 笔记 │ ├── 【MySQL】【持续更新】《高性能MySQL》 学习笔记,第三章.md │ ├── 【MySQL】【持续更新】《高性能MySQL》 学习笔记,第二章.md │ ├── 【MySQL】【持续更新】《高性能MySQL》 学习笔记.md │ └── 配图 │ └── 第三章 │ └── sys_workbench.png ├── 技术文章翻译 ├── 【MySQL】【翻译】 MySQL 5.7 的内部临时表新特性 .md ├── 【MySQL】【翻译】8.0 GA版本的新特性 What’s New in MySQL 8.0 (Generally Available).md ├── 【MySQL】【翻译】Adaptive query routing based on GTID tracking 基于GTID追踪的自适应路由查询.md └── 【MySQL】【翻译】ProxySQL新版本对MGR的原生支持.md └── 求职 └── 面试总结.md /MySQL_Replication_Role.py: -------------------------------------------------------------------------------- 1 | import pymysql 2 | import sys 3 | 4 | 5 | def initialize_conn(): 6 | host = sys.argv[1] 7 | port = sys.argv[2] 8 | user = sys.argv[3] 9 | password = sys.argv[4] 10 | database = sys.argv[5] 11 | db = pymysql.connect(host, int(port), user, password, 12 | database, cursorclass=pymysql.cursors.DictCursor) 13 | return db 14 | 15 | # 简化打印函数 16 | 17 | 18 | def p(arg): 19 | print(arg) 20 | 21 | 22 | # 打印字典内容 23 | 24 | 25 | def p_dict(arg): 26 | for x in arg.items(): 27 | print(x) 28 | 29 | 30 | # 判断实例是否为双线程正常且没有严重落后主库的从库 31 | # 获取slave status 32 | 33 | 34 | def getSlaveInfo(): 35 | sql = "show slave status" 36 | cursor = initialize_conn().cursor() 37 | cursor.execute(sql) 38 | result = cursor.fetchall() 39 | result = result[0] # 将字典从字典列表中取出来 40 | return result 41 | 42 | 43 | # 检查slave status 44 | 45 | 46 | def isNiceSlave(Info): 47 | if Info['Slave_IO_Running'] == 'Yes' and Info['Slave_SQL_Running'] == 'Yes': 48 | return 0 49 | else: 50 | p('Slave_IO_Running', Info['Slave_IO_Running']) 51 | p('Slave_SQL_Running', Info['Slave_SQL_Running']) 52 | return 1 53 | 54 | 55 | Info = getSlaveInfo() 56 | if len(Info) > 2: 57 | a = isNiceSlave(Info) 58 | -------------------------------------------------------------------------------- /MySQL安装/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/naughtyGitCat/MySQLMiscellaneous/46f040da900c0883d4e5b475b1f0c4d030fc7929/MySQL安装/__init__.py -------------------------------------------------------------------------------- /MySQL安装/function_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import os 3 | from start_instance import add_mysql_user, del_mysql_user, download, untar 4 | 5 | 6 | # 用户模块测试 7 | # del_mysql_user() 8 | # add_mysql_user() 9 | # 10 | # cmd_getuid = 'id -u mysql' 11 | # cmd_getgid = 'id -g mysql' 12 | # uid = os.popen(cmd_getuid).read().strip('\n') 13 | # gid = os.popen(cmd_getgid).read().strip('\n') 14 | # print('uid:', uid, 'gid:', gid) 15 | 16 | # OK 17 | 18 | # 下载模块测试 19 | # download() 20 | 21 | # OK，但是不能实时显示下载进度条 22 | 23 | 24 | # 解压模块测试 25 | 26 | untar() 27 | -------------------------------------------------------------------------------- /MySQL安装/initialize_linux4mysql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | #coding=utf-8 3 | import subprocess 4 | def suggest(a, b, c): 5 | (status,output) = subprocess.getstatusoutput(a) 6 | print("######{}#######".format(b)) 7 | print(output) 8 | print("*******{}*******".format(c)) 9 | suggest("df -Th|awk '{print $1,$2}'|grep -v 'tmpfs'","1.查看文件系统","建议data分区为xfs") 10 | suggest("cat /sys/block/sda/queue/scheduler","2.查看IO调度算法","建议采用deadline算法，不要用cfg算法") 11 | suggest("ulimit -a|grep 'open files'","3.查看文件打开数","建议设置为系统最大65535") 12 | suggest("grep -i numa /var/log/dmesg","4.NUMA是否开启","强烈建议关闭NUMA") 13 | suggest("sysctl -a | grep swappiness","5.swap占用比","建议值设置为1-10") 14 | suggest("sysctl -a | grep dirty_ratio","6.dirty刷新脏页比1","设置为10比较好") 15 | suggest("sysctl -a | grep dirty_background_ratio","7.dirty刷新脏页比2","设置为5比较好") 16 | 17 | # (命令正常执行返回0，报错则返回1) 18 | def IO_scheduler(): 19 | x=input('输入yes进行IO调度算法的优化：') 20 | command = '''echo 'deadline' >/sys/block/sda/queue/scheduler''' 21 | if x == 'yes': 22 | print('now do:', command) 23 | (status, output)=subprocess.getstatusoutput(command) 24 | if status == 0: 25 | print('done') 26 | if status != 0: 27 | print('fail,please check manually') 28 | 29 | 30 | 31 | def open_files(): 32 | x = input('输入yes优化文件打开数量：') 33 | command1 = '''echo '* soft nofile 65536' >>/etc/security/limits.conf''' 34 | command2 = '''echo '* hard nofile 65536' >>/etc/security/limits.conf''' 35 | if x == 'yes': 36 | print('now do:', command1) 37 | (status, output)=subprocess.getstatusoutput(command1) 38 | if status == 0: 39 | print('done') 40 | if status != 0: 41 | print('fail,please check manually') 42 | print('now do:', command2) 43 | (status, output)=subprocess.getstatusoutput(command2) 44 | if status == 0: 45 | print('done') 46 | if status != 0: 47 | print('fail,please check manually') 48 | 49 | def disable_NUMA(): 50 | x = input('输入yes关闭NUMA(需联网)：') 51 | command1 = '''yum -y install numactl''' 52 | command2 = '''numactl --interleave=all''' 53 | if x == 'yes': 54 | print('now do:', command1) 55 | (status, output)=subprocess.getstatusoutput(command1) 56 | if status == 0: 57 | print('done') 58 | if status != 0: 59 | print('fail,please check manually') 60 | if x == 'yes': 61 | print('now do:', command2) 62 | (status, output)=subprocess.getstatusoutput(command2) 63 | if status == 0: 64 | print('done') 65 | if status != 0: 66 | print('please restart system,check again') 67 | 68 | def swappiness_ratio(): 69 | x=input('输入yes更改swap阙值：') 70 | command = '''echo 'vm.swappiness = 10'>>/etc/sysctl.conf''' 71 | if x == 'yes': 72 | print('now do:', command) 73 | (status, output)=subprocess.getstatusoutput(command) 74 | if status == 0: 75 | print('done') 76 | if status != 0: 77 | print('fail,please check manually') 78 | 79 | 80 | def dirty_ratio(): 81 | x = input('输入yes更改脏页比：') 82 | command1 = '''echo 'vm.dirty_background_ratio = 10' >>/etc/sysctl.conf''' 83 | command2 = '''echo 'vm.dirty_ratio = 10' >>/etc/sysctl.conf''' 84 | command3 = '''systcl -p''' 85 | if x == 'yes': 86 | print('now do:', command1) 87 | (status, output)=subprocess.getstatusoutput(command1) 88 | if status == 0: 89 | print('done') 90 | if status != 0: 91 | print('fail,please check manually') 92 | 93 | if x == 'yes': 94 | print('now do:', command2) 95 | (status, output)=subprocess.getstatusoutput(command2) 96 | if status == 0: 97 | print('done') 98 | if status != 0: 99 | print('fail,please check manually') 100 | if x == 'yes': 101 | print('now do:', command3) 102 | (status, output)=subprocess.getstatusoutput(command2) 103 | if status == 0: 104 | print('done') 105 | if status != 0: 106 | print('fail,please check manually') 107 | 108 | IO_scheduler() 109 | open_files() 110 | disable_NUMA() 111 | swappiness_ratio() 112 | dirty_ratio() 113 | print('请重启后复查') 114 | -------------------------------------------------------------------------------- /MySQL安装/start_instance.py: -------------------------------------------------------------------------------- 1 | # TODO:根据机器与是否多实例生成my.cnf 2 | # TODO:根据指定的MySQL版本下载二进制包 3 | import subprocess 4 | import os 5 | import urllib.request # 下载 6 | import shutil # 复制文件 7 | import sys, getopt 8 | 9 | # 10 | # 设计用途： 11 | # 快速安装3306实例 12 | 13 | 14 | # ########参数定义段############ 15 | mysql_ver = 'mysql-5.7.21-linux-glibc2.12-x86_64' # 暂未替换到各个函数中 16 | miscellaneous_path = '/data/mysql/{}/'.format(port) # 暂未替换到各个函数中 17 | data_path = '{}/data'.format(miscellaneous_path) # 暂未替换到各个函数中 18 | binlog_path = '{}/logs'.format(miscellaneous_path) # 暂未替换到各个函数中 19 | base_path = 20 | 21 | # ################ 22 | 23 | def get_port(): 24 | port = input('please set the service port') 25 | print('you have set {} as the service port：'.format(port)) 26 | return port 27 | 28 | def get_password(): 29 | password = input('please set the root@localhost`s password') 30 | print('The password you have inputted is：',password) 31 | 32 | # 判断能否上网 33 | 34 | 35 | def net_status(): 36 | yellow_website = 'http://www.baidu.com' 37 | conn = urllib.request.urlopen(yellow_website).code 38 | return conn 39 | 40 | 41 | # 调用wget下载 42 | # TODO:需要对wget命令是否存在进行判断 43 | 44 | def download(): 45 | if net_status() == 200: 46 | print('network is ok,start download') 47 | subprocess.getoutput('yum -y install wget') 48 | source_dir = '/usr/local/src/' 49 | source = 'https://dev.mysql.com/get/Downloads/MySQL-5.7/mysql-5.7.21-linux-glibc2.12-x86_64.tar.gz' 50 | # binary = urllib.request.urlopen(source) 51 | # with open('mysql57_21.tar.gz', 'wb') as file: 52 | # file.write(binary) 53 | print('set download command') 54 | download_cmd = 'wget {} -P {}'.format(source, source_dir) 55 | 56 | (status, output) = subprocess.getstatusoutput(download_cmd) 57 | print('status:', status, 'output:', output) 58 | else: 59 | print('请使用-f命令指定二进制安装包位置') 60 | 61 | 62 | # 解压下载的二进制安装包 63 | # TODO:询问是否创建超链接为mysql 64 | def untar(): 65 | try: 66 | tar_path = '/usr/local/src/mysql-5.7.21-linux-glibc2.12-x86_64.tar.gz' 67 | bin_path = '/usr/local/' 68 | untar_cmd = 'tar -zxf {} -C {}'.format(tar_path, bin_path) 69 | (status, output) = subprocess.getstatusoutput(untar_cmd) 70 | if status == 0: 71 | print('untar finished') 72 | else: 73 | raise Exception 74 | except Exception: 75 | print('exec untar across a error') 76 | 77 | 78 | # 创建MySQL用户和用户组,并返回uid,gid 79 | 80 | def add_mysql_user(): 81 | try: 82 | cmd = 'useradd -d /usr/local/mysql/ -s /sbin/nologin -U -M mysql' 83 | (status, output) = subprocess.getstatusoutput(cmd) 84 | if status == 0: 85 | print('create user mysql success') 86 | else: 87 | raise Exception 88 | cmd_getuid = 'id -u mysql' 89 | cmd_getgid = 'id -g mysql' 90 | # os.popen输出的结果并不是char类型，需要read()出来，并截取换行符 91 | uid = os.popen(cmd_getuid).read().strip('\n') 92 | gid = os.popen(cmd_getgid).read().strip('\n') 93 | print('uid:', uid, 'gid:', gid) 94 | except Exception: 95 | print('create user mysql across a error,please check') 96 | 97 | 98 | # 创建MySQL用户和用户组 99 | 100 | 101 | def del_mysql_user(): 102 | a = os.popen('userdel -r mysql').read().strip('\n') 103 | print('userdel complete', a) 104 | b = os.popen('groupdel mysql').read().strip('\n') 105 | print('groupdel executed', b) 106 | 107 | 108 | # 创建数据文件夹,并归属到mysql用户下 109 | 110 | 111 | def prepare(port, uid, gid): 112 | try: 113 | os.makedirs('/data/mysql/{}/data'.format(port)) 114 | os.mkdir('/data/mysql/{}/logs'.format(port)) 115 | os.mkdir('/data/mysql/{}/tmp'.format(port)) 116 | os.chown('/data/mysql/{}/', uid, gid) 117 | except OSError: 118 | print('create dir error,please check') 119 | return '/data/mysql/{}' 120 | 121 | 122 | # 接收指定的my.cnf文件，并复制到相关数据文件夹下 123 | # TODO:自动询问关键参数，并生成my_$port.cnf 124 | 125 | 126 | def cp_cnf(file_path, data_path): 127 | if os.path.exists(file_path): 128 | try: 129 | shutil.copy(file_path, data_path) 130 | except OSError: 131 | print('copy failed,please check it') 132 | else: 133 | print('file does`s not exist') 134 | 135 | 136 | # 生成start_$port.sh启动文件 137 | # TODO:把参数带进来，缩短cmd长度 138 | def write_start_shell(port): 139 | cmd = '''echo '/usr/local/{}/bin/mysqld --defaults-file=/data/mysql/{}/my_{}.cnf &' >/root/start_{}.sh'''.format(port, port, port, port) 140 | os.popen(cmd) 141 | 142 | # 生成conn_$port.sh连接文件 143 | # TODO:把参数带进来，缩短cmd长度 144 | def write_connect_shell(port): 145 | cmd = '''echo '/usr/local/{}/bin/mysql --defaults-file=/data/mysql/{}/my_{}.cnf -S /data/mysql/{}/tmp/mysql_{}.sock' >/root/start_{}.sh'''.format(port, port, port, port,port,port) 146 | os.popen(cmd) 147 | 148 | 149 | # 初始化实例并读取临时密码 150 | def initialize_instance(port): 151 | cmd = 'echo /usr/local/{}/bin/mysqld --defaults-file=/data/mysql/{}/my_{}.cnf --initialize'.format(port, port, port) 152 | (status, output) = subprocess.getstatusoutput(cmd) 153 | print('status:', status, 'detail:', output) 154 | if status ==0: 155 | print('Initialize finished,now read temporary password') 156 | # 读取error log中的最后一个password信息 157 | 158 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MySQL_Tools 2 | MySQL小工具 3 | -------------------------------------------------------------------------------- /RRD/main.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # !/usr/bin/python 3 | import rrdtool 4 | import time 5 | import psutil 6 | # ##创建rrd 7 | def create_rrdb(): 8 | rrdb = rrdtool.create('rest.rrd', '--step', '60', '--start', '1369982786', 9 | 'DS:input:GAUGE:120:U:U', 10 | 'DS:output:GAUGE:120:U:U', 11 | 'RRA:LAST:0.5:1:600', 12 | 'RRA:AVERAGE:0.5:5:600', 13 | 'RRA:MAX:0.5:5:600', 14 | 'RRA:MIN:0.5:5:600') 15 | if rrdb: 16 | print rrdtool.error() 17 | 18 | # ##rrd插入数据 19 | 20 | def insert_data() 21 | for keys in psutil.network_io_counters(pernic=True): 22 | if keys == 'em1': 23 | sent = psutil.network_io_counters(pernic=True)[keys][0] 24 | recv = psutil.network_io_counters(pernic=True)[keys][1] 25 | up = rrdtool.updatev('rest.rrd', 'N:%d:%d' % (sent, recv)) 26 | print up 27 | 28 | # ##根据rrd绘图 29 | 30 | def draw_graph(): 31 | rrdtool.graph('rest.png', '--start', '1369983960', 32 | '--title', 'my rrd graph test', 33 | '--vertical-label', 'bits', 34 | 'DEF:input=rest.rrd:input:LAST', 35 | 'DEF:output=rest.rrd:output:LAST', 36 | 'LINE1:input#0000FF:In traffic', 37 | 'LINE1:output#00FF00:Out traffic\\r', 38 | 'CDEF:bytes_in=input,8,*', 39 | 'CDEF:bytes_out=output,8,*', 40 | 'COMMENT:\\n', 41 | 'GPRINT:bytes_in:LAST:LAST in traffic\: %6.2lf %Sbps', 42 | 'COMMENT: ', 43 | 'GPRINT:bytes_out:LAST:LAST out traffic\: %6.2lf %Sbps') -------------------------------------------------------------------------------- /Replication_ParalleltoChain_Py2/new_main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | # coding=utf-8 3 | # TODO:多样本检测 4 | # TODO:增加错误回滚步骤异常的提醒 5 | # TODO:未摘出GTID信息，需要针对GTID进行另外一套流程 6 | # TODO:将print改为记录日志 7 | # TODO:将创建实例连接，执行SQL，检查自身Binlog和slave信息做成一个class 8 | 9 | # import pymysql 10 | import time 11 | from functions import print_slave_status, print_selfbinlog_status, get_newest_file_pos, read_args, print_rep_info 12 | import MySQLdb as pymysql 13 | 14 | # 全局参数：从库切换用 15 | middle_master_file = '' 16 | middle_master_pos = '' 17 | orgin_file = '' 18 | origin_pos = '' 19 | error_code = '' 20 | # 读入连接参数 21 | (master_host, master_port, middle_master_host, middle_master_port, slave_host, slave_port, opreate_user, 22 | opreate_password) = read_args() 23 | 24 | middle_master = pymysql.Connect(host=middle_master_host, port=middle_master_port, user=opreate_user, passwd=opreate_password 25 | ) 26 | slave = pymysql.Connect(host=slave_host, port=slave_port, user=opreate_user, passwd=opreate_password 27 | ) 28 | master = pymysql.Connect(host=master_host, port=master_port, user=opreate_user, passwd=opreate_password 29 | ) 30 | 31 | 32 | (master_host_N, master_port_N, middle_master_io_file, middle_master_io_pos, middle_master_sql_file, 33 | middle_master_sql_pos) = print_slave_status(middle_master) 34 | (master_host_S, master_port_S, slave_io_file, slave_io_pos, slave_sql_file, slave_sql_pos) = print_slave_status(slave) 35 | (io_thread_running_n, sql_thread_running_n) = print_rep_info(middle_master) 36 | (io_thread_running_s, sql_thread_running_s) = print_rep_info(slave) 37 | 38 | 39 | # 预检模块 40 | 41 | 42 | def pre_check(): 43 | global error_code 44 | # ##################先预检，判断当前复制拓补是否为一主两从，两个从库是否存在slave线程（SQL,IO任一）停止的情况########### 45 | (master_host_N, master_port_N, middle_master_io_file, middle_master_io_pos, middle_master_sql_file, 46 | middle_master_sql_pos) = print_slave_status(middle_master) 47 | (master_host_S, master_port_S, slave_io_file, slave_io_pos, slave_sql_file, slave_sql_pos) = print_slave_status( 48 | slave) 49 | (io_thread_running_n, sql_thread_running_n) = print_rep_info(middle_master) 50 | (io_thread_running_s, sql_thread_running_s) = print_rep_info(slave) 51 | print('########################预检模块开始###########################') 52 | if master_host_N == master_host_S and master_port_N == master_port_S \ 53 | and io_thread_running_n == 'Yes' and io_thread_running_n == 'Yes' \ 54 | and io_thread_running_s == 'Yes' and io_thread_running_s == 'Yes': 55 | print('复制链路正确，slave线程全部running') 56 | print('master_port_N:', master_port_N, 'master_port_S:', master_port_S) 57 | else: 58 | error_code = 'pre_check' 59 | raise Exception 60 | 61 | 62 | # 连到从库，停止IO进程 63 | 64 | 65 | def stop_slave_io_thread(): 66 | # #################################连接到从库上停止IO进程############################### 67 | try: 68 | global error_code 69 | print('# #################################连接到从库上停止IO进程###############################') 70 | # #被切换的从库 71 | slave = pymysql.Connect(host=slave_host, port=slave_port, user=opreate_user, passwd=opreate_password 72 | ) 73 | # ###游标 74 | cursor_slave = slave.cursor() 75 | # ##停止SLAVE IO进程,但不停止SQL进程，防止SQL线程落后于IO进程 76 | cursor_slave.execute('stop slave io_thread') 77 | time.sleep(3) # 等待3秒，让中间库的IO进度快于自己的IO 78 | # ###输出执行后的slave进程状态 79 | (a, b, slave_io_file, slave_io_pos, slave_sql_file, slave_sql_pos) = print_slave_status(slave) 80 | except Exception: 81 | error_code = 'stop_slave_io_thread' 82 | raise Exception 83 | 84 | # 停止中间库的IO进程，待执行完毕后，show slave status获取其pos,file，然后show master status 获取其pos，file用于从库切换 85 | def stop_the_middle(): 86 | print('# #################################连接到中间库上停止IO进程获取其pos,file,start###############################') 87 | try: 88 | global middle_master_file 89 | global middle_master_pos 90 | global orgin_file 91 | global origin_pos 92 | global error_code 93 | middle_master = pymysql.Connect(host=slave_host, port=slave_port, user=opreate_user, passwd=opreate_password 94 | ) 95 | # ###游标 96 | cursor_middle_master = middle_master.cursor() 97 | # ##停止IO进程，等待SQL线程完成复制 98 | cursor_middle_master.execute('stop slave io_thread') 99 | # ##检查SQL线程是否完成复制 100 | while 1: 101 | print('checking replication process') 102 | time.sleep(5) # 循环检查的间隔5s 103 | (a, b, slave_io_file, slave_io_pos, slave_sql_file, slave_sql_pos) = print_slave_status(slave) 104 | if slave_io_file == slave_sql_file and slave_io_pos == slave_sql_pos: 105 | print('sql thread finished replicate') 106 | break 107 | (orgin_file, origin_pos) = (slave_io_file, slave_io_pos) 108 | (middle_master_file, middle_master_pos) = print_selfbinlog_status(middle_master) 109 | cursor_middle_master.execute('start slave') 110 | except Exception: 111 | error_code = 'stop_the_middle' 112 | raise Exception 113 | 114 | 115 | # 停在上面选定的位置上 116 | def stop_at_chose_pos(): 117 | global orgin_file 118 | global origin_pos 119 | global error_code 120 | # ########################停在同一个位置 121 | try: 122 | print('停止SQL线程') 123 | start_until_sql = '''start slave until MASTER_LOG_FILE = '{}', MASTER_LOG_POS = {}''' \ 124 | .format(orgin_file, int(origin_pos)) 125 | print(start_until_sql) 126 | print('开始停止到中间库的指定位置') 127 | # #被切换的从库 128 | slave = pymysql.Connect(host=slave_host, port=slave_port, user=opreate_user, passwd=opreate_password 129 | ) 130 | # ###游标 131 | cursor_slave = slave.cursor() 132 | cursor_slave.execute(start_until_sql) 133 | # ###检查是否到达指定位置 134 | while 1: 135 | print('checking replication process') 136 | time.sleep(5) # 循环检查的间隔5s 137 | (a, b, slave_io_file, slave_io_pos, slave_sql_file, slave_sql_pos) = print_slave_status(slave) 138 | if slave_io_file == slave_sql_file and slave_io_pos == slave_sql_pos: 139 | print('sql thread finished replicate') 140 | break 141 | 142 | except Exception: 143 | error_code = 'stop_at_same_pos' 144 | raise Exception 145 | 146 | 147 | 148 | # 切换GO！！！ 149 | def change_to_middle_master(): 150 | global middle_master_file 151 | global middle_master_pos 152 | global error_code 153 | try: 154 | # ######################切换 155 | slave = pymysql.Connect(host=slave_host, port=slave_port, user=opreate_user, passwd=opreate_password) 156 | # ###游标 157 | cursor_slave = slave.cursor() 158 | # 在从库上reset slave 159 | print('在从库上reset slave') 160 | # 因为start slave until到达指定位置后只会停止SQL_THREAD,而reset slave需要全部停止，所以需要再次全部停止 161 | cursor_slave.execute('stop slave') 162 | cursor_slave.execute('reset slave all') 163 | print('now begin change') 164 | change_sql = '''change master to MASTER_HOST='{}',MASTER_PORT={},MASTER_USER='{}',MASTER_PASSWORD='{}',MASTER_LOG_FILE='{}',MASTER_LOG_POS={}''' \ 165 | .format(middle_master_host, middle_master_port, opreate_user, opreate_password, middle_master_file, int(middle_master_pos)) 166 | print(change_sql) 167 | print('开始切换到新主库') 168 | cursor_slave.execute(change_sql) 169 | cursor_slave.execute('start slave') 170 | print_slave_status(slave) 171 | except Exception: 172 | error_code = 'change_to_middle_master' 173 | raise Exception 174 | 175 | 176 | 177 | # 检查切换后的sql与slave线程情况 178 | def after_change(): 179 | global error_code 180 | try: 181 | time.sleep(3) 182 | slave = pymysql.Connect(host=slave_host, port=slave_port, user=opreate_user, passwd=opreate_password) 183 | # ###游标 184 | cursor_slave = slave.cursor() 185 | (master_host_S, master_port_S, a, b, c, d) = print_slave_status(slave) 186 | (io_thread_running_s, sql_thread_running_s) = print_rep_info(slave) 187 | if master_port_S == middle_master_host and master_port_S == middle_master_port \ 188 | and io_thread_running_s == 'Yes' and io_thread_running_s == 'Yes': 189 | print('change OK,slave running!!!') 190 | except Exception: 191 | error_code = 'after_change' 192 | raise Exception 193 | 194 | def change_back(): 195 | global orgin_file 196 | global origin_pos 197 | print('切换回去') 198 | slave = pymysql.Connect(host=slave_host, port=slave_port, user=opreate_user, passwd=opreate_password) 199 | # ###游标 200 | cursor_slave = slave.cursor() 201 | cursor_slave.execute('stop slave') 202 | cursor_slave.execute('reset slave all') 203 | change_sql = '''change master to MASTER_HOST='{}',MASTER_PORT={},MASTER_USER='{}',MASTER_PASSWORD='{}',MASTER_LOG_FILE='{}',MASTER_LOG_POS={}''' \ 204 | .format(master_host, master_port, opreate_user, opreate_password, orgin_file, 205 | int(origin_pos)) 206 | cursor_slave.execute('start slave') 207 | 208 | def start_all_slave(): 209 | slave = pymysql.Connect(host=slave_host, port=slave_port, user=opreate_user, passwd=opreate_password) 210 | middle_master = pymysql.Connect(host=slave_host, port=slave_port, user=opreate_user, passwd=opreate_password 211 | ) 212 | # ###游标 213 | cursor_slave = slave.cursor() 214 | cursor_middle_master = middle_master.cursor() 215 | cursor_slave.execute('start slave') 216 | cursor_middle_master.execute('start slave') 217 | 218 | ### main 219 | try: 220 | pre_check() 221 | stop_slave_io_thread() 222 | stop_the_middle() 223 | stop_at_chose_pos() 224 | change_to_middle_master() 225 | after_change() 226 | except Exception: 227 | if error_code == 'change_to_middle_master': 228 | print('已经发生切换，尝试切换回去') 229 | change_back() 230 | elif error_code == 'after_change': 231 | print('切换成功，但SLAVE线程有问题，请手工检查show slave status') 232 | else: 233 | print('尚未切换，尝试重新启动两个从库的slave进程') 234 | start_all_slave() 235 | 236 | middle_master.close() 237 | slave.close() 238 | master.close() -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-slate -------------------------------------------------------------------------------- /bulk_insert_to_mysql/functions.py: -------------------------------------------------------------------------------- 1 | # 批量插入MySQL 2 | import pymysql 3 | import json 4 | 5 | 6 | def getmailJSON(): 7 | with open('mail.json', 'rt', encoding="utf-8") as open_file: 8 | context = open_file.read() 9 | JSON_CONTEXT = json.loads(context.encode("utf-8")) 10 | print(JSON_CONTEXT) 11 | FROM = JSON_CONTEXT['FROM'] 12 | PASSWORD = JSON_CONTEXT['Q4SSW0RD'] 13 | SUBJECT = JSON_CONTEXT['SUBJECT'] 14 | TO = JSON_CONTEXT['TO'] 15 | NAME = JSON_CONTEXT['NAME'] 16 | #print('发信人：{},\n密码：{},\n主题：{},\n收件人：{}'.format(FROM,PASSWORD,SUBJECT,TO)) 17 | return FROM, PASSWORD, SUBJECT, TO, NAME 18 | -------------------------------------------------------------------------------- /bulk_insert_to_mysql/main.py: -------------------------------------------------------------------------------- 1 | # 批量插入MySQL 2 | import pymysql 3 | 4 | import json 5 | user = 'python' 6 | password = '123456' 7 | host = '192.168.1.8' 8 | port = 3307 9 | conn = pymysql.Connect(host=host, port=port, user=user, password=password 10 | ) 11 | cursor_conn = conn.cursor() 12 | for i in range(200): 13 | insert_sql = '''insert into d20180416.hash_on_all_node_instance1(fid,c2,c3,c4) 14 | values ('{}','c2','c3','c4')'''\ 15 | .format(i+1) 16 | print(insert_sql) 17 | cursor_conn.execute('select 1 from mysql.user') 18 | cursor_conn.execute(insert_sql) 19 | conn.commit() 20 | print(cursor_conn.fetchall()) 21 | 22 | conn.close() -------------------------------------------------------------------------------- /check_MySQL_slave.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding: utf-8 3 | # GRANT REPLICATION CLIENT ON *.* TO 'monitor'@'%' IDENTIFIED BY 'm0n1tor'; 4 | 5 | import sys 6 | import getopt 7 | import pymysql 8 | import logging 9 | 10 | dbhost = '' 11 | dbport = 3306 12 | dbuser = "monitor" 13 | dbpassword = "m0n1tor" 14 | 15 | # allow max Seconds_Behind_Master 16 | MaxSBM = 300 17 | 18 | 19 | class Logger: 20 | Logger = None 21 | 22 | def __init__(self): 23 | fmt = '%(asctime)s - %(filename)s:%(lineno)s - %(name)s - %(message)s' 24 | 25 | 26 | def checkSlave(): 27 | global dbhost 28 | global dbport 29 | global dbuser 30 | global dbpassword 31 | 32 | shortargs = 'h:P:' 33 | opts, args = getopt.getopt(sys.argv[1:], shortargs) 34 | for opt, value in opts: 35 | if opt == '-h': 36 | dbhost = value 37 | elif opt == '-P': 38 | dbport = value 39 | # print "host : %s, port: %d, user: %s, password: %s" % (dbhost, int(dbport), dbuser, dbpassword) 40 | db = pymysql.connect(dbhost, dbport, dbuser, dbpassword) 41 | if db.connect() != 0: 42 | return 1 43 | slave_status = db.getSlave() 44 | return slave_status 45 | 46 | 47 | class instanceMySQL: 48 | conn = None 49 | 50 | def __init__(self, host=None, port=None, user=None, passwd=None): 51 | self.dbhost = host 52 | self.dbport = int(port) 53 | self.dbuser = user 54 | self.dbpassword = passwd 55 | 56 | def connect(self): 57 | # print "in db conn" 58 | # print "host : %s, port: %d, user: %s, password: %s" % (self.dbhost, self.dbport, self.dbuser, self.dbpassword) 59 | try: 60 | self.conn = MySQLdb.connect(host="%s" % self.dbhost, port=self.dbport, user="%s" % dbuser, 61 | passwd="%s" % self.dbpassword) 62 | except Exception, e: 63 | # print " Error" 64 | print 65 | e 66 | return 1 67 | return 0 68 | 69 | def getSlave(self): 70 | sql = "show slave status" 71 | cursor = self.conn.cursor(cursorclass=MySQLdb.cursors.DictCursor) 72 | cursor.execute(sql) 73 | result = cursor.fetchall() 74 | row = result[0] 75 | if (row['Slave_IO_Running'] != 'Yes'): 76 | return 1 77 | if (row['Slave_SQL_Running'] != 'Yes'): 78 | return 1 79 | if (row['Seconds_Behind_Master'] > MaxSBM): 80 | return 1 81 | cursor.close() 82 | self.disconnect() 83 | return 0 84 | 85 | def getWsrepState(self): 86 | sql = "SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_STATUS WHERE VARIABLE_NAME IN ('WSREP_CLUSTER_STATUS', 'WSREP_LOCAL_STATE_COMMENT');" 87 | cursor = self.conn.cursor() 88 | cursor.execute(sql) 89 | alldata = cursor.fetchall() 90 | if (alldata[0][0] != 'Synced'): 91 | return 1 92 | if (alldata[1][0] != 'Primary'): 93 | return 1 94 | return 0 95 | cursor.close() 96 | 97 | def disconnect(self): 98 | if (self.conn): 99 | self.conn.close() 100 | self.conn = None 101 | 102 | 103 | if __name__ == "__main__": 104 | st = checkSlave() 105 | # print st 106 | sys.exit(st) 107 | -------------------------------------------------------------------------------- /pmm/CrossServerGraphs.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "$$hashKey": "object:1403", 6 | "builtIn": 1, 7 | "datasource": "-- Grafana --", 8 | "enable": false, 9 | "hide": false, 10 | "iconColor": "#e0752d", 11 | "limit": 100, 12 | "name": "PMM Annotations", 13 | "showIn": 0, 14 | "tags": [ 15 | "pmm_annotation" 16 | ], 17 | "type": "tags" 18 | }, 19 | { 20 | "$$hashKey": "object:1404", 21 | "builtIn": 1, 22 | "datasource": "-- Grafana --", 23 | "enable": true, 24 | "hide": true, 25 | "iconColor": "#6ed0e0", 26 | "limit": 100, 27 | "name": "Annotations & Alerts", 28 | "showIn": 0, 29 | "tags": [], 30 | "type": "dashboard" 31 | } 32 | ] 33 | }, 34 | "editable": true, 35 | "gnetId": null, 36 | "graphTooltip": 1, 37 | "id": 110, 38 | "iteration": 1547012413746, 39 | "links": [ 40 | { 41 | "$$hashKey": "object:1564", 42 | "icon": "dashboard", 43 | "includeVars": true, 44 | "keepTime": true, 45 | "tags": [ 46 | "QAN" 47 | ], 48 | "targetBlank": false, 49 | "title": "Query Analytics", 50 | "type": "link", 51 | "url": "/graph/dashboard/db/_pmm-query-analytics" 52 | }, 53 | { 54 | "$$hashKey": "object:1565", 55 | "asDropdown": true, 56 | "includeVars": true, 57 | "keepTime": true, 58 | "tags": [ 59 | "OS" 60 | ], 61 | "targetBlank": false, 62 | "title": "OS", 63 | "type": "dashboards" 64 | }, 65 | { 66 | "$$hashKey": "object:1566", 67 | "asDropdown": true, 68 | "includeVars": true, 69 | "keepTime": true, 70 | "tags": [ 71 | "MySQL" 72 | ], 73 | "targetBlank": false, 74 | "title": "MySQL", 75 | "type": "dashboards" 76 | }, 77 | { 78 | "$$hashKey": "object:1567", 79 | "asDropdown": true, 80 | "includeVars": true, 81 | "keepTime": true, 82 | "tags": [ 83 | "MongoDB" 84 | ], 85 | "targetBlank": false, 86 | "title": "MongoDB", 87 | "type": "dashboards" 88 | }, 89 | { 90 | "$$hashKey": "object:1568", 91 | "asDropdown": true, 92 | "includeVars": true, 93 | "keepTime": true, 94 | "tags": [ 95 | "PostgreSQL" 96 | ], 97 | "targetBlank": false, 98 | "title": "PostgreSQL", 99 | "type": "dashboards" 100 | }, 101 | { 102 | "$$hashKey": "object:1569", 103 | "asDropdown": true, 104 | "includeVars": true, 105 | "keepTime": true, 106 | "tags": [ 107 | "HA" 108 | ], 109 | "targetBlank": false, 110 | "title": "HA", 111 | "type": "dashboards" 112 | }, 113 | { 114 | "$$hashKey": "object:1570", 115 | "asDropdown": true, 116 | "includeVars": true, 117 | "keepTime": true, 118 | "tags": [ 119 | "Cloud" 120 | ], 121 | "targetBlank": false, 122 | "title": "Cloud", 123 | "type": "dashboards" 124 | }, 125 | { 126 | "$$hashKey": "object:1571", 127 | "asDropdown": true, 128 | "includeVars": true, 129 | "keepTime": true, 130 | "tags": [ 131 | "Insight" 132 | ], 133 | "targetBlank": false, 134 | "title": "Insight", 135 | "type": "dashboards" 136 | }, 137 | { 138 | "$$hashKey": "object:1572", 139 | "asDropdown": true, 140 | "includeVars": true, 141 | "keepTime": true, 142 | "tags": [ 143 | "PMM" 144 | ], 145 | "targetBlank": false, 146 | "title": "PMM", 147 | "type": "dashboards" 148 | } 149 | ], 150 | "panels": [ 151 | { 152 | "aliasColors": {}, 153 | "bars": false, 154 | "dashLength": 10, 155 | "dashes": false, 156 | "datasource": "Prometheus", 157 | "decimals": 2, 158 | "description": "**Load Average**\n\nOn Linux, this is the number of processes which are in “running” state or in “uninterruptible sleep” state which typically corresponds to disk IO. You can also map LoadAvg to VMSTAT output – it is something like moving average of sum of “r” and “b” columns from VMSTAT.\n\nThis chart is best used for trends. If you notice the load average rising, it may be due to innefficient queries. In that case, it is a good idea to look at PMM's Query Analytics.", 159 | "editable": true, 160 | "error": false, 161 | "fill": 2, 162 | "grid": {}, 163 | "gridPos": { 164 | "h": 7, 165 | "w": 12, 166 | "x": 0, 167 | "y": 0 168 | }, 169 | "id": 6, 170 | "legend": { 171 | "alignAsTable": false, 172 | "avg": false, 173 | "current": false, 174 | "hideEmpty": false, 175 | "max": false, 176 | "min": false, 177 | "rightSide": false, 178 | "show": true, 179 | "total": false, 180 | "values": false 181 | }, 182 | "lines": true, 183 | "linewidth": 2, 184 | "links": [ 185 | { 186 | "dashUri": "db/_pmm-query-analytics", 187 | "dashboard": "_PMM Query Analytics", 188 | "title": "PMM Query Analytics", 189 | "type": "dashboard", 190 | "url": "/qan/" 191 | }, 192 | { 193 | "dashboard": "https://www.percona.com/blog/2006/12/04/using-loadavg-for-performance-optimization/", 194 | "title": "Using LoadAvg for Performance Optimization.", 195 | "type": "absolute", 196 | "url": "https://www.percona.com/blog/2006/12/04/using-loadavg-for-performance-optimization/" 197 | } 198 | ], 199 | "nullPointMode": "null", 200 | "percentage": false, 201 | "pointradius": 5, 202 | "points": false, 203 | "renderer": "flot", 204 | "seriesOverrides": [], 205 | "spaceLength": 10, 206 | "stack": false, 207 | "steppedLine": false, 208 | "targets": [ 209 | { 210 | "calculatedInterval": "10s", 211 | "datasourceErrors": {}, 212 | "errors": {}, 213 | "expr": "node_load1{IP=~\"$IP\"}", 214 | "format": "time_series", 215 | "interval": "$interval", 216 | "intervalFactor": 1, 217 | "legendFormat": "{{ IP }}", 218 | "metric": "", 219 | "refId": "A", 220 | "step": 300, 221 | "target": "" 222 | } 223 | ], 224 | "thresholds": [ 225 | { 226 | "colorMode": "custom", 227 | "line": true, 228 | "lineColor": "rgb(241, 34, 15)", 229 | "op": "gt", 230 | "value": 15 231 | } 232 | ], 233 | "timeFrom": null, 234 | "timeShift": null, 235 | "title": "Load Average", 236 | "tooltip": { 237 | "msResolution": false, 238 | "shared": true, 239 | "sort": 0, 240 | "value_type": "individual" 241 | }, 242 | "transparent": false, 243 | "type": "graph", 244 | "xaxis": { 245 | "buckets": null, 246 | "mode": "time", 247 | "name": null, 248 | "show": true, 249 | "values": [] 250 | }, 251 | "yaxes": [ 252 | { 253 | "format": "none", 254 | "label": "", 255 | "logBase": 1, 256 | "max": null, 257 | "min": 0, 258 | "show": true 259 | }, 260 | { 261 | "format": "none", 262 | "logBase": 1, 263 | "max": null, 264 | "min": 0, 265 | "show": true 266 | } 267 | ], 268 | "yaxis": { 269 | "align": false, 270 | "alignLevel": null 271 | } 272 | }, 273 | { 274 | "aliasColors": {}, 275 | "bars": false, 276 | "dashLength": 10, 277 | "dashes": false, 278 | "datasource": "Prometheus", 279 | "decimals": 2, 280 | "editable": true, 281 | "error": false, 282 | "fill": 2, 283 | "grid": {}, 284 | "gridPos": { 285 | "h": 7, 286 | "w": 12, 287 | "x": 12, 288 | "y": 0 289 | }, 290 | "id": 7, 291 | "legend": { 292 | "alignAsTable": false, 293 | "avg": false, 294 | "current": false, 295 | "hideEmpty": false, 296 | "max": false, 297 | "min": false, 298 | "rightSide": false, 299 | "show": true, 300 | "total": false, 301 | "values": false 302 | }, 303 | "lines": true, 304 | "linewidth": 2, 305 | "links": [], 306 | "nullPointMode": "null", 307 | "percentage": false, 308 | "pointradius": 5, 309 | "points": false, 310 | "renderer": "flot", 311 | "seriesOverrides": [], 312 | "spaceLength": 10, 313 | "stack": false, 314 | "steppedLine": false, 315 | "targets": [ 316 | { 317 | "calculatedInterval": "10s", 318 | "datasourceErrors": {}, 319 | "errors": {}, 320 | "expr": "100 - 100 * (node_memory_MemAvailable{IP=~\"$IP\"} or (node_memory_MemFree{IP=~\"$IP\",PORT=~\"$PORT\"} + node_memory_Buffers{IP=~\"$IP\",PORT=~\"$PORT\"} + node_memory_Cached{IP=~\"$IP\"})) / (node_memory_MemTotal{IP=~\"$IP\"} + 0.1)", 321 | "format": "time_series", 322 | "interval": "$interval", 323 | "intervalFactor": 1, 324 | "legendFormat": "{{ IP }}", 325 | "metric": "", 326 | "refId": "A", 327 | "step": 300, 328 | "target": "" 329 | } 330 | ], 331 | "thresholds": [ 332 | { 333 | "colorMode": "custom", 334 | "line": true, 335 | "lineColor": "rgb(248, 8, 48)", 336 | "op": "gt", 337 | "value": 95 338 | } 339 | ], 340 | "timeFrom": null, 341 | "timeShift": null, 342 | "title": "Memory Usage", 343 | "tooltip": { 344 | "msResolution": false, 345 | "shared": true, 346 | "sort": 0, 347 | "value_type": "individual" 348 | }, 349 | "transparent": false, 350 | "type": "graph", 351 | "xaxis": { 352 | "buckets": null, 353 | "mode": "time", 354 | "name": null, 355 | "show": true, 356 | "values": [] 357 | }, 358 | "yaxes": [ 359 | { 360 | "format": "percent", 361 | "label": "", 362 | "logBase": 1, 363 | "max": null, 364 | "min": 0, 365 | "show": true 366 | }, 367 | { 368 | "format": "none", 369 | "logBase": 1, 370 | "max": null, 371 | "min": 0, 372 | "show": true 373 | } 374 | ], 375 | "yaxis": { 376 | "align": false, 377 | "alignLevel": null 378 | } 379 | }, 380 | { 381 | "aliasColors": {}, 382 | "bars": false, 383 | "dashLength": 10, 384 | "dashes": false, 385 | "datasource": "Prometheus", 386 | "editable": true, 387 | "error": false, 388 | "fill": 2, 389 | "grid": {}, 390 | "gridPos": { 391 | "h": 7, 392 | "w": 12, 393 | "x": 0, 394 | "y": 7 395 | }, 396 | "id": 3, 397 | "legend": { 398 | "avg": false, 399 | "current": false, 400 | "max": false, 401 | "min": false, 402 | "show": true, 403 | "total": false, 404 | "values": false 405 | }, 406 | "lines": true, 407 | "linewidth": 2, 408 | "links": [], 409 | "nullPointMode": "null", 410 | "percentage": false, 411 | "pointradius": 5, 412 | "points": false, 413 | "renderer": "flot", 414 | "seriesOverrides": [], 415 | "spaceLength": 10, 416 | "stack": false, 417 | "steppedLine": false, 418 | "targets": [ 419 | { 420 | "calculatedInterval": "2m", 421 | "datasourceErrors": {}, 422 | "errors": {}, 423 | "expr": "rate(mysql_global_status_connections{IP=~\"$IP\",PORT=~\"$PORT\"}[$interval]) or irate(mysql_global_status_connections{IP=~\"$IP\",PORT=~\"$PORT\"}[5m])", 424 | "format": "time_series", 425 | "interval": "$interval", 426 | "intervalFactor": 1, 427 | "legendFormat": "{{ instance }}", 428 | "metric": "", 429 | "refId": "A", 430 | "step": 300 431 | } 432 | ], 433 | "thresholds": [], 434 | "timeFrom": null, 435 | "timeShift": null, 436 | "title": "MySQL Connections", 437 | "tooltip": { 438 | "msResolution": false, 439 | "shared": true, 440 | "sort": 0, 441 | "value_type": "cumulative" 442 | }, 443 | "type": "graph", 444 | "xaxis": { 445 | "buckets": null, 446 | "mode": "time", 447 | "name": null, 448 | "show": true, 449 | "values": [] 450 | }, 451 | "yaxes": [ 452 | { 453 | "format": "short", 454 | "logBase": 1, 455 | "max": null, 456 | "min": 0, 457 | "show": true 458 | }, 459 | { 460 | "format": "short", 461 | "logBase": 1, 462 | "max": null, 463 | "min": 0, 464 | "show": true 465 | } 466 | ], 467 | "yaxis": { 468 | "align": false, 469 | "alignLevel": null 470 | } 471 | }, 472 | { 473 | "aliasColors": {}, 474 | "bars": false, 475 | "dashLength": 10, 476 | "dashes": false, 477 | "datasource": "Prometheus", 478 | "decimals": 2, 479 | "description": "**MySQL Queries**\n\nBased on the queries reported by MySQL's ``SHOW STATUS`` command, it is the average number of statements executed by the server. This variable includes statements executed within stored programs, unlike the Questions variable. It does not count COM_PING or COM_STATISTICS commands.", 480 | "editable": true, 481 | "error": false, 482 | "fill": 0, 483 | "grid": {}, 484 | "gridPos": { 485 | "h": 7, 486 | "w": 12, 487 | "x": 12, 488 | "y": 7 489 | }, 490 | "id": 4, 491 | "legend": { 492 | "alignAsTable": false, 493 | "avg": false, 494 | "current": false, 495 | "hideEmpty": false, 496 | "max": false, 497 | "min": false, 498 | "rightSide": false, 499 | "show": true, 500 | "sort": null, 501 | "sortDesc": null, 502 | "total": false, 503 | "values": false 504 | }, 505 | "lines": true, 506 | "linewidth": 2, 507 | "links": [ 508 | { 509 | "title": "Server Status Variables (Queries)", 510 | "type": "absolute", 511 | "url": "https://dev.mysql.com/doc/refman/5.6/en/server-status-variables.html#statvar_Queries" 512 | } 513 | ], 514 | "nullPointMode": "null", 515 | "percentage": false, 516 | "pointradius": 5, 517 | "points": false, 518 | "renderer": "flot", 519 | "seriesOverrides": [], 520 | "spaceLength": 10, 521 | "stack": false, 522 | "steppedLine": false, 523 | "targets": [ 524 | { 525 | "calculatedInterval": "2m", 526 | "datasourceErrors": {}, 527 | "errors": {}, 528 | "expr": "rate(mysql_global_status_queries{IP=~\"$IP\",PORT=~\"$PORT\"}[$interval]) or irate(mysql_global_status_queries{IP=~\"$IP\",PORT=~\"$PORT\"}[5m])", 529 | "format": "time_series", 530 | "interval": "$interval", 531 | "intervalFactor": 1, 532 | "legendFormat": "{{ instance }}", 533 | "metric": "", 534 | "refId": "A", 535 | "step": 300 536 | } 537 | ], 538 | "thresholds": [], 539 | "timeFrom": null, 540 | "timeShift": null, 541 | "title": "MySQL Queries", 542 | "tooltip": { 543 | "msResolution": false, 544 | "shared": true, 545 | "sort": 0, 546 | "value_type": "cumulative" 547 | }, 548 | "type": "graph", 549 | "xaxis": { 550 | "buckets": null, 551 | "mode": "time", 552 | "name": null, 553 | "show": true, 554 | "values": [] 555 | }, 556 | "yaxes": [ 557 | { 558 | "format": "short", 559 | "logBase": 1, 560 | "max": null, 561 | "min": 0, 562 | "show": true 563 | }, 564 | { 565 | "format": "short", 566 | "logBase": 1, 567 | "max": null, 568 | "min": 0, 569 | "show": true 570 | } 571 | ], 572 | "yaxis": { 573 | "align": false, 574 | "alignLevel": null 575 | } 576 | }, 577 | { 578 | "aliasColors": {}, 579 | "bars": false, 580 | "dashLength": 10, 581 | "dashes": false, 582 | "datasource": "Prometheus", 583 | "decimals": 2, 584 | "description": "**MySQL Traffic**\n\nNetwork traffic used by the MySQL process.", 585 | "editable": true, 586 | "error": false, 587 | "fill": 2, 588 | "grid": {}, 589 | "gridPos": { 590 | "h": 7, 591 | "w": 12, 592 | "x": 0, 593 | "y": 14 594 | }, 595 | "id": 11, 596 | "legend": { 597 | "alignAsTable": true, 598 | "avg": true, 599 | "current": false, 600 | "max": true, 601 | "min": true, 602 | "rightSide": false, 603 | "show": true, 604 | "sort": "avg", 605 | "sortDesc": true, 606 | "total": false, 607 | "values": true 608 | }, 609 | "lines": true, 610 | "linewidth": 2, 611 | "links": [], 612 | "nullPointMode": "null", 613 | "percentage": false, 614 | "pointradius": 5, 615 | "points": false, 616 | "renderer": "flot", 617 | "seriesOverrides": [], 618 | "spaceLength": 10, 619 | "stack": false, 620 | "steppedLine": false, 621 | "targets": [ 622 | { 623 | "calculatedInterval": "2m", 624 | "datasourceErrors": {}, 625 | "errors": {}, 626 | "expr": "(rate(mysql_global_status_bytes_received{IP=~\"$IP\",PORT=~\"$PORT\"}[$interval]) + rate(mysql_global_status_bytes_sent{IP=~\"$IP\",PORT=~\"$PORT\"}[$interval])) or (irate(mysql_global_status_bytes_received{IP=~\"$IP\",PORT=~\"$PORT\"}[5m]) + irate(mysql_global_status_bytes_sent{IP=~\"$IP\",PORT=~\"$PORT\"}[5m]))", 627 | "format": "time_series", 628 | "interval": "$interval", 629 | "intervalFactor": 1, 630 | "legendFormat": "{{ instance }}", 631 | "metric": "", 632 | "refId": "A", 633 | "step": 300 634 | } 635 | ], 636 | "thresholds": [], 637 | "timeFrom": null, 638 | "timeShift": null, 639 | "title": "MySQL Traffic", 640 | "tooltip": { 641 | "msResolution": true, 642 | "shared": true, 643 | "sort": 0, 644 | "value_type": "individual" 645 | }, 646 | "type": "graph", 647 | "xaxis": { 648 | "buckets": null, 649 | "mode": "time", 650 | "name": null, 651 | "show": true, 652 | "values": [] 653 | }, 654 | "yaxes": [ 655 | { 656 | "format": "Bps", 657 | "logBase": 1, 658 | "max": null, 659 | "min": 0, 660 | "show": true 661 | }, 662 | { 663 | "format": "none", 664 | "logBase": 1, 665 | "max": null, 666 | "min": 0, 667 | "show": true 668 | } 669 | ], 670 | "yaxis": { 671 | "align": false, 672 | "alignLevel": null 673 | } 674 | }, 675 | { 676 | "aliasColors": {}, 677 | "bars": false, 678 | "dashLength": 10, 679 | "dashes": false, 680 | "datasource": "Prometheus", 681 | "decimals": 2, 682 | "editable": true, 683 | "error": false, 684 | "fill": 2, 685 | "grid": {}, 686 | "gridPos": { 687 | "h": 7, 688 | "w": 12, 689 | "x": 12, 690 | "y": 14 691 | }, 692 | "id": 10, 693 | "legend": { 694 | "alignAsTable": true, 695 | "avg": true, 696 | "current": false, 697 | "hideEmpty": false, 698 | "max": true, 699 | "min": true, 700 | "rightSide": false, 701 | "show": true, 702 | "sort": "avg", 703 | "sortDesc": true, 704 | "total": false, 705 | "values": true 706 | }, 707 | "lines": true, 708 | "linewidth": 2, 709 | "links": [], 710 | "nullPointMode": "null", 711 | "percentage": false, 712 | "pointradius": 5, 713 | "points": false, 714 | "renderer": "flot", 715 | "seriesOverrides": [], 716 | "spaceLength": 10, 717 | "stack": false, 718 | "steppedLine": false, 719 | "targets": [ 720 | { 721 | "calculatedInterval": "2s", 722 | "datasourceErrors": {}, 723 | "errors": {}, 724 | "expr": "sum(rate(node_network_receive_bytes{IP=~\"$IP\", device!=\"lo\"}[$interval]) + rate(node_network_transmit_bytes{IP=~\"$IP\", device!=\"lo\"}[$interval])) by (IP) or sum(irate(node_network_receive_bytes{P=~\"$IP\", device!=\"lo\"}[5m]) + irate(node_network_transmit_bytes{P=~\"$IP\", device!=\"lo\"}[5m])) by (IP) or\nsum(max_over_time(rdsosmetrics_network_rx{IP=~\"$IP\"}[$interval]) + max_over_time(rdsosmetrics_network_tx{IP=~\"$IP\"}[$interval])) by (instance) or sum(max_over_time(rdsosmetrics_network_rx{IP=~\"$IP\"}[5m]) + max_over_time(rdsosmetrics_network_tx{IP=~\"$IP\"}[5m])) by (IP)", 725 | "format": "time_series", 726 | "interval": "$interval", 727 | "intervalFactor": 1, 728 | "legendFormat": "{{IP }}", 729 | "metric": "", 730 | "refId": "B", 731 | "step": 300, 732 | "target": "" 733 | } 734 | ], 735 | "thresholds": [], 736 | "timeFrom": null, 737 | "timeShift": null, 738 | "title": "Network Traffic", 739 | "tooltip": { 740 | "msResolution": false, 741 | "shared": true, 742 | "sort": 0, 743 | "value_type": "individual" 744 | }, 745 | "transparent": false, 746 | "type": "graph", 747 | "xaxis": { 748 | "buckets": null, 749 | "mode": "time", 750 | "name": null, 751 | "show": true, 752 | "values": [] 753 | }, 754 | "yaxes": [ 755 | { 756 | "format": "Bps", 757 | "label": "", 758 | "logBase": 1, 759 | "max": null, 760 | "min": 0, 761 | "show": true 762 | }, 763 | { 764 | "format": "bytes", 765 | "logBase": 1, 766 | "max": null, 767 | "min": 0, 768 | "show": true 769 | } 770 | ], 771 | "yaxis": { 772 | "align": false, 773 | "alignLevel": null 774 | } 775 | }, 776 | { 777 | "columns": [ 778 | { 779 | "text": "Current", 780 | "value": "current" 781 | } 782 | ], 783 | "datasource": "Prometheus", 784 | "editable": true, 785 | "error": false, 786 | "fontSize": "90%", 787 | "gridPos": { 788 | "h": 7, 789 | "w": 24, 790 | "x": 0, 791 | "y": 21 792 | }, 793 | "height": "", 794 | "id": 12, 795 | "links": [], 796 | "pageSize": null, 797 | "scroll": false, 798 | "showHeader": true, 799 | "sort": { 800 | "col": 0, 801 | "desc": false 802 | }, 803 | "styles": [], 804 | "targets": [ 805 | { 806 | "expr": "node_uname_info{IP=~\"$IP\"}", 807 | "format": "table", 808 | "interval": "5m", 809 | "intervalFactor": 1, 810 | "legendFormat": "{{ instance }} | Hostname: {{ nodename }} | Kernel: {{ release }}", 811 | "refId": "A", 812 | "step": 300 813 | } 814 | ], 815 | "title": "System Info", 816 | "transform": "table", 817 | "transparent": true, 818 | "type": "table" 819 | }, 820 | { 821 | "columns": [ 822 | { 823 | "text": "Current", 824 | "value": "current" 825 | } 826 | ], 827 | "datasource": "Prometheus", 828 | "editable": true, 829 | "error": false, 830 | "fontSize": "90%", 831 | "gridPos": { 832 | "h": 7, 833 | "w": 24, 834 | "x": 0, 835 | "y": 28 836 | }, 837 | "id": 13, 838 | "links": [], 839 | "pageSize": null, 840 | "scroll": false, 841 | "showHeader": true, 842 | "sort": { 843 | "col": 0, 844 | "desc": false 845 | }, 846 | "styles": [], 847 | "targets": [ 848 | { 849 | "expr": "mysql_version_info{IP=~\"$IP\",PORT=~\"$PORT\"}", 850 | "format": "table", 851 | "interval": "5m", 852 | "intervalFactor": 1, 853 | "legendFormat": "{{ instance }} | {{ version }} | {{ version_comment }}", 854 | "refId": "A", 855 | "step": 300 856 | } 857 | ], 858 | "title": "MySQL Info", 859 | "transform": "table", 860 | "transparent": true, 861 | "type": "table" 862 | } 863 | ], 864 | "refresh": "1m", 865 | "schemaVersion": 16, 866 | "style": "dark", 867 | "tags": [ 868 | "Percona", 869 | "Insight", 870 | "V2" 871 | ], 872 | "templating": { 873 | "list": [ 874 | { 875 | "allFormat": "glob", 876 | "auto": true, 877 | "auto_count": 200, 878 | "auto_min": "1s", 879 | "current": { 880 | "text": "auto", 881 | "value": "$__auto_interval_interval" 882 | }, 883 | "datasource": "Prometheus", 884 | "hide": 0, 885 | "includeAll": false, 886 | "label": "Interval", 887 | "multi": false, 888 | "multiFormat": "glob", 889 | "name": "interval", 890 | "options": [ 891 | { 892 | "selected": true, 893 | "text": "auto", 894 | "value": "$__auto_interval_interval" 895 | }, 896 | { 897 | "selected": false, 898 | "text": "1s", 899 | "value": "1s" 900 | }, 901 | { 902 | "selected": false, 903 | "text": "5s", 904 | "value": "5s" 905 | }, 906 | { 907 | "selected": false, 908 | "text": "1m", 909 | "value": "1m" 910 | }, 911 | { 912 | "selected": false, 913 | "text": "5m", 914 | "value": "5m" 915 | }, 916 | { 917 | "selected": false, 918 | "text": "1h", 919 | "value": "1h" 920 | }, 921 | { 922 | "selected": false, 923 | "text": "6h", 924 | "value": "6h" 925 | }, 926 | { 927 | "selected": false, 928 | "text": "1d", 929 | "value": "1d" 930 | } 931 | ], 932 | "query": "1s,5s,1m,5m,1h,6h,1d", 933 | "refresh": 2, 934 | "regex": "", 935 | "type": "interval" 936 | }, 937 | { 938 | "allFormat": "pipe", 939 | "allValue": null, 940 | "current": { 941 | "text": "All", 942 | "value": "$__all" 943 | }, 944 | "datasource": "Prometheus", 945 | "hide": 0, 946 | "includeAll": true, 947 | "label": "Host", 948 | "multi": true, 949 | "multiFormat": "pipe", 950 | "name": "host", 951 | "options": [], 952 | "query": "label_values({__name__=~\"node_load1|mysql_up\", job!=\"prometheus\"}, instance)", 953 | "refresh": 1, 954 | "regex": "", 955 | "sort": 1, 956 | "tagValuesQuery": null, 957 | "tags": [], 958 | "tagsQuery": null, 959 | "type": "query", 960 | "useTags": false 961 | }, 962 | { 963 | "allValue": null, 964 | "current": { 965 | "text": "All", 966 | "value": [ 967 | "$__all" 968 | ] 969 | }, 970 | "datasource": "Prometheus", 971 | "hide": 0, 972 | "includeAll": true, 973 | "label": null, 974 | "multi": true, 975 | "name": "IP", 976 | "options": [], 977 | "query": "label_values(mysql_up,IP)", 978 | "refresh": 1, 979 | "regex": "", 980 | "sort": 0, 981 | "tagValuesQuery": "", 982 | "tags": [], 983 | "tagsQuery": "", 984 | "type": "query", 985 | "useTags": false 986 | }, 987 | { 988 | "allValue": null, 989 | "current": { 990 | "text": "4306", 991 | "value": [ 992 | "4306" 993 | ] 994 | }, 995 | "datasource": "Prometheus", 996 | "hide": 0, 997 | "includeAll": false, 998 | "label": null, 999 | "multi": true, 1000 | "name": "PORT", 1001 | "options": [], 1002 | "query": "label_values(mysql_up,PORT)", 1003 | "refresh": 1, 1004 | "regex": "", 1005 | "sort": 0, 1006 | "tagValuesQuery": "", 1007 | "tags": [], 1008 | "tagsQuery": "", 1009 | "type": "query", 1010 | "useTags": false 1011 | } 1012 | ] 1013 | }, 1014 | "time": { 1015 | "from": "now-5m", 1016 | "to": "now" 1017 | }, 1018 | "timepicker": { 1019 | "collapse": false, 1020 | "enable": true, 1021 | "hidden": false, 1022 | "notice": false, 1023 | "now": true, 1024 | "refresh_intervals": [ 1025 | "5s", 1026 | "10s", 1027 | "30s", 1028 | "1m", 1029 | "5m", 1030 | "15m", 1031 | "30m", 1032 | "1h", 1033 | "2h", 1034 | "1d" 1035 | ], 1036 | "status": "Stable", 1037 | "time_options": [ 1038 | "5m", 1039 | "15m", 1040 | "1h", 1041 | "6h", 1042 | "12h", 1043 | "24h", 1044 | "2d", 1045 | "7d", 1046 | "30d" 1047 | ], 1048 | "type": "timepicker" 1049 | }, 1050 | "timezone": "browser", 1051 | "title": "Cross Server Graphs V2", 1052 | "uid": "Hx5Y3Nfmk", 1053 | "version": 5 1054 | } 1055 | -------------------------------------------------------------------------------- /pmm/prometheus.yml: -------------------------------------------------------------------------------- 1 | # Managed by pmm-managed. DO NOT EDIT. 2 | --- 3 | global: 4 | scrape_interval: 1m 5 | scrape_timeout: 10s 6 | evaluation_interval: 1m 7 | scrape_configs: 8 | - job_name: prometheus 9 | scrape_interval: 1s 10 | scrape_timeout: 1s 11 | metrics_path: /prometheus/metrics 12 | scheme: http 13 | static_configs: 14 | - targets: 15 | - localhost:9090 16 | labels: 17 | instance: pmm-server 18 | - job_name: grafana 19 | scrape_interval: 5s 20 | scrape_timeout: 4s 21 | metrics_path: /metrics 22 | scheme: http 23 | static_configs: 24 | - targets: 25 | - localhost:3000 26 | labels: 27 | instance: pmm-server 28 | - job_name: linux 29 | scrape_interval: 1s 30 | scrape_timeout: 1s 31 | metrics_path: /metrics 32 | scheme: http 33 | static_configs: 34 | - targets: 35 | - localhost:9100 36 | labels: 37 | instance: pmm-server 38 | consul_sd_configs: 39 | - server: localhost:8500 40 | datacenter: dc1 41 | tag_separator: ',' 42 | scheme: http 43 | services: 44 | - linux:metrics 45 | basic_auth: 46 | username: pmm 47 | password: pmm 48 | tls_config: 49 | insecure_skip_verify: true 50 | relabel_configs: 51 | - source_labels: [__meta_consul_tags] 52 | separator: ; 53 | regex: .*,alias_([-\w:\.]+),.* 54 | target_label: instance 55 | replacement: $1 56 | action: replace 57 | - source_labels: [__meta_consul_tags] 58 | separator: ; 59 | regex: .*,scheme_([\w]+),.* 60 | target_label: __scheme__ 61 | replacement: $1 62 | action: replace 63 | - source_labels: [__meta_consul_tags] 64 | separator: ; 65 | regex: .*,alias_([-\w:\.]+):.* 66 | target_label: IP 67 | replacement: $1 68 | action: replace 69 | - job_name: proxysql 70 | scrape_interval: 1s 71 | scrape_timeout: 1s 72 | metrics_path: /metrics 73 | scheme: http 74 | consul_sd_configs: 75 | - server: localhost:8500 76 | datacenter: dc1 77 | tag_separator: ',' 78 | scheme: http 79 | services: 80 | - proxysql:metrics 81 | basic_auth: 82 | username: pmm 83 | password: pmm 84 | tls_config: 85 | insecure_skip_verify: true 86 | relabel_configs: 87 | - source_labels: [__meta_consul_tags] 88 | separator: ; 89 | regex: .*,alias_([-\w:\.]+),.* 90 | target_label: instance 91 | replacement: $1 92 | action: replace 93 | - source_labels: [__meta_consul_tags] 94 | separator: ; 95 | regex: .*,scheme_([\w]+),.* 96 | target_label: __scheme__ 97 | replacement: $1 98 | action: replace 99 | - job_name: mongodb 100 | scrape_interval: 1s 101 | scrape_timeout: 1s 102 | metrics_path: /metrics 103 | scheme: http 104 | consul_sd_configs: 105 | - server: localhost:8500 106 | datacenter: dc1 107 | tag_separator: ',' 108 | scheme: http 109 | services: 110 | - mongodb:metrics 111 | basic_auth: 112 | username: pmm 113 | password: pmm 114 | tls_config: 115 | insecure_skip_verify: true 116 | relabel_configs: 117 | - source_labels: [__meta_consul_tags] 118 | separator: ; 119 | regex: .*,alias_([-\w:\.]+),.* 120 | target_label: instance 121 | replacement: $1 122 | action: replace 123 | - source_labels: [__meta_consul_tags] 124 | separator: ; 125 | regex: .*,scheme_([\w]+),.* 126 | target_label: __scheme__ 127 | replacement: $1 128 | action: replace 129 | - source_labels: [__meta_consul_tags] 130 | separator: ; 131 | regex: .*,cluster_([-\w:\.]+),.* 132 | target_label: cluster 133 | replacement: $1 134 | action: replace 135 | - source_labels: [__meta_consul_tags] 136 | separator: ; 137 | regex: .*,alias_([-\w:\.]+):.* 138 | target_label: IP 139 | replacement: $1 140 | action: replace 141 | - source_labels: [__meta_consul_tags] 142 | separator: ; 143 | regex: .*:([-\w:\.]+),.* 144 | target_label: PORT 145 | replacement: $1 146 | action: replace 147 | - job_name: postgresql 148 | scrape_interval: 1s 149 | scrape_timeout: 1s 150 | metrics_path: /metrics 151 | scheme: http 152 | consul_sd_configs: 153 | - server: localhost:8500 154 | datacenter: dc1 155 | tag_separator: ',' 156 | scheme: http 157 | services: 158 | - postgresql:metrics 159 | basic_auth: 160 | username: pmm 161 | password: pmm 162 | tls_config: 163 | insecure_skip_verify: true 164 | relabel_configs: 165 | - separator: ; 166 | regex: (.*) 167 | target_label: job 168 | replacement: postgresql 169 | action: replace 170 | - source_labels: [__meta_consul_tags] 171 | separator: ; 172 | regex: .*,alias_([-\w:\.]+),.* 173 | target_label: instance 174 | replacement: $1 175 | action: replace 176 | - source_labels: [__meta_consul_tags] 177 | separator: ; 178 | regex: .*,scheme_([\w]+),.* 179 | target_label: __scheme__ 180 | replacement: $1 181 | action: replace 182 | - job_name: mysql-hr 183 | scrape_interval: 1s 184 | scrape_timeout: 1s 185 | metrics_path: /metrics-hr 186 | scheme: http 187 | consul_sd_configs: 188 | - server: localhost:8500 189 | datacenter: dc1 190 | tag_separator: ',' 191 | scheme: http 192 | services: 193 | - mysql:metrics 194 | basic_auth: 195 | username: pmm 196 | password: pmm 197 | tls_config: 198 | insecure_skip_verify: true 199 | relabel_configs: 200 | - separator: ; 201 | regex: (.*) 202 | target_label: job 203 | replacement: mysql 204 | action: replace 205 | - source_labels: [__meta_consul_tags] 206 | separator: ; 207 | regex: .*,alias_([-\w:\.]+),.* 208 | target_label: instance 209 | replacement: $1 210 | action: replace 211 | - source_labels: [__meta_consul_tags] 212 | separator: ; 213 | regex: .*,scheme_([\w]+),.* 214 | target_label: __scheme__ 215 | replacement: $1 216 | action: replace 217 | - source_labels: [__meta_consul_tags] 218 | separator: ; 219 | regex: .*,alias_([-\w:\.]+):.* 220 | target_label: IP 221 | replacement: $1 222 | action: replace 223 | - source_labels: [__meta_consul_tags] 224 | separator: ; 225 | regex: .*:([-\w:\.]+),.* 226 | target_label: PORT 227 | replacement: $1 228 | action: replace 229 | - job_name: mysql-mr 230 | scrape_interval: 5s 231 | scrape_timeout: 1s 232 | metrics_path: /metrics-mr 233 | scheme: http 234 | consul_sd_configs: 235 | - server: localhost:8500 236 | datacenter: dc1 237 | tag_separator: ',' 238 | scheme: http 239 | services: 240 | - mysql:metrics 241 | basic_auth: 242 | username: pmm 243 | password: pmm 244 | tls_config: 245 | insecure_skip_verify: true 246 | relabel_configs: 247 | - separator: ; 248 | regex: (.*) 249 | target_label: job 250 | replacement: mysql 251 | action: replace 252 | - source_labels: [__meta_consul_tags] 253 | separator: ; 254 | regex: .*,alias_([-\w:\.]+),.* 255 | target_label: instance 256 | replacement: $1 257 | action: replace 258 | - source_labels: [__meta_consul_tags] 259 | separator: ; 260 | regex: .*,scheme_([\w]+),.* 261 | target_label: __scheme__ 262 | replacement: $1 263 | action: replace 264 | - source_labels: [__meta_consul_tags] 265 | separator: ; 266 | regex: .*,alias_([-\w:\.]+):.* 267 | target_label: IP 268 | replacement: $1 269 | action: replace 270 | - source_labels: [__meta_consul_tags] 271 | separator: ; 272 | regex: .*:([-\w:\.]+),.* 273 | target_label: PORT 274 | replacement: $1 275 | action: replace 276 | - job_name: mysql-lr 277 | scrape_interval: 1m 278 | scrape_timeout: 5s 279 | metrics_path: /metrics-lr 280 | scheme: http 281 | consul_sd_configs: 282 | - server: localhost:8500 283 | datacenter: dc1 284 | tag_separator: ',' 285 | scheme: http 286 | services: 287 | - mysql:metrics 288 | basic_auth: 289 | username: pmm 290 | password: pmm 291 | tls_config: 292 | insecure_skip_verify: true 293 | relabel_configs: 294 | - separator: ; 295 | regex: (.*) 296 | target_label: job 297 | replacement: mysql 298 | action: replace 299 | - source_labels: [__meta_consul_tags] 300 | separator: ; 301 | regex: .*,alias_([-\w:\.]+),.* 302 | target_label: instance 303 | replacement: $1 304 | action: replace 305 | - source_labels: [__meta_consul_tags] 306 | separator: ; 307 | regex: .*,scheme_([\w]+),.* 308 | target_label: __scheme__ 309 | replacement: $1 310 | action: replace 311 | - source_labels: [__meta_consul_tags] 312 | separator: ; 313 | regex: .*,alias_([-\w:\.]+):.* 314 | target_label: IP 315 | replacement: $1 316 | action: replace 317 | - source_labels: [__meta_consul_tags] 318 | separator: ; 319 | regex: .*:([-\w:\.]+),.* 320 | target_label: PORT 321 | replacement: $1 322 | action: replace 323 | - job_name: rds-mysql-hr 324 | honor_labels: true 325 | scrape_interval: 1s 326 | scrape_timeout: 1s 327 | metrics_path: /metrics-hr 328 | scheme: http 329 | relabel_configs: 330 | - separator: ; 331 | regex: (.*) 332 | target_label: job 333 | replacement: mysql 334 | action: replace 335 | - job_name: rds-mysql-mr 336 | honor_labels: true 337 | scrape_interval: 5s 338 | scrape_timeout: 1s 339 | metrics_path: /metrics-mr 340 | scheme: http 341 | relabel_configs: 342 | - separator: ; 343 | regex: (.*) 344 | target_label: job 345 | replacement: mysql 346 | action: replace 347 | - job_name: rds-mysql-lr 348 | honor_labels: true 349 | scrape_interval: 1m 350 | scrape_timeout: 5s 351 | metrics_path: /metrics-lr 352 | scheme: http 353 | relabel_configs: 354 | - separator: ; 355 | regex: (.*) 356 | target_label: job 357 | replacement: mysql 358 | action: replace 359 | - job_name: rds-basic 360 | honor_labels: true 361 | scrape_interval: 1m 362 | scrape_timeout: 55s 363 | metrics_path: /basic 364 | scheme: http 365 | - job_name: rds-enhanced 366 | honor_labels: true 367 | scrape_interval: 10s 368 | scrape_timeout: 9s 369 | metrics_path: /enhanced 370 | scheme: http 371 | - job_name: remote-mysql-hr 372 | scrape_interval: 1s 373 | scrape_timeout: 1s 374 | metrics_path: /metrics-hr 375 | scheme: http 376 | relabel_configs: 377 | - separator: ; 378 | regex: (.*) 379 | target_label: job 380 | replacement: mysql 381 | action: replace 382 | - job_name: remote-mysql-mr 383 | scrape_interval: 5s 384 | scrape_timeout: 1s 385 | metrics_path: /metrics-mr 386 | scheme: http 387 | relabel_configs: 388 | - separator: ; 389 | regex: (.*) 390 | target_label: job 391 | replacement: mysql 392 | action: replace 393 | - job_name: remote-mysql-lr 394 | scrape_interval: 1m 395 | scrape_timeout: 5s 396 | metrics_path: /metrics-lr 397 | scheme: http 398 | relabel_configs: 399 | - separator: ; 400 | regex: (.*) 401 | target_label: job 402 | replacement: mysql 403 | action: replace 404 | - job_name: remote-postgresql 405 | scrape_interval: 1s 406 | scrape_timeout: 1s 407 | metrics_path: /metrics 408 | relabel_configs: 409 | - target_label: job 410 | replacement: postgresql 411 | remote_read: 412 | - url: http://127.0.0.1:9094/prometheus1/api/v1/read 413 | remote_timeout: 1m 414 | -------------------------------------------------------------------------------- /scan/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/naughtyGitCat/MySQLMiscellaneous/46f040da900c0883d4e5b475b1f0c4d030fc7929/scan/__init__.py -------------------------------------------------------------------------------- /scan/scan_port.py: -------------------------------------------------------------------------------- 1 | 2 | from ping3 import ping, verbose_ping 3 | import socket 4 | from openpyxl import Workbook 5 | from openpyxl import load_workbook 6 | 7 | 8 | def ifping(ipaddress): #检查IP地址是否连通 9 | result=ping( ipaddress) 10 | return result 11 | 12 | def checkport(ipaddress,port): #检查端口是否开放 13 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 14 | result = sock.connect_ex((ipaddress, port)) 15 | return result 16 | 17 | 18 | 19 | 20 | if __name__=="__main__": 21 | #ipaddresses=['10.65.206.23','10.65.1.168','10.65.202.84'] 22 | dbs=['1521:oracle','1522:oracle','1523:oracle','1525:oracle','1527:oracle','1529:oracle','1433:sqlserver','3306:mysql'] # 将需要检查的数据库端口放入列表 23 | networks=['192.168.1'] #将需要检查的网段放入列表 24 | path=r'scan.xlsx' 25 | wb = load_workbook(path) #打开excel文件，需要事先存在 26 | 27 | # 初始化第一个sheet 28 | #work_sheet = wb.active 29 | #work_sheet.title = "range names" 30 | for network in networks: 31 | ## 新建sheet用于存放各个网段并初始化 32 | if network in wb.sheetnames: #wb.sheetnames为excel的所有sheet名称 33 | work_sheet = wb.get_sheet_by_name(network) #指定数据库信息写入的sheet名称 34 | else: 35 | wb.create_sheet(title=network) #新建sheet，title指定sheet名称 36 | work_sheet = wb.get_sheet_by_name(network) 37 | work_sheet.append(['IP地址', '数据库']) #在新建sheet第一行设置标题 38 | wb.save(path) #保存文件 39 | 40 | for i in range(1,255): 41 | ipaddress=network+'.'+str(i) 42 | print ('now checking '+ipaddress) 43 | result = ifping(ipaddress) 44 | if result is None: 45 | #print(ipaddress + ' is not reachable') 46 | work_sheet.append([ipaddress, 'not reachable']) #如不通则写入相关信息至excel 47 | wb.save(path) 48 | 49 | else: 50 | #print(ipaddress + ' is reachable') 51 | opened = 0 # 用于判断是否无DB端口开放 52 | for db in dbs: 53 | port=int(db.split(':')[0]) 54 | name=db.split(':')[1] 55 | ifopen=checkport(ipaddress, port) 56 | #print (result) 57 | if ifopen==0: 58 | #print (ipaddress +' have '+name +' installed') 59 | opened=opened+1 60 | work_sheet.append([ipaddress,name]) #写入开放的数据库端口 61 | wb.save(path) 62 | if opened==0: 63 | work_sheet.append([ipaddress, 'No DBs Found']) #如没有端口开放则写入未找到数据库 64 | wb.save(path) 65 | #wb.save(path) 66 | wb.close() #关闭文件 -------------------------------------------------------------------------------- /separateSQL/README.md: -------------------------------------------------------------------------------- 1 | 从全备SQL文件中分离出各个库的语句 2 | -------------------------------------------------------------------------------- /separateSQL/functions.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | # 函数，按行读出sql文件所有内容，省内存 4 | 5 | 6 | def read_sql(filename): 7 | total = '' 8 | with open(filename, 'rt', encoding='utf8') as open_file: 9 | for line in open_file: 10 | total += line 11 | return total 12 | 13 | # 函数，取出最后数据相关语句的最后位置 14 | 15 | 16 | def get_data_tail(total): 17 | a = re.compile('''SET @@SESSION.SQL_LOG_BIN = @MYSQLDUMP_TEMP_LOG_BIN;''') 18 | b = a.search(total) 19 | start = b.end()-1 20 | return start 21 | 22 | # 函数，掐头去尾，移除末尾杂项语句，输出前面的复制信息 23 | 24 | 25 | def get_context(total): 26 | a = re.compile('''Current Database:''') 27 | b = a.search(total) 28 | start = b.end()-1 29 | m = re.compile('''Current Database:''') 30 | n = m.search(total) 31 | end = n.start()-1 32 | a = total[start:end] 33 | return a 34 | 35 | # 函数，输入文本与关键字，取出关键字在文本中的的每个起始位置，以数组的形式输出 36 | 37 | 38 | def get_pos(KEY_WORD, total): 39 | KEY_WORD_LENGTH = len(KEY_WORD) 40 | t = [] # 将匹配到的位置放入这个数组中 41 | pos_start = 0 42 | pos_end = len(total) 43 | while 1: 44 | a = total.find(KEY_WORD, pos_start, pos_end) 45 | if a == -1: # 如果匹配不到就退出匹配 46 | break 47 | t.append(a) 48 | pos_start = a + KEY_WORD_LENGTH # 偏移量为匹配词的长度 49 | print(pos_start) 50 | print(t) 51 | return t 52 | 53 | 54 | # 函数，输入文本与关键字位置数组，取出关键字 55 | def check_keyword(t, total): 56 | for x in t: 57 | print(total[x:x+8]) 58 | if x == t[-1]: 59 | print('over') 60 | 61 | # 函数，输入文本与关键字位置数组，取出关键字之间的文本 62 | 63 | 64 | def save_paragraph(t, total): 65 | x = 0 66 | while 1: 67 | if x == len(t)-1: 68 | with open('db'+str(x), 'wt', encoding='utf-8') as fout: 69 | fout.write(total[t[x]:]) 70 | print('over') 71 | break 72 | print('x:', x) 73 | print('len_t', len(t)) 74 | print('check', t[x], t[x+1]) 75 | print(total[t[x]:t[x+1]]) # 取出第一段内容 76 | with open('output/db'+str(x)+'.sql', 'wt', encoding='utf-8') as fout: 77 | fout.write(total[t[x]-3:t[x+1]]) 78 | 79 | x += 1 -------------------------------------------------------------------------------- /separateSQL/separateDatabase.py: -------------------------------------------------------------------------------- 1 | from functions import * 2 | 3 | # 从mysqldump全备文件中分离特定库的SQL语句 4 | filename = 'django.sql' 5 | 6 | DATABASE = 'django' 7 | KEY_WORD = 'Current Database:' 8 | 9 | total = read_sql(filename) 10 | t = get_pos(KEY_WORD, total) 11 | check_keyword(t, total) 12 | 13 | # ###################保存每段关键字之后的内容到单独的SQL文本中#################### 14 | 15 | save_paragraph(t, total) 16 | 17 | # ##################备注####################### 18 | # 1.考虑到windows上不区分大小写，所以表名就不用库名命名了，会重复覆盖 19 | -------------------------------------------------------------------------------- /separateSQL/separateSpecifiedDatabase.py: -------------------------------------------------------------------------------- 1 | #先取出所有 2 | from functions import * 3 | import json,re 4 | 5 | # 从mysqldump全备文件中分离特定库的SQL语句 6 | filename = 'django.sql' 7 | 8 | # TODO: '这儿要改成接收输入的方式传入数据库名' 9 | DATABASE = input('请再次输入数据库名（部分也可以，但只能输出一个，\n所以：请输入部分但唯一的名称）：') 10 | PREFIX = 'Current Database:' 11 | KEY_WORD = PREFIX+'.*'+DATABASE 12 | total = read_sql(filename) 13 | 14 | 15 | def get_head(total): 16 | a = re.compile(KEY_WORD) 17 | b = a.search(total) 18 | start = b.start() 19 | return start 20 | 21 | 22 | start = get_head(total) 23 | 24 | 25 | def get_tail(total): # 取下次create database语句前面一个字母的位置 26 | a = re.compile(PREFIX) 27 | b = a.search(total, start+len(KEY_WORD)) # 错开开头 28 | while b is not None: 29 | end = b.start() - 1 30 | print('{} is not last db'.format(DATABASE)) 31 | break 32 | if b is None: 33 | print('{} is the last db'.format(DATABASE)) 34 | end = get_data_tail(total)+1 #原本已经减掉了1个，不能再减，先加上一个 35 | 36 | return end 37 | 38 | end = get_tail(total) 39 | 40 | # print(start,end) 检查检出的位置的正确性用 41 | 42 | def get_db_sql(total, start, end): 43 | with open('output/'+DATABASE+'.sql', 'wt', encoding= 'utf-8') as fout: 44 | fout.write(total[start-3:end]) 45 | 46 | 47 | get_db_sql(total,start,end) -------------------------------------------------------------------------------- /压测/Gnuplot_5.2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/naughtyGitCat/MySQLMiscellaneous/46f040da900c0883d4e5b475b1f0c4d030fc7929/压测/Gnuplot_5.2.pdf -------------------------------------------------------------------------------- /压测/gnuplot_readonly.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/naughtyGitCat/MySQLMiscellaneous/46f040da900c0883d4e5b475b1f0c4d030fc7929/压测/gnuplot_readonly.jpg -------------------------------------------------------------------------------- /压测/gnuplot绘图示例.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/naughtyGitCat/MySQLMiscellaneous/46f040da900c0883d4e5b475b1f0c4d030fc7929/压测/gnuplot绘图示例.jpg -------------------------------------------------------------------------------- /压测/gp522-win64-mingw.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/naughtyGitCat/MySQLMiscellaneous/46f040da900c0883d4e5b475b1f0c4d030fc7929/压测/gp522-win64-mingw.exe -------------------------------------------------------------------------------- /压测/收集与分析的脚本/README.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/naughtyGitCat/MySQLMiscellaneous/46f040da900c0883d4e5b475b1f0c4d030fc7929/压测/收集与分析的脚本/README.txt -------------------------------------------------------------------------------- /压测/收集与分析的脚本/hi_analyze.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | awk ' 3 | BEGIN{ 4 | printf "#ts date time load QPS"; 5 | fmt = " %.2f"; 6 | } 7 | /^TS/ { # The timestamp lines begin with TS. 8 | ts = substr($2, 1, index($2,".") - 1); 9 | load = NF - 2; 10 | diff = ts -prev_ts; 11 | prev_ts = ts; 12 | printf "\n%s %s %s %s",ts,$3,$4,substr($load, 1, length($load)-1); 13 | } 14 | /Queries/ { 15 | printf fmt, ($2-Queries)/diff; 16 | Queries=$2 17 | } 18 | ' "$@" 19 | 20 | -------------------------------------------------------------------------------- /压测/收集与分析的脚本/hi_performance.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | INTERVAL=5 4 | PREFIX=$INTERVAL-sec-status 5 | RUNFILE=/root/running 6 | mysql -e 'show global variables'>>mysql-variables 7 | while test -e $RUNFILE; do 8 | file=$(date +%F_%H) 9 | sleep=$(date +%s.%N |awk "{print $INTERVAL -(\$1 % $INTERVAL)}") 10 | sleep $sleep 11 | ts="$(date +"TS %s.%N %F %T")" 12 | loadavg="$(uptime)" 13 | echo "$ts $loadavg">> $PREFIX-${file}-status 14 | mysql -e "show global status" >> $PREFIX-${file}-status & 15 | echo "$ts $loadavg">> $PREFIX-${file}-innodbstatus 16 | mysql -e "show engine innodb status\G" >> $PREFIX-${file}-innodbstatus & 17 | echo "$ts $loadavg">> $PREFIX-${file}-processlist 18 | mysql -e "show full processlist\G" >>$PREFIX-${file}-processlist & 19 | echo $ts 20 | done 21 | echo Exiting because $RUNFILE not exist 22 | 23 | -------------------------------------------------------------------------------- /性能优化/qps_thread.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | mysqladmin ext -i1 |awk ' 3 | /Queries/{q=$4-qp;qp=$4} 4 | /Threads_connected/{tc=$4} 5 | /Threads_running/{printf "%5d %5d %5d\n",q, tc, $4}' 6 | #输出如下： 7 | # 19 7 2 8 | # 15 7 2 9 | # 21 7 2 10 | # 19 7 2 11 | # 32 7 2 12 | # 18 7 2 13 | # 19 7 2 14 | # 20 7 2 15 | # 15 7 2 16 | # 30 7 2 17 | # 21 7 2 18 | # 18 7 2 19 | #这个命令每秒捕获一次show global status的数据，输出给awk计算并输出每秒的查询数，连接线程，活跃线程。 20 | #这三个数据的趋势对于数据库级别偶尔停顿的敏感性很高。 21 | -------------------------------------------------------------------------------- /性能优化/高性能MySQL 笔记/【MySQL】【持续更新】《高性能MySQL》 学习笔记,第三章.md: -------------------------------------------------------------------------------- 1 | # 第三章：服务器性能剖析 2 | 3 | ​ 本章将针对如下三个问题进行解答： 4 | 5 | ​ 如何确认服务器是否达到了性能最佳的状态 6 | 7 | ​ 找出某条语句为什么执行不够快 8 | 9 | ​ 诊断被用户描述成“停顿”，“堆积”，“卡死”的某些间歇性疑难故障 10 | 11 | ## 1.性能优化简介： 12 | 13 | ​ 针对性能问题，1000个DBA，有1000个回答。诸如，“QPS”,"CPU Load"，“可扩展性”之类。 14 | 15 | ##### 原则一：我们将性能定义为完成某件任务所需要的时间度量。即：“性能就是响应时间” 。 16 | 17 | ​ 通过任务和时间来度量性能而不是资源。数据库的目的是执行SQL语句，上句中的“任务”即是查询或者SQL语句（SELECT UPDATE DELETE等等），综上，数据库服务器的性能用查询的响应时间度量，单位则是每个查询所花费的时间。在这里我们先假设性能优化就是在一定工作负载的情况下尽可能的降低响应时间。 18 | 19 | ​ CPU使用率只是一种现象，而不是很好的可度量的目标。 20 | 21 | ​ 吞吐量的提升可以看作性能优化的副产品，对查询的优化可以让服务器每秒执行更多的查询。因为每秒查询执行的时间更短了。（吞吐量即是性能的倒数：单位时间内查询的数量，QPS,QPM等等） 22 | 23 | ##### 原则二：无法测量就无法有效地优化。第一步应当测量时间花在什么地方。 24 | 25 | ​ 完成一项任务所需要的时间可以分成两部分：执行时间和等待时间。 26 | 27 | ​ 如果要优化任务的执行时间，最好的办法是通过测量定位不同的子任务花费的时间，然后优化去掉一些子任务，降低子任务的执行频率或者提升子任务的执行效率。 28 | 29 | ​ 如果要优化任务的等待时间，则相对复杂一些，等待可能是因为其他系统间接影响所致。 30 | 31 | ### 1.1.通过性能剖析进行优化。 32 | 33 | ​ 性能剖析（Profiling）分为两个步骤：先测量任务所花费的时间，然后对结果进行统计和排序，讲重要的任务排到前面。 34 | 35 | ​ 性能剖析报告会列出所有任务列表。每行记录一个任务，包括任务名，任务的执行时间，任务的消耗时间，任务的平均时间，以及该任务执行时间占全部时间的百分比。 36 | 37 | ​ 基于执行时间的分析研究的是什么任务的执行时间最长。 38 | 39 | ​ 基于等待的分析则是判断任务在什么阶段被阻塞的时间最长。 40 | 41 | ​ 事实上，当基于执行时间的分析发现一个任务需要花费时间花费太多时间的时候，应该深入去分析一下，可能会发现某些“执行时间”其实是在等待。 42 | 43 | ### 1.2.理解性能剖析 44 | 45 | ​ 尽管性能剖析输出了排名，总计和平均值，但还是有许多重要的信息是缺失的。 46 | 47 | ​ 值得优化的查询：一些只占总响应时间比重很小的查询是不值得优化的。如果花费了1000美元去优化一个任务，单业务的收入没有任何增加，那么可以说是打水漂了。如果优化的成本大于收益，就应当停止优化。 48 | 49 | ​ 异常情况:某些任务即使没有出现在性能剖析输出的前面也需要优化。比如，某些任务执行次数很少，单每次执行都非常慢，严重影响用户体验。 50 | 51 | ​ 被掩藏的细节：性能剖析无法显示所有响应时间的冯部，只相信平均值是非常危险的。正如医院内所有病人的平均体温一样毫无意义。 52 | 53 | ## 2.对应用系统进行性能剖析 54 | 55 | ​ 实际上，剖析应用程序一般比剖析数据库服务器更容易，且回报率更高。建议对系统进行自上而下的性能分析，这样可以追踪自用户发起到服务器响应的整个流程，虽然性能问题大多数情况下都和数据库有关，但是应用导致的问题也不少。 56 | 57 | ```shell 58 | # 应该尽可能地测量一切可以测量的地方，并且接受这些测量带来的额外开销。 59 | # Oracle 的性能优化大师Tom Kyte 曾被问到Oracle中的测量点开销，他的回答是，测量点至少为性能优化共享10% 60 | # 大多数应用并不需要每天都运行详尽的性能测量，所以实际上贡献至少超过10% 61 | ``` 62 | 63 | ## 3.剖析MySQL查询 64 | 65 | ### 3.1剖析服务器负载 66 | 67 | ​ MySQL的每一个新版本都增加了更多的可测量点。但是如果只是需要剖析并找出和代价高的查询，慢查询日志应该就能满足我们的需求。可以通过将"long_query_time"设为0来捕获所有的查询，并且查询的响应时间已经可以做到微秒级 。在当前的版本中，慢日志是开销最低，同时精度最高的测量查询时间的工具。如果长期开启慢查询日志，主要要配合logrotate工具一同使用（ 68 | 69 | [使用logrotate工具切割MySQL日志与慢日志分析发送到邮箱]: http://blog.51cto.com/l0vesql/2047599 70 | 71 | ）。Percona分支的MySQL比起官方社区版本记录了更多更有价值的信息。如查询计划，锁，I/O活动等。总的来说，慢日志是一种轻量且全面的性能剖析工具。 72 | 73 | ​ 可以使用pt-query-digest分析慢查询日志，如下图： 74 | 75 | ```shell 76 | pt-query-digest slow.log >slow_log_analyze.log 77 | /data/mysql/3109/slow.log: 53% 00:25 remain 78 | /data/mysql/3109/slow.log: 98% 00:00 remain 79 | 80 | cat slow_log_analyze.log 81 | # 75.3s user time, 2s system time, 41.28M rss, 235.76M vsz 82 | # Current date: Sun Feb 25 15:43:11 2018 83 | # Hostname: MySQL-Cent7-IP001109 84 | # Files: /data/mysql/3109/slow.log 85 | # Overall: 445.27k total, 59 unique, 0.03 QPS, 0.04x concurrency _________ 86 | # Time range: 2017-09-28T16:00:25 to 2018-02-25T07:27:18 87 | # Attribute total min max avg 95% stddev median 88 | # ============ ======= ======= ======= ======= ======= ======= ======= 89 | # Exec time 461284s 100ms 150s 1s 3s 1s 740ms 90 | # Lock time 1154s 0 10s 3ms 57us 83ms 21us 91 | # Rows sent 426.70M 0 9.54M 1004.84 97.36 76.26k 0.99 92 | # Rows examine 465.04M 0 9.54M 1.07k 299.03 76.26k 0.99 93 | # Query size 4.55G 6 1022.79k 10.71k 76.28 73.23k 36.69 94 | 95 | # Profile 96 | # Rank Query ID Response time Calls R/Call V/M Item 97 | # ==== ================== ================= ====== ======= ===== ========= 98 | # 1 0x558CAEF5F387E929 238431.3966 51.7% 294383 0.8099 0.62 SELECT sbtest? 99 | # 2 0x84D1DEE77FA8D4C3 53638.8398 11.6% 33446 1.6037 1.14 SELECT sbtest? 100 | # 3 0x3821AE1F716D5205 53362.1845 11.6% 33504 1.5927 1.11 SELECT sbtest? 101 | # 4 0x737F39F04B198EF6 53244.4816 11.5% 33378 1.5952 1.14 SELECT sbtest? 102 | # 5 0x6EEB1BFDCCF4EBCD 53036.2877 11.5% 33539 1.5813 1.10 SELECT sbtest? 103 | # 6 0x67A347A2812914DF 2619.2344 0.6% 200 13.0962 67.98 SELECT tpcc?.order_line 104 | # 7 0x28FC5B5D583E2DA6 2377.9580 0.5% 215 11.0603 11.53 SHOW GLOBAL STATUS 105 | # 10 0xE730A9F41A4AB139 259.9002 0.1% 355 0.7321 0.42 SHOW INNODB STATUS 106 | # 11 0x88901A51719CB50B 131.1035 0.0% 39 3.3616 21.74 SELECT information_schema.tables 107 | # 12 0x16F46891A99F2C89 127.1865 0.0% 88 1.4453 1.15 SELECT performance_schema.events_statements_history 108 | # 14 0x153F1CE7D660AE82 79.2867 0.0% 46 1.7236 1.47 SELECT information_schema.processlist 109 | # MISC 0xMISC 3976.0946 0.9% 16077 0.2473 0.0 <47 ITEMS> 110 | 111 | # Query 1: 0.17 QPS, 0.14x concurrency, ID 0x558CAEF5F387E929 at byte 4877477857 112 | # This item is included in the report because it matches --limit. 113 | # Scores: V/M = 0.62 114 | # Time range: 2018-02-03T11:26:24 to 2018-02-23T13:03:23 115 | # Attribute pct total min max avg 95% stddev median 116 | : 117 | 118 | ``` 119 | 120 | ​ 121 | 122 | ​ 除了慢日志之外Percona Toolkit工具包中的pt-query-digest工具也可以进行剖析，使用--processlist参数可以不断的分析"show processlist"的输出。但是“show processlist”的输出瞬息万变。即使每秒收集一次仍会遗漏很多有用的信息，因此并不十分推荐这种方式。另一种方式是使用--type=tcpdump选项对网络抓包数据进行分析。 123 | 124 | ### 3.2 剖析单条查询 125 | 126 | #### 使用SHOW PROFILE 127 | 128 | ​ 默认禁用，但它是会话级别的参数。`set profiling=1` ，然后在服务器上直送所有的语句，都会测量其耗费的时间和其他一些查询执行状态变更相关的数据。 129 | 130 | ​ 当一条查询提交给服务器时，此工具会记录剖析信息到一张临时表，并给查询赋一个从1开始的整数标识符。 131 | 132 | ​ 如： 133 | 134 | ```mysql 135 | set profiling=1 136 | select * from t_Order; 137 | select * from t_Product 138 | show profiles 139 | +----------+------------+-------------------------+ 140 | | Query_ID | Duration | Query | 141 | +----------+------------+-------------------------+ 142 | | 1 | 9.75e-05 | SHOW WARNINGS | 143 | | 2 | 0.00052075 | select * from t_order | 144 | | 3 | 0.000511 | select * from t_product | 145 | | 4 | 5.3e-05 | SHOW WARNINGS | 146 | +----------+------------+-------------------------+ 147 | show profile for query 3 148 | +----------------------+----------+ 149 | | Status | Duration | 150 | +----------------------+----------+ 151 | | starting | 0.000065 | 152 | | checking permissions | 0.000009 | 153 | | Opening tables | 0.000142 | 154 | | init | 0.000022 | 155 | | System lock | 0.000010 | 156 | | optimizing | 0.000008 | 157 | | statistics | 0.000013 | 158 | | preparing | 0.000012 | 159 | | executing | 0.000007 | 160 | | Sending data | 0.000154 | 161 | | end | 0.000010 | 162 | | query end | 0.000011 | 163 | | closing tables | 0.000010 | 164 | | freeing items | 0.000016 | 165 | | cleaning up | 0.000012 | 166 | +----------------------+----------+ 167 | ``` 168 | 169 | ​ 剖析报告给出了查询执行的每个步骤及其花费的时间，看结果很难快速地确定哪个步骤花费的时间最多。输出是按照执行顺序进行排序，而不是按照花费的时间排序的。下面给出使用INFORMATION_SHCEMA来查询剖析报告的办法： 170 | 171 | ```mysql 172 | set @query_id=1 173 | SELECT STATE,SUM(DURATION) AS Total_R, 174 | ROUND( 175 | 100*SUM(DURATION)/(SELECT SUM(DURATION) FROM INFORMATION_SCHEMA.PROFILING WHERE QUERY_ID = @query_id),2 176 | ) AS Pct_R,COUNT(*) AS Calls,SUM(DURATION)/COUNT(*) AS "R/Call" 177 | FROM INFORMATION_SCHEMA.PROFILING 178 | WHERE QUERY_ID=@query_id 179 | GROUP BY STATE 180 | ORDER BY Total_R DESC 181 | # 输出如下： 182 | +----------------------+----------+-------+-------+--------------+ 183 | | STATE | Total_R | Pct_R | Calls | R/Call | 184 | +----------------------+----------+-------+-------+--------------+ 185 | | starting | 0.000072 | 20.45 | 1 | 0.0000720000 | 186 | | Sending data | 0.000047 | 13.35 | 1 | 0.0000470000 | 187 | | init | 0.000030 | 8.52 | 1 | 0.0000300000 | 188 | | Opening tables | 0.000026 | 7.39 | 1 | 0.0000260000 | 189 | | checking permissions | 0.000025 | 7.10 | 1 | 0.0000250000 | 190 | | cleaning up | 0.000023 | 6.53 | 1 | 0.0000230000 | 191 | | System lock | 0.000019 | 5.40 | 1 | 0.0000190000 | 192 | | statistics | 0.000018 | 5.11 | 1 | 0.0000180000 | 193 | | preparing | 0.000016 | 4.55 | 1 | 0.0000160000 | 194 | | optimizing | 0.000015 | 4.26 | 1 | 0.0000150000 | 195 | | freeing items | 0.000014 | 3.98 | 1 | 0.0000140000 | 196 | | query end | 0.000013 | 3.69 | 1 | 0.0000130000 | 197 | | closing tables | 0.000012 | 3.41 | 1 | 0.0000120000 | 198 | | executing | 0.000011 | 3.13 | 1 | 0.0000110000 | 199 | | end | 0.000011 | 3.13 | 1 | 0.0000110000 | 200 | +----------------------+----------+-------+-------+--------------+ 201 | #通过这个结果可以很容易看到查询时间长主要是因为花了很大时间在sending data上 202 | #这个状态 代表的原因非常多，可能是各种不同的服务器活动，包括在关联时搜索匹配的行记录等，这部分很难说能优化节省多少消耗的时间。 203 | #若Sorting result花费的时间比较多，则可以考虑增大sort buffer size 204 | ``` 205 | 206 | #### 使用show status 207 | 208 | ​ MySQL的`show status` 命令返回了一些计数器，既有服务器级别的全局技术去，也有基于某个连接的会话级别的计数器。MySQL官方手册对与所有的变量是全局还是会话级别的做了详细的说明。 209 | 210 | ​ `show status`的大部分结果都是一个计数器，可以显示某些活动如读索引的频繁程度，但无法给出消耗了多少时间。`show status`的结果中只有一条Innodb_row_lock_time指的是操作时间，而且这个是全局性的，还是无法测量会话级别的工作。最有用的计数器包括句柄计数器，临时文件和表计数器等。将会话级别的计数器重置为0，然后查询前面提到的视图，再检查计数器的结果： 211 | 212 | ```mysql 213 | flush status; 214 | select * from sakila.nicer_but_slower_film_list; 215 | #............... 216 | show status where variable_name like "Handler%" or Variable_name like "Created%"; 217 | +----------------------------+-------+ 218 | | Variable_name | Value | 219 | +----------------------------+-------+ 220 | | Created_tmp_disk_tables | 2 | 221 | | Created_tmp_files | 2 | 222 | | Created_tmp_tables | 3 | 223 | | Handler_commit | 1 | 224 | | Handler_delete | 0 | 225 | | Handler_discover | 0 | 226 | | Handler_external_lock | 10 | 227 | | Handler_mrr_init | 0 | 228 | | Handler_prepare | 0 | 229 | | Handler_read_first | 3 | 230 | | Handler_read_key | 12942 | 231 | | Handler_read_last | 0 | 232 | | Handler_read_next | 6462 | 233 | | Handler_read_prev | 0 | 234 | | Handler_read_rnd | 5462 | 235 | | Handler_read_rnd_next | 6478 | 236 | | Handler_rollback | 0 | 237 | | Handler_savepoint | 0 | 238 | | Handler_savepoint_rollback | 0 | 239 | | Handler_update | 0 | 240 | | Handler_write | 0 | 241 | +----------------------------+-------+ 242 | ``` 243 | 244 | 从结果可以看到该查询使用了三个临时表，其中两个是磁盘临时表，并且有很多的没有用到索引的读操作（Handler_read_rnd_next）。假设我们不知道这个视图的具体定义，仅从结果来推测，这个查询可能做了多表关联查询，且没有合适的索引，可能是其中一个子查询创建了临时表，然后和其他表做联合查询，而用于保存子查询结果的临时表没有索引。 245 | 246 | ​ 但是，请注意，使用`show status`本身也会创建一个临时表，而且也会通过句柄操作访问此临时表，也会影响到`show status`结果中对应的数字，而且不同的版本可能行为也不尽相同，比较前面通过`show profiles`获得的查询的的执行计划结果来看，至少临时表的计数器多加了2。 247 | ​ 通过`explain`看到的查询执行计划也可以获得和`show status`大部分相同的信息，但是通过`explain`是估计得到的结果，而通过`show status`则是实际测量的结果。比如，`explain`无法告诉你临时表是否是磁盘表。 248 | 249 | #### 使用performance_schema和sys视图库 250 | 251 | ​ 在5.6中，引入了成熟的performance_schema视图库，在5.7时为了方便performance_schema的使用，引入了建立在performance_schema上面的sys库。通过sys库，我们可以方便的观测很多基础数据，同时，可以使用MySQL WorkBench来方便的查看。如图： 252 | 253 | 254 | 255 | -------------------------------------------------------------------------------- /性能优化/高性能MySQL 笔记/【MySQL】【持续更新】《高性能MySQL》 学习笔记,第二章.md: -------------------------------------------------------------------------------- 1 | # 第二章：MySQL基准测试 2 | 3 | ## 1.为什么要进行基准测试 4 | 5 | ​ 基准测试是唯一方便有效的可以观察系统在不同压力下的行为，评估系统的容量的方法 6 | 7 | ​ 在新系统正式上线到生产环境之前，进行基准测试是个好习惯。切勿相信云RDS提供商或者主机提供商的所为多快多稳定的说法。 8 | 9 | ​ 基准测试并不是基于真实压力的测试，其压力通常较为单调简单。基准测试在实际情况下通常会要求快速完成，压测实施者往往会施以实际中远远不能达到的且单调请求的压力，虽然有益于测量系统最大性能容量，但往往会使系统或者压测软件崩溃，得不到真正符合实际的数值 10 | 11 | ​ 真实测试是不可预期且变化多端的，有时情况会复杂且难以解释。 12 | 13 | ​ 当压测发现新系统可以支持原系统40倍的TPS时，不能简单的就认为新系统可以支持40倍的业务增长。因为系统是个复合的系统，除了单个业务的TPS还要考虑该业务与其他业务或者系统交互之间的交互，甚至网络和磁盘的因素都要考虑到其中。 14 | 15 | ​ 结论：我们只能进行大概的测试，得出系统的大致余量，基准测试要尽量简单直接，结果之间要容易相互比较，成本低且易于执行。可以使用控制变量法。 16 | 17 | ## 2.基准测试的策略 18 | 19 | ​ 集成式测试：测试整个系统，发现各部分之间缓存带来的影响。整体应用的集成式测试更能揭示应用的真是表现。一般测试组的同事会偏向于使用此策略。 20 | 21 | ​ 单组件测试：只关注MySQL的性能，进行比较不同的数据库或者查询的性能，针对应用中某个具体问题进行测试，通过一个短期的单组件基准测试，做快速的周期循环，检测某些调整后的效果。 22 | 23 | ### 测试目标 24 | 25 | ​ 有的放矢，测试之前一定要明确测试目的与目标。测试目标决定了选择什么样的测试工具和技术。 26 | 27 | ​ 吞吐量：单位时间事务处理数，指标为TPS（每秒事务数）。 28 | 29 | ​ 响应时间：任务所需的整体时间，通常为平均响应时间，最小/最大响应时间。但最大响应时间往往被百分比响应时间代替，例如，若95%的响应时间都是5毫秒，则表示任务在95%的时间段内都可以在5毫秒之内完成。 30 | 31 | ​ 并发性：Web服务器的并发性是指在任意时间有多少同时发生的并发请求，其高并发一般会导致数据库的高并发。但一个Web站点“同时有5W个用户”访问，却可能只有10-15个并发请求到MySQL数据库。并发性基准测试需要关注的是正在工作中的并发操作，或者同时工作中的线程或者连接数，当并发性增加时，需要关注并测量吞吐量是否随之下降，响应时间变长。并发性测试通常是为了测试在不同并发下的性能。 32 | 33 | ​ 可扩展性：可以理解为，给系统增加一倍的工作，在理想情况下就能获得两倍的结果，即吞吐量增加一倍。或者说给系统增加一倍的资源就能获得两倍的吞吐量。可扩展性指标可以帮助发现应用性能的平静。 34 | 35 | ​ 总结：确定测试对象为对于用户来说最重要的指标，然后收集应用方的需求，例如：什么样的响应时间是可以接受的，期待多少的并发性等等，最后基于这些需求进行设计基准测试。 36 | 37 | ### 测试方法 38 | 39 | ​ 测试时应避免如下的常见错误： 40 | 41 | ​ 使用部分的真实数据进行测试和扩展性有关的项目。 42 | 43 | ​ 错误的数据分布 44 | 45 | ​ 测试数据相对生产数据而言分布不均匀 46 | 47 | ​ 在多用户的场景中，却只测试单用户的情景。 48 | 49 | ​ 在单台服务器上测试分布式应用 50 | 51 | ​ 测试脚本或者软件模拟的用户行为不真实，不现实 52 | 53 | ​ 短时间内反复执行同一个查询，会因为查询缓存的存在导致结果不准确 54 | 55 | ​ 在一轮结果异常的测试结束后不分析异常的原因就直接进入下一轮 56 | 57 | ​ 不进行预热就直接进行测试 58 | 59 | ​ 未使用合理的数据库参数 60 | 61 | ​ 测试时间太短 62 | 63 | #### 设计和规划基准测试 64 | 65 | ​ 第一步先提出问题并明确目标，然后觉得采用标准的基准测试还是设计特定情景的测试 66 | 67 | ​ 基准测试软件：TPCH,即席查询和决策支持型应用的基准测试。即OLAP 68 | 69 | ​ TPCC,订单型业务交易系统应用的基准测试。即OLTP 70 | 71 | ​ LinkBench，人际关系型应用的基准测试。等等 72 | 73 | ​ 特定情境测试：首先获得生产数据集的快照 74 | 75 | ​ 然后只针对数据进行查询。 76 | 77 | ​ 提前做好测试规划：应包括准备测试数据，系统配置的步骤，设计预热方案，测量并分析测试结果 78 | 79 | #### 基准测试应运行足够长的时间 80 | 81 | ​ 大部分系统都会有一些应对突发情况的余量，能够收集性能尖峰，将一些工作延迟到高峰期之后执行，但当对机器加压足够长的时间后，这些余量会被消耗殆尽，系统的短期高峰也就无法维持原来的高性能。至少应当让测试一直运行到确认系统已经稳定运行之后。其中，系统看起来稳定的时间至少大于等于系统预热所需要的时间。 82 | 83 | ​ 如果没有时间去完成准确完整的基准测试，那么已经花费的所有时间都是一种浪费。 84 | 85 | #### 获取系统性能和状态 86 | 87 | ​ 执行基准测试时，需要尽可能多的收集被测试系统的信息，最好为测试建立一个专用的目录，每次执行一轮测试都创建对应的独立子目录，将测试结果，配置文件，测试指标，脚本和其他相关说明都保存在其中。 88 | 89 | ​ 需要着重考虑的系统状态和指标：CPU使用率，磁盘IO，网络流量统计，数据库status杂项等。 90 | 91 | ​ 下面给出一个数据收集脚本范例： 92 | 93 | ```shell 94 | #!/bin/sh 95 | 96 | INTERVAL=5 97 | PREFIX=$INTERVAL-sec-status 98 | RUNFILE=/root/running 99 | mysql -e 'show global variables'>>mysql-variables 100 | while test -e $RUNFILE; do 101 | file=$(date +%F_%H) 102 | sleep=$(date +%s.%N |awk "{print $INTERVAL -(\$1 % $INTERVAL)}") 103 | sleep $sleep 104 | ts="$(date +"TS %s.%N %F %T")" 105 | loadavg="$(uptime)" 106 | echo "$ts $loadavg">> $PREFIX-${file}-status 107 | mysql -e "show global status" >> $PREFIX-${file}-status & 108 | echo "$ts $loadavg">> $PREFIX-${file}-innodbstatus 109 | mysql -e "show engine innodb status\G" >> $PREFIX-${file}-innodbstatus & 110 | echo "$ts $loadavg">> $PREFIX-${file}-processlist 111 | mysql -e "show full processlist\G" >>$PREFIX-${file}-processlist & 112 | echo $ts 113 | done 114 | echo Exiting because $RUNFILE not exist 115 | 116 | ``` 117 | 118 | #### 获取准确的测试结果 119 | 120 | ​ 第一步：审查如下项目 121 | 122 | ​ 是否选择了正确的基准测试类型？ 123 | 124 | ​ 是否为问题收集了相关的线上数据？ 125 | 126 | ​ 是否采用了正确的测试标准？ 127 | 128 | ​ 第二步：确认测试结果是否有可重复性。每次重新测试之前都要确保系统的状态是一致的，甚至每次测试之前都重启系统。还要确保每次预热的时间足够长。 129 | 130 | ​ 如果预热测试用的是随机查询，那么测试结果可能就不是可重复的。如果测试的过程中会修改数据或者DDL，那么每次测试之前，需要利用快照还原数据。 131 | 132 | ​ 可以利用控制变量法，每次测试中，修改的参数应该尽量少。一般情况下，都是通过迭代逐步地修改基准测试的参数。 133 | 134 | ​ 如果在测试中出现异常结果，不要轻易当作垃圾结果直接丢弃，应当认真研究并找到产生这种结果的原因。如果对测试结果不了解，就不要轻易公布。 135 | 136 | #### 运行测试并分析结果 137 | 138 | ​ 基准测试通常需要运行多次，具体运行多少次要看对结果的计分方式，以及测试结果的重要程度。若需要提高测试的准确度就需要多运行几次。一般在测试的实践中，可以取最好的结果值，或者所有结果的平均值，抑或是从五个测试结果里去最好三个值的平均值。只要测试的结果能满足目前的需求，简单地运行几轮测试，看看结果的变化就可以了。若结果变动很大，可以再多运行几次，或者运行更长的时间，以求获得更确定的结果。 139 | 140 | ​ 在运行完测试，获得测试结果后，要把测试结果的数字，通过分析变成知识或者经验，并回答在设计测试时提出的问题或者目标。可以通过编写脚本来抽象分析测试结果中的数据。这里给出一个针对前面采集脚本的一个分析脚本。 141 | 142 | ```shell 143 | #!/bin/sh 144 | awk ' 145 | BEGIN{ 146 | printf "#ts date time load QPS"; 147 | fmt = " %.2f"; 148 | } 149 | /^TS/ { # The timestamp lines begin with TS. 150 | ts = substr($2, 1, index($2,".") - 1); 151 | load = NF - 2; 152 | diff = ts -prev_ts; 153 | prev_ts = ts; 154 | printf "\n%s %s %s %s",ts,$3,$4,substr($load, 1, length($load)-1); 155 | } 156 | /Queries/ { 157 | printf fmt, ($2-Queries)/diff; 158 | Queries=$2 159 | } 160 | ' "$@" 161 | 162 | ``` 163 | 164 | ​ 运行方式：`sh hi_anaylyze.sh 5-sec-status-2018-02-22_14_status >>4plot.log` （将分析后的结果记入4plot.log中） 165 | 166 | ​ 167 | 168 | #### 绘图 169 | 170 | ​ 绘图有很多种方式，EXCEL,rrdtools,gnuplot都可以。 171 | 172 | ​ 这里介绍gnuplot的绘图方式。 173 | 174 | ​ `gunplot>plot '4plot' using 5 with lines title 'QPS', 4 with lines title 'load' ` 175 | 176 | ​ 解析：using 5 表示使用第5列数据作图 177 | 178 | ​ with lines 定义图中的趋势使用线来表示 179 | 180 | ​ title 'QPS' 定义线的名称 181 | 182 | ​ 使用,（逗号）分割，进行多列数据的绘制 183 | 184 | ​ 如图：![gnuplot绘图示例](https://github.com/naughtyGitCat/ToolsForMySQL/raw/master/压测/gnuplot绘图示例.jpg) 185 | 186 | ### 测试工具 187 | 188 | ​ 常用集成式测试工具： 189 | 190 | ​ ab 191 | 192 | ​ http_load 193 | 194 | ​ Jmeter 195 | 196 | ​ 常用单组件测试工具： 197 | 198 | ​ mysqlslap：包含在MySQL的安装包中 199 | 200 | ​ sql-bench：包含在MySQL的安装包中（5.6及以前） 201 | 202 | ​ TPCC,percona`s TPCC：针对订单交易型 203 | 204 | ​ sysbench：全能的多线程系统压测工具，可以配合Lua脚本进行多样化的系统软硬件测试。 205 | 206 | ## 3.压力测试测试实操 207 | 208 | [tpcc-mysql]: http://imysql.cn/2014/10/10/tpcc-mysql-full-user-manual.shtml 209 | [sysbench]: http://blog.51cto.com/l0vesql/2072412 210 | 211 | ​ -------------------------------------------------------------------------------- /性能优化/高性能MySQL 笔记/【MySQL】【持续更新】《高性能MySQL》 学习笔记.md: -------------------------------------------------------------------------------- 1 | 【MySQL】【持续更新】《高性能MySQL》 学习笔记 2 | 3 | # 第一章：MySQL架构和历史 4 | 5 | ## 1.处理和存储分离的设计 6 | 7 | ​ MySQL最重要，最与众不同的特性是它的可插拔式存储引擎架构（将查询处理，系统任务，数据的存储，提取相分离）。 8 | 9 | ## 2.架构： 10 | 11 | | 层级 | 作用 | 备注 | 12 | | ---- | ----------------------- | ---------------------------------------- | 13 | | 连接层 | 连接处理，授权认证等 | RDMS共有设计 | 14 | | 服务层 | 查询解析，缓存，分析优化，内置函数，跨引擎功能 | MySQL核心服务与功能，服务层未设计外键 | 15 | | 引擎层 | 负责数据存储与提取 | 底层函数，不会解析SQL，不同引擎直接不在此层级互相通信，仅响应来自上层的请求。处理外键 | 16 | 17 | ## 3.优化与执行 18 | 19 | ​ 服务层解析查询，创建解析树，然后进行重写查询，决定表的读取顺序，选择合适的索引等。在此步，用户可以通过hint关键字或者force index影响优化器的决策，也可以通过explain命令查看优化器如何优化决策。 20 | 21 | ​ 优化器不关心表使用什么引擎，但存储引擎对优化查询是有影响的。 22 | 23 | ## 4.锁 24 | 25 | ​ 读锁是共享的，相互不阻塞。 26 | 27 | ​ 写锁是排他的，相互阻塞，在给定的时间内，同一行数据，只能有一个用户正在进行写入。 28 | 29 | ​ 大多数时候，MySQL锁的内部管理都是透明的。 30 | 31 | ​ 在给定的资源上，锁定的数据量越少，则系统的并发读越高。 32 | 33 | ​ 加锁需要消耗资源，如果系统花费大量时间来管理锁，而不是存取数据，则系统的性能可能会因此收到影响。 34 | 35 | ​ 一般会在锁的开销和数据的安全性之间寻求平衡，即在表上施加行级锁。 36 | 37 | ​ 每种MySQL引擎都可以实现自己的锁倾向和锁粒度。 38 | 39 | ​ 写锁比读锁有更高的优先级，并发时，写锁请求可能会被插入到队列的最前端，但读锁最多只能排在其他读锁的最前端。 40 | 41 | ​ 服务层会对涉及到整个表的内容的更改使用表级锁，引擎层的锁机制被忽略。 42 | 43 | ​ 行级锁只在存储引擎层实现。 44 | 45 | ​ InnoDB采用两段锁定协议，在事务执行过程中，随时都可以执行锁定，但只有在整个事务提交或者回滚时才会同一时间释放该事务占有的所有锁。‘隐式锁’ 46 | 47 | ​ 服务层可以使用LOCK TABLE或者UNLOCK TABLE，但可能会和事务产生相互影响产生不可预料的后果，尽量不要在业务进行时中使用。 48 | 49 | 50 | 51 | ## 5.事务 52 | 53 | ​ 事务就是一组原子性的SQL查询，一组独立的工作单元。事务内的语句要么全部执行完，要么全部失败。 54 | 55 | ​ 一个完备的数据库系统需要满足ACID特征： 56 | 57 | ​ Atomictiy 原子性：一个事务视为不可分割的最小单元。 58 | 59 | ​ Consistency 一致性: 数据库总是从一个一致性的状态转换到另一个一致性的状态。 60 | 61 | Isolation 隔离性: 一个事务所做的修改在最终提交之前，对其事务是不可见的。 62 | 63 | ​ Durability 持久性：一旦提交，永久保存，不受系统崩溃影响。 64 | 65 | ​ 一个实现了ACID特性的数据库，需要更强的硬件。 66 | 67 | ​ 即使存储引擎不支持事务，但还可以通过lock table来提供部分ACID特性。 68 | 69 | ​ 非事务型的表只能自动提交。 70 | 71 | ​ 事务型的表在执行DDL操作或者lock table时会强行commit当前的活动事务 72 | 73 | ### 隔离级别（ANSI） 74 | 75 | ​ READ UNCOMMITED(未提交读)：事务中未提交的修改也会被其他事务读到，’脏读‘，性能不会好太多。 76 | 77 | ​ READ COMMITED(提交读)：事务中提交后的修改会被其他事务读到,但会造成不可重复读。MSSQL,ORACLE 78 | 79 | ​ READ REPEATABLE(可重复读):事务中多次读取同一条数据值相同，但新插入的不算，即主键范围读可能不一致，'幻读',MySQL默认级别。 80 | 81 | ​ SEARIALIZE(串行)：取消并行，完全串行，悲观锁。 82 | 83 | ​ 可以通过`set [session] transaction isolation level [RU|RC|RR|SX]`设置隔离级别，全局性设置在下一个事务开始时生效，会话级设置只对当前事务有效 84 | 85 | ​ 86 | 87 | | 隔离级别 | 脏读可能性 | 不可重复读 | 幻读可能性 | 加锁读 | 88 | | ---- | ----- | ----- | ----- | ---- | 89 | | RU | Y | Y | Y | N | 90 | | RC | N | Y | Y | N | 91 | | RR | N | N | Y | N | 92 | | SX | N | N | N | Y | 93 | 94 | ### 死锁 95 | 96 | ​ 两个或以上的事务争用同一组资源，并请求锁定对方占用的资源。 97 | 98 | ​ 处理办法：完备的RDMS包含了死锁检测与死锁超时机制。 99 | 100 | ​ InnoDB:将持有最少行级X锁的事务进行回滚。 101 | 102 | ​ 死锁在事务型RDMS中是无法避免的，死锁发生后，只有部分或者完全回滚其中一个事务才能打破窘境。 103 | 104 | ### 事务日志（redo） 105 | 106 | ​ 可以提高事务的效率，存储引擎在修改表的数据时只需要修改其内存拷贝，再批量地把修改的行为记录到硬盘上的事务日志文件中。 107 | 108 | ​ 事务日志采用了正向追加的方式，保证了写入时的 顺序IO。批量记录修改的行为时，事务日志持久后，可以在后台将内存中的数据慢慢刷回磁盘 109 | 110 | ## 6.多版本并发控制（MVCC） 111 | 112 | ​ MVCC是行级锁的一个变种，在很多情况下避免了加锁的操作，且只在RR和RC隔离级别下工作。 113 | 114 | ​ MVCC是通过保存数据行在某个时间点的快照来实现的。 115 | 116 | ​ 根据事务开始时间的不同，每个事务对同一张表，同一时刻看到的数据可能是不一样的。 117 | 118 | ​ InnoDB的MVCC，是通过在每个记录（包括已经存入undo中的记录）后面保存两个隐藏的版本号来实现的。即：该数据创建时的系统版本号和该数据失效时的系统版本号。 119 | 120 | ​ 其他事务： 121 | 122 | ​ SELECT时：只查找比当前事务创建更早的行（极端情况下，在此事务中对此行进行了修改或创建，即读取本事务中被修改后此行的数据）。 123 | 124 | ​ INSERT时：为新插入的此行保存本事务版本的版本号作为此行的创建版本号。同时若在此事务中不在对此行进行修改，则行过期版本号留空 125 | 126 | ​ DELETE时：将本事务的版本号赋给被删除行的失效版本号 127 | 128 | ​ UPDATE时：将本事务的版本号赋给被修改行的失效版本号，同时插入被修改后的数据，同时把本事务的版本号付给新插入的修改后的数据的创建版本号。 129 | 130 | ​ 保留这两个额外版本号，可以使大多数读操作都不用加S锁，但这些多余的行会占用undo空间。建议将undo从共享表空间中独立出来，并减小事务长度。 131 | 132 | ## 7.存储引擎 133 | 134 | ​ 不同引擎保存数据和索引的方式是不同的，但表的定义是在服务层统一处理的。 135 | 136 | ​ 通过show table status显示表的相关信息 137 | 138 | ​ 139 | 140 | | 属性 | 属性值 | 说明 | 141 | | --------------- | -------------------- | ------------------ | 142 | | Name: | user | 表名 | 143 | | Engine: | InnoDB | 存储引擎 | 144 | | Row_format: | Dynamic | 行格式，变长行 | 145 | | Rows | 3 | 对应InnoDB，估计的行数值 | 146 | | Avg_row_length | 5461 | 平均每行字节数 | 147 | | Data_length | 16384 | 表数据字节数 | 148 | | Max_data_length | 0 | 表最大字节数（InnoDB无限制） | 149 | | Index_length | 0 | 非主键索引占用字节数 | 150 | | Data_free | 4194304 | 已分配但未使用的字节数 | 151 | | Auto_increment | NULL | 下个自增值的起始点 | 152 | | Create_time | 2018-01-24 20:02:01 | 创建时间 | 153 | | Update_time | NULL | 最后一次的更新时间 | 154 | | Check_time | NULL | CHECK TABLE命令使用的时间 | 155 | | Collation | utf8_bin | 该表默认字符集与排序规则 | 156 | | Checksum | NULL | （若启用校验）校验和 | 157 | | Create_options | stats_persistent=0 | 建表时的其他非默认选项 | 158 | | Comment | Users and privileges | 表的备注 | 159 | 160 | ​ InnoDB引擎被设计用来处理大量短期事务（大部分情况下正常提交），正常情况下默认使用InnoDB引擎，MySQL8.0版本，将mysql库中的表也从MyISAM更换为InnoDB引擎。 161 | 162 | ​ InnoDB引擎一直朝着可测量性，可扩展性，可配置化，性能，更多新特性的方向演进。 163 | 164 | ### InnoDB引擎概览 165 | 166 | ​ InnoDB的数据存储在表空间中，推荐使用独立表空间，即将各个表的数据和索引放在单独的文件中。 167 | 168 | ​ InnoDB使用MVCC来支持高并发，并且实现了四个标准的隔离级别。默认隔离级别为可重复读RR，并且通过间隙锁的机制解决了范围读可能因为新插入值导致出现幻读的问题。 169 | 170 | ​ 间隙锁通过在本来锁定查询设计的行同事，还会对索引中的间隙进行锁定，防止插入新的行。 171 | 172 | ​ InnoDB表是基于聚簇索引建立的，即索引组织表。这种设计，使得对主键的进行的查询效率很高。其二级索引，即非聚集索引（普通索引，非主键索引）是通过链接的方式指向其对应的主键位置，即二级索引中包含了主键列。这就会产生一个主键列长度很大，其他索引的大小也会同样很大的问题。这个特性要求我们主键列的最大长度尽量减小。 173 | 174 | ​ InnoDB引擎表的文件存储格式是独立的，某种程度上可以跨平台使用。 175 | 176 | ​ InnoDB的显著优化特点：从磁盘读取数据时的可预测性读，引入了能够在内存中创建Hash索引来加速读操作的自适应哈希索引（adaptive hash index），能够加速插入操作的插入缓冲区(insert buffer，对非主键非唯一性索引进行缓存，每10s合并到非主键索引中)。 177 | 178 | ​ InnoDB引擎的表支持热物理备份（非逻辑备份成sql文件）即XtraBackup或者官方的Enterprise Backup 179 | 180 | ### 有趣的CSV引擎 181 | 182 | ​ 可以将Excel文件另存为CSV格式，放入MySQL数据目录下，即可在MySQL中打开使用。 183 | 184 | ### 慢慢弃用的引擎： 185 | 186 | ​ Federated引擎，本来设计用于建立Oracle，SQL server到MySQL的数据联系纽带，后被用于创建跨实例连接（类似于SQL server中的同义词或者链接服务器），但经常带来问题，默认禁用，MariaDB提供了一个改进版的FederatedX。 187 | 188 | ​ Memory引擎，数据只存在内存中，重启后表结构留存，但数据会全部丢失。支持HASH索引，表级锁，并发低，行长度固定。 189 | 190 | ### 第三方常用存储引擎： 191 | 192 | ​ XtraDB引擎，Percona公司基于InnoDB引擎的改进版本 193 | 194 | ​ TokuDB引擎，基于分形树索引的引擎，具有较高的压缩比，可以在很大的数据量上创建索引。 195 | 196 | ​ Infobright引擎，列组织的引擎，面向大数据设计。 197 | 198 | ​ 199 | 200 | ​ 更改表的存储引擎：`alter table t1 engine = InnoDB;` 执行过程中会创建相同表结构不同引擎的一张新表，然后按行将数据从原表读入到新表中，会进行锁表，消耗大量系统IO，尽量不要在业务高峰期进行，或者使用pt-online-schema-change的工具进行在线修改。 -------------------------------------------------------------------------------- /性能优化/高性能MySQL 笔记/配图/第三章/sys_workbench.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/naughtyGitCat/MySQLMiscellaneous/46f040da900c0883d4e5b475b1f0c4d030fc7929/性能优化/高性能MySQL 笔记/配图/第三章/sys_workbench.png -------------------------------------------------------------------------------- /技术文章翻译/【MySQL】【翻译】 MySQL 5.7 的内部临时表新特性 .md: -------------------------------------------------------------------------------- 1 | # 【MySQL】【翻译】MySQL Internal Temporary Tables in MySQL 5.7（MySQL 5.7 内部临时表） 2 | 3 | [Alexander Rubin](https://www.percona.com/blog/author/alexanderrubin/) | December 4, 2017 | Posted In: [Insight for DBAs](https://www.percona.com/blog/category/dba-insight/), [MySQL](https://www.percona.com/blog/category/mysql/), [Percona Monitoring and Management](https://www.percona.com/blog/category/percona-monitoring-and-management/) 4 | 5 | 翻译:A529 张锐志 [博文地址](http://blog.51cto.com/l0vesql) 6 | 7 | In this blog post, I investigate a case of spiking InnoDB Rows inserted in the absence of a write query, and find internal temporary tables to be the culprit. 8 | 9 | 本文中研究了在没有写查询的情况下，InnoDB行插入却因内部临时表的问题发生性能尖刺的情形。 10 | 11 | Recently I was investigating an interesting case for a customer. We could see the regular spikes on a graph depicting “InnoDB rows inserted” metric (jumping from 1K/sec to 6K/sec), however we were not able to correlate those spikes with other activity. The innodb_row_inserted graph (picture from [PMM demo](http://pmmdemo.percona.com/)) looked similar to this (but on a much larger scale): 12 | 13 | 事情发生在我研究一个客户的案例时，在”InnoDB行插入“指标图上，发现了从1k行每秒激增到6K行每秒的尖刺，但却无法和其他活动或者现象连接起来，PMM监控图形上也有同样的反映。 14 | 15 | ![InnoDB row operations graph from PMM](https://www.percona.com/blog/wp-content/uploads/2017/08/Screen-Shot-2017-08-28-at-3.09.12-PM-1024x376.png) 16 | 17 | Other graphs (Com_*, Handler_*) did not show any spikes like that. I’ve examined the logs (we were not able to enable general log or change the threshold of the slow log), performance_schema, triggers, stored procedures, prepared statements and even reviewed the binary logs. However, I was not able to find any single **\*write*** query which could have caused the spike to 6K rows inserted. 18 | 19 | 其他例如句柄和接口的图形都没有显示同样的尖刺，在无法开启general log的情况下，我们尝试检查了所有的日志，performance_schema，触发器，存储过程，预编译语句，甚至包括binlog后发现没有任何单个的写查询语句可以导致每秒插入飙升到6K行。 20 | 21 | Finally, I figured out that I was focusing on the wrong queries. I was trying to correlate the spikes on the InnoDB Rows inserted graph to the DML queries (writes). However, the spike was caused by SELECT queries! But why would SELECT queries cause the massive InnoDB insert operation? How is this even possible? 22 | 23 | 在最后才发现，行插入飙升一定和DML有关的这种想法是错误的，出乎意料的是，尖刺是由于SELECT查询导致的，但为何SELECT查询会导致大量的InnoDB行插入操作呢？ 24 | 25 | It turned out that this is related to temporary tables on disk. In MySQL 5.7 the default setting for [internal_tmp_disk_storage_engine](http://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_internal_tmp_disk_storage_engine) is set for InnoDB. That means that if the SELECT needs to create a temporary table on disk (e.g., for GROUP BY) it will use the InnoDB storage engine. 26 | 27 | 原来是与磁盘临时表有关。在MySQL 5.7版本中，内部磁盘临时表的默认引擎是InnoDB引擎，这就意味着当SELECT操作需要在磁盘上创建临时表时（例如GROUP BY操作），就会使用到InnoDB引擎。 28 | 29 | Is that bad? Not necessarily. Krunal Bauskar published a blog post originally about the [InnoDB Intrinsic Tables performance](http://mysqlserverteam.com/mysql-5-7-innodb-intrinsic-tables/) in MySQL 5.7. The InnoDB internal temporary tables are not redo/undo logged. So in general performance is better. However, here is what we need to watch out for: 30 | 31 | 但这种尖刺就一定意味着性能的下降吗？Krunal Bauskar曾经写过一篇关于[5.7 InnoDB原生表性能](http://mysqlserverteam.com/mysql-5-7-innodb-intrinsic-tables/)的文章,InnoDB的内部临时表的操作并不会记录在redo和undo中，一般情况下相比原本MyISAM引擎的临时表性能更好点，但是仍需注意一下几点： 32 | 33 | 1. Change of the place where MySQL stores temporary tables. InnoDB temporary tables are stored in ibtmp1 tablespace file. There are a number of challenges with that: 34 | 35 | 更改MySQL存储临时表的位置，原本InnoDB临时表被存储在ibtmp1表空间中，可能遇到以下的问题： 36 | 37 | - Location of the ibtmp1 file. By default it is located[ inside the innodb datadir](https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_temp_data_file_path). Originally MyISAM temporary tables were stored in tmpdir. We can configure the size of the file, but the location is always relative to InnoDB datadir, so to move it to tmpdir we need something like this: innodb_temp_data_file_path=../../../tmp/ibtmp1:12M:autoextend 38 | 39 | ibtmp1文件默认保存在InnoDB的数据目录，原本MyISAM临时表被放在MySQL的tmp目录，如若像MyISAM一样把临时表文件存储在MySQL的tmp目录，需要更改为`innodb_temp_data_file_path=../../../tmp/ibtmp1:12M:autoextend` 40 | 41 | - Like other tablespaces it never shrinks back (though it is truncated on restart). The huge temporary table can fill the disk and hang MySQL ([bug opened](https://bugs.mysql.com/bug.php?id=82556)). One way to fix that is to set the maximum size of ibtmp1 file: innodb_temp_data_file_path=ibtmp1:12M:autoextend:max:1G 42 | 43 | 临时表空间和其他的表空间一样都不会自动缩小其占用容量，可能会发生临时表空间容量占满磁盘，MySQL挂掉的情况，可以通过控制其最大的容量来解决：` innodb_temp_data_file_path=ibtmp1:12M:autoextend:max:1G` 44 | 45 | - Like other InnoDB tables it has all the InnoDB limitations, i.e., InnoDB row or column limits. If it exceeds these, it will return [“Row size too large” or “Too many columns” errors](https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_internal_tmp_disk_storage_engine). The workaround is to set internal_tmp_disk_storage_engine to MYISAM. 46 | 47 | 内部临时InnoDB表同样共享常规的InnoDB表的限制，如行或列的最大数量限制，超过最大值后，会返回Row size too large” or “Too many columns”的错误，遇到此种情况，可以将默认临时表引擎改回MyISAM 48 | 49 | 2. When all temp tables go to InnoDB, it may increase the total engine load as well as affect other queries. For example, if originally all datasets fit into buffer_pool and temporary tables were created outside of the InnoDB, it will not affect the**\* InnoDB*** memory footprint. Now, if a huge temporary table is created as an InnoDB table it will use innodb_buffer_pool and may “evict” the existing pages so that other queries may perform slower. 50 | 51 | 当所有的临时表都改成InnoDB引擎后，会增加引擎的负载，影响到其他的查询。例如：当所有的表都放入buffer_pool中，且临时表都不是InnoDB引擎，那么不会对InnoDB的内存占用造成任何影响，但是临时表改成InnoDB引擎后，会和普通InnoDB表一样占用InnoDB_buffer_pool的空间，而且可能因为临时表空间占用过大挤出真正的热数据，让某些高频查询变慢 52 | 53 | #### Conclusion 结论 54 | 55 | Beware of the new change in MySQL 5.7, the internal temporary tables (those that are created for selects when a temporary table is needed) are stored in InnoDB ibtmp file. In most cases this is faster. However, it can change the original behavior. If needed, you can switch the creation of internal temp tables back to MyISAM: set globalinternal_tmp_disk_storage_engine=MYISAM 56 | 57 | 内部InnoDB临时表（可能仅仅因为是SELECT查询导致）被保存在InnoDB的ibtmp文件中，在大部分情况下，会加速临时表或者查询的速度，但是会影响到原本InnoDB内存的占用情况和原本临时表处理的逻辑，如果在某种情况确实需要规避的话，可以尝试将临时表的引擎改回MyISAM。`set global internal_tmp_disk_storage_engine=MYISAM` 。这个案例要求我们要对MySQL 5.7的特性要有所注意和了解。 -------------------------------------------------------------------------------- /技术文章翻译/【MySQL】【翻译】8.0 GA版本的新特性 What’s New in MySQL 8.0 (Generally Available).md: -------------------------------------------------------------------------------- 1 | # 嗦一嗦 MySQL 8.0的新特性 What’s New in MySQL 8.0? (Generally Available) 2 | 3 | 原文链接：https://mysqlserverteam.com/whats-new-in-mysql-8-0-generally-available 4 | 5 | 原文作者：Geir Hoydalsvik （Oracle官方开发工程师） 6 | 7 | 翻译：张锐志 8 | 9 | [TOC] 10 | 11 | 非常高兴的向大家宣布MySQL 8.0 GA版本发布，MySQL 8.0是一个得到全面增强且极具吸引力的新版本。不限于下面几点： 12 | 13 | We proudly announce General Availability of MySQL 8.0. [Download now!](http://dev.mysql.com/downloads/mysql/) MySQL 8.0 is an extremely exciting new version of the world’s most popular open source database with improvements across the board. Some key enhancements include: 14 | 15 | ​     **SQL方面**：窗口函数，公共表达式，NOWAIT,SKIPLOCKED,降序索引，分组，正则表达式，字符集，基于性能损耗的优化模式，直方图 16 | 17 | 1. **SQL** Window functions, Common Table Expressions, NOWAIT and SKIP LOCKED, Descending Indexes, Grouping, Regular Expressions, Character Sets, Cost Model, and Histograms. 18 | 19 | **对JSON的支持**：扩充语法，新功能，增强排序，部分更新性能，基于JSON表的特性，可以使用SQL处理工具处理JSON数据。 20 | 21 | 2. **JSON** Extended syntax, new functions, improved sorting, and partial updates. With JSON table functions you can use the SQL machinery for JSON data. 22 | 23 | **对地理信息系统的支持** 24 | 25 | 3. **GIS** Geography support. Spatial Reference Systems (SRS), as well as SRS aware spatial datatypes, spatial indexes, and spatial functions. 26 | 27 | **可靠性**：DDL语句现在实现原子性和故障恢复（元信息数据被存在了一个基于InnoDB的单独事务性数据字典中）。 28 | 29 | 4. **Reliability** DDL statements have become atomic and crash safe, meta-data is stored in a single, transactional data dictionary. Powered by InnoDB! 30 | 31 | **可观察性**：对P_S,I_S,配置参数，错误日志的记录有了极其重要的增强 32 | 33 | 5. **Observability** Significant enhancements to Performance Schema, Information Schema, Configuration Variables, and Error Logging. 34 | 35 | **可管理性**：远程管理，Undo表空间管理，快速DDL 36 | 37 | 6. **Manageability** Remote management, Undo tablespace management, and new instant DDL. 38 | 39 | **安全性**：OpenSSL的改进，新的默认验证方式，SQL角色权限，分解super权限，密码强度等等 40 | 41 | 7. **Security** OpenSSL improvements, new default authentication, SQL Roles, breaking up the super privilege, password strength, and more. 42 | 43 | **性能**：InnoDB在读写，带宽限制，业务热数据集中的场景上上有着举足轻重的优点，新增的资源组特性额外给用户一个在特定负载和特定硬件情况下将用户线程映射到指定的CPU上的调节选项 44 | 45 | 8. **Performance** InnoDB is significantly better at Read/Write workloads, IO bound workloads, and high contention “hot spot” workloads. Added Resource Group feature to give users an option optimize for specific workloads on specific hardware by mapping user threads to CPUs. 46 | 47 | 以上是8.0版本的部分亮点，我（原文作者）推荐您仔细阅读GA版本前几个版本的发布信息，甚至这些特性和实现方法的的项目日志。或者您可以选择直接在Github上阅读源码。 48 | 49 | The above represents some of the highlights and I encourage you to further drill into the complete series of Milestone blog posts—[8.0.0](http://mysqlserverteam.com/the-mysql-8-0-0-milestone-release-is-available/), [8.0.1](http://mysqlserverteam.com/the-mysql-8-0-1-milestone-release-is-available/), [8.0.2](http://mysqlserverteam.com/the-mysql-8-0-2-milestone-release-is-available/), [8.0.3](https://mysqlserverteam.com/the-mysql-8-0-3-release-candidate-is-available/), and [8.0.4](https://mysqlserverteam.com/the-mysql-8-0-4-release-candidate-is-available/) —and even further down in to the individual [worklogs](http://dev.mysql.com/worklog/) with their specifications and implementation details. Or perhaps you prefer to just look at the source code at [github.com/mysql](https://github.com/mysql). 50 | 51 | ## 面向开发人员的特性 **Developer features** 52 | 53 | MySQL 8.0应面向MySQL开发人员的需求，带来了SQL，JSON，公共表达式，地理信息系统等方面的特性，因为很多开发人员有存储EmoJi表情的需求，在新版本中UTF8MB4成为默认的字符集。除此之外，还有对Binary数据类型按位操作，和改进对IPV6和UUID函数的改进。 54 | 55 | MySQL Developers want new features and MySQL 8.0 delivers many new and much requested features in areas such as SQL, JSON, Regular Expressions, and GIS. Developers also want to be able to store Emojis, thus UTF8MB4 is now the default character set in 8.0. Finally there are improvements in Datatypes, with bit-wise operations on BINARY datatypes and improved IPv6 and UUID functions. 56 | 57 | ### SQL 58 | 59 | ###### 窗口函数 **Window Functions** 60 | 61 | MySQL 8.0带来了标准SQL的窗口函数功能，窗口函数与分组聚合函数相类似的是都提供了对一组行数据的统计计算。但与分组聚合函数将多行合并成一行不同是窗口函数会在结果结果集中展现每一行的聚合。 62 | 63 | MySQL 8.0 delivers SQL window functions. Similar to grouped aggregate functions, window functions perform some calculation on a set of rows, e.g. `COUNT` or `SUM`. But where a grouped aggregate collapses this set of rows into a single row, a window function will perform the aggregation for each row in the result set. 64 | 65 | 窗口函数有两种使用方式，首先是常规的SQL聚合功能函数和特殊的窗口函数。常规的聚合功能函数如：`COUNT`,`SUM`等函数。而窗口函数专有的则是`RANK`, `DENSE_RANK`, `PERCENT_RANK`, `CUME_DIST`, `NTILE`, `ROW_NUMBER`, `FIRST_VALUE`, `LAST_VALUE`, `NTH_VALUE`, `LEAD `and `LAG`等函数 66 | 67 | Window functions come in two flavors: SQL aggregate functions used as window functions and specialized window functions. This is the set of aggregate functions in MySQL that support windowing: `COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, `BIT_OR`, `BIT_AND`, `BIT_XOR`, `STDDEV_POP` (and its synonyms `STD`, `STDDEV`), `STDDEV_SAMP`, `VAR_POP` (and its synonym `VARIANCE`) and `VAR_SAMP`. The set of specialized window functions are: `RANK`, `DENSE_RANK`, `PERCENT_RANK`, `CUME_DIST`, `NTILE`, `ROW_NUMBER`, `FIRST_VALUE`, `LAST_VALUE`, `NTH_VALUE`, `LEAD `and `LAG` 68 | 69 | 对窗口函数的支持上，是用户呼声比较频繁。窗口函数早在SQL2003规范中就成为了标准SQL的一部分。 70 | 71 | Support for window functions (a.k.a. analytic functions) is a frequent user request. Window functions have long been part of standard SQL (SQL 2003). See blog post by Dag Wanvik [here](http://mysqlserverteam.com/mysql-8-0-2-introducing-window-functions/) as well as blog post by Guilhem Bichot [here](https://mysqlserverteam.com/row-numbering-ranking-how-to-use-less-user-variables-in-mysql-queries/). 72 | 73 | ###### 公用表达式 74 | 75 | ###### **Common Table Expression** 76 | 77 | MySQL 8.0 带来了支持递归的公用表达式的功能。非递归的公用表达式由于允许由form子句派生的临时表的原因可以被多次引用，因而被解释为改进型的派生表（from子句中的临时表）。而递归的公用表达式则由一组原始住居，经过处理后得到新的一组数据，再被带入处理得到更多的新数据，循环往复直到再也无法产生更多新数据为止。公用表达式也是一个用户呼声频繁的SQL功能。 78 | 79 | MySQL 8.0 delivers [Recursive] Common Table Expressions (CTEs). Non-recursive CTEs can be explained as “improved derived tables” as it allow the derived table to be referenced more than once. A recursive CTE is a set of rows which is built iteratively: from an initial set of rows, a process derives new rows, which grow the set, and those new rows are fed into the process again, producing more rows, and so on, until the process produces no more rows. CTE is a commonly requested SQL feature, see for example feature request [16244 ](https://bugs.mysql.com/bug.php?id=16244)and [32174 ](https://bugs.mysql.com/bug.php?id=32174). See blog posts by Guilhem Bichot [here](http://mysqlserverteam.com/mysql-8-0-labs-recursive-common-table-expressions-in-mysql-ctes/), [here](http://mysqlserverteam.com/mysql-8-0-labs-recursive-common-table-expressions-in-mysql-ctes-part-two-how-to-generate-series/), [here](http://mysqlserverteam.com/mysql-8-0-labs-recursive-common-table-expressions-in-mysql-ctes-part-three-hierarchies/), and [here](http://mysqlserverteam.com/mysql-8-0-1-recursive-common-table-expressions-in-mysql-ctes-part-four-depth-first-or-breadth-first-traversal-transitive-closure-cycle-avoidance/). 80 | 81 | ###### 立即报错或者跳过锁持有的行 82 | 83 | ###### NOWAIT and SKIP LOCKED** 84 | 85 | MySQL 8.0 给SQL的上锁子句带来了`NOWAIT`和`SKIP LOCKED`两个额外的可选项。在原来的版本中，当行数据被`UPDATE`或者`SELECT ... FOR UPDATE`语句上锁后，其他的事务需要等待锁释放才能访问这行数据。但在某些场景下，有着要马上获取反馈的（不等待锁）需求。使用`NOWAIT`参数后如果请求的数据中包括了被锁住的行，将马上会收到查询失败的报错信息。使用`SKIP LOCKED`参数后，返回的数据将会跳过被锁住的行。 86 | 87 | MySQL 8.0 delivers `NOWAIT` and `SKIP LOCKED` alternatives in the SQL locking clause. Normally, when a row is locked due to an `UPDATE` or a `SELECT ... FOR UPDATE`, any other transaction will have to wait to access that locked row. In some use cases there is a need to either return immediately if a row is locked or ignore locked rows. A locking clause using `NOWAIT` will never wait to acquire a row lock. Instead, the query will fail with an error. A locking clause using `SKIP LOCKED` will never wait to acquire a row lock on the listed tables. Instead, the locked rows are skipped and not read at all. NOWAIT and SKIP LOCKED are frequently requested SQL features. See for example feature request [49763 ](https://bugs.mysql.com/bug.php?id=49763). We also want to say thank you to *Kyle Oppenheim* for his code contribution! See blog post by Martin Hansson [here](https://mysqlserverteam.com/mysql-8-0-1-using-skip-locked-and-nowait-to-handle-hot-rows/). 88 | 89 | ###### 降序索引 90 | 91 | ###### Descending Indexes** 92 | 93 | MySQL 8.0 带来了对降序索引的支持。在 8.0降序索引中，数据被倒序组织，正向查找。而在之前的版本中，虽然支持创建降序排列的索引，但其实现方式是通过创建常见的正序索引，然后进行反向查找来实现的。一方面来说，正序查找要比逆序查找更快，另一方面来说，真正的降序索引在复合的order by语句（即有`asc`又有`desc`）中,可以提高索引利用率，减少`filesort`。 94 | 95 | MySQL 8.0 delivers support for indexes in descending order. Values in such an index are arranged in descending order, and we scan it forward. Before 8.0, when a user create a descending index, we created an ascending index and scanned it backwards. One benefit is that forward index scans are faster than backward index scans. Another benefit of a real descending index is that it enables us to use indexes instead of filesort for an `ORDER BY` clause with mixed `ASC/DESC` sort key parts. [Descending Indexes](https://dev.mysql.com/doc/refman/8.0/en/descending-indexes.html) is a frequently requested SQL feature. See for example feature request [13375](https://bugs.mysql.com/bug.php?id=13375) . See blog post by Chaithra Gopalareddy [here](http://mysqlserverteam.com/mysql-8-0-labs-descending-indexes-in-mysql/). 96 | 97 | ###### **分组函数 GROUPING** 98 | 99 | MySQL 8.0 带来了`GROUPING()`分组函数，这个功能可以把`group by`子句的扩展功能（如`ROLLUP`）产生的过聚合的NULL值，通过0和1进行区分，1为NULL，[这样就可以在having子句中对过聚合的无效值进行过滤](https://www.cnblogs.com/li-peng/p/3298303.html)。 100 | 101 | MySQL 8.0 delivers `GROUPING()`, `SQL_FEATURE T433`. The `GROUPING()` function distinguishes super-aggregate rows from regular grouped rows. `GROUP BY` extensions such as `ROLLUP` produce super-aggregate rows where the set of all values is represented by null. Using the `GROUPING()` function, you can distinguish a null representing the set of all values in a super-aggregate row from a `NULL` in a regular row. GROUPING is a frequently requested SQL feature. See feature requests [3156 ](https://bugs.mysql.com/bug.php?id=3156)and [46053](https://bugs.mysql.com/bug.php?id=46053). Thank you to *Zoe Dong* and *Shane Adams* for code contributions in feature request [46053 ](https://bugs.mysql.com/bug.php?id=46053)! See blog post by Chaithra Gopalareddy [here](https://mysqlserverteam.com/mysql-8-0-grouping-function/). 102 | 103 | ###### **优化器建议 Optimizer Hints** 104 | 105 | 在5.7版本中我们引入了新的优化器建议的语法，借助这个新的语法，优化器建议可以被用`/*+ */`包裹起来直接放在`SELECT | INSERT | REPLACE | UPDATE | DELETE`关键字的后面。在8.0的版本中我们又加入了新的姿势。 106 | 107 | In 5.7 we introduced a new hint syntax for [optimizer hints](https://dev.mysql.com/doc/refman/5.7/en/optimizer-hints.html). With the new syntax, hints can be specified directly after the `SELECT | INSERT | REPLACE | UPDATE | DELETE`keywords in an SQL statement, enclosed in `/*+ */` style comments. (See 5.7 blog post by Sergey Glukhov [here](https://mysqlserverteam.com/new-optimizer-hints-in-mysql/)). In MySQL 8.0 we complete the picture by fully utilizing this new style: 108 | 109 | - 8.0版本增加了`INDEX_MERGE`和`NO_INDEX_MERGE`,允许用户在单个查询中控制是否使用索引合并特性。 110 | - MySQL 8.0 adds hints for `INDEX_MERGE` and `NO_INDEX_MERGE`. This allows the user to control index merge behavior for an individual query without changing the optimizer switch. 111 | - 8.0版本增加了`JOIN_FIXED_ORDER`, `JOIN_ORDER`, `JOIN_PREFIX`, 和 `JOIN_SUFFIX`，允许用户控制join表关联的顺序。 112 | - MySQL 8.0 adds hints for `JOIN_FIXED_ORDER`, `JOIN_ORDER`, `JOIN_PREFIX`, and `JOIN_SUFFIX`. This allows the user to control table order for the join execution. 113 | - 8.0版本增加了`SET_VAR`，该优化器建议可以设定一个只在下一条语句中生效的的系统参数。 114 | - MySQL 8.0 adds a hint called `SET_VAR`. The `SET_VAR` hint will set the value for a given system variable for the next statement only. Thus the value will be reset to the previous value after the statement is over. See blog post by Sergey Glukhov [here](https://mysqlserverteam.com/new-optimizer-hint-for-changing-the-session-system-variable/). 115 | 116 | 相对于之前的优化器建议和优化器特性开关参数，我们更倾向于推荐新形式的优化器建议，新形式的优化器建议可以在不侵入SQL语句（指修改语句的非注释的业务部分）的情况下，注入查询语句的很多位置。与直接修改语句的优化器建议相比，新形势的优化器建议在SQL语义上更加清晰。 117 | 118 | We prefer the new style of optimizer hints as preferred over the old-style hints and setting of [`optimizer_switch`](https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_optimizer_switch) values. By not being inter-mingled with SQL, the new hints can be injected in many places in a query string. They also have clearer semantics in being a hint (vs directive). 119 | 120 | ### **JSON** 121 | 122 | 8.0版本追加了新的JSON函数，并可以提高在排序与分组JSON数据情况下的性能。 123 | 124 | MySQL 8.0 adds new JSON functions and improves performance for sorting and grouping JSON values. 125 | 126 | 127 | 128 | ###### JSON path表达式中扩展的范围性语法 Extended Syntax for Ranges in JSON path expressions** 129 | 130 | MySQL 8.0 扩展了JSON path表达式中范围性的语法，比如：`SELECT JSON_EXTRACT('[1, 2, 3, 4, 5]', '$[1 to 3]');`可以得出`[2, 3, 4]`的结果 131 | 132 | MySQL 8.0 extends the syntax for ranges in JSON path expressions. For example `SELECT JSON_EXTRACT('[1, 2, 3, 4, 5]', '$[1 to 3]');` results in `[2, 3, 4]`. The new syntax introduced is a subset of the SQL standard syntax, described in SQL:2016, 9.39 SQL/JSON path language: syntax and semantics. See also [Bug#79052](https://bugs.mysql.com/bug.php?id=79052)reported by Roland Bouman. 133 | 134 | ###### **JSON表函数 JSON Table Functions** 135 | 136 | MySQL 8.0 增加了可以在JSON数据上使用SQL处理工具的JSON 表函数。`JSON_TABLE()`函数可以创建JSON数据的关系型视图。可以将JSON数据估算到关系型的行列之中，用户可以对此函数返回的数据按照常规关系型数据表的方式进行SQL运算。 137 | 138 | MySQL 8.0 adds JSON table functions which enables the use of the SQL machinery for JSON data. `JSON_TABLE()` creates a relational view of JSON data. It maps the result of a JSON data evaluation into relational rows and columns. The user can query the result returned by the function as a regular relational table using SQL, e.g. join, project, and aggregate. 139 | 140 | ###### **JSON 聚合函数 JSON Aggregation Functions** 141 | 142 | MySQL 8.0 增加了用于生成JSON阵列的聚合函数`JSON_ARRAYAGG()`，和用于生成JSON对象的`JSON_OBJECTAGG()`函数，令多行的JSON文档组合成JSON阵列或者JSON对象成为可能。 143 | 144 | MySQL 8.0 adds the aggregation functions `JSON_ARRAYAGG()` to generate JSON arrays and `JSON_OBJECTAGG()` to generate JSON objects . This makes it possible to combine JSON documents in multiple rows into a JSON array or a JSON object. See blog post by Catalin Besleaga [here](http://mysqlserverteam.com/mysql-8-0-labs-json-aggregation-functions/). 145 | 146 | ###### **JSON 合并函数 JSON Merge Functions** 147 | 148 | `JSON_MERGE_PATCH()` 函数可执行JavaScript的语法，在合并时发生重复键值对时将会优先选用第二个文档的键值对，并删除第一个文档对应的重复键值。 149 | 150 | The `JSON_MERGE_PATCH()` function implements the semantics of JavaScript (and other scripting languages) specified by [RFC7396](https://tools.ietf.org/html/rfc7396), i.e. it removes duplicates by precedence of the second document. For example, `JSON_MERGE('{"a":1,"b":2 }','{"a":3,"c":4 }');# returns {"a":3,"b":2,"c":4}`. 151 | 152 | `JSON_MERGE_PRESERVE()`函数与5.7版本中的` JSON_MERGE()`含义相同，都是在合并的时候保留所有值。 153 | 154 | The `JSON_MERGE_PRESERVE()` function has the semantics of JSON_MERGE() implemented in MySQL 5.7 which preserves all values, for example `JSON_MERGE('{"a": 1,"b":2}','{"a":3,"c":4}'); # returns {"a":[1,3],"b":2,"c":4}.` 155 | 156 | 5.7原来的`JSON_MERGE()` 函数在8.0版本中为减少merge操作的不明确，而被弃用。 157 | 158 | The existing `JSON_MERGE()` function is deprecated in MySQL 8.0 to remove ambiguity for the merge operation. See also proposal in [Bug#81283](http://bugs.mysql.com/bug.php?id=81283) and blog post by Morgan Tocker [here](https://mysqlserverteam.com/proposal-to-change-the-behavior-of-json_merge/). 159 | 160 | ###### **JSON 美化函数 JSON Pretty Function** 161 | 162 | 8.0版本增加了可以接收JSON原生数据类型和字符串表达的JSON，并返回一行缩进的易读的JSON格式化后的的字符串。 163 | 164 | MySQL 8.0 adds a `JSON_PRETTY()` function in MySQL. The function accepts either a JSON native data-type or string representation of JSON and returns a JSON formatted string in a human-readable way with new lines and indentation. 165 | 166 | ###### **JSON 文件大小函数 JSON Size Functions** 167 | 168 | 8.0版本增加了和指定JSON对象空间占用相关的函数，`JSON_STORAGE_SIZE()` 可以用字节为单位返回JSON某个数据类型的实际大小， `JSON_STORAGE_FREE()` 可以返回该JSON数据类型的剩余空间（包括碎片和用来适应更改后发生长度变化的预备空间） 169 | 170 | MySQL 8.0 adds JSON functions related to space usage for a given JSON object. The `JSON_STORAGE_SIZE()` returns the actual size in bytes for a JSON datatype. The `JSON_STORAGE_FREE()` returns the free space of a JSON binary type in bytes, including fragmentation and padding saved for inplace update. 171 | 172 | ###### **JSON 改进型的排序 JSON Improved Sorting** 173 | 174 | 8.0版本通过使用变长的排序键提升了JSON排序分组的性能。在某些场景下，Preliminary 的压测结果出现了1.2到18倍的提升。 175 | 176 | MySQL 8.0 gives better performance for sorting/grouping JSON values by using variable length sort keys. Preliminary benchmarks shows from 1.2 to 18 times improvement in sorting, depending on use case. 177 | 178 | ###### **JSON的部分更新 JSON Partial Update** 179 | 180 | 8.0版本增加了对 `JSON_REMOVE()`, `JSON_SET()` and `JSON_REPLACE()` 函数的部分更新的支持。如果JSON文档的某部分被更新，我们会将更改的详情给到句柄。这样存储引擎和复制关系就不必写入整个JSON文档。在之前的复制环境中由于无法确保JSON文档的排列(layout)在主从上完全一致，所以在基于行的复制情况下物理文件的差异并不能用来削减传输复制信息带来的网络IO消耗。因此，8.0版本提供了在逻辑上区分差异的方法，可以在行复制的情况下传输并应用到从库上 181 | 182 | MySQL 8.0 adds support for partial update for the `JSON_REMOVE()`, `JSON_SET()` and `JSON_REPLACE()` functions. If only some parts of a JSON document are updated, we want to give information to the handler about what was changed, so that the storage engine and replication don’t need to write the full document. In a replicated environment, it cannot be guaranteed that the layout of a JSON document is exactly the same on the slave and the master, so the physical diffs cannot be used to reduce the network I/O for row-based replication. Thus, MySQL 8.0 provides logical diffs that row-based replication can send over the wire and reapply on the slave. See blog post by Knut Anders Hatlen [here](https://mysqlserverteam.com/partial-update-of-json-values/). 183 | 184 | ### **地理信息系统 GIS** 185 | 186 | 8.0 版本提供对地形的支持，其中包括了对空间参照系的数据源信息的支持，SRS aware spatial数据类型，空间索引，空间函数。总而言之，8.0版本可以理解地球表面的经纬度信息，而且可以在任意受支持的5000个空间参照系中计算地球上任意两点之间的距离。 187 | 188 | MySQL 8.0 delivers geography support. This includes meta-data support for Spatial Reference System (SRS), as well as SRS aware spatial datatypes, spatial indexes, and spatial functions. In short, MySQL 8.0 understands latitude and longitude coordinates on the earth’s surface and can, for example, correctly calculate the distances between two points on the earths surface in any of the about 5000 supported spatial reference systems. 189 | 190 | ###### **空间参照系 Spatial Reference System (SRS)** 191 | 192 | `ST_SPATIAL_REFERENCE_SYSTEMS` 存在于information schema视图库中，提供了可供使用的SRS坐标系统的名称。每个SRS坐标系统都有一个SRID编号。8.0版本支持EPSG Geodetic Parameter Dataseset中的5千多个坐标系统（包括立体模和2D平面地球模型） 193 | 194 | The `ST_SPATIAL_REFERENCE_SYSTEMS` information schema view provides information about available spatial reference systems for spatial data. This view is based on the SQL/MM (ISO/IEC 13249-3) standard. Each spatial reference system is identified by an SRID number. MySQL 8.0 ships with about 5000 SRIDs from the [EPSG Geodetic Parameter Dataset](http://www.epsg.org/EPSGhome.aspx), covering georeferenced ellipsoids and 2d projections (i.e. all 2D spatial reference systems). 195 | 196 | ###### **SRID 地理数据类型 SRID aware spatial datatypes** 197 | 198 | 空间类的数据类型可以直接从SRS坐标系统的定义中获取，例如：使用SRID 4326定义进行建表： `CREATE TABLE t1 (g GEOMETRY SRID 4326);` 。SRID是适用于地理类型的数据类型。只有同一SRID的的数据才会被插入到行中。与当前SRID数据类型的数据尝试插入时，会报错。未定义SRID编号的表将可以接受所有SRID编号的数据。 199 | 200 | Spatial datatypes can be attributed with the spatial reference system definition, for example with SRID 4326 like this: `CREATE TABLE t1 (g GEOMETRY SRID 4326);` The SRID is here a SQL type modifier for the GEOMETRY datatype. Values inserted into a column with an SRID property must be in that SRID. Attempts to insert values with other SRIDs results in an exception condition being raised. Unmodified types, i.e., types with no SRID specification, will continue to accept all SRIDs, as before. 201 | 202 | 8.0版本增加了 `INFORMATION_SCHEMA.ST_GEOMETRY_COLUMNS` 视图，可以显示当前实例中所有地理信息的数据行及其对应的SRS名称，编号，地理类型名称。 203 | 204 | MySQL 8.0 adds the `INFORMATION_SCHEMA.ST_GEOMETRY_COLUMNS` view as specified in SQL/MM Part 3, Sect. 19.2. This view will list all GEOMETRY columns in the MySQL instance and for each column it will list the standard `SRS_NAME` , `SRS_ID` , and `GEOMETRY_TYPE_NAME`. 205 | 206 | ###### **SRID 空间索引 SRID aware spatial indexes** 207 | 208 | 在空间数据类型上可以创建空间索引，创建空间索引的列必须非空，例如： `CREATE TABLE t1 (g GEOMETRY SRID 4326 NOT NULL, SPATIAL INDEX(g));` 209 | 210 | Spatial indexes can be created on spatial datatypes. Columns in spatial indexes must be declared NOT NULL. For example like this: `CREATE TABLE t1 (g GEOMETRY SRID 4326 NOT NULL, SPATIAL INDEX(g));` 211 | 212 | 创建空间索引的列必须具有SRID数据标识以用于优化器使用，如果将空间索引建在没有SRID数据标识的列上，将输出waring信息。 213 | 214 | Columns with a spatial index should have an SRID type modifier to allow the optimizer to use the index. If a spatial index is created on a column that doesn’t have an SRID type modifier, a warning is issued. 215 | 216 | ###### **SRID 空间函数 SRID aware spatial functions** 217 | 218 | 8.0 增加了诸如 `ST_Distance()` 和 `ST_Length()` 等用于判断数据的参数是否在SRS中，并计算其空间上的距离。到目前为止，`ST_Distance`和其他的空间关系型函数诸如`ST_Within`,`ST_Intersects`,`ST_Contains`,`ST_Crosses`都支持地理计算。其运算逻辑与行为参见 SQL/MM Part 3 Spatial 219 | 220 | MySQL 8.0 extends spatial functions such as `ST_Distance()` and `ST_Length()` to detect that its parameters are in a geographic (ellipsoidal) SRS and to compute the distance on the ellipsoid. So far, `ST_Distance` and spatial relations such as `ST_Within`, `ST_Intersects`, `ST_Contains`, `ST_Crosses`, etc. support geographic computations. The behavior of each ST function is as defined in SQL/MM Part 3 Spatial. 221 | 222 | ### 字符集 Character Sets 223 | 224 | 8.0版本默认使用UTF8MB4作为默认字符集。相比较5.7版本，SQL性能（诸如排序UTF8MB4字符串）得到了很大的提升。UTF8MB4类型在网页编码上正占据着举足轻重的地位，将其设为默认数据类型后，将会给绝大多数的MySQL用户带来遍历。 225 | 226 | MySQL 8.0 makes [UTF8MB4](https://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html) the default character set. SQL performance – such as sorting UTF8MB4 strings – has been improved by a factor of 20 in 8.0 as compared to 5.7. UTF8MB4 is the dominating character encoding for the web, and this move will make life easier for the vast majority of MySQL users. 227 | 228 | - 默认的字符集从`latin1`变为 `utf8mb4` ,默认排序校对规则从 `latin1_swedish_ci` 变为`utf8mb4_800_ci_ai`。 229 | - The default character set has changed from `latin1` to `utf8mb4` and the default collation has changed from `latin1_swedish_ci` to `utf8mb4_800_ci_ai`. 230 | - `utf8mb4`同样也成为libmysql，服务端命令行工具,server层的默认编码 231 | - The changes in defaults applies to libmysql and server command tools as well as the server itself. 232 | - `utf8mb4`同样也成为MySQL测试框架的默认编码 233 | - The changes are also reflected in MTR tests, running with new default charset. 234 | - 排序校对规则的权重与大小写基于Unicode委员会16年公布的Unicode 9.0.0版本。 235 | - The collation weight and case mapping are based on [Unicode 9.0.0](http://www.unicode.org/versions/Unicode9.0.0/) , announced by the Unicode committee on Jun 21, 2016. 236 | - 在以往的MySQL版本中，`latin1`编码中的21种语言的特殊大小写和排序校对规则被引入了 `utf8mb4` 排序校对规则。例如：捷克语的排序校对规则变成了` utf8mb4_cs_800_ai_ci`。 237 | - The 21 language specific case insensitive collations available for latin1 (MySQL legacy) have been implemented for `utf8mb4` collations, for example the Czech collation becomes utf8mb4_cs_800_ai_ci. See complete list in [WL#9108](http://dev.mysql.com/worklog/task/?id=9108) . See blog post by Xing Zhang [here](http://mysqlserverteam.com/new-collations-in-mysql-8-0-0/) . 238 | - 增加了对特殊语境和重音敏感的排序校对规则的支持。8.0版本支持 DUCET (Default Unicode Collation Entry Table)全部三级排序校对规则。 239 | - Added support for case and accent sensitive collations. MySQL 8.0 supports all 3 levels of collation weight defined by DUCET (Default Unicode Collation Entry Table). See blog post by Xing Zhang [here](http://mysqlserverteam.com/mysql-8-0-1-accent-and-case-sensitive-collations-for-utf8mb4/). 240 | - `utf8mb4` 的 `utf8mb4_ja_0900_as_cs` 排序校验规则对日语字符支持三级权重的排序。 241 | - Japanese `utf8mb4_ja_0900_as_cs` collation for `utf8mb4` which sorts characters by using three levels’ weight. This gives the correct sorting order for Japanese. See blog post by Xing Zhang [here](http://mysqlserverteam.com/mysql-8-0-1-japanese-collation-for-utf8mb4/). 242 | - 对日语有额外的假名支持特性， `utf8mb4_ja_0900_as_cs_ks`中的ks表示假名区分。 243 | - Japanese with additional kana sensitive feature, `utf8mb4_ja_0900_as_cs_ks`, where ‘ks’ stands for ‘kana sensitive’. See blog post by Xing Zhang [here](http://mysqlserverteam.com/mysql-8-0-kana-sensitive-collation-for-japanese/). 244 | - 把 Unicode 9.0.0之前所有排序校验规则中的不填补变成填补字符，此举有利于提升字符串的一致性和性能。例如把字符串末尾的空格按照其他字符对待。之前的排序校验规则在处理这种情况时保留字符串原样。 245 | - Changed all new collations, from Unicode 9.0.0 forward, to be `NO PAD` instead of `PAD STRING`, ie., treat spaces at the end of a string like any other character. This is done to improve consistency and performance. Older collations are left in place. 246 | 247 | See also blog posts by Bernt Marius Johnsen [here](http://mysqlserverteam.com/debugging-character-set-issues-by-example/), [here](http://mysqlserverteam.com/mysql-8-0-collations-migrating-from-older-collations/) and [here](http://mysqlserverteam.com/mysql-8-0-collations-the-devil-is-in-the-details/). 248 | 249 | ### **数据类型 Datatypes** 250 | 251 | ###### **二进制数据类型的Bit-wise操作 Bit-wise operations on binary data types** 252 | 253 | 8.0版本扩展了 `bit-wise`操作（如`bit-wise AND`等）的使用范围，使得其在所有 `BINARY` 数据类型上都适用。在此之前只支持整型数据，若强行在二进制数据类型上使用`Bit-wise`操作，将会隐式转换为64位的`BITINT`类型，并可能丢失若干位的数据。从8.0版本之后，bit-wise操作可以在 `BINARY` 和`BLOB`类型上使用，且不用担心精确度下降的问题。 254 | 255 | MySQL 8.0 extends the bit-wise operations (‘bit-wise AND’, etc) to also work with `[VAR]BINARY/[TINY|MEDIUM|LONG]BLOB`. Prior to 8.0 bit-wise operations were only supported for integers. If you used bit-wise operations on binaries the arguments were implicitly cast to `BIGINT` (64 bit) before the operation, thus possibly losing bits. From 8.0 and onward bit-wise operations work for all `BINARY` and `BLOB` data types, casting arguments such that bits are not lost. 256 | 257 | ###### **IPV6操作 IPV6 manipulation** 258 | 259 | 8.0版本通过支持 `BINARY` 上的`Bit-wise`操作提升了IPv6数据的可操作性。5.6版本中引入了支持IPv6地址和16位二进制数据的互相转换的`INET6_ATON()` 和 `INET6_NTOA()` 函数。但是直到8.0之前，由于上一段中的问题我们都无法讲IPv6转换函数和`bit-wise`操作结合起来。由于 `INET6_ATON()` 可以正确的返回128bit的`VARBINARY(16)`，如果我们想要将一个IPv6地址与网关地址进行比对，现在就可以使用 `INET6_ATON(address)& INET6_ATON(network)` 操作。 260 | 261 | MySQL 8.0 improves the usability of IPv6 manipulation supporting bit-wise operations on BINARY data types. In MySQL 5.6 we introduced the `INET6_ATON()` and `INET6_NTOA()` functions which convert IPv6 addresses between text form like `'fe80::226:b9ff:fe77:eb17'` and `VARBINARY(16)`. However, until now we could not combine these IPv6 functions with bit-wise operations since such operations would – wrongly – convert output to `BIGINT`. For example, if we have an IPv6 address and want to test it against a network mask, we can now use `INET6_ATON(address)& INET6_ATON(network)` because `INET6_ATON()` correctly returns the `VARBINARY(16)`datatype (128 bits). See blog post by Catalin Besleaga [here](https://mysqlserverteam.com/mysql-8-0-storing-ipv6/). 262 | 263 | ###### **UUID 操作UUID manipulations** 264 | 265 | 8.0版本通过增加了三个新的函数（`UUID_TO_BIN()`, `BIN_TO_UUID()`, 和 `IS_UUID()`）提升了UUID的可用性。`UUID_TO_BIN()`可以将UUID格式的文本转换成`VARBINARY(16)`, `BIN_TO_UUID()`则与之相反， `IS_UUID()`用来校验UUID的有效性。将UUID以 `VARBINARY(16)` 的方式存储后，就可以使用实用的索引了。 `UUID_TO_BIN()` 函数可以原本转换后的二进制数值中的时间相关位（UUID生成时有时间关联）移到数据的开头，这样对索引来说更加友好而且可以减少在B树中的随机插入，从而减少了插入耗时。 266 | 267 | MySQL 8.0 improves the usability of UUID manipulations by implementing three new SQL functions: `UUID_TO_BIN()`, `BIN_TO_UUID()`, and `IS_UUID()`. The first one converts from UUID formatted text to `VARBINARY(16)`, the second one from `VARBINARY(16)` to UUID formatted text, and the last one checks the validity of an UUID formatted text. The UUID stored as a `VARBINARY(16)` can be indexed using functional indexes. The functions `UUID_TO_BIN()` and `UUID_TO_BIN()` can also shuffle the time-related bits and move them at the beginning making it index friendly and avoiding the random inserts in the B-tree, this way reducing the insert time. The lack of such functionality has been mentioned as one of the [drawbacks of using UUID’s](https://www.percona.com/blog/2014/12/19/store-uuid-optimized-way/). See blog post by Catalin Besleaga [here](https://mysqlserverteam.com/mysql-8-0-uuid-support/). 268 | 269 | ### **消耗敏感的模型 Cost Model** 270 | 271 | ###### **查询优化器将会照顾到数据缓冲的状况 Query Optimizer Takes Data Buffering into Account** 272 | 273 | 8.0版本自动地根据数据是否存在于内存中而选择查询计划，在以往的版本中，消耗敏感的模型始终假设数据在磁盘上。正因为现在查询内存数据和查询硬盘数据的消耗常数不同，因此优化器会根据数据的位置选择更加优化的读取数据方式。 274 | 275 | MySQL 8.0 chooses query plans based on knowledge about whether data resides in-memory or on-disk. This happens automatically, as seen from the end user there is no configuration involved. Historically, the MySQL cost model has assumed data to reside on spinning disks. The cost constants associated with looking up data in-memory and on-disk are now different, thus, the optimizer will choose more optimal access methods for the two cases, based on knowledge of the location of data. See blog post by Øystein Grøvlen [here](https://mysqlserverteam.com/mysql-8-0-query-optimizer-takes-data-buffering-into-account/). 276 | 277 | ###### **查询优化器的直方图 Optimizer Histograms** 278 | 279 | 8.0版本加入了直方图统计数据。用户可以根据直方图针对表中的某列（一般为非索引列）生成数据分布统计信息，这样优化器就可以利用这些信息去寻觅更加优化的查询计划。直方图最常见的使用场景就是计算字段的选择性。 280 | 281 | MySQL 8.0 implements histogram statistics. With Histograms, the user can create statistics on the data distribution for a column in a table, typically done for non-indexed columns, which then will be used by the query optimizer in finding the optimal query plan. The primary use case for histogram statistics is for calculating the selectivity (filter effect) of predicates of the form “COLUMN operator CONSTANT”. 282 | 283 | 用以创建直方图的 `ANALYZE TABLE` 语法现已被扩展了两个新子句： `UPDATE HISTOGRAM ON column [, column] [WITH n BUCKETS]` 和`DROP HISTOGRAM ON column [, column]`。直方图的总计总数（桶）是可以选的，默认100。直方图的统计信息被存储在词典表`column_statistics`中，并可以使用`information_schema.COLUMN_STATISTICS`进行查看。由于JSON数据格式的灵活性，直方图现在以JSON对象存储。根据表的大小，`ANALYZE TABLE`命令会自动的判断是否要表进行采样，甚至会根据表中数据的分布情况和统计总量来决定创建等频或者等高的直方图。 284 | 285 | The user creates a histogram by means of the `ANALYZE TABLE` syntax which has been extended to accept two new clauses: `UPDATE HISTOGRAM ON column [, column] [WITH n BUCKETS]` and `DROP HISTOGRAM ON column [, column]`. The number of buckets is optional, the default is 100. The histogram statistics are stored in the dictionary table “column_statistics” and accessible through the view `information_schema.COLUMN_STATISTICS`. The histogram is stored as a JSON object due to the flexibility of the JSON datatype. `ANALYZE TABLE` will automatically decide whether to sample the base table or not, based on table size. It will also decide whether to build a *singleton* or a *equi-height* histogram based on the data distribution and the number of buckets specified. See blog post by Erik Frøseth [here](https://mysqlserverteam.com/histogram-statistics-in-mysql/). 286 | 287 | ### **正则表达式 Regular Expressions** 288 | 289 | 与UTF8MB4的正则支持一同，8.0版本也增加了诸如 `REGEXP_INSTR()`, `REGEXP_LIKE()`, `REGEXP_REPLACE()`, 和`REGEXP_SUBSTR()`等新函数。另外，系统中还增加了用以控制正则表达式致性的 [regexp_stack_limit](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_regexp_stack_limit) (默认`8000000`比特) 和 [regexp_time_limit](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_regexp_time_limit) (默认32步) 参数。`REGEXP_REPLACE()`也是社区中受呼声比较高的特性。 290 | 291 | MySQL 8.0 supports regular expressions for UTF8MB4 as well as new functions like `REGEXP_INSTR()`, `REGEXP_LIKE()`, `REGEXP_REPLACE()`, and `REGEXP_SUBSTR()`. The system variables [regexp_stack_limit](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_regexp_stack_limit) (default `8000000` bytes) and [regexp_time_limit](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_regexp_time_limit) (default 32 steps) have been added to control the execution. The `REGEXP_REPLACE()` function is one of the most requested features by the MySQL community, for example see feature request reported as [BUG #27389](http://bugs.mysql.com/bug.php?id=27389) by Hans Ginzel. See also blog posts by Martin Hansson [here](https://mysqlserverteam.com/new-regular-expression-functions-in-mysql-8-0/) and Bernt Marius Johnsen [here](https://mysqlserverteam.com/mysql-8-0-regular-expressions-and-character-properties/). 292 | 293 | ## **运维自动化特性 Dev Ops features** 294 | 295 | 开发向的运维关心数据库实例的可操作型，通常即可靠性，可用性，性能，安全，可观测性，可管理性。关于InnoDB Cluster和MGR的可靠性我们将会另起新篇单独介绍，接下来的段落将会介绍关于8.0版本针对表在其他可操作性上的改变。 296 | 297 | Dev Ops care about operational aspects of the database, typically about reliability, availability, performance, security, observability, and manageability. High Availability comes with MySQL InnoDB Cluster and MySQL Group Replication which will be covered by a separate blog post. Here follows what 8.0 brings to the table in the other categories. 298 | 299 | ### **可靠性 Reliability** 300 | 301 | 8.0版本在整体上 增加了可靠性，原因如下： 302 | 303 | MySQL 8.0 increases the overall reliability of MySQL because : 304 | 305 | ​ 8.0版本将元信息存储与久经考验的事务性存储引擎InnoDB中。诸如用户权限表，数据字典表，现在都使用 InnoDB进行存储。 306 | 307 | 1. MySQL 8.0 stores its meta-data into InnoDB, a proven transactional storage engine. System tables such as Users and Privileges as well as Data Dictionary tables now reside in InnoDB. 308 | 309 | 8.0版本消除了会导致非一致性的一处隐患。在5.7及以前的版本中，存在着服务层和引擎层两份数据字典，因而可能导致在故障情况下的数据字典间的同步失败。在8.0版本中，只有一份数据字典。 310 | 311 | 2. MySQL 8.0 eliminates one source of potential inconsistency. In 5.7 and earlier versions there are essentially two data dictionaries, one for the Server layer and one for the InnoDB layer, and these can get out of sync in some crashing scenarios. In 8.0 there is only one data dictionary. 312 | 313 | 8.0版本实现了原子化，无惧宕机的DDL。根据这个特性，DDL语句要么被全部执行，要么全部未执行。对于复制环境来说这是至关重要的，否则会导致主从之间因为表结构不一致，数据漂移的情况。 314 | 315 | 3. MySQL 8.0 ensures atomic, crash safe DDL. With this the user is guaranteed that any DDL statement will either be executed fully or not at all. This is particularly important in a replicated environment, otherwise there can be scenarios where masters and slaves (nodes) get out of sync, causing data-drift. 316 | 317 | 基于新的事务型数据字典，可靠性得到了提高。 318 | 319 | This work is done in the context of the new, transactional data dictionary. See blog posts by Staale Deraas [here](https://mysqlserverteam.com/atomic-ddl-in-mysql-8-0/) and [here](https://mysqlserverteam.com/mysql-8-0-data-dictionary-status-in-the-8-0-0-dmr/). 320 | 321 | ### **可观测性 Observability** 322 | 323 | ###### **提高信息视图库的性能 Information Schema (speed up)** 324 | 325 | 8.0版本重新实现了信息视图库，在新的实现中，信息视图库的表都是基于InnoDB存储的数据字典表的简单视图。这比之前有了百倍的性能提升。让信息视图库可以更加实用性的被外部工具引用。 326 | 327 | MySQL 8.0 reimplements Information Schema. In the new implementation the Information Schema tables are simple views on data dictionary tables stored in InnoDB. This is by far more efficient than the old implementation with up to 100 times speedup. This makes Information Schema practically usable by external tooling. See blog posts by Gopal Shankar [here](http://mysqlserverteam.com/mysql-8-0-improvements-to-information_schema/) and [here](http://mysqlserverteam.com/mysql-8-0-scaling-and-performance-of-information_schema/) , and the blog post by Ståle Deraas [here](https://mysqlserverteam.com/further-improvements-on-information_schema-in-mysql-8-0-3/). 328 | 329 | ###### **提高性能信息库的速度 Performance Schema (speed up)** 330 | 331 | 8.0版本通过在性能信息库上增加了100多个索引完成了其查询性能的提升。这些索引是预设，且无法被删除，修改，增加。相对于在单独的数据结构上进行便利，其索引是通过在既存数据表上过滤扫描来实现的。不需要管理B树，或者散列表的重建，更新及其他操作。性能信息库的索引从行为上更像散列索引：1，可以快速返回数据，2，不支持排序操作（推到服务层处理）。根据查询，索引会避免全表扫描的需求，而且会返回一个相当精巧的数据集。对`show indexes`来说，性能信息库的索引是不可见的，但是会出现在`explain`的结果中和其有关的部分。 332 | 333 | MySQL 8.0 speeds up performance schema queries by adding more than 100 indexes on performance schema tables. The indexes on performance schema tables are predefined. They cannot be deleted,added or altered. A performance schema index is implemented as a filtered scan across the existing table data, rather than a traversal through a separate data structure. There are no B-trees or hash tables to be constructed, updated or otherwise managed. Performance Schema tables indexes behave like hash indexes in that a) they quickly retrieve the desired rows, and b) do not provide row ordering, leaving the server to sort the result set if necessary. However, depending on the query, indexes obviate the need for a full table scan and will return a considerably smaller result set. Performance schema indexes are visible with `SHOW INDEXES` and are represented in the `EXPLAIN` output for queries that reference indexed columns. See [comment](http://mysqlserverteam.com/using-sys-session-as-an-alternative-to-show-processlist/#comment-11037) from Simon Mudd. See blog post by Marc Alff [here](https://mysqlserverteam.com/mysql-8-0-performance-schema-now-with-indexes/). 334 | 335 | ###### **参数配置 Configuration Variables** 336 | 337 | 8.0版本增加了关于配置参数的有用信息，比如变量名，最大最小值，当前值的来源，更改用户，更改时间等等。可以在一个新增的性能信息表`variables_info`表中进行查询。 338 | 339 | MySQL 8.0 adds useful information about configuration variables, such as the variable name, *min/max* values, *where* the current value came from, *who* made the change and *when* it was made. This information is found in a new performance schema table called `variables_info`. See blog post by Satish Bharathy [here](https://mysqlserverteam.com/mysql-8-0-persisting-configuration-variables/). 340 | 341 | ###### **客户端错误报告信息统计 Client Error Reporting – Message Counts** 342 | 343 | 8.0版本使得查看服务端曝出的客户端错误信息汇总统计成为可能。用户可以在五个不同的表中查看统计信息：`Global count`, `summary per thread`, `summary per user`, `summary per host`, 和`summary per account`。 344 | 345 | 用户可以查看单个错误信息的次数，被`SQL exception`句柄处理过的数量，第一次发生的时间戳，最近一次的时间戳。给予用户适当的权限后，用户既可以用`select`从中查询，也可以使用`truncate`进行重置统计数据。 346 | 347 | MySQL 8.0 makes it possible to look at aggregated counts of client [error messages](http://dev.mysql.com/doc/refman/5.7/en/error-messages-server.html)reported by the server.The user can look at statistics from 5 different tables: Global count, summary per thread, summary per user, summary per host, or summary per account. For each error message the user can see the number of errors raised, the number of errors handled by the SQL exception handler, “first seen” timestamp, and “last seen” timestamp. Given the right privileges the user can either `SELECT` from these tables or `TRUNCATE` to reset statistics. See blog post by Mayank Prasad [here](https://mysqlserverteam.com/mysql-8-0-performance-schema-instrumentation-of-server-errors/). 348 | 349 | ###### **语句延迟直方图 Statement Latency Histograms** 350 | 351 | 为了更高的观察查询相应时间，8.0版本在性能信息库中提供了语句延迟的直方图，同时会从直方图中计算95%，99%，9999%比例的信息。这些百分比用来作为服务质量的指示器。 352 | 353 | MySQL 8.0 provides performance schema histograms of statements latency, for the purpose of better visibility of query response times. This work also computes “P95”, “P99” and “P999” percentiles from collected histograms. These percentiles can be used as indicators of quality of service. See blog post by Frédéric Descamps [here](http://lefred.be/content/mysql-8-0-statements-latency-histograms/). 354 | 355 | ###### **图形化数据依赖关系锁 Data Locking Dependencies Graph** 356 | 357 | 8.0版本在性能信息库中增加了数据锁生产者身份。当事务A锁住行R时，事务B正在等待一行被A锁住的行。新增的生产者身份将会那些数据被锁住（本例中为行R），谁拥有锁（本例中为事务A），谁在等待被锁住的事务（本例中为事务B） 358 | 359 | MySQL 8.0 instruments data locks in the performance schema. When transaction A is locking row R, and transaction B is waiting on this very same row, B is effectively blocked by A. The added instrumentation exposes which data is locked (R), who owns the lock (A), and who is waiting for the data (B). See blog post by Frédéric Descamps [here](http://lefred.be/content/mysql-8-0-data-locking-visibility/). 360 | 361 | ###### **查询样例摘要 Digest Query Sample** 362 | 363 | 8.0版本为了捕获完成的查询样例和此查询案例的一些关键信息，针对性能信息库中的` events_statements_summary_by_digest`表做了一些改动。为了捕获一个真实的查询，并让用户进行`explain`，并获取查询计划，现增加了一列`QUERY_SAMPLE_TEXT` 。为了捕获查询样例时间戳，增加了一列`QUERY_SAMPLE_SEEN` 。为了捕获查询执行时间，增加了一列 `QUERY_SAMPLE_TIMER_WAIT` 。同时`FIRST_SEEN` 和 `LAST_SEEN` 列被修改为可以使用带小数的秒。 364 | 365 | MySQL 8.0 makes some changes to the [events_statements_summary_by_digest](https://dev.mysql.com/doc/refman/8.0/en/statement-summary-tables.html)performance schema table to capture a full example query and some key information about this query example. The column `QUERY_SAMPLE_TEXT` is added to capture a query sample so that users can run EXPLAIN on a real query and to get a query plan. The column `QUERY_SAMPLE_SEEN` is added to capture the query sample timestamp. The column `QUERY_SAMPLE_TIMER_WAIT` is added to capture the query sample execution time. The columns `FIRST_SEEN` and `LAST_SEEN` have been modified to use fractional seconds. See blog post by Frédéric Descamps [here](http://lefred.be/content/mysql-8-0-digest-query-samples-in-performance_schema/). 366 | 367 | ###### **生产者的元信息 Meta-data about Instruments** 368 | 369 | 8.0版本在性能信息库的 [setup_instruments](https://dev.mysql.com/doc/refman/8.0/en/setup-instruments-table.html)表上增加了诸如属性，易变的，文档等元信息。这些只读信息作为生产者的在线文档被用户或者工具查阅。 370 | 371 | MySQL 8.0 adds meta-data such as *properties*, *volatility*, and *documentation* to the performance schema table [setup_instruments](https://dev.mysql.com/doc/refman/8.0/en/setup-instruments-table.html). This read only meta-data act as online documentation for instruments, to be looked at by users or tools. See blog post by Frédéric Descamps [here](http://lefred.be/content/mysql-8-0-meta-data-added-to-performance_schemas-instruments/). 372 | 373 | ###### 错误记录 Error Logging** 374 | 375 | 8.0版本 带来了对错误日志的重要的改革。从软件架构的角度来说，错误日志成为了新的服务架构的一部分。这意味着高级用户可以根据需要写出自己的错误日志实现。虽然大多数用户并不想这么做，但是或许会在如何写，写在何处上要求有些变通的空间。因此8.0版本为用户提供了sinks和filters来实现。8.0版本实行了一个过滤服务（API），和一个默认的过滤服务实现（组件）。这里的过滤器是指抑制某些特定错误信息的数据和或给定错误信息的某部分的输出。8.0版本实行了一个日志撰写（API），和一个默认的日志撰写服务实现（组件）。日志撰写器可以接收日志信息，并将其写入到日志中，这里的日志可以指典型的文件日志，syslog日志，或者JSON日志。 376 | 377 | MySQL 8.0 delivers a major overhaul of the MySQL [error log](https://dev.mysql.com/doc/refman/8.0/en/error-log.html). From a software architecture perspective the error log is made a component in the new service infrastructure. This means that advanced users can write their own error log implementation if desired. Most users will not want to write their own error log implementation but still want some flexibility in what to write and where to write it. Hence, 8.0 offers users facilities to add *sinks (where)* and *filters (what)*. MySQL 8.0 implements a filtering service (API) and a default filtering service implementation (component). Filtering here means to suppress certain log messages (selection) and/or fields within a given log message (projection). MySQL 8.0 implements a log writer service (API) and a default log writer service implementation (component). Log writers accept a log event and write it to a log. This log can be a classic file, syslog, EventLog and a new JSON log writer. 378 | 379 | 不做任何设置默认的话，8.0版本带来了开箱即用的错误日志改进。比如： 380 | 381 | By default, without any configuration, MySQL 8.0 delivers many out-of-the-box error log improvements such as: 382 | 383 | - **错误编码：** 编码格式现在为MY开头的10000系列数据。比如： “MY-10001”。错误编号在GA版本中将不会变化，但是其代表的含义可能在之后的维护性版本发布中做一些改变。 384 | - **Error numbering:** The format is a number in the 10000 series preceded by “MY-“, for example “MY-10001”. Error numbers will be stable in a GA release, but the corresponding error texts are allowed to change (i.e. improve) in maintenance releases. 385 | - **系统信息：**系统性的信息[System]替换了之前的 [Error]*（指之前错误日志是系统性相关，但是前缀为[Error]的错误信息）*,增加到错误日志中。 386 | - **System messages:** System messages are written to the error log as [System] instead of [Error], [Warning], [Note]. [System] and [Error] messages are printed regardless of verbosity and cannot be suppressed. [System] messages are only used in a few places, mainly associated with major state transitions such as starting or stopping the server. 387 | - **错误日志详细度减弱：** 默认的错误日志详细信息级别` log_error_verbosity` 从3(输出)改成了2（输出警告级别及以上） 388 | - **Reduced verbosity:** The default of [log_error_verbosity](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_log_error_verbosity) changes from 3 (Notes) to 2 (Warning). This makes MySQL 8.0 error log less verbose by default. 389 | - **信息源部分：**每个信息前面都加了[Server], [InnoDB], [Replic] 三个其中之一的注释，以显示信息是从哪个子系统中输出的。 390 | - **Source Component:** Each message is annotated with one of three values [Server], [InnoDB], [Replic] showing which sub-system the message is coming from. 391 | 392 | 8.0GA版本错误日志中的启动信息： 393 | 394 | This is what is written to the error log in 8.0 GA after startup : 395 | 396 | | 1234 | 2018-03-08T10:14:29.289863Z 0 [System][MY-010116] [Server] /usr/sbin/mysqld (mysqld 8.0.5) starting as process 8063 2018-03-08T10:14:29.745356Z 0 [Warning][MY-010068] [Server] CA certificate ca.pem is self signed. 2018-03-08T10:14:29.765159Z 0 [System][MY-010931] [Server] /usr/sbin/mysqld: ready for connections. Version: '8.0.5' socket: '/tmp/mysql.sock' port: 3306 Source distribution. 2018-03-08T10:16:51.343979Z 0 [System][MY-010910] [Server] /usr/sbin/mysqld: Shutdown complete (mysqld 8.0.5) Source distribution. | 397 | | ---- | ------------------------------------------------------------ | 398 | | | | 399 | 400 | 新引入的错误编码方式允许 MySQL在需要的情况下在以后的维护性版本发布中，不改变错误编号的情况下改进错误详细信息。同时错误编号还可以用于在客制化中作为过滤/屏蔽错误信息的实施基础。 401 | 402 | The introduction of error numbering in the error log allows MySQL to improve an error text in upcoming maintenance releases (if needed) while keeping the error number (ID) unchanged. Error numbers also act as the basis for filtering/suppression and internationalization/localization. 403 | 404 | ### **可管理性 Manageability** 405 | 406 | ###### **不可见索引 INVISIBLE Indexes** 407 | 408 | 8.0版本将索引的可见性管理变为可能。一个不可见索引在优化器指定查询执行计划时不会被纳入考虑。但是这个索引仍会在后台维护，因此让其变得可见比删除再增加索引代价更小。设计不可见索引的目的时为了帮助DBA来判断索引能否被删除。如果你假设索引已经不会被用到，可以先将其设为不可见，然后观察查询性能，如果相关的查询性能没有下降的话，最后可以将其删除。这个功能也是万众期盼的。 409 | 410 | MySQL 8.0 adds the capability of toggling the visibility of an index (visible/invisible). An invisible index is not considered by the optimizer when it makes the query execution plan. However, the index is still maintained in the background so it is cheap to make it visible again. The purpose of this is for a DBA / DevOp to determine whether an index can be dropped or not. If you suspect an index of not being used you first make it invisible, then monitor query performance, and finally remove the index if no query slow down is experienced. This feature has been asked for by many users, for example through [Bug#70299](https://bugs.mysql.com/bug.php?id=70299). See blog post by Martin Hansson [here](https://mysqlserverteam.com/mysql-8-0-invisible-indexes/). 411 | 412 | ###### **灵活的Undo表空间管理 Flexible Undo Tablespace Management** 413 | 414 | 8.0版本让用户可以全权管理Undo表空间，比如表空间的数量，存放位置，每个undo表空间有多少回滚段。 415 | 416 | MySQL 8.0 gives the user full control over Undo tablespaces, i.e. *how many*tablespaces, *where* are they placed, and *how many rollback segments* in each. 417 | 418 | ​ *Undo日志不在存放于系统表空间中：*在版本升级的过程中，Undo日志被从系统表空间中分离出来，并放到Undo表空间中。这为现存的非独立Undo表空间的升级留了后路。 419 | 420 | 1. *No more Undo log in the System tablespace.* Undo log is migrated out of the System tablespace and into Undo tablespaces during upgrade. This gives an upgrade path for existing 5.7 installation using the system tablespace for undo logs. 421 | 422 | *Undo表空间可以单独管理*：比如放到更快的磁盘存储上。 423 | 424 | 2. *Undo tablespaces can be managed separately from the System tablespace.*For example, Undo tablespaces can be put on fast storage. 425 | 426 | *在线Undo表空间回收：*为了Undo表空间清理的需要，生成了了两个小的Undo表空间，这样可以让InnoDB在一个活跃一个清理的情况下在线收缩Undo表空间。 427 | 428 | 3. *Reclaim space taken by unusually large transactions (online).* A minimum of two Undo tablespaces are created to allow for tablespace truncation. This allows InnoDB to shrink the undo tablespace because one Undo tablespace can be active while the other is truncated. 429 | 430 | *增多回滚段，减少争用：*现在用户可以选择使用最多127个Undo表空间，每个表空间都有最多可达128个回滚段。更多的回滚段可以让并行的事务更可能地为其undo日志使用独立的回滚段，这样可以减少对同个资源的争用。 431 | 432 | 4. *More rollback segments results in less contention.* The user might choose to have up to 127 Undo tablespaces, each one having up to 128 rollback segments. More rollback segments mean that concurrent transactions are more likely to use separate rollback segments for their undo logs which results in less contention for the same resources. 433 | 434 | See blog post by Kevin Lewis [here](https://mysqlserverteam.com/mysql-8-0-2-more-flexible-undo-tablespace-management/). 435 | 436 | ###### **将全局参数设置持久化 SET PERSIST for global variables** 437 | 438 | 在正常的情况下，全局的动态的参数可以在线更改，但是实例重启后，这些没有写入配置文件或者与配置文件冲突的参数设定值有可能会丢失。8.0版本中可以将全局的动态参数的更改持久化。 439 | 440 | MySQL 8.0 makes it possible to persist global, dynamic server variables. Many server variables are both GLOBAL and DYNAMIC and can be reconfigured while the server is running. For example: `SET GLOBAL sql_mode='STRICT_TRANS_TABLES'; `However, such settings are lost upon a server restart. 441 | 442 | 这就让 `SET PERSIST sql_mode='STRICT_TRANS_TABLES'; `的写法成为可能。这样就可以在重启后，参数设定值仍会留存。这个功能的使用场景很多，但是最重要的是给出了一个在更改配置文件不方便或者根本无法做到的情况下的选择。比如某些托管的场景下，你根本没有文件系统的访问权限，仅有连入数据库服务的权限。和 `SET GLOBAL` 相同，`SET PERSIST`也需要`super`权限。 443 | 444 | This work makes it possible to write `SET PERSIST sql_mode='STRICT_TRANS_TABLES'; `The effect is that the setting will survive a server restart. There are many usage scenarios for this functionality but most importantly it gives a way to manage server settings when editing the configuration files is inconvenient or not an option. For example in some hosted environments you don’t have file system access, all that you have is the ability to connect to one or more servers. As for `SET GLOBAL` you need the super privilege for `SET PERSIST`. 445 | 446 | 除此之外还有 `RESET PERSIST` 命令，可以将之前使用`SET PERSIST`命令更改后参数值的持久化特性取消掉，让这个参数值和 `SET GLOBAL`设置的一样，下次启动可能会丢失。 447 | 448 | There is also the `RESET PERSIST` command. The `RESET PERSIST` command has the semantic of removing the configuration variable from the persist configuration, thus converting it to have similar behavior as `SET GLOBAL`. 449 | 450 | 8.0版本也允许对大部分当前启动环境下的只读参数进行 `SET PERSIST` 更改，在下次启动时会生效。当然了部分只读参数是无法被设置更改的。 451 | 452 | MySQL 8.0 allows `SET PERSIST` to set most read-only variables as well, the new values will here take effect at the next server restart. Note that a small subset of read-only variables are left intentionally not settable. See blog post by Satish Bharathy [here](https://mysqlserverteam.com/mysql-8-0-persisting-configuration-variables/). 453 | 454 | ###### **远程管理 Remote Management** 455 | 456 | 8.0版本增加了` RESTART`命令。其用途是用来允许通过SQL连接来允许远程管理MySQL实例。比如通过 `SET PERSIST` 命令更改了只读参数后，可以用这个命令来重启MySQL实例。当然，需要`shutdown`权限（译者注：同时mysqld进程需要被supervisor进程管理才可以。）。 457 | 458 | MySQL 8.0 implements an SQL RESTART command. The purpose is to enable remote management of a MySQL server over an SQL connection, for example to set a non-dynamic configuration variable by `SET PERSIST` followed by a `RESTART`. See blog post [MySQL 8.0: changing configuration easily and cloud friendly !](http://lefred.be/content/mysql-8-0-changing-configuration-easily-and-cloud-friendly/) by Frédéric Descamps. 459 | 460 | ###### **重命名表空间 Rename Tablespace (SQL DDL)** 461 | 462 | 8.0版本增加了 `ALTER TABLESPACE s1 RENAME TO s2;`功能，通用表空间或者共享表空间都可以被用户创建，更改，删除。 463 | 464 | MySQL 8.0 implements `ALTER TABLESPACE s1 RENAME TO s2;` A shared/general tablespace is a user-visible entity which users can CREATE, ALTER, and DROP. See also [Bug#26949](https://bugs.mysql.com/bug.php?id=26949), [Bug#32497](https://bugs.mysql.com/bug.php?id=32497), and [Bug#58006](https://bugs.mysql.com/bug.php?id=58006). 465 | 466 | ###### **列重命名 Rename Column (SQL DDL)** 467 | 468 | 8.0版本支持 `ALTER TABLE ... RENAME COLUMN old_name TO new_name;`，这相对于现有的` ALTER TABLE CHANGE … `语法（需要注明当前列所有属性）来说是个改进。应用程序可能由于某些原因并不能获取所该列的所有信息，因此，这是之前语法的弊端。同时之前的语法也可能会偶然导致数据丢失（rename应该是在线DDL，不需要重建表） 469 | 470 | MySQL 8.0 implements `ALTER TABLE ... RENAME COLUMN old_name TO new_name;`This is an improvement over existing syntax ALTER TABLE CHANGE … which requires re-specification of all the attributes of the column. The old/existing syntax has the disadvantage that all the column information might not be available to the application trying to do the rename. There is also a risk of accidental data type change in the old/existing syntax which might result in data loss. 471 | 472 | ### **安全特性 Security features** 473 | 474 | ###### **新的默认验证插件 New Default Authentication Plugin** 475 | 476 | 8.0版本将默认的验证插件从`mysql_native_password`改成了`caching_sha2_password`。与此对应的，libmysqlclient也会采用`caching_sha2_password `验证机制。新的`caching_sha2_password`结合了更高的安全特性（SHA2算法）和高性能（缓存）。我们目前建议所有用户在网络通信上使用TLS/SSL。 477 | 478 | MySQL 8.0 changes the default authentication plugin from [mysql_native_password](https://dev.mysql.com/doc/refman/8.0/en/native-pluggable-authentication.html)to [caching_sha2_password](https://dev.mysql.com/doc/refman/8.0/en/caching-sha2-pluggable-authentication.html). Correspondingly, libmysqlclient will use caching_sha2_password as the default authentication mechanism, too. The new caching_sha2_password combines better security (SHA2 algorithm) with high performance (caching). The general direction is that we recommend all users to use TLS/SSL for all their network communication. See blog post by Harin Vadodaria [here](https://mysqlserverteam.com/mysql-8-0-4-new-default-authentication-plugin-caching_sha2_password/). 479 | 480 | ###### **社区版本默认使用OpenSSL OpenSSL by Default in Community Edition** 481 | 482 | 8.0版本正在把OpenSSL作为企业版本和社区版本的统一默认TLS/SSL库。之前社区版本使用的是YaSSL。支持OpenSSL也是社区呼声比较多的请求了。 483 | 484 | MySQL 8.0 is unifying on [OpenSSL](https://en.wikipedia.org/wiki/OpenSSL) as the default TLS/SSL library for both MySQL Enterprise Edition and MySQL Community Edition. Previously, MySQL Community Edition used [YaSSL](https://en.wikipedia.org/wiki/WolfSSL). Supporting OpenSSL in the MySQL Community Edition has been one of the most frequently requested features. See blog post by Frédéric Descamps [here](https://mysqlserverteam.com/mysql-8-0-4-openssl-and-mysql-community-edition/). 485 | 486 | ###### **动态链接的OpenSSL OpenSSL is Dynamically Linked** 487 | 488 | 8.0版本动态的和OpenSSL链接在一起。从MySQL软件源的使用者角度来看，现在MySQL包依赖于系统提供的OpenSSL文件。动态的链接之后，OpenSSL的更新就不需要要求MySQL也一同更新或者打补丁。 489 | 490 | MySQL 8.0 is linked dynamically with OpenSSL. Seen from the [MySQL Repository](https://dev.mysql.com/downloads/repo/)users perspective , the MySQL packages depends on the OpenSSL files provided by the Linux system at hand. By dynamically linking, OpenSSL updates can be applied upon availability without requiring a MySQL upgrade or patch. See blog post by Frédéric Descamps [here](https://mysqlserverteam.com/mysql-8-0-4-openssl-and-mysql-community-edition/). 491 | 492 | ###### **对Undo和Redo日志的加密 Encryption of Undo and Redo log** 493 | 494 | 8.0版本加入了对Undo和Redo日志的静态加密。在5.7版本中，我们引入了对使用了独立表空间的InnoDB表的加密。其可为物理的表空间数据文件提供静态加密。在8.0版本中我们将这个贴行扩展到了Undo和Redo日志上。 495 | 496 | MySQL 8.0 implements data-at-rest encryption of UNDO and REDO logs. In 5.7 we introduced [Tablespace Encryption](https://dev.mysql.com/doc/refman/5.7/en/innodb-tablespace-encryption.html) for InnoDB tables stored in file-per-table tablespaces. This feature provides at-rest encryption for physical tablespace data files. In 8.0 we extend this to include UNDO and REDO logs. See documentation [here](https://dev.mysql.com/doc/refman/8.0/en/innodb-tablespace-encryption.html). 497 | 498 | ###### **SQL 角色SQL roles** 499 | 500 | 8.0版本引入了SQL角色。角色是一组权限的集合。其目的是为了简化用户权限管理系统。可以角色赋给用户的身上，给角色授权，创建角色，删除角色，定义哪些角色对于当前会话是合适的。 501 | 502 | MySQL 8.0 implements SQL Roles. A role is a named collection of privileges. The purpose is to simplify the user access right management. One can grant roles to users, grant privileges to roles, create roles, drop roles, and decide what roles are applicable during a session. See blog post by Frédéric Descamps [here](http://lefred.be/content/mysql-8-0-listing-roles/). 503 | 504 | ###### **允许为公用角色授权或收回权限 Allow grants and revokes for PUBLIC** 505 | 506 | 8.0版本引入了可配置的参数 [`mandatory-roles`](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_mandatory_roles)，用于自动给新创建的用户加上角色。授予给所有用户指定的角色的权限不可以被再次分发。但是这些角色除非被设置为默认角色，否则还是需要进行激活操作。当然也可以把新引入的 `activate-all-roles-on-login`参数设为`ON`，这样在用户验证通过连接进来后，所有授权角色都会自动激活。 507 | 508 | MySQL 8.0 introduces the configuration variable [`mandatory-roles`](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_mandatory_roles) which can be used for automatic assignment and granting of *default roles* when new users are created. Example: `role1@%,role2,role3,role4@localhost`. All the specified roles are always considered granted to every user and they can’t be revoked. These roles still require activation unless they are made into default roles. When the new server configuration variable [`activate-all-roles-on-login`](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_activate_all_roles_on_login) is set to “ON”, all granted roles are always activated after the user has authenticated. 509 | 510 | ###### **分解super权限 Breaking up the super privileges** 511 | 512 | 8.0版本定义了在很多方面上一批新粒度的权限用以代替之前版本使用的`SUPER`权限。其本意是用于限制用户仅获得和自己工作相关的权限。比如 [BINLOG_ADMIN](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_binlog-admin), [CONNECTION_ADMIN](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_connection-admin), and [ROLE_ADMIN](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_role-admin). 513 | 514 | MySQL 8.0 defines a set of new granular privileges for various aspects of what SUPER is used for in previous releases. The purpose is to limit user access rights to what is needed for the job at hand and nothing more. For example [BINLOG_ADMIN](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_binlog-admin), [CONNECTION_ADMIN](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_connection-admin), and [ROLE_ADMIN](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_role-admin). 515 | 516 | ###### **用于管理XA事务的授权模型 Authorization model to manage XA-transactions** 517 | 518 | 8.0版本引入了一个新的系统权限 `XA_RECOVER_ADMIN` ，用于控制执行 `XA RECOVER`语句的权限。所有尝试执行 `XA RECOVER` 语句的非授权用户会引起报错。 519 | 520 | MySQL 8.0 introduces a new system privilege `XA_RECOVER_ADMIN` which controls the capability to execute the statement `XA RECOVER`. An attempt to do `XA RECOVER` by a user who wasn’t granted the new system privilege `XA_RECOVER_ADMIN` will cause an error. 521 | 522 | ###### **密码轮换策略 Password rotation policy** 523 | 524 | 8.0版本引入了对密码重新使用的限制，既可以在全局层级也可以在单独的用户等级上配置。过往的历史密码由于安全的原因（会泄露密习惯，或者词组）会被加密保存。密码轮换策略对于其他策略来说是叠加的。可以和现有的机制（如，密码过期和密码安全策略等）共存。 525 | 526 | MySQL 8.0 introduces restrictions on password reuse. Restrictions can be configured at global level as well as individual user level. Password history is kept secure because it may give clues about habits or patterns used by individual users when they change their password. The *password rotation policy* comes in addition to other, existing mechanisms such as the *password expiration policy* and *allowed password policy*. See [Password Management](https://dev.mysql.com/doc/refman/8.0/en/password-management.html). 527 | 528 | ###### **减缓对用户密码的暴力破解 Slow down brute force attacks on user passwords** 529 | 530 | 8.0版本引入了在连续的错误登陆尝试后的等待验证过程。其设计目的用于减缓哦对用户密码的暴力破解。密码连续错误次数和连续错误之后的等待时间是可以配置的。 531 | 532 | MySQL 8.0 introduces a delay in the authentication process based on consecutive unsuccessful login attempts. The purpose is to slow down brute force attacks on user passwords. It is possible to configure the number of consecutive unsuccessful attempts before the delay is introduced and the maximum amount of delay introduced. 533 | 534 | ###### **撤销skip-grant-tables（指远程连接情况下） Retire skip-grant-tables** 535 | 536 | 8.0版本禁止当实例以`–skip-grant-tables`参数启动时的远程用户连接 537 | 538 | MySQL 8.0 disallows remote connections when the server is started with `–skip-grant-tables`. See also [Bug#79027](https://bugs.mysql.com/bug.php?id=79027) reported by Omar Bourja. 539 | 540 | ###### **为实例增加mysqld_safe的部分功能 Add mysqld_safe-functionality to server** 541 | 542 | 8.0版本在实例中引入了部分之前mysqld_safe中的逻辑。可以改善当使用 `--daemonize` 启动参数时在某些情况下的可用性。这也减轻了用户对我们即将移除的`mysqld-safe`脚本的依赖。 543 | 544 | MySQL 8.0 implement parts of the logic currently found in the `mysqld_safe` script inside the server. The work improves server usability in some scenarios for example when using the `--daemonize` startup option. The work also make users less dependent upon the `mysqld_safe script`, which we hope to remove in the future. It also fixes [Bug#75343](https://bugs.mysql.com/bug.php?id=75343) reported by Peter Laursen. 545 | 546 | ### **性能 Performance** 547 | 548 | 8.0 版本带来更好的读写负载，IO依赖性工作负载，和业务热数据集中的负载。另外新增的资源组特性给用户带来在特定硬件特定负载下将用户线程分配给指定CPU的选项。 549 | 550 | MySQL 8.0 comes with better performance for Read/Write workloads, IO bound workloads, and high contention “hot spot” workloads. In addition, the new Resource Group feature gives users an option to optimize for specific workloads on specific hardware by mapping user threads to CPUs. 551 | 552 | ###### **可伸缩的读写负载 Scaling Read/Write Workloads** 553 | 554 | 8.0版本对于读写皆有和高写负载的拿捏恰到好处。在集中的读写均有的负载情况下，我们观测到在4个用户并发的情况下，对于高负载，和5.7版本相比有着两倍性能的提高。在5.7上我们显著了提高了只读情况下的性能，8.0则显著提高了读写负载的可扩展性。为MySQL提升了硬件性能的利用率，其改进是基于重新设计了InnoDB写入Redo日志的方法。对比之前用户线程之前互相争抢着写入其数据变更，在新的Redo日志解决方案中，现在Re'do日志由于其写入和刷缓存的操作都有专用的线程来处理。用户线程之间不在持有Redo写入相关的锁，整个Redo处理过程都是时间驱动。。 555 | 556 | MySQL 8.0 scales well on RW and heavy write workloads. On intensive RW workloads we observe better performance already from 4 concurrent users and more than 2 times better performance on high loads comparing to MySQL 5.7. We can say that while 5.7 significantly improved scalability for Read Only workloads, 8.0 significantly improves scalability for Read/Write workloads. The effect is that MySQL improves hardware utilization (efficiency) for standard server side hardware (like systems with 2 CPU sockets). This improvement is due to re-designing how InnoDB writes to the REDO log. In contrast to the historical implementation where user threads were constantly fighting to log their data changes, in the new REDO log solution user threads are now lock-free, REDO writing and flushing is managed by dedicated background threads, and the whole REDO processing becomes event-driven. See blog post by Dimitri Kravtchuk [here](http://dimitrik.free.fr/blog/archives/2017/10/mysql-performance-80-redesigned-redo-log-readwrite-workloads-scalability.html). 557 | 558 | ###### 榨干IO能力（在更快速的存储设备上） Utilizing IO Capacity (Fast Storage)** 559 | 560 | 8.0版本允许马力全开的使用存储设备，比如使用英特尔奥腾闪存盘的时候，我们可以在IO敏感的负载情况下获得1百万的采样 QPS（这里说的IO敏感是指不在IBP中，且必须从二级存储设备中获取）。这个改观是由于我们摆脱了 `file_system_mutex`全局锁的争用。 561 | 562 | MySQL 8.0 allows users to use every storage device to its full power. For example, testing with Intel Optane flash devices we were able to out-pass 1M Point-Select QPS in a fully IO-bound workload. (IO bound means that data are not cached in buffer pool but must be retrieved from secondary storage). This improvement is due to getting rid of the `fil_system_mutex` global lock. 563 | 564 | ###### **在高争用（热点数据）负载情况下的更优性能 Better Performance upon High Contention Loads (“hot rows”)** 565 | 566 | 8.0版本显著地提升了高争用负载下的性能。高争用负载通常发生在许多事务争用同一行数据的锁，导致了事务等待队列的产生。在实际情景中，负载并不是平稳的，负载可能在特定的时间内爆发（80/20法则）。8.0版本针对短时间的爆发负载无论在每秒处理的事务数（换句话，延迟）还是95%延迟上都处理的更好。对于终端用户来说体现在更好的硬件资源利用率（效率）上。因为系统需要尽量使用榨尽硬件性能，才可以提供更高的平均负载。 567 | 568 | MySQL 8.0 significantly improves the performance for high contention workloads. A high contention workload occurs when multiple transactions are waiting for a lock on the same row in a table, causing queues of waiting transactions. Many real world workloads are not smooth over for example a day but might have bursts at certain hours ([Pareto distributed](https://en.wikipedia.org/wiki/Pareto_distribution)). MySQL 8.0 deals much better with such bursts both in terms of transactions per second, mean latency, and 95th percentile latency. The benefit to the end user is better hardware utilization (efficiency) because the system needs less spare capacity and can thus run with a higher average load. The original patch was contributed by Jiamin Huang ([Bug#84266](https://bugs.mysql.com/bug.php?id=84266)). Please study the [Contention-Aware Transaction Scheduling](https://arxiv.org/pdf/1602.01871.pdf) (CATS) algorithm and read the MySQL blog post by Jiamin Huang and Sunny Bains [here](https://mysqlserverteam.com/contention-aware-transaction-scheduling-arriving-in-innodb-to-boost-performance/). 569 | 570 | ###### **资源组 Resource Groups** 571 | 572 | 8.0版本为MySQL引入了全局资源组。有了资源组的概念后，管理人员可以管理用户线程和系统线程对CPU的分配。这个功能可以被用来按CPU分割负载以在某些使用情景下获取更高的效率和性能。因此DBA的工具箱中又多了一把可以帮助自己提升硬件使用率或者增加查询性能的工具。比如在英特尔志强E7-4860 2.27GHz,40核超线程处理器上运行Sysbench读写负载时，我们可以通过将写负载限制在10个核心上从而将总体请求输入量提升一倍。资源组是相当先进的工具，但由于效果会因为负载的类型和手头的硬件的千差万别，这就要求经验经验丰富的管理人员因地制宜地进行使用。 573 | 574 | MySQL 8.0 introduces global Resource Groups to MySQL. With Resource Groups, DevOps/DBAs can manage the mapping between user/system threads and CPUs. This can be used to split workloads across CPUs to obtain better efficiency and/or performance in some use cases. Thus, Resource Groups adds a tool to the DBA toolbox, a tool which can help the DBA to increase hardware utilization or to increase query stability. As an example, with a Sysbench RW workload running on a Intel(R) Xeon (R) CPU E7-4860 2.27 GHz 40 cores-HT box we doubled the overall throughput by limiting the Write load to 10 cores. Resource Groups is a fairly advanced tool which requires skilled DevOps/DBA to be used effectively as effects will vary with type of load and with the hardware at hand. 575 | 576 | ## **其他特性 Other Features** 577 | 578 | ###### **更优的默认值 Better Defaults** 579 | 580 | 在MySQL了团队中，为了更可能地让用户体验开箱即用，我们对MySQL的默认值保持着紧密的关注。我们已经更改了8.0版本中30多处参数为我们认为更合适的默认参数值。 581 | 582 | In the MySQL team we pay close attention to the default configuration of MySQL, and aim for users to have the best out of the box experience possible. MySQL 8.0 has changed more than 30 default values to what we think are better values. See blog post [New Defaults in MySQL 8.0](https://mysqlserverteam.com/new-defaults-in-mysql-8-0/). The motivation for this is outlined in a blog post by Mogan Tocker [here](https://mysqlserverteam.com/planning-the-defaults-for-mysql-5-8/). 583 | 584 | ###### **协议 Protocol** 585 | 586 | 8.0版本增加了关闭元数据产生并转化成结果集的选项。构建，解析，发送，接收元数据结果集都会消耗实例，客户端和网络的资源。在某些场景下，元数据大小可能比实际的结果集更大，且不必要。因此在我们关闭了元数据产生和存储后，显著了地提升了查询结果集的转化。客户端如果不想接收于数据一同传回的元数据信息，可以设置 `CLIENT_OPTIONAL_RESULTSET_METADATA` 587 | 588 | MySQL 8.0 adds an option to turn off metadata generation and transfer for resultsets. Constructing/parsing and sending/receiving resultset metadata consumes server, client and network resources. In some cases the metadata size can be much bigger than actual result data size and the metadata is just not needed. We can significantly speed up the query result transfer by completely disabling the generation and storage of these data. Clients can set the `CLIENT_OPTIONAL_RESULTSET_METADATA` flag if they do not want meta-data back with the resultset. 589 | 590 | ###### **C语言客户端 API C Client API** 591 | 592 | 8.0版本扩展了 libmysql的C语言API，使其更加稳定地从服务器流式获取复制事务。其设计目的在于为了实现基于binlog的程序（类似Hadoop接收MySQL数据的工具） 进而需要避免对非正式API的调用和对内部头文件的打包 593 | 594 | MySQL 8.0 extends libmysql’s C API with a stable interface for getting replication events from the server as a stream of packets. The purpose is to avoid having to call undocumented APIs and package internal header files in order to implement binlog based programs like the MySQL Applier for Hadoop. 595 | 596 | ###### **内存缓存 Memcached** 597 | 598 | 8.0版本利用多样的的get操作和对范围查询的支持，提升了InnoDB的内存缓存技能。我们通过对多种get操作的支持还提升了读性能，如用户可以在单次内存缓存查询中获取多个键值对。对范围查询的支持是应脸书的Yoshinori 所请求（译者注：MHA作者）。利用范围查询，用户可以指定一个特定的范围，并获取此区间所有合乎需要的键值。这两个特性对于减少客户端和服务端的来回交互来说都是至关重要的。 599 | 600 | MySQL 8.0 enhances the InnoDB Memcached functionalities with *multiple get*operations and support for *range queries*. We added support for the *multiple get*operation to further improve the read performance, i.e. the user can fetch multiple key value pairs in a single memcached query. Support for *range queries* has been requested by Yoshinori @ Facebook. With range queries, the user can specify a particular range, and fetch all the qualified values in this range. Both features can significantly reduce the number of roundtrips between the client and the server. 601 | 602 | ###### **持久化的自增计数器 Persistent Autoinc Counters** 603 | 604 | 8.0版本通过写入redo日志，持久化了自增计数器。持久化的自增计数器解决了一个年代悠久的BUG（译者注：实例在重启后，自增计数器重复利用了之前被使用后删除的自增值）。MySQL恢复进程会重放redo日志，以确保自增计数器的准确数值。不再会出现自增值计数器数值的回滚。这意味着数据库实例在故障恢复时会将redo日志中最后一次发放的自增值作为重建计数器的起点。确保了自增计数器的值不会重复发放，计数器的数值是单调递增的。但要注意可能会有空洞（即发放但是未使用的自增值）。未持久化的自增值在过去一直被认为是一个的有麻烦的故障点。 605 | 606 | MySQL 8.0 persists the `AUTOINC` counters by writing them to the redo log. This is a fix for the very old [Bug#199](http://bugs.mysql.com/bug.php?id=199). The MySQL recovery process will replay the redo log and ensure correct values of the `AUTOINC` counters. There won’t be any rollback of `AUTOINC`counters. This means that database recovery will reestablish the last known counter value after a crash. It comes with the guarantee that the `AUTOINC` counter cannot get the same value twice. The counter is monotonically increasing, but note that there can be gaps (unused values). The lack of persistent `AUTOINC` has been seen as troublesome in the past, e.g. see [Bug#21641](http://bugs.mysql.com/bug.php?id=21641) reported by Stephen Dewey in 2006 or [this](http://mysqlinsights.blogspot.co.uk/2009/03/auto-increment-stability.html) blog post . 607 | 608 | ## 总结 Summary 609 | 610 | 综上所述，8.0版本带来了一大堆新特性和性能提升，马上从dev.mysql.com上下载并尝试一下吧(￣▽￣)"。 611 | 612 | As shown above, MySQL 8.0 comes with a large set of new features and performance improvements. Download it from [dev.mysql.com](http://dev.mysql.com/downloads/mysql/) and try it out ! 613 | 614 | 当然你也可以从既存的5.7实例升级到8.0版本。在此过程中，你可能需要试试我们随着shell工具包一同下发的新的升级检查工具。这个工具可以帮你检查既存5.7版本的8.0版本升级兼容性（译者附：8.0.3到8.0.11不可以直接升级，或许我用到了旧的工具，改天再试）。可以试试Frédéric Descamps写的t[ Migrating to MySQL 8.0 without breaking old application](http://lefred.be/content/migrating-to-mysql-8-0-without-breaking-old-application/) 博文。 615 | 616 | You can also [upgrade](https://dev.mysql.com/doc/refman/8.0/en/upgrading.html) an existing MySQL 5.7 to MySQL 8.0. In the process you might want to try our new [Upgrade Checker](https://mysqlserverteam.com/mysql-shell-8-0-4-introducing-upgrade-checker-utility/) that comes with the new MySQL Shell (mysqlsh). This utility will analyze your existing 5.7 server and tell you about potential 8.0 incompatibilities. Another good resource is the blog post[ Migrating to MySQL 8.0 without breaking old application](http://lefred.be/content/migrating-to-mysql-8-0-without-breaking-old-application/) by Frédéric Descamps. 617 | 618 | 在这篇文章中我们主要覆盖了服务端的新特性。不仅如此，我们还写了很多关于其他方面的文章，如复制，组复制，InnoDB集群，MySQL Shell,开发API及其相关的连接组件（([Connector/Node.js](https://insidemysql.com/introducing-connector-node-js-for-mysql-8-0/), [Connector/Python](https://insidemysql.com/using-mysql-connector-python-8-0-with-mysql-8-0/), [PHP](https://insidemysql.com/introducing-the-mysql-x-devapi-php-extension-for-mysql-8-0/), [Connector/NET](https://insidemysql.com/introducing-connector-net-with-full-support-for-mysql-8-0/), [Connector/ODBC](https://insidemysql.com/what-is-new-in-connector-odbc-8-0/),[Connector/C++](https://insidemysql.com/what-is-new-in-connector-c-8-0/), and [Connector/J](https://insidemysql.com/connector-j-8-0-11-the-face-for-your-brand-new-document-oriented-database/)） 619 | 620 | In this blog post we have covered Server features. There is much more! We will also publish blog posts for other features such as [Replication](https://mysqlhighavailability.com/mysql-8-0-new-features-in-replication/), Group Replication, InnoDB Cluster, [Document Store](https://mysqlserverteam.com/mysql-8-0-announcing-ga-of-the-mysql-document-store/), MySQL Shell, [DevAPI](https://insidemysql.com/mysql-8-0-welcome-to-the-devapi/), and DevAPI based Connectors ([Connector/Node.js](https://insidemysql.com/introducing-connector-node-js-for-mysql-8-0/), [Connector/Python](https://insidemysql.com/using-mysql-connector-python-8-0-with-mysql-8-0/), [PHP](https://insidemysql.com/introducing-the-mysql-x-devapi-php-extension-for-mysql-8-0/), [Connector/NET](https://insidemysql.com/introducing-connector-net-with-full-support-for-mysql-8-0/), [Connector/ODBC](https://insidemysql.com/what-is-new-in-connector-odbc-8-0/),[Connector/C++](https://insidemysql.com/what-is-new-in-connector-c-8-0/), and [Connector/J](https://insidemysql.com/connector-j-8-0-11-the-face-for-your-brand-new-document-oriented-database/)). 621 | 622 | 就这么多，感谢您使用MySQL 623 | 624 | That’s it for now, and **thank you** for using **MySQL** ! 625 | 626 | 627 | 628 | 注：拖拖拉拉一周终于翻译完，困得要命，手头有其他的事情，暂时没时间精力进行复核，请各位见谅。文章分享在github和blog上，大家可以提request，留言，或者通过QQ微信等工具共同交流，共同学习。 629 | -------------------------------------------------------------------------------- /技术文章翻译/【MySQL】【翻译】Adaptive query routing based on GTID tracking 基于GTID追踪的自适应路由查询.md: -------------------------------------------------------------------------------- 1 | # GTID consistent reads 基于GTID的一致性读 2 | 3 | 作者：René Cannaò ProxySQL的作者 4 | 5 | 翻译：张锐志 6 | 7 | 小记：原文的标题和部分段落都有些混乱，已经尽量按作者的想法意义。所以没有和英文完全匹配逐句翻译。 8 | 9 | [原文链接](http://www.proxysql.com/blog/proxysql-gtid-causal-reads) 10 | 11 | # Adaptive query routing based on GTID tracking 基于GTID追踪的自适应路由查询 12 | 13 | ProxySQL是一个工作在第七层(应用层)且支持MySQL 协议的的数据库代理，ProxySQL本身自带了高可用和高性能的功能，并且包含了丰富的功能集。在即将到来的2.0版本中的功能将会更加excited！ 14 | 15 | ProxySQL is a layer 7 database proxy that understands the MySQL Protocol. It provides high availability and high performance out of the box. ProxySQL has a rich set of features, and the upcoming version 2.0 has new exciting features, like the one described in this blog post. 16 | 17 | ProxySQL中最常用的功能当属查询的分析统计和基于路由查询做到的读写分离。 18 | 19 | One of the most commonly used feature is query analysis and query routing in order to provide read/write split. 20 | 21 | 当客户端连接到ProxySQL执行查询时，ProxySQL 会先根据预先设定好的路由规则进行路由检查，并分发到这条语句应该被执行的实例上。最简单的例子就是将所有的读查询全部路由到从库，所有的写查询全部路由到主库。当然，由于读查询有可能在从库上读到非最新的数据，这个案例在生产上并不是实用的。因此，我们建议将所有的请求都发到主库上，同时由于ProxySQL 对SQL统计信息的支持，DBA可以针对性的创建更加精准的路由规则，将特定的查询路由到从库。详细信息和其他案例可以参考[之前的文章](http://www.proxysql.com/blog/configure-read-write-split) 22 | 23 | When a client connected to ProxySQL executes a query, ProxySQL will first check its rules (configured by a DBA) and determine on which MySQL server the query needs to be executed. A minimalistic example is to send all reads to slaves, and writes to master. Of course, this example is not to be used in production, because sending reads to slaves may return stale data, and while stale data is ok for some queries, it is not ok for other queries. For this reason, we generally suggest to send all traffic to master, and thanks to the statistics that ProxySQL makes available, a DBA can create more precise routing rules in which only specific set of queries are routed to the slaves. More details and examples are available in a previous [blog post](http://www.proxysql.com/blog/configure-read-write-split). 24 | 25 | 在ProxySQL支持高客制化的路由规则设定，甚至可以通过`next_query_flagIN`参数指定当前查询连接的下一个查询仍然在同一组(hostgroup)实例上执行，比如可以通过这个特性，指定插入数据语句的后面的查询语句仍然在同一个实例上被执行。但只有在DBA对应用逻辑非常清楚的情况下，才能知道哪些功能或者查询是先写后查的，因此这个特性实际应用起来比较复杂。 26 | 27 | ProxySQL routing can be customized a lot: it is even possible to use `next_query_flagIN` to specific that after a specific query, the next query (or the next set of queries) should be executed on a determined hostgroup. For example, you can specify that after a specific `INSERT`, the following `SELECT` should be executed on the same server. Although this solution is advanced, it is also complex to configure because the DBA should know the application logic to determine which are the read-after-write queries. 28 | 29 | # Causal consistency reads 上下文一致性读 30 | 31 | 虽然某些应用对数据实时性要求极高，但其对上下文一致性读还是兼容的，即客户端可以读到同一个ProxySQL连接中被自己修改后的数据。这个特性在MySQL默认的异步复制结构是无法保证的，异步复制情况下，在主上提交写后，从库上有可能因为传输时间占用或者执行速度的差异导致客户端并不能同时读到刚刚修改的最新数据。 32 | 33 | Some application cannot work properly with stale data, but they can operate properly with causal consistency reads: this happens when a client is able to read the data that has written. Standard MySQL asynchronous replication cannot guarantee that. In fact, if a client writes on master, and immediately tries to read it from a slave, it is possible that the data is yet not replicated to slave. 34 | 35 | 那么我们如何保证只有当写事件被完全同步到从库后，查询才会由ProxySQL路由到从库呢？ 36 | 37 | How can we guarantee that a query is executed on slave only if a certain write event was already replicated? 38 | 39 | 基于GTID 40 | 41 | GTID helps in this. 42 | 43 | 在MySQL 5.7.5更新后，客户端可以知道其最后写入事务的GTID，而且可以在任何当前GTID代表的事务已经被执行的从库上进行读操作。Lefred 在博客中举例描述了这个过程，如下：MySQL实例服务端开启session_track_gtids参数（这是个覆盖全局和线程级的参数，用于返回当前事务成功执行的标记和GTID编号）后，客户端就可以在从库上使用`SELECT WAIT_FOR_EXECUTED_GTID_SET('3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5')`的函数来判断刚刚在主库执行的写事务是否在从库上已经被执行。 44 | 45 | Since MySQL 5.7.5 , a client can know (in the OK packet returned by MySQL Server) which is the GTID of its last write, and can execute a read on any slave where that GTID is already executed. Lefred described the process in a [blog post](http://lefred.be/content/mysql-group-replication-read-your-own-write-across-the-group/) with an example: the server needs to have [session_track_gtids](https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_session_track_gtids) enabled, and the client can execute [WAIT_FOR_EXECUTED_GTID_SET](https://dev.mysql.com/doc/refman/5.7/en/gtid-functions.html#function_wait-for-executed-gtid-set) on a slave and wait for the replication to apply the event on slave. 46 | 47 | 虽然可以通过这种操作来保证上下文一致性读，但由于以下原因，这个方法对生产环境来说还是比较麻烦和不实用的： 48 | 49 | Although this solution works, it is very trivial and not usable in production, mostly for the following reasons/drawbacks: 50 | 51 | ​ 在从库上执行业务查询时每次都要加上WAIT_FOR_EXECUTED_GTID_SET的话，会增加单个查询的响应时间 52 | 53 | - executing a query with WAIT_FOR_EXECUTED_GTID_SET before the real query will add latency before any real query 54 | 55 | 出于效率，若想从若干个从实例中找到已经执行完指定GTID的实例，可能需要在所有的从库上都执行一遍 56 | 57 | - the client doesn’t know which slave will be in sync first, so it needs to check many/all slaves 58 | 59 | 很可能所有的从库在可接受的延迟等待时间内都没有完成该GTID的同步 60 | 61 | - it is possible that none of the slaves will be in sync in an acceptable amount of time (will you wait few seconds before running the query?) 62 | 63 | 也就是说上述的操作在生产环境中并不实用 64 | 65 | That being said, the above solution is not usable in production and is mostly a POC. 66 | 67 | # Can ProxySQL tracks GTID? ProxySQL 可以追踪GTID吗? 68 | 69 | 由于ProxySQL饰演着MySQL客户端的角色，当`session_track_gtids` 开启后，ProxySQL 也可以跟踪所有前端发来的所有请求的GTID，而且可以准确的获知每个前端连接最后的GTID值。这样就可以使用这些信息去路由读请求到正确的从实例（指已经执行完某个线程指定的GTID事务的从实例）上。那么，ProxySQL 怎么追踪指定的GTID是否在从库上已经被执行呢？ 70 | 71 | ProxySQL acts as client for MySQL . Therefore, if `session_track_gtids` is enabled, ProxySQL can track the GTID of all the clients’ requests, and know exactly the last GTID for each client’s connection. ProxySQL can then use this information to send a read to the right slave where the GTID was already executed. How can ProxySQL track GTID executed on slaves? 72 | 73 | 大概分为两种办法： 74 | 75 | There are mostly two approaches: 76 | 77 | ​ 主动问询：ProxySQL 定期的查询所有实例的GTID执行情况 78 | 79 | - Pull : at regular interval, ProxySQL queries all the MySQL servers to retrieve the GTID executed set 80 | 81 | 聆听告知，每当一个新的写事件产生且GTID编号被生成后，ProxySQL 会立刻接到告知 82 | 83 | - Push : every time a new write event is executed on a MySQL server (master or slave) and a new GTID is generated, ProxySQL is immediately notified 84 | 85 | 需要注意的是，由于每次问询之间总是有一定的间隔，导致主动问询方式总是不可避免的存在着基于问询间隔的延迟，问询的越频繁，获取的信息就越准确，但会增加MySQL实例的负载，且当ProxySQL实例过多时会占用查询带宽和流量。总而言之，理论上来说，这种办法效率又低，架构可扩展性又差。 86 | 87 | No need to say, the pull method is not very efficient because it will almost surely introduce latency based on how frequently ProxySQL will query the MySQL servers to retrieve the GTID executed set. The less frequent the check, the less accurate it will be. The more frequent the check, the more precise, yet it will cause load on MySQL servers and inefficiently use a lot of bandwidth if there are hundreds of ProxySQL instances. In other words, this solution is not efficient neither scalable. 88 | 89 | 那么聆听告知的情况 90 | 91 | What about pull method? 92 | 93 | # Real time retrieval of GTID executed set 实时获取已经执行过的GTID值 94 | 95 | 从技术上来说获取当前已经执行过的GTID 值很简单，只要实时消费（分析）binlog即可。但是这种方法需要把ProxySQL实例所在的机器模拟成一个从库，如若单个ProxySQL负载了很多个MySQL实例，那么势必会对提升CPU的消耗。更进一步，如果一个机柜或者交换机上部署了很多ProxySQL 实例，那么传输binlog也会对整个网络的带宽带来考验。举个例子，现有4个集群，每个集群中的主库每天产生40GB的binlog并且挂了5个从库，附加30个ProxySQL实例，那么，每个ProxySQL实例需要把自己模拟成24个主从实例的从库。总计每天要消耗30TB的网络带宽。对自建机房的话或许可以直接纵向或者横向加硬件解决，但对云主机来说，每天将会无法避免巨额的流量费用。 96 | 97 | Real time retrieval of GTID executed set is technically simple: consume and parse binlog in real time! Although, if ProxySQL becomes a slave of every MySQL server, it is easy to conclude that this solution will consume CPU resources on ProxySQL instance. To make things worse, in a setup with a lot of ProxySQL instances, if each ProxySQL instance needs to get replication events via replication from every MySQL server, network bandwidth will soon become a bottleneck. For example, think what will happen if you have 4 clusters, and each cluster has 1 master generating 40GB of binlog per day and 5 slaves, and a total of 30 proxysql instances. If each proxysql instance needs to become a slave of the 24 MySQL servers, this solution will consume nearly 30TB of network bandwidth in 1 day (and you don’t want this if you pay for bandwidth usage). 98 | 99 | # ProxySQL Binlog Reader 100 | 101 | 主动问询GTID执行情况在上个段落中被认为是没有扩展性，且消耗较大的资源的方法。ProxySQL Binlog Reader工具应运而生： 102 | 103 | The pull method described above doesn’t scale and it consumes too many resources. For this reason, a new tool was implemented: ProxySQL Binlog Reader. 104 | 105 | ​ ProxySQL Binlog Reader是一个轻型，运行在MySQL实例机器上，且通过把自己模拟成一个从实例连接到MySQL实例的跟踪所有GTID事件的进程。 106 | 107 | - ProxySQL Binlog Reader is a lightweight process that run on the MySQL server, it connects to the MySQL server as a slave, and tracks all the GTID events. 108 | 109 | ProxySQL Binlog Reader 本身就是一个服务器，每当前端连接进入时，就会开始以一个高效节省带宽的方式进行Binlog的流传输。 110 | 111 | - ProxySQL Binlog Reader is itself a server: when a client connects to it, it will start streaming GTID in a very efficient format to reduce bandwidth. 112 | 113 | 说到这里，我想聪明的您应该猜到了ProxySQL实例饰演着ProxySQL Binlog Reader服务的客户端的角色。 114 | 115 | By now you can easily guess who will be the clients of ProxySQL Binlog Reader: all the ProxySQL instances. 116 | 117 | ![ProxySQL Binlog Reader](http://www.proxysql.com/content/2-blog/34-proxysql-gtid-causal-reads/proxysql-binlog-reader.png) 118 | 119 | # Real time routing 实时路由 120 | 121 | ProxySQL Binlog Reader 让ProxySQL 可以知道每个MySQL实例当前GTID 的执行情况。那么，当客户端执行一条读写分离查询时，ProxySQL就会马上知道这个请求该被路由到哪台从服务器上。退一步说，即使当前所有的从实例都没有完成该GTID的执行，那么ProxySQL 也会明白这是个主写，从读的查询。 122 | 123 | ProxySQL Binlog Reader allows ProxySQL to know in real time which GTID was been executed on every MySQL server, slaves and master itself. Thanks to this, when a client executes a reads that needs to provide causal consistency reads, ProxySQL immediately knows on which server the query can be executed. If for whatever reason the writes was not executed on any slave yet, ProxySQL will know that the write was executed on master and send the read there. 124 | 125 | ![ProxySQL GTID Consistency](http://www.proxysql.com/content/2-blog/34-proxysql-gtid-causal-reads/proxysql-gtid.png) 126 | 127 | # Advanced configuration, and support for many clusters 细致的设置项，支持多集群 128 | 129 | ProxySQL是高客制化的，本文所介绍的功能同样如此。最重要的就是可以设置读请求是否支持上下文一致性（以ProxySQL中的组为对象）。不能设置完上下文一致性就完事大吉，例如，针对一个读请求，需要指定B组对A组满足上下文一致性（这里的组指的是ProxySQL中的Hostgroup，A组应该是写组，B组则可写，可读，只要保证同一连接如果在A上写，但读被路由到B的时候，能够读到刚刚写入的操作即可）。不仅支持主从之间上下文一致性读，还支持切片集群A+切片集群B构成的A-B高可用架构中，上下文一致性读查询（被路由到A或B中）， 130 | 131 | ProxySQL is extremely configurable, and this is true also for this feature. The most important tuning is that you can configure if a read should provide causal consistency or not, and if causal consistency is required you need to specify to which hostgroup should be consistent. This last detail is very important: you don’t simply enable causal consistency, but you need to specify that a read to hostgroup B should be causal consistent to hostgroup A. This allows ProxySQL to implement the algorithm on any number of hostgroups and clusters, and also allows a single client to execute queries on multiple clusters (sharding) knowing that the causal consistency read will be executed on the right cluster. 132 | 133 | # Requirements 要求 134 | 135 | 基于GTID的上下文一致性读需满足如下条件： 136 | 137 | Casual reads using GTID is only possible if: 138 | 139 | ​ ProxySQL 2.0 以后的版本 140 | 141 | - ProxySQL 2.0 is used (older versions do not support it) 142 | 143 | 后端使用MySQL 5.7.5以上的版本，老版本不支持`session_track_gtids` 144 | 145 | - the backend is MySQL version 5.7.5 or newer. Older versions of MySQL do not have capability to use this functionality. 146 | 147 | binlog格式为行格式 148 | 149 | - replication binlog format is ROW 150 | 151 | 开启GTID 152 | 153 | - GTID is enabled (that sounds almost obvious). 154 | 155 | 后端仅限于Oracle或者Percona分支的MySQL，MariaDB不支持`session_track_gtids` 156 | 157 | - backends are either Oracle’s or Percona’s MySQL Server: MariaDB Server does not support `session_track_gtids`, but I hope it will be available soon. 158 | 159 | # Conclusion 结论 160 | 161 | 马上就要发布的ProxySQL 2.0版本可以实时追踪复制架构中中各个实例当前的GTID执行情况。基于GITD的自适应查询路由可以满足上下文一致性读，ProxySQL 可以路由查询请求到指定的GTID已经被执行的从实例。本解决方案扩展性极佳，网络资源占用低，且已经在实际环境中进行验证。 162 | 163 | The upcoming release of ProxySQL 2.0 is able to track executed GTID in real-time from all the MySQL servers in a replication topology. Adaptive query routing based on GTID tracking allows to provide causal reads, and ProxySQL can route reads to the slave where the needed GTID event was already executed. This solutions scales very well with limited network usage, and is being already tested in production environments. -------------------------------------------------------------------------------- /技术文章翻译/【MySQL】【翻译】ProxySQL新版本对MGR的原生支持.md: -------------------------------------------------------------------------------- 1 | # MySQL Group Replication: native support in ProxySQL【ProxySQL新版本对MGR的原生支持】 2 | 3 | Posted by [lefred](http://lefred.be/content/author/lefred/) on March 23, 2017 4 | 5 | 作者：[lefred](http://lefred.be/content/author/lefred/) MySQL官方开发组成员 6 | 7 | 翻译：张锐志，知数堂学员 8 | 9 | [ProxySQL](http://www.proxysql.com/) is the leader in proxy and load balancing solution for MySQL. It has great [features](http://www.proxysql.com/#features) like query caching, multiplexing, mirroring, read/write splitting, routing, etc… The latest enhancement in ProxySQL is the **native support** of **MySQL Group Replication**. No more need to use an external script within the scheduler like I explained in [this previous post](http://lefred.be/content/ha-with-mysql-group-replication-and-proxysql/). 10 | 11 | ProxySQL在MySQL的代理和负载均衡中一直处于领先地位。其中包含了诸如缓存查询，多路复用，流量镜像，读写分离，路由等等的强力功能。在最新的功能性增强中，包含了对MGR的原生支持，不在需要使用第三方脚本进行适配。 12 | 13 | This implementation supports Groups in Single-Primary and in Multi-Primary mode. It is even possible to setup a Multi-Primary Group but dedicate writes on only one member. 14 | 15 | 最新的增强中，提供了对单写和多写集群组的支持，甚至可以在多写组上指定只由某个成员进行写入操作。 16 | 17 | [René](https://twitter.com/rene_cannao), the main developer of ProxySQL, went even further. For example in a 7 nodes clusters (Group of 7 members) where all nodes are writers (Multi-Primary mode), it’s possible to decide to have only 2 writers, 3 readers and 2 backup-writers. This mean that ProxySQL will see all the nodes as possible writers but will only route writes on 2 nodes (add them in the writer hostgroup, because we decided to limit it to 2 writers for example), then it will add the others in the backup-writers group, this group defines the other writer candidates. An finally add 2 in the readers hostgroup. 18 | 19 | ProxySQL的主要开发者René，更进一步的可以（利用ProxySQL）做到例如在一个七个节点的多写集群中，指定2组写节点，2组备用写节点，3个只读节点的操作。即ProxySQL虽然识别出来所有的节点皆为写节点，但只路由写操作到选定的两个写节点（通过Hostgroup的方式），同时将另外两个写节点添加到备用写节点组中，最后三个读节点加入读组。（本段中的组皆为ProxySQL中的hostgroup含义）。 20 | 21 | It’s also possible to limit the access to a member that is slower in applying the replicated transactions (applying queue reaching a threshold). 22 | 23 | 除此之外，还可以限制连接访问集群中超出最大设定落后事务值的慢节点。 24 | 25 | It is time to have a look at this new ProxySQL version. The version supporting MySQL Group Replication is 1.4.0 and currently is only available on [github](https://github.com/sysown/proxysql/tree/v1.4.0-GR) (but stay tuned for a new release soon). 26 | 27 | ProxySQL从1.4.0版本开始增加对MGR的原生支持，若发行版中没有，可以从GitHub中编译获取。 28 | 29 | So let’s have a look at what is new for users. When you connect to the admin interface of ProxySQL, you can see a new table: `mysql_group_replication_hostgroups` 30 | 31 | 下面我们看下对于用户来说有哪些明显的变化，开始进行admin端口连接后会发现比之前多了一个`mysql_group_replication_hostgroups`表 32 | 33 | ``` 34 | ProxySQL> show tables ; 35 | +--------------------------------------------+ 36 | | tables | 37 | +--------------------------------------------+ 38 | | global_variables | 39 | | mysql_collations | 40 | | mysql_group_replication_hostgroups | 41 | | mysql_query_rules | 42 | | mysql_replication_hostgroups | 43 | | mysql_servers | 44 | | mysql_users | 45 | ... 46 | | scheduler | 47 | +--------------------------------------------+ 48 | 15 rows in set (0.00 sec) 49 | 50 | ``` 51 | 52 | This is the table we will use to setup in which hostgroup a node will belongs. 53 | 54 | 我们将在这个表中进行节点的归属组（hostgroup）的设置。 55 | 56 | To illustrate how ProxySQL supports MySQL Group Replication, I will use a cluster of 3 nodes: 57 | 58 | 为了阐明ProxySQL 对MGR支持的原理，下面我会用到一个三节点的集群。 59 | 60 | | name | ip | 61 | | ------ | ------------ | 62 | | mysql1 | 192.168.90.2 | 63 | | mysql2 | 192.168.90.3 | 64 | | mysql3 | 192.168.90.4 | 65 | 66 | So first, as usual we need to add our 3 members into the `mysql_servers` table: 67 | 68 | 首先，我们照旧插入三个节点的信息到`mysql_servers`表中。 69 | 70 | ``` 71 | mysql> insert into mysql_servers (hostgroup_id,hostname,port) values (2,'192.168.90.2',3306); 72 | Query OK, 1 row affected (0.00 sec) 73 | 74 | mysql> insert into mysql_servers (hostgroup_id,hostname,port) values (2,'192.168.90.3',3306); 75 | Query OK, 1 row affected (0.00 sec) 76 | 77 | mysql> insert into mysql_servers (hostgroup_id,hostname,port) values (2,'192.168.90.4',3306); 78 | Query OK, 1 row affected (0.00 sec) 79 | 80 | 81 | mysql> select * from mysql_servers; 82 | +--------------+--------------+------+--------+--------+-------------+-----------------+---------------------+---------+----------------+---------+ 83 | | hostgroup_id | hostname | port | status | weight | compression | max_connections | max_replication_lag | use_ssl | max_latency_ms | comment | 84 | +--------------+--------------+------+--------+--------+-------------+-----------------+---------------------+---------+----------------+---------+ 85 | | 2 | 192.168.90.2 | 3306 | ONLINE | 1 | 0 | 1000 | 0 | 0 | 0 | | 86 | | 2 | 192.168.90.3 | 3306 | ONLINE | 1 | 0 | 1000 | 0 | 0 | 0 | | 87 | | 2 | 192.168.90.4 | 3306 | ONLINE | 1 | 0 | 1000 | 0 | 0 | 0 | | 88 | +--------------+--------------+------+--------+--------+-------------+-----------------+---------------------+---------+----------------+---------+ 89 | 90 | ``` 91 | 92 | Now we can setup ProxySQL’s behavior with our Group Replication cluster, but before let’s check the definition of the new `mysql_group_replication_hostgroups` table: 93 | 94 | 在设置MGR节点在ProxySQL中的行为之前，先查看下新加入的`mysql_group_replication_hostgroups`表的DDL。 95 | 96 | ``` 97 | ProxySQL> show create table mysql_group_replication_hostgroups\G 98 | *************************** 1. row *************************** 99 | table: mysql_group_replication_hostgroups 100 | Create Table: CREATE TABLE mysql_group_replication_hostgroups ( 101 | writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY, 102 | backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL, 103 | reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0), 104 | offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0), 105 | active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1, 106 | max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1, 107 | writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0, 108 | max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0, 109 | comment VARCHAR, 110 | UNIQUE (reader_hostgroup), 111 | UNIQUE (offline_hostgroup), 112 | UNIQUE (backup_writer_hostgroup)) 113 | 114 | ``` 115 | 116 | There are many new columns, let’s have a look at their meaning: 117 | 118 | 看一下之前没有出现过的新列的含义 119 | 120 | | Column Name | Description | 121 | | ----------------------- | ------------------------------------------------------------ | 122 | | writer_hostgroup | the id of the hostgroup that will contain all the members that are writer MGR写节点都应被包含在这个组中 | 123 | | backup_writer_hostgroup | if the group is running in multi-primary mode, there are multi writers (read_only=0) but if the amount of these writer is larger than the max_writers, the extra nodes are located in that backup writer group 在MGR多写的模式下，如果可以提供写属性的节点超过实际使用的写节点数，剩下的节点将在这个备用写节点组中存放。 | 124 | | reader_hostgroup | the id of the hostgroup that will contain all the members in read_only 该组将会包含所有具有只读属性的MGR节点 | 125 | | offline_hostgroup | the id of the hostgroup that will contain the host not being online or not being part of the Group 改组将会包含所有无法提供服务或者不处于online情况下的节点 | 126 | | active | when enabled, ProxySQL monitors the Group and move the server according in the appropriate hostgroups 当该列属性启动时，ProxySQL将会监察整个集权，并根据hostgroup和节点的属性，进行匹配。 | 127 | | max_writers | limit the amount of nodes in the writer hostgroup in case of group in multi-primary mode 控制MGR多写模式下实际对外提供写服务的节点数量 | 128 | | writer_is_also_reader | boolean value, 0 or 1, when enabled, a node in the writer hostgroup will also belongs the the reader hostgroup 布尔值0或1，当启动时写节点组中的节点会同时出现在读组中 | 129 | | max_transactions_behind | if the value is greater than 0, it defines how much a node can be lagging in applying the transactions from the Group, [see this post for more info](http://lefred.be/content/mysql-group-replication-synchronous-or-asynchronous-replication/) 定义节点最大落后整个集群的事务数量（ProxySQL内部，非MGR中的） | 130 | 131 | Now that we are (or should be) more familiar with that table, we will set it up like this: 132 | 133 | 熟悉了表的定义后，整个拓补将会如下图所示： 134 | 135 | ![img](http://lefred.be/wp-content/uploads/2017/03/proxySQL_GR_hostgroup.png) 136 | 137 | So let’s add this: 138 | 139 | 下面我们将MGR集群的分组定义和关键参数写入`mysql_group_replication_hostgroups`表中 140 | 141 | ``` 142 | ProxySQL> insert into mysql_group_replication_hostgroups (writer_hostgroup,backup_writer_hostgroup, 143 | reader_hostgroup, offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind) 144 | values (2,4,3,1,1,1,0,100); 145 | 146 | ``` 147 | 148 | We should not forget to save our mysql servers to disk and load them on runtime: 149 | 150 | 然后将新更改的配置保存到磁盘上，并加载到运行环境。 151 | 152 | ``` 153 | ProxySQL> save mysql servers to disk; 154 | Query OK, 0 rows affected (0.01 sec) 155 | 156 | ProxySQL> load mysql servers to runtime; 157 | Query OK, 0 rows affected (0.00 sec) 158 | 159 | ``` 160 | 161 | It’s also important with the current version of MySQL Group Replication to add a view and its dependencies in sys schema: [addition_to_sys.sql](https://gist.github.com/lefred/77ddbde301c72535381ae7af9f968322): 162 | 163 | 同时，我们需要在MGR中添加如下的视图，及其依赖的存储过程。 164 | 165 | ``` 166 | # mysql -p < addition_to_sys.sql 167 | ``` 168 | 169 | So now from every members of the group, we can run the following statement. ProxySQL based its internal monitoring this same view: 170 | 171 | 如此，我们便可以从MGR集群中任意一个节点上执行下面的语句获取MGR成员的基本信息，ProxySQL 也是根据这个办法进行监测节点的健康与落后情况。 172 | 173 | ``` 174 | mysql> select * from gr_member_routing_candidate_status; 175 | +------------------+-----------+---------------------+----------------------+ 176 | | viable_candidate | read_only | transactions_behind | transactions_to_cert | 177 | +------------------+-----------+---------------------+----------------------+ 178 | | YES | YES | 40 | 0 | 179 | +------------------+-----------+---------------------+----------------------+ 180 | 181 | ``` 182 | 183 | We also must not forget to create in our cluster the ** monitor user needed by ProxySQL**: 184 | 185 | 同时，我们需要讲sys库的读权限赋给ProxySQL配置的监控MySQL的账户： 186 | 187 | ``` 188 | mysql> GRANT SELECT on sys.* to 'monitor'@'%' identified by 'monitor'; 189 | 190 | ``` 191 | 192 | We can immediately check how ProxySQL has distributed the servers in the hostgroups : 193 | 194 | 接下来，我们马上检查下ProxySQL是如何将MGR节点分发到ProxySQL各个组中： 195 | 196 | ``` 197 | ProxySQL> select hostgroup_id, hostname, status from runtime_mysql_servers; 198 | +--------------+--------------+--------+ 199 | | hostgroup_id | hostname | status | 200 | +--------------+--------------+--------+ 201 | | 2 | 192.168.90.2 | ONLINE | 202 | | 3 | 192.168.90.3 | ONLINE | 203 | | 3 | 192.168.90.4 | ONLINE | 204 | +--------------+--------------+--------+ 205 | 206 | ``` 207 | 208 | The Writer (Primary-Master) is mysql1 (192.168.90.2 in hostgroup 2) and the others are in the read hostgroup (id=3). 209 | 210 | 写节点被分配到之前定义好的ID为2的写组中，其他所有的节点被分配到ID为3的只读组中。（单写模式） 211 | 212 | As you can see, there is no more need to create a scheduler calling an external script with complex rules to move the servers in the right hostgroup. 213 | 214 | 这样，我们就省掉了通过定时器去调用第三方复杂定义的脚本将MGR节点匹配并分配到对应的ProxySQL组中的操作。 215 | 216 | Now to use the proxy, it’s exactly as usual, you need to create users associated to default hostgroup or add routing rules. 217 | 218 | 接下来，你就可以按照之前的做法对ProxySQL进行配置，例如关联用户到默认ProxySQL组中，或者添加查询路由规则。 219 | 220 | An extra table has also been added for monitoring: 221 | 222 | 另外，ProxySQL比之前多了一个监控MySQL实例的表，具体信息如下面所示： 223 | 224 | ``` 225 | ProxySQL> SHOW TABLES FROM monitor ; 226 | +------------------------------------+ 227 | | tables | 228 | +------------------------------------+ 229 | | mysql_server_connect | 230 | | mysql_server_connect_log | 231 | | mysql_server_group_replication_log | 232 | | mysql_server_ping | 233 | | mysql_server_ping_log | 234 | | mysql_server_read_only_log | 235 | | mysql_server_replication_lag_log | 236 | +------------------------------------+ 237 | 7 rows in set (0.00 sec) 238 | 239 | ProxySQL> select * from mysql_server_group_replication_log order by time_start_us desc limit 5 ; 240 | +--------------+------+------------------+-----------------+------------------+-----------+---------------------+-------+ 241 | | hostname | port | time_start_us | success_time_us | viable_candidate | read_only | transactions_behind | error | 242 | +--------------+------+------------------+-----------------+------------------+-----------+---------------------+-------+ 243 | | 192.168.90.4 | 3306 | 1490187314429511 | 1887 | YES | NO | 0 | NULL | 244 | | 192.168.90.3 | 3306 | 1490187314429141 | 1378 | YES | YES | 0 | NULL | 245 | | 192.168.90.2 | 3306 | 1490187314428743 | 1478 | NO | NO | 0 | NULL | 246 | | 192.168.90.4 | 3306 | 1490187309406886 | 3639 | YES | NO | 0 | NULL | 247 | | 192.168.90.3 | 3306 | 1490187309406486 | 2444 | YES | YES | 0 | NULL | 248 | +--------------+------+------------------+-----------------+------------------+-----------+---------------------+-------+ 249 | 250 | ``` 251 | 252 | Enjoy MySQL Group Replication & ProxySQL ! 253 | 254 | #### follow me 255 | 256 | [Follow @lefred](https://twitter.com/intent/follow?screen_name=lefred) 257 | 258 | [译者： @naughtyGitCat](https://github.com/naughtyGitCat) -------------------------------------------------------------------------------- /求职/面试总结.md: -------------------------------------------------------------------------------- 1 | 面试总结： 2 | 3 | ​ [持续更新地址](https://github.com/naughtyGitCat/ToolsForMySQL/blob/master/%E6%B1%82%E8%81%8C/%E9%9D%A2%E8%AF%95%E6%80%BB%E7%BB%93.md) 4 | 5 | [TOC] 6 | 7 | ## 基本盘： 8 | 9 | ​ 在原公司半年自己的成长与大环境相比，退步 10 | 11 | ​ 没有接触过分库分表等进阶技术。 12 | 13 | ​ 目标城市：杭州，南京，目标：初级>中级 14 | 15 | ## 面试难点： 16 | 17 | ### 1.原公司的业务量，会仔细问到每个表的大小，若有意掺水，可以先计算好。 18 | 19 | ### 2.慢查询思路 20 | 21 | ​ 系统： 22 | 23 | ​ PMM监控系统 24 | 25 | ​ 每周切割一次慢日志，pt-query-digest分析后，发送到DBA组邮箱 26 | 27 | ​ 优化： 28 | 29 | ​ 先看执行计划，观察是否用尽可能用到索引 30 | 31 | ​ 同时要关注预计扫描的行数，以及是否产生了临时表，或则是否需要进行排序 32 | 33 | #### show processlist中需要关注的项： 34 | 35 | ​ copy to tmp table，建议把alter table操作放到低峰期运行 36 | 37 | ​ Copying to tmp table ，拷贝数据到内存中的临时表，常见于group by操作，加索引 38 | 39 | ​ Copying to tmp table on disk，临时表结果集太大，需要转换成临时表，增大sort buffer和tmp_table_size 40 | 41 | ​ Creating sort index，当前的SELECT需要用到临时表进行order by，建议加索引 42 | 43 | ​ Creating tmp table，建议创建适当的索引，少用UNION,VIEW,SUBQUERY 44 | 45 | ​ Sending data，从硬盘读取数据发送给客户端，建议通过索引，或者加上Limit，减少需要扫描并发送给客户端的数量 46 | 47 | ​ Sorting Result：正在对结果进行排序，类似Creating sort index，但是是在表内，而不是在临时表中，建议加索引。 48 | 49 | #### explain中需要关注的项： 50 | 51 | ##### select type： 52 | 53 | ​ primary ，主键查询 54 | 55 | ​ simply，无表连接和子查询 56 | 57 | ​ UNION，出现在UNION后面的子句 58 | 59 | ​ SUBQUERY，子查询的语句 60 | 61 | ##### type（MySQL在子查询中找到所需行的方式）： 62 | 63 | ​ all，全表扫描，遍历整个表 64 | 65 | ​ index，索引全扫描，遍历整个列 66 | 67 | ​ range，索引范围扫描，（range,between,><） 68 | 69 | ​ ref，使用非唯一索引，或者复合的唯一索引的前缀列进行扫描 70 | 71 | ​ eq_ref，使用唯一索引（多表连接中使用主键或者唯一性索引关联） 72 | 73 | ​ const/system，单表使用主键或者唯一性索引进行查询 74 | 75 | ​ NULL，查询常量或者不设计表中数据 76 | 77 | #### extra(附加信息)： 78 | 79 | ​ using index，使用覆盖索引的时候会出现 80 | 81 | ​ using where，全索引扫描或者全表扫描(type列为ALL或者 index), 又加上where条件 82 | 83 | ​ using index condition 新特性ICP，把过滤推到引擎层处理 84 | 85 | #### SQL解析顺序 86 | 87 | ​ 8 SELECT 9 DISTINCT *selected column* 88 | 89 | ​ 1 FROM *left table* 90 | 91 | ​ 3 *join type* JOIN *right table* 92 | 93 | ​ 2 ON *join condition* 94 | 95 | ​ 4 WHERE *where condition* 96 | 97 | ​ 5 GROUP BY *group by list* 98 | 99 | ​ 6 HAVING *having condition* 100 | 101 | ​ 10 ORDER BY 11 LIMIT 102 | 103 | #### SQL改写点 104 | 105 | ​ 通常情况下，子查询的性能太差，建议改造成JOIN写法 106 | 107 | ​ 多表连接时，关联字段类型尽量一致，并且都要有索引；同时把过滤后结果集较小的表作为驱动表；排序字段应为驱动表的索引列 108 | 109 | ​ 类似分页功能的SQL，先用主键关联后，再返回结果集，效率会高很多 110 | 111 | ​ 多用复合索引，少用单列索引 112 | 113 | ​ varchar列不排序或者分组时，索引可以建成前缀索引 114 | 115 | ​ 116 | 117 | ### 3.高负载下MySQL的排查思路 118 | 119 | ​ 1.先确认系统本身有无问题，是不是确实从应用端发来了大量请求 120 | 121 | ​ 2.查看活跃线程数，活跃线程数陡升表示可能redis或者memcache缓存集穿 122 | 123 | ​ 3.看show processlist中有没有大SQL 124 | 125 | ​ 4.explain show processlist中的ID 126 | 127 | ​ 5.情况异常严重可以kill掉连接上面的查询或者这个线程，甚至禁用这个事务的对应的账号或者主机 128 | 129 | ​ 一般可能是不合理的SQL，或者没有用到合适的索引 130 | 131 | ### 4.docker/整个业务系统的纵向思考 132 | 133 | ​ docker:资源隔离。模板化，但是维护上可能更麻烦，且数据的安全性和性能更是个问题 134 | 135 | ### 5.B+树，索引，隔离级别 136 | 137 | #### InnoDB表的特点： 138 | 139 | ​ 基于B+树的索引表 140 | 141 | ​ 每个表都需要有一个聚集索引 142 | 143 | ​ 所有行记录都存储在B+树的叶子节点 144 | 145 | ​ 基于聚集索引的增删改查效率是最高的 146 | 147 | ​ 如果我们定义了主键，则InnoDB会选择其作为聚集索引 148 | 149 | ​ 如果没有显式定义主键，则InnoDB会选择第一个非空的唯一索引作为聚集索引 150 | 151 | ​ 如果上面的也不满足，则InnoDB会选择内置的6字节的ROWID作为隐含的索引。 152 | 153 | #### 综上所述： 154 | 155 | ​ 如果InnoDB表的数据写入顺序能和B+树节点顺序保持一致的话，这时候存取效率是最高的。 156 | 157 | ​ 1.使用自增列做主键，这时候写入顺序是自增的，和B+树叶子节点分裂顺序一致 158 | 159 | ​ 2.该表不指定自增列作主键，同时也没有可以选为主键的非空唯一索引，这时候InnoDB会选择内置的ROWID作为主键，写入顺序和ROWID增长顺序一致。 160 | 161 | ​ 3.若一个InnoDB表没有显式主键，却有可以被指定为主键的非空唯一性索引，但可能不是递增关系时（如字符串，UUID，多字段联合索引的情况），那么该表的存取效率就可能会比较差 162 | 163 | ​ page的结构是B+树，page内部则是链表 164 | 165 | #### InnoDB索引： 166 | 167 | ​ 主键索引，普通索引，普通索引通过存储主键列的值完成，本质上是组合索引。 168 | 169 | ​ 单列索引，组合索引，组合索引的最左匹配原则 170 | 171 | ​ 约束性索引，最常见的是非空索引 172 | 173 | ​ 自适应哈希索引，InnoDB会自动根据访问的频率和模式来为某些页建立哈希索引，本质是针对B+树Search Path的优化，所有涉及到Search Path的操作，均可以被优化 174 | 175 | #### 非InnoDB索引： 176 | 177 | ​ 哈希索引：存在于heap或者memory表中，只可以等值查询 178 | 179 | ​ 全文索引：不推荐使用 180 | 181 | ​ 空间索引：不经常使用 182 | 183 | #### 无法使用索引的场合： 184 | 185 | ​ 通过索引扫描的记录超过全表的30% 186 | 187 | ​ 联合索引中，无等值查询，第一个排序条件不是最左索引列 188 | 189 | ​ 字符串模糊查询以%开头，且结果列 超出联合索引的范围 190 | 191 | ​ 两表关联或者单表两列比较时的列的类型不同或字符长度不同导致的隐式转换 192 | 193 | ​ 两个索引列，一个用于检索，一个排序，无法使用Index merge 194 | 195 | #### 隔离级别： 196 | 197 | ​ RU>RC>RR>Seriallize 198 | 199 | ​ RC>RR：通过MVCC和间隙锁完成可重复读，间隙锁加在普通索引的范围的两端 200 | 201 | ​ RR开始一个事务时就会创建snapshot，RC隔离级别是在语句开始时刻创建read_view 202 | 203 | #### 锁： 204 | 205 | ​ 表锁，不使用索引时，超过表大小的30%时 206 | 207 | ​ 行锁，通过给索引上的索引值加锁实现的，只有通过索引检索，才会使用行锁 208 | 209 | ​ 元信息锁(meta_lock)，表级锁，互斥锁 210 | 211 | ​ 全局锁（FTWRL，服务层级别），自增锁（表级>线程级），自旋锁（引擎内部） 212 | 213 | ​ 意向锁：表级别，上在树根上，可以消灭DDL进行（加metalock）前的全表扫描表中锁。 214 | 215 | ​ 当发生锁冲突时，原本的隐式锁会变成显示锁 216 | 217 | ​ 查看锁：`show tables like "%lock%" from information_schema`; 218 | 219 | ​ 220 | 221 | ### 6.高可用 222 | 223 | #### GTID 224 | 225 | ​ 全局事务ID，根据GTID可以知道事务最初是在哪个实例上提交的，防止环回复制 226 | 227 | ​ 方便复制的故障转移 228 | 229 | #### 复制延迟 230 | 231 | ##### 原因： 232 | 233 | ​ 从库硬件差距 234 | 235 | ​ 从库单个SQL_THREAD回放 236 | 237 | ​ 从库不执行业务，导致IBP中没有真正的热数据，每次执行都会执行大几率磁盘读 238 | 239 | ##### 解决办法： 240 | 241 | ​ 给从库提升硬件性能，增大change buffer 242 | 243 | ​ 大事务拆分 244 | 245 | ​ 在主库上开启组提交（5.7默认开启），在从库上开启并行复制 246 | 247 | slave-parallel-type=LOGICAL_CLOCK 248 | slave-parallel-workers=4（HDD）或8（SSD） 249 | master_info_repository=TABLE 250 | 251 | ​ slave_preserve_commit_order=1 252 | relay_log_info_repository=TABLE 253 | relay_log_recovery=ON 254 | 255 | ​ 从库执行relay log时，走的主键索引，删除从库的普通索引可以加快，但需慎重 256 | 257 | ##### 落后多少秒，秒是怎么算出来的 258 | 259 | ​ Seconds_Behind_Master = io_thread.timestamp - sql_thread.timestamp 260 | 261 | ​ The value of Seconds_Behind_Master is based on the timestamps stored in events, which are 262 | preserved through replication. This means that if a master M1 is itself a slave of M0, any event from 263 | M1's binary log that originates from M0's binary log has M0's timestamp for that event. This enables 264 | MySQL to replicate TIMESTAMP successfully. However, the problem for Seconds_Behind_Master 265 | is that if M1 also receives direct updates from clients, the Seconds_Behind_Master value 266 | randomly fluctuates because sometimes the last event from M1 originates from M0 and sometimes is 267 | the result of a direct update on M1. 268 | 269 | ##### 大事务卡住时： 270 | 271 | ​ show slave status查看从库SLAVE执行到的位置 272 | 273 | ​ `mysqlbinlog -v --base64-output=decode-rows --start-position=Exec_Master_Log_Pos Relay_Master_Log_File |more ` 274 | 275 | ##### 死锁： 276 | 277 | ​ 加索引或者DDL重放 278 | 279 | #### 半同步复制 280 | 281 | ##### 参数：repl_semi_sync_master_wait_point 282 | 283 | ##### 选项： 284 | 285 | ​ after commit 286 | 287 | ​ 主库在执行完commit后，才会等待从库的ACK消息，这个时候若主库failover，且从库由于不可抗的原因没有收到传过来的数据时，这个已经在主库上commit的事务会彻底丢失，无法被恢复。 288 | 289 | ​ after sync 290 | 291 | ​ 主库在执行完，会等待从库ACK，然后进行commit，这个时候若主库failover,且从库由于不可抗的原因没有收到传过来的数据，这个事务由于没有被提交，防范了事务丢失。 292 | 293 | #### MHA 294 | 295 | ##### MHA的架构 296 | 297 | ##### MHA的切换原理 298 | 299 | ##### MHA的VIP挂载与卸除 300 | 301 | ​ 见我的博客 [【MySQL】【高可用】基于MHA架构的MySQL高可用故障自动切换架构](http://blog.51cto.com/l0vesql/2068659 ) 302 | 303 | [【MySQL】【高可用】从masterha_master_switch工具简单分析MHA的切换逻辑](http://blog.51cto.com/l0vesql/2060910 ) 304 | 305 | ##### MHA的防脑裂： 306 | 307 | ​ 在main.conf中指定的master_ip_failover_script或者shutdown_script中，写入ping所在网关，若不通就关闭实例的逻辑 308 | 309 | 310 | 311 | #### Keepalived 312 | 313 | ##### Keepalived判断MySQL存活的逻辑: 314 | 315 | ​ 1.判断机器本身是否存在mysqld进程 316 | 317 | ​ 2.根据配置的账号进行连接mysqld，是否能连上 318 | 319 | ##### 如何判断脑裂？ 320 | 321 | ​ 在上面的逻辑的最后加上ping网关的逻辑，若网关也ping不通则是本身网络有问题，不抢断VIP 322 | 323 | ##### 数据安全性？ 324 | 325 | ​ 半同步来确保，或者监控判断延时，总之相比MHA，少了补数据的操作。 326 | 327 | #### 发生切换时新主热数据的加载 328 | 329 | ​ 我认为热数据的加载应该定时在从库或者备库上执行，如若等到failover时再进行热数据加载，势必会增加故障恢复时间。 330 | 331 | ##### 热数据加载的两种方式 332 | 333 | ​ 1.观察SQL统计或者与开发讨论热表，按小时定期SELECT * 查出到IBP中 334 | 335 | ​ 2.定期在主库上执行`innodb_buffer_pool_dump_now=1 `,然后把数据目录中的`ib_buffer_pool`文件复制到备库或者从库的目录下覆盖，执行`innodb_buffer_pool_load_now =1`进行加载热数据。 336 | 337 | ​ 注意点：innodb_buffer_pool_dump_pct 控制dump出ibp热数据的比例，越大dump越多 338 | 339 | ​ `SHOW STATUS LIKE 'Innodb_buffer_pool_dump_status';`查看dump进度 340 | 341 | ​ `SHOW STATUS LIKE 'Innodb_buffer_pool_dump_status';`查看load进度 342 | 343 | ## 7.MySQL集群+分库分表+读写分离负载均衡 344 | 345 | ### 集群： 346 | 347 | #### MGR: 348 | 349 | ​ 组提交（GTID事务组传输）+多数派+GTID离散范围分配与使用+增强半同步 350 | 351 | #### PXC: 352 | 353 | ​ galera 写集（数据块传输）+ wrep全局编号集群内协调 354 | 355 | ​ galera：说出写集提交验证的机制 356 | 357 | ### 分库分表： 358 | 359 | #### 垂直拆分： 360 | 361 | ​ 优点：拆分后业务清晰，拆分规则明确 362 | 363 | ​ 系统之间整合或扩展容易 364 | 365 | ​ 数据维护简单 366 | 367 | ​ 缺点：部分业务表无法Join只能通过接口的方式解决 368 | 369 | ​ 受每种业务不同的限制存在单库性能瓶颈 370 | 371 | ​ 事务处理复杂 372 | 373 | #### 水平拆分： 374 | 375 | ​ 优点：拆分规则抽象好，join操作可以在数据库里面做 376 | 377 | ​ 不存在单库大数据，高并发的性能瓶颈 378 | 379 | ​ 应用端改造比较少 380 | 381 | ​ 提升来了系统的稳定性与负载能力 382 | 383 | ​ 缺点：拆分规则难以抽象 384 | 385 | ​ 分片事务一致性难以解决 386 | 387 | ​ 数据多次扩展维护量大 388 | 389 | ​ 跨库join性能差 390 | 391 | ### 切分原则： 392 | 393 | ​ 能不切分就不切分 394 | 395 | ​ 如果要切分一定要选择合适的切分规则，提前规划好 396 | 397 | ​ 数据切分尽量通过数据冗余或者表分组来降低跨库JOIN的可能 398 | 399 | ​ 由于中间件对数据JOIN实现的优劣难以把握，而且实现高性能难度极大，业务读取尽量少使用多表JOIN 400 | 401 | ### 中间件： 402 | 403 | ​ MyCat： 404 | 405 | ### 读写分离负载均衡 406 | 407 | ​ proxySQL： 408 | 409 | ## 8.重要参数 410 | 411 | #### 全局参数： 412 | 413 | ​ table_cache，每个线程可以打开表的数量 414 | 415 | ​ thread_cache_size，可以复用的缓存起来的线程数，对高并发有奇效 416 | 417 | ​ back_log，MySQL暂停回答新请求前，短时间多少个请求可以被存在栈中 418 | 419 | ​ **InnoDB_open_files**，MySQL可以同时打开的ibd文件数 420 | 421 | ​ InnoDB_additional_mem_pool_size**，设置了InnoDB引擎来存放数据字典信息以及一些内部数据结构的内存空间大小。当一个MySQL实例中的数据库对象非常多的的时候，需要调整该大小确保所有的数据都能放在内存中以提高访问效率。 422 | 423 | ​ InnoDB buffer pool，内存的50-70% 424 | 425 | ​ InnoDB log file size**，redo的大小，1GB 426 | 427 | ​ **InnoDB log buffer size**，redo buffer的大小，8-16M 428 | 429 | ​ **InnoDB max_dirty_page_pct**，脏页比，超出后会触发脏页落盘操作，50% 430 | 431 | ​ InnoDB lock wait timeout**，行锁等待时间，超出后会报错，若高并发，行锁等待严重，可减少此值 432 | 433 | #### 线程参数： 434 | 435 | ​ thread_stack**，分配给单个线程的内存大小，512KB 436 | 437 | ​ sort buffer，分配给每个线程处理排序 438 | 439 | ​ **read_buffer**，读入缓冲区大小，顺序扫描时用到，频繁扫描需加大 440 | 441 | ​ **read_rnd_buffer**，二级索引非顺序扫描时的读入缓冲区大小 442 | 443 | ​ join_buffer，表连接的缓冲 444 | 445 | ​ tmp_table_size，每个线程创建临时表的最大大小，超过这个大小后，会转成磁盘临时表 446 | 447 | ​ 注意：所有的线程级别的缓存加在一起x最大线程数+IBP<系统内存*90% 448 | 449 | #### SSD优化： 450 | 451 | ​ 随机IO文件，ibd表数据文件，独立的undo表空间，临时表空间文件 452 | 453 | ​ 顺序IO文件，InnoDB系统表空间（change buffer或者double write的原因），redo,undo 454 | 455 | ​ innodb_io_capacity，HDD设置1000以下，SSD设置2000以上，增加后，checkpoint间隔更短，脏数据落盘更频繁，以减少因脏集中落盘导致的系统性能下降 456 | 457 | ​ innodb_flush_neighbors，关闭，不刷新相邻的脏页 458 | 459 | ​ innodb_page_size，若SSD以4K为最小格式化页，则InnoDB默认的16K页会多刷12K的存储，造成超写，降低SSD寿命，建议两个数值保持一致 460 | 461 | ​ innodb_log_file_size，可用空间低于7/8时会触发checkpoint。SSD可以考虑减小 462 | 463 | ​ innodb_max_dirty_pages_pct，可以考虑减小。 464 | 465 | ​ 总之，原来因HDD盘随机写性能低所做的随机写合并为集中写的操作，在SSD上面可以考虑恢复或者减小集中写的大小。 466 | 467 | 468 | 469 | ​ 470 | 471 | ## 9.5.6与5.7的区别 472 | 473 | ​ 提供了对JSON的支持 474 | 475 | ​ 可以explain一个正在运行的SQL（explain show processlist） 476 | 477 | ​ ***在线修改buffer pool的大小*** 478 | 479 | ​ ***Online DDL的支持（不完全，只支持重命名索引，或者增加varchar长度）*** 480 | 481 | ​ 临时表只在当前会话中可见，生命周期为当前连接 482 | 483 | ​ ***支持多源、多线程复制、增强半同步复制、组复制（MGR）*** 484 | 485 | ​ sys库的加入 486 | 487 | ​ ***SQL MODE :Only full group by*** 488 | 489 | ## 10.监控模板 490 | 491 | ​ 个别公司会问有没有自定义zabbix的模板，别管。 492 | 493 | ​ 原理是通过在监控脚本里面配置账号密码，每隔5S，查询一次全局参数，然后分割得出各个参数项。 494 | 495 | ## 11.备份，mysqldump为什么要rr,savepoint的应用方式，xtrabackup的最后结束时间 496 | 497 | ​ rr:防止不可重复读，幻读，产生数据不一致， 498 | 499 | ​ savepoint:如果不执行rollback to savpoint sp，DDL操作会一直Hang住，执行该操作后，DDL操作可以继续执行了。总之，提高了DDL的并发性，不会阻塞在备份期间对以备份表的DDL操作 500 | 501 | ​ xtrabackup的备份文件实际时间为结束备份的时间 502 | 503 | #### 备份流程： 504 | 505 | ​ mysqldump:关闭实例上所有已经打开的表， 506 | 507 | ​ 加全局读锁 508 | 509 | ​ 设置线程的隔离级别为rr，防止幻读和不可重复读 510 | 511 | ​ 开启一致性快照事务 512 | 513 | ​ 获取复制信息 514 | 515 | ​ 释放全局读锁 516 | 517 | ​ 结合savepoint开始一个一个表的select * ，减少单个表的DDL等待时间 518 | 519 | ​ 结束，断开连接，原事务会自动停止 520 | 521 | ​ xtrabackup：生成一个xtrabackup.log的文件捕获redo文件 522 | 523 | ​ 拷贝InnoDB相关文件（表数据文件，ibdata1回滚空间） 524 | 525 | ​ 全局读锁 526 | 527 | ​ 拷贝InnoDB表结构文件frm,非事务表的全部文件 528 | 529 | ​ 获取Binlog位置 530 | 531 | ​ 释放全局读锁，停止xtrabackup.log 532 | 533 | ​ 534 | 535 | ​ 536 | 537 | ## 12.InnoDB线程，drop表的引擎内部动作。 538 | 539 | #### master thread 540 | 541 | ​ 刷新redo日志缓冲到磁盘（包含了未提交的事务），合并change buffer并刷新脏页到磁盘，删除无用的undo页 542 | 543 | ​ 产生一个检查点 544 | 545 | #### IO thread 546 | 547 | ​ insert buffer thread1个 548 | 549 | ​ Log buffer thread 1个 550 | 551 | ​ read thread 8个 552 | 553 | ​ write thread 8个 554 | 555 | #### Purge thread 556 | 557 | ​ 回收已经使用的undo页，支持多个Purge thread 558 | 559 | #### Page Cleaner Thread 560 | 561 | ​ MySQL5.6版本以上引入，将之前版本中脏页刷新的操作都放到单独的线程中完成，目的是为了减轻原Master thread的工作及对于用户查询线程的阻塞，进一步提升InnoDB的性能 562 | 563 | 564 | 565 | #### drop表的操作： 566 | 567 | ​ 2次锁定LRU list，分别进行清除自适应哈希索引和LRU和flush list。 568 | 569 | ## 13.JSON格式的支持 570 | 571 | ​ 5.7版本中增加了对JSON格式的支持，面试官问到是否有过尝试优化mysql中JSON的思路。 572 | 573 | ​ 答：只是有了解过，插入过数据。 574 | 575 | ​ 但是术业有专攻，JSON最好还是用MongoDB来存储。现阶段业界暂未出现MySQL替代MongoDB做存储的知名案例。等到成熟了再测试，现在进行测试，浪费学习的精力。 576 | 577 | ​ 贵公司为什么会考虑放弃MongoDB？ 578 | 579 | ## 14.InnoDB如何判断是否热数据，热数据是否在InnoDB buffer pool中？ 580 | 581 | ​ LRU List按照功能被划分为两部分：LRU_young与LRU_old，默认情况下，LRU_old为链表长度的3/8。 页面读取时(get/read ahead)，首先链入LRU_old的头部。页面第一次访问时(read/write)，从LRU_old链表移动到LRU_young的头部(整个LRU链表头) 582 | 583 | ​ 热数据，一秒一次，否则进入Lru淘汰链表 584 | 585 | 586 | 587 | ## 15.InnoDB crash recovery 588 | 589 | ​ 1.从redo log文件里面得到最后一次checkpoint发生的LSN 590 | 591 | ​ 2.从这个点开始应用重做日志 592 | 593 | ​ 3.进行undo，反做未提交的事务 594 | 595 | ​ 4.若开启了Binlog，那么在恢复过程中判断哪些事务未提交时会利用binlog 596 | 597 | ​ 5.若未开启Binlog，那么会拿redo log entry的LSN与这行日志对应被修改页的LSN进行比较，若页的LSN大于等于redo log entry，那么表示这个页是干净的，不需要被回滚掉 598 | 599 | ## 工作中遇到的问题（部分为虚构或者其他人的案例） 600 | 601 | ### 1.误删除一张大表 602 | 603 | #### 恢复方法： 604 | 605 | ​ 1.xtrabackup的备份拿出来，删除其他所有业务表，只剩下被误删除的一张表。apply-log 606 | 607 | ​ 2.挂到主库下，start slave until到被删除的时间点 608 | 609 | ​ 3.交换表空间 610 | 611 | ### 2.负载过高，报max_connection 612 | 613 | #### 描述： 614 | 615 | ​ 秒杀环境下，大量连接进来，新的连接无法连接数据库。 616 | 617 | #### 处理： 618 | 619 | ​ 1.不能切换到从库，从库IBP没有热数据，从库会无法连接的同时整个库卡死 620 | 621 | ​ 2.使用socket方式接入，更改最大并发连接数。 622 | 623 | ​ 3.告诉系统运维限制前端最大并发连接数。 624 | 625 | ### 3.误删ibdata1 626 | 627 | #### 描述： 628 | 629 | ​ 测试环境不规范，测试人员误删了ibdata1文件，删除后及时通知到我 630 | 631 | #### 处理： 632 | 633 | ​ 1.进入cli，上全局读锁 634 | 635 | ​ 2.找到mysqld所在的进程号 636 | 637 | ​ 3.按进程号到/proc/目录下将文件找回 638 | 639 | ​ 4.拷贝回原来的位置，释放读锁 640 | 641 | ### 4.强制关机或者杀进程导致无法启动 642 | 643 | #### 描述： 644 | 645 | ​ 启动时报错：`The log sequence numbers 842222557 and 842222557 in ibdata files do not match the log sequence number 856874471 in the ib_logfiles!` 646 | 647 | #### 处理： 648 | 649 | ​ 1.是因为ibdata1中的double write文件丢失或损坏导致的checksum校验失败 650 | 651 | ​ 2.double write负责保证脏页刷入的稳定性，无法打开double write buffer，导致无法启动 652 | 653 | ​ 3.`innodb_force_recovery=(0-6)`每个级别尝试进入基于redo和undo的innodb crash recovery，同时建议恢复过程中中关闭p_s，以加速恢复进程。 654 | 655 | ### 5.错误日志报page_cleaner使用了比预计更长的时间 656 | 657 | #### 描述： 658 | 659 | ```sh 660 | [Note] InnoDB: page_cleaner: 1000ms intended loop took 8351ms. The settings might not be optimal. (flushed=3507 and evicted=0, during the time.) 661 | ``` 662 | 663 | #### 分析思路与处理办法： 664 | 665 | ​ 1.`InnoDB: page_cleaner`说明问题出在InnoDB引擎上。 666 | 667 | ​ 2.脏页清理比预定的时间长了很多，问题可能是设定的时间过于严苛，或者刷新的速度过慢 668 | 669 | ​ 3.总的来说还是过快的脏页清理落盘操作和过低的硬盘IO性能之间的矛盾 670 | 671 | ​ 4.对于HDD来说，优化写入IO基本就是把离散随机的IO操作变成顺序的IO操作。 672 | 673 | ​ 5.将InnoDB_io_capacity的值由原来不合理的4000改成HDD适用的1000，引擎会自动增大脏页刷新和落盘的间隔，尽量将离散的IO顺序化，在HDD上提升写入性能。 674 | 675 | 676 | 677 | ​ 678 | 679 | ## 附： 680 | 681 | ​ 面了很多家之后感觉，每个公司考察DBA的侧重点都不同，有些地方疏漏遗忘在所难免，回来笔记即可。 682 | 683 | ​ 面试的时候，要问下对方的规模和组织情况，问有没有自动化运维平台，问下结果的截止日期。 684 | 685 | ​ 心态稳住，因为个人品格上的缺陷，加上之前的遗留问题，在一段时间内陷入抑郁状态。多谢各位兄弟的帮助，建议大家多跑步，[多听些歌](http://t3.kugou.com/song.html?id=4hBvE42t5V2 "世界第一等") 686 | 687 | ​ 688 | 689 | 690 | 691 | 692 | 693 | --------------------------------------------------------------------------------