├── README.md ├── mysql-benchmark ├── mysql.fio ├── sysbench-prepare-1.0.20.sh ├── sysbench-run-1.0.20.sh ├── sysbench05-oltp.sh ├── sysbench10-oltp.sh ├── sysbench10_prepare.sh └── tpcc-oltp.sh ├── mysql-blog └── 201707 │ ├── 20170721-table-join-column-data-type-should-be-same.md │ └── 20170726-mysql-8.0-new-index.md ├── mysql-notes ├── 1-MySQL安装部署规范.md └── mysql_db_design_guide.md ├── mysql-snapshot ├── db_pool.py ├── pip3.6-requirements.txt ├── readme.md ├── snapshot.py └── utils.py ├── mysql-tools ├── MySQL巡检怎么做？.md ├── README.md ├── check_mysql.py ├── check_security_v1.py ├── dba_login.sh ├── diskcheck_megacli.sh ├── dstat-plugins │ ├── README.md │ ├── dstat_innodb_checkpoint.py │ ├── dstat_innodb_list.py │ └── dstat_mysql5_rowlockwaits.py ├── ibd-analyzer.rb ├── monitor_error.py ├── mysql-toolkit-sql.md ├── mysql_sos ├── open-falcon │ ├── .DS_Store │ ├── 127.0.0.1_190409_innodb_status │ ├── 127.0.0.1_190409_processlist │ ├── README.md │ ├── metric示例.txt │ ├── mymon.cnf │ ├── mymon.py │ └── mymon_utils.py ├── ps_mem.py ├── sql.txt └── sql_result_between_two_instance.py ├── nginx └── blackhole.sh └── tpch ├── README.md ├── SQLs ├── tpch-ddl.sql ├── tpch_queries.sql ├── tpch_queries_01.sql ├── tpch_queries_02.sql ├── tpch_queries_03.sql ├── tpch_queries_04.sql ├── tpch_queries_05.sql ├── tpch_queries_06.sql ├── tpch_queries_07.sql ├── tpch_queries_08.sql ├── tpch_queries_09.sql ├── tpch_queries_10.sql ├── tpch_queries_11.sql ├── tpch_queries_12.sql ├── tpch_queries_13.sql ├── tpch_queries_14.sql ├── tpch_queries_15.sql ├── tpch_queries_16.sql ├── tpch_queries_17.sql ├── tpch_queries_18.sql ├── tpch_queries_19.sql ├── tpch_queries_20.sql ├── tpch_queries_21.sql └── tpch_queries_22.sql └── dbgen ├── makefile └── tpcd.h /README.md: -------------------------------------------------------------------------------- 1 | # mysqldba 2 | > MySQL DBA用得到的东西集锦 3 | 4 | ## mysql-snapshot 5 | > 定时采集系统状态、MySQL状态，便于后期进行性能瓶颈、故障问题分析所用 6 | 7 | ## mysql-benchmark 8 | > MySQL压测脚本，sybench和tpcc-mysql 9 | 10 | ## mysql-notes 11 | > MySQL知识库、文档等 12 | 13 | ## mysql-tools 14 | > 一些信手拈来即可用的脚本、SQL代码等 15 | -------------------------------------------------------------------------------- /mysql-benchmark/mysql.fio: -------------------------------------------------------------------------------- 1 | # 2 | # fio 3 | # 4 | 5 | [global] 6 | runtime=900 7 | time_based 8 | group_reporting 9 | directory=/data/ssd/fio 10 | ioscheduler=noop 11 | refill_buffers 12 | 13 | [mysql-binlog] 14 | filename=test-mysql-bin.log 15 | bsrange=512-1024 16 | ioengine=sync 17 | rw=write 18 | size=1G 19 | fsync=1 20 | overwrite=1 21 | fsync=100 22 | rate_iops=64 23 | invalidate=1 24 | numjobs=64 25 | 26 | [innodb-data] 27 | filename=test-innodb.dat 28 | bs=16K 29 | ioengine=psync 30 | rw=randrw 31 | size=50G 32 | direct=1 33 | rwmixread=80 34 | numjobs=64 35 | 36 | thinktime=900 37 | thinktime_spin=200 38 | thinktime_blocks=2 39 | -------------------------------------------------------------------------------- /mysql-benchmark/sysbench-prepare-1.0.20.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # for sysbench 1.0.20 4 | # 5 | export LD_LIBRARY_PATH=/usr/local/mysql/lib/ 6 | 7 | . ~/.bash_profile 8 | 9 | # 需要启用DEBUG模式时将下面三行注释去掉即可 10 | #set -u 11 | #set -x 12 | #set -e 13 | 14 | BASEDIR="/data/sysbench" 15 | if [ ! -d $BASEDIR ] 16 | then 17 | mkdir $BASEDIR -p 18 | fi 19 | cd $BASEDIR 20 | 21 | # 记录所有错误及标准输出到 sysbench.log 中 22 | #exec 3>&1 4>&2 1>> sysbench_prepare.log 2>&1 23 | 24 | DBIP=192.168.1.109 25 | DBPORT=3109 26 | DBUSER='proxysql' 27 | DBPASSWD='123456' 28 | SOCKET='/tmp/mysql3306.sock' 29 | NOW=`date +'%Y%m%d%H%M'` 30 | DBNAME="sysbench" 31 | TBLCNT=10 32 | WARMUP=300 33 | DURING=1800 34 | ROWS=10000000 35 | MAXREQ=1000000 36 | 37 | #create db 38 | echo 'now create db' 39 | mysql -h$DBIP -P$DBPORT -u$DBUSER -p$DBPASSWD -e 'create database sysbench' 40 | echo 'create ok' 41 | ## prepare data 42 | echo 'now prepare data' 43 | sysbench /usr/share/sysbench/oltp_read_only.lua \ 44 | --mysql-host=$DBIP \ 45 | --mysql-socket=$SOCKET \ 46 | --mysql-port=$DBPORT \ 47 | --mysql-user=$DBUSER \ 48 | --mysql-password=$DBPASSWD \ 49 | --mysql-db=$DBNAME \ 50 | --db-driver=mysql \ 51 | --oltp_tables_count=$TBLCNT \ 52 | --oltp_table_size=$ROWS \ 53 | --time=$DURING prepare 54 | -------------------------------------------------------------------------------- /mysql-benchmark/sysbench-run-1.0.20.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## 用sysbench执行MySQL OLTP基准测试的脚本。 4 | ## 适用于1.0.20版本 5 | ## 6 | ## 叶金荣, 知数堂培训联合创始人, 资深MySQL专家, MySQL布道师, Oracle MySQL ACE 7 | ## 8 | ## 几个注意事项： 9 | ## 1、运行sysbench的客户机和MySQL DB服务器尽量不要在同一台主机上，也包括一台宿主机上启动两个虚机的情形； 10 | ## 2、测试表的数量不宜太少，至少要求20个表以上； 11 | ## 3、每个表的数据量不宜太少，通常至少要求1千万以上，当然了，也要根据DB服务器的配置适当调整； 12 | ## 4、每次进行基准压测的时长不宜过短，通常要求持续15分钟以上； 13 | ## 5、每轮测试完毕后，中间至少暂停5分钟，或者确认系统负载完全恢复空跑状态为止； 14 | ## 6、测试DB服务器要是专用的，不能和其他业务混跑，否则测试结果就不靠谱了； 15 | ## 7、其余未尽事宜，后续再行补充。 16 | ## 17 | ## created by yejinrong@zhishutang.com, 2017/6/3 18 | ## update, 2020/5/24 19 | ## 20 | ## 21 | ## sysbench项目地址： https://github.com/akopytov/sysbench 22 | ## 23 | 24 | export LD_LIBRARY_PATH=/usr/local/mysql/lib/ 25 | 26 | . ~/.bash_profile 27 | 28 | # 需要启用DEBUG模式时将下面三行注释去掉即可 29 | #set -u 30 | #set -x 31 | #set -e 32 | 33 | BASEDIR="/data/sysbench" 34 | if [ ! -d $BASEDIR ] 35 | then 36 | mkdir $BASEDIR -p 37 | fi 38 | cd $BASEDIR 39 | #清理之前的遗留记录 40 | rm -rf $BASEDIR/logs* 41 | # 记录所有错误及标准输出到 sysbench.log 中 42 | exec 3>&1 4>&2 1>> sysbench.log 2>&1 43 | 44 | #时间单位秒 45 | DBIP=192.168.1.109 46 | DBPORT=3109 47 | DBUSER='proxysql' 48 | DBPASSWD='123456' 49 | SOCKET='/tmp/mysql3306.sock' 50 | NOW=`date +'%Y%m%d%H%M'` 51 | DBNAME="sysbench" 52 | REPORT_INTERVAL=1 53 | TBLCNT=10 54 | WARMUP=300 55 | DURING=1800 56 | ROWS=10000000 57 | MAXREQ=1000000 58 | 59 | # 并发压测的线程数，根据机器配置实际情况进行调整 60 | THERAD_NUMBER="8 64 128" 61 | 62 | #初始次数 63 | round=0 64 | # 一般至少跑3轮测试，我正常都会跑10轮以上 65 | while [ $round -lt 4 ] 66 | do 67 | #每回合日志位置： 68 | rounddir=$BASEDIR/logs-round${round} 69 | mkdir -p ${rounddir} 70 | 71 | for thread in `echo "${THERAD_NUMBER}"` 72 | do 73 | #常用可选项： 74 | #oltp_read_only 75 | #oltp_read_write 76 | #oltp_update_non_index 77 | 78 | sysbench /usr/share/sysbench/oltp_read_only.lua \ 79 | --mysql-host=$DBIP \ 80 | --mysql-socket=$SOCKET \ 81 | --mysql-port=$DBPORT \ 82 | --mysql-user=$DBUSER \ 83 | --mysql-password=$DBPASSWD \ 84 | --mysql-db=$DBNAME \ 85 | --db-driver=mysql \ 86 | --oltp_tables_count=$TBLCNT \ 87 | --oltp_table_size=$ROWS \ 88 | --report-interval=$REPORT_INTERVAL \ 89 | --threads=${thread} \ 90 | --rand-type=uniform \ 91 | --time=$DURING run >> ${rounddir}/sysbench_${thread}.log 92 | 93 | 94 | sleep 300 95 | done 96 | 97 | round=`expr $round + 1` 98 | sleep 300 99 | done 100 | -------------------------------------------------------------------------------- /mysql-benchmark/sysbench05-oltp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## 用sysbench执行MySQL OLTP基准测试的脚本。 4 | ## 5 | ## 叶金荣, 知数堂培训联合创始人, 资深MySQL专家, MySQL布道师, Oracle MySQL ACE 6 | ## 7 | ## 几个注意事项： 8 | ## 1、运行sysbench的客户机和MySQL DB服务器尽量不要在同一台主机上，也包括一台宿主机上启动两个虚机的情形； 9 | ## 2、测试表的数量不宜太少，至少要求20个表以上； 10 | ## 3、每个表的数据量不宜太少，通常至少要求1千万以上，当然了，也要根据DB服务器的配置适当调整； 11 | ## 4、每次进行基准压测的时长不宜过短，通常要求持续15分钟以上； 12 | ## 5、每轮测试完毕后，中间至少暂停5分钟，或者确认系统负载完全恢复空跑状态为止； 13 | ## 6、测试DB服务器要是专用的，不能和其他业务混跑，否则测试结果就不靠谱了； 14 | ## 7、其余未尽事宜，后续再行补充。 15 | ## 16 | ## created by yejinrong@zhishutang.com 17 | ## 2017/6/3 18 | ## 19 | ## sysbench项目地址： https://github.com/akopytov/sysbench 20 | ## 21 | 22 | export LD_LIBRARY_PATH=/usr/local/mysql/lib/ 23 | 24 | . ~/.bash_profile 25 | 26 | # 需要启用DEBUG模式时将下面三行注释去掉即可 27 | #set -u 28 | #set -x 29 | #set -e 30 | 31 | BASEDIR="/data/sysbench" 32 | cd $BASEDIR 33 | 34 | # 记录所有错误及标准输出到 sysbench.log 中 35 | exec 3>&1 4>&2 1>> sysbench.log 2>&1 36 | 37 | DBIP=10.10.10.1 38 | DBPORT=3306 39 | DBUSER='yejr' 40 | DBPASSWD='yejr' 41 | NOW=`date +'%Y%m%d%H%M'` 42 | DBNAME="sbtest" 43 | TBLCNT=50 44 | WARMUP=300 45 | DURING=900 46 | ROWS=20000000 47 | MAXREQ=20000000 48 | 49 | # 并发压测的线程数，根据机器配置实际情况进行调整 50 | RUN_NUMBER="8 64 128 256 512 768 1024 1664 2048 4096" 51 | 52 | ## prepare data 53 | ## sysbench --mysql-host=x --mysql-port=x --mysql-user=x --mysql-password=x --mysql-db=x --oltp_tables_count=50 \ 54 | ## --oltp-table-size=20000000 prepare 55 | 56 | round=1 57 | # 一般至少跑3轮测试，我正常都会跑10轮以上 58 | while [ $round -lt 4 ] 59 | do 60 | 61 | rounddir=logs-round${round} 62 | mkdir -p ${rounddir} 63 | 64 | for thread in `echo "${RUN_NUMBER}"` 65 | do 66 | 67 | ./bin/sysbench ./tests/include/oltp_legacy/oltp.lua --db-driver=mysql --mysql-host=${DBIP} --mysql-port=${DBPORT} \ 68 | --mysql-user=${DBUSER} --mysql-password=${DBPASSWD} --mysql-db=${DBNAME} --oltp_tables_count=${TBLCNT} \ 69 | --oltp-table-size=${ROWS} --rand-init=on --threads=${thread} --oltp-read-only=off --report-interval=1 \ 70 | --rand-type=uniform --max-time=${DURING} --max-requests=0 --percentile=99 run >> ./${rounddir}/sbtest_${thread}.log 71 | 72 | sleep 300 73 | done 74 | 75 | round=`expr $round + 1` 76 | sleep 300 77 | done 78 | -------------------------------------------------------------------------------- /mysql-benchmark/sysbench10-oltp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## 用sysbench执行MySQL OLTP基准测试的脚本。 4 | ## 5 | ## 叶金荣, 知数堂培训联合创始人, 资深MySQL专家, MySQL布道师, Oracle MySQL ACE 6 | ## 7 | ## 几个注意事项： 8 | ## 1、运行sysbench的客户机和MySQL DB服务器尽量不要在同一台主机上，也包括一台宿主机上启动两个虚机的情形； 9 | ## 2、测试表的数量不宜太少，至少要求20个表以上； 10 | ## 3、每个表的数据量不宜太少，通常至少要求1千万以上，当然了，也要根据DB服务器的配置适当调整； 11 | ## 4、每次进行基准压测的时长不宜过短，通常要求持续15分钟以上； 12 | ## 5、每轮测试完毕后，中间至少暂停5分钟，或者确认系统负载完全恢复空跑状态为止； 13 | ## 6、测试DB服务器要是专用的，不能和其他业务混跑，否则测试结果就不靠谱了； 14 | ## 7、其余未尽事宜，后续再行补充。 15 | ## 16 | ## created by yejinrong@zhishutang.com 17 | ## 2017/6/3 18 | ## 19 | ## sysbench项目地址： https://github.com/akopytov/sysbench 20 | ## 21 | 22 | export LD_LIBRARY_PATH=/usr/local/mysql/lib/ 23 | 24 | . ~/.bash_profile 25 | 26 | # 需要启用DEBUG模式时将下面三行注释去掉即可 27 | #set -u 28 | #set -x 29 | #set -e 30 | 31 | BASEDIR="/data/sysbench" 32 | if [ ! -d $BASEDIR ] 33 | then 34 | mkdir $BASEDIR -p 35 | fi 36 | cd $BASEDIR 37 | #清理之前的遗留记录 38 | rm -rf $BASEDIR/logs* 39 | # 记录所有错误及标准输出到 sysbench.log 中 40 | exec 3>&1 4>&2 1>> sysbench.log 2>&1 41 | 42 | #时间单位秒 43 | DBIP=192.168.1.109 44 | DBPORT=3109 45 | DBUSER='proxysql' 46 | DBPASSWD='123456' 47 | NOW=`date +'%Y%m%d%H%M'` 48 | DBNAME="sysbench" 49 | REPORT_INTERVAL=1 50 | TBLCNT=10 51 | WARMUP=300 52 | DURING=1800 53 | ROWS=10000000 54 | MAXREQ=1000000 55 | 56 | # 并发压测的线程数，根据机器配置实际情况进行调整 57 | THERAD_NUMBER="8 64 128" 58 | 59 | #初始次数 60 | round=0 61 | # 一般至少跑3轮测试，我正常都会跑10轮以上 62 | while [ $round -lt 4 ] 63 | do 64 | #每回合日志位置： 65 | rounddir=$BASEDIR/logs-round${round} 66 | mkdir -p ${rounddir} 67 | 68 | for thread in `echo "${THERAD_NUMBER}"` 69 | do 70 | #常用可选项： 71 | #oltp_read_only 72 | #oltp_read_write 73 | #oltp_update_non_index 74 | sysbench /usr/share/sysbench/oltp_read_only.lua \ 75 | --mysql-host=$DBIP \ 76 | --mysql-port=$DBPORT \ 77 | --mysql-user=$DBUSER \ 78 | --mysql-password=$DBPASSWD \ 79 | --mysql-db=$DBNAME \ 80 | --db-driver=mysql \ 81 | --tables=$TBLCNT \ 82 | --table-size=$ROWS \ 83 | --report-interval=$REPORT_INTERVAL \ 84 | --threads=${thread} \ 85 | --rand-type=uniform \ 86 | --time=$DURING run >> ${rounddir}/sysbench_${thread}.log 87 | 88 | 89 | sleep 300 90 | done 91 | 92 | round=`expr $round + 1` 93 | sleep 300 94 | done 95 | -------------------------------------------------------------------------------- /mysql-benchmark/sysbench10_prepare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | export LD_LIBRARY_PATH=/usr/local/mysql/lib/ 3 | 4 | . ~/.bash_profile 5 | 6 | # 需要启用DEBUG模式时将下面三行注释去掉即可 7 | #set -u 8 | #set -x 9 | #set -e 10 | 11 | BASEDIR="/data/sysbench" 12 | if [ ! -d $BASEDIR ] 13 | then 14 | mkdir $BASEDIR -p 15 | fi 16 | cd $BASEDIR 17 | 18 | # 记录所有错误及标准输出到 sysbench.log 中 19 | #exec 3>&1 4>&2 1>> sysbench_prepare.log 2>&1 20 | 21 | DBIP=192.168.1.109 22 | DBPORT=3109 23 | DBUSER='proxysql' 24 | DBPASSWD='123456' 25 | NOW=`date +'%Y%m%d%H%M'` 26 | DBNAME="sysbench" 27 | TBLCNT=10 28 | WARMUP=300 29 | DURING=1800 30 | ROWS=10000000 31 | MAXREQ=1000000 32 | 33 | #create db 34 | echo 'now create db' 35 | mysql -h$DBIP -P$DBPORT -u$DBUSER -p$DBPASSWD -e 'create database sysbench' 36 | echo 'create ok' 37 | ## prepare data 38 | echo 'now prepare data' 39 | sysbench /usr/share/sysbench/oltp_read_only.lua \ 40 | --mysql-host=$DBIP \ 41 | --mysql-port=$DBPORT \ 42 | --mysql-user=$DBUSER \ 43 | --mysql-password=$DBPASSWD \ 44 | --mysql-db=$DBNAME \ 45 | --db-driver=mysql \ 46 | --tables=10 \ 47 | --table-size=$ROWS \ 48 | --time=$DURING prepare 49 | -------------------------------------------------------------------------------- /mysql-benchmark/tpcc-oltp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## 用sysbench执行MySQL OLTP基准测试的脚本。 4 | ## 5 | ## 叶金荣, 知数堂培训联合创始人, 资深MySQL专家, MySQL布道师, Oracle MySQL ACE 6 | ## 7 | ## 几个注意事项： 8 | ## 1、运行tpcc的客户机和MySQL DB服务器尽量不要在同一台主机上，也包括一台宿主机上启动两个虚机的情形； 9 | ## 2、tpcc仓库数不宜太少，通常至少要求1千个以上，当仓库数太少的话，tpcc压测时，warehouse表会成为最大行锁等待瓶颈。 10 | ## 当然了，也要根据DB服务器的配置适当调整； 11 | ## 3、每次进行基准压测的时长不宜过短，通常要求持续15分钟以上； 12 | ## 4、每轮测试完毕后，中间至少暂停5分钟，或者确认系统负载完全恢复空跑状态为止； 13 | ## 5、测试DB服务器要是专用的，不能和其他业务混跑，否则测试结果就不靠谱了； 14 | ## 6、其余未尽事宜，后续再行补充。 15 | ## 16 | ## created by yejinrong@zhishutang.com 17 | ## 2017/6/3 18 | ## 19 | ## 老叶改造后的tpcc-mysql项目地址： http://github.com/yejr/tpcc-mysql-autoinc-pk 20 | ## 原生tpcc-mysql项目地址： https://github.com/Percona-Lab/tpcc-mysql 21 | ## 22 | 23 | export LD_LIBRARY_PATH=/usr/local/mysql/lib/ 24 | 25 | . ~/.bash_profile 26 | 27 | # 需要启用DEBUG模式时将下面三行注释去掉即可 28 | #set -u 29 | #set -x 30 | #set -e 31 | 32 | BASEDIR="/data/tpcc-mysql/" 33 | cd $BASEDIR 34 | 35 | exec 3>&1 4>&2 1>> run_tpcc.log 2>&1 36 | 37 | DBIP=10.10.10.1 38 | DBPORT=3306 39 | DBUSER='yejr' 40 | DBPASSWD='yejr' 41 | WAREHOUSE=1000 42 | DBNAME="tpcc${WIREHOUSE}" 43 | WARMUP=300 44 | DURING=900 45 | # 并发压测的线程数，根据机器配置实际情况进行调整 46 | RUN_NUMBER="8 64 128 256 512 768 1024 1664 2048 4096" 47 | 48 | # prepare data 49 | # ./tpcc_load $DBIP:$DBPORT $DBNAME $DBUSER $DBPASSWD $WIREHOUSE 50 | 51 | round=1 52 | # 一般至少跑3轮测试，我正常都会跑10轮以上 53 | while [ $round -lt 4 ] 54 | do 55 | 56 | NOW=`date +'%Y%m%d%H%M'` 57 | rounddir=logs-round${round} 58 | mkdir -p ${rounddir} 59 | 60 | 61 | for thread in `echo "${RUN_NUMBER}"` 62 | do 63 | 64 | ./tpcc_start -h ${DBIP -P ${DBPORT} -d ${DBNAME} -u ${DBUSER} -p "${DBPASSWD}" -w ${WAREHOUSE} -c ${THREADS} -r ${WARMUP} \ 65 | -l ${DURING} -i 1 > ${rounddir}/tpcc_${THREADS}.log 2>&1 66 | 67 | sleep 300 68 | done 69 | 70 | round=`expr $round + 1` 71 | sleep 300 72 | done 73 | -------------------------------------------------------------------------------- /mysql-blog/201707/20170721-table-join-column-data-type-should-be-same.md: -------------------------------------------------------------------------------- 1 | # 听说JOIN的列类型一定要一样？ 2 | ### 导读 3 | > 我们在制定表DDL设计规范时，通常都会要求一条：如果有两个表要做JOIN，那么关联条件列类型最好完全一样，才能保证查询效率，真的如此吗？ 4 | 5 | 相信不少朋友主动或被动告知这样一个规范要求（其实我也制定过这个规范），当多表JOIN时，关联条件列类型最好是完全一样的，这样才可以确保查询效率。果真如此吗？ 6 | 7 | ### 关于多表JOIN的几点结论及建议 8 | 为了节省大家时间，我先把几点结论写在前面，没耐心的同学可忽略后面测试过程。 9 | - 当被驱动表的列是字符串类型，而驱动表的列类型是非字符串时，则会发生类型隐式转换，无法使用索引； 10 | - 当被驱动表和驱动表的列都是字符串类型，两边无论是 CHAR 还是 VARCHAR，均不会发生类型隐式转换，都可以使用索引； 11 | - 当被驱动表的列是字符串且其字符集比驱动表的列采用的字符集更小或无法被包含时（latin比utf8mb4小，gb2312 比 utf8mb4 小，另外 gb2312 虽然比 latin1 大，但并不兼容，也不行，详见下方测试 ），则会发生类型隐式转换，无法使用索引； 12 | - 综上，虽然有很多场景下，JOIN列类型不一致也能用到索引，但保不准啥时候就掉坑了。因此，最后回答一下本文题目，**JOIN列的类型定义完全一致，包括长度、字符集**。 13 | 14 | 几点说明 15 | - 测试表t1、t2表均为**UTF8MB4字符集**。 16 | - 字符串类型列char_col默认设置**VARCHAR(20)**。 17 | - 测试MySQL 版本 5.7.18。 18 | 19 | **场景1：驱动表列是MEDIUMINT/INT/BIGINT** 20 | 21 | 子场景 | 驱动表(t1)列 | 被驱动表(t2)列 | 是否可用索引 22 | ---|---|---|--- 23 | case1.1 | INT | INT | 可用 24 | case1.2 | INT | CHAR(20) | **不可用** 25 | case1.3 | INT | VARCHAR(20) | **不可用** 26 | case1.4 | INT | MEDIUMINT | 可用 27 | case1.5 | INT | BIGINT | 可用 28 | case1.6 | MEDIUMINT | INT | 可用 29 | case1.7 | MEDIUMINT | BIGINT | 可用 30 | case1.8 | BIGINT | MEDIUMINT | 可用 31 | case1.9 | BIGINT | INT | 可用 32 | 33 | **场景2：驱动列是CHAR(20)** 34 | 35 | 子场景 | 驱动表(t1)列 | 被驱动表(t2)列 | 是否可用索引 36 | ---|---|---|--- 37 | case2.1 | CHAR(20) | CHAR(20) | 可用 38 | case2.2 | CHAR(20) UTF8 | CHAR(20) | 可用 39 | case2.3 | CHAR(20) | CHAR(20) UTF8 | **不可用** 40 | case2.4 | CHAR(20) UTF8MB4 | CHAR(20) LATIN1 | **不可用** 41 | case2.5 | CHAR(20) UTF8MB4 | CHAR(20) GB2312 | **不可用** 42 | case2.6 | CHAR(20) LATIN1 | CHAR(20) UTF8MB4 | 可用 43 | case2.7 | CHAR(20) GB2312 | CHAR(20) UTF8MB4 | 可用 44 | case2.8 | CHAR(20) GB2312 | CHAR(20) LATIN1 | **SQL报错，要先转字符集** 45 | case2.9 | CHAR(20) LATIN1 | CHAR(20) GB2312 | **SQL报错，要先转字符集** 46 | case2.10 | CHAR(20) | VARCHAR(20) | 可用 47 | case2.11 | CHAR(20) | VARCHAR(30) | 可用 48 | case2.12 | CHAR(20) | CHAR(30) | 可用 49 | case2.13 | CHAR(20) | VARCHAR(260) | 可用 50 | 51 | **场景3：驱动列是VARCHAR(20)** 52 | 53 | 子场景 | 驱动表(t1)列 | 被驱动表(t2)列 | 是否可用索引 54 | ---|---|---|--- 55 | case3.1 | VARCHAR(20) | CHAR(20) | 可用 56 | case3.2 | VARCHAR(20) | VARCHAR(20) | 可用 57 | case3.3 | VARCHAR(20) | VARCHAR(260) | 可用 58 | 59 | **场景4：驱动列是VARCHAR(260)/VARCHAR(270)** 60 | 61 | 子场景 | 驱动表(t1)列 | 被驱动表(t2)列 | 是否可用索引 62 | ---|---|---|--- 63 | case4.1 | VARCHAR(260) | CHAR(20) | 可用 64 | case4.2 | VARCHAR(260) | VARCHAR(20) | 可用 65 | case4.3 | VARCHAR(260) | VARCHAR(260) | 可用 66 | case4.4 | VARCHAR(260) | VARCHAR(270) | 可用 67 | case4.5 | VARCHAR(270) | VARCHAR(260) | 可用 68 | 69 | **场景5：驱动列是VARCHAR(30)** 70 | 71 | 子场景 | 驱动表(t1)列 | 被驱动表(t2)列 | 是否可用索引 72 | ---|---|---|--- 73 | case5.1 | CHAR(30) | CHAR(20) | 可用 74 | case5.2 | CHAR(30) | VARCHAR(20) | 可用 75 | 76 | **场景6：最后有排序的情况** 77 | 最后的排序列不属于驱动表 78 | ``` 79 | yejr@imysql.com[yejr]> EXPLAIN SELECT * FROM t1 LEFT JOIN 80 | t2 ON (t1.int_col = t2.int_col) WHERE 81 | t1.id >= 5000 ORDER BY t2.id\G 82 | *************************** 1. row *************************** 83 | id: 1 84 | select_type: SIMPLE 85 | table: t1 86 | partitions: NULL 87 | type: range 88 | possible_keys: PRIMARY 89 | key: PRIMARY 90 | key_len: 8 91 | ref: NULL 92 | rows: 51054 93 | filtered: 100.00 94 | Extra: Using where; Using temporary; Using filesort 95 | *************************** 2. row *************************** 96 | id: 1 97 | select_type: SIMPLE 98 | table: t2 99 | partitions: NULL 100 | type: ref 101 | possible_keys: int_col 102 | key: int_col 103 | key_len: 4 104 | ref: yejr.t1.int_col 105 | rows: 10 106 | filtered: 100.00 107 | Extra: NULL 108 | ``` 109 | 小结：**当最后的排序列不属于驱动表时，则会生成临时表，且又有额外排序**。 110 | 111 | 最后的排序列属于驱动表 112 | ``` 113 | yejr@imysql.com[yejr]> EXPLAIN SELECT * FROM t1 LEFT JOIN 114 | t2 ON (t1.int_col = t2.int_col) WHERE 115 | t1.id >= 5000 ORDER BY t1.id\G 116 | *************************** 1. row *************************** 117 | id: 1 118 | select_type: SIMPLE 119 | table: t1 120 | partitions: NULL 121 | type: range 122 | possible_keys: PRIMARY 123 | key: PRIMARY 124 | key_len: 8 125 | ref: NULL 126 | rows: 51054 127 | filtered: 100.00 128 | Extra: Using where 129 | *************************** 2. row *************************** 130 | id: 1 131 | select_type: SIMPLE 132 | table: t2 133 | partitions: NULL 134 | type: ref 135 | possible_keys: int_col 136 | key: int_col 137 | key_len: 4 138 | ref: yejr.t1.int_col 139 | rows: 10 140 | filtered: 100.00 141 | Extra: NULL 142 | ``` 143 | 小结：**当最后的排序列属于驱动表时，则不会生成临时表，也不需要额外排序**。 144 | 145 | 更多的组合测试场景，请各位亲自行完成哈。 146 | 147 | ### 附录 ### 148 | 测试表DDL 149 | ``` 150 | CREATE TABLE `t1` ( 151 | `id` bigint(20) NOT NULL AUTO_INCREMENT, 152 | `int_col` int(20) unsigned NOT NULL DEFAULT '0', 153 | `char_col` char(20) NOT NULL DEFAULT '', 154 | ... 155 | PRIMARY KEY (`id`), 156 | KEY `int_col` (`int_col`), 157 | KEY `char_col` (`char_col`) 158 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 159 | 160 | CREATE TABLE `t2` ( 161 | `id` bigint(20) NOT NULL AUTO_INCREMENT, 162 | `int_col` int(8) unsigned NOT NULL DEFAULT '0', 163 | `char_col` char(20) NOT NULL DEFAULT '', 164 | ... 165 | PRIMARY KEY (`id`), 166 | KEY `int_col` (`int_col`), 167 | KEY `char_col` (`char_col`) 168 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 169 | ``` 170 | 171 | 修改列字符集定义的DDL样例 172 | ``` 173 | /* 174 | - 只修改长度 175 | */ 176 | ALTER TABLE t1 MODIFY char_col 177 | VARCHAR(260) NOT NULL DEFAULT ''; 178 | 179 | /* 180 | - 同时修改字符集 181 | */ 182 | ALTER TABLE t2 MODIFY char_col 183 | VARCHAR(20) CHARACTER SET UTF8 NOT NULL DEFAULT ''; 184 | ``` 185 | 186 | 修改完列定义后，还记得要重新执行 ANALYZE TABLE 重新统计索引信息哟。 187 | ``` 188 | yejr@imysql.com[yejr]> ANALYZE TABLE t1, t2; 189 | +---------+---------+----------+----------+ 190 | | Table | Op | Msg_type | Msg_text | 191 | +---------+---------+----------+----------+ 192 | | yejr.t1 | analyze | status | OK | 193 | | yejr.t2 | analyze | status | OK | 194 | +---------+---------+----------+----------+ 195 | ``` 196 | 197 | 执行测试的SQL样例 198 | ``` 199 | /* 200 | - char_col 可以自行替换成 int_col 201 | - 加上 t1.id >= 5000 是为了避免预估扫描数据量太多，变成全表扫描 202 | */ 203 | EXPLAIN SELECT * FROM t1 LEFT JOIN 204 | t2 ON (t1.char_col = t2.char_col) WHERE 205 | t1.id >= 5000\G 206 | ``` 207 | 208 | **参考** 209 | 210 | - [UTF8字符集的表怎么直接转UTF8MB4？](http://mp.weixin.qq.com/s?__biz=MjM5NzAzMTY4NQ==&mid=2653930515&idx=1&sn=edc341d36f076e379ea71903e31762a0&chksm=bd3b5e798a4cd76fd63e7f6c9f47eaca47cf66761c6c24479d170a16b1934f139278896658e5&scene=21#wechat_redirect) 211 | - [细说ANALYZE TABLE](http://mp.weixin.qq.com/s?__biz=MjM5NzAzMTY4NQ==&mid=2653930467&idx=1&sn=2511653a476245e96389655fd670671e&chksm=bd3b59898a4cd09f2e2823d75f9d0f3acd76442b547bede0178130bc202f79927c1ac1b6dde3&scene=21#wechat_redirect) 212 | - [优化系列 | DELETE子查询改写优化](http://mp.weixin.qq.com/s?__biz=MjM5NzAzMTY4NQ==&mid=2653928676&idx=1&sn=c8d1f5f12f234738bdf6da844ab4994e&scene=21#wechat_redirect) 213 | - [MySQL优化案例系列 — 分页优化](http://mp.weixin.qq.com/s?__biz=MjM5NzAzMTY4NQ==&mid=207345129&idx=1&sn=0c0ab0a302f0c963845544545ce7f893&scene=21#wechat_redirect) 214 | - [MySQL优化案例系列 — discuz!热帖翻页优化](http://mp.weixin.qq.com/s?__biz=MjM5NzAzMTY4NQ==&mid=207345129&idx=2&sn=a37781cc9a73d145c11f396d289c9955&scene=21#wechat_redirect) 215 | -------------------------------------------------------------------------------- /mysql-blog/201707/20170726-mysql-8.0-new-index.md: -------------------------------------------------------------------------------- 1 | # MySQL 8.0索引新玩法 2 | ## 导读 3 | > MySQL 8.0开始支持倒序索引和不可见索引，和叶师傅一起来耍耍。 4 | 5 | MySQL版本号 6 | ``` 7 | Server version: 8.0.1-dmr-log MyS QL Community Server (GPL) 8 | ``` 9 | 10 | 测试表结构 11 | ``` 12 | CREATE TABLE `t1` ( 13 | `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, 14 | `u1` int(10) unsigned NOT NULL DEFAULT '0', 15 | `u2` int(10) unsigned NOT NULL DEFAULT '0', 16 | `u3` varchar(20) NOT NULL DEFAULT '', 17 | `u4` varchar(35) NOT NULL DEFAULT '', 18 | PRIMARY KEY (`id`), 19 | KEY `u1` (`u1` DESC,`u2`) 20 | ) ENGINE=InnoDB AUTO_INCREMENT=131054 DEFAULT CHARSET=utf8mb4 21 | ``` 22 | 23 | 表统计信息： 24 | ``` 25 | yejr@imysql.com[yejr]> show table status like 't1'\G 26 | *************************** 1. row *************************** 27 | Name: t1 28 | Engine: InnoDB 29 | Version: 10 30 | Row_format: Dynamic 31 | Rows: 101970 32 | Avg_row_length: 87 33 | Data_length: 8929280 34 | Max_data_length: 0 35 | Index_length: 3686400 36 | Data_free: 4194304 37 | Auto_increment: 131054 38 | Create_time: 2017-05-31 11:57:48 39 | Update_time: 2017-06-10 12:08:09 40 | Check_time: NULL 41 | Collation: utf8mb4_0900_ai_ci 42 | Checksum: NULL 43 | Create_options: 44 | Comment: 45 | ``` 46 | 47 | ## 1、倒序索引（Descending Indexes） 48 | ### case1、MySQL 8.0下利用倒序索引消除排序 49 | ``` 50 | # 查看执行计划 51 | yejr@imysql.com[yejr]> desc select * from t1 where u1 <= 70 order by u1 desc, u2\G 52 | *************************** 1. row *************************** 53 | id: 1 54 | select_type: SIMPLE 55 | table: t1 56 | partitions: NULL 57 | type: range 58 | possible_keys: u1 59 | key: u1 60 | key_len: 4 61 | ref: NULL 62 | rows: 27742 63 | filtered: 100.00 64 | Extra: Using index condition 65 | 66 | # 清除status，方便后面看SQL执行代价 67 | yejr@imysql.com[yejr]> flush status; 68 | 69 | # 临时修改pager，将查询结果重定向到 /dev/null，而不是发送到标准输出设备 70 | yejr@imysql.com[test]> pager cat - > /dev/null 71 | PAGER set to 'cat - > /dev/null' 72 | 73 | # 执行SQL 74 | yejr@imysql.com[yejr]> select * from t1 where u1 <= 70 order by u1 desc, u2; 75 | 16384 rows in set (0.05 sec) 76 | 77 | # 查看SQL代价 78 | yejr@imysql.com[yejr]> show status like 'handler%read%'; 79 | +-----------------------+-------+ 80 | | Variable_name | Value | 81 | +-----------------------+-------+ 82 | | Handler_read_first | 0 | 83 | | Handler_read_key | 1 | 84 | | Handler_read_last | 0 | 85 | | Handler_read_next | 16384 | 86 | | Handler_read_prev | 0 | 87 | | Handler_read_rnd | 0 | 88 | | Handler_read_rnd_next | 0 | 89 | +-----------------------+-------+ 90 | ``` 91 | 92 | ### case2、MySQL 5.7下不支持倒序索引，有额外排序 93 | ``` 94 | # 执行计划中有 **Using filesort** 95 | yejr@imysql.com[yejr]> desc select * from t1 where u1 <= 70 order by u1 desc, u2\G 96 | *************************** 1. row *************************** 97 | id: 1 98 | select_type: SIMPLE 99 | table: t1 100 | partitions: NULL 101 | type: ALL 102 | possible_keys: u1 103 | key: NULL 104 | key_len: NULL 105 | ref: NULL 106 | rows: 102176 107 | filtered: 33.29 108 | Extra: Using where; Using filesort 109 | 110 | yejr@imysql.com[yejr]> select * from t1 where u1 <= 70 order by u1 desc, u2; 111 | 16383 rows in set (0.05 sec) 112 | 113 | # 可以看到有大量的 read_rnd_next 114 | yejr@zhishutang.com[yejr]> show status like 'handler%read%'; 115 | +-----------------------+--------+ 116 | | Variable_name | Value | 117 | +-----------------------+--------+ 118 | | Handler_read_first | 1 | 119 | | Handler_read_key | 1 | 120 | | Handler_read_last | 0 | 121 | | Handler_read_next | 0 | 122 | | Handler_read_prev | 0 | 123 | | Handler_read_rnd | 0 | 124 | | Handler_read_rnd_next | 102401 | 125 | +-----------------------+--------+ 126 | ``` 127 | 128 | ### case3、MySQL 5.7下不支持倒序索引，但强制指定索引 129 | ``` 130 | # 还是有 **Using filesort** 啊 131 | yejr@imysql.com[yejr]> desc select * from t1 force index(u1) where u1 <= 70 order by u1 desc, u2\G 132 | *************************** 1. row *************************** 133 | id: 1 134 | select_type: SIMPLE 135 | table: t1 136 | partitions: NULL 137 | type: range 138 | possible_keys: u1 139 | key: u1 140 | key_len: 4 141 | ref: NULL 142 | rows: 34014 143 | filtered: 100.00 144 | Extra: Using index condition; Using filesort 145 | 146 | yejr@imysql.com[yejr]> select * from t1 force index(u1) where u1 <= 70 order by u1 desc, u2; 147 | 16383 rows in set (0.08 sec) 148 | 149 | # status瓶颈从 read_rnd_next 变成了 read_next 150 | TODO 151 | yejr@imysql.com[yejr]> show status like 'handler%read%'; 152 | +-----------------------+-------+ 153 | | Variable_name | Value | 154 | +-----------------------+-------+ 155 | | Handler_read_first | 1 | 156 | | Handler_read_key | 1 | 157 | | Handler_read_last | 0 | 158 | | Handler_read_next | 16384 | 159 | | Handler_read_prev | 0 | 160 | | Handler_read_rnd | 0 | 161 | | Handler_read_rnd_next | 0 | 162 | +-----------------------+-------+ 163 | ``` 164 | 165 | 关于 Handler_read_rnd、Handler_read_rnd_next 的解释 ，请看文档说明。 166 | 167 | **Handler_read_rnd** 168 | > The number of requests to read a row based on a fixed position. This value is high if you are doing a lot of queries that require sorting of the result. You probably have a lot of queries that require MySQL to scan entire tables or you have joins that do not use keys properly. 169 | 170 | **Handler_read_rnd_next** 171 | > The number of requests to read the next row in the data file. This value is high if you are doing a lot of table scans. Generally this suggests that your tables are not properly indexed or that your queries are not written to take advantage of the indexes you have. 172 | 173 | 关于倒序索引有几点说明： 174 | - 只支持InnoDB引擎； 175 | - 只支持B+TREE，不支持HASH、FULLTEXT、SPATIAL索引； 176 | - change buffer无法支持有倒序的二级索引（secondary index）； 177 | - 正常正序索引支持的数据类型，倒序索引也都支持； 178 | - 除了原生列之外，还支持虚拟列（VIRTUAL、STORED都支持）； 179 | - InnoDB全文索引的表中，FTS_DOC_ID列也不能设定为倒序； 180 | 181 | ## 2、不可见索引(INVISIBLE INDEX) 182 | 当我们发现冗余索引或者个别不再需要的索引时，首先想到的是直接删除。 183 | 184 | 但是直接删除索引肯定是有风险的，难免什么时候某个老业务又需要用到这个索引了，这时候invisible index就非常实用了。 185 | 186 | 这时候就可以把这个索引设置为 visible/invisible，修改索引的 visible 属性是可以in-place的，非常快。这相比删除、再次创建索引要快得多了。 187 | 188 | 一起来看看 invisible index的玩法： 189 | ``` 190 | # 设置 invisible 191 | yejr@imysql.com[yejr]> alter table t1 alter index u1 INVISIBLE; 192 | 193 | # 查看DDL 194 | yejr@imysql.com[yejr]> show create table t1\G 195 | *************************** 1. row *************************** 196 | Table: t1 197 | Create Table: CREATE TABLE `t1` ( 198 | ... 199 | KEY `u1` (`u1` DESC,`u2`) /*!80000 INVISIBLE */ 200 | ) ENGINE=InnoDB AUTO_INCREMENT=131054 DEFAULT CHARSET=utf8mb4 201 | 202 | # 查看索引状态 203 | yejr@imysql.com[yejr]> desc select * from t1 force index(u1) where u1 <= 70 order by u1 desc, u2\G 204 | ERROR 1176 (42000): Key 'u1' doesn't exist in table 't1' 205 | yejr@imysql.com[yejr]> SELECT INDEX_NAME, IS_VISIBLE 206 | -> FROM INFORMATION_SCHEMA.STATISTICS 207 | -> WHERE table_name = 't1'; 208 | +------------+------------+ 209 | | INDEX_NAME | IS_VISIBLE | 210 | +------------+------------+ 211 | | PRIMARY | YES | 212 | | u1 | NO | 213 | | u1 | NO | 214 | +------------+------------+ 215 | 216 | yejr@imysql.com[yejr]> show index from t1; 217 | +-------+------------+----------+-------------+-------------+---------+ 218 | | Table | Non_unique | Key_name | Column_name | Cardinality | Visible | 219 | +-------+------------+----------+-------------+-------------+---------+ 220 | | t1 | 0 | PRIMARY | id | 101970 | YES | 221 | | t1 | 1 | u1 | u1 | 24 | NO | 222 | | t1 | 1 | u1 | u2 | 24 | NO | 223 | +-------+------------+----------+-------------+-------------+---------+ 224 | 225 | # 执行SQL时即便force index也不可用 226 | yejr@imysql.com[yejr]>select * from t1 force index(u1) where u1 <= 70 order by u1 desc, u2\G 227 | ERROR 1176 (42000): Key 'u1' doesn't exist in table 't1' 228 | ``` 229 | 230 | 关于invisible index的几点补充： 231 | - 即便是 invisible，也还要保持索引的更新； 232 | - 主键或被选中作为聚集索引的唯一索引（这种称为隐含的聚集索引），都不能 invisible； 233 | - 任何存储引擎都支持，不仅限于InnoDB； 234 | 235 | 最后，我要再次安利下MySQL 8.0版本，在这个版本中，有几个重大（激动人心的）新特性： 236 | - 新增User Roles特性； 237 | - 数据字典采用InnoDB表存储，不再使用frm文件了（MyISAM退出历史舞台进入倒计时～）； 238 | - 在线执行SET GLOBAL xx修改某个 option 后，可以实现该修改持久化（太实用了，拯救健忘症重度患者）； 239 | - InnoDB表的 AUTO_INCREMENT 值也可以实现持久化（重启后不再发生变化）； 240 | - InnoDB的REDO、UNDO LOG也支持加密； 241 | - InnoDB表支持NOWAIT、SKIP LOCKED语法； 242 | - 支持倒序索引、不可见索引； 243 | - Optimizer、P_S、JSON、GIS功能持续增强； 244 | - 可想而知的是，对Group Replicatioin也会重点增强； 245 | 246 | 其他新特性不一一列出，我粗略的看了下，这些不都是在向Oracle学习嘛，哈哈。 247 | 248 | **延伸阅读** 249 | 250 | - [**Descending Indexes**](https://dev.mysql.com/doc/refman/8.0/en/descending-indexes.html) 251 | - [**invisible index**](https://dev.mysql.com/doc/refman/8.0/en/invisible-indexes.html) 252 | - [**【重磅】MySQL 8.0 pre release看点**](http://mp.weixin.qq.com/s/z6-ivNmjpSiB8mKyKRJyTA) 253 | -------------------------------------------------------------------------------- /mysql-notes/1-MySQL安装部署规范.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /mysql-notes/mysql_db_design_guide.md: -------------------------------------------------------------------------------- 1 | MySQL数据库设计规范 2 | 3 | 规范背景与目的 4 | ============== 5 | 6 | MySQL数据库与oracle、sqlserver等数据库相比，有其内核上的优势与劣势。我们在使用MySQL数据库的时候需要遵循一定规范，扬长避短。本规范旨在帮助或指导RD、QA、OP等技术人员做出适合线上业务的数据库设计。在数据库变更和处理流程、数据库表设计、SQL编写等方面予以规范，从而为公司业务系统稳定、健康地运行提供保障。MySQL版本之间可能会有部分内容稍有出入，读者自行甄别，本文仅限于共享参考之用。 7 | 8 | 设计规范 9 | ======== 10 | 11 | 2.1 数据库设计 12 | -------------- 13 | 14 | 以下所有规范会按照【高危】、【强制】、【建议】三个级别进行标注，遵守优先级从高到低。 15 | 16 | 对于不满足【高危】和【强制】两个级别的设计，DBA会强制打回要求修改。 17 | 18 | ### 2.1.1 库名 19 | 20 | 1.【强制】库的名称必须控制在32个字符以内，相关模块的表名与表名之间尽量提现join的关系，如user表和user_login表。 21 | 22 | 2.【强制】库的名称格式：业务系统名称_子系统名，同一模块使用的表名尽量使用统一前缀。 23 | 24 | 3.【强制】一般分库名称命名格式是“库通配名_编号”，编号从“0”开始递增，比如“wenda_001” 25 | 26 | 以时间进行分库的名称格式是“库通配名_时间” 27 | 28 | 4.【强制】创建数据库时必须显式指定字符集，并且字符集只能是utf8或者utf8mb4 29 | 30 | 创建数据库SQL举例： 31 | 32 | Create database db1 default character set utf8; 33 | 34 | ### 2.1.2 表结构 35 | 36 | 1.【强制】表和列的名称必须控制在32个字符以内，表名只能使用字母、数字和下划线，一律小写。 37 | 38 | 2.【强制】表名要求模块名强相关，如师资系统采用”sz”作为前缀，渠道系统采用”qd”作为前缀等。 39 | 40 | 3.【强制】创建表时必须显式指定字符集为utf8或utf8mb4。 41 | 42 | 4.【强制】创建表时必须显式指定表存储引擎类型，如无特殊需求，一律为InnoDB。当需要使用除InnoDB/MyISAM/Memory以外的存储引擎时，必须通过DBA审核才能在生产环境中使用。 43 | 44 | 因为Innodb表支持事务、行锁、宕机恢复、MVCC等关系型数据库重要特性，为业界使用最多的MySQL存储引擎。而这是其他大多数存储引擎不具备的，因此首推InnoDB。 45 | 46 | 5.【强制】建表必须有comment，这个很重要。 47 | 48 | 6.【建议】建表时关于主键： 49 | > (1)强制要求主键为id，类型为int或bigint，且为auto_increment 50 | > (2)标识表里每一行主体的字段不要设为主键，建议设为其他字段如user_id，order_id等，并建立unique 51 | > key索引（可参考cdb.teacher表设计）。 52 | > 因为如果设为主键且主键值为随机插入，则会导致innodb内部page分裂和大量随机I/O，性能下降。 53 | 54 | 7.【建议】核心表（如用户表，金钱相关的表）必须有行数据的创建时间字段create_time和最后更新时间字段update_time，便于查问题。 55 | 56 | 8.【建议】表中所有字段必须都是NOT NULL属性，业务可以根据需要定义DEFAULT值。 57 | > 因为使用NULL值会存在每一行都会占用额外存储空间、数据迁移容易出错、聚合函数计算结果偏差等问题。 58 | 59 | 9.【建议】建议对表里的blob、text等大字段，垂直拆分到其他表里，仅在需要读这些对象的时候才去select。 60 | 61 | 10.【建议】反范式设计：把经常需要join查询的字段，在其他表里冗余一份。 62 | 如user_name属性在user_account，user_login_log等表里冗余一份，减少join查询。 63 | 64 | 11.【强制】中间表用于保留中间结果集，名称必须以“tmp_”开头。 65 | > 备份表用于备份或抓取源表快照，名称必须以“bak_”开头。 66 | > 中间表和备份表定期清理。 67 | 68 | 12.【强制】对于超过100W行的大表进行alter table，必须经过DBA审核，并在业务低峰期执行。 69 | > 因为alter table会产生表锁，期间阻塞对于该表的所有写入，对于业务可能会产生极大影响。 70 | 71 | ### 2.1.3 列数据类型优化 72 | 73 | 1.【建议】表中的自增列（auto_increment属性），推荐使用bigint类型。 74 | > 因为无符号int存储范围为-2147483648~2147483647（大约21亿左右），溢出后会导致报错。 75 | 76 | 2.【建议】业务中选择性很少的状态status、类型type等字段推荐使用tinytint或者smallint类型节省存储空间。 77 | 78 | 3.【建议】业务中IP地址字段推荐使用int类型，不推荐用char(15) 79 | > 因为int只占4字节，可以用如下函数相互转换，而char(15)占用至少15字节。一旦表数据行数到了1亿，那么要多用1.1G存储空间！ 80 | > SQL：select inet_aton('192.168.2.12'); select inet_ntoa(3232236044); 81 | > Php: ip2long(‘192.168.2.12’); long2ip(3530427185); 82 | 83 | 4.【建议】不推荐使用enum，set 84 | > 因为它们浪费空间，且枚举值写死了，变更不方便。推荐使用tinyint或smallint 85 | 86 | 5.【建议】不推荐使用blob，text等类型 87 | > 它们都比较浪费硬盘和内存空间。在加载表数据时，会读取大字段到内存里从而浪费内存空间，影响系统性能。建议和PM、RD沟通，是否真的需要这么大字段？ 88 | > Innodb中当一行记录超过8098字节时，会将该记录中选取最长的一个字段将其768字节放在原始page里，该字段余下内容放在overflow-page里。不幸的是在compact行格式下，原始page和overflow-page都会加载。 89 | 90 | 6.【建议】存储金钱的字段，建议用int，程序端乘以100和除以100进行存取。 91 | > 因为int占用4字节，而double占用8字节，空间浪费。 92 | 93 | 7.【建议】文本数据尽量用varchar存储 94 | > 因为varchar是变长存储，比char更省空间。MySQL 95 | > server层规定一行所有文本最多存65535字节，因此在utf8字符集下最多存21844个字符，超过会自动转换为mediumtext字段。而text在utf8字符集下最多存21844个字符，mediumtext最多存2^24/3个字符，longtext最多存2^32个字符。 96 | > 一般建议用varchar类型，字符数不要超过2700 97 | 98 | 8.【建议】时间类型尽量选取timestamp 99 | > 因为datetime占用8字节，timestamp仅占用4字节，但是范围为1970-01-01 00:00:01到2038-01-01 00:00:00。 100 | > 更为高阶的方法，选用int来存储时间，使用SQL函数unix_timestamp()和from_unixtime()来进行转换。 101 | 102 | 详细存储大小参加下图： 103 | ![MySQL DataType](http://jiayifei.cn/wp-content/uploads/2017/11/MySQL-DataType.png) 104 | ### 2.1.4 索引设计 105 | 106 | 1.【强制】InnoDB表必须主键为id int/bigint auto_increment,且主键值禁止被更新。 107 | 108 | 2.【建议】主键的名称以“pk_”开头，唯一键以“uk_”或“uq_”开头，普通索引以“idx_”开头，一律使用小写格式，以表名/字段的名称或缩写作为后缀。 109 | 110 | 3.【强制】InnoDB和MyISAM存储引擎表，索引类型必须为BTREE；MEMORY表可以根据需要选择HASH或者BTREE类型索引。 111 | 112 | 4.【强制】单个索引中每个索引记录的长度不能超过64KB 113 | 114 | 5.【建议】单个表上的索引个数不能超过7个 115 | 116 | 6.【建议】在建立索引时，多考虑建立联合索引，并把区分度最高的字段放在最前面。 117 | 如列userid的区分度可由select count(distinct userid)计算出来。 118 | 119 | 7.【建议】在多表join的SQL里，保证被驱动表的连接列上有索引，这样join执行效率最高。 120 | 121 | 8.【建议】建表或加索引时，保证表里互相不存在冗余索引。 122 | > 对于MySQL来说，如果表里已经存在key(a,b)，则key(a)为冗余索引，需要删除。 123 | 124 | ### 2.1.5 分库分表、分区表 125 | 126 | 1.【强制】分区表的分区字段（partition-key）必须有索引，或者是组合索引的首列。 127 | 128 | 2.【强制】单个分区表中的分区（包括子分区）个数不能超过1024。 129 | 130 | 3.【强制】上线前RD或者DBA必须指定分区表的创建、清理策略。 131 | 132 | 4.【强制】访问分区表的SQL必须包含分区键。 133 | 134 | 5.【建议】单个分区文件不超过2G，总大小不超过50G。建议总分区数不超过20个。 135 | 136 | 6.【强制】对于分区表执行alter table操作，必须在业务低峰期执行。 137 | 138 | 7.【强制】采用分库策略的，库的数量不能超过1024 139 | 140 | 8.【强制】采用分表策略的，表的数量不能超过4096 141 | 142 | 9.【建议】单个分表不超过500W行，ibd文件大小不超过2G，这样才能让数据分布式变得性能更佳。 143 | 144 | 10.【建议】水平分表尽量用取模方式，日志、报表类数据建议采用日期进行分表。 145 | 146 | ### 2.1.6 字符集 147 | 148 | 1.【强制】数据库本身库、表、列所有字符集必须保持一致，为utf8或utf8mb4 149 | 150 | 2.【强制】前端程序字符集或者环境变量中的字符集，与数据库、表的字符集必须一致，统一为utf8 151 | 152 | ### 2.1.7 程序DAO层设计建议 153 | 154 | 1.【建议】新的代码不要用model，推荐使用手动拼SQL+绑定变量传入参数的方式。 155 | > 因为model虽然可以使用面向对象的方式操作db，但是其使用不当很容易造成生成的SQL非常复杂， 156 | > 且model层自己做的强制类型转换性能较差，最终导致数据库性能下降。 157 | 158 | 1.【建议】前端程序连接MySQL或者redis，必须要有连接超时和失败重连机制，且失败重试必须有间隔时间。 159 | 160 | 2.【建议】前端程序报错里尽量能够提示MySQL或redis原生态的报错信息，便于排查错误。 161 | 162 | 3.【建议】对于有连接池的前端程序，必须根据业务需要配置初始、最小、最大连接数，超时时间以及连接回收机制，否则会耗尽数据库连接资源，造成线上事故。 163 | 164 | 4.【建议】对于log或history类型的表，随时间增长容易越来越大，因此上线前RD或者DBA必须建立表数据清理或归档方案。 165 | 166 | 5.【建议】在应用程序设计阶段，RD必须考虑并规避数据库中主从延迟对于业务的影响。 167 | > 尽量避免从库短时延迟（20秒以内）对业务造成影响，建议强制一致性的读开启事务走主库，或更新后过一段时间再去读从库。 168 | 169 | 6.【建议】多个并发业务逻辑访问同一块数据（innodb表）时，会在数据库端产生行锁甚至表锁导致并发下降，因此建议更新类SQL尽量基于主键去更新。 170 | 171 | 7.【建议】业务逻辑之间加锁顺序尽量保持一致，否则会导致死锁。 172 | 173 | 8.【建议】对于单表读写比大于10:1的数据行或单个列，可以将热点数据放在缓存里（如mecache或redis），加快访问速度，降低MySQL压力。 174 | 175 | ### 2.1.8 一个规范的建表语句示例 176 | 177 | 一个较为规范的建表语句为： 178 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 179 | CREATE TABLE user ( 180 | `id` bigint(11) NOT NULL AUTO_INCREMENT, 181 | `user_id` bigint(11) NOT NULL COMMENT ‘用户id’ 182 | `username` varchar(45) NOT NULL COMMENT '真实姓名', 183 | `email` varchar(30) NOT NULL DEFAULT '' COMMENT ‘用户邮箱’, 184 | `nickname` varchar(45) NOT NULL DEFAULT '' COMMENT '昵称', 185 | `avatar` int(11) NOT NULL COMMENT '头像', 186 | `birthday` date NOT NULL DEFAULT '1992-4-12' COMMENT '生日', 187 | `sex` tinyint(4) DEFAULT '0' COMMENT '性别0女1男', 188 | `short_introduce` varchar(150) DEFAULT NULL COMMENT '一句话介绍自己，最多50个汉字', 189 | `user_resume` varchar(300) NOT NULL COMMENT '用户提交的简历存放地址', 190 | `user_register_ip` int NOT NULL COMMENT ‘用户注册时的源ip’, 191 | `create_time` timestamp NOT NULL COMMENT ‘用户记录创建的时间’, 192 | `update_time` timestamp NOT NULL COMMENT ‘用户资料修改的时间’, 193 | `user_review_status` tinyint NOT NULL COMMENT ‘用户资料审核状态，1为通过，2为审核中，3为未通过，4为还未提交审核’, 194 | PRIMARY KEY (`id`), 195 | UNIQUE KEY `idx_user_id` (`user_id`), 196 | KEY `idx_username`(`username`), 197 | KEY `idx_create_time`(`create_time`,`user_review_status`) 198 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='网站用户基本信息'; 199 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 200 | 201 | 2.2 SQL编写 202 | ----------- 203 | 204 | ### 2.2.1 DML语句 205 | 206 | 1.【强制】SELECT语句必须指定具体字段名称，禁止写成“*” 207 | > 因为select * 会将不该读的数据也从MySQL里读出来，造成网卡压力。且表字段一旦更新，但model层没有来得及更新的话，系统会报错。 208 | 209 | 2.【强制】insert语句指定具体字段名称，不要写成insert into t1 values(…)，道理同上。 210 | 211 | 3.【建议】insert into…values(XX),(XX),(XX).. 这里XX的值不要超过5000个。 212 | > 值过多虽然上线很很快，但可能会引起主从同步延迟。 213 | 214 | 4.【建议】SELECT语句不要使用UNION，推荐使用UNION ALL，并且UNION子句个数限制在5个以内。 215 | > 因为union all不需要去重，节省数据库资源，提高性能。 216 | 217 | 5.【建议】in值列表限制在500以内。 218 | > 例如select… where userid in(….500个以内…)，这么做是为了减少底层扫描，减轻数据库压力从而加速查询。 219 | 220 | 6.【建议】事务里批量更新数据需要控制数量，进行必要的sleep，做到少量多次。 221 | 222 | 7.【强制】事务涉及的表必须全部是innodb表。 223 | > 否则一旦失败不会全部回滚，且易造成主从库同步终端。 224 | 225 | 8.【强制】写入和事务发往主库，只读SQL发往从库。 226 | 227 | 9.【强制】除静态表或小表（100行以内），DML语句必须有where条件，且使用索引查找。 228 | 229 | 10.【强制】生产环境禁止使用hint，如sql_no_cache，force index，ignore key，straight join等。 230 | > 因为hint是用来强制SQL按照某个执行计划来执行，但随着数据量变化我们无法保证自己当初的预判是正确的，因此我们要相信MySQL优化器！ 231 | 232 | 11.【强制】where条件里等号左右字段类型必须一致，多数情况无法利用索引。 233 | 234 | 12.【建议】SELECT|UPDATE|DELETE|REPLACE要有WHERE子句，且WHERE子句的条件必需使用索引查找。 235 | 236 | 13.【强制】生产数据库中强烈不推荐大表上发生全表扫描，但对于100行以下的静态表可以全表扫描。查询数据量不要超过表行数的25%，否则不会利用索引。 237 | 238 | 14.【强制】WHERE 子句中禁止只使用全模糊的LIKE条件进行查找，必须有其他等值或范围查询条件，否则无法利用索引。 239 | 240 | 15.【建议】索引列不要使用函数或表达式，否则无法利用索引。 241 | > 如where length(name)=’Admin’或where user_id+2=10023。 242 | 243 | 16.【建议】减少使用or语句，可将or语句优化为union，然后在各个where条件上建立索引。 244 | > 如where a=1 or b=2优化为where a=1… union …where b=2, key(a),key(b) 245 | 246 | 17.【建议】分页查询，当limit起点较高时，可先用过滤条件进行过滤。 247 | > 如select a,b,c from t1 limit 10000,20;优化为: 248 | > Select a,b,c from t1 where id>10000 limit 20; 249 | 250 | ### 2.2.2 多表连接 251 | 252 | 1.【强制】禁止跨db的join语句。 253 | > 因为这样可以减少模块间耦合，为数据库拆分奠定坚实基础。 254 | 255 | 2.【强制】禁止在业务的更新类SQL语句中使用join，比如update t1 join t2… 256 | 257 | 3.【建议】不建议使用子查询，建议将子查询SQL拆开结合程序多次查询，或使用join来代替子查询。 258 | 259 | 4.【建议】线上环境，多表join不要超过3个表。 260 | 261 | 5.【建议】多表连接查询推荐使用别名，且SELECT列表中要用别名引用字段，数据库.表格式， 262 | > 如“select a from db1.table1 alias1 where …” 263 | 264 | 6.【建议】在多表join中，尽量选取结果集较小的表作为驱动表，来join其他表。 265 | 266 | ### 2.2.3 事务 267 | 268 | 1.【建议】事务中INSERT|UPDATE|DELETE|REPLACE语句操作的行数控制在2000以内，以及WHERE子句中IN列表的传参个数控制在500以内。 269 | 270 | 2.【建议】批量操作数据时，需要控制事务处理间隔时间，进行必要的sleep，一般建议值5-10秒。 271 | 272 | 3.【建议】对于有auto_increment属性字段的表的插入操作，并发需要控制在200以内。 273 | 274 | 4.【强制】程序设计必须考虑“数据库事务隔离级别”带来的影响，包括脏读、不可重复读和幻读。 275 | > 线上建议事务隔离级别为repeatable-read，在binlog日志row格式下针对不同业务场景可以考虑read-committed。 276 | 277 | 5.【建议】事务里包含SQL不超过5个（支付业务除外） 278 | > 因为过长的事务会导致锁数据较久，MySQL内部缓存、连接消耗过多等雪崩问题。 279 | 280 | 6.【建议】事务里更新语句尽量基于主键或unique key 281 | > 如update … where id=XX; 否则会产生间隙锁，内部扩大锁定范围，导致系统性能下降，产生死锁。 282 | 283 | 7.【建议】尽量把一些典型外部调用移出事务，如调用webservice，访问文件存储等，从而避免事务过长。 284 | 285 | 8.【建议】对于MySQL主从延迟严格敏感的select语句，请开启事务强制访问主库。 286 | 287 | ### 2.2.4 排序和分组 288 | 289 | 1.【建议】减少使用order by，和业务沟通能不排序就不排序，或将排序放到程序端去做。 290 | > Order by、group by、distinct这些语句较为耗费CPU，数据库的CPU资源是极其宝贵的。 291 | 292 | 2.【建议】order by、group by、distinct这些SQL尽量利用索引直接检索出排序好的数据。 293 | > 如where a=1 order by可以利用key(a,b)。 294 | 295 | 3.【建议】包含了order by、group by、distinct这些查询的语句，where条件过滤出来的结果集请保持在1000行以内，否则SQL会很慢。 296 | 297 | ### 2.2.5 线上业务禁止使用的SQL语句 298 | 299 | 1.【高危】禁用update|delete t1 … where a=XX limit XX; 这种带limit的更新语句。 300 | > 非row格式binlog因为会导致主从不一致，导致数据错乱。建议加上order by PK 301 | 302 | 2.【高危】禁止使用关联子查询 303 | > update t1 set … where name in(select name from user where…);效率极其低下。 304 | 305 | 3.【强制】禁用procedure、function、trigger、views、event、外键约束。 306 | > 因为他们消耗数据库资源，降低数据库集群可扩展性。推荐都在程序端实现，OLAP业务系统由资深dba来具体把控是否使用。 307 | 308 | 4.【强制】禁用insert into …on duplicate key update… 309 | > 在高并发环境下，非row格式binlog会造成主从不一致。 310 | 311 | 5.【强制】禁止联表更新语句 312 | > 如update t1,t2 where t1.id=t2.id… 313 | -------------------------------------------------------------------------------- /mysql-snapshot/db_pool.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #coding=utf-8 3 | 4 | import logging 5 | import traceback 6 | import pymysql 7 | import warnings 8 | from retry import retry 9 | warnings.filterwarnings("ignore") 10 | 11 | from DBUtils.PooledDB import PooledDB 12 | 13 | db_pool_ins = None 14 | 15 | 16 | class DBPool: 17 | def __init__(self, host=None, port=None, user=None, password=None): 18 | self.host = host 19 | self.host = port 20 | self.host = user 21 | self.host = password 22 | self.pool = PooledDB(creator=pymysql, mincached=1, maxcached=10, maxconnections=10, maxusage=0, 23 | blocking=True, host=host, port=int(port), user=user, passwd=password, 24 | db="information_schema", charset='utf8') 25 | def get_connection(self): 26 | return self.pool.connection() 27 | 28 | 29 | class DBAction: 30 | def __init__(self, conn_setting=None): 31 | host = conn_setting['host'] 32 | port = conn_setting['port'] 33 | user = conn_setting['user'] 34 | password = conn_setting['password'] 35 | global db_pool_ins 36 | if db_pool_ins is None: 37 | db_pool_ins = DBPool(host, port, user, password) 38 | self.conn = db_pool_ins.get_connection() 39 | self.cursor = self.conn.cursor() 40 | 41 | def close_database(self): 42 | self.cursor.close() 43 | self.conn.close() 44 | 45 | def data_operate(self, sql, params=()): 46 | """ 47 | 数据的插入，更新，删除 48 | """ 49 | try: 50 | self.cursor.execute(sql, params) 51 | self.conn.commit() 52 | return True 53 | except: 54 | logging.error("sql is %s, params is %s error. %s" % (sql, params, traceback.format_exc())) 55 | self.conn.rollback() 56 | raise Exception 57 | 58 | def data_operate_count(self, sql, params=()): 59 | """ 60 | 数据的插入，更新，删除 61 | :return: 受影响的条数 62 | """ 63 | count = self.cursor.execute(sql, params) 64 | self.conn.commit() 65 | return count 66 | 67 | @retry(delay=1) 68 | def data_inquiry(self, sql, params=()): 69 | """ 70 | SELECT 操作 71 | """ 72 | self.cursor.execute(sql, params) 73 | result = self.cursor.fetchall() 74 | description = self.cursor.description 75 | return result, description 76 | 77 | def commit(self): 78 | self.conn.commit() 79 | 80 | def rollback(self): 81 | self.conn.rollback() 82 | -------------------------------------------------------------------------------- /mysql-snapshot/pip3.6-requirements.txt: -------------------------------------------------------------------------------- 1 | configparser==3.7.4 2 | DBUtils==1.3 3 | decorator==4.4.0 4 | psutil==5.6.3 5 | py==1.8.0 6 | PyMySQL==0.9.3 7 | retry==0.9.2 -------------------------------------------------------------------------------- /mysql-snapshot/readme.md: -------------------------------------------------------------------------------- 1 | # MySQL Snapshot（MySQL数据库快照） 2 | 3 | > 4 | > 当MySQL数据库出现性能瓶颈时，可根据预设条件对当前数据库状态做一份快照，方便DBA进行排查。 5 | > 6 | > 本程序用Python开发，运行在Python 3.6环境中。 7 | > 8 | 9 | ## 1. 安装部署 10 | ** 1.1 设置Python运行环境 11 | 说明： 12 | ```python 13 | pip3.6 install -r pip3.6-requirements.txt 14 | ``` 15 | 16 | ** 1.2 在MySQL里创建专用账号 17 | ``` 18 | CREATE USER snapshot@'%' IDENTIFIED WITH mysql_native_password by '76950ed38752421e29c397beff9157ac'; 19 | GRANT SELECT, PROCESS, REPLICATION CLIENT ON *.* TO `snapshot`@`%`; 20 | GRANT SELECT, EXECUTE ON `sys`.* TO `snapshot`@`%`; 21 | ``` 22 | **备注：** 23 | 1. 必须使用 mysql_native_password，Python 3.6还不支持 caching_sha2_password 插件。 24 | 2. 创建账号的允许连接IP及密码请自行修改。 25 | 26 | ** 1.3 运行程序（示例） 27 | 28 | 29 | ```python 30 | cd /data/mysqldba/mysql-snapshot; 31 | python3.6 ./snapshot.py --host=127.0.0.1 --port=3306 --user=snapshot --password='37b7412fef6f48b' --conditions="{'Threads_connected': 500, 'Threads_running': 20, 'Innodb_row_lock_current_waits': 5, 'Slow_queries': 5, 'Innodb_buffer_pool_wait_free': 5, 'cpu_user':10, 'cpu_sys':10, 'cpu_iowait':5, 'sys_iops':5000, 'sql_delay':60}" --interval=30 --storedir=/data/mysqldba/mysql-snapshot/logs 32 | ``` 33 | 34 | 触发条件说明： 35 | 36 | - 'Threads_connected':500，MySQL实例总连接数超过500 37 | - 'Threads_running':20，MySQL实例当前活跃线程数超过20 38 | - 'Innodb_row_lock_current_waits':5，MySQL实例当前活跃行锁等待数超过5 39 | - 'Slow_queries':5，MySQL实例1秒内新增慢查询数大于5 40 | - 'Innodb_buffer_pool_wait_free':1，MySQL实例每秒产生ibp等待事件大于5 41 | - 'cpu_user':10，系统CPU %user消耗超过10% 42 | - 'cpu_sys':10，系统CPU %sys消耗超过10% 43 | - 'cpu_iowait':5，系统CPU %iowait消耗超过5% 44 | - 'sys_iops':5000，系统磁盘iops（读 + 写）总和超过5000 45 | - 'sql_delay':60，MySQL主从复制延迟时间超过60秒（取值自Seconds_Behind_Master，5.6后很不靠谱，不建议使用，未来会改进） 46 | 47 | 其他参数说明： 48 | - host、user、port、password均是指连接MySQL实例的几个相关参数，不多解释 49 | - interval=N，单位：秒，每隔N秒检查一次状态，判断达到触发条件的话就创建一份状态快照 50 | - storedir，指定快照日志存储的路径，默认路径：/data/mysqldba/mysql-snapshot/logs 51 | 52 | **生成快照文件** 53 | 54 | 规则： 55 | 56 | - 在日志目录`storedir`下，每次达到触发条件时，创建相应的时间戳目录 57 | - 在新建的目录中，保存MySQL当时的状态信息，主要包括：系统负载数据，MySQL连接、事务、锁、错误日志、慢查询日志等信息 58 | 59 | 60 | ```shell 61 | [root@~]# ls -l 62 | -rw-r--r--. 1 root root 13635 Dec 9 02:30 innodb_status 63 | -rw-r--r--. 1 root root 42188 Dec 9 02:30 interrupts 64 | -rw-r--r--. 1 root root 10841 Dec 9 02:31 iostat 65 | -rw-r--r--. 1 root root 1312 Dec 9 02:30 meminfo 66 | -rw-r--r--. 1 root root 28560 Dec 9 02:31 mpstat 67 | -rw-r--r--. 1 root root 8841 Dec 9 02:30 netstat 68 | -rw-r--r--. 1 root root 6246 Dec 9 02:30 processlist 69 | -rw-r--r--. 1 root root 29247 Dec 9 02:30 ps 70 | -rw-r--r--. 1 root root 1415 Dec 9 02:30 slave_status 71 | -rw-r--r--. 1 root root 612 Dec 9 02:30 slowlog 72 | -rw-r--r--. 1 root root 13120 Dec 9 02:30 status 73 | -rw-r--r--. 1 root root 1079 Dec 9 02:30 top 74 | -rw-r--r--. 1 root root 21676 Dec 9 02:30 transactions 75 | -rw-r--r--. 1 root root 18300 Dec 9 02:30 variables 76 | -rw-r--r--. 1 root root 2950 Dec 9 02:31 vmstat 77 | ``` 78 | 79 | 如果在创建快照是加上tcpdump的结果（默认没启用），可以修改源码文件 ```snapshot.py``` 约 85~88 行，修改成如下内容即可： 80 | ```Python 81 | #旧 82 | """ 83 | sys_func_list = [mysql_slow_log, mysql_error_log, system_message, system_dmesg, system_top, system_iostat, system_mpstat, 84 | system_mem_info, system_interrupts, system_ps, system_netstat, system_vmstat] 85 | sys_arg_list = [slow_log, error_log, '/var/log/messages', '/var/log/dmesg', '', self.interval, self.interval, 86 | '', '', '', '', self.interval] 87 | """ 88 | 89 | #新 90 | sys_func_list = [mysql_slow_log, mysql_error_log, system_message, system_dmesg, system_top, system_iostat, system_mpstat, 91 | system_mem_info, system_interrupts, system_ps, system_netstat, system_vmstat, system_tcpdump] 92 | sys_arg_list = [slow_log, error_log, '/var/log/messages', '/var/log/dmesg', '', self.interval, self.interval, 93 | '', '', '', '', self.interval, self.conn_setting['port']] 94 | ``` 95 | 96 | 97 | 说明：有bug请提交[issue](https://github.com/zhishutech/mysqldba/issues)，谢谢。 98 | 99 | 本人QQ：573242930 100 | -------------------------------------------------------------------------------- /mysql-snapshot/snapshot.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*-# 2 | 3 | # ------------------------------------------------------------------------------- 4 | # Name: snapshot 5 | # Description: 6 | # Author: xucl 7 | # Date: 2019-04-17 8 | # ------------------------------------------------------------------------------- 9 | 10 | from utils import * 11 | from db_pool import DBAction 12 | from threading import Lock 13 | 14 | #日至保留时长，默认7天 15 | expire_logs_days=7 16 | 17 | class Snapshot(object): 18 | def __init__(self, connection_settings, interval=None, conditions=None, storedir=None): 19 | global logfile 20 | if not conditions: 21 | raise ValueError('Lack of parameter: conditions') 22 | if not storedir: 23 | raise ValueError('Lack of parameter: storedir') 24 | self.conn_setting = connection_settings 25 | if not interval: 26 | self.interval = 10 27 | else: 28 | self.interval = interval 29 | self.conditions = conditions 30 | self.stordir = storedir 31 | print("") 32 | print("starting mysql snapshot") 33 | print("loging into %s"%logfile) 34 | print("snapshot file stored in %s"%storedir) 35 | 36 | def run(self): 37 | condition_dict = eval(self.conditions) 38 | 39 | while True: 40 | lock = Lock() 41 | global dbaction 42 | 43 | # 第一次获取MySQL&系统状态 44 | dbaction = DBAction(self.conn_setting) 45 | status_dict1 = get_mysql_status(dbaction) 46 | slow_log, error_log = get_log_dir(dbaction) 47 | slave_status_dict = get_slave_status(dbaction) 48 | 49 | sys_dict1 = get_sys_status() 50 | 51 | time.sleep(1) 52 | 53 | # 1秒钟后再次获取MySQL状态 54 | status_dict2 = get_mysql_status(dbaction) 55 | 56 | # 以下指标只需根据原始值判断是否达到触发条件 57 | origin_status_list = ['Threads_connected', 'Threads_running', 'Innodb_row_lock_current_waits'] 58 | # 以下指标需要根据两次差值进行判断是否达到触发条件 59 | diff_status_list = ['Slow_queries', 'Innodb_buffer_pool_wait_free'] 60 | 61 | # 构造数组 62 | origin_status_dict = get_origin_status( status_dict1, origin_status_list) 63 | 64 | # 计算两次MySQL状态中的差值 65 | diff_status_dict = get_diff_status( status_dict1, status_dict2, diff_status_list) 66 | 67 | # 传入要检查的状态指标(MySQL、系统、slave status等) 68 | check_dict = dict(origin_status_dict, ** diff_status_dict, ** sys_dict1, **slave_status_dict) 69 | 70 | # 检查哪些条件被触发 71 | collect_flag = check_conditions(check_dict, condition_dict) 72 | 73 | time_now = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') 74 | 75 | if collect_flag: 76 | filedir = create_unique_dir(self.stordir, time_now) 77 | lock.acquire() 78 | thread_objs = [] 79 | 80 | # 采集MySQL状态函数&传入参数列表 81 | mysql_func_list = [mysql_variables, mysql_status, mysql_innodb_status, 82 | mysql_slave_status, mysql_processlist, mysql_transactions, mysql_lock_info] 83 | 84 | #采集系统状态函数&传入参数列表 85 | """ 86 | sys_func_list = [mysql_slow_log, mysql_error_log, system_message, system_dmesg, system_top, system_iostat, system_mpstat, 87 | system_mem_info, system_interrupts, system_ps, system_netstat, system_vmstat, system_tcpdump] 88 | sys_arg_list = [slow_log, error_log, '/var/log/messages', '/var/log/dmesg', '', self.interval, self.interval, 89 | '', '', '', '', self.interval, self.conn_setting['port']] 90 | """ 91 | sys_func_list = [mysql_slow_log, mysql_error_log, system_message, system_dmesg, system_top, system_mem_info, 92 | system_interrupts, system_ps, system_netstat, system_iostat, system_mpstat, system_vmstat] 93 | sys_arg_list = [slow_log, error_log, '/var/log/messages', '/var/log/dmesg', '', '', '', '', '', '', '', ''] 94 | 95 | #循环执行 96 | for func in mysql_func_list: 97 | dbaction = DBAction(self.conn_setting) 98 | t = do_in_thread(func, dbaction, filedir) 99 | thread_objs.append(t) 100 | 101 | for index, func in enumerate(sys_func_list): 102 | if sys_arg_list[index]: 103 | t = do_in_thread(func, sys_arg_list[index], filedir) 104 | else: 105 | t = do_in_thread(func, filedir) 106 | thread_objs.append(t) 107 | 108 | for thread_obj in thread_objs: 109 | thread_obj.join() 110 | 111 | #清除过期日志，如果不放心可注释掉改成手动清理 112 | self.clear_expire_logs() 113 | 114 | lock.release() 115 | 116 | # 周期休眠 117 | time.sleep(self.interval) 118 | 119 | 120 | # 清除过期日志 121 | def clear_expire_logs(self): 122 | global expire_logs_days 123 | print("clear logs befor %s days" % expire_logs_days) 124 | cmd = "cd %s; find ./ -type d -mtime +%s | xargs rm -fr" % (self.stordir, expire_logs_days) 125 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 126 | pro.terminate() 127 | 128 | if __name__ == '__main__': 129 | args = command_line_args(sys.argv[1:]) 130 | conn_setting = {'host': args.host, 'port': args.port, 131 | 'user': args.user, 'password': args.password, 'charset': 'utf8'} 132 | snapshot = Snapshot(connection_settings=conn_setting, interval=args.interval, conditions=args.conditions, 133 | storedir=args.storedir) 134 | snapshot.run() 135 | -------------------------------------------------------------------------------- /mysql-snapshot/utils.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | 3 | import time 4 | import threading 5 | import getpass 6 | import argparse 7 | import sys 8 | import os 9 | import datetime 10 | import traceback 11 | import subprocess 12 | import psutil 13 | import logging 14 | from collections import Counter 15 | 16 | # 日志文件 17 | logfile="/tmp/mysql-snapshot.log" 18 | # vmstat/iostat/mpstat run time 5s 19 | stats_log_sec=5 20 | 21 | # logging配置 22 | logging.basicConfig(level=logging.DEBUG, 23 | format='%(asctime)s %(filename)s %(levelname)s %(message)s', 24 | datefmt='%a,%d %b %Y %H:%M:%S', 25 | filename=logfile, 26 | filemode='a') 27 | 28 | 29 | class FuncThread(threading.Thread): 30 | 31 | def __init__(self, func, *args, **kwargs): 32 | super(FuncThread, self).__init__() 33 | self.func = func 34 | self.args = args 35 | self.kwargs = kwargs 36 | self.finished = False 37 | self.result = None 38 | 39 | def run(self): 40 | self.result = self.func(*self.args, **self.kwargs) 41 | self.finished = True 42 | 43 | def is_finished(self): 44 | return self.finished 45 | 46 | def get_result(self): 47 | return self.result 48 | 49 | 50 | def do_in_thread(func, *args, **kwargs): 51 | ft = FuncThread(func, *args, **kwargs) 52 | ft.start() 53 | return ft 54 | 55 | 56 | def handle_timeout(func, timeout, *args, **kwargs): 57 | interval = 1 58 | 59 | ret = None 60 | while timeout > 0: 61 | begin_time = time.time() 62 | ret = func(*args, **kwargs) 63 | if ret: 64 | break 65 | time.sleep(interval) 66 | timeout -= time.time() - begin_time 67 | 68 | return ret 69 | 70 | 71 | def create_unique_dir(dirname, snaptime): 72 | result_dir = dirname + '/' + snaptime 73 | # if filedir is not exist than create 74 | if not os.path.exists(result_dir): 75 | os.mkdir(result_dir) 76 | return result_dir 77 | 78 | 79 | def parse_args(): 80 | """parse args for snapshot""" 81 | 82 | parser = argparse.ArgumentParser(description='Snapshot your Database and Server', add_help=False) 83 | connect_setting = parser.add_argument_group('connect setting') 84 | connect_setting.add_argument('-h', '--host', dest='host', type=str, 85 | help='Host the MySQL database server located', default='127.0.0.1') 86 | connect_setting.add_argument('-u', '--user', dest='user', type=str, 87 | help='MySQL Username to log in as', default='root') 88 | connect_setting.add_argument('-p', '--password', dest='password', type=str, nargs='*', 89 | help='MySQL Password to use', default='') 90 | connect_setting.add_argument('-P', '--port', dest='port', type=int, 91 | help='MySQL port to use', default=3306) 92 | 93 | parser.add_argument('--interval', dest='interval', type=int, default=10, 94 | help="interval time to check snapshot condition") 95 | parser.add_argument('--help', dest='help', action='store_true', help='help information', default=False) 96 | parser.add_argument('--conditions', dest='conditions', default=False, 97 | help="Specify trigger conditions") 98 | parser.add_argument('--storedir', dest='storedir', default=False, 99 | help="Specify datadir to store snapshot files") 100 | return parser 101 | 102 | 103 | def command_line_args(args): 104 | need_print_help = False if args else True 105 | parser = parse_args() 106 | args = parser.parse_args(args) 107 | if args.help or need_print_help: 108 | parser.print_help() 109 | sys.exit(1) 110 | if not args.conditions: 111 | raise ValueError('Lack of parameter: conditions') 112 | if not args.storedir: 113 | raise ValueError('Lack of parameter: storedir') 114 | if not args.password: 115 | args.password = getpass.getpass() 116 | else: 117 | args.password = args.password[0] 118 | return args 119 | 120 | 121 | def dt2str(data): 122 | if isinstance(data, datetime.datetime): 123 | return data.strftime('%Y-%m-%d %H:%M:%S') 124 | else: 125 | return str(data) 126 | 127 | 128 | def mysql_variables(dbaction, filedir): 129 | filename = filedir + '/' + 'variables' 130 | sql = 'select * from performance_schema.global_variables;' 131 | try: 132 | logging.info('开始记录MySQL variables') 133 | var_obj, desc = dbaction.data_inquiry(sql) 134 | except: 135 | error_msg = str(traceback.format_exc()) 136 | logging.info('连接池获取连接出错:%s' % error_msg) 137 | var_obj = () 138 | if var_obj: 139 | with open(filename, 'w') as f: 140 | for item in var_obj: 141 | var_string = item[0] + ':' + item[1] + '\n' 142 | f.write(var_string) 143 | 144 | 145 | def mysql_status(dbaction, filedir): 146 | filename = filedir + '/' + 'status' 147 | sql = 'show global status' 148 | try: 149 | logging.info('开始记录MySQL status') 150 | status_obj, desc = dbaction.data_inquiry(sql) 151 | except: 152 | error_msg = str(traceback.format_exc()) 153 | logging.info('连接池获取连接出错:%s' % error_msg) 154 | status_obj = () 155 | if status_obj: 156 | with open(filename, 'w') as f: 157 | for item in status_obj: 158 | status_string = item[0] + ':' + item[1] + '\n' 159 | f.write(status_string) 160 | 161 | 162 | def mysql_innodb_status(dbaction, filedir): 163 | filename = filedir + '/' + 'innodb_status' 164 | sql = 'show engine innodb status' 165 | try: 166 | logging.info('开始记录innodb status') 167 | status_obj, desc = dbaction.data_inquiry(sql) 168 | except: 169 | error_msg = str(traceback.format_exc()) 170 | logging.info('连接池获取连接出错:%s' % error_msg) 171 | status_obj = () 172 | if status_obj: 173 | status_list = str(status_obj[0]).split('\\n')[1:-1] 174 | with open(filename, 'w') as f: 175 | for item in status_list: 176 | status_string = item + '\n' 177 | f.write(status_string) 178 | 179 | 180 | def mysql_slave_status(dbaction, filedir): 181 | filename = filedir + '/' + 'slave_status' 182 | sql = 'show slave status' 183 | try: 184 | logging.info('开始记录slave status') 185 | status_obj, desc = dbaction.data_inquiry(sql) 186 | except: 187 | error_msg = str(traceback.format_exc()) 188 | logging.info('连接池获取连接出错:%s' % error_msg) 189 | status_obj = () 190 | desc_list = [] 191 | if status_obj: 192 | with open(filename, 'w') as f: 193 | for item in desc: 194 | desc_list.append(item[0]) 195 | for index, item in enumerate(status_obj[0]): 196 | row_string = desc_list[index] + ':' + str(item) + '\n' 197 | f.write(row_string) 198 | 199 | 200 | def mysql_processlist(dbaction, filedir): 201 | filename = filedir + '/' + 'processlist' 202 | sql = 'show full processlist' 203 | try: 204 | logging.info('开始记录processlist') 205 | processlist_obj, desc = dbaction.data_inquiry(sql) 206 | except: 207 | error_msg = str(traceback.format_exc()) 208 | logging.info('连接池获取连接出错:%s' % error_msg) 209 | processlist_obj = () 210 | if processlist_obj: 211 | with open(filename, 'w') as f: 212 | for item in desc: 213 | desc_string = item[0] + ' ' 214 | f.write(desc_string) 215 | f.write('\n') 216 | for item in processlist_obj: 217 | processlist_string = map(lambda x: str(x) + ' ', item) 218 | f.write(''.join(processlist_string) + '\n') 219 | 220 | 221 | def mysql_transactions(dbaction, filedir): 222 | filename = filedir + '/' + 'transactions' 223 | desc_list = [] 224 | sql = 'select * from information_schema.innodb_trx' 225 | try: 226 | logging.info('开始记录transactions') 227 | trans_obj, desc = dbaction.data_inquiry(sql) 228 | except: 229 | error_msg = str(traceback.format_exc()) 230 | logging.info('连接池获取连接出错:%s' % error_msg) 231 | trans_obj = () 232 | 233 | if trans_obj: 234 | with open(filename, 'w') as f: 235 | for item in desc: 236 | desc_list.append(item[0]) 237 | for index, trans in enumerate(trans_obj): 238 | row_string = '************** rows: %s **************** \n' % str(index) 239 | f.write(row_string) 240 | for index, item in enumerate(trans): 241 | trans_string = desc_list[index] + ':' + dt2str(item) + '\n' 242 | f.write(trans_string) 243 | 244 | 245 | def mysql_lock_info(dbaction, filedir): 246 | filename = filedir + '/' + 'innodb_locks' 247 | desc_list = [] 248 | sql = 'select * from sys.innodb_lock_waits' 249 | try: 250 | logging.info('开始记录innodb locks') 251 | lock_obj, desc = dbaction.data_inquiry(sql) 252 | except: 253 | error_msg = str(traceback.format_exc()) 254 | logging.info('连接池获取连接出错:%s' % error_msg) 255 | lock_obj = () 256 | if lock_obj: 257 | with open(filename, 'w') as f: 258 | for item in desc: 259 | desc_list.append(item[0]) 260 | for index, trans in enumerate(lock_obj): 261 | row_string = '************** rows: %s **************** \n' % str(index) 262 | f.write(row_string) 263 | for index, item in enumerate(trans): 264 | trans_string = desc_list[index] + ':' + dt2str(item) + '\n' 265 | f.write(trans_string) 266 | 267 | 268 | def mysql_error_log(errorlog, filedir): 269 | error_file = '%s/errorlog' % filedir 270 | start_pos = os.path.getsize(errorlog) 271 | time.sleep(5) 272 | stop_pos = os.path.getsize(errorlog) 273 | offset_size = int(stop_pos) - int(start_pos) 274 | if offset_size: 275 | f = open(errorlog, "r") 276 | f.seek(int(start_pos), 0) 277 | error_text = f.read(offset_size) 278 | f.close() 279 | fp = open(error_file, "w+") 280 | error_list = error_text.strip('\n').split('\n') 281 | for line in error_list: 282 | line = line + '\n' 283 | fp.write(line) 284 | fp.close() 285 | 286 | 287 | def mysql_slow_log(slowlog, filedir): 288 | slow_file = '%s/slowlog' % filedir 289 | start_pos = os.path.getsize(slowlog) 290 | time.sleep(5) 291 | stop_pos = os.path.getsize(slowlog) 292 | offset_size = int(stop_pos) - int(start_pos) 293 | if offset_size: 294 | f = open(slowlog, "r") 295 | f.seek(int(start_pos), 0) 296 | slow_text = f.read(offset_size) 297 | f.close() 298 | fp = open(slow_file, "w+") 299 | slow_list = slow_text.strip('\n').split('\n') 300 | for line in slow_list: 301 | line = line + '\n' 302 | fp.write(line) 303 | fp.close() 304 | 305 | 306 | def system_disk_space(filedir): 307 | logging.info('开始记录磁盘空间') 308 | cmd = 'df -k >> %s/disk-space' % filedir 309 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 310 | pro.terminate() 311 | 312 | 313 | def system_message(message, filedir): 314 | logging.info('开始记录message') 315 | message_file = '%s/message' % filedir 316 | start_pos = os.path.getsize(message) 317 | time.sleep(5) 318 | stop_pos = os.path.getsize(message) 319 | offset_size = int(stop_pos) - int(start_pos) 320 | if offset_size: 321 | f = open(message, "r") 322 | f.seek(int(start_pos), 0) 323 | message_text = f.read(offset_size) 324 | f.close() 325 | fp = open(message_file, "w+") 326 | message_list = message_text.strip('\n').split('\n') 327 | for line in message_list: 328 | line = line + '\n' 329 | fp.write(line) 330 | fp.close() 331 | 332 | 333 | def system_dmesg(dmesg, filedir): 334 | logging.info('开始记录dmesg') 335 | dmesg_file = '%s/dmesg' % filedir 336 | start_pos = os.path.getsize(dmesg) 337 | time.sleep(5) 338 | stop_pos = os.path.getsize(dmesg) 339 | offset_size = int(stop_pos) - int(start_pos) 340 | if offset_size: 341 | f = open(dmesg, "r") 342 | f.seek(int(start_pos), 0) 343 | dmesg_text = f.read(offset_size) 344 | f.close() 345 | fp = open(dmesg_file, "w+") 346 | dmesg_list = dmesg_text.strip('\n').split('\n') 347 | for line in dmesg_list: 348 | line = line + '\n' 349 | fp.write(line) 350 | fp.close() 351 | 352 | 353 | def system_top(filedir): 354 | logging.info('开始记录top输出') 355 | cmd = 'top -bn5 | head -n 15 >> %s/top' % filedir 356 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 357 | if pro.poll() != None: 358 | ErrCode = "top error" 359 | print(ErrCode) 360 | else: 361 | ErrCode = "top success" 362 | time.sleep(5) 363 | pro.terminate() 364 | 365 | 366 | def system_iostat(filedir): 367 | global stats_log_sec 368 | logging.info('开始记录iostat信息') 369 | cmd = 'iostat -m -x 1 %s >> %s/iostat' % (stats_log_sec, filedir) 370 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 371 | if pro.poll() != None: 372 | ErrCode = "iostat error" 373 | print(ErrCode) 374 | else: 375 | ErrCode = "iostat success" 376 | time.sleep(5) 377 | 378 | pro.terminate() 379 | 380 | 381 | def system_mpstat(filedir): 382 | global stats_log_sec 383 | logging.info('开始记录mpstat信息') 384 | cmd = 'mpstat -I SUM -P ALL 1 %s >> %s/mpstat' % (stats_log_sec, filedir) 385 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 386 | if pro.poll() != None: 387 | ErrCode = "mpstat error" 388 | print(ErrCode) 389 | else: 390 | ErrCode = "mpstat success" 391 | time.sleep(5) 392 | 393 | pro.terminate() 394 | 395 | 396 | def system_tcpdump(mysqld_port, filedir): 397 | logging.info('开始记录tcpdump信息') 398 | cmd = 'tcpdump -i any -s 4096 -c 200 -w %s/tcpdump port %s' % (filedir, mysqld_port) 399 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 400 | if pro.poll() != None: 401 | ErrCode = "tcpdump error" 402 | print(ErrCode) 403 | else: 404 | ErrCode = "tcpdump success" 405 | time.sleep(5) 406 | pro.terminate() 407 | 408 | 409 | def system_mem_info(filedir): 410 | logging.info('开始记录内存信息') 411 | cmd = 'cat /proc/meminfo >> %s/meminfo' % filedir 412 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 413 | pro.terminate() 414 | 415 | 416 | def system_interrupts(filedir): 417 | logging.info('开始记录中断信息') 418 | cmd = 'cat /proc/interrupts >> %s/interrupts' % filedir 419 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 420 | pro.terminate() 421 | 422 | 423 | def system_ps(filedir): 424 | logging.info('开始记录ps信息') 425 | cmd = 'ps -eaF >> %s/ps' % filedir 426 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 427 | pro.terminate() 428 | 429 | 430 | def system_netstat(filedir): 431 | logging.info('开始记录netstat信息') 432 | cmd = 'netstat -antp >> %s/netstat' % filedir 433 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 434 | pro.terminate() 435 | 436 | 437 | def system_vmstat(filedir): 438 | global stats_log_sec 439 | logging.info('开始记录vmstat信息') 440 | cmd = 'vmstat 1 %s >> %s/vmstat' % (stats_log_sec, filedir) 441 | pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 442 | if pro.poll() != None: 443 | ErrCode = "vmstat error" 444 | print(ErrCode) 445 | else: 446 | ErrCode = "vmstat success" 447 | time.sleep(5) 448 | pro.terminate() 449 | 450 | 451 | def check_conditions(check_dict, condition_dict): 452 | logging.info('开始检查触发条件') 453 | result = dict(Counter(check_dict) - Counter(condition_dict)) 454 | print("condition: %s" % ([{item:check_dict[item]} for item in result.keys()])) 455 | logging.info('触发条件: %s' % ([{item:check_dict[item]} for item in result.keys()])) 456 | return result 457 | 458 | 459 | def get_mysql_status(dbaction): 460 | status_dict = {} 461 | sql = 'show global status;' 462 | try: 463 | logging.info('开始获取mysql status') 464 | status_obj, desc = dbaction.data_inquiry(sql) 465 | except: 466 | error_msg = str(traceback.format_exc()) 467 | logging.info('连接池获取连接出错:%s' % error_msg) 468 | status_obj = () 469 | 470 | if status_obj: 471 | for item in status_obj: 472 | status_dict[item[0]] = item[1] 473 | return status_dict 474 | else: 475 | return {} 476 | 477 | 478 | def get_origin_status(status_dict, origin_status_list): 479 | origin_status_dict = {} 480 | for item in origin_status_list: 481 | origin_status_dict[item] = int(status_dict[item]) 482 | return origin_status_dict 483 | 484 | 485 | def get_origin_sys_status(status_dict, origin_status_list): 486 | origin_status_dict = {} 487 | for item in origin_status_list: 488 | origin_status_dict[item] = float(status_dict[item]) 489 | return origin_status_dict 490 | 491 | 492 | def get_diff_status(status_dict1, status_dict2, diff_status_list): 493 | diff_status_dict = {} 494 | for item in diff_status_list: 495 | diff_status_dict[item] = int(status_dict2[item]) - int(status_dict1[item]) 496 | return diff_status_dict 497 | 498 | 499 | def get_sys_diff_status(status_dict1, status_dict2, diff_status_list): 500 | diff_status_dict = {} 501 | for item in diff_status_list: 502 | diff_status_dict[item] = float(status_dict2[item]) - float(status_dict1[item]) 503 | return diff_status_dict 504 | 505 | 506 | def get_sys_status(): 507 | logging.info('开始获取系统状态') 508 | sys_status_dict = {} 509 | sys_status_dict['loadavg'] = psutil.getloadavg()[0] 510 | sys_status_dict['cpu_user'] = psutil.cpu_times_percent(interval=0.1,percpu=False).user 511 | sys_status_dict['cpu_sys'] = psutil.cpu_times_percent(interval=0.1,percpu=False).system 512 | sys_status_dict['cpu_iowait'] = psutil.cpu_times_percent(interval=0.1,percpu=False).iowait 513 | 514 | diskio1 = psutil.disk_io_counters(perdisk=False) 515 | time.sleep(1) 516 | diskio2 = psutil.disk_io_counters(perdisk=False) 517 | rio1, wio1 = diskio1.read_count, diskio1.write_count 518 | rio2, wio2 = diskio2.read_count, diskio2.write_count 519 | sys_status_dict['sys_iops'] = (rio2 - rio1) + (wio2 - wio1) 520 | 521 | return sys_status_dict 522 | 523 | 524 | def get_slave_status(dbaction): 525 | slave_status_dict = {} 526 | sql = 'show slave status' 527 | try: 528 | logging.info('开始获取slave status') 529 | status_obj, desc = dbaction.data_inquiry(sql) 530 | except: 531 | error_msg = str(traceback.format_exc()) 532 | logging.info('连接池获取连接出错:%s' % error_msg) 533 | if status_obj: 534 | if not status_obj[0][32]: 535 | #set default value = 0 536 | sql_delay = 0 537 | else: 538 | sql_delay = status_obj[0][32] 539 | slave_status_dict['sql_delay'] = sql_delay 540 | else: 541 | slave_status_dict['sql_delay'] = 0 542 | 543 | return slave_status_dict 544 | 545 | 546 | def get_log_dir(dbaction): 547 | sql = """select * from performance_schema.global_variables where VARIABLE_NAME in ('log_error', 'slow_query_log_file', 'datadir');""" 548 | try: 549 | logging.info('开始获取log位置') 550 | var_obj, desc = dbaction.data_inquiry(sql) 551 | except: 552 | error_msg = str(traceback.format_exc()) 553 | logging.info('连接池获取连接出错:%s' % error_msg) 554 | 555 | for item in var_obj: 556 | if item[0] == 'slow_query_log_file': 557 | slow_log = item[1] 558 | elif item[0] == 'log_error': 559 | error_log = item[1] 560 | else: 561 | data_dir = item[1] 562 | 563 | slow_log = data_dir + slow_log 564 | error_log = data_dir + error_log.split('/')[1] 565 | return slow_log, error_log 566 | -------------------------------------------------------------------------------- /mysql-tools/MySQL巡检怎么做？.md: -------------------------------------------------------------------------------- 1 | ## MySQL巡检怎么做？ 2 | 3 | 刚到新公司？刚来客户现场？有朋友请你帮忙优化数据库？如何快速对现有的实例有个大概的了解，下面我来带你从MySQL数据库层做一次快速的巡检。 4 | 5 | ### 一、巡检内容 6 | 7 | - 表巡检 8 | - 大小超过10G的表 9 | - 索引超过6个的表 10 | - 碎片率超过50%的表 11 | - 行数超过1000万行的表 12 | - 非默认字符集的表 13 | - 含有大字段的表 14 | - varchar定义超长的表 15 | - 无主键/索引的表 16 | - 索引巡检 17 | - 重复索引 18 | - 索引列超过5个的索引 19 | - 无用索引 20 | - 重要参数 21 | - version 22 | - innodb_buffer_pool_size 23 | - innodb_flush_log_at_trx_commit 24 | - innodb_log_file_size 25 | - innodb_log_files_in_group 26 | - innodb_file_per_table 27 | - innodb_max_dirty_pages_pct 28 | - sync_binlog 29 | - max_connections 30 | - query_cache_type 31 | - table_open_cache 32 | - table_definition_cache 33 | - 重要状态指标 34 | - Uptime 35 | - Opened_files 36 | - Opened_table_definitions 37 | - Opened_tables 38 | - Max_used_connections 39 | - Threads_created 40 | - Threads_connected 41 | - Aborted_connects 42 | - Aborted_clients 43 | - Table_locks_waited 44 | - Innodb_buffer_pool_wait_free 45 | - Innodb_log_waits 46 | - Table_locks_waited 47 | - Innodb_row_lock_waits 48 | - Innodb_row_lock_time_avg 49 | - Binlog_cache_disk_use 50 | - Created_tmp_disk_tables 51 | - 用户检查 52 | - 无密码用户 53 | - %用户 54 | - 权限检查 55 | 56 | ### 二、检查脚本 57 | 58 | git地址：https://github.com/zhishutech/mysqldba/blob/master/mysql-tools/check_mysql.py 59 | 60 | ### 三、如何使用检查脚本 61 | 62 | 3.1 创建巡检用户 63 | 64 | `grant select,process on *.* to monitor@localhost identified by '123456'` 65 | 66 | 3.2 巡检脚本填入相应ip、端口号、账号、密码 67 | 68 | 3.3 执行巡检 69 | 70 | ### 四、巡检效果 71 | 72 | ```version : 5.7.23-log 73 | innodb_buffer_pool_size : 134217728 74 | innodb_flush_log_at_trx_commit : 1 75 | innodb_log_file_size : 134217728 76 | innodb_log_files_in_group : 3 77 | innodb_file_per_table : ON 78 | innodb_max_dirty_pages_pct : 50.000000 79 | sync_binlog : 1 80 | max_connections : 512 81 | query_cache_type : OFF 82 | table_open_cache : 1024 83 | table_definition_cache : 1024 84 | Uptime : 1103289 85 | Opened_files : 4741 86 | Opened_table_definitions : 1155 87 | Opened_tables : 34582 88 | Max_used_connections : 8 89 | Threads_created : 8 90 | Threads_connected : 2 91 | Aborted_connects : 67417 92 | Aborted_clients : 674 93 | Table_locks_waited : 0 94 | Innodb_buffer_pool_wait_free : 0 95 | Innodb_log_waits : 0 96 | Innodb_row_lock_waits : 0 97 | Innodb_row_lock_time_avg : 0 98 | Binlog_cache_disk_use : 1 99 | Created_tmp_disk_tables : 256641 100 | historylength : 41 101 | Log sequence number : 1410769049 102 | Log flushed up to : 1410768979 103 | Last checkpoint at : 1410768970 104 | 检查无密码用户 105 | 结果不存在 106 | 检查%用户 107 | user: monitor host: % 108 | 检查用户权限 109 | GRANT REPLICATION CLIENT ON *.* TO 'monitor'@'%' 110 | GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' WITH GRANT OPTION 111 | GRANT PROXY ON ''@'' TO 'root'@'localhost' WITH GRANT OPTION 112 | ``` 113 | 114 | -------------------------------------------------------------------------------- /mysql-tools/README.md: -------------------------------------------------------------------------------- 1 | ## 数据库兼容性测试 2 | - 将线上sql放入sql.txt文件，可以在qa环境打开general log记录 3 | - 更改脚本内的连接地址、用户名、密码等信息 4 | - 执行脚本，如果sql在两边输出结果一致，返回true，否则返回false及对应的sql 5 | - 说明：只对sql结果的前1000行做md5 6 | -------------------------------------------------------------------------------- /mysql-tools/check_mysql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*-# 3 | # auth:xucl 4 | 5 | import sys 6 | import time 7 | from datetime import datetime 8 | import MySQLdb 9 | 10 | 11 | class DBUtil: 12 | def __init__(self, user=None, passwd=None, host=None, port=None, db=None): 13 | self.user = user 14 | self.passwd = passwd 15 | self.host = host 16 | self.port = port 17 | self.db = db 18 | self._conn = None 19 | self._cursor = None 20 | 21 | def __enter__(self): 22 | self._conn = MySQLdb.connect(host=self.host, port=self.port, user=self.user, passwd=self.passwd, db=self.db) 23 | self._cursor = self._conn.cursor() 24 | return self 25 | 26 | def __exit__(self, exc_type, exc_val, exc_tb): 27 | self._conn.close() 28 | self._conn = None 29 | 30 | def check_table_size(self): 31 | try: 32 | sql = "select table_schema,table_name,concat(round((data_length+index_length)/1024/1024,2),'M') FROM \ 33 | information_schema.tables where (DATA_LENGTH+INDEX_LENGTH) > 10*1024*1024*1024 and table_schema not in \ 34 | ('information_schema','mysql','performance_schema','sys')" 35 | self._cursor.execute(sql) 36 | result = self._cursor.fetchall() 37 | print('检查超过10G大小的表') 38 | if not result: 39 | print("结果不存在") 40 | for row in result: 41 | print(("schema: %s tablename: %s size: %s") % (row[0], row[1], row[2])) 42 | except Exception as e: 43 | raise (e) 44 | 45 | def check_table_index(self): 46 | try: 47 | sql = "select t1.name,t2.num from information_schema.innodb_sys_tables t1, (select table_id,count(*) as num from \ 48 | information_schema.innodb_sys_indexes group by table_id having count(*) >=6) t2 where t1.table_id =t2.table_id" 49 | self._cursor.execute(sql) 50 | result = self._cursor.fetchall() 51 | print('检查索引超过6个的表') 52 | if not result: 53 | print("结果不存在") 54 | for row in result: 55 | print() 56 | print(("schema: %s tablename: %s index_num: %s") % (row[0].split('/')[0], row[0].split('/')[1], row[1])) 57 | except Exception as e: 58 | raise (e) 59 | 60 | def check_table_fragment_pct(self): 61 | try: 62 | sql = "SELECT TABLE_SCHEMA as `db`, TABLE_NAME as `tbl`, \ 63 | 1-(TABLE_ROWS*AVG_ROW_LENGTH)/(DATA_LENGTH + INDEX_LENGTH + DATA_FREE) AS `fragment_pct` \ 64 | FROM information_schema.TABLES WHERE TABLE_SCHEMA not in ('information_schema','mysql','performance_schema','sys') \ 65 | and (1-(TABLE_ROWS*AVG_ROW_LENGTH)/(DATA_LENGTH + INDEX_LENGTH + DATA_FREE)) > 0.5 and (DATA_LENGTH + INDEX_LENGTH + DATA_FREE) > 1024*1024*1024 ;" 66 | self._cursor.execute(sql) 67 | result = self._cursor.fetchall() 68 | print('检查碎片率超过50%的表') 69 | if not result: 70 | print("结果不存在") 71 | for row in result: 72 | print(("schema: %s tablename: %s fragment_pct: %s") % (row[0], row[1], row[2])) 73 | except Exception as e: 74 | raise (e) 75 | 76 | def check_table_rows(self): 77 | try: 78 | sql = "select table_schema,table_name,table_rows from \ 79 | information_schema.TABLES where table_schema not in ('information_schema','mysql','performance_schema','sys') \ 80 | and table_rows > 10000000 order by table_rows desc;" 81 | self._cursor.execute(sql) 82 | result = self._cursor.fetchall() 83 | print('检查行数超过1000万行的表') 84 | if not result: 85 | print("结果不存在") 86 | for row in result: 87 | print(("schema: %s tablename: %s rows: %s") % (row[0], row[1], row[2])) 88 | except Exception as e: 89 | raise (e) 90 | 91 | def check_table_chaset(self): 92 | try: 93 | self._cursor.execute("show variables like 'character_set_server';") 94 | default_charset = str(self._cursor.fetchone()[1]) 95 | default_charset = default_charset + "_general_ci" 96 | sql = "select table_schema,table_name,table_collation from information_schema.tables where table_schema not \ 97 | in ('information_schema','mysql','performance_schema','sys') and table_collation !='" + default_charset + "';" 98 | result = self._cursor.fetchall() 99 | print('检查非默认字符集的表') 100 | if not result: 101 | print("结果不存在") 102 | for row in result: 103 | print(("schema: %s tablename: %s fragment_pct: %s") % (row[0], row[1], row[2])) 104 | except Exception as e: 105 | raise (e) 106 | 107 | def check_table_big_columns(self): 108 | try: 109 | sql = "select table_schema,table_name,column_name,data_type from information_schema.columns where data_type in \ 110 | ('blob','clob','text','medium text','long text') and table_schema not in \ 111 | ('information_schema','performance_schema','mysql','sys')" 112 | self._cursor.execute(sql) 113 | result = self._cursor.fetchall() 114 | print('检查含大字段的表') 115 | if not result: 116 | print("结果不存在") 117 | for row in result: 118 | print(("schema: %s tablename: %s column_name: %s data_type: %s") % (row[0], row[1], row[2], row[3])) 119 | except Exception as e: 120 | raise (e) 121 | 122 | def check_table_long_varchar(self): 123 | try: 124 | sql = "select table_schema,table_name,column_name,data_type,CHARACTER_MAXIMUM_LENGTH from information_schema.columns \ 125 | where DATA_TYPE='varchar' and CHARACTER_MAXIMUM_LENGTH > 500 and table_schema not in \ 126 | ('information_schema','performance_schema','mysql','sys');" 127 | self._cursor.execute(sql) 128 | result = self._cursor.fetchall() 129 | print('检查varchar定义长的表') 130 | if not result: 131 | print("结果不存在") 132 | for row in result: 133 | print(("schema: %s tablename: %s column_name: %s data_type: %s(%s)") % ( 134 | row[0], row[1], row[2], row[3], row[4])) 135 | except Exception as e: 136 | raise (e) 137 | 138 | def check_table_no_index(self): 139 | try: 140 | sql = "SELECT t.table_schema,t.table_name FROM information_schema.tables AS t LEFT JOIN \ 141 | (SELECT DISTINCT table_schema, table_name FROM information_schema.`KEY_COLUMN_USAGE` ) AS kt ON \ 142 | kt.table_schema=t.table_schema AND kt.table_name = t.table_name WHERE t.table_schema NOT IN \ 143 | ('mysql', 'information_schema', 'performance_schema', 'sys') AND kt.table_name IS NULL;" 144 | self._cursor.execute(sql) 145 | result = self._cursor.fetchall() 146 | print('检查无主键/索引的表') 147 | if not result: 148 | print("结果不存在") 149 | for row in result: 150 | print(("schema: %s tablename: %s") % (row[0], row[1])) 151 | except Exception as e: 152 | raise (e) 153 | 154 | def check_index_redundant(self): 155 | try: 156 | sql = "select table_schema,table_name,redundant_index_name,redundant_index_columns from \ 157 | sys.schema_redundant_indexes group by table_schema,table_name,redundant_index_name,redundant_index_columns;" 158 | self._cursor.execute(sql) 159 | result = self._cursor.fetchall() 160 | print('检查重复索引') 161 | if not result: 162 | print("结果不存在") 163 | for row in result: 164 | print(("schema: %s tablename: %s redundant_index_name：%s redundant_index_columns:%s ") % ( 165 | row[0], row[1], row[2], row[3])) 166 | except Exception as e: 167 | raise (e) 168 | 169 | def check_index_columns(self): 170 | try: 171 | sql = "select s.table_schema,s.table_name,s.index_name,s.column_name from information_schema.STATISTICS s,\ 172 | (select table_name,index_name,count(*) from information_schema.STATISTICS where table_schema not in \ 173 | ('information_schema','performance_schema','mysql','sys') group by table_name,index_name having count(*)>5)t where \ 174 | s.table_name=t.table_name and s.index_name=t.index_name;" 175 | self._cursor.execute(sql) 176 | result = self._cursor.fetchall() 177 | print('检查索引列超过5个的索引') 178 | if not result: 179 | print("结果不存在") 180 | for row in result: 181 | print(("schema: %s tablename: %s index_name：%s column_name:%s ") % (row[0], row[1], row[2], row[3])) 182 | except Exception as e: 183 | raise (e) 184 | 185 | def check_index_unused(self): 186 | try: 187 | sql = "select * from sys.schema_unused_indexes;" 188 | self._cursor.execute(sql) 189 | result = self._cursor.fetchall() 190 | print('检查无用的索引') 191 | if not result: 192 | print("结果不存在") 193 | for row in result: 194 | print(("schema: %s tablename: %s indexname：%s") % (row[0], row[1], row[2])) 195 | except Exception as e: 196 | raise (e) 197 | 198 | def check_important_variables(self): 199 | print('检查重要参数') 200 | variables_list = ['version', 'innodb_buffer_pool_size', 'innodb_flush_log_at_trx_commit', 201 | 'innodb_log_file_size', 'innodb_log_files_in_group', 'innodb_file_per_table', 202 | 'innodb_max_dirty_pages_pct', 'sync_binlog', 'max_connections', 'query_cache_type', 203 | 'table_open_cache', 'table_definition_cache'] 204 | 205 | for variable in variables_list: 206 | try: 207 | sql = ("show global variables like '%s'" % variable) 208 | self._cursor.execute(sql) 209 | result = self._cursor.fetchone()[1] 210 | print(('%s : %s') % (variable, result)) 211 | except Exception as e: 212 | raise (e) 213 | 214 | def check_important_status(self): 215 | print('检查重要状态') 216 | status_list = ['Uptime', 'Opened_files', 'Opened_table_definitions', 'Opened_tables', 'Max_used_connections', 217 | 'Threads_created', 'Threads_connected', 'Aborted_connects', 'Aborted_clients', 218 | 'Table_locks_waited', 'Innodb_buffer_pool_wait_free', 'Innodb_log_waits', 219 | 'Innodb_row_lock_waits', 'Innodb_row_lock_time_avg', 'Binlog_cache_disk_use', 'Created_tmp_disk_tables'] 220 | for status in status_list: 221 | try: 222 | sql = ("show global status like '%s'" % status) 223 | self._cursor.execute(sql) 224 | result = self._cursor.fetchone()[1] 225 | print(('%s : %s') % (status, result)) 226 | except Exception as e: 227 | raise (e) 228 | self._cursor.execute("show engine innodb status") 229 | innodb_status = self._cursor.fetchall() 230 | innodb_status_format = str(innodb_status).split('\\n') 231 | for item in innodb_status_format: 232 | if "Log sequence number" in item: 233 | logsequencenumber = item.split(' ')[3] 234 | print(('%s : %s') % ('Log sequence number', logsequencenumber)) 235 | if "Log flushed up to" in item: 236 | logflushnumber = item.split(' ')[6] 237 | print(('%s : %s') % ('Log flushed up to', logflushnumber)) 238 | if "Last checkpoint at" in item: 239 | checkpoint = item.split(' ')[4] 240 | print(('%s : %s') % ('Last checkpoint at', checkpoint)) 241 | if "History list length" in item: 242 | historylength = item.split(' ')[3] 243 | print(('%s : %s') % ('historylength', historylength)) 244 | 245 | 246 | def check_user_nopass(self): 247 | try: 248 | sql = "select user,host from mysql.user where authentication_string='';" 249 | self._cursor.execute(sql) 250 | result = self._cursor.fetchall() 251 | print('检查无密码用户') 252 | if not result: 253 | print("结果不存在") 254 | for row in result: 255 | print(("user: %s host: %s") % (row[0], row[1])) 256 | except Exception as e: 257 | raise (e) 258 | 259 | def check_user_nowhere(self): 260 | try: 261 | sql = "select user,host from mysql.user where host='%';" 262 | self._cursor.execute(sql) 263 | result = self._cursor.fetchall() 264 | print('检查%用户') 265 | if not result: 266 | print("结果不存在") 267 | for row in result: 268 | print(("user: %s host: %s") % (row[0], row[1])) 269 | except Exception as e: 270 | raise (e) 271 | 272 | def check_user_privileges(self): 273 | try: 274 | sql = "select user,host from mysql.user where user not in ('mysql.session','mysql.sys');" 275 | self._cursor.execute(sql) 276 | result = self._cursor.fetchall() 277 | user_list = [] 278 | for row in result: 279 | user_list.append("'" + row[0] + "'" + "@" + "'" + row[1] + "'") 280 | print('检查用户权限') 281 | for user in user_list: 282 | sql = "show grants for %s;" % user 283 | # print(sql) 284 | self._cursor.execute(sql) 285 | result = self._cursor.fetchall() 286 | for row in result: 287 | print(row[0]) 288 | except Exception as e: 289 | raise (e) 290 | 291 | 292 | if __name__ == '__main__': 293 | with DBUtil('user', 'password', 'hostip', 3306, 'information_schema') as client: 294 | client.check_table_size() 295 | client.check_table_index() 296 | client.check_table_fragment_pct() 297 | client.check_table_rows() 298 | client.check_table_chaset() 299 | client.check_table_big_columns() 300 | client.check_table_long_varchar() 301 | client.check_table_no_index() 302 | client.check_index_redundant() 303 | client.check_index_columns() 304 | client.check_index_unused() 305 | client.check_important_variables() 306 | client.check_important_status() 307 | client.check_user_nopass() 308 | client.check_user_nowhere() 309 | client.check_user_privileges() 310 | -------------------------------------------------------------------------------- /mysql-tools/check_security_v1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # author:郑松华，知数堂SQL优化班老师，网名：骑龟的兔子 4 | # 5 | # 脚本作用说明 6 | # 针对MySQL做安全检查，包括账号、授权，以及几个重要参数的设置 7 | # Last Update, 2020/08/14 8 | 9 | import sys 10 | import os 11 | import datetime 12 | import pymysql 13 | import argparse 14 | import getpass 15 | import pandas as pd 16 | 17 | class Monitor(object): 18 | 19 | def __init__(self, connection_settings): 20 | """ 21 | conn_setting: {'host': 127.0.0.1, 'port': 3306, 'user': user, 'passwd': passwd, 'charset': 'utf8'} 22 | """ 23 | self.conn_setting = connection_settings 24 | self.connection = pymysql.connect(**self.conn_setting) 25 | 26 | pd.set_option('display.max_columns',None) 27 | #显示所有行 28 | pd.set_option('display.max_rows',None) 29 | #设置数据的显示长度，默认为50 30 | pd.set_option('max_colwidth',100) 31 | #禁止自动换行(设置为Flase不自动换行，True反之) 32 | pd.set_option('expand_frame_repr', False) 33 | 34 | 35 | def process_run(self): 36 | sql1 = ''' 37 | select now() time ; 38 | ''' 39 | cursor1 = self.connection.cursor() 40 | cursor1.execute(sql1) 41 | 42 | res1 = cursor1.fetchall() 43 | res1 = pd.DataFrame(list(res1), columns=['time']) 44 | 45 | print("This is MySQL 5.7 version security Report version 1.0 ") 46 | print(res1) 47 | return True 48 | 49 | def process_d02(self): 50 | sql1 = ''' 51 | select host,user , authentication_string , password_expired ,password_last_changed ,password_lifetime ,account_locked from mysql.user ; 52 | ''' 53 | cursor1 = self.connection.cursor() 54 | cursor1.execute(sql1) 55 | 56 | res1 = cursor1.fetchall() 57 | res1 = pd.DataFrame(list(res1), columns=['host','user','authentication_string','password_expired','password_last_changed','password_lifetime','account_locked']) 58 | print("\n") 59 | print(" User Account about password , change lifetime ,locked info ") 60 | print("\n") 61 | print(res1) 62 | return True 63 | 64 | def process_d03(self): 65 | sql1 = ''' 66 | show global variables like '%plu%'; 67 | ''' 68 | cursor1 = self.connection.cursor() 69 | cursor1.execute(sql1) 70 | 71 | res1 = cursor1.fetchall() 72 | res1 = pd.DataFrame(list(res1), columns=['Variable_name','Value']) 73 | print("\n") 74 | print(" authentication_plugin info ") 75 | print("\n") 76 | print(res1) 77 | return True 78 | 79 | def process_d04(self): 80 | sql1 = ''' 81 | show global variables like '%vali%'; 82 | ''' 83 | cursor1 = self.connection.cursor() 84 | cursor1.execute(sql1) 85 | 86 | res1 = cursor1.fetchall() 87 | res1 = pd.DataFrame(list(res1), columns=['Variable_name','Value']) 88 | print("\n") 89 | print("\n") 90 | print(res1) 91 | return True 92 | 93 | 94 | def process_d05(self): 95 | sql1 = ''' 96 | select 97 | host 98 | ,user 99 | ,Select_priv 100 | ,Insert_priv 101 | ,Update_priv 102 | ,Delete_priv 103 | ,Create_priv 104 | ,Drop_priv 105 | ,Reload_priv 106 | ,Shutdown_priv 107 | ,Process_priv 108 | ,File_priv 109 | ,Grant_priv 110 | ,References_priv 111 | ,Index_priv 112 | ,Alter_priv 113 | ,Show_db_priv 114 | ,Super_priv 115 | ,Create_tmp_table_priv 116 | ,Lock_tables_priv 117 | ,Execute_priv 118 | ,Repl_slave_priv 119 | ,Repl_client_priv 120 | ,Create_view_priv 121 | ,Show_view_priv 122 | ,Create_routine_priv 123 | ,Alter_routine_priv 124 | ,Create_user_priv 125 | ,Event_priv 126 | ,Trigger_priv 127 | ,Create_tablespace_priv 128 | from mysql.user 129 | where host = '%'; 130 | ''' 131 | cursor1 = self.connection.cursor() 132 | cursor1.execute(sql1) 133 | 134 | res1 = cursor1.fetchall() 135 | res1 = pd.DataFrame(list(res1), columns=['host','user','Select_priv','Insert_priv','Update_priv','Delete_priv','Create_priv' 136 | ,'Drop_priv','Reload_priv','Shutdown_priv','Process_priv','File_priv','Grant_priv','References_priv','Index_priv','Alter_priv' 137 | ,'Show_db_priv','Super_priv','Create_tmp_table_priv','Lock_tables_priv','Execute_priv','Repl_slave_priv','Repl_client_priv' 138 | ,'Create_view_priv','Show_view_priv','Create_routine_priv','Alter_routine_priv','Create_user_priv','Event_priv','Trigger_priv' 139 | ,'Create_tablespace_priv' 140 | ]) 141 | print("\n") 142 | print(" host % priv list ") 143 | print("\n") 144 | print(res1) 145 | return True 146 | 147 | def process_d06(self): 148 | sql1 = ''' 149 | show variables like '%general%'; 150 | ''' 151 | cursor1 = self.connection.cursor() 152 | cursor1.execute(sql1) 153 | 154 | res1 = cursor1.fetchall() 155 | res1 = pd.DataFrame(list(res1), columns=['Variable_name','Value']) 156 | print("\n") 157 | print(" general log on/off status ") 158 | print("\n") 159 | print(res1) 160 | return True 161 | 162 | def process_d07(self): 163 | sql1 = ''' 164 | show variables like '%timeout%'; 165 | ''' 166 | cursor1 = self.connection.cursor() 167 | cursor1.execute(sql1) 168 | 169 | res1 = cursor1.fetchall() 170 | res1 = pd.DataFrame(list(res1), columns=['Variable_name','Value']) 171 | print("\n") 172 | print(" timeout variables list ") 173 | print("\n") 174 | print(res1) 175 | return True 176 | 177 | def __del__(self): 178 | pass 179 | 180 | 181 | 182 | def is_valid_datetime(string): 183 | try: 184 | datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") 185 | return True 186 | except: 187 | return False 188 | 189 | 190 | def parse_args(): 191 | """parse args for binlog2sql""" 192 | 193 | parser = argparse.ArgumentParser(description='Parse MySQL binlog to SQL you want', add_help=False) 194 | connect_setting = parser.add_argument_group('connect setting') 195 | connect_setting.add_argument('-h', '--host', dest='host', type=str, 196 | help='Host the MySQL database server located', default='127.0.0.1') 197 | connect_setting.add_argument('-u', '--user', dest='user', type=str, 198 | help='MySQL Username to log in as', default='root') 199 | connect_setting.add_argument('-p', '--password', dest='password', type=str, nargs='*', 200 | help='MySQL Password to use', default='') 201 | connect_setting.add_argument('-P', '--port', dest='port', type=int, 202 | help='MySQL port to use', default=3306) 203 | 204 | parser.add_argument('--help', dest='help', action='store_true', help='help information', default=False) 205 | 206 | schema = parser.add_argument_group('schema filter') 207 | schema.add_argument('-d', '--databases', dest='databases', type=str, nargs='*', 208 | help='dbs you want to process', default='') 209 | schema.add_argument('-t', '--tables', dest='tables', type=str, nargs='*', 210 | help='tables you want to process', default='') 211 | return parser 212 | 213 | 214 | def command_line_args(args): 215 | need_print_help = False if args else True 216 | parser = parse_args() 217 | args = parser.parse_args(args) 218 | if args.help or need_print_help: 219 | parser.print_help() 220 | sys.exit(1) 221 | if not args.password: 222 | args.password = getpass.getpass() 223 | else: 224 | args.password = args.password[0] 225 | return args 226 | 227 | if __name__ == '__main__': 228 | args = command_line_args(sys.argv[1:]) 229 | conn_setting = {'host': args.host, 'port': args.port, 'user': args.user, 'passwd': args.password, 'charset': 'utf8'} 230 | mo = Monitor( connection_settings=conn_setting) 231 | mo.process_run() 232 | mo.process_d02() 233 | mo.process_d03() 234 | mo.process_d04() 235 | mo.process_d05() 236 | mo.process_d06() 237 | mo.process_d07() 238 | 239 | 240 | # C:/Python38/python.exe check_ security_v1.py --host=127.0.0.1 --port=3306 --user=test --password='test' --database=mysql 241 | -------------------------------------------------------------------------------- /mysql-tools/dba_login.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## MySQL DBA登入服务器后, 建议关注的一些重要信息, 欢迎分享及补充。 4 | ## 5 | ## 叶金荣, 知数堂培训联合创始人, 资深MySQL专家, MySQL布道师, Oracle MySQL ACE 6 | ## 7 | ## created by yejinrong@zhishutang.com 8 | ## 2017/5/22 9 | ## 10 | 11 | echo 12 | echo "Welcome to MySQL Server, make sure all of the check result is OK" 13 | echo 14 | 15 | . ~/.bash_profile 16 | #1. check who is login 17 | echo "#1# check who is login" 18 | w 19 | echo 20 | echo 21 | 22 | #2. check system's load and others 23 | echo "#2# check disk free" 24 | df -hT | grep -v 'Filesystem.*Type.*Size'|sort -rn -k 6|head -n 3 25 | echo 26 | echo 27 | 28 | echo "#3# check memory and swap" 29 | echo "show memory & swap usage, check if memory leak" 30 | free -h 31 | echo 32 | echo 33 | 34 | #3. check which prog's load is high 35 | echo "#4# check which prog's load is high" 36 | ps -eo pid,pcpu,size,rss,cmd | sort -rn -k 2 | head -n 5 | grep -iv 'PID.*CPU.*SIZE' 37 | echo 38 | echo 39 | 40 | #4. check mysql status 41 | echo "#5# check MySQL status" 42 | mysqladmin pr | egrep -v 'Sleep|\-\-\-\-\-' | sort -rn -k 12 | head -n 5 43 | -------------------------------------------------------------------------------- /mysql-tools/diskcheck_megacli.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #zabbix监控硬盘信息脚本 3 | #By xiangjunyu 20151101 4 | 5 | . ~/.bash_profile > /dev/null 6 | 7 | #获取磁盘信息 8 | /opt/MegaRAID/MegaCli/MegaCli64 -Pdlist -a0|grep -Ei '(Slot Number|Media Error Count|Other Error Count|Predictive Failure Count|Raw Size|Firmware state)'|sed -e "s:\[0x.*Sectors\]::g" >/tmp/pdinfo.txt 9 | 10 | #将每块磁盘信息拆分，进行逐盘分析 11 | split -l 6 -d /tmp/pdinfo.txt /tmp/pdinfo 12 | 13 | #获取磁盘数量（实际数量=PDNUM+1） 14 | PDNUM=`/opt/MegaRAID/MegaCli/MegaCli64 -PDGetNum -aAll|grep Physical|awk '{ print $8 }'` 15 | 16 | #磁盘分块后文件名规范统一化 17 | for((i=0;i<${PDNUM};i++)) 18 | do 19 | mv /tmp/pdinfo0${i} /tmp/pdinfo${i} >/dev/null 2>&1 20 | #ls /tmp/pdinfo${i} 21 | done 22 | SLOT_NUM=$2 23 | DATAFORMATE() 24 | { 25 | while read LINE 26 | do 27 | if [[ ${LINE} == Slot* ]]; 28 | then 29 | SLOTNUMNAME=`echo ${LINE}|awk -F: '{ print $1 }'` 30 | SLOTNUM=`echo ${LINE}|awk -F: '{ print $2 }'` 31 | elif [[ ${LINE} == Media* ]]; 32 | then 33 | MECNAME=`echo ${LINE}|awk -F: '{ print $1 }'` 34 | MEC=`echo ${LINE}|awk -F: '{ print $2 }'` 35 | elif [[ ${LINE} == Other* ]]; 36 | then 37 | OECNAME=`echo ${LINE}|awk -F: '{ print $1 }'` 38 | OEC=`echo ${LINE}|awk -F: '{ print $2 }'` 39 | elif [[ ${LINE} == Predictive* ]]; 40 | then 41 | PFCNAME=`echo ${LINE}|awk -F: '{ print $1 }'` 42 | PFC=`echo ${LINE}|awk -F: '{ print $2 }'` 43 | elif [[ ${LINE} == Raw* ]]; 44 | then 45 | RAWNAME=`echo ${LINE}|awk -F: '{ print $1 }'` 46 | SIZE=`echo ${LINE}|awk -F: '{ print $2 }'` 47 | elif [[ ${LINE} == Firmware* ]]; 48 | then 49 | FIRMWARENAME=`echo ${LINE}|awk -F: '{ print $1 }'` 50 | FIRMWARESTATUS=`echo ${LINE}|awk -F: '{ print $2 }'` 51 | fi 52 | done 1: print '%s: lost pipe to mysql, %s' % (self.filename, e) 49 | for name in self.vars: self.val[name] = -1 50 | 51 | except Exception, e: 52 | if op.debug > 1: print '%s: exception' % (self.filename, e) 53 | for name in self.vars: self.val[name] = -1 54 | -------------------------------------------------------------------------------- /mysql-tools/dstat-plugins/dstat_innodb_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2019/2/15 14:02 4 | # @Author : xucl 5 | # @Email : xuchenliang123@live.com 6 | # @howtouse: dstat --innodb-list 7 | global mysql_options 8 | mysql_options = os.getenv('DSTAT_MYSQL') 9 | 10 | class dstat_plugin(dstat): 11 | def __init__(self): 12 | self.name = 'innodb list length' 13 | self.nick = ('unpurged', 'free') 14 | self.vars = ('u_list', 'f_list') 15 | self.type = 'd' 16 | self.width = 10 17 | self.scale = 1000 18 | 19 | def check(self): 20 | if os.access('/usr/bin/mysql', os.X_OK): 21 | try: 22 | self.stdin, self.stdout, self.stderr = dpopen('/usr/bin/mysql -n %s' % mysql_options) 23 | except IOError: 24 | raise Exception, 'Cannot interface with MySQL binary' 25 | return True 26 | raise Exception, 'Needs MySQL binary' 27 | 28 | def extract(self): 29 | try: 30 | self.stdin.write('show engine innodb status\G\n') 31 | line1 = greppipe(self.stdout, 'History list length') 32 | line2 = greppipe(self.stdout, 'free list len') 33 | 34 | if line1: 35 | l1 = line1.split() 36 | self.set2['u_list'] = int(l1[3].rstrip(' ')) 37 | if line2: 38 | l2 = line2.split() 39 | self.set2['f_list'] = int((l2[6]).rstrip(',')) 40 | 41 | #for name in self.vars: 42 | # self.val[name] = (self.set2[name] - self.set1[name]) * 1.0 / elapsed 43 | self.val['u_list'] = (self.set2['u_list']) 44 | self.val['f_list'] = (self.set2['f_list']) 45 | 46 | except IOError, e: 47 | if op.debug > 1: print '%s: lost pipe to mysql, %s' % (self.filename, e) 48 | for name in self.vars: self.val[name] = -1 49 | 50 | except Exception, e: 51 | if op.debug > 1: print '%s: exception' % (self.filename, e) 52 | for name in self.vars: self.val[name] = -1 53 | -------------------------------------------------------------------------------- /mysql-tools/dstat-plugins/dstat_mysql5_rowlockwaits.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2019/2/15 14:02 4 | # @Author : xucl 5 | # @Email : xuchenliang123@live.com 6 | # @howtouse: dstat --mysql5-rowlockwaits 7 | global mysql_user 8 | mysql_user = os.getenv('DSTAT_MYSQL_USER') or os.getenv('USER') 9 | 10 | global mysql_pwd 11 | mysql_pwd = os.getenv('DSTAT_MYSQL_PWD') 12 | 13 | global mysql_host 14 | mysql_host = os.getenv('DSTAT_MYSQL_HOST') 15 | 16 | global mysql_port 17 | mysql_port = os.getenv('DSTAT_MYSQL_PORT') 18 | 19 | class dstat_plugin(dstat): 20 | """ 21 | Plugin for MySQL 5 connections. 22 | """ 23 | 24 | def __init__(self): 25 | self.name = 'innodb lock waits' 26 | self.nick = ('current_waits', 'total_waits') 27 | self.vars = ('current_waits', 'waits') 28 | self.type = 'd' 29 | self.width = 15 30 | self.scale = 1 31 | 32 | def check(self): 33 | global MySQLdb 34 | import MySQLdb 35 | try: 36 | self.db = MySQLdb.connect(user=mysql_user, passwd=mysql_pwd, host=mysql_host, port=int(mysql_port)) 37 | except Exception, e: 38 | raise Exception, 'Cannot interface with MySQL server, %s' % e 39 | 40 | def extract(self): 41 | try: 42 | c = self.db.cursor() 43 | c.execute("""show global status like 'Innodb_row_lock_current_waits';""") 44 | current_waits = c.fetchone() 45 | c.execute("""show global status like 'Innodb_row_lock_waits';""") 46 | waits = c.fetchone() 47 | if current_waits: 48 | self.set2['current_waits'] = int(current_waits[1]) 49 | if waits: 50 | self.set2['waits'] = int(waits[1]) 51 | #for name in self.vars: 52 | # self.val[name] = self.set2[name] * 1.0 / elapsed 53 | self.val['current_waits'] = self.set2['current_waits'] 54 | self.val['waits'] = self.set2['waits'] 55 | if step == op.delay: 56 | self.set1.update(self.set2) 57 | 58 | except Exception, e: 59 | for name in self.vars: 60 | self.val[name] = -1 61 | 62 | # vim:ts=4:sw=4:et 63 | -------------------------------------------------------------------------------- /mysql-tools/ibd-analyzer.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | # 3 | # Author: Libing Song(宋利兵) 4 | # Wixin Official Accounts: mysqlcode 5 | # 分析InnoDB数据文件中B+Tree数据结构脚本 6 | # 相关阅读： 7 | # 8 | 9 | class IBD_page 10 | PAGE_SIZE = 16 * 1024 11 | 12 | @@INVALID_PAGE_NO = 0xFFFFFFFF 13 | 14 | @@FIL_PAGE_SPACE_OR_CHKSUM = 0 15 | @@FIL_PAGE_OFFSET = 4 16 | @@FIL_PAGE_PREV = 8 17 | @@FIL_PAGE_NEXT = 12 18 | @@FIL_PAGE_LSN = 16 19 | @@FIL_PAGE_TYPE = 24 20 | 21 | # Page type of index 22 | @@FIL_PAGE_INDEX = 17855 23 | 24 | # Offset of the data on the page 25 | @@FIL_PAGE_DATA = 38 26 | 27 | # On a page of any file segment, data may be put starting from this offset 28 | @@FSEG_PAGE_DATA = @@FIL_PAGE_DATA 29 | @@FSEG_HEADER_SIZE = 10 30 | 31 | # Offset of page header 32 | @@PAGE_HEADER = @@FIL_PAGE_DATA 33 | # Offset of data on the page 34 | @@PAGE_DATA = @@PAGE_HEADER + 36 + 2 * @@FSEG_HEADER_SIZE 35 | 36 | # Records in the page 37 | @@PAGE_N_RECS = 16 38 | #Level of the node in the index tree; the leaf level is the level 0 */ 39 | @@PAGE_LEVEL = 26 40 | 41 | @@REC_N_NEW_EXTRA_BYTES = 5 42 | @@PAGE_NEW_INFIMUM = @@PAGE_DATA + @@REC_N_NEW_EXTRA_BYTES 43 | @@PAGE_NEW_SUPREMUM = @@PAGE_DATA + 2 * @@REC_N_NEW_EXTRA_BYTES + 8 44 | 45 | @@REC_NEXT = 2 46 | @@REC_NEW_INFO_BITS = 5 47 | @@REC_INFO_BITS_MASK = 0xF0 48 | @@REC_INFO_MIN_REC_FLAG = 0x10 49 | @@REC_INFO_DELETED_FLAG = 0x20 50 | 51 | def initialize(buf, page_no) 52 | @buf = buf 53 | @page_no = page_no 54 | end 55 | 56 | def type 57 | get_int2(@@FIL_PAGE_TYPE) 58 | end 59 | 60 | def in_btr() 61 | type() == @@FIL_PAGE_INDEX 62 | end 63 | 64 | def is_btr_root() 65 | in_btr() and prev_page() == @@INVALID_PAGE_NO and next_page() == @@INVALID_PAGE_NO 66 | end 67 | 68 | def level 69 | get_int2(@@PAGE_HEADER + @@PAGE_LEVEL) 70 | end 71 | 72 | def record_count 73 | get_int2(@@PAGE_HEADER + @@PAGE_N_RECS) 74 | end 75 | 76 | def prev_page() 77 | get_int4(@@FIL_PAGE_PREV) 78 | end 79 | 80 | def next_page() 81 | get_int4(@@FIL_PAGE_NEXT) 82 | end 83 | 84 | def first_rec() 85 | next_rec(@@PAGE_NEW_INFIMUM) 86 | end 87 | 88 | def next_rec(rec) 89 | # The offset of next record is against to current record's position 90 | offset = get_int2(rec - @@REC_NEXT) + rec 91 | offset = (offset + PAGE_SIZE * @page_no) & (PAGE_SIZE - 1) 92 | 93 | if offset == @@PAGE_NEW_SUPREMUM 94 | offset = 0 95 | end 96 | return offset 97 | end 98 | 99 | def rec_key_str(rec, len) 100 | get_str(rec, len) 101 | end 102 | 103 | def rec_pointer(rec, key_len) 104 | if level == 0 105 | return 0 106 | else 107 | # Pointer is the last field of index row 108 | get_int4(rec+key_len) 109 | end 110 | end 111 | 112 | def rec_flags(rec) 113 | flag = get_int1(rec - @@REC_NEW_INFO_BITS) 114 | str="" 115 | if flag & @@REC_INFO_DELETED_FLAG == @@REC_INFO_DELETED_FLAG 116 | str = "D" 117 | end 118 | if flag & @@REC_INFO_MIN_REC_FLAG == @@REC_INFO_MIN_REC_FLAG 119 | str = str + "M" 120 | end 121 | str 122 | end 123 | 124 | # Get a 1 byte number from page buffer 125 | def get_int1(offset) 126 | @buf[offset] 127 | end 128 | 129 | # Get a 2 byte number from page buffer 130 | def get_int2(offset) 131 | s = @buf[offset..offset+2] 132 | s = s.unpack("n") 133 | s.to_s.to_i 134 | end 135 | 136 | # Get a 4 byte number from page buffer 137 | def get_int4(offset) 138 | s = @buf[offset..offset+4] 139 | s = s.unpack("N") 140 | s.to_s.to_i 141 | end 142 | 143 | # Get a 'len' long string from page buffer 144 | def get_str(offset, len) 145 | @buf[offset..offset+len-1] 146 | end 147 | end 148 | 149 | class IBD_file 150 | def initialize(filename) 151 | @size = File.size?(filename) 152 | @file = File.new(filename, "r") 153 | end 154 | 155 | def read_page(page_no) 156 | if (page_no * IBD_page::PAGE_SIZE > @size) 157 | printf("Page No.(%d) is too large", page_no) 158 | return nil 159 | end 160 | 161 | @file.pos = page_no * IBD_page::PAGE_SIZE 162 | IBD_page.new(@file.read(IBD_page::PAGE_SIZE), page_no) 163 | end 164 | 165 | def page_count 166 | @size / IBD_page::PAGE_SIZE 167 | end 168 | 169 | def btr_roots() 170 | a = [] 171 | page_count().times do |i| 172 | page = read_page(i) 173 | if page.is_btr_root() 174 | a = a + [i] 175 | end 176 | end 177 | return a 178 | end 179 | 180 | def print_btr_roots(print_key_len) 181 | btr_roots.each do |page_no| 182 | page = read_page(page_no) 183 | 184 | printf("B-Tree Root Page: %d, Level: %d, Records: %d\n", page_no.to_s, 185 | page.level, page.record_count) 186 | 187 | rec = page.first_rec 188 | while rec != 0 189 | printf(" Key(%s)\n", page.rec_key_str(rec, print_key_len)) 190 | rec = page.next_rec(rec) 191 | end 192 | end 193 | end 194 | 195 | def print_btr(root, key_len, print_key_len) 196 | page_no = root 197 | next_level_page_no = -1 198 | while page_no > 0 199 | page = read_page(page_no) 200 | if (!page) 201 | puts "Reading page failed. You probable set a wrong key length" 202 | exit 1 203 | end 204 | 205 | printf("Page: %d, Level: %d, Records: %d\n", page_no.to_s, page.level, 206 | page.record_count) 207 | 208 | rec = page.first_rec 209 | while rec != 0 210 | if page.level == 0 211 | printf(" Flags(%s), Key(%s)\n", page.rec_flags(rec), 212 | page.rec_key_str(rec, print_key_len)) 213 | else 214 | printf(" Flags(%s), Key(%s), Pointer(%d)\n", page.rec_flags(rec), 215 | page.rec_key_str(rec, print_key_len), 216 | page.rec_pointer(rec, key_len)) 217 | end 218 | 219 | if next_level_page_no == -1 220 | next_level_page_no = page.rec_pointer(rec, key_len) 221 | end 222 | rec = page.next_rec(rec) 223 | end 224 | 225 | page_no = page.next_page 226 | if page_no == 0xFFFFFFFF 227 | page_no = next_level_page_no 228 | next_level_page_no = -1 229 | end # if 230 | end # while 231 | end # def 232 | 233 | end 234 | 235 | def print_help() 236 | puts <<-EOF 237 | ibd-analizer useage: 238 | ./ibd-analizer.rb [OPTIONS] 239 | OPTIONS 240 | ======= 241 | -h, --help: 242 | show help. 243 | -R, --roots: 244 | print all root pages of b-trees. 245 | -r n, --root-page n: 246 | print the b-tree that starts at 'n' page. it will be ignored if '-R' is set. 247 | -L n, --key-len n: 248 | the key of the b-tree is 'n' bytes long. It has to be set with '-r' together. 249 | In a clustered index b-tree, it is the sum of all primary key fields(length). 250 | In a secondary index b-tree, it is the sum of all the secondary key fields and 251 | primary key fields. E.g. 252 | CREATE TABLE t1(c1 char(4) PRIMARY KEY, c2 char(5), INDEX(c2)); 253 | In clustered index b-tree, the key length is 4. 254 | In secondary index b-tree, the key length is 9. 255 | -l n, --print-key-len n: 256 | print only the first 'n' bytes of the key. The first 4 bytes will be printed, 257 | if it is not set. 258 | OUTPUT FIELDS 259 | ============= 260 | Page 261 | Page No. of current page. 262 | Level 263 | B-Tree level of current page. 264 | Records 265 | How many records(include deleted records) in the page. 266 | Flags 267 | M - It is the minimum key record in the same level of the b-tree. 268 | D - The record was marked as 'deleted'. It has been delete by user. 269 | Key 270 | Value of the key. 271 | Pointer 272 | Page No. of the next level page belongs to the key. 273 | EOF 274 | end 275 | 276 | require 'getoptlong' 277 | 278 | def main 279 | roots = nil 280 | root_page = nil 281 | key_len = nil 282 | print_key_len = 4 # Print only the first 4 bytes of keys, by default 283 | 284 | opts = GetoptLong.new( 285 | ['--help', '-h', GetoptLong::NO_ARGUMENT], 286 | ['--roots', '-R', GetoptLong::NO_ARGUMENT], 287 | ['--root-page', '-r', GetoptLong::REQUIRED_ARGUMENT], 288 | ['--key-len', '-L', GetoptLong::REQUIRED_ARGUMENT], 289 | ['--print-key-len', '-l', GetoptLong::REQUIRED_ARGUMENT] 290 | ) 291 | 292 | opts.each do |opt, arg| 293 | case opt 294 | when '--help' 295 | print_help 296 | when '--roots' 297 | root = true 298 | when '--root-page' 299 | root_page = arg.to_i 300 | when '--key-len' 301 | key_len = arg.to_i() 302 | when '--print-key-len' 303 | print_key_len = arg.to_i 304 | end 305 | end 306 | 307 | if ARGV.length != 1 308 | puts "Missing ibdfile argument!" 309 | print_help 310 | exit 0 311 | end 312 | 313 | f = IBD_file.new(ARGV.shift) 314 | if roots or !root_page 315 | f.print_btr_roots(print_key_len) 316 | else 317 | if !key_len 318 | puts "--key-len is not set. It has to be set when you print a b-tree." 319 | exit 1 320 | end 321 | f.print_btr(root_page, key_len, print_key_len) 322 | end 323 | end 324 | 325 | main 326 | -------------------------------------------------------------------------------- /mysql-tools/monitor_error.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*-# 2 | 3 | # ------------------------------------------------------------------------------- 4 | # Name: monitor_error 5 | # Description: 6 | # Author: xucl 7 | # Date: 2019-03-28 8 | # ------------------------------------------------------------------------------- 9 | 10 | import datetime 11 | import json 12 | import configparser 13 | import os 14 | 15 | # basic settings 16 | errorlog = "data/error.log" 17 | message = "/var/log/messages" 18 | dmesg = "/var/log/dmesg" 19 | dbid = 1 20 | 21 | def get_error_text(error_last_pos): 22 | f = open(errorlog, "r") 23 | error_pos = os.path.getsize(errorlog) 24 | offset_size = int(error_pos) - int(error_last_pos) 25 | f.seek(int(error_last_pos), 0) 26 | errortext = f.read(offset_size).strip('\n') 27 | f.close() 28 | return errortext, error_pos 29 | 30 | 31 | def get_message_text(message_last_pos): 32 | f = open(message, "r") 33 | meg_pos = os.path.getsize(message) 34 | offset_size = int(meg_pos) - int(message_last_pos) 35 | f.seek(int(message_last_pos), 0) 36 | megtext = f.read(offset_size).strip('\n') 37 | f.close() 38 | return megtext, meg_pos 39 | 40 | 41 | def get_dmesg_text(dmesg_last_pos): 42 | f = open(dmesg, "r") 43 | dmesg_pos = os.path.getsize(dmesg) 44 | offset_size = int(dmesg_pos) - int(dmesg_last_pos) 45 | f.seek(int(dmesg_last_pos), 0) 46 | dmesgtext = f.read(offset_size).strip('\n') 47 | f.close() 48 | return dmesgtext, dmesg_pos 49 | 50 | def update_position(cf, error_pos, meg_pos, dmesg_pos): 51 | cf.set('error', 'position', str(error_pos)) 52 | cf.set('message', 'position', str(meg_pos)) 53 | cf.set('dmesg', 'position', str(dmesg_pos)) 54 | cf.write(open('last_position', 'w')) 55 | 56 | def generate_data(errtext, mestext, dmsgtext): 57 | monitor_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") 58 | errordata = {} 59 | errordata['dbid'] = dbid 60 | errordata['time'] = monitor_time 61 | errordata['error'] = errtext 62 | errordata['meg'] = mestext 63 | errordata['dmsg'] = dmsgtext 64 | errordata = json.dumps(errordata) 65 | return errordata 66 | 67 | 68 | if __name__ == "__main__": 69 | if not os.path.exists('last_position'): 70 | f = open('last_position', "w") 71 | init_pos = """ 72 | [error] 73 | position = 0 74 | 75 | [message] 76 | position = 0 77 | 78 | [dmesg] 79 | position = 0 80 | """ 81 | f.write(init_pos) 82 | f.close() 83 | cf = configparser.ConfigParser() 84 | cf.read('last_position') 85 | 86 | error_last_pos = cf.get('error', 'position') 87 | message_last_pos = cf.get('message', 'position') 88 | dmesg_last_pos = cf.get('dmesg', 'position') 89 | 90 | errtext, error_pos = get_error_text(error_last_pos) 91 | mestext, meg_pos = get_message_text(message_last_pos) 92 | dmsgtext, dmesg_pos = get_dmesg_text(dmesg_last_pos) 93 | update_position(cf, error_pos, meg_pos, dmesg_pos) 94 | errordata = generate_data(errtext, mestext, dmsgtext) 95 | print(errordata) 96 | -------------------------------------------------------------------------------- /mysql-tools/mysql-toolkit-sql.md: -------------------------------------------------------------------------------- 1 | ## 关于 2 | 3 | > - 作者：叶金荣, 知数堂培训（http:/zhishutang.com）联合创始人, 资深MySQL专家, MySQL布道师, Oracle MySQL ACE 4 | > - 分享工作、教学中用到的&收集到的一些实用SQL脚本命令，有需自取 5 | > - 这些脚本在MySQL 5.7/8.0版本下均测试通过 6 | > - 最后更新时间：2019-7-7 7 | > - QQ群：579036588 8 | > - 微信公众号：「老叶茶馆」、「知数堂」、「pai3306」 9 | 10 | * ## 查看哪些索引采用部分索引（前缀索引） 11 | > 优化建议：检查部分索引长度是否还可以进一步缩小 12 | ``` 13 | SELECT TABLE_SCHEMA, TABLE_NAME, INDEX_NAME, 14 | SEQ_IN_INDEX, COLUMN_NAME, CARDINALITY, SUB_PART 15 | FROM INFORMATION_SCHEMA.STATISTICS WHERE 16 | SUB_PART > 10 ORDER BY SUB_PART DESC; 17 | ``` 18 | 19 | * ## 查看哪些索引长度超过30字节，重点查CHAR/VARCHAR/TEXT/BLOB等类型 20 | > 优化建议：超过20字节长度的索引，都应该考虑进一步缩短，否则效率不佳 21 | ``` 22 | select c.table_schema as `db`, c.table_name as `tbl`, 23 | c.COLUMN_NAME as `col`, c.DATA_TYPE as `col_type`, 24 | c.CHARACTER_MAXIMUM_LENGTH as `col_len`, 25 | c.CHARACTER_OCTET_LENGTH as `col_len_bytes`, 26 | s.NON_UNIQUE as `isuniq`, s.INDEX_NAME, s.CARDINALITY, 27 | s.SUB_PART, s.NULLABLE 28 | from information_schema.COLUMNS c inner join information_schema.STATISTICS s 29 | using(table_schema, table_name, COLUMN_NAME) where 30 | c.table_schema not in ('mysql', 'sys', 'performance_schema', 'information_schema', 'test') and 31 | c.DATA_TYPE in ('varchar', 'char', 'text', 'blob') and 32 | ((CHARACTER_OCTET_LENGTH > 20 and SUB_PART is null) or 33 | SUB_PART * CHARACTER_OCTET_LENGTH/CHARACTER_MAXIMUM_LENGTH >20); 34 | ``` 35 | 36 | * ## 查看未完成的事务列表 37 | > 优化建议：若有长时间未完成的事务，可能会导致： 38 | > - undo不能被及时purge，undo表空间不断增长； 39 | > - 持有行锁，其他事务被阻塞。 40 | > 应该及时提交或回滚这些事务，或者直接kill释放之。 41 | > 42 | > 参考： 43 | > - [FAQ系列 | 如何避免ibdata1文件大小暴涨 http://mp.weixin.qq.com/s/KD2qLrmWY80yFxUtxJVNMA] 44 | > - [是谁，把InnoDB表上的DML搞慢的？ http://mp.weixin.qq.com/s/wEPKgPo1dMsxTedjvulSlQ] 45 | ``` 46 | select b.host, b.user, b.db, b.time, b.COMMAND, 47 | a.trx_id, a. trx_state from 48 | information_schema.innodb_trx a left join 49 | information_schema.PROCESSLIST b on a.trx_mysql_thread_id = b.id; 50 | ``` 51 | 52 | 53 | * ## 查看当前有无行锁等待事件 54 | > 优化建议： 55 | > - 若当前有行锁等待，则有可能导致锁超时被回滚，事务失败； 56 | > - 有时候，可能是因为某个终端/会话开启事务，对数据加锁后，忘记提交/回滚，导致行锁不能释放。 57 | > 58 | > 参考： 59 | > [是谁，把InnoDB表上的DML搞慢的？ http://mp.weixin.qq.com/s/wEPKgPo1dMsxTedjvulSlQ] 60 | > [FAQ系列 | 是什么导致MySQL数据库服务器磁盘I/O高？ http://mp.weixin.qq.com/s/sAGFo-h1GCBhad1r1cEWTg] 61 | > [[译文]MySQL发生死锁肿么办？ http://mp.weixin.qq.com/s/oUSdfv0qlrxCearFw-_XZw] 62 | > [都是主键惹的祸-记一次死锁分析过程 http://mp.weixin.qq.com/s/7VmlqcqTQH7ITnmUKCdw5Q] 63 | ``` 64 | SELECT lw.requesting_trx_id AS request_XID, 65 | trx.trx_mysql_thread_id as request_mysql_PID, 66 | trx.trx_query AS request_query, 67 | lw.blocking_trx_id AS blocking_XID, 68 | trx1.trx_mysql_thread_id as blocking_mysql_PID, 69 | trx1.trx_query AS blocking_query, lo.lock_index AS lock_index FROM 70 | information_schema.innodb_lock_waits lw INNER JOIN 71 | information_schema.innodb_locks lo 72 | ON lw.requesting_trx_id = lo.lock_trx_id INNER JOIN 73 | information_schema.innodb_locks lo1 74 | ON lw.blocking_trx_id = lo1.lock_trx_id INNER JOIN 75 | information_schema.innodb_trx trx 76 | ON lo.lock_trx_id = trx.trx_id INNER JOIN 77 | information_schema.innodb_trx trx1 78 | ON lo1.lock_trx_id = trx1.trx_id; 79 | ``` 80 | 其实，在MySQL 5.7下，也可以直接查看 sys.innodb_lock_waits 视图： 81 | ``` 82 | SELECT * FROM sys.innodb_lock_waits\G 83 | ``` 84 | 85 | * ## 检查哪些表没有显式创建主键索引 86 | > 优化建议： 87 | > - 选择自增列做主键； 88 | > - 或者其他具备单调递增特点的列做主键； 89 | > - 主键最好不要有业务用途，避免后续会更新。 90 | > 91 | > 参考： 92 | > - [MySQL FAQ系列 — 为什么InnoDB表要建议用自增列做主键 http://mp.weixin.qq.com/s/GpOzU9AqhWPj6bj9C5yXmw] 93 | ``` 94 | SELECT 95 | a.TABLE_SCHEMA as `db`, 96 | a.TABLE_NAME as `tbl` 97 | FROM 98 | ( 99 | SELECT 100 | TABLE_SCHEMA, 101 | TABLE_NAME 102 | FROM 103 | information_schema.TABLES 104 | WHERE 105 | TABLE_SCHEMA NOT IN ( 106 | 'mysql', 107 | 'sys', 108 | 'information_schema', 109 | 'performance_schema' 110 | ) AND 111 | TABLE_TYPE = 'BASE TABLE' 112 | ) AS a 113 | LEFT JOIN ( 114 | SELECT 115 | TABLE_SCHEMA, 116 | TABLE_NAME 117 | FROM 118 | information_schema.TABLE_CONSTRAINTS 119 | WHERE 120 | CONSTRAINT_TYPE = 'PRIMARY KEY' 121 | ) AS b ON a.TABLE_SCHEMA = b.TABLE_SCHEMA 122 | AND a.TABLE_NAME = b.TABLE_NAME 123 | WHERE 124 | b.TABLE_NAME IS NULL; 125 | ``` 126 | 127 | 128 | * ## 查看表数据列元数据信息 129 | ``` 130 | select a.table_id, a.name, b.name, b.pos, b.mtype, b.prtype, b.len from 131 | information_schema.INNODB_SYS_TABLES a left join 132 | information_schema.INNODB_SYS_COLUMNS b 133 | using(table_id) where a.name = 'yejr/t1'; 134 | ``` 135 | 136 | 137 | * ## 查看InnoDB表碎片率 138 | > 优化建议： 139 | ``` 140 | SELECT TABLE_SCHEMA as `db`, TABLE_NAME as `tbl`, 141 | 1-(TABLE_ROWS*AVG_ROW_LENGTH)/(DATA_LENGTH + INDEX_LENGTH + DATA_FREE) AS `fragment_pct` 142 | FROM information_schema.TABLES WHERE 143 | TABLE_SCHEMA = 'yejr' ORDER BY fragment_pct DESC; 144 | ``` 145 | 146 | 147 | * ## 查某个表在innodb buffer pool中的new block、old block比例 148 | ``` 149 | select table_name, count(*), sum(NUMBER_RECORDS), 150 | if(IS_OLD='YES', 'old', 'new') as old_block from 151 | information_schema.innodb_buffer_page where 152 | table_name = '`yejr`.`t1`' group by old_block; 153 | ``` 154 | 155 | * ## 查看某个数据库里所有表的索引统计情况，重点是关注 stat_pct 列值较低的索引 156 | ``` 157 | # 工作方式 158 | # 1、扫描所有索引统计信息 159 | # 2、包含主键列的辅助索引统计值，对比主键索引列的统计值，得到一个百分比stat_pct 160 | # 3、根据stat_pct排序，值越低说明辅助索引统计信息越不精确，越是需要关注 161 | 162 | set @statdb = 'yejr'; 163 | select 164 | a.database_name , 165 | a.table_name , 166 | a.index_name , 167 | a.stat_value SK, 168 | b.stat_value PK, 169 | round((a.stat_value/b.stat_value)*100,2) stat_pct 170 | from 171 | ( 172 | select 173 | b.database_name , 174 | b.table_name , 175 | b.index_name , 176 | b.stat_value 177 | from 178 | ( 179 | select database_name , 180 | table_name , 181 | index_name , 182 | max(stat_name) stat_name 183 | from innodb_index_stats 184 | where database_name = @statdb 185 | and stat_name not in ( 'size' ,'n_leaf_pages' ) 186 | group by 187 | database_name , 188 | table_name , 189 | index_name 190 | ) a join innodb_index_stats b on a.database_name=b.database_name 191 | and a.table_name=b.table_name 192 | and a.index_name=b.index_name 193 | and a.stat_name=b.stat_name 194 | and b.index_name !='PRIMARY' 195 | ) a left join 196 | ( 197 | select 198 | b.database_name , 199 | b.table_name , 200 | b.index_name , 201 | b.stat_value 202 | from 203 | ( 204 | select database_name , 205 | table_name , 206 | index_name , 207 | max(stat_name) stat_name 208 | from innodb_index_stats 209 | where database_name = @statdb 210 | and stat_name not in ( 'size' ,'n_leaf_pages' ) 211 | group by 212 | database_name , 213 | table_name , 214 | index_name 215 | ) a join innodb_index_stats b 216 | on a.database_name=b.database_name 217 | and a.table_name=b.table_name 218 | and a.index_name=b.index_name 219 | and a.stat_name=b.stat_name 220 | and b.index_name ='PRIMARY' 221 | ) b 222 | on a.database_name=b.database_name 223 | and a.table_name=b.table_name 224 | where b.stat_value is not null 225 | and a.stat_value >0 226 | order by stat_pct; 227 | 228 | +---------------+-------------------+--------------+--------+--------+----------+ 229 | | database_name | table_name | index_name | SK | PK | stat_pct | 230 | +---------------+-------------------+--------------+--------+--------+----------+ 231 | | zhishutang | t_json_vs_vchar | c1vc | 37326 | 39825 | 93.73 | 232 | | zhishutang | t_json_vs_vchar | c2vc | 37371 | 39825 | 93.84 | 233 | | zhishutang | t1 | name | 299815 | 299842 | 99.99 | 234 | | zhishutang | t4 | c2 | 2 | 2 | 100.00 | 235 | +---------------+-------------------+--------------+--------+--------+----------+ 236 | ``` 237 | -------------------------------------------------------------------------------- /mysql-tools/mysql_sos: -------------------------------------------------------------------------------- 1 | !/usr/bin/env bash 2 | #紧急救援，故障收集 3 | #根据自己的情况调整用户名 密码 和路径 4 | 5 | mysql_port="3309" 6 | ip="192.168.80.129" 7 | time="2018" 8 | file=${ip}_${mysql_port}_${time} 9 | 10 | mkdir -p /data/mysql_pack/log/$file/log 11 | 12 | for((i=1;i<=2;i++)); 13 | do 14 | pt-mysql-summary --user=admin --password=redhat --host=127.0.0.1 --port=$mysql_port >> /data/mysql_pack/log/$file/mysql-summary.log 15 | pt-summary >> /data/mysql_pack/log/$file/pt-summary .log 16 | done 17 | 18 | top -b -n 10 -d 3 > /data/mysql_pack/log/$file/top.log 19 | free -m > /data/mysql_pack/log/$file/free.log 20 | free -g >> /data/mysql_pack/log/$file/free.log 21 | df -h > /data/mysql_pack/log/$file/df.log 22 | df -i >> /data/mysql_pack/log/$file/df.log 23 | vmstat -a 2 5 > /data/mysql_pack/log/$file/vmstat.log 24 | iostat -x 10 5 > /data/mysql_pack/log/$file/iostat.log 25 | iotop -b --iter=10 > /data/mysql_pack/log/$file/iotop.log 26 | /usr/bin/dmesg > /data/mysql_pack/log/$file/dmesg.log 27 | cp -rf /var/log/messages* /data/mysql_pack/log/$file/log/ 28 | cat /proc/meminfo > /data/mysql_pack/log/$file/meminfo.log 29 | cat /proc/cpuinfo > /data/mysql_pack/log/$file/cpuinfo.log 30 | mysql_pid=`cat /data/mysql/3309/data/localhost.pid` 31 | lsof -p $mysql_pid > /data/mysql_pack/log/$file/lsof.log 32 | pstack $mysql_pid > /data/mysql_pack/log/$file/pstack.log 33 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "show GLOBAL VARIABLES \G" >> /data/mysql_pack/log/$file/variables.log 34 | 35 | for((i=1;i<=10;i++)); 36 | do 37 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "SHOW FULL PROCESSLIST" >> /data/mysql_pack/log/$file/processlist.log 38 | sleep 1 39 | done 40 | 41 | for((i=1;i<=3;i++)); 42 | do 43 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "show GLOBAL status\G " >> /data/mysql_pack/log/$file/status.log 44 | sleep 5 45 | 46 | done 47 | 48 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "show slave status\G" >> /data/mysql_pack/log/$file/slave.log 49 | 50 | 51 | for((i=1;i<=3;i++)); 52 | do 53 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "select * from sys.schema_table_lock_waits\G " >> /data/mysql_pack/log/$file/locks.log 54 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "select * from sys.io_global_by_file_by_bytes\G " >> /data/mysql_pack/log/$file/IO_global_latency.log 55 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "select * from sys.io_global_by_file_by_latency\G " >> /data/mysql_pack/log/$file/IO_File_latency.log 56 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "select * from sys.memory_by_thread_by_current_bytes\G " >> /data/mysql_pack/log/$file/Mem_Thread_Current.log 57 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "select * from sys.memory_global_by_current_bytes\G " >> /data/mysql_pack/log/$file/Mem_Global_Current.log 58 | sleep 1 59 | 60 | done 61 | for((i=1;i<=5;i++)); 62 | do 63 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "show ENGINE innodb status\G " >> /data/mysql_pack/log/$file/innodb_status.log 64 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "show ENGINE innodb MUTEX\G " >> /data/mysql_pack/log/$file/innodb_status_MUTEX.log 65 | sleep 1 66 | done 67 | 68 | 69 | for((i=1;i<=10;i++)); 70 | do 71 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "show GLOBAL status like '%Questions%'\G " >> /data/mysql_pack/log/$file/pqs.log 72 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "show GLOBAL status like '%Handler_commit%'\G " >> /data/mysql_pack/log/$file/commit.log 73 | /usr/local/mysql/bin/mysql -u root -predhat -S /data/mysql/$mysql_port/data/mysql.sock -e "show GLOBAL status like '%Handler_rollback%'\G " >> /data/mysql_pack/log/$file/rollback.log 74 | 75 | sleep 1 76 | done 77 | 78 | 79 | cp -rf /data/mysql/$mysql_port/data/error.log /data/mysql_pack/log/$file/log/ 80 | cp -rf /data/mysql/$mysql_port/data/slow.log /data/mysql_pack/log/$file/log/ 81 | /usr/bin/dmesg > /data/mysql_pack/log/$file/dmesg.log 82 | -------------------------------------------------------------------------------- /mysql-tools/open-falcon/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhishutech/mysqldba/4c04ecbc77ceb0ed9a75575f0b037f33dbe51f23/mysql-tools/open-falcon/.DS_Store -------------------------------------------------------------------------------- /mysql-tools/open-falcon/127.0.0.1_190409_processlist: -------------------------------------------------------------------------------- 1 | 2019-04-09 19:59:19 2 | 2019-04-09 19:59:49 3 | (7315, 'xucl', '112.10.240.98:3666', 'information_schema', 'Query', 0, 'executing', 'select * from information_schema.processlist') 4 | (7197, 'monitor', '127.0.0.1:60556', None, 'Sleep', 0, '', None) 5 | -------------------------------------------------------------------------------- /mysql-tools/open-falcon/README.md: -------------------------------------------------------------------------------- 1 | | metric name | 指标说明 | 取值 | 类型 | 2 | | ------------------------------ | ----------------------------------------- | --------- | ------- | 3 | | mysql_alive | mysql存活状态，1为存活，0为宕机 | 自定义 | GAUGE | 4 | | responsetime | 响应时间，单位：秒 | 自定义 | GAUGE | 5 | | uptime | 存活时间，单位：秒 | status | GAUGE | 6 | | role | 实例角色，1为master，0为slave | 自定义 | GAUGE | 7 | | read_only | 只读 | variables | GAUGE | 8 | | super_read_only | 管理员只读 | variables | GAUGE | 9 | | aborted_client | 已连接断开数 | status | COUNTER | 10 | | aborted_connections | 为连接断开数 | status | COUNTER | 11 | | threads_connected | 已连接线程数 | status | GAUGE | 12 | | threads_running | 活跃线程数 | status | GAUGE | 13 | | threads_created | 历史上创建过的线程数 | status | COUNTER | 14 | | Max_used_connections | 历史最大连接数 | status | GAUGE | 15 | | max_user_connections | 最大用户连接数 | variables | GAUGE | 16 | | max_connections | 最大连接数 | variables | GAUGE | 17 | | innodb_buffer_pool_size | innodb_buffer_pool大小 | variables | GAUGE | 18 | | qps | 每秒查询数，queries/s | 自定义 | GAUGE | 19 | | tps | 每秒事务数，commit+rollback/s | 自定义 | GAUGE | 20 | | created_tmp_disk_tables | 创建磁盘临时表数 | status | COUNTER | 21 | | created_tmp_files | 创建临时文件数 | status | COUNTER | 22 | | created_tmp_tables | 创建内存临时表数 | status | COUNTER | 23 | | select_full_join | 全表join数 | status | COUNTER | 24 | | select_full_range | 全表range数 | status | COUNTER | 25 | | select_scan | 全表扫描数 | status | COUNTER | 26 | | Slow_queries | 慢查询数 | status | COUNTER | 27 | | table_locks_immediate | 立即锁表次数 | status | GAUGE | 28 | | table_locks_waited | 等待表锁次数 | status | COUNTER | 29 | | innodb_row_lock_current_waits | 当前行锁等待数 | status | GAUGE | 30 | | innodb_row_lock_time | 行锁等待时间 | status | COUNTER | 31 | | innodb_row_lock_time_avg | 平均行锁等待时间 | status | GAUGE | 32 | | innodb_row_lock_waits | 行锁等待次数 | status | COUNTER | 33 | | bytes_received | 接受字节数 | status | COUNTER | 34 | | bytes_sent | 发送字节数 | status | COUNTER | 35 | | innodb_buffer_pool_wait_free | 等待buffer_pool分配新page数 | status | COUNTER | 36 | | innodb_log_waits | 等待刷redo log次数 | status | COUNTER | 37 | | history_list_length | 等待被purge的队列长度，来自innodb status | 自定义 | GAUGE | 38 | | loglsn | 当前lsn值 | 自定义 | GAUGE | 39 | | logflushlsn | 刷新到redo的lsn值 | 自定义 | GAUGE | 40 | | checkpointlsn | check point lsn值 | 自定义 | GAUGE | 41 | | open_files | 当前打开文件数 | status | GAUGE | 42 | | opened_files | 历史打开文件数 | status | COUNTER | 43 | | open_tables | 当前打开表数 | status | GAUGE | 44 | | opened_tables | 历史打开表数 | status | COUNTER | 45 | | table_open_cache | 表缓存数 | variables | GAUGE | 46 | | Open_table_definitions | 当前打开表定义数 | status | GAUGE | 47 | | opened_table_definitions | 历史打开表定义数 | status | COUNTER | 48 | | table_definition_cache | 表定义数 | variables | GAUGE | 49 | | Handler_read_key | 读取索引次数 | status | COUNTER | 50 | | Handler_read_next | 读取下一个值次数 | status | COUNTER | 51 | | Handler_read_rnd | 随机读次数 | status | COUNTER | 52 | | Handler_read_rnd_next | 全表扫描或者排序或者读下一行 | status | COUNTER | 53 | | sort_merge_pass | 归并排序次数 | status | COUNTER | 54 | | binlog_cache_disk_use | 使用磁盘存储binlog | status | COUNTER | 55 | | slave_io_running | slave上io线程状态，来源show slave status | 自定义 | GAUGE | 56 | | slave_sql_running | slave上sql线程状态,来源show slave status | 自定义 | GAUGE | 57 | | Seconds_behind_master | slave延迟（暂时），后期改为pt-slave-delay | 自定义 | GAUGE | 58 | | innodb_flush_log_at_trx_commit | innodb_flush_log_at_trx_commit | variables | GAUGE | 59 | | sync_binlog | sync_binlog | variables | GAUGE | 60 | | Com_select | select次数 | status | COUNTER | 61 | | Com_update | update次数 | status | COUNTER | 62 | | Com_insert | insert次数 | status | COUNTER | 63 | | Com_delete | delete次数 | status | COUNTER | 64 | | Com_commit | commit次数 | status | COUNTER | 65 | | Com_rollback | rollback次数 | status | COUNTER | 66 | 67 | -------------------------------------------------------------------------------- /mysql-tools/open-falcon/metric示例.txt: -------------------------------------------------------------------------------- 1 | [{'endpoint': '127.0.0.1', 'metric': 'read_only', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 2 | {'endpoint': '127.0.0.1', 'metric': 'super_read_only', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 3 | {'endpoint': '127.0.0.1', 'metric': 'innodb_buffer_pool_size', 'timestamp': 1554378553, 'step': 60, 'value': 134217728, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 4 | {'endpoint': '127.0.0.1', 'metric': 'table_open_cache', 'timestamp': 1554378553, 'step': 60, 'value': 1024, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 5 | {'endpoint': '127.0.0.1', 'metric': 'table_definition_cache', 'timestamp': 1554378553, 'step': 60, 'value': 1024, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 6 | {'endpoint': '127.0.0.1', 'metric': 'innodb_flush_log_at_trx_commit', 'timestamp': 1554378553, 'step': 60, 'value': 1, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 7 | {'endpoint': '127.0.0.1', 'metric': 'sync_binlog', 'timestamp': 1554378553, 'step': 60, 'value': 1, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 8 | {'endpoint': '127.0.0.1', 'metric': 'max_connections', 'timestamp': 1554378553, 'step': 60, 'value': 512, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 9 | {'endpoint': '127.0.0.1', 'metric': 'max_user_connections', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 10 | {'endpoint': '127.0.0.1', 'metric': 'Uptime', 'timestamp': 1554378553, 'step': 60, 'value': 96591, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 11 | {'endpoint': '127.0.0.1', 'metric': 'Threads_connected', 'timestamp': 1554378553, 'step': 60, 'value': 3, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 12 | {'endpoint': '127.0.0.1', 'metric': 'Threads_created', 'timestamp': 1554378553, 'step': 60, 'value': 3, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 13 | {'endpoint': '127.0.0.1', 'metric': 'Threads_running', 'timestamp': 1554378553, 'step': 60, 'value': 1, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 14 | {'endpoint': '127.0.0.1', 'metric': 'Innodb_row_lock_current_waits', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 15 | {'endpoint': '127.0.0.1', 'metric': 'Innodb_row_lock_time_avg', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 16 | {'endpoint': '127.0.0.1', 'metric': 'Open_files', 'timestamp': 1554378553, 'step': 60, 'value': 11, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 17 | {'endpoint': '127.0.0.1', 'metric': 'Open_tables', 'timestamp': 1554378553, 'step': 60, 'value': 90, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 18 | {'endpoint': '127.0.0.1', 'metric': 'Open_table_definitions', 'timestamp': 1554378553, 'step': 60, 'value': 108, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 19 | {'endpoint': '127.0.0.1', 'metric': 'Max_used_connections', 'timestamp': 1554378553, 'step': 60, 'value': 3, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 20 | {'endpoint': '127.0.0.1', 'metric': 'Aborted_clients', 'timestamp': 1554378553, 'step': 60, 'value': 4, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 21 | {'endpoint': '127.0.0.1', 'metric': 'Aborted_connects', 'timestamp': 1554378553, 'step': 60, 'value': 9, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 22 | {'endpoint': '127.0.0.1', 'metric': 'Created_tmp_disk_tables', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 23 | {'endpoint': '127.0.0.1', 'metric': 'Created_tmp_files', 'timestamp': 1554378553, 'step': 60, 'value': 5, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 24 | {'endpoint': '127.0.0.1', 'metric': 'Created_tmp_tables', 'timestamp': 1554378553, 'step': 60, 'value': 176, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 25 | {'endpoint': '127.0.0.1', 'metric': 'Select_full_join', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 26 | {'endpoint': '127.0.0.1', 'metric': 'Select_full_range_join', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 27 | {'endpoint': '127.0.0.1', 'metric': 'Select_scan', 'timestamp': 1554378553, 'step': 60, 'value': 352, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 28 | {'endpoint': '127.0.0.1', 'metric': 'Slow_queries', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 29 | {'endpoint': '127.0.0.1', 'metric': 'Table_locks_immediate', 'timestamp': 1554378553, 'step': 60, 'value': 274, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 30 | {'endpoint': '127.0.0.1', 'metric': 'Table_locks_waited', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 31 | {'endpoint': '127.0.0.1', 'metric': 'Innodb_row_lock_time', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 32 | {'endpoint': '127.0.0.1', 'metric': 'Innodb_row_lock_waits', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 33 | {'endpoint': '127.0.0.1', 'metric': 'Bytes_received', 'timestamp': 1554378553, 'step': 60, 'value': 2824338, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 34 | {'endpoint': '127.0.0.1', 'metric': 'Bytes_sent', 'timestamp': 1554378553, 'step': 60, 'value': 57086730, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 35 | {'endpoint': '127.0.0.1', 'metric': 'Innodb_buffer_pool_wait_free', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 36 | {'endpoint': '127.0.0.1', 'metric': 'Innodb_log_waits', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 37 | {'endpoint': '127.0.0.1', 'metric': 'Opened_files', 'timestamp': 1554378553, 'step': 60, 'value': 179, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 38 | {'endpoint': '127.0.0.1', 'metric': 'Opened_tables', 'timestamp': 1554378553, 'step': 60, 'value': 183, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 39 | {'endpoint': '127.0.0.1', 'metric': 'Opened_table_definitions', 'timestamp': 1554378553, 'step': 60, 'value': 108, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 40 | {'endpoint': '127.0.0.1', 'metric': 'Handler_read_key', 'timestamp': 1554378553, 'step': 60, 'value': 27, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 41 | {'endpoint': '127.0.0.1', 'metric': 'Handler_read_next', 'timestamp': 1554378553, 'step': 60, 'value': 2, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 42 | {'endpoint': '127.0.0.1', 'metric': 'Handler_read_rnd', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 43 | {'endpoint': '127.0.0.1', 'metric': 'Handler_read_rnd_next', 'timestamp': 1554378553, 'step': 60, 'value': 137317, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 44 | {'endpoint': '127.0.0.1', 'metric': 'Sort_merge_passes', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 45 | {'endpoint': '127.0.0.1', 'metric': 'Binlog_cache_disk_use', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 46 | {'endpoint': '127.0.0.1', 'metric': 'Com_select', 'timestamp': 1554378553, 'step': 60, 'value': 64403, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 47 | {'endpoint': '127.0.0.1', 'metric': 'Com_update', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 48 | {'endpoint': '127.0.0.1', 'metric': 'Com_insert', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 49 | {'endpoint': '127.0.0.1', 'metric': 'Com_delete', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 50 | {'endpoint': '127.0.0.1', 'metric': 'Com_commit', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 51 | {'endpoint': '127.0.0.1', 'metric': 'Com_rollback', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'COUNTER', 'tags': 'idc=dx,region=hangzhou'}, 52 | {'endpoint': '127.0.0.1', 'metric': 'qps', 'timestamp': 1554378553, 'step': 60, 'value': 3, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 53 | {'endpoint': '127.0.0.1', 'metric': 'tps', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 54 | {'endpoint': '127.0.0.1', 'metric': 'historylength', 'timestamp': 1554378553, 'step': 60, 'value': 20, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 55 | {'endpoint': '127.0.0.1', 'metric': 'loglsn', 'timestamp': 1554378553, 'step': 60, 'value': 1412905856, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 56 | {'endpoint': '127.0.0.1', 'metric': 'logflushlsn', 'timestamp': 1554378553, 'step': 60, 'value': 1412905856, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 57 | {'endpoint': '127.0.0.1', 'metric': 'checkpintlsn', 'timestamp': 1554378553, 'step': 60, 'value': 1412905847, 'counterType': 'GAUGE', 'tags': 'idc=dx,region=hangzhou'}, 58 | {'endpoint': '127.0.0.1', 'metric': 'role', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 59 | {'endpoint': '127.0.0.1', 'metric': 'Slave_IO_Running', 'timestamp': 1554378553, 'step': 60, 'value': 1, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 60 | {'endpoint': '127.0.0.1', 'metric': 'Slave_SQL_Running', 'timestamp': 1554378553, 'step': 60, 'value': 1, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 61 | {'endpoint': '127.0.0.1', 'metric': 'Seconds_Behind_Master', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 62 | {'endpoint': '127.0.0.1', 'metric': 'reponsetime', 'timestamp': 1554378553, 'step': 60, 'value': 0, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}, 63 | {'endpoint': '127.0.0.1', 'metric': 'mysql_alive', 'timestamp': 1554378553, 'step': 60, 'value': 1, 'counterType': 'GAGUE', 'tags': 'idc=dx,region=hangzhou'}] -------------------------------------------------------------------------------- /mysql-tools/open-falcon/mymon.cnf: -------------------------------------------------------------------------------- 1 | [mysql] 2 | HOST=127.0.0.1 3 | PORT=3306 4 | USERNAME=xucl 5 | PASSWORD=xuclxucl 6 | ENDPOINT=127.0.0.1 7 | TAGS=idc=dx,region=hangzhou 8 | -------------------------------------------------------------------------------- /mysql-tools/open-falcon/mymon.py: -------------------------------------------------------------------------------- 1 | #! /bin/env python 2 | # coding:gbk 3 | # author:xucl 4 | import configparser 5 | import requests 6 | import pymysql 7 | import time, datetime 8 | import json 9 | import os 10 | 11 | from mymon_utils import map_value, get_var_metric, get_sta_metric, get_statusdiff_metric, get_innodb_metric, get_slave_metric, get_other_metric, generate_metric 12 | 13 | 14 | class DBUtil: 15 | def __init__(self, user=None, passwd=None, host=None, port=None, db=None, endpoint=None, tags=None): 16 | self.user = user 17 | self.passwd = passwd 18 | self.host = host 19 | self.port = port 20 | self.db = db 21 | self.endpoint = endpoint 22 | self.tags = tags 23 | self._conn = None 24 | self._cursor = None 25 | 26 | def __enter__(self): 27 | self._conn = pymysql.connect(host=self.host, port=self.port, user=self.user, passwd=self.passwd, db=self.db) 28 | self._cursor = self._conn.cursor() 29 | return self 30 | 31 | def __exit__(self, exc_type, exc_val, exc_tb): 32 | self._conn.close() 33 | self._conn = None 34 | 35 | def get_status(self): 36 | self._cursor.execute('show global status;') 37 | data_list = self._cursor.fetchall() 38 | data_dict = {} 39 | for item in data_list: 40 | data_dict[item[0]] = item[1] 41 | return data_dict 42 | 43 | def get_innodb_status(self): 44 | self._cursor.execute('show engine innodb status;') 45 | innodb_status = self._cursor.fetchall() 46 | innodb_status_format = str(innodb_status).split('\\n') 47 | return innodb_status_format 48 | 49 | def get_slave_status(self): 50 | column_list = [] 51 | data_dict = {} 52 | self._cursor.execute('show slave status;') 53 | data_list = self._cursor.fetchall() 54 | if data_list: 55 | data_list = list(data_list[0]) 56 | for index, desc in enumerate(self._cursor.description): 57 | data_dict[desc[0]] = data_list[index] 58 | return data_dict 59 | 60 | def get_variables(self, variable_list): 61 | self._cursor.execute('show global variables;') 62 | data_list = self._cursor.fetchall() 63 | data_dict = {} 64 | for item in data_list: 65 | data_dict[item[0]] = item[1] 66 | metric_list = get_var_metric(self, variable_list, data_dict, ts) 67 | return metric_list 68 | 69 | def get_status_metric(self, status, data_dict, type): 70 | metric_list = get_sta_metric(self, status, data_dict, ts, type) 71 | return metric_list 72 | 73 | def get_custom_status(self, data_dict1, data_dict2): 74 | metric_list = get_statusdiff_metric(self, data_dict1, data_dict2, ts) 75 | return metric_list 76 | 77 | def get_custom_metrics(self, data_dict, innodb_status_format, slave_status_format): 78 | metrc_list = [] 79 | start_time = int(time.time()) 80 | self._cursor.execute("select benchmark(10000000,'select 1+1');") 81 | end_time = int(time.time()) 82 | rt = end_time - start_time 83 | innodb_metric_list = get_innodb_metric(self, innodb_status_format, ts) 84 | slave_metric_list = get_slave_metric(self, slave_status_format, ts) 85 | othter_metric_list = get_other_metric(self, "reponsetime", rt, ts) 86 | alive_metrics = get_other_metric(self, "mysql_alive", 1, ts) 87 | metrc_list.extend(innodb_metric_list) 88 | metrc_list.extend(slave_metric_list) 89 | metrc_list.extend(othter_metric_list) 90 | metrc_list.extend(alive_metrics) 91 | return metrc_list 92 | 93 | def record_innodb_status(self, innodb_status_format): 94 | status_file = self.endpoint + '_' + datetime.datetime.now().strftime('%y%m%d') + '_innodb_status' 95 | innodb_status_format = innodb_status_format[1:-1] 96 | f = open(status_file, "a+") 97 | monitor_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') 98 | f.write(monitor_time + '\n') 99 | for item in innodb_status_format: 100 | f.write(item + '\n') 101 | f.close() 102 | 103 | def record_processlist(self): 104 | self._cursor.execute('select * from information_schema.processlist;') 105 | processlist = self._cursor.fetchall() 106 | processlist_file = self.endpoint + '_' + datetime.datetime.now().strftime('%y%m%d') + '_processlist' 107 | fp = open(processlist_file, "a+") 108 | monitor_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') 109 | fp.write(monitor_time + '\n') 110 | for item in processlist: 111 | fp.write(str(item) + '\n') 112 | fp.close() 113 | 114 | 115 | if __name__ == '__main__': 116 | 117 | cf = configparser.ConfigParser() 118 | cf.read('mymon.cnf') 119 | 120 | hostname = cf.get('mysql', 'HOST') 121 | port = cf.get('mysql', 'PORT') 122 | username = cf.get('mysql', 'USERNAME') 123 | password = cf.get('mysql', 'PASSWORD') 124 | endpoint = cf.get('mysql', 'ENDPOINT') 125 | tags = cf.get('mysql', 'TAGS') 126 | 127 | ts = int(time.time()) 128 | payload = [] 129 | try: 130 | with DBUtil(username, password, hostname, int(port), 'information_schema', endpoint, tags) as client: 131 | data_dict1 = client.get_status() 132 | innodb_status_format = client.get_innodb_status() 133 | slave_status_format = client.get_slave_status() 134 | time.sleep(1) 135 | data_dict2 = client.get_status() 136 | client.record_processlist() 137 | 138 | variables = ['read_only', 'super_read_only', 'innodb_buffer_pool_size', 'table_open_cache', 'table_definition_cache', 139 | 'innodb_flush_log_at_trx_commit', 'sync_binlog', 'max_connections', 'max_user_connections'] 140 | gague_status = ['Uptime', 'Threads_connected', 'Threads_created', 'Threads_running','Innodb_row_lock_current_waits', 141 | 'Innodb_row_lock_time_avg', 'Open_files', 'Open_tables','Open_table_definitions', 'Max_used_connections'] 142 | counter_status = ['Aborted_clients', 'Aborted_connects', 'Created_tmp_disk_tables', 'Created_tmp_files', 'Created_tmp_tables', 143 | 'Select_full_join', 'Select_full_range_join', 'Select_scan', 'Slow_queries', 'Table_locks_immediate', 144 | 'Table_locks_waited', 'Innodb_row_lock_time', 'Innodb_row_lock_waits', 'Bytes_received', 'Bytes_sent', 'Innodb_buffer_pool_wait_free', 145 | 'Innodb_log_waits', 'Opened_files', 'Opened_tables', 'Opened_table_definitions', 'Handler_read_key', 146 | 'Handler_read_next', 'Handler_read_rnd','Handler_read_rnd_next', 'Sort_merge_passes', 'Binlog_cache_disk_use', 147 | 'Com_select', 'Com_update', 'Com_insert', 'Com_delete', 'Com_commit', 'Com_rollback'] 148 | var_metrics = client.get_variables(variables) 149 | guage_metrics = client.get_status_metric(gague_status, data_dict1, "GAUGE") 150 | counter_metrics = client.get_status_metric(counter_status, data_dict1, "COUNTER") 151 | custom_status = client.get_custom_status(data_dict1, data_dict2) 152 | custom_metrics = client.get_custom_metrics(data_dict1, innodb_status_format, slave_status_format) 153 | payload.extend(var_metrics) 154 | payload.extend(guage_metrics) 155 | payload.extend(counter_metrics) 156 | payload.extend(custom_status) 157 | payload.extend(custom_metrics) 158 | client.record_innodb_status(innodb_status_format) 159 | 160 | except: 161 | alive_metrics = generate_metric(endpoint, "mysql_alive", ts, 60, 0, "GAUGE", tags) 162 | payload.append(alive_metrics) 163 | 164 | r = requests.post("http://127.0.0.1:1988/v1/push", data=json.dumps()) 165 | print(r.text) 166 | -------------------------------------------------------------------------------- /mysql-tools/open-falcon/mymon_utils.py: -------------------------------------------------------------------------------- 1 | #! /bin/env python 2 | # coding:gbk 3 | # author:xucl 4 | 5 | 6 | def map_value(value): 7 | if value == 'ON' or value == 'Yes': 8 | return 1 9 | elif value == 'OFF' or value == 'No': 10 | return 0 11 | else: 12 | return int(value) 13 | 14 | 15 | def generate_metric(endpoint, metric, ts, step, value, type, tags): 16 | metric_dict = {} 17 | metric_dict['endpoint'] = endpoint 18 | metric_dict['metric'] = metric 19 | metric_dict['timestamp'] = ts 20 | metric_dict['step'] = step 21 | metric_dict['value'] = value 22 | metric_dict['counterType'] = type 23 | metric_dict['tags'] = tags 24 | return metric_dict 25 | 26 | 27 | def get_var_metric(self, variables, data_dict, ts): 28 | metric_list = [] 29 | for var in variables: 30 | try: 31 | value = map_value(data_dict[var]) 32 | except: 33 | value = -1 34 | metric = generate_metric(self.endpoint, var, ts, 60, value, "GAUGE", self.tags) 35 | metric_list.append(metric) 36 | return metric_list 37 | 38 | 39 | def get_sta_metric(self, status, data_dict, ts, type): 40 | metric_list = [] 41 | for sta in status: 42 | try: 43 | value = map_value(data_dict[sta]) 44 | except: 45 | value = -1 46 | metric = generate_metric(self.endpoint, sta, ts, 60, value, type, self.tags) 47 | metric_list.append(metric) 48 | return metric_list 49 | 50 | 51 | def get_statusdiff_metric(self, data_dict1, data_dict2, ts): 52 | metric_list = [] 53 | try: 54 | qps_value = map_value(data_dict2['Queries']) - map_value(data_dict1['Queries']) 55 | tps_value = map_value(data_dict2['Handler_commit']) - map_value(data_dict1['Handler_commit']) + map_value(data_dict2['Handler_rollback']) - map_value(data_dict2['Handler_rollback']) 56 | except: 57 | qps_value = -1 58 | tps_value = -1 59 | qps = generate_metric(self.endpoint, "qps", ts, 60, qps_value, "GAUGE", self.tags) 60 | tps = generate_metric(self.endpoint, "tps", ts, 60, tps_value, "GAUGE", self.tags) 61 | metric_list.append(qps) 62 | metric_list.append(tps) 63 | return metric_list 64 | 65 | 66 | def get_innodb_metric(self, innodb_status_format, ts): 67 | metric_list = [] 68 | for item in innodb_status_format: 69 | if "Log sequence number" in item: 70 | value = map_value(item.split(' ')[3]) 71 | metric = generate_metric(self.endpoint, "loglsn", ts, 60, value, "GAUGE", self.tags) 72 | metric_list.append(metric) 73 | if "Log flushed up to" in item: 74 | value = map_value(item.split(' ')[6]) 75 | metric = generate_metric(self.endpoint, "logflushlsn", ts, 60, value, "GAUGE", self.tags) 76 | metric_list.append(metric) 77 | if "Last checkpoint at" in item: 78 | value = map_value(item.split(' ')[4]) 79 | metric = generate_metric(self.endpoint, "checkpointlsn", ts, 60, value, "GAUGE", self.tags) 80 | metric_list.append(metric) 81 | elif "History list length" in item: 82 | value = map_value(item.split(' ')[3]) 83 | metric = generate_metric(self.endpoint, "historylength", ts, 60, value, "GAUGE", self.tags) 84 | metric_list.append(metric) 85 | return metric_list 86 | 87 | 88 | def get_slave_metric(self, slave_status_format, ts): 89 | metric_list = [] 90 | if slave_status_format: 91 | metric_name = "role" 92 | value = 0 93 | else: 94 | metric_name = "role" 95 | value = 1 96 | metric = generate_metric(self.endpoint, metric_name, ts, 60, value, "GAUGE", self.tags) 97 | metric_list.append(metric) 98 | 99 | if slave_status_format: 100 | for item in ['Slave_IO_Running', 'Slave_SQL_Running', 'Seconds_Behind_Master']: 101 | value = map_value(slave_status_format[item]) 102 | metric = generate_metric(self.endpoint, item, ts, 60, value, "GAUGE", self.tags) 103 | metric_list.append(metric) 104 | return metric_list 105 | 106 | 107 | def get_other_metric(self, metric_name, value, ts): 108 | metric_list = [] 109 | metric = generate_metric(self.endpoint, metric_name, ts, 60, value, "GAUGE", self.tags) 110 | metric_list.append(metric) 111 | return metric_list 112 | -------------------------------------------------------------------------------- /mysql-tools/sql.txt: -------------------------------------------------------------------------------- 1 | select * from t1 2 | select * from t2 3 | select * from t3 4 | -------------------------------------------------------------------------------- /mysql-tools/sql_result_between_two_instance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*-# 3 | # auth:xucl 4 | 5 | import sys 6 | import time 7 | from datetime import datetime 8 | import MySQLdb 9 | import hashlib 10 | 11 | class DBUtil: 12 | def __init__(self, src_host=None, src_port=None, src_user=None, src_passwd=None, src_db=None, tar_host=None, tar_port=None, tar_user=None, tar_passwd=None, tar_db=None): 13 | self.src_host = src_host 14 | self.src_port = src_port 15 | self.src_user = src_user 16 | self.src_passwd = src_passwd 17 | self.src_db = src_db 18 | self.tar_host = tar_host 19 | self.tar_port = tar_port 20 | self.tar_user = tar_user 21 | self.tar_passwd = tar_passwd 22 | self.tar_db = tar_db 23 | 24 | def __enter__(self): 25 | self._conn1 = MySQLdb.connect(host=self.src_host, port=self.src_port, user=self.src_user, passwd=self.src_passwd, db=self.src_db) 26 | self._conn2 = MySQLdb.connect(host=self.tar_host, port=self.tar_port, user=self.tar_user, passwd=self.tar_passwd, db=self.tar_db) 27 | self._cursor1 = self._conn1.cursor() 28 | self._cursor2 = self._conn2.cursor() 29 | return self 30 | 31 | def __exit__(self, exc_type, exc_val, exc_tb): 32 | self._conn1.close() 33 | self._conn1 = None 34 | self._conn2.close() 35 | self._conn2 = None 36 | 37 | 38 | def get_sql_md5(self, sql): 39 | self._cursor1.execute(sql) 40 | result1 = self._cursor1.fetchmany(size=1000) 41 | m1 = hashlib.md5() 42 | m1.update(str(result1).encode('utf-8')) 43 | hash1 = m1.hexdigest() 44 | 45 | self._cursor2.execute(sql) 46 | result2 = self._cursor2.fetchmany(size=1000) 47 | m2 = hashlib.md5() 48 | m2.update(str(result2).encode('utf-8')) 49 | hash2 = m2.hexdigest() 50 | 51 | if hash1 == hash2: 52 | return True 53 | else: 54 | return (sql, False) 55 | 56 | 57 | if __name__ == '__main__': 58 | with DBUtil('127.0.0.1', 3306, 'xucl', 'xuclxucl', 'xucl', '127.0.0.1', 3307, 'xucl', 'xuclxucl', 'xucl') as client: 59 | with open('sql.txt', 'r') as f: 60 | sql_list = f.readlines() 61 | for sql in sql_list: 62 | print(client.get_sql_md5(sql.strip('\n'))) 63 | -------------------------------------------------------------------------------- /nginx/blackhole.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## 搜索nginx access log，发现一些不安全的请求URL时，将请求IP加到路由黑洞中 4 | ## 本想重新编译nginx + ModSecurity，但编译时报错，懒得折腾，就写个脚本先做简单处理 5 | ## 6 | ## created by yejinrong@zhishutang.com 7 | ## 2020/6/1 8 | 9 | . ~/.bash_profile 10 | 11 | ## 定义nginx log路径 12 | logdir=/logs/nginx 13 | 14 | ## 记录ip路由黑洞文件 15 | iplist=ip-route.txt 16 | 17 | ## 扫描最近的1000行日志，可以自定义大小 18 | ln=1000 19 | 20 | cd $logdir 21 | 22 | exec 3>&1 4>&2 1>> blackhole.log 2>&1 23 | 24 | date 25 | ## 扫描所有access log 26 | for f in `ls *access*log` 27 | do 28 | ## 搜索关键字 29 | if [ ! -z "`tail -n $ln $f | egrep -i 'base64.*code|union.*select|eval\(|sleep.*28.*29'`" ] ; then 30 | echo "checking logs..." 31 | tail -n $ln $f | egrep -i 'base64.*code|union.*select|eval\(|sleep.*28.*29' 32 | 33 | ## 如果符合条件就加入黑名单 34 | for ip in `tail -n $ln $f | egrep -i 'base64.*code|union.*select|eval\(|sleep.*28.*29' | awk '{print $1}' | sort -u` 35 | do 36 | ## 排除本机等白名单 37 | if [ -z "`grep '^$ip\$' $iplist`" ] && [ $ip != "yejr.run" ] && [ $ip != "127.0.0.1" ] ; then 38 | echo "adding to blackhole list..." 39 | /sbin/ip route add blackhole $ip 40 | fi 41 | done 42 | fi 43 | done 44 | 45 | # 导出所有列表 46 | /sbin/ip route | grep blackhole | awk '{print $2}' > $iplist 47 | 48 | date 49 | echo 50 | echo 51 | echo 52 | -------------------------------------------------------------------------------- /tpch/README.md: -------------------------------------------------------------------------------- 1 | # tpc-h for mysql 2 | 3 | 参考文章：https://imysql.com/2012/12/21/tpch-for-mysql-manual.html 4 | 5 | 本项目是我已经修改后的可以用来运的各个SQL脚本，需要的自取。 6 | 有问题的话，请提issue。 7 | 8 | ## 编译 9 | 1. 安装gcc 10 | ``` 11 | $ yum install -y gcc 12 | ``` 13 | 14 | 2. 编译 15 | ``` 16 | $ cd TPC-H_Tools_v3.0.0/dbgen/ 17 | $ make 18 | ``` 19 | ## 使用 20 | 1. 创建测试库表 21 | ``` 22 | $ mysql -f < SQLs/tpch-ddl.sql 23 | ``` 24 | 25 | 2. 生成测试数据 26 | ``` 27 | $ cd TPC-H_Tools_v3.0.0/dbgen/ 28 | $ ./dbgen -vf -s 1 #一般建议至少10或50、100等，产生足够数据量 29 | ``` 30 | 31 | 3. 导入测试数据 32 | ``` 33 | mysql> load data infile 'path/region.tbl' into table region; 34 | ``` 35 | 其他表同理。 36 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch-ddl.sql: -------------------------------------------------------------------------------- 1 | 2 | drop database tpch; 3 | create database tpch default character set latin1; 4 | use tpch; 5 | 6 | -- sCCSID: @(#)DSS.DDL 2.1.8.1 7 | create table nation ( n_nationkey integer not null, 8 | n_name char(25) not null, 9 | n_regionkey integer not null, 10 | n_comment varchar(152)); 11 | 12 | create table region ( r_regionkey integer not null, 13 | r_name char(25) not null, 14 | r_comment varchar(152)); 15 | 16 | create table part ( p_partkey integer not null, 17 | p_name varchar(55) not null, 18 | p_mfgr char(25) not null, 19 | p_brand char(10) not null, 20 | p_type varchar(25) not null, 21 | p_size integer not null, 22 | p_container char(10) not null, 23 | p_retailprice decimal(15,2) not null, 24 | p_comment varchar(23) not null ); 25 | 26 | create table supplier ( s_suppkey integer not null, 27 | s_name char(25) not null, 28 | s_address varchar(40) not null, 29 | s_nationkey integer not null, 30 | s_phone char(15) not null, 31 | s_acctbal decimal(15,2) not null, 32 | s_comment varchar(101) not null); 33 | 34 | create table partsupp ( ps_partkey integer not null, 35 | ps_suppkey integer not null, 36 | ps_availqty integer not null, 37 | ps_supplycost decimal(15,2) not null, 38 | ps_comment varchar(199) not null ); 39 | 40 | create table customer ( c_custkey integer not null, 41 | c_name varchar(25) not null, 42 | c_address varchar(40) not null, 43 | c_nationkey integer not null, 44 | c_phone char(15) not null, 45 | c_acctbal decimal(15,2) not null, 46 | c_mktsegment char(10) not null, 47 | c_comment varchar(117) not null); 48 | 49 | create table orders ( o_orderkey integer not null, 50 | o_custkey integer not null, 51 | o_orderstatus char(1) not null, 52 | o_totalprice decimal(15,2) not null, 53 | o_orderdate date not null, 54 | o_orderpriority char(15) not null, 55 | o_clerk char(15) not null, 56 | o_shippriority integer not null, 57 | o_comment varchar(79) not null); 58 | 59 | create table lineitem ( l_orderkey integer not null, 60 | l_partkey integer not null, 61 | l_suppkey integer not null, 62 | l_linenumber integer not null, 63 | l_quantity decimal(15,2) not null, 64 | l_extendedprice decimal(15,2) not null, 65 | l_discount decimal(15,2) not null, 66 | l_tax decimal(15,2) not null, 67 | l_returnflag char(1) not null, 68 | l_linestatus char(1) not null, 69 | l_shipdate date not null, 70 | l_commitdate date not null, 71 | l_receiptdate date not null, 72 | l_shipinstruct char(25) not null, 73 | l_shipmode char(10) not null, 74 | l_comment varchar(44) not null); 75 | 76 | -- sCCSID: @(#)DSS.RI 2.1.8.1 77 | -- tpch bENCHMARK vERSION 8.0 78 | 79 | use tpch; 80 | 81 | -- alter table tpch.region drop primary key; 82 | -- alter table tpch.nation drop primary key; 83 | -- alter table tpch.part drop primary key; 84 | -- alter table tpch.supplier drop primary key; 85 | -- alter table tpch.partsupp drop primary key; 86 | -- alter table tpch.orders drop primary key; 87 | -- alter table tpch.lineitem drop primary key; 88 | -- alter table tpch.customer drop primary key; 89 | 90 | 91 | -- fOR TABLE region 92 | alter table tpch.region 93 | add primary key (r_regionkey); 94 | 95 | -- fOR TABLE nation 96 | alter table tpch.nation 97 | add primary key (n_nationkey); 98 | 99 | commit work; 100 | 101 | -- fOR TABLE part 102 | alter table tpch.part 103 | add primary key (p_partkey); 104 | 105 | commit work; 106 | 107 | -- fOR TABLE supplier 108 | alter table tpch.supplier 109 | add primary key (s_suppkey); 110 | 111 | commit work; 112 | 113 | -- fOR TABLE partsupp 114 | alter table tpch.partsupp 115 | add primary key (ps_partkey,ps_suppkey); 116 | 117 | commit work; 118 | 119 | -- fOR TABLE customer 120 | alter table tpch.customer 121 | add primary key (c_custkey); 122 | 123 | commit work; 124 | 125 | -- fOR TABLE lineitem 126 | alter table tpch.lineitem 127 | add primary key (l_orderkey,l_linenumber); 128 | 129 | commit work; 130 | 131 | -- fOR TABLE orders 132 | alter table tpch.orders 133 | add primary key (o_orderkey); 134 | 135 | commit work; 136 | 137 | 138 | 139 | alter table tpch.nation 140 | add foreign key nation_fk1 (n_regionkey) REFERENCES tpch.region(R_REGIONKEY); 141 | 142 | alter table tpch.supplier 143 | add foreign key supplier_fk1 (s_nationkey) REFERENCES tpch.nation(N_NATIONKEY); 144 | 145 | 146 | alter table tpch.customer 147 | add foreign key customer_fk1 (c_nationkey) REFERENCES tpch.nation(N_NATIONKEY); 148 | 149 | alter table tpch.partsupp 150 | add foreign key partsupp_fk1 (ps_suppkey) REFERENCES tpch.supplier(S_SUPPKEY); 151 | 152 | alter table tpch.partsupp 153 | add foreign key partsupp_fk2 (ps_partkey) REFERENCES tpch.part(P_PARTKEY); 154 | 155 | alter table tpch.orders 156 | add foreign key orders_fk1 (o_custkey) REFERENCES tpch.customer(C_CUSTKEY); 157 | 158 | 159 | alter table tpch.lineitem 160 | add foreign key lineitem_fk1 (l_orderkey) REFERENCES tpch.orders(O_ORDERKEY); 161 | 162 | alter table tpch.lineitem 163 | add foreign key lineitem_fk2 (l_partkey,l_suppkey) REFERENCES 164 | tpch.partsupp(PS_PARTKEY,PS_SUPPKEY); 165 | 166 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries.sql: -------------------------------------------------------------------------------- 1 | -- using 1602316083 as a seed to the RNG 2 | 3 | use tpch; 4 | 5 | -- Q1 6 | select 7 | l_returnflag, 8 | l_linestatus, 9 | sum(l_quantity) as sum_qty, 10 | sum(l_extendedprice) as sum_base_price, 11 | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, 12 | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, 13 | avg(l_quantity) as avg_qty, 14 | avg(l_extendedprice) as avg_price, 15 | avg(l_discount) as avg_disc, 16 | count(*) as count_order 17 | from 18 | lineitem 19 | where 20 | l_shipdate <= date '1998-12-01' - interval '88' day 21 | group by 22 | l_returnflag, 23 | l_linestatus 24 | order by 25 | l_returnflag, 26 | l_linestatus limit 1; 27 | 28 | 29 | -- Q2 30 | select 31 | s_acctbal, 32 | s_name, 33 | n_name, 34 | p_partkey, 35 | p_mfgr, 36 | s_address, 37 | s_phone, 38 | s_comment 39 | from 40 | part, 41 | supplier, 42 | partsupp, 43 | nation, 44 | region 45 | where 46 | p_partkey = ps_partkey 47 | and s_suppkey = ps_suppkey 48 | and p_size = 8 49 | and p_type like '%BRASS' 50 | and s_nationkey = n_nationkey 51 | and n_regionkey = r_regionkey 52 | and r_name = 'AFRICA' 53 | and ps_supplycost = ( 54 | select 55 | min(ps_supplycost) 56 | from 57 | partsupp, 58 | supplier, 59 | nation, 60 | region 61 | where 62 | p_partkey = ps_partkey 63 | and s_suppkey = ps_suppkey 64 | and s_nationkey = n_nationkey 65 | and n_regionkey = r_regionkey 66 | and r_name = 'AFRICA' 67 | ) 68 | order by 69 | s_acctbal desc, 70 | n_name, 71 | s_name, 72 | p_partkey limit 100; 73 | 74 | 75 | -- Q3 76 | select 77 | l_orderkey, 78 | sum(l_extendedprice * (1 - l_discount)) as revenue, 79 | o_orderdate, 80 | o_shippriority 81 | from 82 | customer, 83 | orders, 84 | lineitem 85 | where 86 | c_mktsegment = 'MACHINERY' 87 | and c_custkey = o_custkey 88 | and l_orderkey = o_orderkey 89 | and o_orderdate < date '1995-03-01' 90 | and l_shipdate > date '1995-03-01' 91 | group by 92 | l_orderkey, 93 | o_orderdate, 94 | o_shippriority 95 | order by 96 | revenue desc, 97 | o_orderdate limit 10; 98 | 99 | 100 | -- Q4 101 | select 102 | o_orderpriority, 103 | count(*) as order_count 104 | from 105 | orders 106 | where 107 | o_orderdate >= date '1993-01-01' 108 | and o_orderdate < date '1993-01-01' + interval '3' month 109 | and exists ( 110 | select 111 | * 112 | from 113 | lineitem 114 | where 115 | l_orderkey = o_orderkey 116 | and l_commitdate < l_receiptdate 117 | ) 118 | group by 119 | o_orderpriority 120 | order by 121 | o_orderpriority limit 1; 122 | 123 | 124 | -- Q5 125 | select 126 | n_name, 127 | sum(l_extendedprice * (1 - l_discount)) as revenue 128 | from 129 | customer, 130 | orders, 131 | lineitem, 132 | supplier, 133 | nation, 134 | region 135 | where 136 | c_custkey = o_custkey 137 | and l_orderkey = o_orderkey 138 | and l_suppkey = s_suppkey 139 | and c_nationkey = s_nationkey 140 | and s_nationkey = n_nationkey 141 | and n_regionkey = r_regionkey 142 | and r_name = 'EUROPE' 143 | and o_orderdate >= date '1993-01-01' 144 | and o_orderdate < date '1993-01-01' + interval '1' year 145 | group by 146 | n_name 147 | order by 148 | revenue desc limit 1; 149 | 150 | 151 | -- Q6 152 | select 153 | sum(l_extendedprice * l_discount) as revenue 154 | from 155 | lineitem 156 | where 157 | l_shipdate >= date '1993-01-01' 158 | and l_shipdate < date '1993-01-01' + interval '1' year 159 | and l_discount between 0.02 - 0.01 and 0.02 + 0.01 160 | and l_quantity < 24 limit 1; 161 | 162 | 163 | -- Q7 164 | select 165 | supp_nation, 166 | cust_nation, 167 | l_year, 168 | sum(volume) as revenue 169 | from 170 | ( 171 | select 172 | n1.n_name as supp_nation, 173 | n2.n_name as cust_nation, 174 | extract(year from l_shipdate) as l_year, 175 | l_extendedprice * (1 - l_discount) as volume 176 | from 177 | supplier, 178 | lineitem, 179 | orders, 180 | customer, 181 | nation n1, 182 | nation n2 183 | where 184 | s_suppkey = l_suppkey 185 | and o_orderkey = l_orderkey 186 | and c_custkey = o_custkey 187 | and s_nationkey = n1.n_nationkey 188 | and c_nationkey = n2.n_nationkey 189 | and ( 190 | (n1.n_name = 'ALGERIA' and n2.n_name = 'EGYPT') 191 | or (n1.n_name = 'EGYPT' and n2.n_name = 'ALGERIA') 192 | ) 193 | and l_shipdate between date '1995-01-01' and date '1996-12-31' 194 | ) as shipping 195 | group by 196 | supp_nation, 197 | cust_nation, 198 | l_year 199 | order by 200 | supp_nation, 201 | cust_nation, 202 | l_year limit 1; 203 | 204 | 205 | -- Q8 206 | select 207 | o_year, 208 | sum(case 209 | when nation = 'EGYPT' then volume 210 | else 0 211 | end) / sum(volume) as mkt_share 212 | from 213 | ( 214 | select 215 | extract(year from o_orderdate) as o_year, 216 | l_extendedprice * (1 - l_discount) as volume, 217 | n2.n_name as nation 218 | from 219 | part, 220 | supplier, 221 | lineitem, 222 | orders, 223 | customer, 224 | nation n1, 225 | nation n2, 226 | region 227 | where 228 | p_partkey = l_partkey 229 | and s_suppkey = l_suppkey 230 | and l_orderkey = o_orderkey 231 | and o_custkey = c_custkey 232 | and c_nationkey = n1.n_nationkey 233 | and n1.n_regionkey = r_regionkey 234 | and r_name = 'MIDDLE EAST' 235 | and s_nationkey = n2.n_nationkey 236 | and o_orderdate between date '1995-01-01' and date '1996-12-31' 237 | and p_type = 'ECONOMY BRUSHED COPPER' 238 | ) as all_nations 239 | group by 240 | o_year 241 | order by 242 | o_year limit 1; 243 | 244 | 245 | -- Q9 246 | select 247 | nation, 248 | o_year, 249 | sum(amount) as sum_profit 250 | from 251 | ( 252 | select 253 | n_name as nation, 254 | extract(year from o_orderdate) as o_year, 255 | l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount 256 | from 257 | part, 258 | supplier, 259 | lineitem, 260 | partsupp, 261 | orders, 262 | nation 263 | where 264 | s_suppkey = l_suppkey 265 | and ps_suppkey = l_suppkey 266 | and ps_partkey = l_partkey 267 | and p_partkey = l_partkey 268 | and o_orderkey = l_orderkey 269 | and s_nationkey = n_nationkey 270 | and p_name like '%seashell%' 271 | ) as profit 272 | group by 273 | nation, 274 | o_year 275 | order by 276 | nation, 277 | o_year desc limit 1; 278 | 279 | 280 | -- Q10 281 | select 282 | c_custkey, 283 | c_name, 284 | sum(l_extendedprice * (1 - l_discount)) as revenue, 285 | c_acctbal, 286 | n_name, 287 | c_address, 288 | c_phone, 289 | c_comment 290 | from 291 | customer, 292 | orders, 293 | lineitem, 294 | nation 295 | where 296 | c_custkey = o_custkey 297 | and l_orderkey = o_orderkey 298 | and o_orderdate >= date '1994-05-01' 299 | and o_orderdate < date '1994-05-01' + interval '3' month 300 | and l_returnflag = 'R' 301 | and c_nationkey = n_nationkey 302 | group by 303 | c_custkey, 304 | c_name, 305 | c_acctbal, 306 | c_phone, 307 | n_name, 308 | c_address, 309 | c_comment 310 | order by 311 | revenue desc limit 20; 312 | 313 | 314 | -- Q11 315 | select 316 | ps_partkey, 317 | sum(ps_supplycost * ps_availqty) as value 318 | from 319 | partsupp, 320 | supplier, 321 | nation 322 | where 323 | ps_suppkey = s_suppkey 324 | and s_nationkey = n_nationkey 325 | and n_name = 'CHINA' 326 | group by 327 | ps_partkey having 328 | sum(ps_supplycost * ps_availqty) > ( 329 | select 330 | sum(ps_supplycost * ps_availqty) * 0.0001000000 331 | from 332 | partsupp, 333 | supplier, 334 | nation 335 | where 336 | ps_suppkey = s_suppkey 337 | and s_nationkey = n_nationkey 338 | and n_name = 'CHINA' 339 | ) 340 | order by 341 | value desc limit 1; 342 | 343 | 344 | -- Q12 345 | select 346 | l_shipmode, 347 | sum(case 348 | when o_orderpriority = '1-URGENT' 349 | or o_orderpriority = '2-HIGH' 350 | then 1 351 | else 0 352 | end) as high_line_count, 353 | sum(case 354 | when o_orderpriority <> '1-URGENT' 355 | and o_orderpriority <> '2-HIGH' 356 | then 1 357 | else 0 358 | end) as low_line_count 359 | from 360 | orders, 361 | lineitem 362 | where 363 | o_orderkey = l_orderkey 364 | and l_shipmode in ('TRUCK', 'FOB') 365 | and l_commitdate < l_receiptdate 366 | and l_shipdate < l_commitdate 367 | and l_receiptdate >= date '1996-01-01' 368 | and l_receiptdate < date '1996-01-01' + interval '1' year 369 | group by 370 | l_shipmode 371 | order by 372 | l_shipmode limit 1; 373 | 374 | 375 | -- Q13 376 | select 377 | c_count, 378 | count(*) as custdist 379 | from 380 | ( 381 | select 382 | c_custkey, 383 | count(o_orderkey) as c_count 384 | from 385 | customer left outer join orders on 386 | c_custkey = o_custkey 387 | and o_comment not like '%unusual%accounts%' 388 | group by 389 | c_custkey 390 | ) as c_orders 391 | group by 392 | c_count 393 | order by 394 | custdist desc, 395 | c_count desc limit 1; 396 | 397 | 398 | -- Q14 399 | select 400 | 100.00 * sum(case 401 | when p_type like 'PROMO%' 402 | then l_extendedprice * (1 - l_discount) 403 | else 0 404 | end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue 405 | from 406 | lineitem, 407 | part 408 | where 409 | l_partkey = p_partkey 410 | and l_shipdate >= date '1996-04-01' 411 | and l_shipdate < date '1996-04-01' + interval '1' month limit 1; 412 | 413 | create view revenue0 (supplier_no, total_revenue) as 414 | select 415 | l_suppkey, 416 | sum(l_extendedprice * (1 - l_discount)) 417 | from 418 | lineitem 419 | where 420 | l_shipdate >= date '1997-08-01' 421 | and l_shipdate < date '1997-08-01' + interval '3' month 422 | group by 423 | l_suppkey; 424 | 425 | 426 | -- Q15 427 | select 428 | s_suppkey, 429 | s_name, 430 | s_address, 431 | s_phone, 432 | total_revenue 433 | from 434 | supplier, 435 | revenue0 436 | where 437 | s_suppkey = supplier_no 438 | and total_revenue = ( 439 | select 440 | max(total_revenue) 441 | from 442 | revenue0 443 | ) 444 | order by 445 | s_suppkey; 446 | 447 | drop view revenue0 ; 448 | 449 | 450 | -- Q16 451 | select 452 | p_brand, 453 | p_type, 454 | p_size, 455 | count(distinct ps_suppkey) as supplier_cnt 456 | from 457 | partsupp, 458 | part 459 | where 460 | p_partkey = ps_partkey 461 | and p_brand <> 'Brand#23' 462 | and p_type not like 'ECONOMY ANODIZED%' 463 | and p_size in (10, 16, 21, 20, 37, 31, 47, 30) 464 | and ps_suppkey not in ( 465 | select 466 | s_suppkey 467 | from 468 | supplier 469 | where 470 | s_comment like '%Customer%Complaints%' 471 | ) 472 | group by 473 | p_brand, 474 | p_type, 475 | p_size 476 | order by 477 | supplier_cnt desc, 478 | p_brand, 479 | p_type, 480 | p_size limit 1; 481 | 482 | 483 | -- Q17 484 | select 485 | sum(l_extendedprice) / 7.0 as avg_yearly 486 | from 487 | lineitem, 488 | part 489 | where 490 | p_partkey = l_partkey 491 | and p_brand = 'Brand#34' 492 | and p_container = 'SM DRUM' 493 | and l_quantity < ( 494 | select 495 | 0.2 * avg(l_quantity) 496 | from 497 | lineitem 498 | where 499 | l_partkey = p_partkey 500 | ) limit 1; 501 | 502 | 503 | -- Q18 504 | select 505 | c_name, 506 | c_custkey, 507 | o_orderkey, 508 | o_orderdate, 509 | o_totalprice, 510 | sum(l_quantity) 511 | from 512 | customer, 513 | orders, 514 | lineitem 515 | where 516 | o_orderkey in ( 517 | select 518 | l_orderkey 519 | from 520 | lineitem 521 | group by 522 | l_orderkey having 523 | sum(l_quantity) > 314 524 | ) 525 | and c_custkey = o_custkey 526 | and o_orderkey = l_orderkey 527 | group by 528 | c_name, 529 | c_custkey, 530 | o_orderkey, 531 | o_orderdate, 532 | o_totalprice 533 | order by 534 | o_totalprice desc, 535 | o_orderdate limit 100; 536 | 537 | 538 | -- Q19 539 | select 540 | sum(l_extendedprice* (1 - l_discount)) as revenue 541 | from 542 | lineitem, 543 | part 544 | where 545 | ( 546 | p_partkey = l_partkey 547 | and p_brand = 'Brand#12' 548 | and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') 549 | and l_quantity >= 10 and l_quantity <= 10 + 10 550 | and p_size between 1 and 5 551 | and l_shipmode in ('AIR', 'AIR REG') 552 | and l_shipinstruct = 'DELIVER IN PERSON' 553 | ) 554 | or 555 | ( 556 | p_partkey = l_partkey 557 | and p_brand = 'Brand#22' 558 | and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') 559 | and l_quantity >= 15 and l_quantity <= 15 + 10 560 | and p_size between 1 and 10 561 | and l_shipmode in ('AIR', 'AIR REG') 562 | and l_shipinstruct = 'DELIVER IN PERSON' 563 | ) 564 | or 565 | ( 566 | p_partkey = l_partkey 567 | and p_brand = 'Brand#43' 568 | and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') 569 | and l_quantity >= 22 and l_quantity <= 22 + 10 570 | and p_size between 1 and 15 571 | and l_shipmode in ('AIR', 'AIR REG') 572 | and l_shipinstruct = 'DELIVER IN PERSON' 573 | ) limit 1; 574 | 575 | 576 | -- Q20 577 | select 578 | s_name, 579 | s_address 580 | from 581 | supplier, 582 | nation 583 | where 584 | s_suppkey in ( 585 | select 586 | ps_suppkey 587 | from 588 | partsupp 589 | where 590 | ps_partkey in ( 591 | select 592 | p_partkey 593 | from 594 | part 595 | where 596 | p_name like 'floral%' 597 | ) 598 | and ps_availqty > ( 599 | select 600 | 0.5 * sum(l_quantity) 601 | from 602 | lineitem 603 | where 604 | l_partkey = ps_partkey 605 | and l_suppkey = ps_suppkey 606 | and l_shipdate >= date '1994-01-01' 607 | and l_shipdate < date '1994-01-01' + interval '1' year 608 | ) 609 | ) 610 | and s_nationkey = n_nationkey 611 | and n_name = 'INDONESIA' 612 | order by 613 | s_name limit 1; 614 | 615 | 616 | -- Q21 617 | select 618 | s_name, 619 | count(*) as numwait 620 | from 621 | supplier, 622 | lineitem l1, 623 | orders, 624 | nation 625 | where 626 | s_suppkey = l1.l_suppkey 627 | and o_orderkey = l1.l_orderkey 628 | and o_orderstatus = 'F' 629 | and l1.l_receiptdate > l1.l_commitdate 630 | and exists ( 631 | select 632 | * 633 | from 634 | lineitem l2 635 | where 636 | l2.l_orderkey = l1.l_orderkey 637 | and l2.l_suppkey <> l1.l_suppkey 638 | ) 639 | and not exists ( 640 | select 641 | * 642 | from 643 | lineitem l3 644 | where 645 | l3.l_orderkey = l1.l_orderkey 646 | and l3.l_suppkey <> l1.l_suppkey 647 | and l3.l_receiptdate > l3.l_commitdate 648 | ) 649 | and s_nationkey = n_nationkey 650 | and n_name = 'INDONESIA' 651 | group by 652 | s_name 653 | order by 654 | numwait desc, 655 | s_name limit 100; 656 | 657 | 658 | -- Q22 659 | select 660 | cntrycode, 661 | count(*) as numcust, 662 | sum(c_acctbal) as totacctbal 663 | from 664 | ( 665 | select 666 | substring(c_phone from 1 for 2) as cntrycode, 667 | c_acctbal 668 | from 669 | customer 670 | where 671 | substring(c_phone from 1 for 2) in 672 | ('19', '28', '24', '33', '25', '20', '34') 673 | and c_acctbal > ( 674 | select 675 | avg(c_acctbal) 676 | from 677 | customer 678 | where 679 | c_acctbal > 0.00 680 | and substring(c_phone from 1 for 2) in 681 | ('19', '28', '24', '33', '25', '20', '34') 682 | ) 683 | and not exists ( 684 | select 685 | * 686 | from 687 | orders 688 | where 689 | o_custkey = c_custkey 690 | ) 691 | ) as custsale 692 | group by 693 | cntrycode 694 | order by 695 | cntrycode limit 1; 696 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_01.sql: -------------------------------------------------------------------------------- 1 | use tpch; 2 | 3 | select 4 | l_returnflag, 5 | l_linestatus, 6 | sum(l_quantity) as sum_qty, 7 | sum(l_extendedprice) as sum_base_price, 8 | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, 9 | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, 10 | avg(l_quantity) as avg_qty, 11 | avg(l_extendedprice) as avg_price, 12 | avg(l_discount) as avg_disc, 13 | count(*) as count_order 14 | from 15 | lineitem 16 | where 17 | l_shipdate <= date '1998-12-01' - interval '88' day 18 | group by 19 | l_returnflag, 20 | l_linestatus 21 | order by 22 | l_returnflag, 23 | l_linestatus limit 1; 24 | 25 | 26 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_02.sql: -------------------------------------------------------------------------------- 1 | select 2 | s_acctbal, 3 | s_name, 4 | n_name, 5 | p_partkey, 6 | p_mfgr, 7 | s_address, 8 | s_phone, 9 | s_comment 10 | from 11 | part, 12 | supplier, 13 | partsupp, 14 | nation, 15 | region 16 | where 17 | p_partkey = ps_partkey 18 | and s_suppkey = ps_suppkey 19 | and p_size = 8 20 | and p_type like '%BRASS' 21 | and s_nationkey = n_nationkey 22 | and n_regionkey = r_regionkey 23 | and r_name = 'AFRICA' 24 | and ps_supplycost = ( 25 | select 26 | min(ps_supplycost) 27 | from 28 | partsupp, 29 | supplier, 30 | nation, 31 | region 32 | where 33 | p_partkey = ps_partkey 34 | and s_suppkey = ps_suppkey 35 | and s_nationkey = n_nationkey 36 | and n_regionkey = r_regionkey 37 | and r_name = 'AFRICA' 38 | ) 39 | order by 40 | s_acctbal desc, 41 | n_name, 42 | s_name, 43 | p_partkey limit 100; 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_03.sql: -------------------------------------------------------------------------------- 1 | select 2 | l_orderkey, 3 | sum(l_extendedprice * (1 - l_discount)) as revenue, 4 | o_orderdate, 5 | o_shippriority 6 | from 7 | customer, 8 | orders, 9 | lineitem 10 | where 11 | c_mktsegment = 'MACHINERY' 12 | and c_custkey = o_custkey 13 | and l_orderkey = o_orderkey 14 | and o_orderdate < date '1995-03-01' 15 | and l_shipdate > date '1995-03-01' 16 | group by 17 | l_orderkey, 18 | o_orderdate, 19 | o_shippriority 20 | order by 21 | revenue desc, 22 | o_orderdate limit 10; 23 | 24 | 25 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_04.sql: -------------------------------------------------------------------------------- 1 | select 2 | o_orderpriority, 3 | count(*) as order_count 4 | from 5 | orders 6 | where 7 | o_orderdate >= date '1993-01-01' 8 | and o_orderdate < date '1993-01-01' + interval '3' month 9 | and exists ( 10 | select 11 | * 12 | from 13 | lineitem 14 | where 15 | l_orderkey = o_orderkey 16 | and l_commitdate < l_receiptdate 17 | ) 18 | group by 19 | o_orderpriority 20 | order by 21 | o_orderpriority limit 1; 22 | 23 | 24 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_05.sql: -------------------------------------------------------------------------------- 1 | select 2 | n_name, 3 | sum(l_extendedprice * (1 - l_discount)) as revenue 4 | from 5 | customer, 6 | orders, 7 | lineitem, 8 | supplier, 9 | nation, 10 | region 11 | where 12 | c_custkey = o_custkey 13 | and l_orderkey = o_orderkey 14 | and l_suppkey = s_suppkey 15 | and c_nationkey = s_nationkey 16 | and s_nationkey = n_nationkey 17 | and n_regionkey = r_regionkey 18 | and r_name = 'EUROPE' 19 | and o_orderdate >= date '1993-01-01' 20 | and o_orderdate < date '1993-01-01' + interval '1' year 21 | group by 22 | n_name 23 | order by 24 | revenue desc limit 1; 25 | 26 | 27 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_06.sql: -------------------------------------------------------------------------------- 1 | select 2 | sum(l_extendedprice * l_discount) as revenue 3 | from 4 | lineitem 5 | where 6 | l_shipdate >= date '1993-01-01' 7 | and l_shipdate < date '1993-01-01' + interval '1' year 8 | and l_discount between 0.02 - 0.01 and 0.02 + 0.01 9 | and l_quantity < 24 limit 1; 10 | 11 | 12 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_07.sql: -------------------------------------------------------------------------------- 1 | select 2 | supp_nation, 3 | cust_nation, 4 | l_year, 5 | sum(volume) as revenue 6 | from 7 | ( 8 | select 9 | n1.n_name as supp_nation, 10 | n2.n_name as cust_nation, 11 | extract(year from l_shipdate) as l_year, 12 | l_extendedprice * (1 - l_discount) as volume 13 | from 14 | supplier, 15 | lineitem, 16 | orders, 17 | customer, 18 | nation n1, 19 | nation n2 20 | where 21 | s_suppkey = l_suppkey 22 | and o_orderkey = l_orderkey 23 | and c_custkey = o_custkey 24 | and s_nationkey = n1.n_nationkey 25 | and c_nationkey = n2.n_nationkey 26 | and ( 27 | (n1.n_name = 'ALGERIA' and n2.n_name = 'EGYPT') 28 | or (n1.n_name = 'EGYPT' and n2.n_name = 'ALGERIA') 29 | ) 30 | and l_shipdate between date '1995-01-01' and date '1996-12-31' 31 | ) as shipping 32 | group by 33 | supp_nation, 34 | cust_nation, 35 | l_year 36 | order by 37 | supp_nation, 38 | cust_nation, 39 | l_year limit 1; 40 | 41 | 42 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_08.sql: -------------------------------------------------------------------------------- 1 | select 2 | o_year, 3 | sum(case 4 | when nation = 'EGYPT' then volume 5 | else 0 6 | end) / sum(volume) as mkt_share 7 | from 8 | ( 9 | select 10 | extract(year from o_orderdate) as o_year, 11 | l_extendedprice * (1 - l_discount) as volume, 12 | n2.n_name as nation 13 | from 14 | part, 15 | supplier, 16 | lineitem, 17 | orders, 18 | customer, 19 | nation n1, 20 | nation n2, 21 | region 22 | where 23 | p_partkey = l_partkey 24 | and s_suppkey = l_suppkey 25 | and l_orderkey = o_orderkey 26 | and o_custkey = c_custkey 27 | and c_nationkey = n1.n_nationkey 28 | and n1.n_regionkey = r_regionkey 29 | and r_name = 'MIDDLE EAST' 30 | and s_nationkey = n2.n_nationkey 31 | and o_orderdate between date '1995-01-01' and date '1996-12-31' 32 | and p_type = 'ECONOMY BRUSHED COPPER' 33 | ) as all_nations 34 | group by 35 | o_year 36 | order by 37 | o_year limit 1; 38 | 39 | 40 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_09.sql: -------------------------------------------------------------------------------- 1 | select 2 | nation, 3 | o_year, 4 | sum(amount) as sum_profit 5 | from 6 | ( 7 | select 8 | n_name as nation, 9 | extract(year from o_orderdate) as o_year, 10 | l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount 11 | from 12 | part, 13 | supplier, 14 | lineitem, 15 | partsupp, 16 | orders, 17 | nation 18 | where 19 | s_suppkey = l_suppkey 20 | and ps_suppkey = l_suppkey 21 | and ps_partkey = l_partkey 22 | and p_partkey = l_partkey 23 | and o_orderkey = l_orderkey 24 | and s_nationkey = n_nationkey 25 | and p_name like '%seashell%' 26 | ) as profit 27 | group by 28 | nation, 29 | o_year 30 | order by 31 | nation, 32 | o_year desc limit 1; 33 | 34 | 35 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_10.sql: -------------------------------------------------------------------------------- 1 | select 2 | c_custkey, 3 | c_name, 4 | sum(l_extendedprice * (1 - l_discount)) as revenue, 5 | c_acctbal, 6 | n_name, 7 | c_address, 8 | c_phone, 9 | c_comment 10 | from 11 | customer, 12 | orders, 13 | lineitem, 14 | nation 15 | where 16 | c_custkey = o_custkey 17 | and l_orderkey = o_orderkey 18 | and o_orderdate >= date '1994-05-01' 19 | and o_orderdate < date '1994-05-01' + interval '3' month 20 | and l_returnflag = 'R' 21 | and c_nationkey = n_nationkey 22 | group by 23 | c_custkey, 24 | c_name, 25 | c_acctbal, 26 | c_phone, 27 | n_name, 28 | c_address, 29 | c_comment 30 | order by 31 | revenue desc limit 20; 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_11.sql: -------------------------------------------------------------------------------- 1 | select 2 | ps_partkey, 3 | sum(ps_supplycost * ps_availqty) as value 4 | from 5 | partsupp, 6 | supplier, 7 | nation 8 | where 9 | ps_suppkey = s_suppkey 10 | and s_nationkey = n_nationkey 11 | and n_name = 'CHINA' 12 | group by 13 | ps_partkey having 14 | sum(ps_supplycost * ps_availqty) > ( 15 | select 16 | sum(ps_supplycost * ps_availqty) * 0.0001000000 17 | from 18 | partsupp, 19 | supplier, 20 | nation 21 | where 22 | ps_suppkey = s_suppkey 23 | and s_nationkey = n_nationkey 24 | and n_name = 'CHINA' 25 | ) 26 | order by 27 | value desc limit 1; 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_12.sql: -------------------------------------------------------------------------------- 1 | select 2 | l_shipmode, 3 | sum(case 4 | when o_orderpriority = '1-URGENT' 5 | or o_orderpriority = '2-HIGH' 6 | then 1 7 | else 0 8 | end) as high_line_count, 9 | sum(case 10 | when o_orderpriority <> '1-URGENT' 11 | and o_orderpriority <> '2-HIGH' 12 | then 1 13 | else 0 14 | end) as low_line_count 15 | from 16 | orders, 17 | lineitem 18 | where 19 | o_orderkey = l_orderkey 20 | and l_shipmode in ('TRUCK', 'FOB') 21 | and l_commitdate < l_receiptdate 22 | and l_shipdate < l_commitdate 23 | and l_receiptdate >= date '1996-01-01' 24 | and l_receiptdate < date '1996-01-01' + interval '1' year 25 | group by 26 | l_shipmode 27 | order by 28 | l_shipmode limit 1; 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_13.sql: -------------------------------------------------------------------------------- 1 | select 2 | c_count, 3 | count(*) as custdist 4 | from 5 | ( 6 | select 7 | c_custkey, 8 | count(o_orderkey) as c_count 9 | from 10 | customer left outer join orders on 11 | c_custkey = o_custkey 12 | and o_comment not like '%unusual%accounts%' 13 | group by 14 | c_custkey 15 | ) as c_orders 16 | group by 17 | c_count 18 | order by 19 | custdist desc, 20 | c_count desc limit 1; 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_14.sql: -------------------------------------------------------------------------------- 1 | select 2 | 100.00 * sum(case 3 | when p_type like 'PROMO%' 4 | then l_extendedprice * (1 - l_discount) 5 | else 0 6 | end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue 7 | from 8 | lineitem, 9 | part 10 | where 11 | l_partkey = p_partkey 12 | and l_shipdate >= date '1996-04-01' 13 | and l_shipdate < date '1996-04-01' + interval '1' month limit 1; 14 | 15 | 16 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_15.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE VIEW revenue0 AS 3 | SELECT 4 | l_suppkey AS supplier_no, 5 | sum(l_extendedprice * (1 - l_discount)) AS total_revenue 6 | FROM lineitem 7 | WHERE (l_shipdate >= toDate('1997-08-01')) AND (l_shipdate < (toDate('1997-08-01') + toIntervalMonth('3'))) 8 | GROUP BY l_suppkey; 9 | 10 | 11 | select 12 | s_suppkey, 13 | s_name, 14 | s_address, 15 | s_phone, 16 | total_revenue 17 | from 18 | supplier, 19 | revenue0 20 | where 21 | s_suppkey = supplier_no 22 | and total_revenue = ( 23 | select 24 | max(total_revenue) 25 | from 26 | revenue0 27 | ) 28 | order by 29 | s_suppkey; 30 | 31 | drop view revenue0 ; 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_16.sql: -------------------------------------------------------------------------------- 1 | select 2 | p_brand, 3 | p_type, 4 | p_size, 5 | count(distinct ps_suppkey) as supplier_cnt 6 | from 7 | partsupp, 8 | part 9 | where 10 | p_partkey = ps_partkey 11 | and p_brand <> 'Brand#23' 12 | and p_type not like 'ECONOMY ANODIZED%' 13 | and p_size in (10, 16, 21, 20, 37, 31, 47, 30) 14 | and ps_suppkey not in ( 15 | select 16 | s_suppkey 17 | from 18 | supplier 19 | where 20 | s_comment like '%Customer%Complaints%' 21 | ) 22 | group by 23 | p_brand, 24 | p_type, 25 | p_size 26 | order by 27 | supplier_cnt desc, 28 | p_brand, 29 | p_type, 30 | p_size limit 1; 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_17.sql: -------------------------------------------------------------------------------- 1 | select 2 | sum(l_extendedprice) / 7.0 as avg_yearly 3 | from 4 | lineitem, 5 | part 6 | where 7 | p_partkey = l_partkey 8 | and p_brand = 'Brand#34' 9 | and p_container = 'SM DRUM' 10 | and l_quantity < ( 11 | select 12 | 0.2 * avg(l_quantity) 13 | from 14 | lineitem 15 | where 16 | l_partkey = p_partkey 17 | ) limit 1; 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_18.sql: -------------------------------------------------------------------------------- 1 | select 2 | c_name, 3 | c_custkey, 4 | o_orderkey, 5 | o_orderdate, 6 | o_totalprice, 7 | sum(l_quantity) 8 | from 9 | customer, 10 | orders, 11 | lineitem 12 | where 13 | o_orderkey in ( 14 | select 15 | l_orderkey 16 | from 17 | lineitem 18 | group by 19 | l_orderkey having 20 | sum(l_quantity) > 314 21 | ) 22 | and c_custkey = o_custkey 23 | and o_orderkey = l_orderkey 24 | group by 25 | c_name, 26 | c_custkey, 27 | o_orderkey, 28 | o_orderdate, 29 | o_totalprice 30 | order by 31 | o_totalprice desc, 32 | o_orderdate limit 100; 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_19.sql: -------------------------------------------------------------------------------- 1 | select 2 | sum(l_extendedprice* (1 - l_discount)) as revenue 3 | from 4 | lineitem, 5 | part 6 | where 7 | ( 8 | p_partkey = l_partkey 9 | and p_brand = 'Brand#12' 10 | and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') 11 | and l_quantity >= 10 and l_quantity <= 10 + 10 12 | and p_size between 1 and 5 13 | and l_shipmode in ('AIR', 'AIR REG') 14 | and l_shipinstruct = 'DELIVER IN PERSON' 15 | ) 16 | or 17 | ( 18 | p_partkey = l_partkey 19 | and p_brand = 'Brand#22' 20 | and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') 21 | and l_quantity >= 15 and l_quantity <= 15 + 10 22 | and p_size between 1 and 10 23 | and l_shipmode in ('AIR', 'AIR REG') 24 | and l_shipinstruct = 'DELIVER IN PERSON' 25 | ) 26 | or 27 | ( 28 | p_partkey = l_partkey 29 | and p_brand = 'Brand#43' 30 | and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') 31 | and l_quantity >= 22 and l_quantity <= 22 + 10 32 | and p_size between 1 and 15 33 | and l_shipmode in ('AIR', 'AIR REG') 34 | and l_shipinstruct = 'DELIVER IN PERSON' 35 | ) limit 1; 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_20.sql: -------------------------------------------------------------------------------- 1 | select 2 | s_name, 3 | s_address 4 | from 5 | supplier, 6 | nation 7 | where 8 | s_suppkey in ( 9 | select 10 | ps_suppkey 11 | from 12 | partsupp 13 | where 14 | ps_partkey in ( 15 | select 16 | p_partkey 17 | from 18 | part 19 | where 20 | p_name like 'floral%' 21 | ) 22 | and ps_availqty > ( 23 | select 24 | 0.5 * sum(l_quantity) 25 | from 26 | lineitem 27 | where 28 | l_partkey = ps_partkey 29 | and l_suppkey = ps_suppkey 30 | and l_shipdate >= date '1994-01-01' 31 | and l_shipdate < date '1994-01-01' + interval '1' year 32 | ) 33 | ) 34 | and s_nationkey = n_nationkey 35 | and n_name = 'INDONESIA' 36 | order by 37 | s_name limit 1; 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_21.sql: -------------------------------------------------------------------------------- 1 | select 2 | s_name, 3 | count(*) as numwait 4 | from 5 | supplier, 6 | lineitem l1, 7 | orders, 8 | nation 9 | where 10 | s_suppkey = l1.l_suppkey 11 | and o_orderkey = l1.l_orderkey 12 | and o_orderstatus = 'F' 13 | and l1.l_receiptdate > l1.l_commitdate 14 | and exists ( 15 | select 16 | * 17 | from 18 | lineitem l2 19 | where 20 | l2.l_orderkey = l1.l_orderkey 21 | and l2.l_suppkey <> l1.l_suppkey 22 | ) 23 | and not exists ( 24 | select 25 | * 26 | from 27 | lineitem l3 28 | where 29 | l3.l_orderkey = l1.l_orderkey 30 | and l3.l_suppkey <> l1.l_suppkey 31 | and l3.l_receiptdate > l3.l_commitdate 32 | ) 33 | and s_nationkey = n_nationkey 34 | and n_name = 'INDONESIA' 35 | group by 36 | s_name 37 | order by 38 | numwait desc, 39 | s_name limit 100; 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /tpch/SQLs/tpch_queries_22.sql: -------------------------------------------------------------------------------- 1 | select 2 | cntrycode, 3 | count(*) as numcust, 4 | sum(c_acctbal) as totacctbal 5 | from 6 | ( 7 | select 8 | substring(c_phone from 1 for 2) as cntrycode, 9 | c_acctbal 10 | from 11 | customer 12 | where 13 | substring(c_phone from 1 for 2) in 14 | ('19', '28', '24', '33', '25', '20', '34') 15 | and c_acctbal > ( 16 | select 17 | avg(c_acctbal) 18 | from 19 | customer 20 | where 21 | c_acctbal > 0.00 22 | and substring(c_phone from 1 for 2) in 23 | ('19', '28', '24', '33', '25', '20', '34') 24 | ) 25 | and not exists ( 26 | select 27 | * 28 | from 29 | orders 30 | where 31 | o_custkey = c_custkey 32 | ) 33 | ) as custsale 34 | group by 35 | cntrycode 36 | order by 37 | cntrycode limit 1; 38 | -------------------------------------------------------------------------------- /tpch/dbgen/makefile: -------------------------------------------------------------------------------- 1 | # 2 | # $Id: makefile.suite,v 1.25 2009/10/22 19:10:21 jms Exp $ 3 | # 4 | # Revision History 5 | # =================== 6 | # $Log: makefile.suite,v $ 7 | # Revision 1.25 2009/10/22 19:10:21 jms 8 | # update revision to 2.9.0, disable bug55 fix 9 | # 10 | # Revision 1.24 2009/10/22 19:06:10 jms 11 | # update revision to 2.9.0, disable bug55 fix 12 | # 13 | # Revision 1.23 2009/06/28 14:01:08 jms 14 | # bug fix for DOP 15 | # 16 | # Revision 1.22 2008/09/15 16:37:46 jms 17 | # release 2.8.0 makefile.suite 18 | # 19 | # Revision 1.21 2008/03/21 18:26:54 jms 20 | # recursive zip for reference data, chmod for update_release 21 | # 22 | # Revision 1.20 2008/03/21 17:38:39 jms 23 | # changes for 2.6.3 24 | # 25 | # Revision 1.19 2007/03/08 20:36:03 jms 26 | # update release number to 2.6.1 27 | # 28 | # Revision 1.18 2007/02/06 18:15:56 jms 29 | # remove update release from general target 30 | # 31 | # Revision 1.17 2007/01/25 19:35:50 jms 32 | # add sln file used by VS2005 33 | # 34 | # Revision 1.16 2007/01/05 20:05:41 jms 35 | # update release number 36 | # 37 | # Revision 1.15 2006/09/07 17:25:57 jms 38 | # correct dss.ddl 39 | # 40 | # Revision 1.14 2006/08/01 17:21:22 jms 41 | # fix bad merge 42 | # 43 | # Revision 1.13 2006/08/01 16:55:44 jms 44 | # move to 2.4.1 45 | # 46 | # Revision 1.12 2006/06/29 20:46:17 jms 47 | # 2.4.0 changes from Meikel 48 | # 49 | # Revision 1.10 2006/05/25 22:30:44 jms 50 | # qgen porting for 32b/64b 51 | # 52 | # Revision 1.9 2006/04/26 23:17:09 jms 53 | # checking release.h prior to release build 54 | # 55 | # Revision 1.8 2006/04/26 23:03:00 jms 56 | # release 2.3.4-1 57 | # 58 | # Revision 1.7 2006/04/12 18:13:58 jms 59 | # release 2.3.3 60 | # 61 | # Revision 1.6 2006/03/09 18:59:19 jms 62 | # move to version 2.3.2 63 | # 64 | # Revision 1.5 2006/01/28 23:54:32 jms 65 | # add reference data to release 66 | # 67 | # Revision 1.4 2005/10/28 03:00:32 jms 68 | # fix release target 69 | # 70 | # Revision 1.3 2005/10/28 02:54:14 jms 71 | # increment build count with each release creation 72 | # 73 | # Revision 1.2 2005/01/03 20:08:58 jms 74 | # change line terminations 75 | # 76 | # Revision 1.1.1.1 2004/11/24 23:31:47 jms 77 | # re-establish external server 78 | # 79 | # Revision 1.5 2004/03/26 20:39:23 jms 80 | # add tpch tag to release files 81 | # 82 | # Revision 1.4 2004/03/16 14:45:57 jms 83 | # correct release target in makefile 84 | # 85 | # Revision 1.3 2004/03/02 20:49:01 jms 86 | # simplify distributions, add Windows IDE files 87 | # releases should use make release from now on 88 | # 89 | # Revision 1.2 2004/02/18 14:05:53 jms 90 | # porting changes for LINUX and 64 bit RNG 91 | # 92 | # Revision 1.1.1.1 2003/04/03 18:54:21 jms 93 | # recreation after CVS crash 94 | # 95 | # Revision 1.1.1.1 2003/04/03 18:54:21 jms 96 | # initial checkin 97 | # 98 | # 99 | # 100 | ################ 101 | ## CHANGE NAME OF ANSI COMPILER HERE 102 | ################ 103 | CC = gcc 104 | # Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata) 105 | # SQLSERVER, SYBASE, ORACLE, VECTORWISE 106 | # Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, 107 | # SGI, SUN, U2200, VMS, LINUX, WIN32 108 | # Current values for WORKLOAD are: TPCH 109 | DATABASE= MYSQL 110 | MACHINE = LINUX 111 | WORKLOAD = TPCH 112 | # 113 | CFLAGS = -g -DDBNAME=\"dss\" -D$(MACHINE) -D$(DATABASE) -D$(WORKLOAD) -DRNG_TEST -D_FILE_OFFSET_BITS=64 114 | LDFLAGS = -O 115 | # The OBJ,EXE and LIB macros will need to be changed for compilation under 116 | # Windows NT 117 | OBJ = .o 118 | EXE = 119 | LIBS = -lm 120 | # 121 | # NO CHANGES SHOULD BE NECESSARY BELOW THIS LINE 122 | ############### 123 | VERSION=2 124 | RELEASE=13 125 | PATCH=0 126 | BUILD=`grep BUILD release.h | cut -f3 -d' '` 127 | NEW_BUILD=`expr ${BUILD} + 1` 128 | TREE_ROOT=/tmp/tree 129 | # 130 | PROG1 = dbgen$(EXE) 131 | PROG2 = qgen$(EXE) 132 | PROGS = $(PROG1) $(PROG2) 133 | # 134 | HDR1 = dss.h rnd.h config.h dsstypes.h shared.h bcd2.h rng64.h release.h 135 | HDR2 = tpcd.h permute.h 136 | HDR = $(HDR1) $(HDR2) 137 | # 138 | SRC1 = build.c driver.c bm_utils.c rnd.c print.c load_stub.c bcd2.c \ 139 | speed_seed.c text.c permute.c rng64.c 140 | SRC2 = qgen.c varsub.c 141 | SRC = $(SRC1) $(SRC2) 142 | # 143 | OBJ1 = build$(OBJ) driver$(OBJ) bm_utils$(OBJ) rnd$(OBJ) print$(OBJ) \ 144 | load_stub$(OBJ) bcd2$(OBJ) speed_seed$(OBJ) text$(OBJ) permute$(OBJ) \ 145 | rng64$(OBJ) 146 | OBJ2 = build$(OBJ) bm_utils$(OBJ) qgen$(OBJ) rnd$(OBJ) varsub$(OBJ) \ 147 | text$(OBJ) bcd2$(OBJ) permute$(OBJ) speed_seed$(OBJ) rng64$(OBJ) 148 | OBJS = $(OBJ1) $(OBJ2) 149 | # 150 | SETS = dists.dss 151 | DOC=README HISTORY PORTING.NOTES BUGS 152 | DDL = dss.ddl dss.ri 153 | WINDOWS_IDE = tpch.dsw dbgen.dsp tpch.sln tpch.vcproj qgen.vcproj 154 | OTHER=makefile.suite $(SETS) $(DDL) $(WINDOWS_IDE) 155 | # case is *important* in TEST_RES 156 | TEST_RES = O.res L.res c.res s.res P.res S.res n.res r.res 157 | # 158 | DBGENSRC=$(SRC1) $(HDR1) $(OTHER) $(DOC) $(SRC2) $(HDR2) $(SRC3) 159 | FQD=queries/1.sql queries/2.sql queries/3.sql queries/4.sql queries/5.sql queries/6.sql queries/7.sql \ 160 | queries/8.sql queries/9.sql queries/10.sql queries/11.sql queries/12.sql queries/13.sql \ 161 | queries/14.sql queries/15.sql queries/16.sql queries/17.sql queries/18.sql queries/19.sql queries/20.sql \ 162 | queries/21.sql queries/22.sql 163 | VARIANTS= variants/8a.sql variants/12a.sql variants/13a.sql variants/14a.sql variants/15a.sql 164 | ANS = answers/q1.out answers/q2.out answers/q3.out answers/q4.out answers/q5.out answers/q6.out answers/q7.out answers/q8.out \ 165 | answers/q9.out answers/q10.out answers/q11.out answers/q12.out answers/q13.out answers/q14.out answers/q15.out \ 166 | answers/q16.out answers/q17.out answers/q18.out answers/q19.out answers/q20.out answers/q21.out answers/q22.out 167 | QSRC = $(FQD) $(VARIANTS) $(ANS) 168 | TREE_DOC=tree.readme tree.changes appendix.readme appendix.version answers.readme queries.readme variants.readme 169 | REFERENCE=reference/[tcR]* 170 | REFERENCE_DATA=referenceData/[13]* 171 | SCRIPTS= check55.sh column_split.sh dop.sh gen_tasks.sh last_row.sh load_balance.sh new55.sh check_dirs.sh 172 | ALLSRC=$(DBGENSRC) $(REFERENCE) $(QSRC) $(SCRIPTS) 173 | JUNK = 174 | # 175 | all: $(PROGS) 176 | $(PROG1): $(OBJ1) $(SETS) 177 | $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJ1) $(LIBS) 178 | $(PROG2): permute.h $(OBJ2) 179 | $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJ2) $(LIBS) 180 | clean: 181 | rm -f $(PROGS) $(OBJS) $(JUNK) 182 | lint: 183 | lint $(CFLAGS) -u -x -wO -Ma -p $(SRC1) 184 | lint $(CFLAGS) -u -x -wO -Ma -p $(SRC2) 185 | 186 | tar: $(ALLSRC) 187 | tar cvhf - $(ALLSRC) --exclude .svn\*/\* |gzip - > tpch_${VERSION}_${RELEASE}_${PATCH}.tar.gz 188 | tar cvhf - $(REFERENCE_DATA) --exclude .svn\*/\* |gzip - > reference_${VERSION}_${RELEASE}_${PATCH}.tar.gz 189 | zip: $(ALLSRC) 190 | zip -r tpch_${VERSION}_${RELEASE}_${PATCH}.zip $(ALLSRC) -x *.svn* 191 | zip -r reference_${VERSION}_${RELEASE}_${PATCH}.zip $(REFERENCE_DATA) -x *.svn* 192 | release: 193 | make -f makefile.suite tar 194 | make -f makefile.suite zip 195 | ( cd tests; sh test_list.sh `date '+%Y%m%d'` ) 196 | rnd$(OBJ): rnd.h 197 | $(OBJ1): $(HDR1) 198 | $(OBJ2): dss.h tpcd.h config.h rng64.h release.h 199 | -------------------------------------------------------------------------------- /tpch/dbgen/tpcd.h: -------------------------------------------------------------------------------- 1 | /* 2 | * $Id: tpcd.h,v 1.2 2005/01/03 20:08:59 jms Exp $ 3 | * 4 | * Revision History 5 | * =================== 6 | * $Log: tpcd.h,v $ 7 | * Revision 1.2 2005/01/03 20:08:59 jms 8 | * change line terminations 9 | * 10 | * Revision 1.1.1.1 2004/11/24 23:31:47 jms 11 | * re-establish external server 12 | * 13 | * Revision 1.1.1.1 2003/04/03 18:54:21 jms 14 | * recreation after CVS crash 15 | * 16 | * Revision 1.1.1.1 2003/04/03 18:54:21 jms 17 | * initial checkin 18 | * 19 | * 20 | */ 21 | /***************************************************************** 22 | * Title: tpcd.h for TPC D 23 | ***************************************************************** 24 | */ 25 | #define DFLT 0x0001 26 | #define OUTPUT 0x0002 27 | #define EXPLAIN 0x0004 28 | #define DBASE 0x0008 29 | #define VERBOSE 0x0010 30 | #define TIMING 0x0020 31 | #define LOG 0x0040 32 | #define QUERY 0x0080 33 | #define REFRESH 0x0100 34 | #define ANSI 0x0200 35 | #define SEED 0x0400 36 | #define COMMENT 0x0800 37 | #define INIT 0x1000 38 | #define TERMINATE 0x2000 39 | #define DFLT_NUM 0x4000 40 | 41 | /* 42 | * general defines 43 | */ 44 | #define VTAG ':' /* flags a variable substitution */ 45 | #define ofp stdout /* make the routine a filter */ 46 | #define QDIR_TAG "DSS_QUERY" /* variable to point to queries */ 47 | #define QDIR_DFLT "." /* and its default */ 48 | 49 | /* 50 | * database portability defines 51 | */ 52 | #ifdef VECTORWISE 53 | #define GEN_QUERY_PLAN "EXPLAIN" 54 | #define START_TRAN "" 55 | #define END_TRAN "COMMIT;" 56 | #define SET_OUTPUT "" 57 | #define SET_ROWCOUNT "first %d\n" 58 | #define SET_DBASE "" 59 | #endif /* VECTORWISE */ 60 | 61 | #ifdef DB2 62 | #define GEN_QUERY_PLAN "SET CURRENT EXPLAIN SNAPSHOT ON;" 63 | #define START_TRAN "" 64 | #define END_TRAN "COMMIT WORK;" 65 | #define SET_OUTPUT "" 66 | #define SET_ROWCOUNT "--#SET ROWS_FETCH %d\n" 67 | #define SET_DBASE "CONNECT TO %s ;\n" 68 | #endif 69 | 70 | #ifdef INFORMIX 71 | #define GEN_QUERY_PLAN "SET EXPLAIN ON;" 72 | #define START_TRAN "BEGIN WORK;" 73 | #define END_TRAN "COMMIT WORK;" 74 | #define SET_OUTPUT "OUTPUT TO " 75 | #define SET_ROWCOUNT "FIRST %d" 76 | #define SET_DBASE "database %s ;\n" 77 | #endif 78 | 79 | #ifdef ORACLE 80 | #define GEN_QUERY_PLAN "" 81 | #define START_TRAN "" 82 | #define END_TRAN "" 83 | #define SET_OUTPUT "" 84 | #define SET_ROWCOUNT "where rownum <= %d;\n" 85 | #define SET_DBASE "" 86 | #endif 87 | 88 | #ifdef SQLSERVER 89 | #define GEN_QUERY_PLAN "set showplan on\nset noexec on\ngo\n" 90 | #define START_TRAN "begin transaction\ngo\n" 91 | #define END_TRAN "commit transaction\ngo\n" 92 | #define SET_OUTPUT "" 93 | #define SET_ROWCOUNT "set rowcount %d\ngo\n\n" 94 | #define SET_DBASE "use %s\ngo\n" 95 | #endif 96 | 97 | #ifdef SYBASE 98 | #define GEN_QUERY_PLAN "set showplan on\nset noexec on\ngo\n" 99 | #define START_TRAN "begin transaction\ngo\n" 100 | #define END_TRAN "commit transaction\ngo\n" 101 | #define SET_OUTPUT "" 102 | #define SET_ROWCOUNT "set rowcount %d\ngo\n\n" 103 | #define SET_DBASE "use %s\ngo\n" 104 | #endif 105 | 106 | #ifdef TDAT 107 | #define GEN_QUERY_PLAN "EXPLAIN" 108 | #define START_TRAN "BEGIN TRANSACTION" 109 | #define END_TRAN "END TRANSACTION" 110 | #define SET_OUTPUT ".SET FORMAT OFF\n.EXPORT REPORT file=" 111 | #define SET_ROWCOUNT ".SET RETCANCEL ON\n.SET RETLIMIT %d\n" 112 | #define SET_DBASE ".LOGON %s\n" 113 | #endif 114 | 115 | #define MAX_VARS 8 /* max number of host vars in any query */ 116 | #define QLEN_MAX 2048 /* max length of any query */ 117 | #define QUERIES_PER_SET 22 118 | 119 | EXTERN int flags; 120 | EXTERN int s_cnt; 121 | EXTERN char *osuff; 122 | EXTERN int stream; 123 | EXTERN char *lfile; 124 | EXTERN char *ifile; 125 | EXTERN char *tfile; 126 | 127 | #define MAX_PERMUTE 41 128 | #ifdef DECLARER 129 | int rowcnt_dflt[QUERIES_PER_SET + 1] = 130 | {-1,-1,100,10,-1,-1,-1,-1,-1,-1,20,-1,-1,-1,-1,-1,-1,-1,100,-1,-1,100,-1}; 131 | int rowcnt; 132 | #define SEQUENCE(stream, query) permutation[stream % MAX_PERMUTE][query - 1] 133 | #else 134 | extern int rowcnt_dflt[]; 135 | extern int rowcnt; 136 | #endif 137 | 138 | #ifdef MYSQL 139 | #define GEN_QUERY_PLAN "" 140 | #define START_TRAN "START TRANSACTION" 141 | #define END_TRAN "COMMIT" 142 | #define SET_OUTPUT "" 143 | #define SET_ROWCOUNT "limit %d;\n" 144 | #define SET_DBASE "use %s;\n" 145 | #endif 146 | --------------------------------------------------------------------------------