├── .gitignore ├── Analyze.py ├── Cluster_Region.py ├── Cluster_check.py ├── README.md ├── SplitTable.py ├── Stats_dump.py ├── TiDB_Kill.py ├── TiDB_Outfile.py ├── adminCheck.sh ├── analyze.sh ├── dumpling.sh ├── failoverByCDC.md ├── failoverByCDC.sh ├── out_user.py ├── report_ccsv ├── query.lst ├── report.py └── time.lst ├── requirements.txt ├── restart_tidb_tiflash.py ├── split_hot_region.py ├── split_table_region.py ├── sync_diff.py ├── sync_diff_T2T.py ├── tidb_status.py └── time_to_tso.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows: 2 | Thumbs.db 3 | ehthumbs.db 4 | Desktop.ini 5 | 6 | # python 7 | test.py 8 | test1.py 9 | test*.py 10 | 11 | # file 12 | .DS_Store 13 | *.csv 14 | *.log 15 | .vscode 16 | *.pyc 17 | 18 | # dir 19 | .idea/ 20 | ../.idea/ 21 | 22 | # sql file 23 | *.sql 24 | 25 | *.toml 26 | -------------------------------------------------------------------------------- /Analyze.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import argparse 5 | import pymysql 6 | import time 7 | 8 | # If Python is version 2.7, encoding problems can reload sys configuration 9 | try: 10 | import sys 11 | import os 12 | 13 | reload(sys) 14 | sys.setdefaultencoding('utf-8') 15 | except: 16 | pass 17 | 18 | 19 | def main(): 20 | ''' 21 | When no library name or table name is entered, Analyze operation is not performed. 22 | When you enter a library name without entering a table name, all tables in the library are analyzed. 23 | When no library name is entered, but a table name is entered, the entered table is analyzed. 24 | No other cases will be analyzed. 25 | ''' 26 | args = parse_args() 27 | table, db = args.tables, args.database 28 | healthy = args.stats_healthy 29 | if table is None and db is None: 30 | print("Please enter the relevant information for the analyze table.") 31 | elif table is None and db is not None: 32 | for _db in db.split(","): 33 | analyze_db(_db, healthy) 34 | print( 35 | "Statistics for all tables in Analyze {} library succeeded~\n".format(_db)) 36 | elif table is not None and db is None: 37 | for db_table in table.split(","): 38 | _db, _table = db_table.split(".")[0], db_table.split(".")[1] 39 | analyze_table(_db, _table, healthy) 40 | print("Success Analyze all tables") 41 | else: 42 | print("Please enter the table that requires analyze in the correct format") 43 | 44 | 45 | def parser_stats(db_name, table_name): 46 | # concat sql 47 | _sql = "show stats_healthy where Db_name in ('{}') and Table_name in ('{}');".format( 48 | db_name, table_name) 49 | result = mysql_execute(_sql)[0]["Healthy"] 50 | 51 | return result 52 | 53 | 54 | def analyze_table(db_name, table_name, healthy): 55 | # time format 56 | time_format = time.strftime("%Y-%m-%d %H:%M:%S", 57 | time.localtime()) 58 | # Analyze the table 59 | _health = parser_stats(db_name, table_name) 60 | if _health <= int(healthy): 61 | mysql_execute("use {}".format(db_name), 62 | "analyze table {}".format(table_name)) 63 | print("{} Analyze table {}.{} Sucessful".format( 64 | time_format, db_name, table_name)) 65 | else: 66 | print("{} db: {}, table: {} health: {},skip analyze".format(time_format, 67 | db_name, table_name, _health)) 68 | 69 | 70 | def analyze_db(db_name, healthy): 71 | # Analyze all tables in the database 72 | table_names = mysql_execute("use {}".format(db_name), "show tables;") 73 | for table_name in table_names: 74 | table_name = table_name["Tables_in_{}".format(db_name)] 75 | _health = parser_stats(db_name, table_name) 76 | if _health <= int(healthy): 77 | analyze_table(db_name, table_name, healthy) 78 | else: 79 | print("db: {}, table: {} health: {},skip analyze".format( 80 | db_name, table_name, _health)) 81 | 82 | 83 | def mysql_execute(*_sql): 84 | # Connect to MySQL and execute SQL commands 85 | # content = "" 86 | args = parse_args() 87 | config = { 88 | "host": args.mysql, 89 | "port": int(args.port), 90 | "user": args.user, 91 | "password": args.password, 92 | "charset": 'utf8mb4', 93 | "cursorclass": pymysql.cursors.DictCursor 94 | } 95 | 96 | connection = pymysql.connect(**config) 97 | 98 | # with connection.cursor() as cursor: 99 | cursor = connection.cursor() 100 | for sql in _sql: 101 | cursor.execute(sql) 102 | content = cursor.fetchall() 103 | connection.commit() 104 | cursor.close() 105 | connection.close() 106 | 107 | return content 108 | 109 | 110 | def parse_args(): 111 | # Incoming parameters 112 | parser = argparse.ArgumentParser( 113 | description="Update table statistics manually") 114 | parser.add_argument("-P", 115 | dest="port", 116 | help="tidb port, default: 4000", 117 | default=4000) 118 | parser.add_argument("-H", 119 | dest="mysql", 120 | help="Database address, default: 127.0.0.1", 121 | default="127.0.0.1") 122 | parser.add_argument("-u", 123 | dest="user", 124 | help="Database account, default: root", 125 | default="root") 126 | parser.add_argument("-p", 127 | dest="password", 128 | help="Database password, default: null", 129 | default="") 130 | parser.add_argument( 131 | "-d", 132 | dest="database", 133 | help="Database name, for example: test,test1, default: None", 134 | default=None) 135 | parser.add_argument( 136 | "-t", 137 | dest="tables", 138 | help="Table name (database.table), for example: test.test,test.test2, default: None", 139 | default=None) 140 | parser.add_argument("-sh", dest="stats_healthy", 141 | help="Table stats healthy, If it is below the threshold, then analyze~", default=100) 142 | 143 | args = parser.parse_args() 144 | 145 | return args 146 | 147 | 148 | if __name__ == "__main__": 149 | main() 150 | -------------------------------------------------------------------------------- /Cluster_Region.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import json 5 | import argparse 6 | import shlex 7 | import subprocess 8 | 9 | 10 | def main(): 11 | parsers = {} 12 | args = parse_args() 13 | if args.file is None: 14 | regions = parser_regions(args.pd) 15 | else: 16 | with open(args.file, 'r') as load_f: 17 | regions = json.load(load_f) 18 | 19 | parsers["total"] = regions["count"] 20 | for region in regions["regions"]: 21 | if region.get("approximate_size", None) is None or region.get( 22 | "approximate_keys", None) is None: 23 | parsers["empty"] = 1 + parsers.get("empty", 0) 24 | elif region["approximate_size"] <= int( 25 | args.size) and region["approximate_keys"] <= int(args.keys): 26 | parsers["conform"] = 1 + parsers.get("conform", 0) 27 | elif region["approximate_size"] <= int( 28 | args.size) and region["approximate_keys"] > int(args.keys): 29 | parsers["incompatible-keys"] = 1 + parsers.get("incompatible-keys", 30 | 0) 31 | elif region["approximate_size"] > int( 32 | args.size) and region["approximate_keys"] <= int(args.keys): 33 | parsers["incompatible-size"] = 1 + parsers.get("incompatible-size", 34 | 0) 35 | elif region["approximate_size"] > int( 36 | args.size) and region["approximate_keys"] > int(args.keys): 37 | parsers["incompatible"] = 1 + parsers.get("incompatible", 0) 38 | else: 39 | parsers["errors"] = 1 + parsers.get("error", 0) 40 | 41 | print( 42 | "Total: {} \nNumber of empty regions: {} \nThe regions that can be merged: {} \nSize <= {} and Keys > {}: {} \nSize > {} and Keys <= {}: {} \nSize > {} and Keys > {}: {} \nParser errors: {}".format( 43 | parsers["total"], parsers.get("empty", 0), 44 | parsers.get("conform", 0), args.size, args.keys, 45 | parsers.get("incompatible-keys", 0), args.size, args.keys, 46 | parsers.get("incompatible-size", 0), args.size, args.keys, 47 | parsers.get("incompatible", 0), parsers.get("error", 0))) 48 | 49 | 50 | def parser_regions(pd_Api): 51 | regions_cmd = "../resources/bin/pd-ctl -u http://{} -d region".format(pd_Api) 52 | regions = subprocess.check_output(shlex.split(regions_cmd)) 53 | regions = json.loads(regions) 54 | 55 | return regions 56 | 57 | 58 | def parse_args(): 59 | parser = argparse.ArgumentParser( 60 | description="Show the hot region details and splits") 61 | parser.add_argument("-pd", 62 | dest="pd", 63 | help="pd status url, default: 127.0.0.1:2379", 64 | default="127.0.0.1:2379") 65 | parser.add_argument("-s", 66 | dest="size", 67 | help="Region size(MB), default: 20", 68 | default="20") 69 | parser.add_argument("-k", 70 | dest="keys", 71 | help="Region keys, default: 200000", 72 | default="200000") 73 | parser.add_argument("-file", 74 | dest="file", 75 | help="Files to parse, default: None", 76 | default=None) 77 | args = parser.parse_args() 78 | 79 | return args 80 | 81 | 82 | if __name__ == "__main__": 83 | main() 84 | -------------------------------------------------------------------------------- /Cluster_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import json 5 | import argparse 6 | import shlex 7 | import subprocess 8 | 9 | def main(): 10 | pass 11 | 12 | 13 | 14 | def parse_args(): 15 | parser = argparse.ArgumentParser( 16 | description="Show the hot region details and splits") 17 | parser.add_argument("--th", 18 | dest="tidb", 19 | help="tidb status url, default: 127.0.0.1:10080", 20 | default="127.0.0.1:10080") 21 | parser.add_argument("--ph", 22 | dest="pd", 23 | help="pd status url, default: 127.0.0.1:2379", 24 | default="127.0.0.1:2379") 25 | parser.add_argument("top", help="the top read/write region number") 26 | args = parser.parse_args() 27 | return args 28 | 29 | 30 | if __name__ == "__main__": 31 | main() 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Script specification 2 | 3 | 4 | 5 | ## 1. split_hot_region.py 6 | - 脚本说明 7 | - 主要是为了快速打散读/写热点 8 | - `TiDB 3.0` 版本开始已经有了打散表的命令，可结合使用。 9 | 10 | - 使用说明 11 | - 需要将整个项目 `git clone` 到 `tidb-ansible/` 目录下，脚本中使用的相对路径调用 `pd-ctl` 12 | - 可以使用 `split_hot_region.py -h` 获取帮助 13 | 14 | * 注意事项 15 | + 3.0 较新版本，已经有 `INFORMATION_SCHEMA.TIDB_HOT_REGIONS` 视图可以查看集群热点信息。 16 | + 在 3.0.10 以上的版本，可以使用浏器打开：`http://{{pd_leader_ip}}:{{pd_status_port}}/dashboard/#/keyvis`, 查看热点图，更加方便直观。 17 | 18 | - 使用演示 19 | ```shell 20 | # 查看脚本说明 21 | [tidb@ip-172-16-4-51 scripts]$ ./split_hot_region.py -h 22 | usage: split.py [-h] [--th TIDB] [--ph PD] top 23 | 24 | Show the hot region details and splits 25 | 26 | positional arguments: 27 | top the top read/write region number 28 | 29 | optional arguments: 30 | -h, --help show this help message and exit 31 | --th TIDB tidb status url, default: 127.0.0.1:10080 32 | --ph PD pd status url, default: 127.0.0.1:2379 33 | 34 | # 脚本使用 35 | [tidb@ip-172-16-4-51 scripts]$ ./split_hot_region.py --th 127.0.0.1:10080 --ph 172.16.4.51:2379 1 36 | --------------------TOP 1 Read region messegs-------------------- 37 | leader and region id is [53] [27], Store id is 7 and IP is 172.16.4.58:20160, and Flow valuation is 11.0MB, DB name is mysql, table name is stats_buckets 38 | --------------------TOP 1 Write region messegs-------------------- 39 | leader and region id is [312] [309], Store id is 6 and IP is 172.16.4.54:20160, and Flow valuation is 61.0MB, DB name is mysql, table name is stats_buckets 40 | The top Read 1 Region is 27 41 | The top Write 1 Region is 309 42 | Please Enter the region you want to split(Such as 1,2,3, default is None):27,26 43 | Split Region 27 Command executed 44 | Please check the Region 26 is in Top 45 | ``` 46 | 47 | **注意:** 在脚本使用过程中，如果不进行内容输入，将会退出脚本。如果想选择性分裂 `region`，请严格按照提醒输入，如：1,2。 48 | 49 | ## 2. split_table_region.py 50 | - 脚本说明 51 | - 主要为了分裂小表 `region` 52 | - `TiDB 3.0` 版本开始已经有了打散表的命令，新版本可以忽略该脚本 53 | 54 | - 使用说明 55 | - 需要将整个项目 `git clone` 到 `tidb-ansible/` 目录下，脚本中使用的相对路径调用 `pd-ctl` 56 | - 可以使用 `split_table_region.py -h` 获取帮助 57 | 58 | - 使用演示 59 | ```shell 60 | # 查看帮助 61 | [tidb@ip-172-16-4-51 scripts]$ ./split_table_region.py -h 62 | usage: table_split.py [-h] [--th TIDB] [--ph PD] database table 63 | 64 | Show the hot region details and splits 65 | 66 | positional arguments: 67 | database database name 68 | table table name 69 | 70 | optional arguments: 71 | -h, --help show this help message and exit 72 | --th TIDB tidb status url, default: 127.0.0.1:10080 73 | --ph PD pd status url, default: 127.0.0.1:2379 74 | 75 | # 分裂小表 76 | [tidb@ip-172-16-4-51 scripts]$ ./split_table_region.py --th 127.0.0.1:10080 --ph 172.16.4.51:2379 test t2 77 | Table t2 Info: 78 | Region id is 26627, leader id is 26628, Store id is 8 and IP is 172.16.4.59:20160 79 | 80 | Table t2 Index info: 81 | Index IN_name info, id is 1: 82 | Region id is 26627 and leader id is 26628, Store id is 8 and IP is 172.16.4.59:20160 83 | Index In_age info, id is 2: 84 | Region id is 26627 and leader id is 26628, Store id is 8 and IP is 172.16.4.59:20160 85 | We will Split region: ['26627'], y/n(default is yes): y 86 | Split Region 26627 Command executed 87 | ``` 88 | 89 | ## 3. Stats_dump.py 90 | 91 | * 脚本目的 92 | + 快速拿到相关表的统计信息、表结构、版本信息、生成导入统计信息语句 93 | + 并且内部方便快速导入表结构和统计信息 94 | 95 | * 使用说明 96 | + 该脚本需要访问 `TIDB` 数据库和 `TiDB status url`，需要安装 `pymysql` 包：`sudo pip install pymysql` 97 | + 可以使用 `Stats_dump.py -h` 获取帮助 98 | + 最终会生成一个 `tar` 包，解压后，里面有一个 `schema.sql` 文件，里面有 TiDB 的集群信息。 99 | + 还原统计信息和表结构可以: `mysql -uroot -P4000 -h127.0.0.1 200000: 0 163 | Size > 20 and Keys <= 200000: 0 164 | Size > 20 and Keys > 200000: 0 165 | Parser errors: 0 166 | 167 | [tidb@xiaohou-vm1 scripts]$ python2 Cluster_Region.py -file region.json 168 | Total: 33 169 | Number of empty regions: 17 170 | The regions that can be merged: 16 171 | Size <= 20 and Keys > 200000: 0 172 | Size > 20 and Keys <= 200000: 0 173 | Size > 20 and Keys > 200000: 0 174 | Parser errors: 0 175 | ``` 176 | 177 | * 参数说明 178 | + `-pd` 后填 PD 的 IP 地址和 status 端口，端口默认是 2379 179 | + `-s` 为 region merge 配置 `max-merge-region-size` 大小 180 | + `-k` 为 region merge 配置 `max-merge-region-keys` 大小 181 | + `-file` 用来指定需要解析的 region 信息文件，优先级高于 `-pd` 配置，如果配置，将优先分析 file 文件内容，不访问 pd 182 | 183 | * 注意 184 | + `Number of empty regions` 代表空 region(比如 drop 或者 truncate 之后存留 region，如果很多，则需要开启跨表 region merge，请联系官方)。 185 | + 如果 `The regions that can be merged` 有值，代表着符合 merge 条件，一般是不同表的 region(默认是不会合并)，或者是还没有来得及合并。 186 | + 如果 `Size <= 20 and Keys > 200000` 或者 `Size > 20 and Keys <= 200000` 有值，代表 region merge 配置的 `max-merge-region-keys` 或者 `max-merge-region-size` 配置小了，可以考虑调整。 187 | + `Size > 20 and Keys > 200000` 代表 region merge 条件都不符合 188 | + `Parser errors` 代表解析异常，请联系官方。 189 | + 如果服务器上有 `jq` 命令，也可以使用 `jq` 解析：```./bin/pd-ctl -d region | jq ".regions | map(select(.approximate_size < 20 and .approximate_keys < 200000)) | length"``` 190 | 191 | 192 | ## 5. Outfile_TiDB.py 193 | 194 | * 脚本目的 195 | + 方便导出 TIDB 数据为 CSV 格式 196 | 197 | * 使用说明 198 | + 需要安装 `MySQLdb`: ` sudo yum -y install mysql-devel;pip install mysql` 199 | + 可以使用 `Outfile_TiDB.py -h` 获取帮助 200 | 201 | * 使用演示 202 | 203 | ```shell 204 | [tidb@xiaohou-vm1 scripts]$ ./Outfile_TiDB.py -h 205 | usage: outfile.py [-h] [-tp MYSQL] [-u USER] [-p PASSWORD] [-d DATABASE] 206 | [-t TABLE] [-k FIELD] [-T THREAD] [-B BATCH] [-w WHERE] 207 | [-c COLUMN] 208 | 209 | Export data to CSV 210 | 211 | optional arguments: 212 | -h, --help show this help message and exit 213 | -tp MYSQL TiDB Port, default: 127.0.0.1:4000 214 | -u USER TiDB User, default: root 215 | -p PASSWORD TiDB Password, default: null 216 | -d DATABASE database name, default: test 217 | -t TABLE Table name, default: test 218 | -k FIELD Table primary key, default: _tidb_rowid 219 | -T THREAD Export thread, default: 20 220 | -B BATCH Export batch size, default: 3000 221 | -w WHERE Filter condition， for example: where id >= 1, default: null 222 | -c COLUMN Table Column, for example: id,name, default: all 223 | 224 | [tidb@xiaohou-vm1 scripts]$ python Outfile_TiDB.py 225 | ... 226 | Exiting Main Thread, Total cost time is 17.1548991203 227 | 228 | [tidb@xiaohou-vm1 scripts]$ cat test.test.0.csv 229 | "id","name" 230 | 1,"aa" 231 | 2,"bb" 232 | 3,"" 233 | 234 | [tidb@xiaohou-vm1 scripts]$ ./Outfile_TiDB.py -c 'id' 235 | Write test.test.0.csv is Successful, Cost time is 0.000797271728515625 236 | Retrieved select id from test where _tidb_rowid >= 1 and _tidb_rowid < 100001 237 | Exiting Main Thread 238 | 239 | [tidb@xiaohou-vm1 scripts]$ cat test.test.0.csv 240 | "id" 241 | 1 242 | 2 243 | 3 244 | ``` 245 | 246 | * 参数说明 247 | + `-tp` 后填 TIDB 的 IP 地址和 `status` 端口，端口默认是 `10080` 248 | + `-u` 为 TIDB 的用户，默认为 `root` 249 | + `-p` 为 TiDB 的密码，默认为空 250 | + `-d` 指定库名，默认为 `test` 251 | + `-t` 指定表名，默认为 `test` 252 | + `-c` 指定表字段，默认为 `all`，导出所有 253 | + `-k` 指定主键名，默认使用 `_tidb_rowid` 254 | + `-T` 指定并发数，默认使用 20 255 | + `-B` 指定每次批量导出数据大小，默认使用 3000 256 | + `-w` 可加入判断条件，比如 `where id >= 1`，默认为空。 257 | + `-c` 可添加导出数据的的列 258 | 259 | * 注意 260 | + 多少并发就会生成多少文件，如果数据量很少，`-B` 较大，只有一个文件是正常的 261 | + 当前只支持单表导出。 262 | 263 | # 6. analyze 表相关脚本 264 | 265 | * 脚本目的 266 | + 当我们全量导入一次数据后，表统计信息可能不会非常准确，这个时候最好进行一次全量的 analyze。 267 | 268 | * 为了避免安装过多插件，可以使用 shell 脚本 `analyze.sh` 脚本进行统计信息收集，该脚本会对非 'METRICS_SCHEMA','PERFORMANCE_SCHEMA','INFORMATION_SCHEMA','mysql' 库中的所有表进行 analyze。 269 | 270 | * 相关配置说明 271 | 272 | | 相关参数 | 说明 | 273 | | ----------- | ---------------------------------------- | 274 | | db_user | 数据库登录用户名，默认 root | 275 | | db_port | 数据库登录端口，默认 4000 | 276 | | db_password | 数据库登录密码，默认 123，注意不能为空。 | 277 | | db_ip | 数据库登录 IP，默认为 "127.0.0.1" | 278 | | mysql_path | MySQL 命令的绝对路径 | 279 | 280 | * 使用演示 281 | 282 | ```shell 283 | nohup ./analyze.sh >>& analyze.log & 284 | cat analyze.log 285 | nohup: ignoring input 286 | /usr/local/mysql/bin/mysql is exist~ 287 | mysql: [Warning] Using a password on the command line interface can be insecure. 288 | mysql: [Warning] Using a password on the command line interface can be insecure. 289 | mysql: [Warning] Using a password on the command line interface can be insecure. 290 | Analyze table jh_test.t Sucess~ 291 | mysql: [Warning] Using a password on the command line interface can be insecure. 292 | Analyze table jh_test.t1 Sucess~ 293 | ``` 294 | 295 | **如果习惯使用 Python 脚本，可以使用 Analyze.py 脚本** 296 | 297 | * Analyze.py 脚本和 Analyze.sh 脚本目的一样，功能会更加丰富一点，可以指定表进行 Analyze。 298 | 299 | * 使用说明 300 | * 需要安装 `pymysql` 的包：`sudo pip install pymysql` 301 | 302 | * 参数说明： 303 | 304 | | 参数 | 说明 | 305 | | ---- | ------------------------------------------------------- | 306 | | -h | 显示脚本使用方式 | 307 | | -P | 数据库连接端口，默认 4000 | 308 | | -p | 数据库密码，默认为空 | 309 | | -H | 数据库连接 IP，默认为 127.0.0.1 | 310 | | -u | 数据库连接账户，默认为 root | 311 | | -d | 需要收集统计信息的库，多个使用逗号隔开 | 312 | | -t | 需要收集统计信息的表，多个使用逗号隔开 | 313 | | -sh | health 判断，如果健康度较高，则跳过检查。默认阈值为 100 | 314 | 315 | * 使用演示 316 | 317 | ```shell 318 | # 可以使用 -h 进行帮助查看 319 | $ ./Analyze.py -h 320 | usage: Analyze.py [-h] [-P PORT] [-H MYSQL] [-u USER] [-p PASSWORD] 321 | [-d DATABASE] [-t TABLES] [-sh STATS_HEALTHY] 322 | 323 | Update table statistics manually 324 | 325 | optional arguments: 326 | -h, --help show this help message and exit 327 | -P PORT tidb port, default: 4000 328 | -H MYSQL Database address, default: 127.0.0.1 329 | -u USER Database account, default: root 330 | -p PASSWORD Database password, default: null 331 | -d DATABASE Database name, for example: test,test1, default: None 332 | -t TABLES Table name (database.table), for example: 333 | test.test,test.test2, default: None 334 | -sh STATS_HEALTHY Table stats healthy, If it is below the threshold, then 335 | analyze~ 336 | 337 | # 更新 test、sbtest 两个库中所有表的统计信息 338 | $ ./Analyze.py -d test,sbtest -p123456 339 | 2021-12-24 13:56:48 Analyze table test.t Sucessful 340 | 2021-12-24 13:56:48 Analyze table test.t1 Sucessful 341 | 2021-12-24 13:56:48 Analyze table test.t2 Sucessful 342 | Statistics for all tables in Analyze test library succeeded~ 343 | 344 | 2021-12-24 13:56:48 Analyze table sbtest.sbtest1 Sucessful 345 | 2021-12-24 13:56:48 Analyze table sbtest.sbtest2 Sucessful 346 | 2021-12-24 13:56:48 Analyze table sbtest.sbtest3 Sucessful 347 | 2021-12-24 13:56:48 Analyze table sbtest.t4 Sucessful 348 | Statistics for all tables in Analyze sbtest library succeeded~ 349 | 350 | # 更新 test.t1、sbtest.sbtest1 两张表的统计信息 351 | $ ./Analyze.py -t test.t1,sbtest.sbtest1 -p123456 352 | 2021-12-24 13:56:48 Analyze table test.t1 Sucessful 353 | Success Analyze all tables 354 | 2021-12-24 13:56:48 Analyze table sbtest.sbtest1 Sucessful 355 | Success Analyze all tables 356 | 357 | # 调整阈值 358 | ./Analyze.py -t test.t1,sbtest.sbtest1 -p123456 -sh 20 359 | 2021-12-24 13:56:48 db: test, table: t1 health: 100,skip analyze 360 | Success Analyze all tables 361 | 2021-12-24 13:56:48 db: sbtest, table: sbtest1 health: 100,skip analyze 362 | Success Analyze all tables 363 | ``` 364 | 365 | # 7. 导出数据库账号密码相关脚本 366 | 367 | * 脚本目的 368 | + 一次性备份 TiDB/MySQL 所有账户、密码权限 369 | 370 | * 使用说明 371 | * 需要安装 `pymysql` 的包：`sudo pip install pymysql` 372 | 373 | * 参数说明： 374 | 375 | | 参数 | 说明 | 376 | | ---- | -------------------------------------- | 377 | | -h | 显示脚本使用方式 | 378 | | -P | 数据库连接端口，默认 4000 | 379 | | -p | 数据库密码，默认为空 | 380 | | -H | 数据库连接 IP，默认为 127.0.0.1 | 381 | | -u | 数据库连接账户，默认为 root | 382 | | -d | 需要收集统计信息的库，多个使用逗号隔开 | 383 | | -t | 需要收集统计信息的表，多个使用逗号隔开 | 384 | 385 | * 使用演示 386 | 387 | ```shell 388 | ./out_user.py -h 389 | usage: out_user.py [-h] [-P PORT] [-H MYSQL] [-u USER] [-p PASSWORD] 390 | 391 | Update table statistics manually 392 | 393 | optional arguments: 394 | -h, --help show this help message and exit 395 | -P PORT tidb port, default: 4000 396 | -H MYSQL Database address, default: 127.0.0.1 397 | -u USER Database account, default: root 398 | -p PASSWORD Database password, default: null 399 | 400 | # 导出所有账户密码： 401 | ./out_user.py 402 | User: 'root'@'%' is OK 403 | User: 'tidb'@'%' is OK 404 | User: 'tidb1'@'%' is OK 405 | 406 | # 结果： 407 | cat tidb_users.sql 408 | -- 'root'@'%' 409 | create user 'root'@'%'; 410 | update mysql.user set `authentication_string`='' where user='root' and host='%'; 411 | GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; 412 | 413 | -- 'tidb'@'%' 414 | create user 'tidb'@'%'; 415 | update mysql.user set `authentication_string`='' where user='tidb' and host='%'; 416 | GRANT USAGE ON *.* TO 'tidb'@'%'; 417 | 418 | -- 'tidb1'@'%' 419 | create user 'tidb1'@'%'; 420 | update mysql.user set `authentication_string`='*445AD2DF92D86C174C647766EB0146B6E70341CB' where user='tidb1' and host='%'; 421 | GRANT USAGE ON *.* TO 'tidb1'@'%'; 422 | 423 | ``` 424 | 425 | # 8. 连接指定 tidb 节点执行 kill session 操作 426 | 427 | * 脚本目的 428 | + 由于 TiDB 有多个 TiDB-server 节点，在某些情况下，会有只能通过负载均衡的方式连接数据库的情况。 429 | + 有些操作，需要登录对应服务器进行执行，比如发现某个节点有一个大 SQL，这个时候就需要登录对应节点，使用 `kill tidb session_id;` 来 kill 对应 SQL。 430 | + 本脚本就是为了解决相关情况下，可以快速链接对应节点来执行 kill 命令。 431 | 432 | * 使用说明 433 | * 需要安装 `pymysql` 的包：`sudo pip install pymysql` 434 | * 默认最多重试 100 次，如果 100 次尝试连接都没有连接到对应节点，则自动退出。 435 | 436 | * 参数说明： 437 | 438 | | 参数 | 说明 | 439 | | ---- | -------------------------------------------------------- | 440 | | -h | 显示脚本使用方式 | 441 | | -P | 数据库连接端口，默认 4000 | 442 | | -p | 数据库密码，默认为空 | 443 | | -H | 数据库连接 IP，默认为 127.0.0.1 | 444 | | -u | 数据库连接账户，默认为 root | 445 | | -a | 需要执行 kill 命令的 TiDB server 节点 IP，默认为空 | 446 | | -ap | 需要执行 kill 命令的 TiDB server 节点的端口，默认为 4000 | 447 | | -id | 需要 kill 的 session id | 448 | 449 | * 使用示例 450 | 451 | ```shell 452 | # help 命令 453 | ./TiDB_Kill.py -h 454 | usage: TiDB_Kill.py [-h] [-P PORT] [-H MYSQL] [-u USER] [-p PASSWORD] 455 | [-a ADVERTISEADDRESS] [-ap ADVERTISEPORT] [-id SESSIONID] 456 | 457 | Kill TiDB session by id 458 | 459 | optional arguments: 460 | -h, --help show this help message and exit 461 | -P PORT tidb port, default: 4000 462 | -H MYSQL Database address, default: 127.0.0.1 463 | -u USER Database account, default: root 464 | -p PASSWORD Database password, default: null 465 | -a ADVERTISEADDRESS TiDB Address, default: null 466 | -ap ADVERTISEPORT TiDB Port, default: 4000 467 | -id SESSIONID session id, default: null 468 | 469 | # 使用 470 | ./TiDB_Kill.py -p 123456 -a 172.16.5.120 -ap 4000 -id 1 471 | 172.16.5.120 4000 472 | The TiDB IP is 172.16.5.120 473 | The TiDB IP Port is 4000 474 | Will execute: kill tidb 1, y/n (default:yes) 475 | Connection retries 1 476 | ``` 477 | 478 | # 9. 数据库同步，数据对比 479 | 480 | * 脚本目的 481 | + 当 TiDB 集群做了主从同步，有时候需要对上下游数据做数据对比 482 | 483 | * 使用说明 484 | * 需要安装 `pymysql` 的包：`sudo pip install pymysql` 485 | * 默认对比内容 486 | 487 | * 参数说明： 488 | 489 | | 参数
| 说明 | 490 | | :---------------------------: | -------------------------------------------------------------------------------------------------------------------------------- | 491 | | -h | 显示脚本使用方式 | 492 | | -hf | 上游数据库 IP 和 端口，默认 127.0.0.1:4000 | 493 | | -uf | 上游数据库密码，默认为 root | 494 | | -pf | 上游数据库密码，默认为空 | 495 | | -ht | 下游数据库 IP 和 端口，默认 127.0.0.1:4000 | 496 | | -ut | 下游数据库密码，默认为 root | 497 | | -pt | 下游数据库密码，默认为空 | 498 | | -d | 需要对比的库，逗号隔开，比如 tmp,test | 499 | | -t | 需要对比的表，比如 tmp.t1,tmp.t2，当前未开发 | 500 | | -T | 数据对比并行度，默认 200，约小越慢，约大，TiKV CPU 使用率越高 | 501 | | -m | 同步类型，比如，tidb,tidb: 第一个是上游数据库类型，逗号后是下游数据库类型。tidb,tidb 可以直接对比，有一个为 mysql 的，需要无 udi | 502 | | -v | 对比算法，默认为 xor，对比的是数据内容，可以配置 count，对比的是 kv 数。只能用 `-m tidb,tidb` 模式下 | 503 | 504 | * 使用示例 505 | 506 | ```shell 507 | # help 命令 508 | ./sync_diff.py -h 509 | usage: sync_diff.py [-h] [-hf FMYSQL] [-uf FUSER] [-pf FPASSWORD] [-ht TMYSQL] 510 | [-ut TUSER] [-pt TPASSWORD] [-d DATABASE] [-t TABLES] 511 | [-T THREAD] [-m MODE] [-v VERIFICATION] 512 | 513 | Check tables 514 | 515 | optional arguments: 516 | -h, --help show this help message and exit 517 | -hf FMYSQL Source database address and port, default: 127.0.0.1:4000 518 | -uf FUSER Source database account, default: root 519 | -pf FPASSWORD Source database password, default: null 520 | -ht TMYSQL Target database address and port, default: 127.0.0.1:4000 521 | -ut TUSER Target database account, default: root 522 | -pt TPASSWORD Target database password, default: null 523 | -d DATABASE Database name, for example: test,tmp, default: None 524 | -t TABLES Table name, for example: tmp.t,tmp.t1, default: None 525 | -T THREAD set tidb_distsql_scan_concurrency, for example: 200, 526 | default: 200 527 | -m MODE Compare database types, for example: tidb,mysql, default: 528 | tidb,tidb 529 | -v VERIFICATION Verification method, for example: checksum, default: xor 530 | 531 | # 使用 532 | ./sync_diff.py -d tmp,tmp1 533 | Check sucessfull, Cost time is 0.014266729354858398s, DB name is: tmp, Table name is:t, bit xor:2175547901 534 | Check sucessfull, Cost time is 0.012514114379882812s, DB name is: tmp, Table name is:t1, bit xor:2393996321 535 | Check sucessfull, Cost time is 0.012146711349487305s, DB name is: tmp, Table name is:t2, bit xor:1665511314 536 | Check sucessfull, Cost time is 0.012310504913330078s, DB name is: tmp, Table name is:t_json, bit xor:0 537 | Check sucessfull, Cost time is 0.01268911361694336s, DB name is: tmp, Table name is:order, bit xor:0 538 | Check sucessfull, Cost time is 0.013577938079833984s, DB name is: tmp1, Table name is:test, bit xor:0 539 | ``` 540 | 541 | # 10. 备份相关脚本 542 | 543 | * 脚本目的 544 | + dumpling 备份脚本 545 | 546 | * 为了避免安装过多插件，可以使用 shell 脚本 `dumpling.sh` 脚本进行统计信息收集，该脚本会对非 'METRICS_SCHEMA','PERFORMANCE_SCHEMA','INFORMATION_SCHEMA','mysql' 库中的所有表进行备份 547 | 548 | * 相关配置说明 549 | 550 | | 相关参数 | 说明 | 551 | | ---------- | ---------------------------------------- | 552 | | user | 数据库登录用户名，默认 root | 553 | | port | 数据库登录端口，默认 4000 | 554 | | password | 数据库登录密码，默认 123，注意不能为空。 | 555 | | host | 数据库登录 IP，默认为 "127.0.0.1" | 556 | | filetype | 备份出类型，可以是 sql 或者 csv | 557 | | thread | 线程 | 558 | | backupDir | 备份路径 | 559 | | oldDir | 需要删除的备份路径 | 560 | | logFile | 日志路径 | 561 | | binaryPath | dumpling 包路径 | 562 | 563 | * 使用演示 564 | 565 | ```shell 566 | ➜ PingCAP git:(master) ✗ cat nohup.log 567 | [1] + 21919 done nohup ./dumpling.sh &> nohup.log 568 | ➜ PingCAP git:(master) ✗ cat nohup.log 569 | nohup: ignoring input 570 | /home/db/tidbmgt/tools/dumpling is exist~ 571 | /tidbbackup/fmtfx_back/ exist~ 572 | dumpling start~, backup dir is [ /tidbbackup/fmtfx_back/20211008 ] 573 | You can execute the [ tail -f /tidbbackup/fmtfx_back/main.log ] command to view the progress 574 | ``` 575 | 576 | # time_to_tso.py 577 | 578 | * 脚本目的 579 | * 将时间转为 TSO 580 | 581 | * 使用演示 582 | 583 | ```shell 584 | ./time_to_tso.py '2021-12-13 18:14:18.123' 585 | TSO: 42975637225419571 586 | 587 | # 使用 tidb 解析 tso 588 | (root@127.0.0.1) [(none)]>select tidb_parse_tso(429756372254195712); 589 | +------------------------------------+ 590 | | tidb_parse_tso(429756372254195712) | 591 | +------------------------------------+ 592 | | 2021-12-13 18:14:18.123000 | 593 | +------------------------------------+ 594 | 1 row in set (0.00 sec) 595 | ``` 596 | 597 | **注意事项：** 598 | + 1. TSO = 物理时间 + 逻辑时间 599 | + 2. 当前代码中逻辑时间使用 18 位 0 代替 600 | + 3. 所以生成的和原始的不一样的 601 | 602 | # tidb_status.py 603 | 604 | * 脚本目的 605 | 606 | 定时检查 tidb-server 10080 端口 status api 状态，默认如果进程存在，但是 status api 无法监测到，就默认将 4000 端口添加防火墙。 607 | 608 | 在 kill -19 挂起进程下，端口存活但是进程无响应，模拟假死状况。但是这种情况下，应用或者负载均衡很难判断出异常，因此写这个守护脚本进行监测。 609 | 610 | * 使用演示 611 | 612 | ```shell 613 | ✗ python2 tidb_status.py -h 614 | usage: tidb_status.py [-h] [-P PORT] [-H HOST] [-s STATUS] [-n NUMBER] 615 | [-t STIME] 616 | 617 | Check tidb status 618 | 619 | optional arguments: 620 | -h, --help show this help message and exit 621 | -P PORT tidb port, default: 4000 622 | -H HOST Database address, default: 127.0.0.1 623 | -s STATUS TiDB status port, default: 10080 624 | -n NUMBER try number, default: 3 625 | -t STIME sleep time, default: 3 626 | 627 | # 使用演示 628 | ✗ python2 tidb_status.py 629 | 2022-05-19 15:07:44 630 | 数据库状态正常 631 | 端口 4000 已经开启 632 | 2022-05-19 15:07:47 633 | 数据库状态正常 634 | 端口 4000 已经开启 635 | ... 636 | ``` 637 | 638 | 639 | # restart_tidb_tiflash.py 脚本 640 | 641 | 说明：脚本目的是访问 promethues 执行 query，查询 tiflash proxy 状态，如果 proxy 有重启，对应公式可以根据进程启动时间进行判断，默认如果 5min 内重启成功，就会捕获。 642 | 进行相应 tidb-server 重启，重启后，脚本默认会等待 5min 后再进行监控。 643 | 644 | ## 使用方法 645 | ```shell 646 | # 查看帮助 647 | $ ./restart_tidb_tiflash.py -h 648 | usage: restart_tidb_tiflash.py [-h] [-P TIDBPORT] [-ph PROMADDR] 649 | [-th FLASHADDR] 650 | 651 | 如果 tiflash 重启，重启本地 tidb-server 652 | 653 | optional arguments: 654 | -h, --help show this help message and exit 655 | -P TIDBPORT tidb port, default: 4000 656 | -ph PROMADDR Prometheus ip 和端口, default: 127.0.0.1:9090 657 | -th FLASHADDR tiflash ip 和 proxy status 端口, default: 127.0.0.1:20292 658 | 659 | ``` 660 | 661 | ### 使用案例 662 | ```shell 663 | # 启动监控脚本 664 | $ ./restart_tidb_tiflash.py -ph "172.16.201.210:9291" -th "172.16.201.210:22293" -P 4201 665 | 666 | # 查看日志 667 | $ tail -f logs/2022-08-02-status.log 668 | 2022-08-02 14:41:44: tiflash 状态正常：172.16.201.210:9291 0 669 | 2022-08-02 14:41:51: tiflash 状态正常：172.16.201.210:9291 0 670 | 671 | # 重启 tiflash 672 | $ tiup cluster restart wangjun-tidb -R tiflash -y; date 673 | ... 674 | Restarted cluster `wangjun-tidb` successfully 675 | Tue Aug 2 14:42:19 CST 2022 676 | 677 | # 查看监控脚本日志 678 | $ tail -f logs/2022-08-02-status.log 679 | 2022-08-02 14:42:12: tiflash 状态正常：172.16.201.210:9291 0 680 | 2022-08-02 14:42:19: tiflash 可能发生重启：172.16.201.210:9291 1，开始多次判断：第 1 次 681 | 2022-08-02 14:42:20: tiflash 可能发生重启：172.16.201.210:9291 1，开始多次判断：第 2 次 682 | 2022-08-02 14:42:21: tiflash 可能发生重启：172.16.201.210:9291 1，开始多次判断：第 3 次 683 | 2022-08-02 14:42:22: tiflash 是否重启，已经连续判断 3 次，进行 tidb-server 重启 684 | 2022-08-02 14:42:25: 重启 tidb 成功：sudo systemctl restart tidb-4201.service 685 | 686 | # 查看重启进程 687 | $ ps aux | grep -E "tiflash|tidb-serv" 688 | tidb 873 14.4 6.9 3570304 1131916 ? Ssl 14:42 0:09 bin/tiflash/tiflash server --config-file conf/tiflash.toml 689 | tidb 1370 1.4 0.4 1254884 65488 ? Ssl 14:42 0:00 bin/tidb-server -P 4201 --status=12081 --host=0.0.0.0 --advertise-address=172.16.201.210 --store=tikv --initialize-insecure --path=172.16.201.210:2579 --log-slow-query=/data/wangjun-tidb/tidb-deploy/tidb-4201/log/tidb_slow_query.log --config=conf/tidb.toml --log-file=/data/wangjun-tidb/tidb-deploy/tidb-4201/log/tidb.log 690 | wangjun 2263 0.0 0.0 112708 984 pts/2 S+ 14:43 0:00 grep --color=auto -E tiflash|tidb-serv 691 | wangjun 32611 0.1 0.1 220892 18088 pts/1 S+ 14:41 0:00 python ./restart_tidb_tiflash.py -ph 172.16.201.210:9291 -th 172.16.201.210:22293 -P 4201 692 | ``` 693 | 694 | # SplitTable.py 脚本 695 | 696 | 说明：脚本目的是获取当前表的数据分布，然后生成对应的 split 语法。 697 | 且可以指定语法生成的库/表名 698 | 699 | 使用场景：可以在跑批或者 insert select 下，模仿原表或者原目标表的数据分布，进行 split 700 | 701 | ## 使用方法 702 | ```shell 703 | usage: SplitTable.py [-h] [-P PORT] [-H MYSQL] [-u USER] [-p PASSWORD] 704 | [-t TABLES] [-tt TOTABLES] 705 | 706 | Update table statistics manually 707 | 708 | optional arguments: 709 | -h, --help show this help message and exit 710 | -P PORT tidb port, default: 4000 711 | -H MYSQL Database address, default: 127.0.0.1 712 | -u USER Database account, default: root 713 | -p PASSWORD Database password, default: null 714 | -t TABLES Table name (database.table), for example: test.test default: 715 | None 716 | -tt TOTABLES Table name (database.table), for example: test.test1 default: 717 | None 718 | 719 | ``` 720 | 721 | * 参数： 722 | - `-t`: 解析的表 723 | - `-tt`: 生产 split 的目标表，默认不添加会使用 `-t` 参数的表名 724 | 725 | ### 使用案例 726 | ```shell 727 | ./SplitTable.py -P 4201 -p tidb@123 -t test.sbt3 728 | split table `test`.`sbt3` index `k_3` by ('15082931275-71506150285-51909720602-81342784324-01114645089-73049053419-98304802326-28965349754-62974367746-84533806438'),xxxx; 729 | split table `test`.`sbt3` by (2259465),xxx; 730 | split table `test`.`sbt3` index `k_2` by ('499952'),('499952'); 731 | split table `test`.`sbt3` index `PRIMARY` by ('780336', '58085146578-56026749860-47299980660-03615766659-76405300524-63386680270-70730127379-92873626833-76132976414-20189494100'),xxxx; 732 | 733 | ``` 734 | 735 | 736 | # TiDB 数据库表差异检查 737 | 738 | ## 简介 739 | 740 | TiDB 数据库表差异检查是一个用于比较两个 TiDB 数据库之间的表数据差异的脚本。它可以用于检查源数据库（通常是主数据库）与目标数据库（通常是从数据库）之间的数据是否一致。该工具适用于检查数据的一致性，以确保在数据同步和备份过程中没有发生问题。 741 | 742 | ## 使用方法 743 | 744 | ### 1. 安装依赖 745 | 746 | 确保已经安装了以下依赖项： 747 | 748 | - Python 3.x 749 | - MySQL 客户端 750 | - TiDB 数据库 751 | - 安装 `PyMySQL==1.0.2` 752 | 753 | 754 | ### 2. 运行脚本 755 | 756 | 进入克隆的仓库目录，并运行脚本： 757 | 758 | ```bash 759 | python sync_diff_T2T.py -mh -mu -mp -sh -su -sp -b -dl "db1,db2,db3" 760 | ``` 761 | 762 | 参数说明： 763 | 764 | - `-mh`: 主数据库的地址和端口，默认为 `127.0.0.1:4000` 765 | - `-mu`: 主数据库的用户名，默认为 `root` 766 | - `-mp`: 主数据库的密码，默认为空 767 | - `-sh`: 从数据库的地址和端口，默认为 `127.0.0.1:4000` 768 | - `-su`: 从数据库的用户名，默认为 `root` 769 | - `-sp`: 从数据库的密码，默认为 `tidb@123` 770 | - `-b`: `sync_diff_inspector` 的二进制文件路径，默认为当前目录 `./` 771 | - `-dl`: 需要对比的数据库列表，默认为空，可配置为："db1,db2,db3" 772 | - `-t`: 对比并行，默认为 16 773 | - `-g`: 修改 tidb gc 时间，默认为 24h，上下游都修改。脚本正确执行完成后会修改会以前的 gc 时间。 774 | 775 | ### 3. 查看结果 776 | 777 | 脚本将生成一个配置文件 `config.toml` 和一个输出目录，包含检查结果。你可以查看 `sync-diff.log` 文件以获取详细的比较结果。 778 | -------------------------------------------------------------------------------- /SplitTable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import argparse 5 | import pymysql 6 | 7 | # If Python is version 2.7, encoding problems can reload sys configuration 8 | try: 9 | import sys 10 | import os 11 | 12 | reload(sys) 13 | sys.setdefaultencoding('utf-8') 14 | except: 15 | pass 16 | 17 | 18 | class SplitTable(): 19 | 20 | def __init__(self): 21 | args = self.parse_args() 22 | # db config 23 | self.config = { 24 | "host": args.mysql, 25 | "port": int(args.port), 26 | "user": args.user, 27 | "password": args.password, 28 | "charset": 'utf8mb4', 29 | "cursorclass": pymysql.cursors.DictCursor 30 | } 31 | 32 | def mysqlconnect(self, *_sql): 33 | connection = pymysql.connect(**self.config) 34 | # with connection.cursor() as cursor: 35 | cursor = connection.cursor() 36 | for sql in _sql: 37 | cursor.execute(sql) 38 | content = cursor.fetchall() 39 | connection.commit() 40 | cursor.close() 41 | connection.close() 42 | 43 | return content 44 | 45 | def keys(self, db, table): 46 | # sql 47 | sql = ''' 48 | select 49 | INDEX_ID, 50 | INDEX_NAME, 51 | tidb_decode_key(start_key) start, 52 | tidb_decode_key(END_KEY) end 53 | from 54 | information_schema.tikv_region_status 55 | where 56 | DB_NAME = '{}' 57 | and TABLE_NAME = '{}' 58 | order by 59 | INDEX_ID; 60 | '''.format(db, table) 61 | keys = self.mysqlconnect(sql) 62 | 63 | return keys 64 | 65 | def parserkeys(self, ct): 66 | try: 67 | sct = eval(ct['start']) 68 | except: 69 | sct = {} 70 | if sct.has_key('_tidb_rowid'): 71 | sctv = [] 72 | sctv.append(sct['_tidb_rowid']) 73 | sctv = tuple(sctv) 74 | elif sct.has_key('handle') and ct['INDEX_NAME'] is None: 75 | sctv = sct['handle'].values() 76 | sctv = tuple(sctv[::-1]) 77 | elif sct.has_key('index_vals') and ct['INDEX_ID'] == sct['index_id']: 78 | sctv = sct['index_vals'].values() 79 | sctv = tuple(sctv[::-1]) 80 | else: 81 | sctv = None 82 | 83 | try: 84 | ect = eval(ct['end']) 85 | except: 86 | ect = {} 87 | if ect.has_key('_tidb_rowid'): 88 | ectv = [] 89 | ectv.append(ect['_tidb_rowid']) 90 | ectv = tuple(ectv) 91 | elif ect.has_key('handle') and ct['INDEX_NAME'] is None: 92 | ectv = ect['handle'].values() 93 | ectv = tuple(ectv[::-1]) 94 | elif ect.has_key('index_vals') and ct['INDEX_ID'] == ect['index_id']: 95 | ectv = ect['index_vals'].values() 96 | ectv = tuple(ectv[::-1]) 97 | else: 98 | ectv = None 99 | 100 | if ectv is None and sctv is None: 101 | allct = '' 102 | elif ectv is None: 103 | if len(sctv) == 1: 104 | sctv = str(sctv).replace(',', '') 105 | allct = str(sctv) 106 | elif sctv is None: 107 | if len(ectv) == 1: 108 | ectv = str(ectv).replace(',', '') 109 | allct = str(ectv) 110 | else: 111 | if len(sctv) == 1: 112 | sctv = str(sctv).replace(',', '') 113 | if len(ectv) == 1: 114 | ectv = str(ectv).replace(',', '') 115 | allct = str(sctv) + ',' + str(ectv) 116 | 117 | allct = allct.strip(',') 118 | 119 | type = ct['INDEX_NAME'] 120 | 121 | return type, allct 122 | 123 | def parserSql(self, db, tbl, ct): 124 | for type, val in ct.items(): 125 | sql = '' 126 | val = val.strip(',') 127 | if type == 'None' or type == 'handle': 128 | if val is not '': 129 | sql = 'split table `{}`.`{}` by {};'.format(db, tbl, val) 130 | else: 131 | if val is not '': 132 | sql = 'split table `{}`.`{}` index `{}` by {};'.format(db, tbl, type, val) 133 | 134 | print(sql) 135 | 136 | return sql 137 | 138 | def main(self): 139 | args = self.parse_args() 140 | 141 | db = args.tables.split('.')[0] 142 | tb = args.tables.split('.')[1] 143 | keys = self.keys(db, tb) 144 | parsers = {} 145 | for key in keys: 146 | type, ct = self.parserkeys(key) 147 | type = str(type) 148 | parsers[type] = ct + ',' + parsers.get(type,'') 149 | 150 | if args.totables is '': 151 | self.parserSql(db, tb, parsers) 152 | else: 153 | tdb = args.totables.split('.')[0] 154 | ttb = args.totables.split('.')[1] 155 | 156 | self.parserSql(tdb, ttb, parsers) 157 | 158 | def parse_args(self): 159 | # Incoming parameters 160 | parser = argparse.ArgumentParser( 161 | description="Update table statistics manually") 162 | parser.add_argument("-P", 163 | dest="port", 164 | help="tidb port, default: 4000", 165 | default=4201) 166 | parser.add_argument("-H", 167 | dest="mysql", 168 | help="Database address, default: 127.0.0.1", 169 | default="127.0.0.1") 170 | parser.add_argument("-u", 171 | dest="user", 172 | help="Database account, default: root", 173 | default="root") 174 | parser.add_argument("-p", 175 | dest="password", 176 | help="Database password, default: null", 177 | default="") 178 | parser.add_argument( 179 | "-t", 180 | dest="tables", 181 | help= 182 | "Table name (database.table), for example: test.test default: None", 183 | default='') 184 | parser.add_argument( 185 | "-tt", 186 | dest="totables", 187 | help= 188 | "Table name (database.table), for example: test.test1 default: None", 189 | default='') 190 | 191 | args = parser.parse_args() 192 | 193 | return args 194 | 195 | 196 | if __name__ == "__main__": 197 | main = SplitTable() 198 | main.main() 199 | -------------------------------------------------------------------------------- /Stats_dump.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import json 5 | import argparse 6 | import pymysql 7 | import subprocess 8 | import os 9 | import time 10 | import tarfile 11 | import shutil 12 | 13 | ## If Python is version 2.7, encoding problems can reload sys configuration 14 | try: 15 | import sys 16 | 17 | reload(sys) 18 | sys.setdefaultencoding('utf-8') 19 | except: 20 | pass 21 | 22 | def main(): 23 | args = parse_args() 24 | if args.database is None and args.tables is None: 25 | info = parser_all_info() 26 | elif args.tables is None and args.database is not None: 27 | info = parser_database_info(args.database) 28 | elif args.database is None and args.tables is not None: 29 | info = parser_table_info(args.tables) 30 | else: 31 | print("Please enter the correct library or table name！") 32 | 33 | download_dir = time.strftime("%Y%m%d-%H%M%S", time.localtime()) + "-stats" 34 | dir_stats = os.path.join(download_dir, 'stats') 35 | if not os.path.isdir(download_dir): 36 | os.makedirs(download_dir) 37 | if not os.path.isdir(dir_stats): 38 | os.makedirs(dir_stats) 39 | 40 | file_name = os.path.join(download_dir, 'schema.sql') 41 | content = parser_version() 42 | comments = "TiDB Version" 43 | schema_file(content, comments, file_name) 44 | for database_name in info: 45 | content = get_databas_schema(database_name) + ";\nuse {};".format( 46 | database_name) 47 | comments = "Database {} schema".format(database_name) 48 | schema_file(content, comments, file_name) 49 | for table_name in info[database_name]: 50 | stats_name = os.path.join( 51 | dir_stats, "{}.{}.json".format(database_name, table_name)) 52 | stats_name_tmp = os.path.join( 53 | 'stats', "{}.{}.json".format(database_name, table_name)) 54 | comments = "Table {} schema".format(table_name) 55 | content = get_table_schema( 56 | database_name, 57 | table_name) + ";\nLOAD STATS '{}';".format(stats_name_tmp) 58 | schema_file(content, comments, file_name) 59 | content = parser_stats(database_name, table_name) 60 | stats_file(content, stats_name) 61 | try: 62 | with tarfile.open("{}.tar.gz".format(download_dir), "w:gz") as tar: 63 | tar.add(download_dir, arcname=os.path.basename(download_dir)) 64 | shutil.rmtree(download_dir) 65 | print( 66 | "Directory {} was compressed successfully, and directory {} was deleted~".format( 67 | download_dir, download_dir)) 68 | except: 69 | print("Compression failure!") 70 | 71 | 72 | def parser_stats(database_name, table_name): 73 | args = parse_args() 74 | httpAPI = "http://{}/stats/dump/{}/{}".format(args.tidb, database_name, 75 | table_name) 76 | try: 77 | webContent = subprocess.check_output(["curl", "-sl", httpAPI]) 78 | webContent = json.loads(webContent) 79 | except: 80 | print("Failed to obtain table {}.{} statistics".format(database_name, 81 | table_name)) 82 | 83 | return webContent 84 | 85 | 86 | def stats_file(content, file_name): 87 | try: 88 | with open(file_name, 'a+') as f: 89 | json.dump(content, f) 90 | print(u"File {} written successfully~".format(file_name)) 91 | except: 92 | print(u"Write {} error!".format(file_name)) 93 | 94 | 95 | def schema_file(content, comments, file_name): 96 | try: 97 | with open(file_name, 'a+') as f: 98 | f.write(u"-- {} \n".format(comments)) 99 | f.write(u"{} \n\n".format(content)) 100 | print(u"Write {} Successful~".format(comments)) 101 | except: 102 | print("Write error: {}!".format(content)) 103 | 104 | 105 | def get_table_schema(database_name, table_name): 106 | content = mysql_execute("use {};".format(database_name), 107 | "show create table {};".format(table_name)) 108 | content = content[0]['Create Table'] 109 | 110 | return content 111 | 112 | 113 | def get_databas_schema(database_name): 114 | content = mysql_execute("show create database {}".format(database_name)) 115 | content = content[0]['Create Database'] 116 | 117 | return content 118 | 119 | 120 | def parser_version(): 121 | content = mysql_execute("select tidb_version()") 122 | content = content[0]["tidb_version()"] 123 | content = "/*\n" + content + "\n*/" 124 | 125 | return content 126 | 127 | 128 | def parser_table_info(tables): 129 | info = {} 130 | for content in tables.split(','): 131 | database_name = content.split('.')[0] 132 | table_name = content.split('.')[1] 133 | table = list(info.get(database_name, '')) 134 | table.append(table_name) 135 | info[database_name] = table 136 | 137 | return info 138 | 139 | 140 | def parser_database_info(database_name): 141 | info = {} 142 | for database_name in database_name.split(','): 143 | content = mysql_execute("use {};".format(database_name), "show tables;") 144 | tables = [] 145 | for table in content: 146 | table_name = table["Tables_in_{}".format(database_name)] 147 | tables.append(table_name) 148 | info[database_name] = tables 149 | 150 | return info 151 | 152 | 153 | def parser_all_info(): 154 | info = {} 155 | content = mysql_execute('show databases;') 156 | for database in content: 157 | database_name = database["Database"] 158 | if database_name not in ["INFORMATION_SCHEMA", "PERFORMANCE_SCHEMA", 159 | "mysql", "default"]: 160 | content = mysql_execute("use {};".format(database_name), 161 | "show tables;") 162 | tables = [] 163 | for table in content: 164 | table_name = table["Tables_in_{}".format(database_name)] 165 | tables.append(table_name) 166 | info[database_name] = tables 167 | else: 168 | print( 169 | "Backing up INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql,default libraries is not supported!") 170 | 171 | return info 172 | 173 | 174 | def mysql_execute(*_sql): 175 | args = parse_args() 176 | host = args.mysql.split(':', 1)[0] 177 | port = int(args.mysql.split(':', 1)[1]) 178 | try: 179 | connection = pymysql.connect(host=host, 180 | user=args.user, 181 | password=args.password, 182 | port=port, 183 | charset='utf8mb4', 184 | cursorclass=pymysql.cursors.DictCursor) 185 | except: 186 | print("Connect Database is failed~") 187 | 188 | try: 189 | with connection.cursor() as cursor: 190 | cursor.execute("SET NAMES utf8mb4") 191 | for sql in _sql: 192 | cursor.execute(sql) 193 | content = cursor.fetchall() 194 | connection.commit() 195 | except: 196 | print( 197 | "SQL {} execution failed~ \n Please check table or database exists or not!".format( 198 | _sql)) 199 | 200 | finally: 201 | cursor.close() 202 | connection.close() 203 | 204 | return content 205 | 206 | 207 | def parse_args(): 208 | parser = argparse.ArgumentParser( 209 | description="Export statistics and table structures") 210 | parser.add_argument("-tu", 211 | dest="tidb", 212 | help="tidb status url, default: 127.0.0.1:10080", 213 | default="127.0.0.1:10080") 214 | parser.add_argument( 215 | "-H", 216 | dest="mysql", 217 | help="Database address and port, default: 127.0.0.1:4000", 218 | default="127.0.0.1:4000") 219 | parser.add_argument("-u", 220 | dest="user", 221 | help="Database account, default: root", 222 | default="root") 223 | parser.add_argument("-p", 224 | dest="password", 225 | help="Database password, default: null", 226 | default="") 227 | parser.add_argument( 228 | "-d", 229 | dest="database", 230 | help="Database name, for example: test,test1, default: None", 231 | default=None) 232 | parser.add_argument( 233 | "-t", 234 | dest="tables", 235 | help= 236 | "Table name (database.table), for example: test.test,test.test2, default: None", 237 | default=None) 238 | args = parser.parse_args() 239 | return args 240 | 241 | 242 | if __name__ == "__main__": 243 | main() 244 | -------------------------------------------------------------------------------- /TiDB_Kill.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import argparse 5 | import pymysql 6 | import json 7 | 8 | # If Python is version 2.7, encoding problems can reload sys configuration 9 | try: 10 | import sys 11 | 12 | reload(sys) 13 | sys.setdefaultencoding('utf-8') 14 | except: 15 | pass 16 | 17 | 18 | def main(): 19 | args = parse_args() 20 | ack = "no" 21 | ct = 0 22 | while ack == "no": 23 | ct += 1 24 | ack = mysql_execute() 25 | print("Connection retries {}".format(ct)) 26 | if ct >= 100: 27 | ack = "yes" 28 | 29 | 30 | def mysql_execute(): 31 | # Connect to MySQL and execute SQL commands 32 | args = parse_args() 33 | config = { 34 | "host": args.mysql, 35 | "port": int(args.port), 36 | "user": args.user, 37 | "password": args.password, 38 | "charset": 'utf8mb4', 39 | # "cursorclass": pymysql.cursors.DictCursor 40 | } 41 | 42 | connection = pymysql.connect(**config) 43 | cursor = connection.cursor() 44 | _sql = "select @@tidb_config;" 45 | try: 46 | cursor.execute(_sql) 47 | content = cursor.fetchall() 48 | connection.commit() 49 | except: 50 | print("SQL {} execution failed~".format(_sql)) 51 | finally: 52 | print(args.advertiseAddress, args.advertisePort) 53 | config_c = json.loads(content[0][0]) 54 | if config_c["advertise-address"].strip( 55 | ) == args.advertiseAddress and str( 56 | config_c["port"]).strip() == args.advertisePort: 57 | print("The TiDB IP is {}".format(config_c["advertise-address"])) 58 | print("The TiDB IP Port is {}".format(config_c["port"])) 59 | sql_kill = "kill tidb {}".format(args.sessionId) 60 | _ct = str( 61 | input("Will execute: {}, y/n (default:yes)".format( 62 | sql_kill))) or "y" 63 | if _ct == "y": 64 | cursor.execute(sql_kill) 65 | else: 66 | print("SQL {} cancel execution".format(sql_kill)) 67 | ack = "yes" 68 | cursor.close() 69 | connection.close() 70 | else: 71 | cursor.close() 72 | connection.close() 73 | ack = "no" 74 | 75 | return ack 76 | 77 | 78 | def parse_args(): 79 | # Incoming parameters 80 | parser = argparse.ArgumentParser(description="Kill TiDB session by id") 81 | parser.add_argument("-P", 82 | dest="port", 83 | help="tidb port, default: 4000", 84 | default=4000) 85 | parser.add_argument("-H", 86 | dest="mysql", 87 | help="Database address, default: 127.0.0.1", 88 | default="127.0.0.1") 89 | parser.add_argument("-u", 90 | dest="user", 91 | help="Database account, default: root", 92 | default="root") 93 | parser.add_argument("-p", 94 | dest="password", 95 | help="Database password, default: null", 96 | default="") 97 | parser.add_argument("-a", 98 | dest="advertiseAddress", 99 | help="TiDB Address, default: null", 100 | default="") 101 | parser.add_argument("-ap", 102 | dest="advertisePort", 103 | help="TiDB Port, default: 4000", 104 | default="4000") 105 | parser.add_argument("-id", 106 | dest="sessionId", 107 | help="session id, default: null", 108 | default="") 109 | args = parser.parse_args() 110 | 111 | return args 112 | 113 | 114 | if __name__ == "__main__": 115 | main() 116 | -------------------------------------------------------------------------------- /TiDB_Outfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import mysql.connector 5 | import argparse 6 | import time 7 | import csv 8 | import os 9 | import threading 10 | 11 | from multiprocessing import Process 12 | 13 | try: 14 | import Queue 15 | except: 16 | import queue 17 | 18 | ## If Python is version 2.7, encoding problems can reload sys configuration 19 | try: 20 | import sys 21 | 22 | reload(sys) 23 | sys.setdefaultencoding('utf-8') 24 | except: 25 | pass 26 | 27 | 28 | def outfile_tidb(mode, file_name, fieldnames, queue): 29 | while True: 30 | try: 31 | _sql = None 32 | _sql = queue.get(True, 1) 33 | _cmd = mysql_execute(mode, file_name, fieldnames, _sql) 34 | except: 35 | print("{} get {}".format(threading.current_thread(), _sql)) 36 | break 37 | finally: 38 | print("task is successful~") 39 | queue.task_done() 40 | 41 | 42 | def main(): 43 | args = parse_args() 44 | queue = Queue.Queue() 45 | get_list_time = time.time() 46 | id_list = parser_id() 47 | print("get id cost: {}".format(time.time() - get_list_time)) 48 | 49 | if args.column is "all": 50 | fieldnames = mysql_execute(_sql="desc {};".format(args.table), 51 | dictionary=True) 52 | fieldnames = [i['Field'] for i in fieldnames] 53 | else: 54 | fieldnames = str(args.column).split(',') 55 | 56 | batch = int(args.batch) 57 | for list_id in range(0, len(id_list), batch): 58 | filter_content = list(id_list[list_id:list_id + batch]) 59 | if len(filter_content) == 1: 60 | filter_content = filter_content[0][0] 61 | _sql = 'select {} from {} where {} = {}'.format( 62 | ', '.join(fieldnames), args.table, args.field, filter_content) 63 | else: 64 | filter_content = str(tuple([i[0] for i in filter_content])) 65 | _sql = 'select {} from {} where {} in {}'.format( 66 | ', '.join(fieldnames), args.table, args.field, filter_content) 67 | 68 | queue.put(_sql) 69 | 70 | thread = int(args.thread) 71 | threads = [] 72 | for num in range(thread): 73 | file_name = "{}.{}.{}.csv".format(args.database, args.table, num) 74 | t = threading.Thread(target=outfile_tidb, 75 | args=('csv', file_name, fieldnames, queue)) 76 | threads.append(t) 77 | 78 | for num in range(thread): 79 | threads[num].start() 80 | 81 | queue.join() 82 | # for num in range(thread): 83 | # threads[num].join() 84 | 85 | 86 | def parser_id(): 87 | args = parse_args() 88 | try: 89 | schema = mysql_execute(_sql="desc {};".format(args.table)) 90 | id_list = mysql_execute(_sql="select {} as id from {} {};".format( 91 | args.field, args.table, args.where)) 92 | 93 | except all as error: 94 | print('Error log is: {}'.format(error)) 95 | 96 | return id_list 97 | 98 | 99 | def mysql_execute(mode=None, 100 | file_name=None, 101 | fieldnames=[], 102 | _sql="", 103 | dictionary=False): 104 | args = parse_args() 105 | host = args.mysql.split(':', 1)[0] 106 | port = int(args.mysql.split(':', 1)[1]) 107 | try: 108 | connection = mysql.connector.connect(host=host, 109 | user=args.user, 110 | password=args.password, 111 | port=port, 112 | charset='utf8mb4', 113 | database=args.database) 114 | except all as error: 115 | print("Connect Database is failed~\n", error) 116 | 117 | try: 118 | cursor = connection.cursor(dictionary=dictionary) 119 | # with connection.cursor(dictionary=dictionary) as cursor: 120 | cursor.execute("SET NAMES utf8mb4") 121 | cursor.execute(_sql) 122 | if mode is not 'csv': 123 | content = cursor.fetchall() 124 | else: 125 | start_time = time.time() 126 | with open(file_name, 'a+') as csvfile: 127 | if os.path.getsize(file_name): 128 | pass 129 | else: 130 | writer = csv.DictWriter(csvfile, 131 | fieldnames=fieldnames, 132 | quoting=csv.QUOTE_NONNUMERIC) 133 | writer.writeheader() 134 | 135 | writer = csv.writer(csvfile, 136 | delimiter=',', 137 | lineterminator='\n', 138 | quoting=csv.QUOTE_NONNUMERIC) 139 | content = cursor.fetchall() 140 | print("get content cost time is {}".format( 141 | time.time() - start_time)) 142 | start_time = time.time() 143 | writer.writerows(content) 144 | 145 | end_time = time.time() 146 | print("Write {} is Successful, Cost time is {}".format( 147 | file_name, end_time - start_time)) 148 | 149 | connection.commit() 150 | 151 | except all as error: 152 | print( 153 | "SQL {} execution failed~ \n Please check table or database exists or not!,error: {}".format( 154 | _sql)) 155 | 156 | finally: 157 | cursor.close() 158 | connection.close() 159 | 160 | return content 161 | 162 | 163 | def parse_args(): 164 | parser = argparse.ArgumentParser(description="Export data to CSV") 165 | parser.add_argument("-tp", 166 | dest="mysql", 167 | help="TiDB Port, default: 127.0.0.1:4000", 168 | default="127.0.0.1:4000") 169 | parser.add_argument("-u", 170 | dest="user", 171 | help="TiDB User, default: root", 172 | default="root") 173 | parser.add_argument("-p", 174 | dest="password", 175 | help="TiDB Password, default: null", 176 | default="") 177 | parser.add_argument("-d", 178 | dest="database", 179 | help="database name, default: test", 180 | default="test") 181 | parser.add_argument("-t", 182 | dest="table", 183 | help="Table name, default: test", 184 | default="test") 185 | parser.add_argument("-k", 186 | dest="field", 187 | help="Table primary key, default: _tidb_rowid", 188 | default="_tidb_rowid") 189 | parser.add_argument("-T", 190 | dest="thread", 191 | help="Export thread, default: 20", 192 | default=20) 193 | parser.add_argument("-B", 194 | dest="batch", 195 | help="Export batch size, default: 3000", 196 | default=3000) 197 | parser.add_argument( 198 | "-w", 199 | dest="where", 200 | help="Filter condition， for example: where id >= 1, default: null", 201 | default="") 202 | parser.add_argument( 203 | "-c", 204 | dest="column", 205 | help="Table Column, for example: id,name, default: all", 206 | default="all") 207 | 208 | args = parser.parse_args() 209 | 210 | return args 211 | 212 | 213 | if __name__ == "__main__": 214 | start_time = time.time() 215 | main() 216 | end_time = time.time() 217 | print('Exiting Main Thread, Total cost time is {}'.format( 218 | end_time - start_time)) 219 | -------------------------------------------------------------------------------- /adminCheck.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | settings() { 4 | # Configuration db information 5 | db_user='root' 6 | db_port=4100 7 | # Note: Password cannot be empty ！！！！ 8 | db_password="root" 9 | db_ip='127.0.0.1' 10 | db_name='test test1 test2' 11 | variable='tidb_distsql_scan_concurrency=30' 12 | } 13 | 14 | environment() { 15 | # Configure MySQL command path 16 | mysql_path='/usr/bin/mysql' 17 | ## if MySQL is not exist,then exit 18 | if [ ! -f $mysql_path ]; then 19 | echo "$mysql_path is not exist~" 20 | exit 8 21 | else 22 | echo "$mysql_path is exist~" 23 | fi 24 | } 25 | 26 | getTso() { 27 | # Get TSO 28 | tso=`$mysql_path -u $db_user -p$db_password -h $db_ip -P $db_port -e "show master status;"` 29 | # echo "$tso" | grep binlog | awk -F' ' '{{print $2}}' 30 | } 31 | 32 | main() { 33 | settings; 34 | environment; 35 | getTso; 36 | local tso=`echo "$tso" | grep binlog | awk -F' ' '{{print $2}}'` 37 | 38 | # Get tables 39 | for dbsName in $db_name; do 40 | local sql="SELECT 41 | table_schema AS DatabaseName, 42 | table_name AS TableName, 43 | index_name AS IndexName, 44 | GROUP_CONCAT(column_name ORDER BY seq_in_index ASC) AS Columns 45 | FROM information_schema.statistics 46 | WHERE table_schema = '$dbsName' 47 | GROUP BY table_schema, table_name, index_name 48 | ORDER BY table_schema, table_name, index_name; 49 | " 50 | tables=$($mysql_path -u$db_user -p$db_password -h $db_ip -P $db_port -e "$sql") 51 | echo "$tables" | awk -F' ' '{{print $2,$3,$4}}' | grep -Ev 'TableName' >info.tables 52 | while read -r line; do 53 | # 使用适当的方法来提取信息，假设以空格分隔 54 | table_name=$(echo $line | cut -d ' ' -f 1) 55 | index_type=$(echo $line | cut -d ' ' -f 2) 56 | 57 | # 使用循环来处理剩余的信息 58 | for column in $(echo $line | cut -d ' ' -f 3-); do 59 | echo "Database: $dbsName,Table: $table_name, Index Type: $index_type, Column: $column" 60 | # 在这里可以执行任何需要的操作 61 | local indexInfo=$column 62 | local database=$dbsName 63 | local table=$table_name 64 | local tso=$tso 65 | local othersql="set tidb_snapshot=$tso;set $variable" 66 | local current_datetime=$(date +"%Y-%m-%d %H:%M:%S") 67 | if [ $index_type = "PRIMARY" ];then 68 | echo "$current_datetime 主键跳过"; 69 | # XoRSQL="select (bit_xor(concat($indexInfo))) as bitXor from $database.$table use index(primary);" 70 | else 71 | local XoRSQL="select (bit_xor(concat($indexInfo))) as bitXor from $database.$table ;" 72 | # Get XOR Bit 73 | # echo $XoRSQL 74 | local xor_bit=$($mysql_path -u $db_user -p$db_password -h $db_ip -P $db_port -e "$othersql;$XoRSQL") 75 | local xor=`echo $xor_bit|awk -F' ' '{{print $2}}'` 76 | local XoRSQLPrimary="select (bit_xor(concat($indexInfo))) as bitXor from $database.$table use index(primary);" 77 | local xor_bit=$($mysql_path -u $db_user -p$db_password -h $db_ip -P $db_port -e "$othersql;$XoRSQL") 78 | local xorPrimary=`echo $xor_bit|awk -F' ' '{{print $2}}'` 79 | if [ $xor != $xorPrimary ];then 80 | echo "$current_datetime xor: $xor,xorPrimary: $xorPrimary, 校验不通过" 81 | else 82 | echo "$current_datetime xor: $xor,xorPrimary: $xorPrimary, 校验通过" 83 | fi 84 | fi 85 | 86 | done 87 | done 'VIEW';") 53 | local table_name=($table_name) 54 | # Get all db names in the library 55 | for ((tbi=1;tbi<${#table_name[@]};tbi++)); do 56 | local _table_names=${table_name[tbi]} 57 | analyze; 58 | echo "Analyze table $_dbname.$_table_names Sucess~" 59 | sleep 1 60 | done 61 | done 62 | } 63 | 64 | main; 65 | -------------------------------------------------------------------------------- /dumpling.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | init(){ 4 | # database info 5 | user='tidb' 6 | port=4000 7 | password='tidb@123!@#' 8 | filetype='sql' 9 | thread=4 10 | host='127.0.0.1' 11 | 12 | # backup 13 | _time=$(date '+%Y%m%d') 14 | backupDir='/tidbbackup/fmtfx_back/' 15 | # dir 16 | backupDir+=$_time 17 | 18 | # Old 19 | oldTime=$(date -d '2 day ago' '+%Y%m%d') 20 | oldDir='/tidbbackup/fmtfx_back/' 21 | # dir 22 | oldDir+=$oldTime 23 | 24 | # other 25 | ## log 26 | logFile='/tidbbackup/fmtfx_back/main.log' 27 | ## log time 28 | logTime=$(date '+%Y/%m/%d %H:%M:%S') 29 | ## binary dir 30 | binaryPath='/home/db/tidbmgt/tools' 31 | binaryFile=$binaryPath'/dumpling' 32 | _dir='/tidbbackup/fmtfx_back/' 33 | 34 | ## if dumpling is not exist,then exit 35 | if [ ! -f $binaryFile ];then 36 | echo "$binaryFile is not exist~" 37 | exit 8 38 | else 39 | echo "$binaryFile is exist~" 40 | fi 41 | 42 | } 43 | 44 | mkDir(){ 45 | if [ ! -d $_dir ];then 46 | mkdir -p $_dir 47 | else 48 | echo "$_dir exist~" 49 | fi 50 | } 51 | 52 | rmBackup(){ 53 | # rm -rf Previous backup 54 | rm -rf $oldDir && echo "[$logTime] Delete $oldDir Sucess~" &>> 'execute.log' 55 | } 56 | 57 | dump(){ 58 | # cd base path 59 | cd $binaryPath; 60 | nohup ./dumpling \ 61 | -u $user \ 62 | -P $port \ 63 | -p $password \ 64 | -h $host \ 65 | --filetype $filetype \ 66 | -t $thread \ 67 | -o $backupDir \ 68 | -r 200000 \ 69 | -F 256MiB &>> $logFile & 70 | 71 | echo "dumpling start~, backup dir is [ $backupDir ]" 72 | echo "You can execute the [ tail -f $logFile ] command to view the progress " 73 | # check 74 | sleep 3 75 | ps -ef | grep dumpling | grep -Ev 'grep' 76 | } 77 | 78 | main(){ 79 | init; 80 | mkDir 81 | dump; 82 | rmBackup 83 | 84 | } 85 | 86 | # run scripts 87 | main; 88 | -------------------------------------------------------------------------------- /failoverByCDC.md: -------------------------------------------------------------------------------- 1 | # 使用说明 2 | 这个脚本是一个用于管理 TiDB 集群和 CDC 同步任务的 Bash 脚本。它包含多项功能，例如数据库用户锁定、重启 TiDB 服务器节点、检查连接、管理 CDC 任务等。以下是该脚本的详细使用说明： 3 | 4 | ## 脚本功能 5 | 该脚本主要用于TiDB数据库的主从切换及数据同步任务的管理。它可以执行以下操作： 6 | 7 | 1. 锁定指定的数据库用户 8 | 2. 重启 TiDB 服务节点 9 | 3. 检查是否有连接留存 10 | 4. 等待 CDC 同步任务赶上当前数据状态 11 | 5. 删除现有的 CDC 同步任务 12 | 6. 在从库创建新的 CDC 同步任务 13 | 7. 解锁数据库用户 14 | 15 | ## 使用方法 16 | 脚本通过命令行参数接收配置信息，并根据这些参数执行不同的操作。使用示例如下： 17 | 18 | ```shell 19 | ./failoverByCDC.sh [参数列表] 20 | ``` 21 | 22 | ## 参数列表 23 | 脚本支持以下参数： 24 | - --clusterName=<集群名称>: 指定操作的 TiDB 集群名称。 25 | - --masterDB=<主数据库地址:端口>: 主数据库的地址和端口，格式为 IP:端口。 26 | - --masterUser=<主数据库用户名>: 用于访问主数据库的用户名。 27 | - --masterPassword=<主数据库密码>: 用于访问主数据库的密码。 28 | - --slaveDB=<从数据库地址:端口>: 从数据库的地址和端口，格式同主数据库。 29 | - --slaveUser=<从数据库用户名>: 用于访问从数据库的用户名。 30 | - --slavePassword=<从数据库密码>: 用于访问从数据库的密码。 31 | - --masterPD=: 主 PD (Placement Driver) 服务的地址和端口，PD 是 TiDB 集群的元数据管理组件。 32 | - --slavePD=: 从 PD 服务的地址和端口。 33 | - --version=: 指定 TiCDC 的版本，TiCDC 是 TiDB 集群的变更数据捕获组件。 34 | - --reloadList=<重启节点列表>: 需要重启的 TiDB 节点的地址和端口列表，多个地址用逗号分隔。 35 | - --changefeedID=: CDC 同步任务的唯一标识。 36 | - --cdcConfig=: CDC 同步任务的配置文件路径。 37 | - --lockUser=<锁定的用户列表>: 需要锁定的数据库用户列表，多个用户用逗号分隔，格式为 用户名@'主机'。 38 | - --mod=<执行模式>: 指定脚本执行的具体步骤，用逗号分隔的数字序列表示，每个数字对应脚本中定义的一个操作步骤。 39 | - --sink-uri=<新 CDC 同步的 TiDB 连接 URI>: 新的 CDC 同步任务的目标 TiDB 实例的连接 URI。 40 | 41 | ## 示例 42 | 以下是一个使用示例： 43 | 44 | ```shell 45 | ./failoverByCDC.sh --clusterName="tidb-test" \ 46 | --masterDB='10.xxx:4000' \ 47 | --masterUser='root' \ 48 | --masterPassword='tidbxxx' \ 49 | --masterPD='10.xxx:2379' \ 50 | --slaveDB='10.xxx:4000' \ 51 | --slaveUser='root' \ 52 | --slavePassword='tidb@123' \ 53 | --slavePD='10.102.173.121:2379' \ 54 | --version='v7.1.4' \ 55 | --reloadList='172.xxx:4000,172xxx:4000' \ 56 | --changefeedID='userxxkup' \ 57 | --cdcConfig='cdc.conf' \ 58 | --lockUser="xin@'%',pxlogin@'%',tidb@'%'" \ 59 | --mod='1,2,3' \ 60 | --sink-uri="tidb://root:w@-xxx@1xx.xxx:4000/&transaction-atomicity=none" 61 | ``` 62 | 63 | ## 示例参数解释 64 | - --mod='1,2,3'：这个参数决定了脚本中执行的操作序列。每个数字对应脚本中定义的一个操作步骤。例如，在上面的示例中，1,2,3 分别表示： 65 | - 1 - 锁定用户 66 | - 2 - 重启 TiDB server 节点 67 | - 3 - 检查是否有留存连接 68 | mod 参数的可选值及其对应功能 69 | - 1 锁定用户 70 | - 2 重启 TiDB server 节点 71 | - 3 检查是否有留存连接 72 | - 4 等待 CDC 任务同步追上 73 | - 5 删除 master CDC 任务 74 | - 6 创建从库到主库的同步 75 | - 7 解锁从库应用用户 76 | 您可以根据需要任意组合这些数字来指定执行的步骤。例如，如果只想删除 CDC 任务并创建新的同步任务，可以设置 --mod='5,6'。 77 | 78 | ## 注意事项 79 | - 确保所有涉及的数据库、用户及其他配置信息都是正确的。 80 | - 用逗号分隔 --mod 参数中的数字，不要有空格。 81 | - 确保脚本和相关命令行工具具有执行权限，并在可以访问 TiDB 和 CDC 的环境中执行。 82 | - 确保脚本具有执行权限，可使用 chmod +x failoverByCDC.sh 命令来设置。 83 | - 该脚本需要在具有访问数据库和执行 TiDB、CDC 命令的环境中运行。 84 | - 传入的参数应根据实际环境进行调整。 85 | 86 | 通过适当配置这些参数，灵活地使用此脚本来管理 TiDB 集群和 CDC 任务。 87 | -------------------------------------------------------------------------------- /failoverByCDC.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 全局变量存储配置信息 4 | MASTER_DB="" 5 | MASTER_USER="" 6 | MASTER_PASSWORD="" 7 | SLAVE_DB="" 8 | SLAVE_USER="" 9 | SLAVE_PASSWORD="" 10 | MASTER_PD="" 11 | SLAVE_PD="" 12 | VERSION="" 13 | RELOAD_LIST="" 14 | CHANGEFEED_ID="" 15 | CDC_CONFIG="" 16 | LOCK_USER="" 17 | MOD="" 18 | clusterName="" 19 | SINK_URI="" 20 | 21 | # 参数解析函数 22 | parserArgs() { 23 | while [ "$1" != "" ]; do 24 | case $1 in 25 | --masterDB=*) 26 | MASTER_DB="${1#*=}" 27 | ;; 28 | --masterUser=*) 29 | MASTER_USER="${1#*=}" 30 | ;; 31 | --masterPassword=*) 32 | MASTER_PASSWORD="${1#*=}" 33 | ;; 34 | --slaveDB=*) 35 | SLAVE_DB="${1#*=}" 36 | ;; 37 | --slaveUser=*) 38 | SLAVE_USER="${1#*=}" 39 | ;; 40 | --slavePassword=*) 41 | SLAVE_PASSWORD="${1#*=}" 42 | ;; 43 | --masterPD=*) 44 | MASTER_PD="${1#*=}" 45 | ;; 46 | --slavePD=*) 47 | SLAVE_PD="${1#*=}" 48 | ;; 49 | --version=*) 50 | VERSION="${1#*=}" 51 | ;; 52 | --reloadList=*) 53 | RELOAD_LIST="${1#*=}" 54 | ;; 55 | --changefeedID=*) 56 | CHANGEFEED_ID="${1#*=}" 57 | ;; 58 | --cdcConfig=*) 59 | CDC_CONFIG="${1#*=}" 60 | ;; 61 | --lockUser=*) 62 | LOCK_USER="${1#*=}" 63 | ;; 64 | --mod=*) 65 | MOD="${1#*=}" 66 | ;; 67 | --clusterName=*) 68 | clusterName="${1#*=}" 69 | ;; 70 | --sink-uri=*) 71 | SINK_URI="${1#*=}" 72 | ;; 73 | *) 74 | echo "无效参数：$1" 75 | usage 76 | exit 1 77 | ;; 78 | esac 79 | shift 80 | done 81 | 82 | # 检查是否所有必需的参数都已设置 83 | if [ -z "$MASTER_DB" ] || [ -z "$MASTER_USER" ] || [ -z "$MASTER_PASSWORD" ] || 84 | [ -z "$SLAVE_DB" ] || [ -z "$SLAVE_USER" ] || [ -z "$SLAVE_PASSWORD" ] || 85 | [ -z "$MASTER_PD" ] || [ -z "$SLAVE_PD" ] || [ -z "$VERSION" ] || [ -z "$RELOAD_LIST" ] || 86 | [ -z "$CHANGEFEED_ID" ] || [ -z "$CDC_CONFIG" ] || [ -z "$LOCK_USER" ] || 87 | [ -z "$MOD" ] || [ -z "$clusterName" ] || [ -z "$SINK_URI" ]; then 88 | echo "缺少必需的参数。" 89 | usage 90 | exit 1 91 | fi 92 | } 93 | 94 | # 使用说明函数 95 | usage() { 96 | echo "使用方法: $0 --clusterName=<集群名称> --masterDB=<数据库主机:端口> --masterUser=<用户名> --masterPassword=<密码> --slaveDB=<数据库从机:端口> --slaveUser=<用户名> --slavePassword=<密码> --masterPD= --slavePD= --version=<版本> --reloadList=<主机列表> --changefeedID= --cdcConfig=<配置文件> --lockUser=<锁定用户列表> --mod=<执行模式> --sink-uri=" 97 | # 逻辑代码 98 | echo "主库数据库: $MASTER_DB" 99 | echo "主库用户: $MASTER_USER" 100 | echo "主库密码: $MASTER_PASSWORD" 101 | echo "Master PD 服务: $MASTER_PD" 102 | echo "从库数据库: $SLAVE_DB" 103 | echo "从库用户: $SLAVE_USER" 104 | echo "从库密码: $SLAVE_PASSWORD" 105 | echo "Slave PD 服务: $SLAVE_PD" 106 | echo "CDC 版本: $VERSION" 107 | echo "重载列表: $RELOAD_LIST" 108 | echo "Changefeed 标识: $CHANGEFEED_ID" 109 | echo "CDC 配置文件: $CDC_CONFIG" 110 | echo "锁定用户: $LOCK_USER" 111 | echo "新建 cdc 同步 uri: $SINK_URI" 112 | echo "执行模式: $MOD, 113 | 1. 用户锁定 114 | 2. 重启 tidb server 节点 115 | 3. 确认是否有留存连接 116 | 4. 等待 cdc 任务同步追上 117 | 5. 删除 master cdc 任务 118 | 6. 创建从库到主库的同步 119 | 7. 解锁从库应用用户" 120 | } 121 | 122 | # 锁定账户 123 | userLock() { 124 | # 获取用户 125 | local userAccounts=$LOCK_USER 126 | # 读取账户到数组 127 | IFS=',' read -ra ACCOUNTS <<< "$userAccounts" 128 | # 创建一个新的数组用于存储过滤后的账户 129 | filtered_accounts=() 130 | 131 | # 遍历 ACCOUNTS 数组 132 | for account in "${ACCOUNTS[@]}"; do 133 | # 过滤掉包含 root@'%' 134 | if [[ "$account" != "root@'%'" ]]; then 135 | filtered_accounts+=("$account") 136 | fi 137 | done 138 | 139 | # 锁定操作开始 140 | for account in "${filtered_accounts[@]}"; do 141 | echo "Locking account: $account" 142 | IFS=':' read -r ip port <<< "$MASTER_DB" 143 | mysql -h "$ip" -u "$MASTER_USER" -p"$MASTER_PASSWORD" -P "$port" -e "ALTER USER $account ACCOUNT LOCK;FLUSH PRIVILEGES;" 144 | if [ $? -eq 0 ]; then 145 | echo "Successfully locked: $account" 146 | else 147 | echo "Failed to lock: $account" 148 | exit 1 149 | fi 150 | done 151 | } 152 | 153 | # 重启 tidb-server 节点 154 | restartTidb(){ 155 | local tidbList=$RELOAD_LIST 156 | # 使用数组来安全地存储和执行命令 157 | local command=(~/.tiup/bin/tiup cluster reload "$clusterName" -R tidb -N "$tidbList" -c 200 -y) 158 | 159 | # 使用echo显示命令 160 | echo "执行重启命令：${command[*]}" 161 | 162 | # 执行命令 163 | "${command[@]}" 164 | 165 | # 检查命令执行的返回值 166 | if [ $? -eq 0 ]; then 167 | echo "Reload command executed successfully." 168 | else 169 | echo "Reload command failed." 170 | exit 1 171 | fi 172 | } 173 | 174 | # 确定是否有相关账户连接留存 175 | checkConnect(){ 176 | # master 的 ip 端口 177 | IFS=':' read -r ip port <<< "$MASTER_DB" 178 | # 封的用户：kfc_login@'%',phhs_login@'%' 转为：'kfc_login','phhs_login' 179 | local users=$(echo $LOCK_USER|sed "s/@'%'//g" | sed "s/,/','/g") 180 | # count 行数 181 | local command=(mysql -h "$ip" -u "$MASTER_USER" -p"$MASTER_PASSWORD" -P "$port" -e \ 182 | "select count(1) as count from information_schema.cluster_processlist where USER in ('','$users');") 183 | # 输出命令 184 | echo "查看连接 SQL：" 185 | echo ${command[*]} 186 | # 执行命令 187 | local result 188 | result=$("${command[@]}") 189 | local status=$? # 保存 mysql 命令的退出状态 190 | # 检查命令执行的返回值 191 | if [ $status -eq 0 ]; then 192 | echo "count command executed successfully." 193 | else 194 | echo "count command failed." 195 | exit 1 196 | fi 197 | # 行数判断 198 | echo "'$users' 账户的连接数为:" $result 199 | # 去除结果 \n 200 | local result=$(echo $result | tr -d '\n') 201 | if [ "$result" = "count 0" ]; then 202 | echo "无连接，继续脚本" 203 | else 204 | echo "尚存连接，即将退出" 205 | exit 1 206 | fi 207 | } 208 | 209 | # 获取 cdc tso 210 | getCheckPoint(){ 211 | # 执行查询命令 212 | local command=(~/.tiup/bin/tiup cdc:$VERSION cli changefeed query -s --pd=http://$MASTER_PD --changefeed-id=$CHANGEFEED_ID) 213 | # 输出命令 214 | echo "取 cdc 获取同步任务状态命令: " 215 | echo ${command[*]} 216 | # 执行命令 217 | local result 218 | result=$("${command[@]}") 219 | ## 这里是测试问题看结果 220 | #local result 221 | #local result=$(cat ./tmp.txt) 222 | # 检查命令执行的返回值 223 | if [ $? -eq 0 ]; then 224 | echo "get CDC checkpoint tso command executed successfully." 225 | else 226 | echo "get CDC checkpoint tso command failed." 227 | exit 1 228 | fi 229 | # 获取 tso 230 | cdcCheckPoint=$(echo "$result" | tr '\r' '\n' | grep "checkpoint_tso" | sed -n 's/.*"checkpoint_tso": \([0-9]*\),.*/\1/p') 231 | echo "CDC check Point TSO: $cdcCheckPoint" 232 | echo "$cdcCheckPoint" 233 | } 234 | 235 | # 确定 cdc 任务 236 | checkCdcStatus(){ 237 | # master 的 ip 端口 238 | IFS=':' read -r ip port <<< "$MASTER_DB" 239 | # 获取 master 当前 TSO 240 | local masterCommand=(mysql -h "$ip" -u "$MASTER_USER" -p"$MASTER_PASSWORD" -P "$port" -e 241 | "show master status;") 242 | # 输出命令 243 | echo ${masterCommand[*]} 244 | # 执行命令 245 | local masterResult 246 | masterResult=$("${masterCommand[@]}") 247 | local status=$? 248 | # 检查命令执行的返回值 249 | if [ $status -eq 0 ]; then 250 | echo "get Master tso command executed successfully." 251 | else 252 | echo "get Master tso command failed." 253 | exit 1 254 | fi 255 | # 结果格式化，取第二行最后一个字段内容，也就是 pos 值 256 | local masterPos=$(echo "$masterResult" | awk 'NR==2 {print $NF}') 257 | echo "master pos 为：$masterPos" 258 | 259 | # 定义循环控制变量 260 | ctlNum=5 261 | ctmTmp=1 262 | # 进入循环 263 | while [ $ctmTmp -le $ctlNum ]; do 264 | echo "循环 $ctmTmp 次" 265 | # 等待一小段时间，获取 cdc 同步位点 266 | waitTime=3 267 | echo "等待 $waitTime s" 268 | sleep $waitTime; 269 | 270 | # 获取 checkpoint，获取 func 最后一个 echo 271 | local checkPoint=$(getCheckPoint | tail -n 1) 272 | # echo $checkPoint 273 | if [[ "$checkPoint" =~ ^-?[0-9]+$ ]]; then 274 | echo "Checkpoint: $checkPoint is a integer number." 275 | else 276 | echo "Checkpoint: $checkPoint is not a integer number. exit!!" 277 | exit 1 278 | fi 279 | # 比较 checkpoint 和 master tso，如果 checkpoint 大于等于 master tso 280 | if [[ $checkPoint -ge $masterPos ]]; then 281 | echo "checkPoint ($checkPoint) is greater than or equal to masterPos ($masterPos). Breaking the loop." 282 | break # 如果 checkPoint 大于等于 masterPos，跳出循环 283 | else 284 | echo "checkPoint ($checkPoint) is less than masterPos ($masterPos). Continuing the loop." 285 | fi 286 | # 增加迭代器 287 | ctmTmp=$(( ctmTmp + 1 )) 288 | done 289 | # 添加如果循环判断到最后，同步都没有追上，退出脚本 290 | if [[ $ctmTmp -le $ctlNum ]]; then 291 | # 判断结束 292 | echo "判断结束" 293 | else 294 | echo "超过判断次数，退出整个脚本" 295 | exit 1; 296 | fi 297 | } 298 | 299 | # master cdc 任务删除 300 | deleteCdcTask(){ 301 | # 暂停任务 302 | pauseTaskCommand=(~/.tiup/bin/tiup cdc:$VERSION cli changefeed pause --pd=http://$MASTER_PD --changefeed-id=$CHANGEFEED_ID) 303 | # 输出命令 304 | echo "暂停 cdc 同步任务命令: " 305 | echo ${pauseTaskCommand[*]} 306 | # 执行命令 307 | "${pauseTaskCommand[@]}" 308 | # 检查命令执行的返回值 309 | if [ $? -eq 0 ]; then 310 | echo "Pause CDC task command executed successfully." 311 | else 312 | echo "Pause CDC task command failed." 313 | exit 1 314 | fi 315 | # 删除任务 316 | removeTaskCommand=(~/.tiup/bin/tiup cdc:$VERSION cli changefeed remove --pd=http://$MASTER_PD --changefeed-id=$CHANGEFEED_ID) 317 | # 输出命令 318 | echo "暂停 cdc 同步任务命令: " 319 | echo ${removeTaskCommand[*]} 320 | # 执行命令 321 | "${removeTaskCommand[@]}" 322 | # 检查命令执行的返回值 323 | if [ $? -eq 0 ]; then 324 | echo "Remove CDC task command executed successfully." 325 | else 326 | echo "Remove CDC task command failed." 327 | exit 1 328 | fi 329 | } 330 | 331 | createCdcTask(){ 332 | # slave 的 ip 端口 333 | IFS=':' read -r ip port <<< "$SLAVE_DB" 334 | # 获取 slave 当前 TSO 335 | local slaveCommand=(mysql -h "$ip" -u "$SLAVE_USER" -p"$SLAVE_PASSWORD" -P "$port" -e 336 | "show master status;") 337 | # 输出命令 338 | echo ${slaveCommand[*]} 339 | # 执行命令 340 | local slaveResult 341 | slaveResult=$("${slaveCommand[@]}") 342 | local status=$? 343 | # 检查命令执行的返回值 344 | if [ $status -eq 0 ]; then 345 | echo "get Slave tso command executed successfully." 346 | else 347 | echo "get Slave tso command failed." 348 | exit 1 349 | fi 350 | # 结果格式化，取第二行最后一个字段内容，也就是 pos 值 351 | local slavePos=$(echo "$slaveResult" | awk 'NR==2 {print $NF}') 352 | echo "slave 的 pos 为：$slavePos" 353 | 354 | # 从库创建任务 355 | createTaskCommand=(~/.tiup/bin/tiup cdc:$VERSION cli changefeed create --pd=http://$SLAVE_PD \ 356 | --changefeed-id=$CHANGEFEED_ID --sink-uri="$SINK_URI" --config=$CDC_CONFIG --start-ts="$slavePos") 357 | # 输出命令 358 | echo "创建 cdc 同步任务命令: " 359 | echo ${createTaskCommand[*]} 360 | # 执行命令 361 | "${createTaskCommand[@]}" 362 | # 检查命令执行的返回值 363 | if [ $? -eq 0 ]; then 364 | echo "Create CDC task command executed successfully." 365 | else 366 | echo "Create CDC task command failed." 367 | exit 1 368 | fi 369 | } 370 | 371 | # 解锁从库 user 372 | userUnLock() { 373 | local userAccounts=$LOCK_USER 374 | # 解析数据库用户名 375 | IFS=',' read -ra ACCOUNTS <<< "$userAccounts" 376 | for account in "${ACCOUNTS[@]}"; do 377 | echo "UnLock account: $account" 378 | # 解析数据库账号密码 379 | IFS=':' read -r ip port <<< "$SLAVE_DB" 380 | mysql -h "$ip" -u "$SLAVE_USER" -p"$SLAVE_PASSWORD" -P "$port" -e "ALTER USER $account ACCOUNT UNLOCK;FLUSH PRIVILEGES;" 381 | if [ $? -eq 0 ]; then 382 | echo "Successfully Unlocked: $account" 383 | else 384 | echo "Failed to Unlock: $account" 385 | exit 1 386 | fi 387 | done 388 | } 389 | 390 | # 主逻辑函数 391 | main() { 392 | parserArgs "$@" 393 | # 逻辑代码 394 | echo "主库数据库: $MASTER_DB" 395 | echo "主库用户: $MASTER_USER" 396 | echo "主库密码: $MASTER_PASSWORD" 397 | echo "Master PD 服务: $MASTER_PD" 398 | echo "从库数据库: $SLAVE_DB" 399 | echo "从库用户: $SLAVE_USER" 400 | echo "从库密码: $SLAVE_PASSWORD" 401 | echo "Slave PD 服务: $SLAVE_PD" 402 | echo "CDC 版本: $VERSION" 403 | echo "重载列表: $RELOAD_LIST" 404 | echo "Changefeed 标识: $CHANGEFEED_ID" 405 | echo "CDC 配置文件: $CDC_CONFIG" 406 | echo "锁定用户: $LOCK_USER" 407 | echo "新建 cdc 同步 uri: $SINK_URI" 408 | echo "执行模式: $MOD, 409 | 1. 用户锁定 410 | 2. 重启 tidb server 节点 411 | 3. 确认是否有留存连接 412 | 4. 等待 cdc 任务同步追上 413 | 5. 删除 master cdc 任务 414 | 6. 创建从库到主库的同步 415 | 7. 解锁从库应用用户" 416 | # 开始任务 417 | echo "<-------->开始任务<-------->" 418 | # 1. 用户锁定 419 | # userLock; 420 | # 2. 重启 tidb server 节点 421 | # restartTidb; 422 | # 3. 确认是否有留存连接 423 | # checkConnect; 424 | # 4. 等待 cdc 任务同步追上 425 | # checkCdcStatus; 426 | # 5. 删除 master cdc 任务 427 | # deleteCdcTask; 428 | # 6. 创建从库到主库的同步 429 | # createCdcTask; 430 | # 7. 解锁从库应用用户 431 | # userUnLock; 432 | # 根据 mod 模式来执行步骤 433 | # 根据传入的模式执行对应的函数 434 | # 根据传入的模式执行对应的函数 435 | IFS=',' read -ra ADDR <<< "$MOD" 436 | for i in "${ADDR[@]}"; do 437 | case $i in 438 | 1) 439 | echo "Step 1: 用户锁定" 440 | userLock 441 | ;; 442 | 2) 443 | echo "Step 2: 重启 TiDB server 节点" 444 | restartTidb 445 | ;; 446 | 3) 447 | echo "Step 3: 确认是否有留存连接" 448 | checkConnect 449 | ;; 450 | 4) 451 | echo "Step 4: 等待 CDC 任务同步追上" 452 | checkCdcStatus 453 | ;; 454 | 5) 455 | echo "Step 5: 删除 master CDC 任务" 456 | deleteCdcTask 457 | ;; 458 | 6) 459 | echo "Step 6: 创建从库到主库的同步" 460 | createCdcTask 461 | ;; 462 | 7) 463 | echo "Step 7: 解锁从库应用用户" 464 | userUnLock 465 | ;; 466 | *) 467 | echo "无效的选项: $i" 468 | ;; 469 | esac 470 | done 471 | } 472 | 473 | # 脚本入口点 474 | if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then 475 | main "$@" 476 | fi 477 | 478 | # 测试命令 479 | # ./failoverByCDC.sh --masterDB='10.1xxx02.58.180:4000' --masterUser='root' --masterPassword='tidbxx' --masterPD='10.xxx:2379' --slaveDB='10.102.5xx:4000' --slaveUser='root' --slavePassword='tidb@123' --slavePD='10.1xxx1:2379' --version='v7.1.4' --reloadList='172.xxx:4000,172.xxx.47:4000' --changefeedID='usexbackup' --cdcConfig='cdc.conf' --lockUser="xin@'%',px@'%',tidb@'%'" --mod='1,2,3' --clusterName="tidb-test" --sink-uri="tidb://root:w@-xxxxxxR@172.20.xx:4000/&transaction-atomicity=none" 480 | -------------------------------------------------------------------------------- /out_user.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import argparse 5 | import pymysql 6 | import os 7 | 8 | ## If Python is version 2.7, encoding problems can reload sys configuration 9 | try: 10 | import sys 11 | 12 | reload(sys) 13 | sys.setdefaultencoding('utf-8') 14 | except: 15 | pass 16 | 17 | 18 | def main(): 19 | args = parse_args() 20 | _set_pwd = "select user,host,authentication_string from mysql.user;" 21 | context = mysql_execute(_set_pwd) 22 | for i in context: 23 | write_file(i['user'], i['host'], i['authentication_string']) 24 | print("User: '{}'@'{}' is OK".format(i['user'], i['host'])) 25 | 26 | 27 | def write_file(user_name, hosts, authentication_string): 28 | # update and grants 29 | _show_grants = "SHOW GRANTS FOR '{}'@'{}'".format(user_name, hosts) 30 | grant_context = mysql_execute(_show_grants) 31 | _set_pwd = "update mysql.user set `authentication_string`='{}' where user='{}' and host='{}';".format( 32 | authentication_string, user_name, hosts) 33 | _create_user = "create user '{}'@'{}';".format(user_name, hosts) 34 | with open('tidb_users.sql', 'a+') as f: 35 | f.write("-- '{}'@'{}' \n".format(user_name, hosts)) 36 | f.write("{} \n".format(_create_user)) 37 | f.write("{} \n".format(_set_pwd)) 38 | for i in grant_context: 39 | i = list(i.values()) 40 | f.write("{};\n".format(i[0])) 41 | f.write("\n") 42 | 43 | 44 | def mysql_execute(*_sql): 45 | # Connect to MySQL and execute SQL commands 46 | args = parse_args() 47 | config = { 48 | "host": args.mysql, 49 | "port": int(args.port), 50 | "user": args.user, 51 | "password": args.password, 52 | "charset": 'utf8mb4', 53 | "cursorclass": pymysql.cursors.DictCursor 54 | } 55 | 56 | connection = pymysql.connect(**config) 57 | cursor = connection.cursor() 58 | try: 59 | for sql in _sql: 60 | cursor.execute(sql) 61 | content = cursor.fetchall() 62 | connection.commit() 63 | except: 64 | print( 65 | "SQL {} execution failed~ \nPlease check table or database exists or not!" 66 | .format(_sql)) 67 | finally: 68 | cursor.close() 69 | connection.close() 70 | 71 | return content 72 | 73 | 74 | def parse_args(): 75 | # Incoming parameters 76 | parser = argparse.ArgumentParser( 77 | description="Update table statistics manually") 78 | parser.add_argument("-P", 79 | dest="port", 80 | help="tidb port, default: 4000", 81 | default=4000) 82 | parser.add_argument("-H", 83 | dest="mysql", 84 | help="Database address, default: 127.0.0.1", 85 | default="127.0.0.1") 86 | parser.add_argument("-u", 87 | dest="user", 88 | help="Database account, default: root", 89 | default="root") 90 | parser.add_argument("-p", 91 | dest="password", 92 | help="Database password, default: null", 93 | default="") 94 | 95 | args = parser.parse_args() 96 | 97 | return args 98 | 99 | 100 | if __name__ == "__main__": 101 | main() 102 | -------------------------------------------------------------------------------- /report_ccsv/query.lst: -------------------------------------------------------------------------------- 1 | total-session max|@|max_over_time(sum by (tidb_server_connections)(tidb_server_connections)[2h:])|@| 2 | active-session max|@|max_over_time(sum by (tidb_server_tokens)(tidb_server_tokens)[2h:])|@| 3 | QPS max|@|max_over_time(sum(rate(tidb_server_query_total{result="OK"}[1m]))[2h:])|@| 4 | Transaction-OPS|@|max_over_time(sum(rate(tidb_session_transaction_duration_seconds_count{type="commit",txn_mode="optimistic"}[1m]))[2h:])|@| 5 | 60 duration|@|histogram_quantile(0.60, sum(rate(tidb_server_handle_query_duration_seconds_bucket[2h])) by (le)) * 1000|@|ms 6 | 99 duration|@|histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[2h])) by (le)) * 1000|@|ms 7 | Failed-Query-OPM (TOTAL)|@|max_over_time(sum(increase(tidb_server_execute_error_total[1m]))[2h:])|@| 8 | TiDB-cpu-max|@|max_over_time(max(irate(process_cpu_seconds_total{job="tidb"}[30s]))[2h:]) * 100|@|% 9 | TiDB-Memory-max|@|max_over_time(max(sum(process_resident_memory_bytes{job="tidb"}))[2h:]) / 1024 / 1024 / 1024|@|GB 10 | TiKV-cpu-max|@|max_over_time(max(irate(process_cpu_seconds_total{job="tikv"}[30s]))[2h:]) * 100|@|% 11 | TiKV-Memory-max|@|max_over_time(max(sum(process_resident_memory_bytes{job="tikv"}))[2h:]) / 1024 / 1024 / 1024|@|GB 12 | Network-Traffic-max-out|@|max_over_time(max(irate(node_network_transmit_bytes_total{device!="lo"}[1m]))[2h:]) / 1024 / 1024|@|MIB 13 | Number-of-Regions|@|max_over_time(sum(pd_cluster_status{type="leader_count"})[2h:])|@| 14 | TiCDC-standby-lag|@|max_over_time(max(ticdc_owner_checkpoint_ts_lag{changefeed="standby-task"})[2h:])|@|s 15 | TiCDC-neardb-lag|@|max_over_time(max(ticdc_owner_checkpoint_ts_lag{changefeed="neardb-task"})[2h:])|@|s 16 | TiCDC-kafka-lag|@|max_over_time(max(ticdc_owner_checkpoint_ts_lag{changefeed="kafka-task"})[2h:])|@|s 17 | TiCDC-one-miiror-lag|@|max_over_time(max(ticdc_owner_checkpoint_ts_lag{changefeed="one-mirror-task"})[2h:])|@|s -------------------------------------------------------------------------------- /report_ccsv/report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import requests 5 | import argparse 6 | import csv 7 | from datetime import datetime, timedelta 8 | 9 | def main(): 10 | args = parse_args() 11 | # 读取参数文件 12 | file_path = 'query.lst' 13 | parameters = [] 14 | 15 | with open(file_path, 'r') as file: 16 | lines = file.readlines() 17 | 18 | # 遍历每一行，解析参数 19 | for line in lines: 20 | parts = line.split('|@|') 21 | if len(parts) == 3: 22 | parameters.append({ 23 | 'percentile': parts[0].strip(), 24 | 'query': parts[1].strip(), 25 | 'unit': parts[2].strip() 26 | }) 27 | file.close() 28 | 29 | # 读取时间文件 30 | file_path = 'time.lst' 31 | time_list = [] 32 | 33 | with open(file_path, 'r') as file: 34 | lines = file.readlines() 35 | 36 | # 遍历每一行，将时间字符串添加到列表中 37 | for line in lines: 38 | time_list.append(line.strip()) # 去除换行符并添加到列表 39 | file.close() 40 | 41 | # print(parameters) 42 | url = args.plhost 43 | # query = ''' 44 | # histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[2h])) by (le)) * 1000 45 | # ''' # 替换为你的PromQL查询语句 46 | # time_str='2023-11-18T19:00:00.000Z' 47 | ct_list = {} 48 | for time_str in time_list: 49 | time_str = "{}T{}Z".format(args.date ,time_str) 50 | # values = execute_query(url, query, time_str) 51 | # print(values) 52 | for parameter in parameters: 53 | query = parameter['query'] 54 | percentile = parameter['percentile'] 55 | unit = parameter['unit'] 56 | values = execute_query(url, query, time_str) 57 | # print("Time:{}: 指标：{},values:{}{}".format(time_str, percentile, values, unit)) 58 | if percentile in ct_list: 59 | ct_list[percentile] = ct_list[percentile] + "|@|" + values + unit 60 | else: 61 | ct_list[percentile] = values + unit 62 | 63 | with open('data.csv', 'w', newline='') as csvfile: 64 | for key, ct in ct_list.items(): 65 | data = key + "|@|" + ct 66 | data_list = data.split('|@|') 67 | writer = csv.writer(csvfile) 68 | writer.writerow(data_list) 69 | 70 | print("all sucessfull") 71 | 72 | def parse_args(): 73 | # 创建 ArgumentParser 对象 74 | parser = argparse.ArgumentParser(description='') 75 | # 添加命令行参数 76 | parser.add_argument('--plhost', '-H', type=str, default='127.0.0.1:9090', help='prometheus host:port') 77 | parser.add_argument('--date', '-dt', type=str, default="2023-11-19", help='date time: 2023-11-19') 78 | # 解析命令行参数 79 | args = parser.parse_args() 80 | return args 81 | 82 | """ 83 | Prometheus API 84 | 输入： 85 | API IP address and port 86 | query 87 | time 88 | 89 | 返回： 90 | 结果 91 | """ 92 | def execute_query(url, query, time_str): 93 | prometheus_url = "http://{}".format(url) # 替换为你的Prometheus实例的URL 94 | # query = ''' 95 | # histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[2h])) by (le)) * 1000 96 | # ''' # 替换为你的PromQL查询语句 97 | 98 | # 构建Prometheus查询API的URL 99 | api_url = f"{prometheus_url}/api/v1/query?query={query}" 100 | # time_str='2023-11-18T19:00:00.000Z' 101 | # 解析时间字符串为datetime对象（假设该时间是UTC时间） 102 | time = datetime.strptime(time_str, "%Y-%m-%dT%H:%M:%S.%fZ") 103 | # 将时间标记为UTC时区 104 | time = time - timedelta(hours=8) 105 | 106 | # 转换为字符串格式 107 | result_time_str = time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") 108 | 109 | api_url = api_url + '&time=' + result_time_str 110 | 111 | # 发送GET请求获取查询结果 112 | response = requests.get(api_url) 113 | 114 | # 检查响应状态码 115 | if response.status_code == 200: 116 | result = response.json() # 获取JSON格式的结果 117 | # print(result) # 处理查询结果，这里打印结果 118 | # print('99 duration 为 {} ms'.format(result['data']['result'][0]['value'][1])) 119 | else: 120 | print(f"Failed to execute query. Status code: {response.status_code}") 121 | 122 | try: 123 | rt = result['data']['result'][0]['value'][1] 124 | rt = round(float(rt), 2) 125 | rt = str(rt) 126 | except Exception as e: 127 | rt = "-" 128 | 129 | return rt 130 | 131 | 132 | 133 | if __name__ == "__main__": 134 | # execute_query() 135 | main() 136 | -------------------------------------------------------------------------------- /report_ccsv/time.lst: -------------------------------------------------------------------------------- 1 | 02:00:00.000 2 | 04:00:00.000 3 | 06:00:00.000 4 | 08:00:00.000 5 | 10:00:00.000 6 | 12:00:00.000 7 | 14:00:00.000 8 | 16:00:00.000 9 | 18:00:00.000 10 | 20:00:00.000 11 | 22:00:00.000 12 | 23:59:59.000 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | mysql_connector_repackaged==0.3.1 2 | PyMySQL==1.0.2 3 | requests==2.27.1 4 | -------------------------------------------------------------------------------- /restart_tidb_tiflash.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | # import subprocess 5 | import os 6 | import time 7 | import argparse 8 | import requests 9 | import json 10 | 11 | 12 | def main(): 13 | args = parse_args() 14 | while True: 15 | # 异常下尝试重试次数 16 | num = 3 17 | # 获取 tiflash 状态 18 | stat = wileStatus(args.promAddr, args.flashAddr, num) 19 | if stat == num: 20 | content = "tiflash 是否重启，已经连续判断 {} 次，进行 tidb-server 重启".format(num) 21 | logsFile(content) 22 | # 数据库重启 23 | rt_tidb(args.tidbPort) 24 | # 重启后等待一定时间进入下个轮回 25 | time.sleep(300) 26 | else: 27 | pass 28 | 29 | # 固定时间轮询 30 | time.sleep(7) 31 | 32 | 33 | def rt_tidb(port): 34 | # 根据端口拼 service 文件名 35 | serviceFileName = "tidb-{}.service".format(port) 36 | command = "sudo systemctl restart {}".format(serviceFileName) 37 | try: 38 | os.system(command) 39 | content = "重启 tidb 成功：{}".format(command) 40 | logsFile(content) 41 | except all: 42 | logsFile("重启异常: {}".format(all)) 43 | 44 | 45 | def wileStatus(promAddr, flashAddr, num=3): 46 | wc = 0 47 | while wc < num: 48 | # 如果发现 tiflash 异常，默认进行 3 次重取操作 49 | stat = tiflashStatus(promAddr, flashAddr) 50 | # print("断点: {}".format(stat)) 51 | if stat == 0: 52 | # 状态为 0，代表 tiflash 正常 53 | content = "tiflash 状态正常：{} {}".format(promAddr, stat) 54 | logsFile(content) 55 | return True 56 | elif stat == 1: 57 | wc += 1 58 | content = "tiflash 可能发生重启：{} {}，开始多次判断：第 {} 次".format( 59 | promAddr, stat, wc) 60 | logsFile(content) 61 | # 等待一定时间重新获取 tiflash 状态 62 | time.sleep(3) 63 | else: 64 | logsFile("其他情况，stat 为：{}".format(stat)) 65 | break 66 | 67 | return wc 68 | 69 | 70 | def logsFile(content): 71 | fileName = time.strftime("%Y-%m-%d", time.localtime()) + "-status.log" 72 | dirs = "logs" 73 | # 判断有没有 logs 目录，如果没有就创建一个 74 | if not os.path.exists(dirs): 75 | os.makedirs(dirs) 76 | # 日志文件添加 logs 目录 77 | fileName = os.path.join(dirs, fileName) 78 | timeLocal = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 79 | with open(fileName, "a+") as f: 80 | f.write("{}: {} \n".format(timeLocal, content)) 81 | 82 | 83 | def tiflashStatus(promAddr, flashAddr): 84 | # 5min 内发现 tiflash 重启 85 | cT = 5 86 | promeQuery = "changes(tiflash_proxy_process_start_time_seconds{instance='%s',job='tiflash'}[%sm])" % ( 87 | flashAddr, cT) 88 | stat = checkProm(promAddr, promeQuery) 89 | stat = json.loads(stat) 90 | # 从结果取出结果 0 或者 1 91 | dataLen = len(stat['data']['result']) 92 | if dataLen == 0: 93 | logsFile("tiflash 是否重启无法捕获，tiflash 可能超过 {}min 未启动。stat 状态为：{}".format( 94 | cT, stat)) 95 | else: 96 | status = int(stat['data']['result'][0]['value'][1]) 97 | 98 | return status 99 | 100 | 101 | def checkProm(prometheus_address, query): 102 | # 请求 Prometheus 执行 Prometheus query，返回结果 103 | try: 104 | response = requests.get('http://%s/api/v1/query' % prometheus_address, 105 | params={ 106 | 'query': query 107 | }).text 108 | return response 109 | except all: 110 | logsFile("访问 Prometheus 异常", all) 111 | return None 112 | 113 | 114 | def parse_args(): 115 | # Incoming parameters 116 | parser = argparse.ArgumentParser( 117 | description="如果 tiflash 重启，重启本地 tidb-server") 118 | parser.add_argument("-P", 119 | dest="tidbPort", 120 | help="tidb port, default: 4000", 121 | default=4000) 122 | parser.add_argument("-ph", 123 | dest="promAddr", 124 | help="Prometheus ip 和端口, default: 127.0.0.1:9090", 125 | default="127.0.0.1:9090") 126 | parser.add_argument( 127 | "-th", 128 | dest="flashAddr", 129 | help="tiflash ip 和 proxy status 端口, default: 127.0.0.1:20292", 130 | default="127.0.0.1:20292") 131 | 132 | args = parser.parse_args() 133 | 134 | return args 135 | 136 | 137 | if __name__ == "__main__": 138 | main() 139 | -------------------------------------------------------------------------------- /split_hot_region.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import json 5 | import argparse 6 | import shlex 7 | import subprocess 8 | 9 | 10 | def main(): 11 | args = parse_args() 12 | r_regions, w_regions = region_messges() 13 | print("The top Read {} Region is {}").format(args.top, r_regions) 14 | print("The top Write {} Region is {}").format(args.top, w_regions) 15 | regions = r_regions + "," + w_regions 16 | _region = raw_input( 17 | "Please Enter the region you want to split(Such as 1,2,3, default is None): ") or None 18 | if _region is None: 19 | print("Exit") 20 | else: 21 | _region = _region.split(",") 22 | try: 23 | for region_id in _region: 24 | if region_id in regions: 25 | Split_region(region_id) 26 | else: 27 | print('Please check the Region {} is in Top').format(region_id) 28 | except: 29 | print("Please enter the correct content! Such as 1,2,3") 30 | 31 | 32 | def Split_region(region_id): 33 | args = parse_args() 34 | _split_cmd = "../resources/bin/pd-ctl -u http://{} -d operator add split-region {}".format( 35 | args.pd, region_id) 36 | try: 37 | _sc = subprocess.check_output(shlex.split(_split_cmd)) 38 | print("Split Region {} Command executed {}").format(region_id, _sc) 39 | except: 40 | print("Split Region {} is faild").format(region_id) 41 | 42 | 43 | def region_messges(): 44 | args = parse_args() 45 | h_r_region = "../resources/bin/pd-ctl -u http://{} -d region topread {}".format( 46 | args.pd, args.top) 47 | h_w_region = "../resources/bin/pd-ctl -u http://{} -d region topwrite {}".format( 48 | args.pd, args.top) 49 | _r = subprocess.check_output(shlex.split(h_r_region)) 50 | _w = subprocess.check_output(shlex.split(h_w_region)) 51 | try: 52 | r_regions = json.loads(_r) 53 | w_regions = json.loads(_w) 54 | except: 55 | print("Json format parsing error, please check if the parameters are correct") 56 | _r_regions = [] 57 | print('--------------------TOP {} Read region messegs--------------------').format( 58 | args.top) 59 | for _r_r in r_regions["regions"]: 60 | _flow = round(_r_r.get("read_bytes", 0) / 1024 / 1024) 61 | _db_name, _table_name = table_db_info(_r_r["id"]) 62 | print( 63 | "leader and region id is [{}] [{}], Store id is {} and IP is {}, and Flow valuation is {}MB," 64 | " DB name is {}, table name is {}").format( 65 | _r_r["leader"]["id"], _r_r["id"], _r_r["leader"]["store_id"], 66 | store_info(_r_r["leader"]["store_id"]), _flow, _db_name, 67 | _table_name) 68 | _r_regions.append(str(_r_r["id"])) 69 | 70 | _w_regions = [] 71 | print('--------------------TOP {} Write region messegs--------------------').format( 72 | args.top) 73 | for _W_r in w_regions["regions"]: 74 | _flow = round(_W_r.get("written_bytes", 0) / 1024 / 1024) 75 | _db_name, _table_name = table_db_info(_r_r["id"]) 76 | print( 77 | "leader and region id is [{}] [{}], Store id is {} and IP is {}, and Flow valuation is {}MB," 78 | " DB name is {}, table name is {}").format( 79 | _W_r["leader"]["id"], _W_r["id"], _W_r["leader"]["store_id"], 80 | store_info(_W_r["leader"]["store_id"]), _flow, _db_name, 81 | _table_name) 82 | _w_regions.append(str(_W_r["id"])) 83 | 84 | _r_regions = ','.join(_r_regions) 85 | _w_regions = ','.join(_w_regions) 86 | 87 | return _r_regions, _w_regions 88 | 89 | 90 | def table_db_info(region_id): 91 | args = parse_args() 92 | httpAPI = "http://{}/regions/{}".format(args.tidb, region_id) 93 | webContent = subprocess.check_output(["curl", "-sl", httpAPI]) 94 | _table_info = json.loads(webContent) 95 | if _table_info["frames"] is not None: 96 | for _info in _table_info["frames"]: 97 | _db_name = _info.get("db_name", "null") 98 | _table_name = _info.get("table_name", "null") 99 | else: 100 | _db_name = "NUlL" 101 | _table_name = 'NUll, This table has been drop or truncate' 102 | return _db_name, _table_name 103 | 104 | 105 | def store_info(store_id): 106 | args = parse_args() 107 | _cmd = "../resources/bin/pd-ctl -u http://{} -d store {}".format(args.pd, 108 | store_id) 109 | _sc = subprocess.check_output(shlex.split(_cmd)) 110 | _store_info = json.loads(_sc) 111 | info = _store_info["store"]["address"] 112 | return info 113 | 114 | 115 | def parse_args(): 116 | parser = argparse.ArgumentParser( 117 | description="Show the hot region details and splits") 118 | parser.add_argument("--th", 119 | dest="tidb", 120 | help="tidb status url, default: 127.0.0.1:10080", 121 | default="127.0.0.1:10080") 122 | parser.add_argument("--ph", 123 | dest="pd", 124 | help="pd status url, default: 127.0.0.1:2379", 125 | default="127.0.0.1:2379") 126 | parser.add_argument("top", help="the top read/write region number") 127 | args = parser.parse_args() 128 | return args 129 | 130 | 131 | if __name__ == "__main__": 132 | main() 133 | -------------------------------------------------------------------------------- /split_table_region.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import json 5 | import argparse 6 | import shlex 7 | import subprocess 8 | 9 | 10 | def main(): 11 | args = parse_args() 12 | info = table_region_parser() 13 | _input = raw_input( 14 | 'We will Split region: {}, y/n(default is yes): '.format( 15 | info[args.table])) or 'y' 16 | if _input == 'y': 17 | for _region_id in info[args.table]: 18 | Split_region(int(_region_id)) 19 | else: 20 | print('exit') 21 | 22 | 23 | def table_region_parser(): 24 | args = parse_args() 25 | httpAPI = "http://{}/tables/{}/{}/regions".format(args.tidb, args.database, 26 | args.table) 27 | 28 | webContent = subprocess.check_output(["curl", "-sl", httpAPI]) 29 | region_info = json.loads(webContent) 30 | table_id = region_info['id'] 31 | region_id = [] 32 | index_region_id = [] 33 | Info = {} 34 | print("Table {} Info:").format(args.table) 35 | for regions in region_info['record_regions']: 36 | region_id.append(str(regions["region_id"])) 37 | ip_info = store_info(regions['leader']['store_id']) 38 | print( 39 | " Region id is {}, leader id is {}, Store id is {} and IP is {}").format( 40 | regions["region_id"], regions['leader']['id'], 41 | regions['leader']['store_id'], ip_info) 42 | Info[args.table] = region_id 43 | 44 | if region_info['indices'] is not None: 45 | print("\nTable {} Index info:").format(args.table) 46 | for index_info in region_info['indices']: 47 | print("Index {} info, id is {}:").format(index_info['name'], 48 | index_info['id']) 49 | for index_region in index_info['regions']: 50 | index_region_id.append(str(index_region['region_id'])) 51 | ip_info = store_info(index_region['leader']['store_id']) 52 | print( 53 | ' Region id is {} and leader id is {}, Store id is {} and IP is {}').format( 54 | index_region['region_id'], 55 | index_region['leader']['id'], 56 | index_region['leader']['store_id'], ip_info) 57 | Info[index_info['name']] = index_region_id 58 | return Info 59 | 60 | 61 | def store_info(store_id): 62 | args = parse_args() 63 | _cmd = "../resources/bin/pd-ctl -u http://{} -d store {}".format(args.pd, 64 | store_id) 65 | _sc = subprocess.check_output(shlex.split(_cmd)) 66 | _store_info = json.loads(_sc) 67 | info = _store_info["store"]["address"] 68 | return info 69 | 70 | 71 | def Split_region(region_id): 72 | args = parse_args() 73 | _split_cmd = "../resources/bin/pd-ctl -u http://{} -d operator add split-region {} --policy=approximate".format( 74 | args.pd, region_id) 75 | try: 76 | _sc = subprocess.check_output(shlex.split(_split_cmd)) 77 | print("Split Region {} Command executed {}").format(region_id, _sc) 78 | except: 79 | print("Split Region {} is faild").format(region_id) 80 | 81 | 82 | def parse_args(): 83 | parser = argparse.ArgumentParser( 84 | description="Show the hot region details and splits") 85 | parser.add_argument("--th", 86 | dest="tidb", 87 | help="tidb status url, default: 127.0.0.1:10080", 88 | default="127.0.0.1:10080") 89 | parser.add_argument("--ph", 90 | dest="pd", 91 | help="pd status url, default: 127.0.0.1:2379", 92 | default="127.0.0.1:2379") 93 | parser.add_argument("database", help="database name") 94 | parser.add_argument("table", help="table name") 95 | args = parser.parse_args() 96 | return args 97 | 98 | 99 | if __name__ == "__main__": 100 | main() 101 | -------------------------------------------------------------------------------- /sync_diff.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import argparse 5 | import pymysql 6 | import time 7 | import json 8 | import threading 9 | 10 | 11 | class MyThread(threading.Thread): 12 | def __init__(self, func, args=()): 13 | super(MyThread, self).__init__() 14 | self.func = func 15 | self.args = args 16 | 17 | def run(self): 18 | self.result = self.func(*self.args) 19 | 20 | def get_result(self): 21 | try: 22 | return self.result 23 | except Exception: 24 | return None 25 | 26 | 27 | def main(): 28 | args = parse_args() 29 | if args.tables is None: 30 | db_list = args.database + "," 31 | dbname = tuple(db_list.split(",")) 32 | tb_sql = "select TABLE_SCHEMA,TABLE_NAME from INFORMATION_SCHEMA.tables where TABLE_SCHEMA in {}".format( 33 | dbname) 34 | t_d_list = mysql_execute("f", tb_sql) 35 | else: 36 | pass 37 | tso = mysql_execute("t", "select checkPoint from tidb_binlog.checkpoint") 38 | tso = json.loads(tso[0]["checkPoint"]) 39 | ftso = tso["ts-map"]["primary-ts"] 40 | ttso = tso["ts-map"]["secondary-ts"] 41 | for tb in t_d_list: 42 | tb_name = tb["TABLE_NAME"] 43 | dbname = tb["TABLE_SCHEMA"] 44 | if args.verification == "xor": 45 | _sql = "select table_name, concat('select bit_xor(CAST(CRC32(concat_ws(',group_concat('`',`COLUMN_NAME`,'`'),', concat(',group_concat('ISNULL(`',`COLUMN_NAME`,'`)'),'))) AS unsigned)) as b_xor from `', table_name, '`') as _sql from `COLUMNS` where TABLE_SCHEMA='{}' and table_name='{}' and data_type not in ('json', 'timestamp') group by table_name".format( 46 | dbname, tb_name) 47 | _sql = mysql_execute("f", "use INFORMATION_SCHEMA", _sql) 48 | bit_xor_sql = _sql[0]["_sql"] 49 | else: 50 | bit_xor_sql = "admin checksum table `{}`".format(tb_name) 51 | start = time.time() 52 | data = [] 53 | threads = [] 54 | for mod in ["f", "t"]: 55 | if mod == "f": 56 | bit_xor = MyThread(check_table, 57 | args=(dbname, bit_xor_sql, ftso, mod)) 58 | else: 59 | bit_xor = MyThread(check_table, 60 | args=(dbname, bit_xor_sql, ttso, mod)) 61 | threads.append(bit_xor) 62 | bit_xor.start() 63 | for t in threads: 64 | t.join() 65 | data.append(t.get_result()) 66 | end = time.time() 67 | f_bit_xor_sum = data[0][1] 68 | t_bit_xor_sum = data[1][1] 69 | if args.verification == "xor": 70 | if f_bit_xor_sum == t_bit_xor_sum: 71 | print( 72 | "Check sucessfull, Cost time is {}s, DB name is: {}, Table name is:{}, bit xor:{}" 73 | .format(end - start, dbname, tb_name, f_bit_xor_sum)) 74 | else: 75 | print( 76 | "Check failed, Cost time is {}s, DB name is:{},Table name is:{}, f-bit_xor is:{}, t-bit_xor is:{}" 77 | .format(end - start, dbname, tb_name, f_bit_xor_sum, 78 | t_bit_xor_sum)) 79 | else: 80 | if f_bit_xor_sum == t_bit_xor_sum: 81 | print( 82 | "Check successful, Cost time is {}s, DB name is:{}, Table name is:{}, kvs is {}" 83 | .format(end - start, dbname, tb_name, f_bit_xor_sum)) 84 | else: 85 | print( 86 | "Check failed, Cost time is {}s, DB name is:{},Table name is:{}, f-kvs is {}, t-kvs is {}" 87 | .format(end - start, dbname, tb_name, f_bit_xor_sum, 88 | t_bit_xor_sum)) 89 | 90 | 91 | def check_table(db_name, bit_xor_sql, tso, mode): 92 | args = parse_args() 93 | # tso = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 94 | set_engine = "set tidb_isolation_read_engines='tikv, tidb';" 95 | set_scan = "set tidb_distsql_scan_concurrency = {}".format(args.thread) 96 | set_time = "set tidb_snapshot='{}'".format(tso) 97 | _mode = args.mode.split(',', 1) 98 | if mode == "f" and _mode[0].strip() == "tidb" and _mode[1].strip( 99 | ) == "tidb": 100 | content = mysql_execute(mode, "use {}".format(db_name), set_time, 101 | set_scan, set_engine, bit_xor_sql) 102 | elif mode == "f" and _mode[0].strip() == "mysql" or _mode[1].strip( 103 | ) == "mysql": 104 | content = mysql_execute(mode, "use {}".format(db_name), bit_xor_sql) 105 | elif mode == "t" and _mode[0].strip() == "tidb" and _mode[1].strip( 106 | ) == "tidb": 107 | content = mysql_execute(mode, "use {}".format(db_name), set_time, 108 | set_scan, set_engine, bit_xor_sql) 109 | elif mode == "t" and _mode[0].strip() == "mysql" or _mode[1].strip( 110 | ) == "mysql": 111 | content = mysql_execute(mode, "use {}".format(db_name), bit_xor_sql) 112 | else: 113 | print("unknown~") 114 | if args.verification == "xor": 115 | _result = content[0]["b_xor"] 116 | else: 117 | _result = content[0]["Total_kvs"] 118 | 119 | return mode, _result 120 | 121 | 122 | def mysql_execute(mode, *_sql): 123 | # connect mysql 124 | args = parse_args() 125 | if mode == 'f': 126 | host = args.fmysql.split(':', 1)[0] 127 | port = int(args.fmysql.split(':', 1)[1]) 128 | 129 | config = { 130 | "host": host, 131 | "user": args.fuser, 132 | "password": args.fpassword, 133 | "port": port, 134 | "charset": 'utf8mb4', 135 | "cursorclass": pymysql.cursors.DictCursor 136 | } 137 | else: 138 | args = parse_args() 139 | host = args.tmysql.split(':', 1)[0] 140 | port = int(args.tmysql.split(':', 1)[1]) 141 | 142 | config = { 143 | "host": host, 144 | "user": args.tuser, 145 | "password": args.tpassword, 146 | "port": port, 147 | "charset": 'utf8mb4', 148 | "cursorclass": pymysql.cursors.DictCursor 149 | } 150 | 151 | connection = pymysql.connect(**config) 152 | cursor = connection.cursor() 153 | cursor.execute("set group_concat_max_len=10240000") 154 | for sql in _sql: 155 | cursor.execute(sql) 156 | content = cursor.fetchall() 157 | connection.commit() 158 | connection.close() 159 | 160 | return content 161 | 162 | 163 | def parse_args(): 164 | parser = argparse.ArgumentParser(description="Check tables") 165 | parser.add_argument( 166 | "-hf", 167 | dest="fmysql", 168 | help="Source database address and port, default: 127.0.0.1:4000", 169 | default="127.0.0.1:4000") 170 | parser.add_argument("-uf", 171 | dest="fuser", 172 | help="Source database account, default: root", 173 | default="root") 174 | parser.add_argument("-pf", 175 | dest="fpassword", 176 | help="Source database password, default: null", 177 | default="123456") 178 | parser.add_argument( 179 | "-ht", 180 | dest="tmysql", 181 | help="Target database address and port, default: 127.0.0.1:4000", 182 | default="127.0.0.1:4000") 183 | parser.add_argument("-ut", 184 | dest="tuser", 185 | help="Target database account, default: root", 186 | default="root") 187 | parser.add_argument("-pt", 188 | dest="tpassword", 189 | help="Target database password, default: null", 190 | default="123456") 191 | parser.add_argument( 192 | "-d", 193 | dest="database", 194 | help="Database name, for example: test,tmp, default: None", 195 | default="tmp,tmp1") 196 | parser.add_argument( 197 | "-t", 198 | dest="tables", 199 | help="Table name, for example: tmp.t,tmp.t1, default: None", 200 | default=None) 201 | parser.add_argument( 202 | "-T", 203 | dest="thread", 204 | help= 205 | "set tidb_distsql_scan_concurrency, for example: 200, default: 200", 206 | default=200) 207 | parser.add_argument( 208 | "-m", 209 | dest="mode", 210 | help= 211 | "Compare database types, for example: tidb,mysql, default: tidb,tidb", 212 | default="tidb,tidb") 213 | parser.add_argument( 214 | "-v", 215 | dest="verification", 216 | help="Verification method, for example: checksum, default: xor", 217 | default="xor") 218 | 219 | args = parser.parse_args() 220 | 221 | return args 222 | 223 | 224 | if __name__ == "__main__": 225 | main() 226 | -------------------------------------------------------------------------------- /sync_diff_T2T.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import argparse # 导入用于解析命令行参数的模块 5 | import json # 导入用于处理JSON数据的模块 6 | import pymysql # 导入用于操作MySQL数据库的模块 7 | import os # 导入用于处理文件和目录路径的模块 8 | import shlex # 导入用于解析字符串为命令的模块 9 | import subprocess # 导入用于执行外部命令的模块 10 | from datetime import datetime # 导入处理日期和时间的模块 11 | import sys # 导入用于与Python解释器交互的模块 12 | 13 | def main(): 14 | # 获取当前脚本的绝对路径 15 | script_path = os.path.abspath(__file__) 16 | 17 | # 输出当前脚本的绝对路径 18 | print("当前脚本的绝对路径:", script_path) 19 | script_directory = os.path.dirname(script_path) 20 | 21 | args = parse_args() # 解析命令行参数并存储在args变量中 22 | # 获取上下游 tso mapping 23 | masterTso, slaveTso = parseTso(args.slaveHost, args.slaveUser, args.slavePassword) # 解析上下游数据库的时间戳信息 24 | # 生成配置文件 25 | # 获取当前时间 26 | current_time = datetime.now() 27 | # 格式化时间为精确到秒的字符串 28 | formatted_time = current_time.strftime("%Y%m%d%H%M%S") 29 | outDir = os.path.join(script_directory, formatted_time) # 构建输出目录路径 30 | # 格式化库列表 31 | if len(args.DatabaseList) > 0: 32 | dbList = ['"{}.*"'.format(db) for db in args.DatabaseList.split(',')] 33 | dbList = ",".join(dbList) 34 | else: 35 | dbList = "" 36 | 37 | # 生成配置 38 | formatConfig(args.threadCount, args.masterHost, args.masterUser, args.masterPassword, masterTso, args.slaveHost, args.slaveUser, args.slavePassword, slaveTso, outDir, dbList) # 生成配置文件 39 | 40 | # 执行命令 41 | binaryPath = os.path.join(args.binaryPath, "sync_diff_inspector") 42 | command = "{} --config=./config.toml".format(binaryPath) # 构建执行命令 43 | # 命令 print 44 | print("执行命令：{}".format(command)) 45 | # update gc: changeGc("127.0.0.1:4100", "root", "root", "12h") 46 | print("update gc for master") 47 | masterGcTime = changeGc(args.masterHost, args.masterUser, args.masterPassword, args.gcTime) 48 | print("update gc for slave") 49 | slaveGcTime = changeGc(args.slaveHost, args.slaveUser, args.slavePassword,args.gcTime) 50 | try: 51 | result = subprocess.check_output(shlex.split(command)) # 执行命令并捕获输出 52 | print("执行结果:", result) 53 | except subprocess.CalledProcessError as e: 54 | print(e.output.decode('utf-8')) 55 | # Change back to gc time 56 | print("diff Error, Change back to gc time") 57 | print("update gc for master") 58 | changeGc(args.masterHost, args.masterUser, args.masterPassword, masterGcTime) 59 | print("update gc for slave") 60 | changeGc(args.slaveHost, args.slaveUser, args.slavePassword,slaveGcTime) 61 | exit(1) 62 | 63 | # Change back to gc time 64 | print("update gc for master") 65 | changeGc(args.masterHost, args.masterUser, args.masterPassword, masterGcTime) 66 | print("update gc for slave") 67 | changeGc(args.slaveHost, args.slaveUser, args.slavePassword,slaveGcTime) 68 | 69 | def formatConfig(threadCount, masterHost, masterUser, masterPassword, masterTso, slaveHost, slaveUser, slavePassword, slaveTso, outDir, dbList): 70 | masterPort = int(masterHost.split(":",1)[1]) 71 | slavePort = int(slaveHost.split(":",1)[1]) 72 | config = """ 73 | # Diff Configuration. 74 | 75 | ######################### Global config ######################### 76 | 77 | # how many goroutines are created to check data 78 | check-thread-count = {} 79 | 80 | # set false if just want compare data by checksum, will skip select data when checksum is not equal. 81 | # set true if want compare all different rows, will slow down the total compare time. 82 | export-fix-sql = true 83 | 84 | # ignore check table's data 85 | check-struct-only = false 86 | 87 | ######################### Databases config ######################### 88 | [data-sources] 89 | [data-sources.master] 90 | host = "{}" 91 | port = {} 92 | user = "{}" 93 | password = "{}" 94 | snapshot = "{}" 95 | 96 | [data-sources.slave] 97 | host = "{}" 98 | port = {} 99 | user = "{}" 100 | password = "{}" 101 | snapshot = "{}" 102 | 103 | ######################### Task config ######################### 104 | # Required 105 | [task] 106 | # 1 fix sql: fix-target-TIDB1.sql 107 | # 2 log: sync-diff.log 108 | # 3 summary: summary.txt 109 | # 4 checkpoint: a dir 110 | output-dir = "{}" 111 | 112 | source-instances = ["master"] 113 | 114 | target-instance = "slave" 115 | target-check-tables = ["!INFORMATION_SCHEMA.*","!METRICS_SCHEMA.*","!PERFORMANCE_SCHEMA.*","!mysql.*","!test.*","!tidb_binlog.*",{}] 116 | 117 | """.format(threadCount, masterHost.split(":",1)[0], masterPort, masterUser, masterPassword, masterTso, 118 | slaveHost.split(":",1)[0], slavePort, slaveUser, slavePassword, slaveTso, outDir, dbList) 119 | 120 | # 打开文件以写入内容，如果文件不存在则创建它 121 | with open('config.toml', 'w') as file: 122 | file.write("{}\n".format(config)) 123 | 124 | # 你可以继续写入更多内容 125 | file.close() 126 | 127 | return config 128 | 129 | def parseTso(slaveHost, slaveUser, slavePassword): 130 | port = int(slaveHost.split(":",1)[1]) 131 | # connect mysql 132 | config = { 133 | "host": slaveHost.split(":",1)[0], 134 | "user": slaveUser, 135 | "password": slavePassword, 136 | "port": port, 137 | "charset": 'utf8mb4', 138 | "cursorclass": pymysql.cursors.DictCursor 139 | } 140 | 141 | connection = pymysql.connect(**config) # 连接到MySQL数据库 142 | cursor = connection.cursor() 143 | 144 | sql = """ 145 | select checkPoint from tidb_binlog.checkpoint; 146 | """ 147 | cursor.execute(sql) # 执行SQL查询 148 | content = cursor.fetchall() 149 | connection.commit() 150 | connection.close() 151 | content = content[0]['checkPoint'] 152 | content = json.loads(content) 153 | masterTso = content['ts-map']['primary-ts'] 154 | slaveTso = content['ts-map']['secondary-ts'] 155 | 156 | return masterTso, slaveTso 157 | 158 | def changeGc(dbHost, dbUser, dbPassword, gcTime): 159 | port = int(dbHost.split(":",1)[1]) 160 | # connect mysql 161 | config = { 162 | "host": dbHost.split(":",1)[0], 163 | "user": dbUser, 164 | "password": dbPassword, 165 | "port": port, 166 | "charset": 'utf8mb4', 167 | "cursorclass": pymysql.cursors.DictCursor 168 | } 169 | 170 | connection = pymysql.connect(**config) # 连接到MySQL数据库 171 | cursor = connection.cursor() 172 | # select gc time sql 173 | selectGcSql = """ 174 | select variable_value from mysql.tidb where variable_name='tikv_gc_life_time'; 175 | """ 176 | # update gc time sql 177 | upGcSql = """ 178 | update mysql.tidb set VARIABLE_VALUE='{}' where variable_name='tikv_gc_life_time'; 179 | """.format(gcTime) 180 | # select gc time 181 | cursor.execute(selectGcSql) 182 | selectGc = cursor.fetchone() 183 | # update gc time to gcTime 184 | cursor.execute(upGcSql) 185 | connection.commit() 186 | connection.close() 187 | print("TiDB gc: {}, update gc: {}".format(selectGc['variable_value'], gcTime)) 188 | 189 | # return tidb gc time 190 | return selectGc['variable_value'] 191 | 192 | # 解析命令行参数 193 | def parse_args(): 194 | parser = argparse.ArgumentParser(description="Check tables for TiDB to TiDB.") 195 | parser.add_argument( 196 | "-mh", 197 | dest="masterHost", 198 | help="Source database address and port, default: 127.0.0.1:4000", 199 | default="127.0.0.1:4000") 200 | parser.add_argument("-mu", 201 | dest="masterUser", 202 | help="Source database account, default: root", 203 | default="root") 204 | parser.add_argument("-mp", 205 | dest="masterPassword", 206 | help="Source database password, default: null", 207 | default="") 208 | parser.add_argument("-dl", 209 | dest="DatabaseList", 210 | help="Diff database list, default: null, for example: db1,db2,db3", 211 | default="") 212 | parser.add_argument( 213 | "-sh", 214 | dest="slaveHost", 215 | help="Target database address and port, default: 127.0.0.1:4000", 216 | default="127.0.0.1:4000") 217 | parser.add_argument("-su", 218 | dest="slaveUser", 219 | help="Target database account, default: root", 220 | default="root") 221 | parser.add_argument("-sp", 222 | dest="slavePassword", 223 | help="Target database password, default: null", 224 | default="tidb@123") 225 | parser.add_argument( 226 | "-b", 227 | dest="binaryPath", 228 | help="Sync diff binary path, for example: /user/bin/, default: ./", 229 | default="./") 230 | parser.add_argument( 231 | "-t", 232 | dest="threadCount", 233 | help="set check-thread-count, default: 16", 234 | default="16") 235 | parser.add_argument( 236 | "-g", 237 | dest="gcTime", 238 | help="set tidb gc time, default: 24h", 239 | default="24h") 240 | 241 | args = parser.parse_args() 242 | 243 | return args 244 | 245 | if __name__ == "__main__": 246 | main() 247 | -------------------------------------------------------------------------------- /tidb_status.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import subprocess 5 | import os 6 | import time 7 | import argparse 8 | 9 | 10 | def main(): 11 | args = parse_args() 12 | httpApi = "http://{}:{}/status".format(args.host, args.status) 13 | dropPort = "sudo iptables -A INPUT -p tcp --dport {} -j REJECT --reject-with tcp-reset".format( 14 | args.port) 15 | acceptPort = "sudo iptables -D INPUT -p tcp --dport {} -j REJECT --reject-with tcp-reset".format( 16 | args.port) 17 | portStatus = "sudo iptables -L -n | grep {} | grep reject| wc -l".format( 18 | args.port) 19 | psStatus = "sudo ps aux |grep tidb-server | grep {}|grep -Ev grep|wc -l".format( 20 | args.port) 21 | while True: 22 | time.sleep(int(args.Stime)) 23 | if checkPs(psStatus): 24 | if checkApi(httpApi, args.number): 25 | if checkPs(portStatus): 26 | # 等待 10s 后恢复 27 | time.sleep(10) 28 | os.system(acceptPort) 29 | logsFile("开启端口 {}".format(args.port)) 30 | else: 31 | logsFile("端口 {} 已经开启".format(args.port)) 32 | else: 33 | if checkPs(portStatus): 34 | logsFile("端口 {} 已经关闭".format(args.port)) 35 | else: 36 | # 避免在检测过程中，tidb-server 进程关闭 37 | if checkPs(psStatus): 38 | os.system(dropPort) 39 | logsFile("关闭端口 {}".format(args.port)) 40 | else: 41 | logsFile("tidb 数据库进程不存在, 端口为：{}".format(args.port)) 42 | else: 43 | logsFile("tidb 数据库进程不存在, 端口为：{}".format(args.port)) 44 | 45 | 46 | def checkApi(httpApi, number): 47 | nc = 0 48 | while True: 49 | # 判断 api 状态是否异常 50 | tidbA = tidbActive(httpApi) 51 | if tidbA: 52 | logsFile("数据库状态正常") 53 | rt = True 54 | break 55 | else: 56 | nc += 1 57 | # 控制异常次数，默认连续 3 次才进行关闭 58 | if nc >= number: 59 | rt = False 60 | logsFile("第 {} 次检测不正常".format(nc)) 61 | break 62 | else: 63 | logsFile("第 {} 次检测不正常".format(nc)) 64 | 65 | time.sleep(1) 66 | 67 | return rt 68 | 69 | 70 | def checkPs(command): 71 | sta = os.popen(command) 72 | sta = int(sta.read()) 73 | if sta != 0: 74 | rt = True 75 | else: 76 | rt = False 77 | 78 | return rt 79 | 80 | 81 | def tidbStatus(portStatus): 82 | sta = os.popen(portStatus) 83 | sta = int(sta.read()) 84 | 85 | return sta 86 | 87 | 88 | def tidbActive(httpApi): 89 | try: 90 | subprocess.check_output(["curl", "-slm", "1", httpApi]) 91 | rt = True 92 | except Exception: 93 | rt = False 94 | 95 | return rt 96 | 97 | 98 | def logsFile(content): 99 | fileName = time.strftime("%Y-%m-%d", time.localtime()) + "-status.log" 100 | dirs = "logs" 101 | # 判断有没有 logs 目录，如果没有就创建一个 102 | if not os.path.exists(dirs): 103 | os.makedirs(dirs) 104 | # 日志文件添加 logs 目录 105 | fileName = os.path.join(dirs, fileName) 106 | timeLocal = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 107 | with open(fileName, "a+") as f: 108 | f.write("{}: {} \n".format(timeLocal, content)) 109 | 110 | 111 | def parse_args(): 112 | # Incoming parameters 113 | parser = argparse.ArgumentParser(description="Check tidb status") 114 | parser.add_argument("-P", 115 | dest="port", 116 | help="tidb port, default: 4000", 117 | default=4000) 118 | parser.add_argument("-H", 119 | dest="host", 120 | help="Database address, default: 127.0.0.1", 121 | default="127.0.0.1") 122 | parser.add_argument("-s", 123 | dest="status", 124 | help="TiDB status port, default: 10080", 125 | default="10080") 126 | parser.add_argument("-n", 127 | dest="number", 128 | help="try number, default: 3", 129 | default=3) 130 | parser.add_argument("-t", 131 | dest="Stime", 132 | help="sleep time, default: 3", 133 | default=3) 134 | 135 | args = parser.parse_args() 136 | 137 | return args 138 | 139 | 140 | if __name__ == "__main__": 141 | main() 142 | -------------------------------------------------------------------------------- /time_to_tso.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import time 5 | from datetime import datetime 6 | import sys 7 | 8 | if len(sys.argv) == 2: 9 | timestr = sys.argv[1] 10 | else: 11 | print("Please input time") 12 | sys.exit(0) 13 | 14 | # parser time 15 | try: 16 | datetime_obj = datetime.strptime(timestr, "%Y-%m-%d %H:%M:%S.%f") 17 | except ValueError: 18 | print("Error: {}".format(ValueError)) 19 | sys.exit(0) 20 | 21 | # init time 22 | obj_stamp = int( 23 | time.mktime(datetime_obj.timetuple()) * 1000.0 + 24 | datetime_obj.microsecond / 1000.0) 25 | 26 | # time: 10 to 2 27 | bit_tamp = bin(obj_stamp) 28 | 29 | # Complete 18 bits 30 | bit_tamp = bit_tamp + '0' * 18 31 | 32 | # print("{}, {}".format(obj_stamp, bit_tamp)) 33 | print("TSO: {}".format(int(bit_tamp, 2))) 34 | --------------------------------------------------------------------------------