├── .gitignore ├── CHANGELOG.txt ├── LICENSE ├── README.md ├── binlogstream.py.diff ├── constant.py ├── flashback.py ├── func.py ├── internal └── flashback_internal.ppt ├── joint_sql.py ├── log └── test.txt ├── mysql_table.py ├── mysqlbinlog_back.py ├── pymysqlreplication ├── __init__.py ├── binlogstream.py ├── bitmap.py ├── column.py ├── constants │ ├── BINLOG.py │ ├── FIELD_TYPE.py │ └── __init__.py ├── event.py ├── gtid.py ├── packet.py ├── row_event.py ├── table.py └── tests │ ├── __init__.py │ ├── base.py │ ├── benchmark.py │ ├── test_basic.py │ ├── test_data_objects.py │ └── test_data_type.py └── test ├── log └── test.txt ├── test_generate_table_name.py ├── test_mysql_table.py ├── test_mysqlbinlog_back.py └── test_parameter.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | #remind myself 92 | other/ 93 | -------------------------------------------------------------------------------- /CHANGELOG.txt: -------------------------------------------------------------------------------- 1 | 0.1.3 2016/12/19 2 | 3 | 修复bug: #1生成SQL有个小bug，字段如果有空值，对应的空值会被解析成None而不是null 4 | 5 | 0.1.2 2016/12/19 6 | 增加功能：对字符集 utf8mb4的支持 7 | 8 | 0.1.1 2016/11/23 9 | 增加功能：提供针对update,delete,insert操作类型的过滤 ( --skip_insert --skip_update --skip_delete) 10 | 增加功能：产生的sql可以去除schema名称( --add_schema_name) ，这样用户更好调试 11 | 其他：加版本标识 12 | 其他：单元测试挪到test目录下 13 | 14 | 0.1.0 2016/11/17 15 | 初始版本 16 | 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #目前运行情况 2 | 现在已经在阿里的rds上，db为utf8字符集的生产环境下使用。其他环境没有在生产环境下使用，请小心。 3 | #工具简介 4 | ##概述 5 | mysqlbinlog_back.py 是在线读取row格式的mysqld的binlog,然后生成反向的sql语句的工具。一般用于数据恢复的目的。 6 | 所谓反向的sql语句就是如果是insert，则反向的sql为delete。如果delete,反向的sql是insert,如果是update, 反向的sql还是update,但是update的值是原来的值。 7 | 8 | 最简单的例子为 9 | `python mysqlbinlog_back.py --host="127.0.0.1" --username="root" --port=43306 --password="" --schema=test --table="test5"` 10 | 11 | 下面是程序输出结果 12 | `ls -l log/*` 13 | 14 | -rw-r--r-- 1 root root 2592 Nov 9 15:44 log/save_data_dml_test_20161109_154434.sql 15 | -rw-r--r-- 1 root root 1315 Nov 9 15:44 log/flashback_test_20161109_154434.sql <--- 反向sq文件 16 | -rw-r--r-- 1 root root 441 Nov 9 15:44 log/save_data_create_table_test_20161109_154434.sql 17 | 18 | 它会在线连接参数指定mysql,读取binlog,仅仅抽取对schema为test 表名test5的binlog，生成反向sq文件保存在log目录下,其中flash_开头的文件是反向的sql语句。 19 | 20 | 用mysql命令导入数据是**一定指定字符集为utf8mb4**,比如 21 | 22 | mysql ... --default-character-set=utf8mb4 test < flashback_test_20161109_154434.sql 23 | 24 | 25 | ##详细描述 26 | mysqlbinlog_back.py在线连接参数指定mysql,读取binlog,如果缺省，它通过show binary logs命令找到最近的binlog文件，从文件开头开始解析，一直解析到当前时间退出。 27 | 28 | 如果指定开始binary log文件名和位置（BINLOG_START_FILE_NAME，BINLOG_START_FILE_POSITION），会从指定binary log文件名和位置开始解析，一直BINLOG_END_TIME结束，中间会自动扫描跨多个binlog. 29 | 30 | 生成文件目录可以通过OUTPUT_FILE_PATH来指定。目录下有2个类： 31 | 一类是反向解析的文件,格式为flashback_schema名_当前时间.sql . 32 | 另一类用于审查数据的sql,审查数据的sql用于记录操作类型，sql的老、新值。其中, save_data_create_table_开头的文件用于生成建表语句，save_data_dml用于插入到新的表中。 33 | 34 | ##参数说明 35 | `python mysqlbinlog_back.py --help` 看在线的帮助 36 | 37 | 另外也可以看一下[CHANGELOG.txt](CHANGELOG.txt) 38 | 39 | ##依赖的包和环境 40 | python2.6 41 | 42 | pymysql 43 | 44 | ##内部原理 45 | [内部原理](internal/flashback_internal.ppt) 46 | 47 | #使用限制 48 | 1.支持mysql版本为MySQL 5.5 and 5.6.因为底层使用的是[python-mysql-replication](https://github.com/noplay/python-mysql-replication)包。 49 | 50 | 2.数据库必须是row格式的。原因看这个[链接](https://python-mysql-replication.readthedocs.io/en/latest/limitations.html) 51 | 52 | 3.反向生成的表必须有主键。 53 | 54 | 4.日志必须在主库存在 55 | 56 | 5.反向生成的mysql数据类型列出在下面。没有列出的类型没有经过严格的测试，也许有问题 57 | 58 | 6.支持的类型 59 | 60 | 允许解析的字段类型，不在里面的会报错 61 | 62 | ALLOW_TYPE={ "varchar":True, "char":True, "datetime":True, "date":True, "time":True, "timestamp":True, "bigint":True, "mediumint":True, 63 | "smallint":True, "tinyint":True, "int":True, "smallint":True, "decimal":True, "float":True, "double":True, "longtext":True, "tinytext":True, 64 | "text":True, "mediumtext":True } 65 | 66 | #FAQ 67 | 1.mysqlbinlog_back.py 是否对数据库性能造成影响？ 68 | 69 | 基本没有影响。因为代码对mysql的操作就是2种，第一种是伪装成mysql的从库去在线读取日志，对mysql的压力就是传输一下binlog.第二种会读取information_schema.columns系统表 70 | 71 | 2.对mysql字符集的支持什么 72 | 73 | utf8测试通过。gbk方式没有测试，应该问题不大。 74 | 75 | 原理角度python都用utf8的方式读出数据，内部转换成unicode的方式，然后写文件输出到utf8编码格式的文件 76 | 77 | 如果数据库是utf8mb4的编码格式，因为产生的sql是utf8格式的，包括了utfmb4的编码，所以没有问题，但是特别注意2点 78 | 79 | a.导入mysql时一定指定，mysql ... --default-character-set=utf8mb4 80 | 81 | b.如果需要对产生的sql再次编辑，一定注意，因为包括utfmb4的一些符号对你的编辑器来说是不可见得，所以千万不要搞掉了哦。 82 | 83 | 84 | 3.对mysql时间类型的支持是什么 85 | 86 | datetime没有时区的概念，所以是啥就是啥。 87 | timestamp经过python转换成datetime,转换成运行程序的环境时区相关的时间 88 | 89 | 4.底层是用的python-mysql-replication 包，是否可以用原生态的python-mysql-replication替换呢？ 90 | 91 | 不行，因为原生态的包开发的接口不够多，有些功能不具备。所以在它的代码基础上改了部分 92 | 93 | 5.指定event位置时是否会找出语句的丢失? 94 | 95 | 一定不能指定位置时指定在dml的位置，位置至少应该在dml之前的table_map的位置，当然更加好的位置应该是在事物开始的位置，也就是begin的位置。 96 | 因为一个dml会对应2个event,一个table_map，另一个是dml的event 97 | 98 | #联系方式 99 | mail:laiyi@daojia.com 100 | -------------------------------------------------------------------------------- /binlogstream.py.diff: -------------------------------------------------------------------------------- 1 | --- ./binlogstream.py 2016-05-20 16:29:08.000000000 +0800 2 | +++ ../../pymysqlreplication/binlogstream.py 2016-11-08 17:51:42.000000000 +0800 3 | @@ -1,5 +1,5 @@ 4 | # -*- coding: utf-8 -*- 5 | - 6 | +import pdb 7 | import pymysql 8 | import struct 9 | 10 | @@ -125,7 +125,7 @@ 11 | """Connect to replication stream and read event 12 | """ 13 | report_slave = None 14 | - 15 | + #changed by yilai,change parameter 16 | def __init__(self, connection_settings, server_id, resume_stream=False, 17 | blocking=False, only_events=None, log_file=None, log_pos=None, 18 | filter_non_implemented_events=True, 19 | @@ -133,7 +133,7 @@ 20 | only_tables=None, only_schemas=None, 21 | freeze_schema=False, skip_to_timestamp=None, 22 | report_slave=None, slave_uuid=None, 23 | - pymysql_wrapper=None): 24 | + pymysql_wrapper=None,end_to_timestamp=None,process_interval=0): 25 | """ 26 | Attributes: 27 | resume_stream: Start for event from position or the latest event of 28 | @@ -188,6 +188,11 @@ 29 | self.pymysql_wrapper = pymysql_wrapper 30 | else: 31 | self.pymysql_wrapper = pymysql.connect 32 | + #added by yilai 33 | + #self.current_position=-1 34 | + self.end_to_timestamp=end_to_timestamp 35 | + self.event_count=0 36 | + self.process_interval=process_interval 37 | 38 | def close(self): 39 | if self.__connected_stream: 40 | @@ -346,7 +351,7 @@ 41 | prelude += struct.pack('0: 64 | + if (self.event_count % self.process_interval)==0: 65 | + print("scan {0} events ....".format(self.event_count)) 66 | 67 | if not pkt.is_ok_packet(): 68 | continue 69 | - 70 | + #yilai,for debug 71 | + #print("-----begin event-----") 72 | + #print(':'.join(x.encode('hex') for x in pkt._data)) 73 | + #print(pkt._data) 74 | binlog_event = BinLogPacketWrapper(pkt, self.table_map, 75 | self._ctl_connection, 76 | self.__use_checksum, 77 | @@ -389,6 +403,14 @@ 78 | self.__only_tables, 79 | self.__only_schemas, 80 | self.__freeze_schema) 81 | + #added by yilai 82 | + #print("*****timestamp{0},next pos{1},current pos{2}".format(binlog_event.timestamp, 83 | + # binlog_event.log_pos,self.current_position)) 84 | + #self.current_position=binlog_event.log_pos 85 | + #pdb.set_trace() 86 | + #added by yilai 87 | + if self.end_to_timestamp and binlog_event.timestamp > self.end_to_timestamp: 88 | + return None 89 | 90 | if self.skip_to_timestamp and binlog_event.timestamp < self.skip_to_timestamp: 91 | continue 92 | @@ -401,6 +423,7 @@ 93 | if binlog_event.event_type == ROTATE_EVENT: 94 | self.log_pos = binlog_event.event.position 95 | self.log_file = binlog_event.event.next_binlog 96 | + #binlog_event.event.dump() 97 | # Table Id in binlog are NOT persistent in MySQL - they are in-memory identifiers 98 | # that means that when MySQL master restarts, it will reuse same table id for different tables 99 | # which will cause errors for us since our in-memory map will try to decode row data with 100 | @@ -457,12 +480,13 @@ 101 | self.__connect_to_ctl() 102 | 103 | cur = self._ctl_connection.cursor() 104 | + #changed by yilai,add schema name "information_schema" for rds in alibaba 105 | cur.execute(""" 106 | SELECT 107 | COLUMN_NAME, COLLATION_NAME, CHARACTER_SET_NAME, 108 | COLUMN_COMMENT, COLUMN_TYPE, COLUMN_KEY 109 | FROM 110 | - columns 111 | + information_schema.columns 112 | WHERE 113 | table_schema = %s AND table_name = %s 114 | """, (schema, table)) 115 | -------------------------------------------------------------------------------- /constant.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | 常量定义（正式环境） 6 | v0.1.0 2016/07/20 yilai created 7 | """ 8 | 9 | from __future__ import print_function 10 | 11 | 12 | class Constant(object): 13 | """ 14 | 版本号标识 15 | """ 16 | VERSION="0.1.3" 17 | """ 18 | 存一些全局的变量 19 | :return: 20 | """ 21 | #保存数据的表名的模板 22 | KEEP_DATA_TABLE_TEMPLATE=u"`_{0}_keep_data_`" 23 | #定义程序日志的路径 24 | LOGFILE_PATH="./" 25 | #定义程序日志的文件名 26 | LOGFILE_NAME="mysqlbinlog_flashback.log" 27 | #输出文件的编码方式 28 | FILE_ENCODING="utf-8" 29 | #处理到数量后这个打印一条提示信息 30 | PROCESS_INTERVAL=10000 31 | #PROCESS_INTERVAL=100 32 | #timedelta (minutes)of current time 33 | TIMEDELTA_CURRENT_TIME=0 34 | #是否对update语句生成update pk的字句 35 | UPDATE_PK=True 36 | 37 | #允许解析的字段类型，不在里面的会报错 38 | ALLOW_TYPE={ 39 | "varchar":True, 40 | "char":True, 41 | "datetime":True, 42 | "date":True, 43 | "time":True, 44 | "timestamp":True, 45 | "bigint":True, 46 | "mediumint":True, 47 | "smallint":True, 48 | "tinyint":True, 49 | "int":True, 50 | "smallint":True, 51 | "decimal":True, 52 | "float":True, 53 | "double":True, 54 | "longtext":True, 55 | "tinytext":True, 56 | "text":True, 57 | "mediumtext":True 58 | } 59 | 60 | -------------------------------------------------------------------------------- /flashback.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | flashback的主要处理逻辑和相关的类 5 | v0.1.0 2016/10/20 yilai created 6 | """ 7 | 8 | 9 | from datetime import datetime,timedelta 10 | from time import mktime 11 | import traceback 12 | import logging 13 | from pymysqlreplication import BinLogStreamReader 14 | from pymysqlreplication.row_event import ( 15 | DeleteRowsEvent, 16 | UpdateRowsEvent, 17 | WriteRowsEvent, 18 | ) 19 | 20 | from pymysqlreplication.event import( 21 | QueryEvent, 22 | XidEvent 23 | ) 24 | from joint_sql import joint_update_sql,joint_insert_sql,joint_delete_sql,joint_keep_data_sql,join_create_table 25 | from mysql_table import MysqlTable 26 | from constant import Constant 27 | from mysql_table import MysqlTable 28 | 29 | logger = logging.getLogger("__main__") 30 | #定义保留数据表名的格式 31 | 32 | class Parameter(object): 33 | """ 34 | 存用户的命令行的参数和一些全局的变量 35 | :return: 36 | """ 37 | 38 | def __init__(self,username,password,start_binlog_file,start_position 39 | ,schema,host="127.0.0.1",start_time=None,output_file_path="./",port=3306,tablename=None 40 | ,keep_data=False,keep_current_data=False 41 | ,one_binlog_file=False,dump_event=False,end_to_timestamp=None,start_to_timestamp=None 42 | ,skip_insert=False,skip_update=False,skip_delete=False,add_schema_name=False 43 | ): 44 | 45 | """ 46 | :param username: 连接数据库的用户名 47 | :param password: 48 | :param port: 49 | :param start_binlog_file: 50 | :param start_position: 51 | :param end_time: 需要解析binlog的结束时间 52 | :param output_file_prefix: 53 | :param start_time: 54 | :param tablename: 55 | :param schema: 56 | :param keep_data: 是否生成dml语句的before,after value到单独一个表中 57 | :param keep_current_data: 是否包括当前值 58 | :param one_binlog_file: 是否就解析一个binlog文件 59 | :param dump_event:dump event的信息，用于调试 60 | :return: 61 | """ 62 | self.mysql_setting={} 63 | self.mysql_setting["host"]=host 64 | self.mysql_setting["port"]=port 65 | self.mysql_setting["user"]=username 66 | self.mysql_setting["passwd"]=password 67 | self.mysql_setting["charset"] = "utf8" 68 | self.start_binlog_file=start_binlog_file 69 | self.start_position=start_position 70 | self.output_file_path=output_file_path 71 | self.start_time=start_time 72 | self.schema=schema 73 | self.schema_array=None 74 | if self.schema is not None: 75 | self.schema_array=self.schema.split(",") 76 | self.table_name=tablename 77 | self.table_name_array=None 78 | if self.table_name is not None: 79 | self.table_name_array=self.table_name.split(",") 80 | self.keep_data=keep_data 81 | self.keep_current_data=keep_current_data 82 | self.one_binlog_file=one_binlog_file 83 | self.dump_event=dump_event 84 | self.end_to_timestamp=end_to_timestamp 85 | self.start_to_timestamp=start_to_timestamp 86 | self.skip_insert=skip_insert 87 | self.skip_update=skip_update 88 | self.skip_delete=skip_delete 89 | self.add_schema_name=add_schema_name 90 | 91 | #下面是一些公共的变量 92 | self.stream=None 93 | self.stat={} 94 | #self.stat["commit"]=0 95 | self.stat["flash_sql"]={} 96 | #输出的文件描述符 97 | self.file={} 98 | self.file["flashback"]=None 99 | self.file["data"]=None 100 | self.file["data_create"]=None 101 | 102 | def get_file_name(self,filename_type,current_time=None): 103 | """ 104 | 得到文件名 105 | :param : 106 | :return: 107 | """ 108 | if current_time is None: 109 | current_time=datetime.strftime( datetime.now(), '%Y%m%d_%H%M%S') 110 | filename="{0}/{1}_{2}_{3}.sql".format(self.output_file_path,filename_type,self.schema,current_time) 111 | return filename 112 | 113 | 114 | def check_tables_exist(self): 115 | for table in self.table_name_array: 116 | mysql_table=MysqlTable() 117 | mysql_table.connect(self.mysql_setting) 118 | mysql_table.get_columns(self.schema_array[0],table) 119 | 120 | 121 | 122 | def set_defaut(self): 123 | """ 124 | 访问数据库设置缺省值start_binlog_file，end_to_timestamp 125 | """ 126 | if self.start_binlog_file is None: 127 | mysql_table=MysqlTable() 128 | mysql_table.connect(self.mysql_setting) 129 | log_name=mysql_table.get_last_binary_log_name() 130 | self.start_binlog_file=log_name 131 | 132 | if self.end_to_timestamp is None: 133 | mysql_table=MysqlTable() 134 | mysql_table.connect(self.mysql_setting) 135 | current_dt=mysql_table.get_current_datetime() 136 | current_dt=current_dt - timedelta(minutes = Constant.TIMEDELTA_CURRENT_TIME) 137 | self.end_to_timestamp=convert_datetime_to_timestamp(current_dt) 138 | 139 | 140 | 141 | 142 | # 143 | # class Stat(object): 144 | # """ 145 | # 处理mysqlbinlog形成的统计结果，也会用于产生保留现场数据文件的建表语句 146 | # :return: 147 | # """ 148 | # def __init__(self): 149 | # pass 150 | # 151 | 152 | 153 | 154 | def deal_all_event(parameter): 155 | """ 156 | 读日志，处理mysqlbinlog的日志 157 | :param parameter: 用户的命令行的参数和一些全局的变量 158 | :return: 159 | :except:如果有问题，会抛出异常 160 | """ 161 | events=[] 162 | if parameter.skip_insert is False: 163 | events.append(WriteRowsEvent) 164 | if parameter.skip_update is False: 165 | events.append(UpdateRowsEvent) 166 | if parameter.skip_delete is False: 167 | events.append(DeleteRowsEvent) 168 | stream = BinLogStreamReader( 169 | connection_settings=parameter.mysql_setting, 170 | server_id=1294967255, 171 | #blocking=True, 172 | only_events=events, 173 | #TODO 事件是否需要配置？先不管吧 174 | #only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent,QueryEvent,XidEvent], 175 | only_tables=parameter.table_name_array, 176 | only_schemas=parameter.schema_array, 177 | #log_file='mysql-bin.000840', 178 | log_file=parameter.start_binlog_file, 179 | log_pos=parameter.start_position, 180 | end_to_timestamp=parameter.end_to_timestamp, 181 | #log_pos=6178, 182 | skip_to_timestamp=parameter.start_to_timestamp, 183 | #log_pos=226812, 184 | resume_stream=True, 185 | process_interval=Constant.PROCESS_INTERVAL 186 | ) 187 | parameter.stream=stream 188 | logger.debug("begin to deal event,stream object={0}".format(stream)) 189 | if (parameter.dump_event is True): 190 | dump_events(stream) 191 | return 192 | for event in stream: 193 | #pdb.set_trace() 194 | #TODO begin，commit太多需要在dml的前后加合适,没有做了,先不需要俘获这个事件 195 | logger.debug("event type={0},logfile={1}".format(type(event),stream.log_file)) 196 | if isinstance(event,QueryEvent): 197 | deal_query(event,parameter.file,parameter.stat) 198 | elif isinstance(event,XidEvent): 199 | deal_xid(event,parameter.file,parameter.stat) 200 | elif isinstance(event, DeleteRowsEvent): 201 | deal_delete_rows(event,parameter.file,parameter.stat,logfile=stream.log_file,add_schema_name=parameter.add_schema_name) 202 | elif isinstance(event, UpdateRowsEvent): 203 | deal_update_rows(event,parameter.file,parameter.stat,logfile=stream.log_file,add_schema_name=parameter.add_schema_name) 204 | elif isinstance(event, WriteRowsEvent): 205 | deal_insert_rows(event,parameter.file,parameter.stat,logfile=stream.log_file,add_schema_name=parameter.add_schema_name) 206 | 207 | #logger.debug(stream.table_map[70].__dict__) 208 | 209 | 210 | 211 | 212 | def dump_events(stream): 213 | """ 214 | 调试目的，dump event到屏幕上，不会到日志文件中 215 | :param stream: 216 | :return: 217 | """ 218 | for binlogevent in stream: 219 | #pdb.set_trace() 220 | #print(type(binlogevent)) 221 | if isinstance(binlogevent,QueryEvent): 222 | #pass 223 | print(u"++++{0}".format(binlogevent.query)) 224 | if isinstance(binlogevent,XidEvent): 225 | print("++++{0}".format("commit")) 226 | binlogevent.dump() 227 | stream.close() 228 | 229 | def deal_common_event(event,file_dict,logfile=None): 230 | """ 231 | 处理事件的公共函数，写时间戳，位置等公共信息 232 | :param event: mysql的事件 233 | :param file_dict: 文件描述符，包括：反向sql文件,保留现场数据文件 234 | :return: 235 | """ 236 | 237 | content=u"#end_log_pos {0} {1} {2} {3}".format(event.packet.log_pos, 238 | datetime.fromtimestamp(event.timestamp).isoformat(),event.timestamp,logfile) 239 | write_file(file_dict["flashback"],content) 240 | write_file(file_dict["data"],content) 241 | 242 | 243 | def deal_query(event,file_dict,stat,keep_data=False,keep_current=False): 244 | """ 245 | 处理query事件，仅仅写begin语句到文件中，其他忽略 246 | :param event: mysql的事件 247 | :param file_dict: 文件描述符，包括：反向sql文件,保留现场数据文件 248 | :param stat:mysqlbinlog形成的统计结果 249 | :param keep_data:产生保留现场数据中的老、值和新值 250 | :param keep_current：产生保留现场数据中的当前值 251 | :return: 252 | """ 253 | content=u"{0}".format(event.query) 254 | 255 | if content=='BEGIN': 256 | deal_common_event(event,file_dict) 257 | logger.debug(u"Query event content={0}".format(event.query)) 258 | write_file(file_dict["flashback"],content) 259 | write_file(file_dict["data"],content) 260 | 261 | 262 | 263 | 264 | def deal_xid(event,file_dict,stat,keep_data=False,keep_current=False): 265 | """ 266 | 处理xid事件，仅仅写commit语句到文件中 267 | :param event: mysql的事件 268 | :param file_dict: 文件描述符，包括：反向sql文件,保留现场数据文件 269 | :param stat:mysqlbinlog形成的统计结果 270 | :return: 271 | """ 272 | content="commit" 273 | deal_common_event(event,file_dict) 274 | logger.debug(u"xid event content") 275 | stat["commit"] += 1 276 | write_file(file_dict["flashback"],content) 277 | write_file(file_dict["data"],content) 278 | 279 | 280 | def deal_insert_rows(event,file_dict,stat,keep_data=False,keep_current=False,logfile=None,add_schema_name=False): 281 | """ 282 | 处理delete事件，反向生成insert语句，保留现场数据到文件 283 | :param event: mysql的事件 284 | :param file_dict: 文件描述符，包括：反向sql文件,保留现场数据文件 285 | :param stat:mysqlbinlog形成的统计结果 286 | :return: 287 | """ 288 | deal_common_event(event,file_dict,logfile) 289 | for row in event.rows: 290 | sql=joint_delete_sql(event.schema,event.table,event.primary_key,row,add_schema_name) 291 | sql_keep=joint_keep_data_sql("insert",event.timestamp,event.table,row) 292 | write_file(file_dict["flashback"],sql) 293 | write_file(file_dict["data"],sql_keep) 294 | add_stat(stat,"delete",event.schema,event.table) 295 | 296 | def deal_delete_rows(event,file_dict,stat,keep_data=False,keep_current=False,logfile=None,add_schema_name=False): 297 | """ 298 | 处理delete事件，反向生成insert语句，保留现场数据到文件 299 | :param event: mysql的事件 300 | :param file_dict: 文件描述符，包括：反向sql文件,保留现场数据文件 301 | :param stat:mysqlbinlog形成的统计结果 302 | :return: 303 | """ 304 | deal_common_event(event,file_dict,logfile) 305 | for row in event.rows: 306 | sql=joint_insert_sql(event.schema,event.table,event.primary_key,row,add_schema_name) 307 | sql_keep=joint_keep_data_sql("delete",event.timestamp,event.table,row) 308 | write_file(file_dict["flashback"],sql) 309 | write_file(file_dict["data"],sql_keep) 310 | add_stat(stat,"insert",event.schema,event.table) 311 | 312 | def deal_update_rows(event,file_dict,stat,keep_data=False,keep_current=False,logfile=None,add_schema_name=False): 313 | """ 314 | 处理update事件，反向生成insert语句，保留现场数据到文件 315 | :param event: mysql的事件 316 | :param file_dict: 文件描述符，包括：反向sql文件,保留现场数据文件 317 | :param stat:mysqlbinlog形成的统计结果 318 | :return: 319 | """ 320 | #logger.debug("===logfile {0}".format(logfile)) 321 | deal_common_event(event,file_dict,logfile) 322 | for row in event.rows: 323 | sql=joint_update_sql(event.schema,event.table,event.primary_key,row,Constant.UPDATE_PK,add_schema_name) 324 | sql_keep=joint_keep_data_sql("update",event.timestamp,event.table,row) 325 | write_file(file_dict["flashback"],sql) 326 | write_file(file_dict["data"],sql_keep) 327 | add_stat(stat,"update",event.schema,event.table) 328 | 329 | def add_stat(stat,op_type,schema,table): 330 | """ 331 | 统计生成了多少sql 332 | :param stat:格式为stat["flash_sql"]["schema"]["table"]["操作类型"] ： 333 | :param op_type: update/delete/insert 334 | :param schema: 335 | :param table: 336 | :return: 337 | """ 338 | if not stat["flash_sql"].has_key(schema): 339 | stat["flash_sql"][schema]={} 340 | if not stat["flash_sql"][schema].has_key(table): 341 | stat["flash_sql"][schema][table]={} 342 | stat["flash_sql"][schema][table]["update"]=0 343 | stat["flash_sql"][schema][table]["insert"]=0 344 | stat["flash_sql"][schema][table]["delete"]=0 345 | stat["flash_sql"][schema][table][op_type]+=1 346 | 347 | 348 | def write_file(file,content): 349 | """ 350 | 写文件 351 | :param file: 352 | :param content: 353 | :return: 354 | """ 355 | file.write(content) 356 | file.write(";\n") 357 | 358 | 359 | 360 | 361 | 362 | def generate_create_table(parameter): 363 | """ 364 | 产生建表语句 365 | :param parameter: 366 | :return: 367 | """ 368 | mysql_table=MysqlTable() 369 | for schema in parameter.stat["flash_sql"]: 370 | for table in parameter.stat["flash_sql"][schema]: 371 | mysql_table.connect(parameter.mysql_setting) 372 | arr=mysql_table.get_columns(schema,table) 373 | sql=join_create_table(schema,table,arr) 374 | write_file(parameter.file["data_create"],sql) 375 | 376 | def convert_datetime_to_timestamp(dt,format=None): 377 | if isinstance(dt,str): 378 | end_time=datetime.strptime(dt, format) 379 | else: 380 | end_time=dt 381 | end_to_timestamp=mktime(end_time.timetuple()) 382 | return end_to_timestamp -------------------------------------------------------------------------------- /func.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | 公共函数 6 | v0.1.1 2016/09/21 yilai 加了些参数 7 | v0.1.0 2016/07/20 yilai created 8 | """ 9 | import logging 10 | from logging.handlers import RotatingFileHandler 11 | import traceback 12 | 13 | logger = logging.getLogger("__main__") 14 | 15 | def init_logger(name,log_level=logging.DEBUG,screen_output=True): 16 | """ 17 | ！！注意，这个函数只能一次调用哦 18 | 初始化日志，将来日志会写到文件和打印到屏幕上 19 | :parameter : name:日志的文件名 log_level:日志的级别 screen_output：日志也输出屏幕上 20 | :return: 21 | :exception :需要自己catch异常 22 | """ 23 | 24 | LOGFILE = name 25 | maxBytes = 5 * 1024 * 1024 26 | backupCount = 1 27 | 28 | #logger = logging.getLogger(__name__) 29 | logger.setLevel(log_level) 30 | #logger.setLevel(logging.INFO) 31 | #设置文件输出到文件中 32 | #ch = TimedRotatingFileHandler(LOGFILE, 'S', 5, 1) 33 | ch = RotatingFileHandler(LOGFILE, 'a', maxBytes, backupCount) 34 | 35 | #format='%(asctime)s [%(filename)s][%(process)d][%(levelname)s][%(lineno)d] %(message)s' 36 | format='%(asctime)s [%(filename)s][%(levelname)s][%(lineno)d] %(message)s' 37 | formatter = logging.Formatter(format) 38 | 39 | ch.setFormatter(formatter) 40 | logger.addHandler(ch) 41 | 42 | if screen_output: 43 | #设置log输出到屏幕上 44 | chscreen = logging.StreamHandler() 45 | #chscreen.setLevel(logging.INFO) 46 | chscreen.setLevel(log_level) 47 | logger.addHandler(chscreen) 48 | #print("===log will also write to {0}===".format(LOGFILE)) 49 | logger.info("===log will also write to {0}===".format(LOGFILE)) 50 | 51 | def print_stack(): 52 | """ 53 | 打印堆栈 54 | :return: 55 | """ 56 | logger.error( "=====Additional info:dump stack to diagnose =======") 57 | logger.error(traceback.format_exc()) 58 | #sys.exit(1) 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /internal/flashback_internal.ppt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/58daojia-dba/mysqlbinlog_flashback/555fd1e8c0b49bdf3d7b0f4de94039b957ff3161/internal/flashback_internal.ppt -------------------------------------------------------------------------------- /joint_sql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | v0.1.2 2016/12/19 yilai 建表改成utf8mb4字符集 6 | v0.1.1 2016/09/21 yilai 加了些参数 7 | v0.1.0 2016/07/20 yilai created 8 | """ 9 | from datetime import datetime,date,timedelta 10 | from decimal import Decimal 11 | import traceback 12 | import logging 13 | from pymysqlreplication import BinLogStreamReader 14 | from pymysqlreplication.row_event import ( 15 | DeleteRowsEvent, 16 | UpdateRowsEvent, 17 | WriteRowsEvent, 18 | ) 19 | 20 | from pymysqlreplication.event import( 21 | QueryEvent, 22 | XidEvent 23 | ) 24 | from constant import Constant 25 | 26 | logger = logging.getLogger("__main__") 27 | 28 | def joint_insert_sql(schema,table,pk,row,add_schema_name=False): 29 | """ 30 | 拼接insert语句 31 | :param schema: 32 | :param table: 33 | :param pk: 主键列名，数组类型 34 | :param row: 分析binlog的数据，包括字段名和值 35 | :return: sql 36 | """ 37 | logger.debug(u"schema={0},table={1},pk={2},row={3}".format(schema,table,pk,row)) 38 | #sql=u"update {0} set {1} where {2}" 39 | table_name=generate_table_name(schema,table,False,add_schema_name) 40 | 41 | (columns,values)=generate_two_array_column_and_value(row["values"]) 42 | statment_column=",".join(columns) 43 | statment_value=",".join(values) 44 | sql=u"insert into "+table_name+"("+statment_column+ ") values("+statment_value+")" 45 | logger.debug(sql) 46 | return sql 47 | 48 | 49 | def joint_update_sql(schema,table,pk,row,update_pk=True,add_schema_name=False): 50 | """ 51 | 拼接update语句 52 | :param schema: 53 | :param table: 54 | :param pk: 主键列名，数组类型 55 | :param row: 分析binlog的数据，包括字段名和值 56 | :param update_pk 是否产生更新pk的语句 57 | :return: sql 58 | """ 59 | logger.debug(u"schema={0},table={1},pk={2},row={3}".format(schema,table,pk,row)) 60 | #sql=u"update {0} set {1} where {2}" 61 | table_name=generate_table_name(schema,table,False,add_schema_name) 62 | #TODO ,key没有加escape 63 | 64 | if update_pk: 65 | statment_dict=row["before_values"] 66 | else: 67 | (statment_dict,null)=split_dict_column_value_pair(pk,row["before_values"]) 68 | (null,pk_dict)=split_dict_column_value_pair(pk,row["after_values"]) 69 | statment=generate_array_column_value_pair(statment_dict) 70 | statment=",".join(statment) 71 | where =generate_array_column_value_pair(pk_dict) 72 | where = " and ".join(where) 73 | sql=u"update "+table_name+" set"+statment+ " where "+where 74 | logger.debug(sql) 75 | return sql 76 | 77 | def joint_delete_sql(schema,table,pk,row,add_schema_name=False): 78 | """ 79 | 拼接delete语句 80 | :param schema: 81 | :param table: 82 | :param pk: 主键列名，数组类型 83 | :param row: 分析binlog的数据，包括字段名和值 84 | :return: sql 85 | """ 86 | logger.debug(u"schema={0},table={1},pk={2},row={3}".format(schema,table,pk,row)) 87 | if pk is None or pk is "": 88 | raise ValueError(u"could not find primary key for table,schema={0},table={1}".format(schema,table)) 89 | table_name=generate_table_name(schema,table,False,add_schema_name) 90 | temp=generate_dict_pk(pk,row["values"]) 91 | where =generate_array_column_value_pair(temp) 92 | where = " and ".join(where) 93 | sql=u"delete from "+table_name + " where "+where 94 | logger.debug(sql) 95 | return sql 96 | 97 | 98 | def generate_dict_pk(pk,row): 99 | """ 100 | 产生基于主键的dict类型的{列名，列值} 101 | :param pk: 可能是string或者tuple 102 | :param row: 103 | :return:{列名，列值} 104 | """ 105 | dt={} 106 | if isinstance(pk,unicode): 107 | dt[pk]=row[pk] 108 | else: 109 | for key in pk : 110 | dt[key]=row[key] 111 | return dt 112 | 113 | def split_dict_column_value_pair(pk,row): 114 | """ 115 | 产生基于主键的dict类型的{列名，列值} 116 | :param pk: 可能是string或者tuple 117 | :param row: 118 | :return: 其他列字典{列名，列值} ，主键字典{列名，列值} 119 | """ 120 | pk_orgi={} 121 | pk_dt={} 122 | other_dt={} 123 | if isinstance(pk,unicode): 124 | pk_orgi[pk]="" 125 | else: 126 | for key in pk: 127 | pk_orgi[key]="" 128 | for key in row: 129 | if pk_orgi.has_key(key): 130 | pk_dt[key]=row[key] 131 | else: 132 | other_dt[key]=row[key] 133 | return (other_dt,pk_dt) 134 | 135 | 136 | def generate_table_name(schema,table,keep_data=False,add_schema_name=True): 137 | """ 138 | 拼接表名 139 | :param schema: 140 | :param table: 141 | :param keep_data:是否产生保留数据的表名 142 | :return: 143 | """ 144 | content="" 145 | if keep_data: 146 | content=Constant.KEEP_DATA_TABLE_TEMPLATE.format(table) 147 | else: 148 | if add_schema_name: 149 | content=u'`'+ schema+ '`'+ ".`" + table + "`" 150 | else: 151 | content=u'`'+ table + "`" 152 | return content 153 | 154 | def generate_array_column_value_pair(row): 155 | """ 156 | 产生 “列=列值” 的数组 157 | :param row: 格式为row["列名"]=列值 158 | :return:“列=列值” 的数组 159 | """ 160 | logger.debug(u"dump row={0}".format(row)) 161 | content=[] 162 | for key in row: 163 | logger.debug(u"dump row ele: {0}={1}".format(key,row[key])) 164 | ele="`"+key+"`"+"="+to_string(row[key],"'") 165 | content.append(ele) 166 | return content 167 | 168 | def to_string(val,prefix=""): 169 | """ 170 | 把任何类型转换成string 171 | :param val: 172 | :param prefix:对于字符串，加prefix前缀 173 | :return: 174 | """ 175 | #修复为null的bug 176 | if val is None: 177 | return "null" 178 | if isinstance(val,unicode): 179 | ret= u"%s%s%s" % (prefix,escape_unicode(val),prefix) 180 | #ret= u"'%s'" % escape_unicode(val) 181 | elif isinstance(val,str): 182 | ret= u"%s%s%s" % (prefix,escape_string(val),prefix) 183 | elif isinstance(val,datetime) or isinstance(val,date) or isinstance(val,timedelta): 184 | ret=u"'"+str(val)+u"'" 185 | else: 186 | ret=str(val) 187 | return ret 188 | 189 | def escape_string(value): 190 | """escape_string escapes *value* but not surround it with quotes. 191 | """ 192 | value = value.replace('\\', '\\\\') 193 | value = value.replace('\0', '\\0') 194 | value = value.replace('\n', '\\n') 195 | value = value.replace('\r', '\\r') 196 | value = value.replace('\032', '\\Z') 197 | value = value.replace("'", "\\'") 198 | value = value.replace('"', '\\"') 199 | return value 200 | 201 | 202 | _escape_table = [unichr(x) for x in range(128)] 203 | _escape_table[0] = u'\\0' 204 | _escape_table[ord('\\')] = u'\\\\' 205 | _escape_table[ord('\n')] = u'\\n' 206 | _escape_table[ord('\r')] = u'\\r' 207 | _escape_table[ord('\032')] = u'\\Z' 208 | _escape_table[ord('"')] = u'\\"' 209 | _escape_table[ord("'")] = u"\\'" 210 | #_escape_table[ord("'")] = u"\'" 211 | def escape_unicode(value, mapping=None): 212 | """escapes *value* without adding quote. 213 | 214 | Value should be unicode 215 | """ 216 | return value.translate(_escape_table) 217 | 218 | 219 | 220 | def generate_two_array_column_and_value(row): 221 | """ 222 | 产生 “列 的和“列值”两个数组 223 | :param row: 格式为row["列名"]=列值 224 | :return: 225 | """ 226 | 227 | logger.debug(u"dump row={0}".format(row)) 228 | columns=[] 229 | values=[] 230 | for key in row: 231 | logger.debug(u"dump row ele: {0}={1},type={2}".format(key,row[key],type(row[key]))) 232 | columns.append("`"+key+"`") 233 | values.append(to_string(row[key],"'")) 234 | return (columns,values) 235 | 236 | 237 | 238 | def joint_keep_data_sql(op,op_timestamp,table,row,keep_current=False): 239 | """ 240 | 241 | :param op: 操作类型，insert,update,delete 242 | :param op_timestamp: 243 | :param table 244 | :param row: 数据 245 | :param keep_current: 是否去当前值 246 | :return: insert的语句 247 | """ 248 | op_datetime=datetime.fromtimestamp(op_timestamp) 249 | content_dict={} 250 | content_dict[u"op"]=op 251 | content_dict[u"op_datetime"]=op_datetime 252 | 253 | if (op=="update"): 254 | generate_keep_data_dict(content_dict,"bfr",row["before_values"]) 255 | generate_keep_data_dict(content_dict,"aft",row["after_values"]) 256 | elif (op=="insert"): 257 | generate_keep_data_dict(content_dict,"aft",row["values"]) 258 | else: 259 | generate_keep_data_dict(content_dict,"bfr",row["values"]) 260 | """ 261 | if (op=="update"): 262 | new_row=row["before_values"] 263 | for key in new_row: 264 | content_dict[u"bfr_"+key]=new_row[key] 265 | new_row=row["after_values"] 266 | for key in new_row: 267 | content_dict["aft"+key]=new_row[key] 268 | elif (op=="insert"): 269 | new_row=row["values"] 270 | for key in new_row: 271 | content_dict["aft"+key]=new_row[key] 272 | else: 273 | new_row=row["values"] 274 | for key in new_row: 275 | content_dict["bfr"+key]=new_row[key] 276 | """ 277 | table_name=generate_table_name("",table,True) 278 | (columns,values)=generate_two_array_column_and_value(content_dict) 279 | statment_column=",".join(columns) 280 | statment_value=",".join(values) 281 | sql=u"insert into "+table_name+"("+statment_column+ ") values("+statment_value+")" 282 | logger.debug(sql) 283 | return sql 284 | 285 | 286 | def generate_keep_data_dict(target,prefix,row): 287 | for key in row: 288 | target[u""+prefix+"_"+key]=row[key] 289 | return target 290 | 291 | def join_create_table(schema,table,colums): 292 | """ 293 | 产生建表语句 294 | :param schema: 295 | :param table: 296 | :param colums:列定义的数组 297 | :return: 298 | """ 299 | #TODO timestamp的时区有问题吗？应该不会，python会自动转换为当前的时区 300 | columns_def=[u"op varchar(64)",u"op_datetime datetime"] 301 | table_name=generate_table_name("",table,True) 302 | for row in colums: 303 | check_mysql_type(row["COLUMN_TYPE"]) 304 | columns_def_ele=u"bfr_"+row["COLUMN_NAME"] + " "+row["COLUMN_TYPE"] 305 | columns_def.append(columns_def_ele) 306 | for row in colums: 307 | columns_def_ele=u"aft_"+row["COLUMN_NAME"] + " "+row["COLUMN_TYPE"] 308 | columns_def.append(columns_def_ele) 309 | columns_def_str=",".join(columns_def) 310 | 311 | sql=u"CREATE TABLE "+table_name+" ("+columns_def_str+")"+" ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" 312 | return sql 313 | 314 | 315 | def check_mysql_type(mysql_type): 316 | """ 317 | 检查mysql的类型，因为每种类型都需要详细的测试，防止产生的sql有问题 318 | :param mysql_type: 319 | :return: 320 | """ 321 | #mysql_type="" 322 | ele=mysql_type.split(" ")[0].split("(")[0].lower() 323 | if not Constant.ALLOW_TYPE.has_key(ele): 324 | raise ValueError(u"find unsupport mysql type to flash,type={0}".format(mysql_type)) 325 | 326 | 327 | -------------------------------------------------------------------------------- /log/test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/58daojia-dba/mysqlbinlog_flashback/555fd1e8c0b49bdf3d7b0f4de94039b957ff3161/log/test.txt -------------------------------------------------------------------------------- /mysql_table.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | 提供information_schema.columns的mysql层访问 4 | 对外暴露的方法会自己维护数据库连接（自己打开关闭连接） 5 | v0.1.0 2016/07/20 yilai created 6 | """ 7 | 8 | from __future__ import print_function 9 | import time,os 10 | import datetime 11 | import re 12 | import logging 13 | import pymysql 14 | 15 | 16 | logger = logging.getLogger("__main__") 17 | 18 | class MysqlTable(object): 19 | 20 | 21 | def __init__(self): 22 | self._connection = None 23 | 24 | 25 | def get_columns(self,schema,table): 26 | try: 27 | with self._connection.cursor(pymysql.cursors.DictCursor) as cursor: 28 | sql = """ 29 | SELECT 30 | COLUMN_NAME, COLLATION_NAME, CHARACTER_SET_NAME, 31 | COLUMN_COMMENT, COLUMN_TYPE, COLUMN_KEY 32 | FROM 33 | information_schema.columns 34 | WHERE 35 | table_schema = %s AND table_name = %s 36 | """ 37 | cursor.execute(sql, (schema, table)) 38 | #必须用fetchone,它返回dict,fetchall返回的是list 39 | arr=cursor.fetchall() 40 | if len(arr)==0: 41 | raise ValueError(u"can't find table {0}.{1}".format(schema,table)) 42 | return (arr) 43 | finally: 44 | self.disconnect() 45 | 46 | def get_last_binary_log_name(self): 47 | """ 48 | 得到最后一个binary log name 49 | :return: dict{u'Log_name': u'mysql-bin.000025', u'File_size': 191} 50 | """ 51 | try: 52 | with self._connection.cursor(pymysql.cursors.DictCursor) as cursor: 53 | sql = """ 54 | SHOW binary logs; 55 | """ 56 | cursor.execute(sql) 57 | #必须用fetchone,它返回dict,fetchall返回的是list 58 | arr=cursor.fetchall() 59 | if (len(arr)==0): 60 | raise ValueError("can't get the last binary log name,'SHOW binary logs' return null ") 61 | return(arr[len(arr)-1]["Log_name"]) 62 | finally: 63 | self.disconnect() 64 | 65 | def get_current_datetime(self): 66 | """ 67 | 得到最后一个binary log name 68 | :return: dict{u'Log_name': u'mysql-bin.000025', u'File_size': 191} 69 | """ 70 | try: 71 | with self._connection.cursor(pymysql.cursors.DictCursor) as cursor: 72 | sql = """ 73 | select now() current 74 | """ 75 | cursor.execute(sql) 76 | dt=cursor.fetchone() 77 | return dt["current"] 78 | finally: 79 | self.disconnect() 80 | 81 | 82 | def connect(self,connection_setting): 83 | self._connection = pymysql.connect(**(connection_setting)) 84 | 85 | def disconnect(self): 86 | if self._connection is not None: 87 | self._connection.close() 88 | #print("close") 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /mysqlbinlog_back.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | 产生mysqlbinlog反向的sql的主程序 5 | v0.1.0 2016/10/20 yilai created 6 | """ 7 | 8 | import traceback 9 | import os,sys 10 | import logging 11 | from func import init_logger,print_stack 12 | from flashback import Parameter,deal_all_event,generate_create_table,convert_datetime_to_timestamp 13 | import codecs 14 | from datetime import datetime,timedelta 15 | from time import mktime 16 | from constant import Constant 17 | from optparse import OptionParser 18 | from mysql_table import MysqlTable 19 | 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | def get_check_option(): 25 | """ 26 | 得到和检查用户输入的参数，返回参数对象 27 | """ 28 | logger.debug(sys.argv) 29 | usage = 'usage: python %prog [options]' \ 30 | '\nsample1:python %prog --host="127.0.0.1" --username="root" --port=43306 --password="" --schema=test --table="test5"' \ 31 | '\n' \ 32 | 'sample2:python %prog --host="127.0.0.1" --username="root" --port=43306 --password="" --schema=test --table="test5,test6" ' \ 33 | '--binlog_end_time="2016-11-05 11:27:13" --binlog_start_file_name="mysql-bin.000024" --binlog_start_file_position=4 ' \ 34 | '--binlog_start_time="2016-11-04 11:27:13" --skip_delete --skip_insert --add_schema_name' \ 35 | '\nsample3:python %prog --host="127.0.0.1" --username="root" --port=43306 --password="" --schema=test --table="test5,test6" --binlog_start_file_name="mysql-bin.000022"' 36 | parser = OptionParser(usage) 37 | parser.add_option("-H","--host", type='string', help="mandatory,mysql hostname" ) 38 | parser.add_option("-P","--port", type='int',default=3306,help="mysql port,default 3306" ) 39 | parser.add_option("-u","--username", type='string', help="mandatory,username" ) 40 | parser.add_option("-p","--password", type='string',default="",help="password" ) 41 | #TODO 只能是一个字符串哦，先不支持多个schema?实际能支持，就是文件名有comma 42 | parser.add_option("-s","--schema", type='string',help="mandatory,mysql schema") 43 | parser.add_option("-t","--tables", type='string', 44 | help="mandatory,mysql tables,suport multiple tables,use comma as separator") 45 | parser.add_option("-N","--binlog_end_time", type='string', 46 | help="binlog end time,format yyyy-mm-dd hh24:mi:ss,default is current time ") 47 | parser.add_option("-S","--binlog_start_file_name", type='string', 48 | help="binlog start file name,default is current logfile of db") 49 | #TODO 开始位置不能写在这？ 50 | parser.add_option("-L","--binlog_start_file_position", type='int',default=4, 51 | help="binlog start file name") 52 | parser.add_option("-E","--binlog_start_time", type='string', 53 | help="binlog start time,format yyyy-mm-dd hh24:mi:ss") 54 | parser.add_option("-l","--output_file_path", type='string',default="./log", 55 | help="file path that sql generated,,default ./log") 56 | parser.add_option("-I","--skip_insert", action="store_true",default=False, 57 | help="skip insert(WriteRowsEvent) event") 58 | parser.add_option("-U","--skip_update", action="store_true",default=False, 59 | help="skip update(UpdateRowsEvent) event") 60 | parser.add_option("-D","--skip_delete", action="store_true",default=False, 61 | help="skip delete(DeleteRowsEvent) event") 62 | parser.add_option("-a","--add_schema_name", action="store_true",default=False, 63 | help="add schema name for flashback sql") 64 | parser.add_option("-v","--version",action="store_true",default=False, 65 | help="version info") 66 | (options, args) = parser.parse_args() 67 | if options.version is True: 68 | logger.info("version is {0}".format(Constant.VERSION)) 69 | exit(0) 70 | if options.host is None: 71 | raise ValueError("parameter error:host is mandatory input") 72 | if options.username is None: 73 | raise ValueError("parameter error:username is mandatory input") 74 | if options.schema is None: 75 | raise ValueError("parameter error:schema is mandatory input") 76 | if options.tables is None: 77 | raise ValueError("parameter error:tables is mandatory input") 78 | if not os.path.exists(options.output_file_path) : 79 | raise ValueError("parameter error:output {0} dir is not exists".format(options.output_file_path)) 80 | 81 | if options.skip_insert and options.skip_delete and options.skip_update: 82 | raise ValueError("conld choose at least one event") 83 | 84 | if not options.binlog_end_time is None: 85 | try: 86 | end_to_timestamp=convert_datetime_to_timestamp(options.binlog_end_time, '%Y-%m-%d %H:%M:%S') 87 | except Exception as err: 88 | raise ValueError("binlog_end_time {0} format error,detail error={1}".format(options.binlog_end_time,err.__str__())) 89 | 90 | if not options.binlog_start_time is None: 91 | try: 92 | start_to_timestamp=convert_datetime_to_timestamp(options.binlog_start_time, '%Y-%m-%d %H:%M:%S') 93 | except Exception as err: 94 | raise ValueError("binlog_start_time {0} format error,detail error={1}".format(options.binlog_start_time,err.__str__())) 95 | if not options.binlog_end_time is None: 96 | if start_to_timestamp>=end_to_timestamp: 97 | raise ValueError("binlog_start_time is above binlog_end_time,start_time={0},end_time={1}". 98 | format(options.binlog_start_time,options.binlog_end_time)) 99 | return options 100 | 101 | 102 | 103 | def parse_option(): 104 | """ 105 | 分析用户输入的参数，返回参数对象 106 | """ 107 | 108 | opt=get_check_option() 109 | dict={} 110 | dict["host"]=opt.host 111 | dict["username"]=opt.username 112 | dict["port"]=opt.port 113 | dict["password"]=opt.password 114 | dict["start_binlog_file"]=opt.binlog_start_file_name 115 | dict["start_position"]=opt.binlog_start_file_position 116 | dict["output_file_path"]=opt.output_file_path 117 | dict["schema"]=opt.schema 118 | dict["tablename"]=opt.tables 119 | dict["keep_data"]=True 120 | input_end_to_datetime=opt.binlog_end_time 121 | if not input_end_to_datetime is None: 122 | end_to_timestamp=convert_datetime_to_timestamp(input_end_to_datetime, '%Y-%m-%d %H:%M:%S') 123 | dict["end_to_timestamp"]=int(end_to_timestamp) 124 | input_start_to_datetime=opt.binlog_start_time 125 | if not input_start_to_datetime is None: 126 | start_to_timestamp=convert_datetime_to_timestamp(input_start_to_datetime, '%Y-%m-%d %H:%M:%S') 127 | dict["start_to_timestamp"]=int(start_to_timestamp) 128 | dict["skip_insert"]=opt.skip_insert 129 | dict["skip_update"]=opt.skip_update 130 | dict["skip_delete"]=opt.skip_delete 131 | dict["add_schema_name"]=opt.add_schema_name 132 | 133 | parameter=Parameter(**dict) 134 | parameter.check_tables_exist() 135 | parameter.set_defaut() 136 | 137 | return parameter 138 | 139 | def new_files(parameter): 140 | """ 141 | 建立反向sql文件,保留现场数据文件和保留现场数据文件的建表语句文件 142 | :parameter 用户输入参数的形成的实例 143 | :return: 文件描述符，存在parameter实例中 144 | """ 145 | flash_filename=parameter.get_file_name("flashback") 146 | flashback=codecs.open(flash_filename, "w", encoding=Constant.FILE_ENCODING) 147 | parameter.file["flashback"]=flashback 148 | logger.debug("flashback sql fileno={0}".format(parameter.file["flashback"])) 149 | if parameter.keep_data: 150 | data_filename=parameter.get_file_name("save_data_dml") 151 | data=codecs.open(data_filename, "w", encoding=Constant.FILE_ENCODING) 152 | parameter.file["data"]=data 153 | logger.debug("data sql fileno={0}".format(parameter.file["data"])) 154 | data_create_filename=parameter.get_file_name("save_data_create_table") 155 | data_create=codecs.open(data_create_filename, "w", encoding=Constant.FILE_ENCODING) 156 | parameter.file["data_create"]=data_create 157 | logger.debug("data create sql fileno={0}".format(parameter.file["data_create"])) 158 | 159 | def close_files(parameter): 160 | if parameter.file["data"] is not None: 161 | parameter.file["data"].close() 162 | if parameter.file["data_create"] is not None: 163 | parameter.file["data_create"].close() 164 | if parameter.file["flashback"] is not None: 165 | parameter.file["flashback"].close() 166 | 167 | 168 | def print_stat(parameter): 169 | logger.info("===statistics===") 170 | logger.info("scan {0} events ".format(parameter.stream.event_count)) 171 | logger.info(parameter.stat) 172 | 173 | def main(): 174 | logfilename="{0}/{1}".format(Constant.LOGFILE_PATH,Constant.LOGFILE_NAME) 175 | init_logger(logfilename,logging.INFO) 176 | #init_logger(logfilename) 177 | #TODO 缺少tablemap的报警没有，会导致丢失数据 178 | #TODO 如果表被改动了，基本没有办法哦 179 | try: 180 | parameter=parse_option() 181 | except Exception as err: 182 | logger.error(err.__str__()) 183 | print_stack() 184 | exit(1) 185 | 186 | try: 187 | parameter=parse_option() 188 | logger.info(u"parameter={0}".format(parameter.__dict__)) 189 | new_files(parameter) 190 | deal_all_event(parameter) 191 | generate_create_table(parameter) 192 | print_stat(parameter) 193 | except Exception as err: 194 | logger.error("error:"+err.__str__()) 195 | print_stack() 196 | finally: 197 | parameter.stream.close() 198 | close_files(parameter) 199 | 200 | if __name__=='__main__': 201 | main() -------------------------------------------------------------------------------- /pymysqlreplication/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Python MySQL Replication: 3 | Pure Python Implementation of MySQL replication protocol build on top of 4 | PyMYSQL. 5 | 6 | Licence 7 | ======= 8 | Copyright 2012 Julien Duponchelle 9 | 10 | Licensed under the Apache License, Version 2.0 (the "License"); 11 | you may not use this file except in compliance with the License. 12 | You may obtain a copy of the License at 13 | 14 | http://www.apache.org/licenses/LICENSE-2.0 15 | 16 | Unless required by applicable law or agreed to in writing, software 17 | distributed under the License is distributed on an "AS IS" BASIS, 18 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | See the License for the specific language governing permissions and 20 | limitations under the License. 21 | """ 22 | 23 | from .binlogstream import BinLogStreamReader 24 | -------------------------------------------------------------------------------- /pymysqlreplication/binlogstream.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import pdb 3 | import pymysql 4 | import struct 5 | from datetime import datetime 6 | from pymysql.constants.COMMAND import COM_BINLOG_DUMP, COM_REGISTER_SLAVE 7 | from pymysql.cursors import DictCursor 8 | from pymysql.util import int2byte 9 | 10 | from .packet import BinLogPacketWrapper 11 | from .constants.BINLOG import TABLE_MAP_EVENT, ROTATE_EVENT 12 | from .gtid import GtidSet 13 | from .event import ( 14 | QueryEvent, RotateEvent, FormatDescriptionEvent, 15 | XidEvent, GtidEvent, StopEvent, 16 | BeginLoadQueryEvent, ExecuteLoadQueryEvent, 17 | NotImplementedEvent) 18 | from .row_event import ( 19 | UpdateRowsEvent, WriteRowsEvent, DeleteRowsEvent, TableMapEvent) 20 | 21 | try: 22 | from pymysql.constants.COMMAND import COM_BINLOG_DUMP_GTID 23 | except ImportError: 24 | # Handle old pymysql versions 25 | # See: https://github.com/PyMySQL/PyMySQL/pull/261 26 | COM_BINLOG_DUMP_GTID = 0x1e 27 | 28 | # 2013 Connection Lost 29 | # 2006 MySQL server has gone away 30 | MYSQL_EXPECTED_ERROR_CODES = [2013, 2006] 31 | 32 | 33 | class ReportSlave(object): 34 | 35 | """Represent the values that you may report when connecting as a slave 36 | to a master. SHOW SLAVE HOSTS related""" 37 | 38 | hostname = '' 39 | username = '' 40 | password = '' 41 | port = 0 42 | 43 | def __init__(self, value): 44 | """ 45 | Attributes: 46 | value: string or tuple 47 | if string, then it will be used hostname 48 | if tuple it will be used as (hostname, user, password, port) 49 | """ 50 | 51 | if isinstance(value, (tuple, list)): 52 | try: 53 | self.hostname = value[0] 54 | self.username = value[1] 55 | self.password = value[2] 56 | self.port = int(value[3]) 57 | except IndexError: 58 | pass 59 | elif isinstance(value, dict): 60 | for key in ['hostname', 'username', 'password', 'port']: 61 | try: 62 | setattr(self, key, value[key]) 63 | except KeyError: 64 | pass 65 | else: 66 | self.hostname = value 67 | 68 | def __repr__(self): 69 | return '' %\ 70 | (self.hostname, self.username, self.password, self.port) 71 | 72 | def encoded(self, server_id, master_id=0): 73 | """ 74 | server_id: the slave server-id 75 | master_id: usually 0. Appears as "master id" in SHOW SLAVE HOSTS 76 | on the master. Unknown what else it impacts. 77 | """ 78 | 79 | # 1 [15] COM_REGISTER_SLAVE 80 | # 4 server-id 81 | # 1 slaves hostname length 82 | # string[$len] slaves hostname 83 | # 1 slaves user len 84 | # string[$len] slaves user 85 | # 1 slaves password len 86 | # string[$len] slaves password 87 | # 2 slaves mysql-port 88 | # 4 replication rank 89 | # 4 master-id 90 | 91 | lhostname = len(self.hostname) 92 | lusername = len(self.username) 93 | lpassword = len(self.password) 94 | 95 | packet_len = (1 + # command 96 | 4 + # server-id 97 | 1 + # hostname length 98 | lhostname + 99 | 1 + # username length 100 | lusername + 101 | 1 + # password length 102 | lpassword + 103 | 2 + # slave mysql port 104 | 4 + # replication rank 105 | 4) # master-id 106 | 107 | MAX_STRING_LEN = 257 # one byte for length + 256 chars 108 | 109 | return (struct.pack(' 5.6""" 219 | cur = self._stream_connection.cursor() 220 | cur.execute("SHOW GLOBAL VARIABLES LIKE 'BINLOG_CHECKSUM'") 221 | result = cur.fetchone() 222 | cur.close() 223 | 224 | if result is None: 225 | return False 226 | var, value = result[:2] 227 | if value == 'NONE': 228 | return False 229 | return True 230 | 231 | def _register_slave(self): 232 | if not self.report_slave: 233 | return 234 | 235 | packet = self.report_slave.encoded(self.__server_id) 236 | 237 | if pymysql.__version__ < "0.6": 238 | self._stream_connection.wfile.write(packet) 239 | self._stream_connection.wfile.flush() 240 | self._stream_connection.read_packet() 241 | else: 242 | self._stream_connection._write_bytes(packet) 243 | self._stream_connection._next_seq_id = 1 244 | self._stream_connection._read_packet() 245 | 246 | def __connect_to_stream(self): 247 | # log_pos (4) -- position in the binlog-file to start the stream with 248 | # flags (2) BINLOG_DUMP_NON_BLOCK (0 or 1) 249 | # server_id (4) -- server id of this slave 250 | # log_file (string.EOF) -- filename of the binlog on the master 251 | self._stream_connection = self.pymysql_wrapper(**self.__connection_settings) 252 | 253 | self.__use_checksum = self.__checksum_enabled() 254 | 255 | # If checksum is enabled we need to inform the server about the that 256 | # we support it 257 | if self.__use_checksum: 258 | cur = self._stream_connection.cursor() 259 | cur.execute("set @master_binlog_checksum= @@global.binlog_checksum") 260 | cur.close() 261 | 262 | if self.slave_uuid: 263 | cur = self._stream_connection.cursor() 264 | cur.execute("set @slave_uuid= '%s'" % self.slave_uuid) 265 | cur.close() 266 | 267 | self._register_slave() 268 | 269 | if not self.auto_position: 270 | # only when log_file and log_pos both provided, the position info is 271 | # valid, if not, get the current position from master 272 | if self.log_file is None or self.log_pos is None: 273 | cur = self._stream_connection.cursor() 274 | cur.execute("SHOW MASTER STATUS") 275 | self.log_file, self.log_pos = cur.fetchone()[:2] 276 | cur.close() 277 | 278 | prelude = struct.pack('0: 411 | if (self.event_count % self.process_interval)==0: 412 | stm="scan {0} events ....from binlogfile={1},timestamp={2}".format(self.event_count,self.log_file,datetime.fromtimestamp(binlog_event.timestamp).isoformat()) 413 | print(stm) 414 | if self.end_to_timestamp and binlog_event.timestamp > self.end_to_timestamp: 415 | return None 416 | 417 | if self.skip_to_timestamp and binlog_event.timestamp < self.skip_to_timestamp: 418 | continue 419 | 420 | if binlog_event.event_type == TABLE_MAP_EVENT and \ 421 | binlog_event.event is not None: 422 | self.table_map[binlog_event.event.table_id] = \ 423 | binlog_event.event.get_table() 424 | 425 | if binlog_event.event_type == ROTATE_EVENT: 426 | self.log_pos = binlog_event.event.position 427 | self.log_file = binlog_event.event.next_binlog 428 | #binlog_event.event.dump() 429 | # Table Id in binlog are NOT persistent in MySQL - they are in-memory identifiers 430 | # that means that when MySQL master restarts, it will reuse same table id for different tables 431 | # which will cause errors for us since our in-memory map will try to decode row data with 432 | # wrong table schema. 433 | # The fix is to rely on the fact that MySQL will also rotate to a new binlog file every time it 434 | # restarts. That means every rotation we see *could* be a sign of restart and so potentially 435 | # invalidates all our cached table id to schema mappings. This means we have to load them all 436 | # again for each logfile which is potentially wasted effort but we can't really do much better 437 | # without being broken in restart case 438 | self.table_map = {} 439 | elif binlog_event.log_pos: 440 | self.log_pos = binlog_event.log_pos 441 | 442 | # event is none if we have filter it on packet level 443 | # we filter also not allowed events 444 | if binlog_event.event is None or (binlog_event.event.__class__ not in self.__allowed_events): 445 | continue 446 | 447 | return binlog_event.event 448 | 449 | def _allowed_event_list(self, only_events, ignored_events, 450 | filter_non_implemented_events): 451 | if only_events is not None: 452 | events = set(only_events) 453 | else: 454 | events = set(( 455 | QueryEvent, 456 | RotateEvent, 457 | StopEvent, 458 | FormatDescriptionEvent, 459 | XidEvent, 460 | GtidEvent, 461 | BeginLoadQueryEvent, 462 | ExecuteLoadQueryEvent, 463 | UpdateRowsEvent, 464 | WriteRowsEvent, 465 | DeleteRowsEvent, 466 | TableMapEvent, 467 | NotImplementedEvent)) 468 | if ignored_events is not None: 469 | for e in ignored_events: 470 | events.remove(e) 471 | if filter_non_implemented_events: 472 | try: 473 | events.remove(NotImplementedEvent) 474 | except KeyError: 475 | pass 476 | return frozenset(events) 477 | 478 | def __get_table_information(self, schema, table): 479 | for i in range(1, 3): 480 | try: 481 | if not self.__connected_ctl: 482 | self.__connect_to_ctl() 483 | 484 | cur = self._ctl_connection.cursor() 485 | #changed by yilai,add schema name "information_schema" for rds in alibaba 486 | cur.execute(""" 487 | SELECT 488 | COLUMN_NAME, COLLATION_NAME, CHARACTER_SET_NAME, 489 | COLUMN_COMMENT, COLUMN_TYPE, COLUMN_KEY 490 | FROM 491 | information_schema.columns 492 | WHERE 493 | table_schema = %s AND table_name = %s 494 | """, (schema, table)) 495 | 496 | return cur.fetchall() 497 | except pymysql.OperationalError as error: 498 | code, message = error.args 499 | if code in MYSQL_EXPECTED_ERROR_CODES: 500 | self.__connected_ctl = False 501 | continue 502 | else: 503 | raise error 504 | 505 | def __iter__(self): 506 | return iter(self.fetchone, None) 507 | -------------------------------------------------------------------------------- /pymysqlreplication/bitmap.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | bitCountInByte = [ 4 | 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 5 | 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 6 | 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 7 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 8 | 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 9 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 10 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 11 | 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 12 | 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 13 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 14 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 15 | 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 16 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 17 | 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 18 | 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 19 | 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8, 20 | ] 21 | 22 | # Calculate totol bit counts in a bitmap 23 | def BitCount(bitmap): 24 | n = 0 25 | for i in range(0, len(bitmap)): 26 | bit = bitmap[i] 27 | if type(bit) is str: 28 | bit = ord(bit) 29 | n += bitCountInByte[bit] 30 | return n 31 | 32 | # Get the bit set at offset position in bitmap 33 | def BitGet(bitmap, position): 34 | bit = bitmap[int(position / 8)] 35 | if type(bit) is str: 36 | bit = ord(bit) 37 | return bit & (1 << (position & 7)) 38 | -------------------------------------------------------------------------------- /pymysqlreplication/column.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import struct 4 | 5 | from .constants import FIELD_TYPE 6 | 7 | 8 | class Column(object): 9 | """Definition of a column 10 | """ 11 | 12 | def __init__(self, *args, **kwargs): 13 | if len(args) == 3: 14 | self.__parse_column_definition(*args) 15 | else: 16 | self.__dict__.update(kwargs) 17 | 18 | def __parse_column_definition(self, column_type, column_schema, packet): 19 | self.type = column_type 20 | self.name = column_schema["COLUMN_NAME"] 21 | self.collation_name = column_schema["COLLATION_NAME"] 22 | self.character_set_name = column_schema["CHARACTER_SET_NAME"] 23 | self.comment = column_schema["COLUMN_COMMENT"] 24 | self.unsigned = column_schema["COLUMN_TYPE"].find("unsigned") != -1 25 | self.type_is_bool = False 26 | self.is_primary = column_schema["COLUMN_KEY"] == "PRI" 27 | 28 | if self.type == FIELD_TYPE.VARCHAR: 29 | self.max_length = struct.unpack('> 8 62 | if real_type == FIELD_TYPE.SET or real_type == FIELD_TYPE.ENUM: 63 | self.type = real_type 64 | self.size = metadata & 0x00ff 65 | self.__read_enum_metadata(column_schema) 66 | else: 67 | self.max_length = (((metadata >> 4) & 0x300) ^ 0x300) \ 68 | + (metadata & 0x00ff) 69 | 70 | def __read_enum_metadata(self, column_schema): 71 | enums = column_schema["COLUMN_TYPE"] 72 | if self.type == FIELD_TYPE.ENUM: 73 | self.enum_values = enums.replace('enum(', '')\ 74 | .replace(')', '').replace('\'', '').split(',') 75 | else: 76 | self.set_values = enums.replace('set(', '')\ 77 | .replace(')', '').replace('\'', '').split(',') 78 | 79 | def __eq__(self, other): 80 | return self.data == other.data 81 | 82 | def __ne__(self, other): 83 | return not self.__eq__(other) 84 | 85 | def serializable_data(self): 86 | return self.data 87 | 88 | @property 89 | def data(self): 90 | return dict((k, v) for (k, v) in self.__dict__.items() if not k.startswith('_')) 91 | -------------------------------------------------------------------------------- /pymysqlreplication/constants/BINLOG.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | UNKNOWN_EVENT = 0x00 4 | START_EVENT_V3 = 0x01 5 | QUERY_EVENT = 0x02 6 | STOP_EVENT = 0x03 7 | ROTATE_EVENT = 0x04 8 | INTVAR_EVENT = 0x05 9 | LOAD_EVENT = 0x06 10 | SLAVE_EVENT = 0x07 11 | CREATE_FILE_EVENT = 0x08 12 | APPEND_BLOCK_EVENT = 0x09 13 | EXEC_LOAD_EVENT = 0x0a 14 | DELETE_FILE_EVENT = 0x0b 15 | NEW_LOAD_EVENT = 0x0c 16 | RAND_EVENT = 0x0d 17 | USER_VAR_EVENT = 0x0e 18 | FORMAT_DESCRIPTION_EVENT = 0x0f 19 | XID_EVENT = 0x10 20 | BEGIN_LOAD_QUERY_EVENT = 0x11 21 | EXECUTE_LOAD_QUERY_EVENT = 0x12 22 | TABLE_MAP_EVENT = 0x13 23 | PRE_GA_WRITE_ROWS_EVENT = 0x14 24 | PRE_GA_UPDATE_ROWS_EVENT = 0x15 25 | PRE_GA_DELETE_ROWS_EVENT = 0x16 26 | WRITE_ROWS_EVENT_V1 = 0x17 27 | UPDATE_ROWS_EVENT_V1 = 0x18 28 | DELETE_ROWS_EVENT_V1 = 0x19 29 | INCIDENT_EVENT = 0x1a 30 | HEARTBEAT_LOG_EVENT = 0x1b 31 | IGNORABLE_LOG_EVENT = 0x1c 32 | ROWS_QUERY_LOG_EVENT = 0x1d 33 | WRITE_ROWS_EVENT_V2 = 0x1e 34 | UPDATE_ROWS_EVENT_V2 = 0x1f 35 | DELETE_ROWS_EVENT_V2 = 0x20 36 | GTID_LOG_EVENT = 0x21 37 | ANONYMOUS_GTID_LOG_EVENT = 0x22 38 | PREVIOUS_GTIDS_LOG_EVENT = 0x23 39 | 40 | # INTVAR types 41 | INTVAR_INVALID_INT_EVENT = 0x00 42 | INTVAR_LAST_INSERT_ID_EVENT = 0x01 43 | INTVAR_INSERT_ID_EVENT = 0x02 44 | -------------------------------------------------------------------------------- /pymysqlreplication/constants/FIELD_TYPE.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Original code from PyMySQL 4 | # Copyright (c) 2010 PyMySQL contributors 5 | # 6 | #Permission is hereby granted, free of charge, to any person obtaining a copy 7 | #of this software and associated documentation files (the "Software"), to deal 8 | #in the Software without restriction, including without limitation the rights 9 | #to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | #copies of the Software, and to permit persons to whom the Software is 11 | #furnished to do so, subject to the following conditions: 12 | # 13 | #The above copyright notice and this permission notice shall be included in 14 | #all copies or substantial portions of the Software. 15 | # 16 | #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | #FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | #AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | #THE SOFTWARE. 23 | 24 | DECIMAL = 0 25 | TINY = 1 26 | SHORT = 2 27 | LONG = 3 28 | FLOAT = 4 29 | DOUBLE = 5 30 | NULL = 6 31 | TIMESTAMP = 7 32 | LONGLONG = 8 33 | INT24 = 9 34 | DATE = 10 35 | TIME = 11 36 | DATETIME = 12 37 | YEAR = 13 38 | NEWDATE = 14 39 | VARCHAR = 15 40 | BIT = 16 41 | TIMESTAMP2 = 17 42 | DATETIME2 = 18 43 | TIME2 = 19 44 | NEWDECIMAL = 246 45 | ENUM = 247 46 | SET = 248 47 | TINY_BLOB = 249 48 | MEDIUM_BLOB = 250 49 | LONG_BLOB = 251 50 | BLOB = 252 51 | VAR_STRING = 253 52 | STRING = 254 53 | GEOMETRY = 255 54 | 55 | CHAR = TINY 56 | INTERVAL = ENUM 57 | -------------------------------------------------------------------------------- /pymysqlreplication/constants/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .BINLOG import * 4 | from .FIELD_TYPE import * 5 | -------------------------------------------------------------------------------- /pymysqlreplication/event.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import struct 4 | import datetime 5 | 6 | from pymysql.util import byte2int, int2byte 7 | 8 | 9 | class BinLogEvent(object): 10 | def __init__(self, from_packet, event_size, table_map, ctl_connection, 11 | only_tables = None, 12 | only_schemas = None, 13 | freeze_schema = False): 14 | self.packet = from_packet 15 | self.table_map = table_map 16 | self.event_type = self.packet.event_type 17 | self.timestamp = self.packet.timestamp 18 | self.event_size = event_size 19 | self._ctl_connection = ctl_connection 20 | # The event have been fully processed, if processed is false 21 | # the event will be skipped 22 | self._processed = True 23 | self.complete = True 24 | 25 | def _read_table_id(self): 26 | # Table ID is 6 byte 27 | # pad little-endian number 28 | table_id = self.packet.read(6) + int2byte(0) + int2byte(0) 29 | return struct.unpack('' % self.gtid 73 | 74 | 75 | class RotateEvent(BinLogEvent): 76 | """Change MySQL bin log file 77 | 78 | Attributes: 79 | position: Position inside next binlog 80 | next_binlog: Name of next binlog file 81 | """ 82 | def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): 83 | super(RotateEvent, self).__init__(from_packet, event_size, table_map, 84 | ctl_connection, **kwargs) 85 | self.position = struct.unpack('' % self 49 | 50 | @property 51 | def encoded_length(self): 52 | return (16 + # sid 53 | 8 + # n_intervals 54 | 2 * # stop/start 55 | 8 * # stop/start mark encoded as int64 56 | len(self.intervals)) 57 | 58 | def encode(self): 59 | buffer = b'' 60 | # sid 61 | buffer += binascii.unhexlify(self.sid.replace('-', '')) 62 | # n_intervals 63 | buffer += struct.pack('' % self.gtids 134 | 135 | @property 136 | def encoded_length(self): 137 | return (8 + # n_sids 138 | sum(x.encoded_length for x in self.gtids)) 139 | 140 | def encoded(self): 141 | return b'' + (struct.pack(' 0: 107 | data = self.__data_buffer[:size] 108 | self.__data_buffer = self.__data_buffer[size:] 109 | if len(data) == size: 110 | return data 111 | else: 112 | return data + self.packet.read(size - len(data)) 113 | return self.packet.read(size) 114 | 115 | def unread(self, data): 116 | '''Push again data in data buffer. It's use when you want 117 | to extract a bit from a value a let the rest of the code normally 118 | read the datas''' 119 | self.read_bytes -= len(data) 120 | self.__data_buffer += data 121 | 122 | def advance(self, size): 123 | size = int(size) 124 | self.read_bytes += size 125 | buffer_len = len(self.__data_buffer) 126 | if buffer_len > 0: 127 | self.__data_buffer = self.__data_buffer[size:] 128 | if size > buffer_len: 129 | self.packet.advance(size - buffer_len) 130 | else: 131 | self.packet.advance(size) 132 | 133 | def read_length_coded_binary(self): 134 | """Read a 'Length Coded Binary' number from the data buffer. 135 | 136 | Length coded numbers can be anywhere from 1 to 9 bytes depending 137 | on the value of the first byte. 138 | 139 | From PyMYSQL source code 140 | """ 141 | c = byte2int(self.read(1)) 142 | if c == NULL_COLUMN: 143 | return None 144 | if c < UNSIGNED_CHAR_COLUMN: 145 | return c 146 | elif c == UNSIGNED_SHORT_COLUMN: 147 | return self.unpack_uint16(self.read(UNSIGNED_SHORT_LENGTH)) 148 | elif c == UNSIGNED_INT24_COLUMN: 149 | return self.unpack_int24(self.read(UNSIGNED_INT24_LENGTH)) 150 | elif c == UNSIGNED_INT64_COLUMN: 151 | return self.unpack_int64(self.read(UNSIGNED_INT64_LENGTH)) 152 | 153 | def read_length_coded_string(self): 154 | """Read a 'Length Coded String' from the data buffer. 155 | 156 | A 'Length Coded String' consists first of a length coded 157 | (unsigned, positive) integer represented in 1-9 bytes followed by 158 | that many bytes of binary data. (For example "cat" would be "3cat".) 159 | 160 | From PyMYSQL source code 161 | """ 162 | length = self.read_length_coded_binary() 163 | if length is None: 164 | return None 165 | return self.read(length).decode() 166 | 167 | def __getattr__(self, key): 168 | if hasattr(self.packet, key): 169 | return getattr(self.packet, key) 170 | 171 | raise AttributeError("%s instance has no attribute '%s'" % 172 | (self.__class__, key)) 173 | 174 | def read_int_be_by_size(self, size): 175 | '''Read a big endian integer values based on byte number''' 176 | if size == 1: 177 | return struct.unpack('>b', self.read(size))[0] 178 | elif size == 2: 179 | return struct.unpack('>h', self.read(size))[0] 180 | elif size == 3: 181 | return self.read_int24_be() 182 | elif size == 4: 183 | return struct.unpack('>i', self.read(size))[0] 184 | elif size == 5: 185 | return self.read_int40_be() 186 | elif size == 8: 187 | return struct.unpack('>l', self.read(size))[0] 188 | 189 | def read_uint_by_size(self, size): 190 | '''Read a little endian integer values based on byte number''' 191 | if size == 1: 192 | return self.read_uint8() 193 | elif size == 2: 194 | return self.read_uint16() 195 | elif size == 3: 196 | return self.read_uint24() 197 | elif size == 4: 198 | return self.read_uint32() 199 | elif size == 5: 200 | return self.read_uint40() 201 | elif size == 6: 202 | return self.read_uint48() 203 | elif size == 7: 204 | return self.read_uint56() 205 | elif size == 8: 206 | return self.read_uint64() 207 | 208 | def read_length_coded_pascal_string(self, size): 209 | """Read a string with length coded using pascal style. 210 | The string start by the size of the string 211 | """ 212 | length = self.read_uint_by_size(size) 213 | return self.read(length) 214 | 215 | def read_int24(self): 216 | a, b, c = struct.unpack("BBB", self.read(3)) 217 | res = a | (b << 8) | (c << 16) 218 | if res >= 0x800000: 219 | res -= 0x1000000 220 | return res 221 | 222 | def read_int24_be(self): 223 | a, b, c = struct.unpack('BBB', self.read(3)) 224 | res = (a << 16) | (b << 8) | c 225 | if res >= 0x800000: 226 | res -= 0x1000000 227 | return res 228 | 229 | def read_uint8(self): 230 | return struct.unpack('IB", self.read(5)) 248 | return b + (a << 8) 249 | 250 | def read_uint48(self): 251 | a, b, c = struct.unpack(" 255: 117 | values[name] = self.__read_string(2, column) 118 | else: 119 | values[name] = self.__read_string(1, column) 120 | elif column.type == FIELD_TYPE.NEWDECIMAL: 121 | values[name] = self.__read_new_decimal(column) 122 | elif column.type == FIELD_TYPE.BLOB: 123 | values[name] = self.__read_string(column.length_size, column) 124 | elif column.type == FIELD_TYPE.DATETIME: 125 | values[name] = self.__read_datetime() 126 | elif column.type == FIELD_TYPE.TIME: 127 | values[name] = self.__read_time() 128 | elif column.type == FIELD_TYPE.DATE: 129 | values[name] = self.__read_date() 130 | elif column.type == FIELD_TYPE.TIMESTAMP: 131 | values[name] = datetime.datetime.fromtimestamp( 132 | self.packet.read_uint32()) 133 | 134 | # For new date format: 135 | elif column.type == FIELD_TYPE.DATETIME2: 136 | values[name] = self.__read_datetime2(column) 137 | elif column.type == FIELD_TYPE.TIME2: 138 | values[name] = self.__read_time2(column) 139 | elif column.type == FIELD_TYPE.TIMESTAMP2: 140 | values[name] = self.__add_fsp_to_time( 141 | datetime.datetime.fromtimestamp( 142 | self.packet.read_int_be_by_size(4)), column) 143 | elif column.type == FIELD_TYPE.LONGLONG: 144 | if unsigned: 145 | values[name] = self.packet.read_uint64() 146 | else: 147 | values[name] = self.packet.read_int64() 148 | elif column.type == FIELD_TYPE.YEAR: 149 | values[name] = self.packet.read_uint8() + 1900 150 | elif column.type == FIELD_TYPE.ENUM: 151 | values[name] = column.enum_values[ 152 | self.packet.read_uint_by_size(column.size) - 1] 153 | elif column.type == FIELD_TYPE.SET: 154 | # We read set columns as a bitmap telling us which options 155 | # are enabled 156 | bit_mask = self.packet.read_uint_by_size(column.size) 157 | values[name] = set( 158 | val for idx, val in enumerate(column.set_values) 159 | if bit_mask & 2 ** idx 160 | ) or None 161 | 162 | elif column.type == FIELD_TYPE.BIT: 163 | values[name] = self.__read_bit(column) 164 | elif column.type == FIELD_TYPE.GEOMETRY: 165 | values[name] = self.packet.read_length_coded_pascal_string( 166 | column.length_size) 167 | else: 168 | raise NotImplementedError("Unknown MySQL column type: %d" % 169 | (column.type)) 170 | 171 | nullBitmapIndex += 1 172 | 173 | return values 174 | 175 | def __add_fsp_to_time(self, time, column): 176 | """Read and add the fractional part of time 177 | For more details about new date format: 178 | http://dev.mysql.com/doc/internals/en/date-and-time-data-type-representation.html 179 | """ 180 | microsecond = self.__read_fsp(column) 181 | if microsecond > 0: 182 | time = time.replace(microsecond=microsecond) 183 | return time 184 | 185 | def __read_fsp(self, column): 186 | read = 0 187 | if column.fsp == 1 or column.fsp == 2: 188 | read = 1 189 | elif column.fsp == 3 or column.fsp == 4: 190 | read = 2 191 | elif column.fsp == 5 or column.fsp == 6: 192 | read = 3 193 | if read > 0: 194 | microsecond = self.packet.read_int_be_by_size(read) 195 | if column.fsp % 2: 196 | return int(microsecond / 10) 197 | else: 198 | return microsecond 199 | 200 | return 0 201 | 202 | def __read_string(self, size, column): 203 | string = self.packet.read_length_coded_pascal_string(size) 204 | if column.character_set_name is not None: 205 | string = string.decode(charset_to_encoding(column.character_set_name)) 206 | return string 207 | 208 | def __read_bit(self, column): 209 | """Read MySQL BIT type""" 210 | resp = "" 211 | for byte in range(0, column.bytes): 212 | current_byte = "" 213 | data = self.packet.read_uint8() 214 | if byte == 0: 215 | if column.bytes == 1: 216 | end = column.bits 217 | else: 218 | end = column.bits % 8 219 | if end == 0: 220 | end = 8 221 | else: 222 | end = 8 223 | for bit in range(0, end): 224 | if data & (1 << bit): 225 | current_byte += "1" 226 | else: 227 | current_byte += "0" 228 | resp += current_byte[::-1] 229 | return resp 230 | 231 | def __read_time(self): 232 | time = self.packet.read_uint24() 233 | date = datetime.timedelta( 234 | hours=int(time / 10000), 235 | minutes=int((time % 10000) / 100), 236 | seconds=int(time % 100)) 237 | return date 238 | 239 | def __read_time2(self, column): 240 | """TIME encoding for nonfractional part: 241 | 242 | 1 bit sign (1= non-negative, 0= negative) 243 | 1 bit unused (reserved for future extensions) 244 | 10 bits hour (0-838) 245 | 6 bits minute (0-59) 246 | 6 bits second (0-59) 247 | --------------------- 248 | 24 bits = 3 bytes 249 | """ 250 | data = self.packet.read_int_be_by_size(3) 251 | 252 | sign = 1 if self.__read_binary_slice(data, 0, 1, 24) else -1 253 | if sign == -1: 254 | # negative integers are stored as 2's compliment 255 | # hence take 2's compliment again to get the right value. 256 | data = ~data + 1 257 | 258 | t = datetime.timedelta( 259 | hours=sign*self.__read_binary_slice(data, 2, 10, 24), 260 | minutes=self.__read_binary_slice(data, 12, 6, 24), 261 | seconds=self.__read_binary_slice(data, 18, 6, 24), 262 | microseconds=self.__read_fsp(column) 263 | ) 264 | return t 265 | 266 | def __read_date(self): 267 | time = self.packet.read_uint24() 268 | if time == 0: # nasty mysql 0000-00-00 dates 269 | return None 270 | 271 | year = (time & ((1 << 15) - 1) << 9) >> 9 272 | month = (time & ((1 << 4) - 1) << 5) >> 5 273 | day = (time & ((1 << 5) - 1)) 274 | if year == 0 or month == 0 or day == 0: 275 | return None 276 | 277 | date = datetime.date( 278 | year=year, 279 | month=month, 280 | day=day 281 | ) 282 | return date 283 | 284 | def __read_datetime(self): 285 | value = self.packet.read_uint64() 286 | if value == 0: # nasty mysql 0000-00-00 dates 287 | return None 288 | 289 | date = value / 1000000 290 | time = int(value % 1000000) 291 | 292 | year = int(date / 10000) 293 | month = int((date % 10000) / 100) 294 | day = int(date % 100) 295 | if year == 0 or month == 0 or day == 0: 296 | return None 297 | 298 | date = datetime.datetime( 299 | year=year, 300 | month=month, 301 | day=day, 302 | hour=int(time / 10000), 303 | minute=int((time % 10000) / 100), 304 | second=int(time % 100)) 305 | return date 306 | 307 | def __read_datetime2(self, column): 308 | """DATETIME 309 | 310 | 1 bit sign (1= non-negative, 0= negative) 311 | 17 bits year*13+month (year 0-9999, month 0-12) 312 | 5 bits day (0-31) 313 | 5 bits hour (0-23) 314 | 6 bits minute (0-59) 315 | 6 bits second (0-59) 316 | --------------------------- 317 | 40 bits = 5 bytes 318 | """ 319 | data = self.packet.read_int_be_by_size(5) 320 | year_month = self.__read_binary_slice(data, 1, 17, 40) 321 | try: 322 | t = datetime.datetime( 323 | year=int(year_month / 13), 324 | month=year_month % 13, 325 | day=self.__read_binary_slice(data, 18, 5, 40), 326 | hour=self.__read_binary_slice(data, 23, 5, 40), 327 | minute=self.__read_binary_slice(data, 28, 6, 40), 328 | second=self.__read_binary_slice(data, 34, 6, 40)) 329 | except ValueError: 330 | return None 331 | return self.__add_fsp_to_time(t, column) 332 | 333 | def __read_new_decimal(self, column): 334 | """Read MySQL's new decimal format introduced in MySQL 5""" 335 | 336 | # This project was a great source of inspiration for 337 | # understanding this storage format. 338 | # https://github.com/jeremycole/mysql_binlog 339 | 340 | digits_per_integer = 9 341 | compressed_bytes = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4] 342 | integral = (column.precision - column.decimals) 343 | uncomp_integral = int(integral / digits_per_integer) 344 | uncomp_fractional = int(column.decimals / digits_per_integer) 345 | comp_integral = integral - (uncomp_integral * digits_per_integer) 346 | comp_fractional = column.decimals - (uncomp_fractional 347 | * digits_per_integer) 348 | 349 | # Support negative 350 | # The sign is encoded in the high bit of the the byte 351 | # But this bit can also be used in the value 352 | value = self.packet.read_uint8() 353 | if value & 0x80 != 0: 354 | res = "" 355 | mask = 0 356 | else: 357 | mask = -1 358 | res = "-" 359 | self.packet.unread(struct.pack(' 0: 363 | value = self.packet.read_int_be_by_size(size) ^ mask 364 | res += str(value) 365 | 366 | for i in range(0, uncomp_integral): 367 | value = struct.unpack('>i', self.packet.read(4))[0] ^ mask 368 | res += '%09d' % value 369 | 370 | res += "." 371 | 372 | for i in range(0, uncomp_fractional): 373 | value = struct.unpack('>i', self.packet.read(4))[0] ^ mask 374 | res += '%09d' % value 375 | 376 | size = compressed_bytes[comp_fractional] 377 | if size > 0: 378 | value = self.packet.read_int_be_by_size(size) ^ mask 379 | res += '%0*d' % (comp_fractional, value) 380 | 381 | return decimal.Decimal(res) 382 | 383 | def __read_binary_slice(self, binary, start, size, data_length): 384 | """ 385 | Read a part of binary data and extract a number 386 | binary: the data 387 | start: From which bit (1 to X) 388 | size: How many bits should be read 389 | data_length: data size 390 | """ 391 | binary = binary >> data_length - (start + size) 392 | mask = ((1 << size) - 1) 393 | return binary & mask 394 | 395 | def _dump(self): 396 | super(RowsEvent, self)._dump() 397 | print("Table: %s.%s" % (self.schema, self.table)) 398 | print("Affected columns: %d" % self.number_of_columns) 399 | print("Changed rows: %d" % (len(self.rows))) 400 | print("primary key: {0}".format(self.primary_key)) 401 | 402 | def _fetch_rows(self): 403 | self.__rows = [] 404 | 405 | if not self.complete: 406 | return 407 | 408 | while self.packet.read_bytes + 1 < self.event_size: 409 | self.__rows.append(self._fetch_one_row()) 410 | 411 | @property 412 | def rows(self): 413 | if self.__rows is None: 414 | self._fetch_rows() 415 | return self.__rows 416 | 417 | 418 | class DeleteRowsEvent(RowsEvent): 419 | """This event is trigger when a row in the database is removed 420 | 421 | For each row you have a hash with a single key: values which contain the data of the removed line. 422 | """ 423 | 424 | def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): 425 | super(DeleteRowsEvent, self).__init__(from_packet, event_size, 426 | table_map, ctl_connection, **kwargs) 427 | if self._processed: 428 | self.columns_present_bitmap = self.packet.read( 429 | (self.number_of_columns + 7) / 8) 430 | 431 | def _fetch_one_row(self): 432 | row = {} 433 | 434 | row["values"] = self._read_column_data(self.columns_present_bitmap) 435 | return row 436 | 437 | def _dump(self): 438 | super(DeleteRowsEvent, self)._dump() 439 | print("Values:") 440 | for row in self.rows: 441 | print("--") 442 | for key in row["values"]: 443 | print("*", key, ":", row["values"][key]) 444 | 445 | 446 | class WriteRowsEvent(RowsEvent): 447 | """This event is triggered when a row in database is added 448 | 449 | For each row you have a hash with a single key: values which contain the data of the new line. 450 | """ 451 | 452 | def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): 453 | super(WriteRowsEvent, self).__init__(from_packet, event_size, 454 | table_map, ctl_connection, **kwargs) 455 | if self._processed: 456 | self.columns_present_bitmap = self.packet.read( 457 | (self.number_of_columns + 7) / 8) 458 | 459 | def _fetch_one_row(self): 460 | row = {} 461 | 462 | row["values"] = self._read_column_data(self.columns_present_bitmap) 463 | return row 464 | 465 | def _dump(self): 466 | super(WriteRowsEvent, self)._dump() 467 | print("Values:") 468 | for row in self.rows: 469 | print("--") 470 | for key in row["values"]: 471 | print("*", key, ":", row["values"][key]) 472 | 473 | 474 | class UpdateRowsEvent(RowsEvent): 475 | """This event is triggered when a row in the database is changed 476 | 477 | For each row you got a hash with two keys: 478 | * before_values 479 | * after_values 480 | 481 | Depending of your MySQL configuration the hash can contains the full row or only the changes: 482 | http://dev.mysql.com/doc/refman/5.6/en/replication-options-binary-log.html#sysvar_binlog_row_image 483 | """ 484 | 485 | def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): 486 | super(UpdateRowsEvent, self).__init__(from_packet, event_size, 487 | table_map, ctl_connection, **kwargs) 488 | if self._processed: 489 | #Body 490 | self.columns_present_bitmap = self.packet.read( 491 | (self.number_of_columns + 7) / 8) 492 | self.columns_present_bitmap2 = self.packet.read( 493 | (self.number_of_columns + 7) / 8) 494 | 495 | def _fetch_one_row(self): 496 | row = {} 497 | 498 | row["before_values"] = self._read_column_data(self.columns_present_bitmap) 499 | 500 | row["after_values"] = self._read_column_data(self.columns_present_bitmap2) 501 | return row 502 | 503 | def _dump(self): 504 | super(UpdateRowsEvent, self)._dump() 505 | print("Affected columns: %d" % self.number_of_columns) 506 | print("Values:") 507 | for row in self.rows: 508 | print("--") 509 | for key in row["before_values"]: 510 | print("*%s:%s=>%s" % (key, 511 | row["before_values"][key], 512 | row["after_values"][key])) 513 | 514 | 515 | class TableMapEvent(BinLogEvent): 516 | """This evenement describe the structure of a table. 517 | It's send before a change append on a table. 518 | A end user of the lib should have no usage of this 519 | """ 520 | 521 | def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): 522 | super(TableMapEvent, self).__init__(from_packet, event_size, 523 | table_map, ctl_connection, **kwargs) 524 | self.__only_tables = kwargs["only_tables"] 525 | self.__only_schemas = kwargs["only_schemas"] 526 | self.__freeze_schema = kwargs["freeze_schema"] 527 | 528 | # Post-Header 529 | self.table_id = self._read_table_id() 530 | 531 | if self.table_id in table_map and self.__freeze_schema: 532 | self._processed = False 533 | return 534 | 535 | self.flags = struct.unpack('= 5.6: 70 | return True 71 | return False 72 | 73 | def isMySQL57(self): 74 | version = float(self.getMySQLVersion().rsplit('.', 1)[0]) 75 | return version == 5.7 76 | 77 | @property 78 | def supportsGTID(self): 79 | if not self.isMySQL56AndMore(): 80 | return False 81 | return self.execute("SELECT @@global.gtid_mode ").fetchone()[0] == "ON" 82 | 83 | def connect_conn_control(self, db): 84 | if self.conn_control is not None: 85 | self.conn_control.close() 86 | self.conn_control = pymysql.connect(**db) 87 | 88 | def tearDown(self): 89 | self.conn_control.close() 90 | self.conn_control = None 91 | self.stream.close() 92 | self.stream = None 93 | 94 | def execute(self, query): 95 | c = self.conn_control.cursor() 96 | c.execute(query) 97 | return c 98 | 99 | def resetBinLog(self): 100 | self.execute("RESET MASTER") 101 | if self.stream is not None: 102 | self.stream.close() 103 | self.stream = BinLogStreamReader(self.database, server_id=1024, 104 | ignored_events=self.ignoredEvents()) 105 | 106 | def set_sql_mode(self): 107 | """set sql_mode to test with same sql_mode (mysql 5.7 sql_mode default is changed)""" 108 | version = float(self.getMySQLVersion().rsplit('.', 1)[0]) 109 | if version == 5.7: 110 | self.execute("set @@sql_mode='NO_ENGINE_SUBSTITUTION'") 111 | 112 | def bin_log_format(self): 113 | query = "select @@binlog_format" 114 | cursor = self.execute(query) 115 | result = cursor.fetchone() 116 | return result[0] 117 | 118 | def bin_log_basename(self): 119 | cursor = self.execute('select @@log_bin_basename') 120 | bin_log_basename = cursor.fetchone()[0] 121 | bin_log_basename = bin_log_basename.split("/")[-1] 122 | return bin_log_basename 123 | -------------------------------------------------------------------------------- /pymysqlreplication/tests/benchmark.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # This is a sample script in order to make benchmark 4 | # on library speed. 5 | # 6 | # 7 | 8 | import pymysql 9 | import time 10 | import random 11 | import os 12 | from pymysqlreplication import BinLogStreamReader 13 | from pymysqlreplication.row_event import * 14 | import cProfile 15 | 16 | 17 | def execute(con, query): 18 | c = con.cursor() 19 | c.execute(query) 20 | return c 21 | 22 | def consume_events(): 23 | stream = BinLogStreamReader(connection_settings=database, 24 | server_id=3, 25 | resume_stream=False, 26 | blocking=True, 27 | only_events = [UpdateRowsEvent], 28 | only_tables = ['test'] ) 29 | start = time.clock() 30 | i = 0.0 31 | for binlogevent in stream: 32 | i += 1.0 33 | if i % 1000 == 0: 34 | print("%d event by seconds (%d total)" % (i / (time.clock() - start), i)) 35 | stream.close() 36 | 37 | 38 | database = { 39 | "host": "localhost", 40 | "user": "root", 41 | "passwd": "", 42 | "use_unicode": True, 43 | "charset": "utf8", 44 | "db": "pymysqlreplication_test" 45 | } 46 | 47 | conn = pymysql.connect(**database) 48 | 49 | execute(conn, "DROP DATABASE IF EXISTS pymysqlreplication_test") 50 | execute(conn, "CREATE DATABASE pymysqlreplication_test") 51 | conn = pymysql.connect(**database) 52 | execute(conn, "CREATE TABLE test (i INT) ENGINE = MEMORY") 53 | execute(conn, "INSERT INTO test VALUES(1)") 54 | execute(conn, "CREATE TABLE test2 (i INT) ENGINE = MEMORY") 55 | execute(conn, "INSERT INTO test2 VALUES(1)") 56 | execute(conn, "RESET MASTER") 57 | 58 | 59 | if os.fork() != 0: 60 | print("Start insert data") 61 | while True: 62 | execute(conn, "UPDATE test SET i = i + 1;") 63 | execute(conn, "UPDATE test2 SET i = i + 1;") 64 | else: 65 | consume_events() 66 | #cProfile.run('consume_events()') 67 | 68 | -------------------------------------------------------------------------------- /pymysqlreplication/tests/test_data_objects.py: -------------------------------------------------------------------------------- 1 | import sys 2 | if sys.version_info < (2, 7): 3 | import unittest2 as unittest 4 | else: 5 | import unittest 6 | 7 | from pymysqlreplication.column import Column 8 | from pymysqlreplication.table import Table 9 | from pymysqlreplication.event import GtidEvent 10 | 11 | from pymysqlreplication.tests import base 12 | 13 | __all__ = ["TestDataObjects"] 14 | 15 | 16 | class TestDataObjects(base.PyMySQLReplicationTestCase): 17 | def ignoredEvents(self): 18 | return [GtidEvent] 19 | 20 | def test_column_is_primary(self): 21 | col = Column(1, 22 | {"COLUMN_NAME": "test", 23 | "COLLATION_NAME": "utf8_general_ci", 24 | "CHARACTER_SET_NAME": "UTF8", 25 | "COLUMN_COMMENT": "", 26 | "COLUMN_TYPE": "tinyint(2)", 27 | "COLUMN_KEY": "PRI"}, 28 | None) 29 | self.assertEqual(True, col.is_primary) 30 | 31 | def test_column_not_primary(self): 32 | col = Column(1, 33 | {"COLUMN_NAME": "test", 34 | "COLLATION_NAME": "utf8_general_ci", 35 | "CHARACTER_SET_NAME": "UTF8", 36 | "COLUMN_COMMENT": "", 37 | "COLUMN_TYPE": "tinyint(2)", 38 | "COLUMN_KEY": ""}, 39 | None) 40 | self.assertEqual(False, col.is_primary) 41 | 42 | def test_column_serializable(self): 43 | col = Column(1, 44 | {"COLUMN_NAME": "test", 45 | "COLLATION_NAME": "utf8_general_ci", 46 | "CHARACTER_SET_NAME": "UTF8", 47 | "COLUMN_COMMENT": "", 48 | "COLUMN_TYPE": "tinyint(2)", 49 | "COLUMN_KEY": "PRI"}, 50 | None) 51 | 52 | serialized = col.serializable_data() 53 | self.assertIn("type", serialized) 54 | self.assertIn("name", serialized) 55 | self.assertIn("collation_name", serialized) 56 | self.assertIn("character_set_name", serialized) 57 | self.assertIn("comment", serialized) 58 | self.assertIn("unsigned", serialized) 59 | self.assertIn("type_is_bool", serialized) 60 | self.assertIn("is_primary", serialized) 61 | 62 | self.assertEqual(col, Column(**serialized)) 63 | 64 | def test_table(self): 65 | tbl = Table(1, "test_schema", "test_table", [], []) 66 | 67 | serialized = tbl.serializable_data() 68 | self.assertIn("table_id", serialized) 69 | self.assertIn("schema", serialized) 70 | self.assertIn("table", serialized) 71 | self.assertIn("columns", serialized) 72 | self.assertIn("column_schemas", serialized) 73 | 74 | self.assertEqual(tbl, Table(**serialized)) 75 | 76 | 77 | if __name__ == "__main__": 78 | unittest.main() 79 | -------------------------------------------------------------------------------- /pymysqlreplication/tests/test_data_type.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import copy 4 | import platform 5 | import sys 6 | if sys.version_info < (2, 7): 7 | import unittest2 as unittest 8 | else: 9 | import unittest 10 | 11 | from decimal import Decimal 12 | 13 | from pymysqlreplication.tests import base 14 | from pymysqlreplication.constants.BINLOG import * 15 | from pymysqlreplication.row_event import * 16 | from pymysqlreplication.event import * 17 | 18 | __all__ = ["TestDataType"] 19 | 20 | 21 | class TestDataType(base.PyMySQLReplicationTestCase): 22 | def ignoredEvents(self): 23 | return [GtidEvent] 24 | 25 | def create_and_insert_value(self, create_query, insert_query): 26 | self.execute(create_query) 27 | self.execute(insert_query) 28 | self.execute("COMMIT") 29 | 30 | self.assertIsInstance(self.stream.fetchone(), RotateEvent) 31 | self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) 32 | #QueryEvent for the Create Table 33 | self.assertIsInstance(self.stream.fetchone(), QueryEvent) 34 | 35 | #QueryEvent for the BEGIN 36 | self.assertIsInstance(self.stream.fetchone(), QueryEvent) 37 | 38 | self.assertIsInstance(self.stream.fetchone(), TableMapEvent) 39 | 40 | event = self.stream.fetchone() 41 | if self.isMySQL56AndMore(): 42 | self.assertEqual(event.event_type, WRITE_ROWS_EVENT_V2) 43 | else: 44 | self.assertEqual(event.event_type, WRITE_ROWS_EVENT_V1) 45 | self.assertIsInstance(event, WriteRowsEvent) 46 | return event 47 | 48 | def test_decimal(self): 49 | create_query = "CREATE TABLE test (test DECIMAL(2,1))" 50 | insert_query = "INSERT INTO test VALUES(4.2)" 51 | event = self.create_and_insert_value(create_query, insert_query) 52 | self.assertEqual(event.columns[0].precision, 2) 53 | self.assertEqual(event.columns[0].decimals, 1) 54 | self.assertEqual(event.rows[0]["values"]["test"], Decimal("4.2")) 55 | 56 | def test_decimal_long_values(self): 57 | create_query = "CREATE TABLE test (\ 58 | test DECIMAL(20,10) \ 59 | )" 60 | insert_query = "INSERT INTO test VALUES(42000.123456)" 61 | event = self.create_and_insert_value(create_query, insert_query) 62 | self.assertEqual(event.rows[0]["values"]["test"], Decimal("42000.123456")) 63 | 64 | def test_decimal_long_values_1(self): 65 | create_query = "CREATE TABLE test (\ 66 | test DECIMAL(20,10) \ 67 | )" 68 | insert_query = "INSERT INTO test VALUES(9000000123.123456)" 69 | event = self.create_and_insert_value(create_query, insert_query) 70 | self.assertEqual(event.rows[0]["values"]["test"], Decimal("9000000123.123456")) 71 | 72 | def test_decimal_long_values_2(self): 73 | create_query = "CREATE TABLE test (\ 74 | test DECIMAL(20,10) \ 75 | )" 76 | insert_query = "INSERT INTO test VALUES(9000000123.0000012345)" 77 | event = self.create_and_insert_value(create_query, insert_query) 78 | self.assertEqual(event.rows[0]["values"]["test"], 79 | Decimal("9000000123.0000012345")) 80 | 81 | def test_decimal_negative_values(self): 82 | create_query = "CREATE TABLE test (\ 83 | test DECIMAL(20,10) \ 84 | )" 85 | insert_query = "INSERT INTO test VALUES(-42000.123456)" 86 | event = self.create_and_insert_value(create_query, insert_query) 87 | self.assertEqual(event.rows[0]["values"]["test"], Decimal("-42000.123456")) 88 | 89 | def test_decimal_two_values(self): 90 | create_query = "CREATE TABLE test (\ 91 | test DECIMAL(2,1), \ 92 | test2 DECIMAL(20,10) \ 93 | )" 94 | insert_query = "INSERT INTO test VALUES(4.2, 42000.123456)" 95 | event = self.create_and_insert_value(create_query, insert_query) 96 | self.assertEqual(event.rows[0]["values"]["test"], Decimal("4.2")) 97 | self.assertEqual(event.rows[0]["values"]["test2"], Decimal("42000.123456")) 98 | 99 | def test_decimal_with_zero_scale_1(self): 100 | create_query = "CREATE TABLE test (test DECIMAL(23,0))" 101 | insert_query = "INSERT INTO test VALUES(10)" 102 | event = self.create_and_insert_value(create_query, insert_query) 103 | self.assertEqual(event.rows[0]["values"]["test"], Decimal("10")) 104 | 105 | def test_decimal_with_zero_scale_2(self): 106 | create_query = "CREATE TABLE test (test DECIMAL(23,0))" 107 | insert_query = "INSERT INTO test VALUES(12345678912345678912345)" 108 | event = self.create_and_insert_value(create_query, insert_query) 109 | self.assertEqual(event.rows[0]["values"]["test"], Decimal("12345678912345678912345")) 110 | 111 | def test_decimal_with_zero_scale_3(self): 112 | create_query = "CREATE TABLE test (test DECIMAL(23,0))" 113 | insert_query = "INSERT INTO test VALUES(100000.0)" 114 | event = self.create_and_insert_value(create_query, insert_query) 115 | self.assertEqual(event.rows[0]["values"]["test"], Decimal("100000")) 116 | 117 | def test_decimal_with_zero_scale_4(self): 118 | create_query = "CREATE TABLE test (test DECIMAL(23,0))" 119 | insert_query = "INSERT INTO test VALUES(-100000.0)" 120 | event = self.create_and_insert_value(create_query, insert_query) 121 | self.assertEqual(event.rows[0]["values"]["test"], Decimal("-100000")) 122 | 123 | def test_decimal_with_zero_scale_6(self): 124 | create_query = "CREATE TABLE test (test DECIMAL(23,0))" 125 | insert_query = "INSERT INTO test VALUES(-1234567891234567891234)" 126 | event = self.create_and_insert_value(create_query, insert_query) 127 | self.assertEqual(event.rows[0]["values"]["test"], Decimal("-1234567891234567891234")) 128 | 129 | def test_tiny(self): 130 | create_query = "CREATE TABLE test (id TINYINT UNSIGNED NOT NULL, test TINYINT)" 131 | insert_query = "INSERT INTO test VALUES(255, -128)" 132 | event = self.create_and_insert_value(create_query, insert_query) 133 | self.assertEqual(event.rows[0]["values"]["id"], 255) 134 | self.assertEqual(event.rows[0]["values"]["test"], -128) 135 | 136 | def test_tiny_maps_to_boolean_true(self): 137 | create_query = "CREATE TABLE test (id TINYINT UNSIGNED NOT NULL, test BOOLEAN)" 138 | insert_query = "INSERT INTO test VALUES(1, TRUE)" 139 | event = self.create_and_insert_value(create_query, insert_query) 140 | self.assertEqual(event.rows[0]["values"]["id"], 1) 141 | self.assertEqual(type(event.rows[0]["values"]["test"]), type(1)) 142 | self.assertEqual(event.rows[0]["values"]["test"], 1) 143 | 144 | def test_tiny_maps_to_boolean_false(self): 145 | create_query = "CREATE TABLE test (id TINYINT UNSIGNED NOT NULL, test BOOLEAN)" 146 | insert_query = "INSERT INTO test VALUES(1, FALSE)" 147 | event = self.create_and_insert_value(create_query, insert_query) 148 | self.assertEqual(event.rows[0]["values"]["id"], 1) 149 | self.assertEqual(type(event.rows[0]["values"]["test"]), type(0)) 150 | self.assertEqual(event.rows[0]["values"]["test"], 0) 151 | 152 | def test_tiny_maps_to_none(self): 153 | create_query = "CREATE TABLE test (id TINYINT UNSIGNED NOT NULL, test BOOLEAN)" 154 | insert_query = "INSERT INTO test VALUES(1, NULL)" 155 | event = self.create_and_insert_value(create_query, insert_query) 156 | self.assertEqual(event.rows[0]["values"]["id"], 1) 157 | self.assertEqual(type(event.rows[0]["values"]["test"]), type(None)) 158 | self.assertEqual(event.rows[0]["values"]["test"], None) 159 | 160 | def test_short(self): 161 | create_query = "CREATE TABLE test (id SMALLINT UNSIGNED NOT NULL, test SMALLINT)" 162 | insert_query = "INSERT INTO test VALUES(65535, -32768)" 163 | event = self.create_and_insert_value(create_query, insert_query) 164 | self.assertEqual(event.rows[0]["values"]["id"], 65535) 165 | self.assertEqual(event.rows[0]["values"]["test"], -32768) 166 | 167 | def test_long(self): 168 | create_query = "CREATE TABLE test (id INT UNSIGNED NOT NULL, test INT)" 169 | insert_query = "INSERT INTO test VALUES(4294967295, -2147483648)" 170 | event = self.create_and_insert_value(create_query, insert_query) 171 | self.assertEqual(event.rows[0]["values"]["id"], 4294967295) 172 | self.assertEqual(event.rows[0]["values"]["test"], -2147483648) 173 | 174 | def test_float(self): 175 | create_query = "CREATE TABLE test (id FLOAT NOT NULL, test FLOAT)" 176 | insert_query = "INSERT INTO test VALUES(42.42, -84.84)" 177 | event = self.create_and_insert_value(create_query, insert_query) 178 | self.assertEqual(round(event.rows[0]["values"]["id"], 2), 42.42) 179 | self.assertEqual(round(event.rows[0]["values"]["test"],2 ), -84.84) 180 | 181 | def test_double(self): 182 | create_query = "CREATE TABLE test (id DOUBLE NOT NULL, test DOUBLE)" 183 | insert_query = "INSERT INTO test VALUES(42.42, -84.84)" 184 | event = self.create_and_insert_value(create_query, insert_query) 185 | self.assertEqual(round(event.rows[0]["values"]["id"], 2), 42.42) 186 | self.assertEqual(round(event.rows[0]["values"]["test"],2 ), -84.84) 187 | 188 | def test_timestamp(self): 189 | create_query = "CREATE TABLE test (test TIMESTAMP);" 190 | insert_query = "INSERT INTO test VALUES('1984-12-03 12:33:07')" 191 | event = self.create_and_insert_value(create_query, insert_query) 192 | self.assertEqual(event.rows[0]["values"]["test"], datetime.datetime(1984, 12, 3, 12, 33, 7)) 193 | 194 | def test_timestamp_mysql56(self): 195 | if not self.isMySQL56AndMore(): 196 | self.skipTest("Not supported in this version of MySQL") 197 | self.set_sql_mode() 198 | create_query = '''CREATE TABLE test (test0 TIMESTAMP(0), 199 | test1 TIMESTAMP(1), 200 | test2 TIMESTAMP(2), 201 | test3 TIMESTAMP(3), 202 | test4 TIMESTAMP(4), 203 | test5 TIMESTAMP(5), 204 | test6 TIMESTAMP(6));''' 205 | insert_query = '''INSERT INTO test VALUES('1984-12-03 12:33:07', 206 | '1984-12-03 12:33:07.1', 207 | '1984-12-03 12:33:07.12', 208 | '1984-12-03 12:33:07.123', 209 | '1984-12-03 12:33:07.1234', 210 | '1984-12-03 12:33:07.12345', 211 | '1984-12-03 12:33:07.123456')''' 212 | event = self.create_and_insert_value(create_query, insert_query) 213 | self.assertEqual(event.rows[0]["values"]["test0"], datetime.datetime(1984, 12, 3, 12, 33, 7)) 214 | self.assertEqual(event.rows[0]["values"]["test1"], datetime.datetime(1984, 12, 3, 12, 33, 7, 1)) 215 | self.assertEqual(event.rows[0]["values"]["test2"], datetime.datetime(1984, 12, 3, 12, 33, 7, 12)) 216 | self.assertEqual(event.rows[0]["values"]["test3"], datetime.datetime(1984, 12, 3, 12, 33, 7, 123)) 217 | self.assertEqual(event.rows[0]["values"]["test4"], datetime.datetime(1984, 12, 3, 12, 33, 7, 1234)) 218 | self.assertEqual(event.rows[0]["values"]["test5"], datetime.datetime(1984, 12, 3, 12, 33, 7, 12345)) 219 | self.assertEqual(event.rows[0]["values"]["test6"], datetime.datetime(1984, 12, 3, 12, 33, 7, 123456)) 220 | 221 | def test_longlong(self): 222 | create_query = "CREATE TABLE test (id BIGINT UNSIGNED NOT NULL, test BIGINT)" 223 | insert_query = "INSERT INTO test VALUES(18446744073709551615, -9223372036854775808)" 224 | event = self.create_and_insert_value(create_query, insert_query) 225 | self.assertEqual(event.rows[0]["values"]["id"], 18446744073709551615) 226 | self.assertEqual(event.rows[0]["values"]["test"], -9223372036854775808) 227 | 228 | def test_int24(self): 229 | create_query = "CREATE TABLE test (id MEDIUMINT UNSIGNED NOT NULL, test MEDIUMINT, test2 MEDIUMINT, test3 MEDIUMINT, test4 MEDIUMINT, test5 MEDIUMINT)" 230 | insert_query = "INSERT INTO test VALUES(16777215, 8388607, -8388608, 8, -8, 0)" 231 | event = self.create_and_insert_value(create_query, insert_query) 232 | self.assertEqual(event.rows[0]["values"]["id"], 16777215) 233 | self.assertEqual(event.rows[0]["values"]["test"], 8388607) 234 | self.assertEqual(event.rows[0]["values"]["test2"], -8388608) 235 | self.assertEqual(event.rows[0]["values"]["test3"], 8) 236 | self.assertEqual(event.rows[0]["values"]["test4"], -8) 237 | self.assertEqual(event.rows[0]["values"]["test5"], 0) 238 | 239 | def test_date(self): 240 | create_query = "CREATE TABLE test (test DATE);" 241 | insert_query = "INSERT INTO test VALUES('1984-12-03')" 242 | event = self.create_and_insert_value(create_query, insert_query) 243 | self.assertEqual(event.rows[0]["values"]["test"], datetime.date(1984, 12, 3)) 244 | 245 | def test_zero_date(self): 246 | create_query = "CREATE TABLE test (id INTEGER, test DATE, test2 DATE);" 247 | insert_query = "INSERT INTO test (id, test2) VALUES(1, '0000-01-21')" 248 | event = self.create_and_insert_value(create_query, insert_query) 249 | self.assertEqual(event.rows[0]["values"]["test"], None) 250 | self.assertEqual(event.rows[0]["values"]["test2"], None) 251 | 252 | def test_zero_month(self): 253 | self.set_sql_mode() 254 | create_query = "CREATE TABLE test (id INTEGER, test DATE, test2 DATE);" 255 | insert_query = "INSERT INTO test (id, test2) VALUES(1, '2015-00-21')" 256 | event = self.create_and_insert_value(create_query, insert_query) 257 | self.assertEqual(event.rows[0]["values"]["test"], None) 258 | self.assertEqual(event.rows[0]["values"]["test2"], None) 259 | 260 | def test_zero_day(self): 261 | self.set_sql_mode() 262 | create_query = "CREATE TABLE test (id INTEGER, test DATE, test2 DATE);" 263 | insert_query = "INSERT INTO test (id, test2) VALUES(1, '2015-05-00')" 264 | event = self.create_and_insert_value(create_query, insert_query) 265 | self.assertEqual(event.rows[0]["values"]["test"], None) 266 | self.assertEqual(event.rows[0]["values"]["test2"], None) 267 | 268 | def test_time(self): 269 | create_query = "CREATE TABLE test (test1 TIME, test2 TIME);" 270 | insert_query = "INSERT INTO test VALUES('838:59:59', '-838:59:59')" 271 | event = self.create_and_insert_value(create_query, insert_query) 272 | self.assertEqual(event.rows[0]["values"]["test1"], datetime.timedelta( 273 | microseconds=(((838*60) + 59)*60 + 59)*1000000 274 | )) 275 | self.assertEqual(event.rows[0]["values"]["test2"], datetime.timedelta( 276 | microseconds=(((-838*60) + 59)*60 + 59)*1000000 277 | )) 278 | 279 | def test_time2(self): 280 | if not self.isMySQL56AndMore(): 281 | self.skipTest("Not supported in this version of MySQL") 282 | create_query = "CREATE TABLE test (test1 TIME(6), test2 TIME(6));" 283 | insert_query = """ 284 | INSERT INTO test VALUES('838:59:59.000000', '-838:59:59.000000'); 285 | """ 286 | event = self.create_and_insert_value(create_query, insert_query) 287 | self.assertEqual(event.rows[0]["values"]["test1"], datetime.timedelta( 288 | microseconds=(((838*60) + 59)*60 + 59)*1000000 + 0 289 | )) 290 | self.assertEqual(event.rows[0]["values"]["test2"], datetime.timedelta( 291 | microseconds=(((-838*60) + 59)*60 + 59)*1000000 + 0 292 | )) 293 | 294 | def test_zero_time(self): 295 | create_query = "CREATE TABLE test (id INTEGER, test TIME NOT NULL DEFAULT 0);" 296 | insert_query = "INSERT INTO test (id) VALUES(1)" 297 | event = self.create_and_insert_value(create_query, insert_query) 298 | self.assertEqual(event.rows[0]["values"]["test"], datetime.timedelta(seconds=0)) 299 | 300 | def test_datetime(self): 301 | create_query = "CREATE TABLE test (test DATETIME);" 302 | insert_query = "INSERT INTO test VALUES('1984-12-03 12:33:07')" 303 | event = self.create_and_insert_value(create_query, insert_query) 304 | self.assertEqual(event.rows[0]["values"]["test"], datetime.datetime(1984, 12, 3, 12, 33, 7)) 305 | 306 | def test_zero_datetime(self): 307 | self.set_sql_mode() 308 | create_query = "CREATE TABLE test (id INTEGER, test DATETIME NOT NULL DEFAULT 0);" 309 | insert_query = "INSERT INTO test (id) VALUES(1)" 310 | event = self.create_and_insert_value(create_query, insert_query) 311 | self.assertEqual(event.rows[0]["values"]["test"], None) 312 | 313 | def test_broken_datetime(self): 314 | self.set_sql_mode() 315 | create_query = "CREATE TABLE test (test DATETIME NOT NULL);" 316 | insert_query = "INSERT INTO test VALUES('2013-00-00 00:00:00')" 317 | event = self.create_and_insert_value(create_query, insert_query) 318 | self.assertEqual(event.rows[0]["values"]["test"], None) 319 | 320 | def test_year(self): 321 | if self.isMySQL57(): 322 | # https://dev.mysql.com/doc/refman/5.7/en/migrating-to-year4.html 323 | self.skipTest("YEAR(2) is unsupported in mysql 5.7") 324 | create_query = "CREATE TABLE test (a YEAR(4), b YEAR(2))" 325 | insert_query = "INSERT INTO test VALUES(1984, 1984)" 326 | event = self.create_and_insert_value(create_query, insert_query) 327 | self.assertEqual(event.rows[0]["values"]["a"], 1984) 328 | self.assertEqual(event.rows[0]["values"]["b"], 1984) 329 | 330 | def test_varchar(self): 331 | create_query = "CREATE TABLE test (test VARCHAR(242)) CHARACTER SET latin1 COLLATE latin1_bin;" 332 | insert_query = "INSERT INTO test VALUES('Hello')" 333 | event = self.create_and_insert_value(create_query, insert_query) 334 | self.assertEqual(event.rows[0]["values"]["test"], 'Hello') 335 | self.assertEqual(event.columns[0].max_length, 242) 336 | 337 | def test_bit(self): 338 | create_query = "CREATE TABLE test (test BIT(6), \ 339 | test2 BIT(16), \ 340 | test3 BIT(12), \ 341 | test4 BIT(9), \ 342 | test5 BIT(64) \ 343 | );" 344 | insert_query = "INSERT INTO test VALUES( \ 345 | b'100010', \ 346 | b'1000101010111000', \ 347 | b'100010101101', \ 348 | b'101100111', \ 349 | b'1101011010110100100111100011010100010100101110111011101011011010')" 350 | event = self.create_and_insert_value(create_query, insert_query) 351 | self.assertEqual(event.columns[0].bits, 6) 352 | self.assertEqual(event.columns[1].bits, 16) 353 | self.assertEqual(event.columns[2].bits, 12) 354 | self.assertEqual(event.columns[3].bits, 9) 355 | self.assertEqual(event.columns[4].bits, 64) 356 | self.assertEqual(event.rows[0]["values"]["test"], "100010") 357 | self.assertEqual(event.rows[0]["values"]["test2"], "1000101010111000") 358 | self.assertEqual(event.rows[0]["values"]["test3"], "100010101101") 359 | self.assertEqual(event.rows[0]["values"]["test4"], "101100111") 360 | self.assertEqual(event.rows[0]["values"]["test5"], "1101011010110100100111100011010100010100101110111011101011011010") 361 | 362 | def test_enum(self): 363 | create_query = "CREATE TABLE test (test ENUM('a', 'ba', 'c'), test2 ENUM('a', 'ba', 'c')) CHARACTER SET latin1 COLLATE latin1_bin;" 364 | insert_query = "INSERT INTO test VALUES('ba', 'a')" 365 | event = self.create_and_insert_value(create_query, insert_query) 366 | self.assertEqual(event.rows[0]["values"]["test"], 'ba') 367 | self.assertEqual(event.rows[0]["values"]["test2"], 'a') 368 | 369 | def test_set(self): 370 | create_query = "CREATE TABLE test (test SET('a', 'ba', 'c'), test2 SET('a', 'ba', 'c')) CHARACTER SET latin1 COLLATE latin1_bin;" 371 | insert_query = "INSERT INTO test VALUES('ba,a,c', 'a,c')" 372 | event = self.create_and_insert_value(create_query, insert_query) 373 | self.assertEqual(event.rows[0]["values"]["test"], set(('a', 'ba', 'c'))) 374 | self.assertEqual(event.rows[0]["values"]["test2"], set(('a', 'c'))) 375 | 376 | def test_tiny_blob(self): 377 | create_query = "CREATE TABLE test (test TINYBLOB, test2 TINYTEXT) CHARACTER SET latin1 COLLATE latin1_bin;" 378 | insert_query = "INSERT INTO test VALUES('Hello', 'World')" 379 | event = self.create_and_insert_value(create_query, insert_query) 380 | self.assertEqual(event.rows[0]["values"]["test"], b'Hello') 381 | self.assertEqual(event.rows[0]["values"]["test2"], 'World') 382 | 383 | def test_medium_blob(self): 384 | create_query = "CREATE TABLE test (test MEDIUMBLOB, test2 MEDIUMTEXT) CHARACTER SET latin1 COLLATE latin1_bin;" 385 | insert_query = "INSERT INTO test VALUES('Hello', 'World')" 386 | event = self.create_and_insert_value(create_query, insert_query) 387 | self.assertEqual(event.rows[0]["values"]["test"], b'Hello') 388 | self.assertEqual(event.rows[0]["values"]["test2"], 'World') 389 | 390 | def test_long_blob(self): 391 | create_query = "CREATE TABLE test (test LONGBLOB, test2 LONGTEXT) CHARACTER SET latin1 COLLATE latin1_bin;" 392 | insert_query = "INSERT INTO test VALUES('Hello', 'World')" 393 | event = self.create_and_insert_value(create_query, insert_query) 394 | self.assertEqual(event.rows[0]["values"]["test"], b'Hello') 395 | self.assertEqual(event.rows[0]["values"]["test2"], 'World') 396 | 397 | def test_blob(self): 398 | create_query = "CREATE TABLE test (test BLOB, test2 TEXT) CHARACTER SET latin1 COLLATE latin1_bin;" 399 | insert_query = "INSERT INTO test VALUES('Hello', 'World')" 400 | event = self.create_and_insert_value(create_query, insert_query) 401 | self.assertEqual(event.rows[0]["values"]["test"], b'Hello') 402 | self.assertEqual(event.rows[0]["values"]["test2"], 'World') 403 | 404 | def test_string(self): 405 | create_query = "CREATE TABLE test (test CHAR(12)) CHARACTER SET latin1 COLLATE latin1_bin;" 406 | insert_query = "INSERT INTO test VALUES('Hello')" 407 | event = self.create_and_insert_value(create_query, insert_query) 408 | self.assertEqual(event.rows[0]["values"]["test"], 'Hello') 409 | 410 | def test_geometry(self): 411 | create_query = "CREATE TABLE test (test GEOMETRY);" 412 | insert_query = "INSERT INTO test VALUES(GeomFromText('POINT(1 1)'))" 413 | event = self.create_and_insert_value(create_query, insert_query) 414 | self.assertEqual(event.rows[0]["values"]["test"], b'\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\xf0?') 415 | 416 | def test_null(self): 417 | create_query = "CREATE TABLE test ( \ 418 | test TINYINT NULL DEFAULT NULL, \ 419 | test2 TINYINT NULL DEFAULT NULL, \ 420 | test3 TINYINT NULL DEFAULT NULL, \ 421 | test4 TINYINT NULL DEFAULT NULL, \ 422 | test5 TINYINT NULL DEFAULT NULL, \ 423 | test6 TINYINT NULL DEFAULT NULL, \ 424 | test7 TINYINT NULL DEFAULT NULL, \ 425 | test8 TINYINT NULL DEFAULT NULL, \ 426 | test9 TINYINT NULL DEFAULT NULL, \ 427 | test10 TINYINT NULL DEFAULT NULL, \ 428 | test11 TINYINT NULL DEFAULT NULL, \ 429 | test12 TINYINT NULL DEFAULT NULL, \ 430 | test13 TINYINT NULL DEFAULT NULL, \ 431 | test14 TINYINT NULL DEFAULT NULL, \ 432 | test15 TINYINT NULL DEFAULT NULL, \ 433 | test16 TINYINT NULL DEFAULT NULL, \ 434 | test17 TINYINT NULL DEFAULT NULL, \ 435 | test18 TINYINT NULL DEFAULT NULL, \ 436 | test19 TINYINT NULL DEFAULT NULL, \ 437 | test20 TINYINT NULL DEFAULT NULL\ 438 | )" 439 | insert_query = "INSERT INTO test (test, test2, test3, test7, test20) VALUES(NULL, -128, NULL, 42, 84)" 440 | event = self.create_and_insert_value(create_query, insert_query) 441 | self.assertEqual(event.rows[0]["values"]["test"], None) 442 | self.assertEqual(event.rows[0]["values"]["test2"], -128) 443 | self.assertEqual(event.rows[0]["values"]["test3"], None) 444 | self.assertEqual(event.rows[0]["values"]["test7"], 42) 445 | self.assertEqual(event.rows[0]["values"]["test20"], 84) 446 | 447 | def test_encoding_latin1(self): 448 | db = copy.copy(self.database) 449 | db["charset"] = "latin1" 450 | self.connect_conn_control(db) 451 | 452 | if platform.python_version_tuple()[0] == "2": 453 | string = unichr(233) 454 | else: 455 | string = "\u00e9" 456 | 457 | create_query = "CREATE TABLE test (test CHAR(12)) CHARACTER SET latin1 COLLATE latin1_bin;" 458 | insert_query = b"INSERT INTO test VALUES('" + string.encode('latin-1') + b"');" 459 | event = self.create_and_insert_value(create_query, insert_query) 460 | self.assertEqual(event.rows[0]["values"]["test"], string) 461 | 462 | def test_encoding_utf8(self): 463 | if platform.python_version_tuple()[0] == "2": 464 | string = unichr(0x20ac) 465 | else: 466 | string = "\u20ac" 467 | 468 | create_query = "CREATE TABLE test (test CHAR(12)) CHARACTER SET utf8 COLLATE utf8_bin;" 469 | insert_query = b"INSERT INTO test VALUES('" + string.encode('utf-8') + b"')" 470 | 471 | event = self.create_and_insert_value(create_query, insert_query) 472 | self.assertMultiLineEqual(event.rows[0]["values"]["test"], string) 473 | 474 | 475 | if __name__ == "__main__": 476 | unittest.main() 477 | -------------------------------------------------------------------------------- /test/log/test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/58daojia-dba/mysqlbinlog_flashback/555fd1e8c0b49bdf3d7b0f4de94039b957ff3161/test/log/test.txt -------------------------------------------------------------------------------- /test/test_generate_table_name.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import unittest 4 | import sys 5 | sys.path.append("..") 6 | from joint_sql import * 7 | from datetime import datetime 8 | from decimal import Decimal 9 | from func import init_logger 10 | from time import mktime 11 | import logging 12 | #from parameter import Pama 13 | 14 | class TestGenerate_table_name(unittest.TestCase): 15 | def setUp(self): 16 | init_logger("test1.log",logging.INFO) 17 | 18 | 19 | def test_generate_table_name(self): 20 | table_name=generate_table_name("test","test5") 21 | self.assertEquals(u"`test`.`test5`",table_name) 22 | #print(ret) 23 | 24 | def test_generate_array_column_value_pair(self): 25 | row={u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'afer1_tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 3} 26 | ret=generate_array_column_value_pair(row) 27 | self.assertEquals([u'`tdec`=15.01', u"`tlongtext`='longtext'", u'`tint`=1', u"`tvar`='afer1_tvar'", u'`tfload`=10.0100002289', u"`tdatetime`='2016-10-12 01:00:00'", u"`ttimestamp`='2016-10-13 01:00:00'", u'`id`=3'],ret) 28 | #self.assertEquals([1,2],[1,"2"]) 29 | 30 | def test_to_string(self): 31 | val1=u'test' 32 | self.assertEquals(u"'test'",to_string(val1,"'")) 33 | self.assertEquals(u"test",to_string(val1)) 34 | val2=u"te'st" 35 | #print(u"aaa"+to_string(val2,"'")) 36 | self.assertEquals(u"'te\\'st'",to_string(val2,"'")) 37 | self.assertEquals("15.01",to_string(Decimal('15.01'))) 38 | self.assertEquals(u"'2016-10-13 01:00:00'",to_string(datetime(2016, 10, 13, 1, 0))) 39 | 40 | 41 | 42 | 43 | 44 | def _test_tuple(self): 45 | pk=u"id" 46 | pk1=(u'id', u'tvar') 47 | dt={} 48 | row={u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'afer1_tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 3} 49 | 50 | if isinstance(pk1,tuple): 51 | print "tuple1" 52 | 53 | for key in pk1: 54 | print key 55 | dt[key]=row[key] 56 | print dt 57 | 58 | def test_generate_dict_pk(self): 59 | pk=u"id" 60 | row={u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'afer1_tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 3} 61 | ret=generate_dict_pk(pk,row) 62 | self.assertEquals({u'id': 3},ret) 63 | pk1=(u'id', u'tvar') 64 | ret=generate_dict_pk(pk1,row) 65 | self.assertEquals({u'tvar': u'afer1_tvar', u'id': 3},ret) 66 | 67 | def test_joint_update_sql(self): 68 | schema="test" 69 | table="test5" 70 | pk=u"id" 71 | row={'before_values': {u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 3}, 'after_values': {u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'afer1_tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 3}} 72 | ret=joint_update_sql(schema,table,pk,row,False,True) 73 | self.assertEquals(u"update `test`.`test5` set`tdec`=15.01,`tlongtext`='longtext',`tint`=1,`tvar`='tvar',`tfload`=10.0100002289,`ttimestamp`='2016-10-13 01:00:00',`tdatetime`='2016-10-12 01:00:00' where `id`=3",ret) 74 | ret=joint_update_sql(schema,table,pk,row,False) 75 | self.assertEquals(u"update `test5` set`tdec`=15.01,`tlongtext`='longtext',`tint`=1,`tvar`='tvar',`tfload`=10.0100002289,`ttimestamp`='2016-10-13 01:00:00',`tdatetime`='2016-10-12 01:00:00' where `id`=3",ret) 76 | 77 | 78 | 79 | def test_joint_update_sql_case1(self): 80 | schema="test" 81 | table="test5" 82 | pk=u"id" 83 | row={'before_values': {u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 3}, 'after_values': {u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'afer1_tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 4}} 84 | ret=joint_update_sql(schema,table,pk,row,True,True) 85 | expect=u"update `test`.`test5` set`tdec`=15.01,`tlongtext`='longtext',`tint`=1,`tvar`='tvar',`tfload`=10.0100002289,`tdatetime`='2016-10-12 01:00:00',`ttimestamp`='2016-10-13 01:00:00',`id`=3 where `id`=4" 86 | self.assertEquals(expect,ret) 87 | ret=joint_update_sql(schema,table,pk,row,True,False) 88 | expect=u"update `test5` set`tdec`=15.01,`tlongtext`='longtext',`tint`=1,`tvar`='tvar',`tfload`=10.0100002289,`tdatetime`='2016-10-12 01:00:00',`ttimestamp`='2016-10-13 01:00:00',`id`=3 where `id`=4" 89 | self.assertEquals(expect,ret) 90 | 91 | 92 | def test_generate_two_array_column_and_value(self): 93 | row={u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'afer1_tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 3} 94 | #row={u'tdec': Decimal('15.01'),u'tlongtext': u'longtext'} 95 | (columns,values)=generate_two_array_column_and_value(row) 96 | #print(columns) 97 | #print(values) 98 | 99 | def test_joint_delete_sql(self): 100 | schema="test" 101 | table="test5" 102 | pk=u"id" 103 | row={'values': {u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 3}} 104 | ret=joint_delete_sql(schema,table,pk,row,True) 105 | self.assertEquals(u'delete from `test`.`test5` where `id`=3',ret) 106 | ret=joint_delete_sql(schema,table,pk,row) 107 | self.assertEquals(u'delete from `test5` where `id`=3',ret) 108 | 109 | def test_joint_insert_sql(self): 110 | schema="test" 111 | table="test5" 112 | pk=u"id" 113 | row={'values': {u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 3}} 114 | ret=joint_insert_sql(schema,table,pk,row,True) 115 | self.assertEquals(u"insert into `test`.`test5`(`tdec`,`tlongtext`,`tint`,`tvar`,`tfload`,`tdatetime`,`ttimestamp`,`id`) values(15.01,'longtext',1,'tvar',10.0100002289,'2016-10-12 01:00:00','2016-10-13 01:00:00',3)",ret) 116 | ret=joint_insert_sql(schema,table,pk,row) 117 | self.assertEquals(u"insert into `test5`(`tdec`,`tlongtext`,`tint`,`tvar`,`tfload`,`tdatetime`,`ttimestamp`,`id`) values(15.01,'longtext',1,'tvar',10.0100002289,'2016-10-12 01:00:00','2016-10-13 01:00:00',3)",ret) 118 | 119 | 120 | def test_split_dict_column_value_pair(self): 121 | pk=u"id" 122 | row={u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'afer1_tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0), u'id': 3} 123 | (other_dict,pk_dict)=split_dict_column_value_pair(pk,row) 124 | # print(pk_dict) 125 | # print(other_dict) 126 | self.assertEquals({u'id': 3},pk_dict) 127 | self.assertEquals({u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tvar': u'afer1_tvar', u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0)},other_dict) 128 | pk1=(u'id', u'tvar') 129 | (other_dict1,pk_dict1)=split_dict_column_value_pair(pk1,row) 130 | # print(pk_dict1) 131 | # print(other_dict1) 132 | self.assertEquals({u'tvar': u'afer1_tvar', u'id': 3},pk_dict1) 133 | self.assertEquals({u'tdec': Decimal('15.01'), u'tlongtext': u'longtext', u'tint': 1, u'tfload': 10.010000228881836, u'ttimestamp': datetime(2016, 10, 13, 1, 0), u'tdatetime': datetime(2016, 10, 12, 1, 0)},other_dict1) 134 | 135 | def test_check_mysql_type(self): 136 | check_mysql_type(u"int(10) unsigned") 137 | check_mysql_type(u"bigint(20)") 138 | check_mysql_type(u"BIgINT(20)") 139 | 140 | def _test_date(self): 141 | dateC=datetime.strptime( "2016-10-31 14:12:08", '%Y-%m-%d %H:%M:%S') 142 | timestamp=mktime(dateC.timetuple()) 143 | print(type(timestamp)) 144 | print(timestamp) 145 | dt={} 146 | dt["aaa"]=1 147 | print(dt) 148 | 149 | def _test_key(self): 150 | key="key1" 151 | dict={ 152 | "key1":1, 153 | "key2":2 154 | } 155 | 156 | 157 | if __name__ == "__main__": 158 | #unittest.main() 159 | """ 160 | suite = unittest.TestSuite() 161 | suite.addTest(TestGenerate_table_name("test_check_mysql_type")) 162 | unittest.TextTestRunner(verbosity=3).run(suite) 163 | """ 164 | 165 | suite = unittest.TestLoader().loadTestsFromTestCase(TestGenerate_table_name) 166 | unittest.TextTestRunner(verbosity=3).run(suite) 167 | 168 | -------------------------------------------------------------------------------- /test/test_mysql_table.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | sys.path.append("..") 4 | from flashback import Parameter,deal_all_event,generate_create_table,add_stat 5 | from datetime import datetime 6 | from mysql_table import MysqlTable 7 | 8 | class TestMysqlTable(unittest.TestCase): 9 | def setUp(self): 10 | pass 11 | 12 | def test_get_columns(self): 13 | dict={} 14 | dict["host"]="127.0.0.1" 15 | dict["user"]="root" 16 | dict["port"]=43306 17 | dict["charset"] = "utf8" 18 | mysql_table=MysqlTable() 19 | mysql_table.connect(dict) 20 | dt=mysql_table.get_columns("test","test5") 21 | mysql_table.connect(dict) 22 | dt=mysql_table.get_columns("test","test5") 23 | expect=[{u'COLUMN_TYPE': u'bigint(20)', u'CHARACTER_SET_NAME': None, u'COLUMN_COMMENT': u'', u'COLUMN_KEY': u'PRI', u'COLLATION_NAME': None, u'COLUMN_NAME': u'id'}, {u'COLUMN_TYPE': u'varchar(64)', u'CHARACTER_SET_NAME': u'utf8', u'COLUMN_COMMENT': u'db\u5b9e\u4f8bid', u'COLUMN_KEY': u'MUL', u'COLLATION_NAME': u'utf8_general_ci', u'COLUMN_NAME': u'tvar'}, {u'COLUMN_TYPE': u'datetime', u'CHARACTER_SET_NAME': None, u'COLUMN_COMMENT': u'sql\u6267\u884c\u65f6\u95f4', u'COLUMN_KEY': u'', u'COLLATION_NAME': None, u'COLUMN_NAME': u'tdatetime'}, {u'COLUMN_TYPE': u'float', u'CHARACTER_SET_NAME': None, u'COLUMN_COMMENT': u'sql\u6d88\u8017\u65f6\u95f4\u5fae\u79d2', u'COLUMN_KEY': u'', u'COLLATION_NAME': None, u'COLUMN_NAME': u'tfload'}, {u'COLUMN_TYPE': u'int(11)', u'CHARACTER_SET_NAME': None, u'COLUMN_COMMENT': u'sql\u8fd4\u56de\u884c\u6570', u'COLUMN_KEY': u'', u'COLLATION_NAME': None, u'COLUMN_NAME': u'tint'}, {u'COLUMN_TYPE': u'decimal(10,2)', u'CHARACTER_SET_NAME': None, u'COLUMN_COMMENT': u'sql\u6d88\u8017\u65f6\u95f4\u5fae\u79d2', u'COLUMN_KEY': u'', u'COLLATION_NAME': None, u'COLUMN_NAME': u'tdec'}, {u'COLUMN_TYPE': u'timestamp', u'CHARACTER_SET_NAME': None, u'COLUMN_COMMENT': u'sql\u6d88\u8017\u65f6\u95f4\u5fae\u79d2', u'COLUMN_KEY': u'', u'COLLATION_NAME': None, u'COLUMN_NAME': u'ttimestamp'}, {u'COLUMN_TYPE': u'longtext', u'CHARACTER_SET_NAME': u'utf8', u'COLUMN_COMMENT': u'SQL\u6587\u672c', u'COLUMN_KEY': u'', u'COLLATION_NAME': u'utf8_general_ci', u'COLUMN_NAME': u'tlongtext'}] 24 | self.assertEquals(expect,dt) 25 | 26 | try: 27 | mysql_table.connect(dict) 28 | dt=mysql_table.get_columns("test","test111") 29 | self.assertEquals("not go here","go here") 30 | except ValueError as err: 31 | print err.__str__() 32 | self.assertEquals("correct","correct") 33 | 34 | """ 35 | for row in dt: 36 | print(row["COLUMN_NAME"]+" type "+row["COLUMN_TYPE"]) 37 | """ 38 | 39 | def test_get_last_binary_log_name(self): 40 | dict={} 41 | dict["host"]="127.0.0.1" 42 | dict["user"]="root" 43 | dict["port"]=43306 44 | dict["charset"] = "utf8" 45 | mysql_table=MysqlTable() 46 | mysql_table.connect(dict) 47 | dt=mysql_table.get_last_binary_log_name() 48 | print dt 49 | 50 | def test_get_current_datetime(self): 51 | dict={} 52 | dict["host"]="127.0.0.1" 53 | dict["user"]="root" 54 | dict["port"]=43306 55 | dict["charset"] = "utf8" 56 | mysql_table=MysqlTable() 57 | mysql_table.connect(dict) 58 | dt=mysql_table.get_current_datetime() 59 | print dt 60 | 61 | 62 | if __name__ == "__main__": 63 | unittest.main() -------------------------------------------------------------------------------- /test/test_mysqlbinlog_back.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os,sys 3 | import sys 4 | sys.path.append("..") 5 | from mysqlbinlog_back import get_check_option 6 | 7 | class TestMysqlbinlogBack(unittest.TestCase): 8 | def setUp(self): 9 | pass 10 | 11 | def test_get_check_option(self): 12 | try: 13 | sys.argv=['mysqlbinlog_back.py'] 14 | get_check_option() 15 | self.assertEquals("not go here","go here") 16 | except ValueError as err: 17 | self.assertEquals("correct","correct") 18 | 19 | try: 20 | sys.argv=['mysqlbinlog_back.py', '--host=aaa'] 21 | get_check_option() 22 | self.assertEquals("not go here","go here") 23 | sys.argv=['mysqlbinlog_back.py', '--host=aaa','--username=test'] 24 | get_check_option() 25 | self.assertEquals("not go here","go here") 26 | except ValueError as err: 27 | self.assertEquals("correct","correct") 28 | 29 | try: 30 | sys.argv=['mysqlbinlog_back.py', '--host=aaa','--username=test'] 31 | get_check_option() 32 | self.assertEquals("not go here","go here") 33 | except ValueError as err: 34 | self.assertEquals("correct","correct") 35 | 36 | try: 37 | sys.argv=['mysqlbinlog_back.py', '--host=127.0.0.1', '--username=test', '--schema=test', '--table=test5', '--binlog_end_time=2013-11-05 11:27:13', '--binlog_start_file_name=mysql-bin.000024', '--password=test', '--binlog_start_file_position=5', '--binlog_start_time=2016-11-04 11:27:13'] 38 | get_check_option() 39 | self.assertEquals("not go here","go here") 40 | except ValueError as err: 41 | print err.__str__() 42 | self.assertEquals("correct","correct") 43 | 44 | try: 45 | sys.argv=['mysqlbinlog_back.py', '--host=127.0.0.1', '--username=test', '--schema=test', '--table=test5', '--binlog_end_time=2013-11-05 11:27:13a', '--binlog_start_file_name=mysql-bin.000024', '--password=test', '--binlog_start_file_position=5', '--binlog_start_time=2016-11-04 11:27:13'] 46 | get_check_option() 47 | self.assertEquals("not go here","go here") 48 | except ValueError as err: 49 | print err.__str__() 50 | self.assertEquals("correct","correct") 51 | 52 | try: 53 | sys.argv=['mysqlbinlog_back.py', '--host=127.0.0.1', '--username=test', '--schema=test', '--table=test5', '--binlog_end_time=2013-11-05 11:27:13', '--binlog_start_file_name=mysql-bin.000024', '--password=test', '--binlog_start_file_position=5', '--binlog_start_time=a2016-11-04 11:27:13'] 54 | get_check_option() 55 | self.assertEquals("not go here","go here") 56 | except ValueError as err: 57 | print err.__str__() 58 | self.assertEquals("correct","correct") 59 | 60 | 61 | try: 62 | sys.argv=['mysqlbinlog_back.py', '--host=127.0.0.1', '--username=test', '--schema=test', '--table=test5', '--binlog_end_time=2015-11-05 11:27:13', '--binlog_start_file_name=mysql-bin.000024', '--password=test', '--binlog_start_file_position=5', '--binlog_start_time=2016-11-04 11:27:13'] 63 | get_check_option() 64 | self.assertEquals("not go here","go here") 65 | except ValueError as err: 66 | print err.__str__() 67 | self.assertEquals("correct","correct") 68 | 69 | 70 | sys.argv=['mysqlbinlog_back.py', '--host=127.0.0.1', '--username=test', '--schema=test', '--table=test5', '--binlog_end_time=2016-11-05 11:27:13', '--binlog_start_file_name=mysql-bin.000024'] 71 | opt=get_check_option() 72 | #print opt 73 | self.assertEquals({'username': 'test', 'binlog_start_file_position': 4, 'tables': 'test5', 'skip_update': False, 'output_file_path': './log', 'binlog_end_time': '2016-11-05 11:27:13', 'binlog_start_file_name': 'mysql-bin.000024', 'host': '127.0.0.1', 'version': False, 'skip_insert': False, 'binlog_start_time': None, 'password': '', 'skip_delete': False, 'port': 3306, 'add_schema_name': False, 'schema': 'test'},opt) 74 | 75 | sys.argv=['mysqlbinlog_back.py', '--host=127.0.0.1', '--username=test', '--schema=test', '--table=test5', '--binlog_end_time=2016-11-05 11:27:13', '--binlog_start_file_name=mysql-bin.000024', '--password=test', '--binlog_start_file_position=5', '--binlog_start_time=2016-11-04 11:27:13'] 76 | opt=get_check_option() 77 | #print opt 78 | self.assertEquals({'username': 'test', 'binlog_start_file_position': 5, 'tables': 'test5', 'skip_update': False, 'output_file_path': './log', 'binlog_end_time': '2016-11-05 11:27:13', 'binlog_start_file_name': 'mysql-bin.000024', 'host': '127.0.0.1', 'version': False, 'skip_insert': False, 'binlog_start_time': '2016-11-04 11:27:13', 'password': 'test', 'skip_delete': False, 'port': 3306, 'add_schema_name': False, 'schema': 'test'},opt) 79 | 80 | 81 | if __name__ == "__main__": 82 | unittest.main() -------------------------------------------------------------------------------- /test/test_parameter.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | sys.path.append("..") 4 | from flashback import Parameter,deal_all_event,generate_create_table,add_stat 5 | from datetime import datetime 6 | 7 | class TestParameter(unittest.TestCase): 8 | def setUp(self): 9 | dict={} 10 | dict["host"]="127.0.0.1" 11 | dict["username"]="root" 12 | 13 | dict["password"]="" 14 | dict["start_binlog_file"]="mysql-bin.000008" 15 | dict["start_position"]=3306 16 | #dict["end_time"]="aaa" 17 | #dict["output_file_path"]="./" 18 | dict["schema"]="test" 19 | dict["keep_data"]=True 20 | parameter=Parameter(**dict) 21 | self.parameter=parameter 22 | 23 | def test_get_file_name(self): 24 | curtime=datetime.strftime(datetime(2016, 10, 26, 12, 30),'%Y%m%d_%H%M%S') 25 | #print(curtime) 26 | ret=self.parameter.get_file_name("flashback",curtime) 27 | self.assertEquals(".//flashback_test_20161026_123000.sql",ret) 28 | #print(ret) 29 | 30 | def test_add_stat(self): 31 | stat={} 32 | stat["flash_sql"]={} 33 | op_type="update" 34 | schema=u"test" 35 | table=u"test5" 36 | add_stat(stat,op_type,schema,table) 37 | add_stat(stat,op_type,schema,table) 38 | add_stat(stat,"insert",schema,table) 39 | self.assertEquals({'flash_sql': {u'test': {u'test5': {'insert': 1, 'update': 2, 'delete': 0}}}},stat) 40 | """ 41 | for schema in stat["flash_sql"]: 42 | for table in stat["flash_sql"][schema]: 43 | print("{0}.{1}".format(schema,table)) 44 | """ 45 | if __name__ == "__main__": 46 | unittest.main() --------------------------------------------------------------------------------