├── py_mysql_binlogserver ├── __init__.py ├── _tutorial │ ├── __init__.py │ ├── learn_socket1_client.py │ ├── learn_socket2_server.py │ ├── learn_bin1_charset.py │ ├── learn_socket3_server_mulit_thread.py │ ├── learn_bin2_binlog.py │ ├── learn_socket4_server_mulit_thread.py │ ├── learn_packet3_query.py │ ├── learn_packet4_dump.py │ ├── learn_packet4_dump2.py │ ├── learn_packet1_greeting.py │ ├── learn_packet5_dump_with_semi_ack.py │ └── learn_packet2_auth.py ├── constants │ ├── __init__.py │ ├── FLAG.py │ ├── SERVER_STATUS.py │ ├── FIELD_TYPE.py │ ├── CLIENT.py │ ├── COMMAND.py │ ├── EVENT_TYPE.py │ ├── CR.py │ └── ER.py ├── packet │ ├── __init__.py │ ├── binlog_event.py │ ├── query.py │ ├── semiack.py │ ├── gtid_event.py │ ├── event_header.py │ ├── dump_pos.py │ ├── format_description_event.py │ ├── slave.py │ ├── dump_gtid.py │ ├── challenge.py │ └── response.py ├── protocol │ ├── __init__.py │ ├── ok.py │ ├── err.py │ ├── packet.py │ ├── gtid.py │ ├── Flags.py │ └── proto.py ├── tests │ ├── __init__.py │ └── test_packet.py ├── _playground │ ├── __init__.py │ ├── test_other.py │ ├── test_byte.py │ ├── test_slave.py │ ├── socket_client.py │ ├── socket_client_semi-repl.py │ └── socket_client2.py ├── cap │ ├── 0000_base │ │ ├── 0.cap │ │ ├── 2.cap │ │ ├── sql.txt │ │ ├── 1.cap │ │ └── 3.cap │ ├── auth_result.cap │ ├── 6d90b32a96401b9c81dddd508438943f │ │ ├── 0.cap │ │ ├── sql.txt │ │ ├── 2.cap │ │ ├── 1.cap │ │ └── 3.cap │ ├── afaca7cb78986fd4316977cbd595f018 │ │ ├── 0.cap │ │ ├── 2.cap │ │ ├── sql.txt │ │ ├── 1.cap │ │ └── 3.cap │ ├── b601070dfc14cb85fda3766a69a9e1b3 │ │ ├── 0.cap │ │ ├── 2.cap │ │ ├── sql.txt │ │ ├── 1.cap │ │ └── 3.cap │ ├── e1d4a95c32a61fb17047e58b2a5815b3 │ │ ├── 0.cap │ │ ├── 2.cap │ │ ├── sql.txt │ │ ├── 1.cap │ │ └── 3.cap │ ├── e7c4e133c1fee2dc8295e57fe4a25d27 │ │ ├── 0.cap │ │ ├── 2.cap │ │ ├── sql.txt │ │ ├── 1.cap │ │ └── 3.cap │ ├── e939e866562755b3b12b1fb018ec51dc │ │ ├── 0.cap │ │ ├── 2.cap │ │ ├── sql.txt │ │ ├── 1.cap │ │ └── 3.cap │ ├── ed6e1098afd0dc1a9e17d564282c5276 │ │ ├── 0.cap │ │ ├── 2.cap │ │ ├── sql.txt │ │ ├── 1.cap │ │ └── 3.cap │ ├── 23d75f5315fb3a5d3e6393de2d0fb853 │ │ ├── 0.cap │ │ └── sql.txt │ ├── 5ddb634cbc08b70a5c0c8ec08fa04a03 │ │ ├── 0.cap │ │ └── sql.txt │ ├── 6e5c39fc27c09b95926b5f0697971c3d │ │ ├── 0.cap │ │ └── sql.txt │ └── c64be5c6e91cdad08370b7189c583332 │ │ ├── sql.txt │ │ └── 0.cap ├── binlogs │ └── mysql-bin.index ├── dump │ └── readme.md ├── example.conf ├── example.py ├── proxy.py └── server.py ├── doc ├── T4架构篇-用Python开发MySQL增强半同步BinlogServer.md ├── readme.md └── T1基础篇-用Python开发MySQL增强半同步BinlogServer.md ├── .gitignore └── README.md /py_mysql_binlogserver/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/_tutorial/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/constants/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/packet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/protocol/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/_playground/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/0000_base/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/auth_result.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/binlogs/mysql-bin.index: -------------------------------------------------------------------------------- 1 | mysql-bin.000001 2 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/6d90b32a96401b9c81dddd508438943f/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/afaca7cb78986fd4316977cbd595f018/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/afaca7cb78986fd4316977cbd595f018/2.cap: -------------------------------------------------------------------------------- 1 | NONE -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/b601070dfc14cb85fda3766a69a9e1b3/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e1d4a95c32a61fb17047e58b2a5815b3/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e7c4e133c1fee2dc8295e57fe4a25d27/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e939e866562755b3b12b1fb018ec51dc/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/ed6e1098afd0dc1a9e17d564282c5276/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/ed6e1098afd0dc1a9e17d564282c5276/2.cap: -------------------------------------------------------------------------------- 1 | ON -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/23d75f5315fb3a5d3e6393de2d0fb853/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/5ddb634cbc08b70a5c0c8ec08fa04a03/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/6e5c39fc27c09b95926b5f0697971c3d/0.cap: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e1d4a95c32a61fb17047e58b2a5815b3/2.cap: -------------------------------------------------------------------------------- 1 | 5.7.26 -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e939e866562755b3b12b1fb018ec51dc/2.cap: -------------------------------------------------------------------------------- 1 | 3306192 -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/c64be5c6e91cdad08370b7189c583332/sql.txt: -------------------------------------------------------------------------------- 1 | SET NAMES utf8mb4 -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e1d4a95c32a61fb17047e58b2a5815b3/sql.txt: -------------------------------------------------------------------------------- 1 | select version() -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e7c4e133c1fee2dc8295e57fe4a25d27/2.cap: -------------------------------------------------------------------------------- 1 |  2 | 1569118298 -------------------------------------------------------------------------------- /doc/T4架构篇-用Python开发MySQL增强半同步BinlogServer.md: -------------------------------------------------------------------------------- 1 | # 用Python开发MySQL增强半同步BinlogServer-架构篇 2 | 3 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/0000_base/2.cap: -------------------------------------------------------------------------------- 1 | ,+Could not exec query on this Binlog server. -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/6d90b32a96401b9c81dddd508438943f/sql.txt: -------------------------------------------------------------------------------- 1 | SELECT @@GLOBAL.SERVER_UUID -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e7c4e133c1fee2dc8295e57fe4a25d27/sql.txt: -------------------------------------------------------------------------------- 1 | SELECT UNIX_TIMESTAMP() -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e939e866562755b3b12b1fb018ec51dc/sql.txt: -------------------------------------------------------------------------------- 1 | SELECT @@GLOBAL.SERVER_ID -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/ed6e1098afd0dc1a9e17d564282c5276/sql.txt: -------------------------------------------------------------------------------- 1 | SELECT @@GLOBAL.GTID_MODE -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/afaca7cb78986fd4316977cbd595f018/sql.txt: -------------------------------------------------------------------------------- 1 | SELECT @master_binlog_checksum -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/b601070dfc14cb85fda3766a69a9e1b3/2.cap: -------------------------------------------------------------------------------- 1 | (Py-MySQL-BinlogServer GPL) -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/0000_base/sql.txt: -------------------------------------------------------------------------------- 1 | select 'Could not exec query on this Binlog server.' as msg -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/b601070dfc14cb85fda3766a69a9e1b3/sql.txt: -------------------------------------------------------------------------------- 1 | select @@version_comment limit 1 2 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/dump/readme.md: -------------------------------------------------------------------------------- 1 | This directory is used to save the proxy's network packages. 2 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/23d75f5315fb3a5d3e6393de2d0fb853/sql.txt: -------------------------------------------------------------------------------- 1 | SET @master_heartbeat_period= 30000001024 -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/6d90b32a96401b9c81dddd508438943f/2.cap: -------------------------------------------------------------------------------- 1 | %$18f03682-ab70-11e9-aba4-32068899652e -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/5ddb634cbc08b70a5c0c8ec08fa04a03/sql.txt: -------------------------------------------------------------------------------- 1 | SET @master_binlog_checksum= @@global.binlog_checksum -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/6e5c39fc27c09b95926b5f0697971c3d/sql.txt: -------------------------------------------------------------------------------- 1 | SET @slave_uuid= 'ba66414c-d10d-11e9-b4b0-0800275ae9e7' -------------------------------------------------------------------------------- /py_mysql_binlogserver/_playground/test_other.py: -------------------------------------------------------------------------------- 1 | from _md5 import md5 2 | 3 | print(md5("select @@version_comment limit 1".encode()).hexdigest()) 4 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/0000_base/1.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/0000_base/1.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/0000_base/3.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/0000_base/3.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/c64be5c6e91cdad08370b7189c583332/0.cap: -------------------------------------------------------------------------------- 1 | k@b!character_set_connectionutf8mb4character_set_clientutf8mb4character_set_resultsutf8mb4 -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/6d90b32a96401b9c81dddd508438943f/1.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/6d90b32a96401b9c81dddd508438943f/1.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/6d90b32a96401b9c81dddd508438943f/3.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/6d90b32a96401b9c81dddd508438943f/3.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/afaca7cb78986fd4316977cbd595f018/1.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/afaca7cb78986fd4316977cbd595f018/1.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/afaca7cb78986fd4316977cbd595f018/3.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/afaca7cb78986fd4316977cbd595f018/3.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/b601070dfc14cb85fda3766a69a9e1b3/1.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/b601070dfc14cb85fda3766a69a9e1b3/1.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/b601070dfc14cb85fda3766a69a9e1b3/3.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/b601070dfc14cb85fda3766a69a9e1b3/3.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e1d4a95c32a61fb17047e58b2a5815b3/1.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/e1d4a95c32a61fb17047e58b2a5815b3/1.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e1d4a95c32a61fb17047e58b2a5815b3/3.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/e1d4a95c32a61fb17047e58b2a5815b3/3.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e7c4e133c1fee2dc8295e57fe4a25d27/1.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/e7c4e133c1fee2dc8295e57fe4a25d27/1.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e7c4e133c1fee2dc8295e57fe4a25d27/3.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/e7c4e133c1fee2dc8295e57fe4a25d27/3.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e939e866562755b3b12b1fb018ec51dc/1.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/e939e866562755b3b12b1fb018ec51dc/1.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/e939e866562755b3b12b1fb018ec51dc/3.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/e939e866562755b3b12b1fb018ec51dc/3.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/ed6e1098afd0dc1a9e17d564282c5276/1.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/ed6e1098afd0dc1a9e17d564282c5276/1.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/cap/ed6e1098afd0dc1a9e17d564282c5276/3.cap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvinzane/py-mysql-binlogserver/HEAD/py_mysql_binlogserver/cap/ed6e1098afd0dc1a9e17d564282c5276/3.cap -------------------------------------------------------------------------------- /py_mysql_binlogserver/_tutorial/learn_socket1_client.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | # 创建一个socket对象 4 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 5 | # 建立连接 6 | s.connect(("192.168.1.101", 3306)) 7 | # 接收数据 8 | buf = s.recv(10240) 9 | print(type(buf)) # 10 | # 发关数据 11 | s.send(b'hello') 12 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/constants/FLAG.py: -------------------------------------------------------------------------------- 1 | NOT_NULL = 1 2 | PRI_KEY = 2 3 | UNIQUE_KEY = 4 4 | MULTIPLE_KEY = 8 5 | BLOB = 16 6 | UNSIGNED = 32 7 | ZEROFILL = 64 8 | BINARY = 128 9 | ENUM = 256 10 | AUTO_INCREMENT = 512 11 | TIMESTAMP = 1024 12 | SET = 2048 13 | PART_KEY = 16384 14 | GROUP = 32767 15 | UNIQUE = 65536 16 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/_tutorial/learn_socket2_server.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | 4 | # 创建一个socket对象 5 | s = socket.socket() 6 | # 监听端口 7 | s.bind(('127.0.0.1', 8000)) 8 | s.listen(5) 9 | 10 | while True: 11 | conn, addr = s.accept() 12 | conn.send(bytes('Welcome python socket server.', 'utf8')) 13 | # 关闭链接 14 | conn.close() 15 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/_playground/test_byte.py: -------------------------------------------------------------------------------- 1 | import struct 2 | 3 | pos = bytes.fromhex("9F210000") 4 | print(pos) 5 | 6 | pos = struct.unpack("> 8 & 0xff # 高位 10 | low = row+i & 0xff # 低位 11 | try: 12 | # 用bytes对象转换成GBK字符 13 | print(bytes([high, low]).decode("gbk"), end="") 14 | except: 15 | print(end=" ") 16 | print("") 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | eggs 9 | parts 10 | bin 11 | var 12 | sdist 13 | develop-eggs 14 | .installed.cfg 15 | 16 | # Installer logs 17 | pip-log.txt 18 | 19 | # Unit test / coverage reports 20 | .coverage 21 | .tox 22 | 23 | # Translations 24 | *.mo 25 | 26 | #Mr Developer 27 | .mr.developer.cfg 28 | 29 | # Doc 30 | _build 31 | 32 | # Text Editor Backupfile 33 | *~ 34 | 35 | # Intellij IDE 36 | .idea 37 | *.xml 38 | *.iml 39 | 40 | # Nose 41 | .noseids 42 | 43 | # Pyenv 44 | .python-version 45 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/constants/FIELD_TYPE.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | DECIMAL = 0 4 | TINY = 1 5 | SHORT = 2 6 | LONG = 3 7 | FLOAT = 4 8 | DOUBLE = 5 9 | NULL = 6 10 | TIMESTAMP = 7 11 | LONGLONG = 8 12 | INT24 = 9 13 | DATE = 10 14 | TIME = 11 15 | DATETIME = 12 16 | YEAR = 13 17 | NEWDATE = 14 18 | VARCHAR = 15 19 | BIT = 16 20 | JSON = 245 21 | NEWDECIMAL = 246 22 | ENUM = 247 23 | SET = 248 24 | TINY_BLOB = 249 25 | MEDIUM_BLOB = 250 26 | LONG_BLOB = 251 27 | BLOB = 252 28 | VAR_STRING = 253 29 | STRING = 254 30 | GEOMETRY = 255 31 | 32 | CHAR = TINY 33 | INTERVAL = ENUM 34 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/protocol/ok.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | from py_mysql_binlogserver.protocol.packet import Packet, hex_ba 5 | 6 | 7 | class OK(Packet): 8 | __slots__ = ('errorCode', 'sqlState', 'errorMessage') + Packet.__slots__ 9 | 10 | def __init__(self): 11 | super().__init__() 12 | self.sequenceId = 1 13 | 14 | def getPayload(self): 15 | payload = bytearray() 16 | 17 | payload.extend(hex_ba('07 00 00 02 00 00 00 02 00 00 00')) 18 | 19 | return payload 20 | 21 | 22 | if __name__ == "__main__": 23 | import doctest 24 | doctest.testmod() 25 | -------------------------------------------------------------------------------- /doc/readme.md: -------------------------------------------------------------------------------- 1 | # Readme 2 | 3 | ## 教程 4 | 5 | 6 | ## MySQL protocol 7 | ``` 8 | 3 length 9 | 1 SequenceId 10 | 1 type: 00 OK packet or others without error 11 | 12 | ``` 13 | 14 | ## Binlog protocol 15 | 16 | ### 不用hearbeat会从第一个event开始传送 17 | ### 使用增强半同步后，binlog的packet格式会产生变化 18 | 19 | ### Binlog event 20 | ```sql 21 | 1 OK value 22 | 4 timestamp 23 | 1 event_type 24 | 4 server_id 25 | 4 log_pos 26 | 2 flags 27 | unpack = struct.unpack(' 1 and sys.argv[1] or os.path.dirname(__file__)+"/example.conf" 13 | config.read(conf_file) 14 | 15 | logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s') 16 | logger = logging.getLogger() 17 | logger.setLevel(config["Logging"].getint("level")) 18 | 19 | logger.info("Start Binlog Dumper from %s: %s" % (config["Dumper"]['host'], config["Dumper"]['port'])) 20 | 21 | try: 22 | 23 | server = BinlogServer(config["Server"]) 24 | server.run() 25 | 26 | client = BinlogDumper(config["Dumper"]) 27 | client.run() 28 | 29 | except KeyboardInterrupt: 30 | logger.info("Stop Binlog Dumper from %s: %s at %s %s" % (config["Dumper"]['host'], 31 | config["Dumper"]['port'], 32 | client._log_file, 33 | client._log_pos, 34 | )) 35 | client.close() 36 | 37 | 38 | if __name__ == "__main__": 39 | main() 40 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/protocol/err.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | from py_mysql_binlogserver.protocol.packet import Packet 5 | from py_mysql_binlogserver.protocol.proto import Proto 6 | from py_mysql_binlogserver.protocol import Flags 7 | 8 | 9 | class ERR(Packet): 10 | __slots__ = ('errorCode', 'sqlState', 'errorMessage') + Packet.__slots__ 11 | 12 | def __init__(self, errorCode=0, sqlState="HY000", errorMessage=""): 13 | self.sequenceId = 2 14 | self.errorCode = errorCode 15 | self.sqlState = sqlState 16 | self.errorMessage = errorMessage 17 | 18 | def getPayload(self): 19 | payload = bytearray() 20 | 21 | payload.extend(Proto.build_byte(Flags.ERR)) 22 | payload.extend(Proto.build_fixed_int(2, self.errorCode)) 23 | payload.extend(Proto.build_byte(ord('#'))) 24 | payload.extend(Proto.build_fixed_str(5, self.sqlState)) 25 | payload.extend(Proto.build_eop_str(self.errorMessage)) 26 | 27 | return payload 28 | 29 | @staticmethod 30 | def loadFromPacket(packet): 31 | obj = ERR() 32 | proto = Proto(packet, 3) 33 | 34 | obj.sequenceId = proto.get_fixed_int(1) 35 | proto.get_filler(1) 36 | obj.errorCode = proto.get_fixed_int(2) 37 | proto.get_filler(1) 38 | obj.sqlState = proto.get_fixed_str(5) 39 | obj.errorMessage = proto.get_eop_str() 40 | 41 | return obj 42 | 43 | 44 | if __name__ == "__main__": 45 | import doctest 46 | doctest.testmod() 47 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/_tutorial/learn_socket4_server_mulit_thread.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import threading 3 | import socketserver 4 | 5 | 6 | class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler): 7 | 8 | def handle(self): 9 | data = str(self.request.recv(1024), 'ascii') 10 | response = bytes("{} = {}".format(data, eval(data)), 'ascii') 11 | self.request.sendall(response) 12 | 13 | 14 | class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): 15 | pass 16 | 17 | 18 | def client(ip, port, message): 19 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: 20 | sock.connect((ip, port)) 21 | sock.sendall(bytes(message, 'ascii')) 22 | response = str(sock.recv(1024), 'ascii') 23 | print("Send: {}".format(message)) 24 | print("Received: {}".format(response)) 25 | print("") 26 | 27 | 28 | if __name__ == "__main__": 29 | # Port 0 means to select an arbitrary unused port 30 | HOST, PORT = "localhost", 0 31 | 32 | server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler) 33 | with server: 34 | ip, port = server.server_address 35 | server_thread = threading.Thread(target=server.serve_forever) 36 | # Exit the server thread when the main thread terminates 37 | server_thread.daemon = True 38 | server_thread.start() 39 | print(f"Calculator Server start at {ip} : {port}") 40 | 41 | client(ip, port, "1+1") 42 | client(ip, port, "3+2-5") 43 | client(ip, port, "12345679*81") 44 | 45 | server.shutdown() 46 | 47 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/packet/event_header.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from py_mysql_binlogserver.protocol import Flags 3 | from py_mysql_binlogserver.protocol.packet import Packet 4 | from py_mysql_binlogserver.protocol.proto import Proto 5 | 6 | 7 | class EventHeader(Packet): 8 | ''' 9 | 4 timestamp 10 | 1 event type 11 | 4 server-id 12 | 4 event-size 13 | 4 log pos 14 | 2 flags 15 | ''' 16 | __slots__ = ('timestamp','event_type','server_id','event_size','log_pos','flags') + Packet.__slots__ 17 | 18 | def __init__(self, timestamp, event_type, server_id): 19 | super(EventHeader, self).__init__() 20 | self.timestamp = timestamp 21 | self.event_type = event_type 22 | self.server_id = server_id 23 | self.log_pos = 0 24 | self.flags = 0 25 | 26 | def getEventBody(self): 27 | """ 28 | Return the event body as a bytearray 29 | """ 30 | raise NotImplementedError('getEventBody') 31 | 32 | def getPayload(self): 33 | payload = bytearray() 34 | 35 | payload.extend(b'\x00') # OK 36 | payload.extend(Proto.build_fixed_int(4, self.timestamp)) 37 | payload.extend(Proto.build_fixed_int(1, self.event_type)) 38 | payload.extend(Proto.build_fixed_int(4, self.server_id)) 39 | payload.extend(Proto.build_fixed_int(4, len(self.getEventBody()))) 40 | payload.extend(Proto.build_fixed_int(4, self.log_pos)) 41 | payload.extend(Proto.build_fixed_int(2, self.flags)) 42 | payload.extend(self.getEventBody()) 43 | 44 | return payload 45 | 46 | @staticmethod 47 | def loadFromPacket(packet): 48 | return b'' 49 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/packet/dump_pos.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | import struct 5 | 6 | from py_mysql_binlogserver.constants.COMMAND import COM_BINLOG_DUMP 7 | from py_mysql_binlogserver.protocol.packet import Packet 8 | from py_mysql_binlogserver.protocol.proto import int2byte 9 | 10 | 11 | class DumpPos(Packet): 12 | __slots__ = ('server_id', 'log_file', 'log_pos') + Packet.__slots__ 13 | 14 | def __init__(self, server_id, log_file ,log_pos): 15 | super(DumpPos, self).__init__() 16 | self.server_id = server_id 17 | self.log_file = log_file 18 | self.log_pos = log_pos 19 | 20 | def getPayload(self): 21 | # # only when log_file and log_pos both provided, the position info is 22 | # # valid, if not, get the current position from master 23 | # if self.log_file is None or self.log_pos is None: 24 | # cur = self._stream_connection.cursor() 25 | # cur.execute("SHOW MASTER STATUS") 26 | # master_status = cur.fetchone() 27 | # if master_status is None: 28 | # raise BinLogNotEnabled() 29 | # self.log_file, self.log_pos = master_status[:2] 30 | # cur.close() 31 | 32 | prelude = struct.pack('= len(packet): 20 | dump += ' ' 21 | else: 22 | dump += hex(packet[offset + x])[2:].upper().zfill(2) 23 | dump += ' ' 24 | if x == 7: 25 | dump += ' ' 26 | 27 | dump += ' ' 28 | for x in range(16): 29 | if offset + x >= len(packet): 30 | break 31 | c = chr(packet[offset + x]) 32 | if (len(c) > 1 33 | or packet[offset + x] < 32 34 | or packet[offset + x] == 255): 35 | dump += '.' 36 | else: 37 | dump += c 38 | if x == 7: 39 | dump += ' ' 40 | dump += '\n' 41 | offset += 16 42 | print(dump) 43 | 44 | 45 | def scramble_native_password(password, message): 46 | """ 47 | mysql_native_password 48 | https://dev.mysql.com/doc/internals/en/secure-password-authentication.html#packet-Authentication::Native41 49 | """ 50 | SCRAMBLE_LENGTH = 20 51 | sha1_new = partial(hashlib.new, 'sha1') 52 | 53 | """Scramble used for mysql_native_password""" 54 | if not password: 55 | return b'' 56 | 57 | password = password.encode("utf-8") 58 | message = message.encode("utf-8") 59 | 60 | stage1 = sha1_new(password).digest() 61 | stage2 = sha1_new(stage1).digest() 62 | s = sha1_new() 63 | s.update(message[:SCRAMBLE_LENGTH]) 64 | s.update(stage2) 65 | result = s.digest() 66 | return _my_crypt(result, stage1) 67 | 68 | 69 | def _my_crypt(message1, message2): 70 | result = bytearray(message1) 71 | for i in range(len(result)): 72 | result[i] ^= message2[i] 73 | 74 | return bytes(result) 75 | 76 | 77 | def get_response(s, username, password, challenge1, challenge2): 78 | ''' 79 | https://dev.mysql.com/doc/internals/en/connection-phase-packets.html 80 | 简化版 81 | 4 capability flags, CLIENT_PROTOCOL_41 always set 82 | 4 max-packet size 83 | 1 character set 84 | string[23] reserved (all [0]) 85 | string[NUL] username 86 | lenenc-int length of auth-response 87 | string[n] auth-response 88 | string[NUL] auth plugin name 89 | ''' 90 | scramble_password = scramble_native_password(password, challenge1 + challenge2) 91 | 92 | response = b'' 93 | response += struct.pack('> 16)) 61 | payload.extend(Proto.build_fixed_int(1, self.characterSet)) 62 | payload.extend(Proto.build_fixed_int(2, self.statusFlags)) 63 | payload.extend(Proto.build_fixed_int(2, self.capabilityFlags & 0xffff)) 64 | 65 | if self.hasCapabilityFlag(Flags.CLIENT_PLUGIN_AUTH): 66 | payload.extend(Proto.build_fixed_int(1, self.authPluginDataLength)) 67 | else: 68 | payload.extend(Proto.build_filler(1)) 69 | 70 | payload.extend(Proto.build_filler(10)) 71 | 72 | if self.hasCapabilityFlag(Flags.CLIENT_SECURE_CONNECTION): 73 | payload.extend(Proto.build_fixed_str( 74 | max(13, self.authPluginDataLength - 8), 75 | self.challenge2)) 76 | if self.hasCapabilityFlag(Flags.CLIENT_PLUGIN_AUTH): 77 | payload.extend(Proto.build_null_str(self.authPluginName)) 78 | 79 | return payload 80 | 81 | @staticmethod 82 | def loadFromPacket(packet): 83 | obj = Challenge() 84 | proto = Proto(packet, 3) 85 | 86 | obj.sequenceId = proto.get_fixed_int(1) 87 | obj.protocolVersion = proto.get_fixed_int(1) 88 | obj.serverVersion = proto.get_null_str() 89 | obj.connectionId = proto.get_fixed_int(4) 90 | obj.challenge1 = proto.get_fixed_str(8) 91 | proto.get_filler(1) 92 | obj.capabilityFlags = proto.get_fixed_int(2) << 16 93 | if proto.has_remaining_data(): 94 | obj.characterSet = proto.get_fixed_int(1) 95 | obj.statusFlags = proto.get_fixed_int(2) 96 | obj.setCapabilityFlag(proto.get_fixed_int(2)) 97 | 98 | if obj.hasCapabilityFlag(Flags.CLIENT_PLUGIN_AUTH): 99 | obj.authPluginDataLength = proto.get_fixed_int(1) 100 | else: 101 | proto.get_filler(1) 102 | 103 | proto.get_filler(10) 104 | 105 | if (obj.hasCapabilityFlag(Flags.CLIENT_SECURE_CONNECTION)): 106 | obj.challenge2 = proto.get_fixed_str(max(13, obj.authPluginDataLength - 8)) 107 | 108 | if (obj.hasCapabilityFlag(Flags.CLIENT_PLUGIN_AUTH)): 109 | obj.authPluginName = proto.get_null_str() 110 | 111 | return obj 112 | 113 | 114 | __TEST_PACKETS__ = [ 115 | # 5.5.2-m2 116 | [ 117 | '36 00 00 00 0a 35 2e 35', 118 | '2e 32 2d 6d 32 00 0b 00', 119 | '00 00 64 76 48 40 49 2d', 120 | '43 4a 00 ff f7 08 02 00', 121 | '00 00 00 00 00 00 00 00', 122 | '00 00 00 00 00 2a 34 64', 123 | '7c 63 5a 77 6b 34 5e 5d', 124 | '3a 00 ', 125 | ], 126 | # 5.6.4-m7-log 127 | [ 128 | '50 00 00 00 0a 35 2e 36', 129 | '2e 34 2d 6d 37 2d 6c 6f', 130 | '67 00 56 0a 00 00 52 42', 131 | '33 76 7a 26 47 72 00 ff', 132 | 'ff 08 02 00 0f c0 15 00', 133 | '00 00 00 00 00 00 00 00', 134 | '00 2b 79 44 26 2f 5a 5a', 135 | '33 30 35 5a 47 00 6d 79', 136 | '73 71 6c 5f 6e 61 74 69', 137 | '76 65 5f 70 61 73 73 77', 138 | '6f 72 64 00 ', 139 | ], 140 | ] 141 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/protocol/packet.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | import logging 4 | import logging.handlers 5 | import os 6 | 7 | from py_mysql_binlogserver.protocol.Flags import header_name 8 | from py_mysql_binlogserver.protocol.proto import Proto 9 | 10 | logger = logging.getLogger('py_mysql_binlogserver') 11 | 12 | 13 | class Packet(object): 14 | """ 15 | Basic class for all mysql proto classes to inherit from 16 | """ 17 | __slots__ = ('sequenceId',) 18 | 19 | def __init__(self): 20 | self.sequenceId = None 21 | 22 | def getPayload(self): 23 | """ 24 | Return the payload as a bytearray 25 | """ 26 | raise NotImplementedError('getPayload') 27 | 28 | def toPacket(self): 29 | """ 30 | Convert a Packet object to a byte array stream 31 | """ 32 | payload = self.getPayload() 33 | 34 | # Size is payload + packet size + sequence id 35 | size = len(payload) 36 | 37 | packet = bytearray(size + 4) 38 | 39 | packet[0:2] = Proto.build_fixed_int(3, size) 40 | packet[3] = Proto.build_fixed_int(1, self.sequenceId)[0] 41 | packet[4:] = payload 42 | 43 | return packet 44 | 45 | 46 | def hex_ba(string): 47 | ba = bytearray() 48 | fields = string.strip().split(' ') 49 | for field in fields: 50 | if field == '': 51 | continue 52 | ba_tmp = bytearray(1) 53 | ba_tmp[0] = int(field, 16) 54 | ba.extend(ba_tmp) 55 | return ba 56 | 57 | 58 | def getSize(packet): 59 | """ 60 | Returns a specified packet size 61 | """ 62 | return Proto(packet).get_fixed_int(3) 63 | 64 | 65 | def getType(packet): 66 | """ 67 | Returns a specified packet type 68 | """ 69 | return packet[4] 70 | 71 | 72 | def getSequenceId(packet): 73 | """ 74 | Returns the Sequence ID for the given packet 75 | """ 76 | return Proto(packet, 3).get_fixed_int(1) 77 | 78 | 79 | def dump(packet): 80 | """ 81 | Dumps a packet to the logger 82 | """ 83 | offset = 0 84 | # 85 | if not logger.isEnabledFor(logging.DEBUG): 86 | return 87 | 88 | dump = 'Packet Dump\n' 89 | 90 | while offset < len(packet): 91 | dump += hex(offset)[2:].zfill(8).upper() 92 | dump += ' ' 93 | 94 | for x in range(16): 95 | if offset + x >= len(packet): 96 | dump += ' ' 97 | else: 98 | dump += hex(packet[offset + x])[2:].upper().zfill(2) 99 | dump += ' ' 100 | if x == 7: 101 | dump += ' ' 102 | 103 | dump += ' ' 104 | 105 | for x in range(16): 106 | if offset + x >= len(packet): 107 | break 108 | c = chr(packet[offset + x]) 109 | if (len(c) > 1 110 | or packet[offset + x] < 32 111 | or packet[offset + x] == 255): 112 | dump += '.' 113 | else: 114 | dump += c 115 | 116 | if x == 7: 117 | dump += ' ' 118 | 119 | dump += '\n' 120 | offset += 16 121 | logger.debug(dump) 122 | 123 | 124 | def read_server_packet(socket_in): 125 | """ 126 | Reads a packet from a socket 127 | """ 128 | # Read the size of the packet 129 | psize = bytearray(3) 130 | socket_in.recv_into(psize, 3) 131 | 132 | size = getSize(psize) + 1 133 | 134 | # Read the rest of the packet 135 | packet_payload = bytearray(size) 136 | socket_in.recv_into(packet_payload, size) 137 | 138 | # Combine the chunks 139 | psize.extend(packet_payload) 140 | # if __debug__: 141 | # dump(psize) 142 | 143 | return psize 144 | 145 | 146 | def file2packet(filename): 147 | tmp_dir = os.path.dirname(os.path.dirname(__file__)) + "/cap" 148 | fi = open(tmp_dir + "/" + filename, "r+b") 149 | packet = bytearray(fi.read()) 150 | fi.close() 151 | return packet 152 | 153 | 154 | def dump_my_packet(packet): 155 | """ 156 | Dumps a packet to the string 157 | """ 158 | offset = 0 159 | try: 160 | header = getType(packet) 161 | except: 162 | header = 0 163 | dump = 'Length: %s, SequenceId: %s, Header: %s=%s \n' % ( 164 | getSize(packet), getSequenceId(packet), header_name(header), header,) 165 | 166 | while offset < len(packet): 167 | dump += hex(offset)[2:].zfill(8).upper() 168 | dump += ' ' 169 | 170 | for x in range(16): 171 | if offset + x >= len(packet): 172 | dump += ' ' 173 | else: 174 | dump += hex(packet[offset + x])[2:].upper().zfill(2) 175 | dump += ' ' 176 | if x == 7: 177 | dump += ' ' 178 | 179 | dump += ' ' 180 | 181 | for x in range(16): 182 | if offset + x >= len(packet): 183 | break 184 | c = chr(packet[offset + x]) 185 | if (len(c) > 1 186 | or packet[offset + x] < 32 187 | or packet[offset + x] == 255): 188 | dump += '.' 189 | else: 190 | dump += c 191 | 192 | if x == 7: 193 | dump += ' ' 194 | 195 | dump += '\n' 196 | offset += 16 197 | 198 | print(dump) 199 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/_playground/socket_client_semi-repl.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import os 3 | import socket 4 | import struct 5 | 6 | from py_mysql_binlogserver.packet.challenge import Challenge 7 | from py_mysql_binlogserver.packet.dump_gtid import DumpGtid 8 | from py_mysql_binlogserver.packet.query import Query 9 | from py_mysql_binlogserver.packet.response import Response 10 | from py_mysql_binlogserver.packet.semiack import SemiAck 11 | from py_mysql_binlogserver.packet.slave import Slave 12 | from py_mysql_binlogserver.protocol import Flags 13 | from py_mysql_binlogserver.protocol.err import ERR 14 | from py_mysql_binlogserver.protocol.packet import dump_my_packet, getSequenceId, getType 15 | from py_mysql_binlogserver.protocol.packet import read_server_packet 16 | from py_mysql_binlogserver.protocol.proto import scramble_native_password 17 | from pymysql.util import byte2int 18 | 19 | 20 | def get_socket(host='127.0.0.1', port=3306, user="", password="", schema=""): 21 | 22 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 23 | s.connect((host, port)) 24 | 25 | packet = read_server_packet(s) 26 | print("received 1:") 27 | dump_my_packet(packet) 28 | 29 | challenge = Challenge.loadFromPacket(packet) 30 | 31 | challenge1 = challenge.challenge1 32 | challenge2 = challenge.challenge2 33 | 34 | scramble_password = scramble_native_password(password, challenge1 + challenge2) 35 | response = Response() 36 | response.sequenceId = 1 37 | response.capabilityFlags = 33531397 38 | response.characterSet = 33 39 | response.maxPacketSize = 16777216 40 | response.clientAttributes["_client_name"] = 'pymysql' 41 | response.clientAttributes["_pid"] = str(os.getpid()) 42 | response.clientAttributes["_client_version"] = '5.7' 43 | response.clientAttributes["program_name"] = 'mysql' 44 | response.pluginName = 'mysql_native_password' 45 | response.username = user 46 | response.schema = schema 47 | response.authResponse = scramble_password 48 | response.removeCapabilityFlag(Flags.CLIENT_COMPRESS) 49 | response.removeCapabilityFlag(Flags.CLIENT_SSL) 50 | response.removeCapabilityFlag(Flags.CLIENT_LOCAL_FILES) 51 | 52 | packet = response.toPacket() 53 | print("login 1:") 54 | dump_my_packet(packet) 55 | 56 | s.sendall(packet) 57 | 58 | packet = read_server_packet(s) 59 | print("received 2:") 60 | dump_my_packet(packet) 61 | 62 | return s 63 | 64 | 65 | def read_packet(skt): 66 | while True: 67 | packet = read_server_packet(skt) 68 | sequenceId = getSequenceId(packet) 69 | print("read packet [%s]:" % (sequenceId,)) 70 | dump_my_packet(packet) 71 | packetType = getType(packet) 72 | 73 | if packetType == Flags.ERR: 74 | buf = ERR.loadFromPacket(packet) 75 | print("error:", buf.errorCode, buf.sqlState, buf.errorMessage) 76 | skt.close() 77 | exit(1) 78 | break 79 | 80 | if packetType == Flags.EOF or packetType == Flags.OK: 81 | break 82 | 83 | 84 | def read_binlog(skt): 85 | while True: 86 | packet = read_server_packet(skt) 87 | sequenceId = getSequenceId(packet) 88 | packetType = getType(packet) 89 | 90 | # OK value 91 | # timestamp 92 | # event_type 93 | # server_id 94 | # log_pos 95 | # flags 96 | unpack = struct.unpack('....Œ9t ]!ur2.=. 200 | 00000010 00 00 87 26 00 00 00 00 00 BA 66 41 4C D1 0D 11 ..‡&.... .ºfALÑ.. 201 | 00000020 E9 B4 B0 08 00 27 5A E9 E7 13 00 00 00 00 00 00 é´°..'Zé ç....... 202 | 00000030 00 02 2A 00 00 00 00 00 00 00 2B 00 00 00 00 00 ..*..... ..+..... 203 | 00000040 00 00 .. 204 | 205 | 1567897996 2 3306101 67 9930 206 | Length: 68, SequenceId: 26, Header: OK=0 207 | QUERY_EVENT 208 | 00000000 44 00 00 1A 00 8C 39 74 5D 02 75 72 32 00 43 00 D....Œ9t ].ur2.C. 209 | 00000010 00 00 CA 26 00 00 08 00 1F 02 00 00 00 00 00 00 ..Ê&.... ........ 210 | 00000020 03 00 00 1A 00 00 00 00 00 00 01 00 00 20 40 00 ........ ..... @. 211 | 00000030 00 00 00 06 03 73 74 64 04 E0 00 E0 00 E0 00 64 .....std .à.à.à.d 212 | 00000040 62 31 00 42 45 47 49 4E b1.BEGIN 213 | 214 | 1567897996 19 3306101 43 9973 215 | Length: 44, SequenceId: 27, Header: OK=0 216 | TABLE_MAP_EVENT 217 | 00000000 2C 00 00 1B 00 8C 39 74 5D 13 75 72 32 00 2B 00 ,....Œ9t ].ur2.+. 218 | 00000010 00 00 F5 26 00 00 00 00 0D 01 00 00 00 00 01 00 ..õ&.... ........ 219 | 00000020 03 64 62 31 00 02 74 31 00 02 03 0F 02 28 00 02 .db1..t1 .....(.. 220 | 221 | 1567897996 30 3306101 41 10014 222 | Length: 42, SequenceId: 28, Header: OK=0 223 | WRITE_ROWS_EVENT_V2 224 | 00000000 2A 00 00 1C 00 8C 39 74 5D 1E 75 72 32 00 29 00 *....Œ9t ].ur2.). 225 | 00000010 00 00 1E 27 00 00 00 00 0D 01 00 00 00 00 01 00 ...'.... ........ 226 | 00000020 02 00 02 FF FC 23 00 00 00 04 32 33 33 33 ....ü#.. ..2333 227 | 228 | 1567897996 16 3306101 27 10041 229 | XID_EVENT 230 | Length: 28, SequenceId: 29, Header: OK=0 231 | 00000000 1C 00 00 1D 00 8C 39 74 5D 10 75 72 32 00 1B 00 .....Œ9t ].ur2... 232 | 00000010 00 00 39 27 00 00 00 00 AE 26 00 00 00 00 00 00 ..9'.... ®&...... 233 | ''' 234 | 235 | s.close() 236 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/proxy.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import configparser 3 | import logging 4 | import os 5 | import socket 6 | import socketserver 7 | import struct 8 | import sys 9 | import threading 10 | from _md5 import md5 11 | 12 | from py_mysql_binlogserver.packet.challenge import Challenge 13 | from py_mysql_binlogserver.packet.query import Query 14 | from py_mysql_binlogserver.packet.response import Response 15 | from py_mysql_binlogserver.protocol import Flags 16 | from py_mysql_binlogserver.protocol.err import ERR 17 | from py_mysql_binlogserver.protocol.packet import getSize, getType, file2packet, \ 18 | dump 19 | from py_mysql_binlogserver.protocol.proto import scramble_native_password 20 | 21 | SocketServer = socketserver 22 | connection_counter = 0 23 | logger = logging.getLogger() 24 | 25 | 26 | class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler): 27 | connection_id = 0 28 | timeout = 5 29 | user_id = 0 30 | logger = logging.getLogger('server') 31 | server_id = 0 32 | dir_name = os.path.dirname(__file__) 33 | upstream = None 34 | 35 | def setup(self): 36 | global connection_counter 37 | connection_counter += 1 38 | self.connection_id = connection_counter 39 | 40 | def send_packet(self, packet): 41 | self.request.sendall(packet) 42 | 43 | def read_packet(self): 44 | """ 45 | Reads a packet from a socket 46 | """ 47 | # Read the size of the packet 48 | socket_in = self.request 49 | psize = bytearray(3) 50 | socket_in.recv_into(psize, 3) 51 | 52 | size = getSize(psize) + 1 53 | 54 | # Read the rest of the packet 55 | packet_payload = bytearray(size) 56 | socket_in.recv_into(packet_payload, size) 57 | 58 | # Combine the chunks 59 | psize.extend(packet_payload) 60 | 61 | return psize 62 | 63 | def init_upstream(self): 64 | conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 65 | host = self.server.settings["server_host"] 66 | port = self.server.settings.getint("server_port") 67 | user = self.server.settings["server_user"] 68 | password = self.server.settings["server_password"] 69 | 70 | schema = "" 71 | conn.connect((host, port)) 72 | self.upstream = conn 73 | 74 | challenge = Challenge.loadFromPacket(self.upstream.recv(10240)) 75 | logger.debug("== Greeting ==") 76 | dump(challenge.toPacket()) 77 | 78 | challenge1 = challenge.challenge1 79 | challenge2 = challenge.challenge2 80 | 81 | scramble_password = scramble_native_password(password, challenge1 + challenge2) 82 | response = Response() 83 | response.sequenceId = 1 84 | response.capabilityFlags = 33531397 85 | response.characterSet = 33 86 | response.maxPacketSize = 16777216 87 | response.clientAttributes["_client_name"] = 'pymysql' 88 | response.clientAttributes["_pid"] = str(os.getpid()) 89 | response.clientAttributes["_client_version"] = '5.7' 90 | response.clientAttributes["program_name"] = 'mysql' 91 | response.pluginName = 'mysql_native_password' 92 | response.username = user 93 | response.schema = schema 94 | response.authResponse = scramble_password 95 | response.removeCapabilityFlag(Flags.CLIENT_COMPRESS) 96 | response.removeCapabilityFlag(Flags.CLIENT_SSL) 97 | response.removeCapabilityFlag(Flags.CLIENT_LOCAL_FILES) 98 | 99 | logger.debug("== Auth ==") 100 | dump(response.toPacket()) 101 | 102 | self.upstream.send(response.toPacket()) 103 | 104 | _packet = self.upstream.recv(10240) 105 | 106 | logger.debug("== Result ==") 107 | dump(_packet) 108 | packetType = getType(_packet) 109 | 110 | if packetType == Flags.ERR: 111 | buf = ERR.loadFromPacket(_packet) 112 | logger.error("Upstream error:", buf.errorCode, buf.sqlState, buf.errorMessage) 113 | self.upstream.close() 114 | self.finish() 115 | exit(1) 116 | 117 | def handle(self): 118 | 119 | # 认证 120 | challenge1 = '12345678' 121 | challenge2 = '123456789012' 122 | challenge = self.create_challenge(challenge1, challenge2) 123 | self.send_packet(challenge.toPacket()) 124 | 125 | packet = self.read_packet() 126 | response = Response() 127 | response = response.loadFromPacket(packet) 128 | 129 | username = response.username 130 | self.logger.info("login user:" + username) 131 | 132 | password = self.server.settings["proxy_password"] 133 | self.user_id = username 134 | 135 | # 验证密码 136 | native_password = scramble_native_password(password, challenge1 + challenge2) 137 | 138 | if self.server.settings["proxy_user"] != username or response.authResponse.encode("iso-8859-1") != native_password: 139 | err = ERR(9001, '28000', '[%s] Access denied.' % username) 140 | buff = err.toPacket() 141 | self.send_packet(buff) 142 | self.finish() 143 | return 144 | 145 | buff = file2packet("auth_result.cap") 146 | self.send_packet(buff) 147 | 148 | # 初始化后端服务器 149 | self.init_upstream() 150 | 151 | # 查询 152 | while True: 153 | 154 | packet = self.read_packet() 155 | if len(packet) < 4: 156 | continue 157 | packet_type = getType(packet) 158 | 159 | if packet_type == Flags.COM_QUIT: 160 | self.upstream.close() 161 | self.finish() 162 | 163 | elif packet_type == Flags.COM_QUERY: 164 | self.handle_query(packet) 165 | else: 166 | self.dispatch_packet(packet) 167 | 168 | def create_challenge(self, challenge1, challenge2): 169 | # 认证 170 | challenge = Challenge() 171 | challenge.protocolVersion = 10 172 | challenge.serverVersion = '5.7.20-log' 173 | challenge.connectionId = self.connection_id 174 | challenge.challenge1 = challenge1 175 | challenge.challenge2 = challenge2 176 | challenge.capabilityFlags = 4160717151 177 | challenge.characterSet = 224 178 | challenge.statusFlags = 2 179 | challenge.authPluginDataLength = 21 180 | challenge.authPluginName = 'mysql_native_password' 181 | challenge.sequenceId = 0 182 | return challenge 183 | 184 | def dispatch_packet(self, packet, sql=None): 185 | self.upstream.send(packet) 186 | logger.debug(f"== Send packet ==") 187 | dump(packet) 188 | 189 | if self.server.settings["dump_packet_to_file"] == "1" and sql: 190 | dir_name = self.dir_name + "/dump/" + md5(sql.encode()).hexdigest() 191 | if not os.path.isdir(dir_name): 192 | os.mkdir(dir_name) 193 | with open(dir_name + "/" + "sql.txt", "w") as sql_write: 194 | sql_write.write(sql) 195 | 196 | while True: 197 | _header = self.upstream.recv(5) 198 | _length = struct.unpack(" 1 and sys.argv[1] or os.path.dirname(__file__)+"/example.conf" 247 | config.read(conf_file) 248 | 249 | logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s") 250 | logger = logging.getLogger() 251 | logger.setLevel(config["Logging"].getint("level")) 252 | 253 | server_settings = config["Proxy"] 254 | 255 | proxy = MyProxy(server_settings) 256 | proxy.settings = server_settings 257 | proxy.run() 258 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/protocol/gtid.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import re 4 | import struct 5 | import binascii 6 | from io import BytesIO 7 | 8 | 9 | def overlap(i1, i2): 10 | return i1[0] < i2[1] and i1[1] > i2[0] 11 | 12 | 13 | def contains(i1, i2): 14 | return i2[0] >= i1[0] and i2[1] <= i1[1] 15 | 16 | 17 | class Gtid(object): 18 | """A mysql GTID is composed of a server-id and a set of right-open 19 | intervals [a,b), and represent all transactions x that happened on 20 | server SID such as 21 | 22 | x <= a < b 23 | 24 | The human representation of it, though, is either represented by a 25 | single transaction number A=a (when only one transaction is covered, 26 | ie b = a+1) 27 | 28 | SID:A 29 | 30 | Or a closed interval [A,B] for at least two transactions (note, in that 31 | case, that b=B+1) 32 | 33 | SID:A-B 34 | 35 | We can also have a mix of ranges for a given SID: 36 | SID:1-2:4:6-74 37 | 38 | For convenience, a Gtid accepts adding Gtid's to it and will merge 39 | the existing interval representation. Adding TXN 3 to the human 40 | representation above would produce: 41 | 42 | SID:1-4:6-74 43 | 44 | and adding 5 to this new result: 45 | 46 | SID:1-74 47 | 48 | Adding an already present transaction number (one that overlaps) will 49 | raise an exception. 50 | 51 | Adding a Gtid with a different SID will raise an exception. 52 | """ 53 | 54 | @staticmethod 55 | def parse_interval(interval): 56 | """ 57 | We parse a human-generated string here. So our end value b 58 | is incremented to conform to the internal representation format. 59 | """ 60 | m = re.search('^([0-9]+)(?:-([0-9]+))?$', interval) 61 | if not m: 62 | raise ValueError('GTID format is incorrect: %r' % (interval,)) 63 | a = int(m.group(1)) 64 | b = int(m.group(2) or a) 65 | return (a, b + 1) 66 | 67 | @staticmethod 68 | def parse(gtid): 69 | m = re.search('^([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})' 70 | '((?::[0-9-]+)+)$', gtid) 71 | if not m: 72 | raise ValueError('GTID format is incorrect: %r' % (gtid,)) 73 | 74 | sid = m.group(1) 75 | intervals = m.group(2) 76 | 77 | intervals_parsed = [Gtid.parse_interval(x) 78 | for x in intervals.split(':')[1:]] 79 | 80 | return (sid, intervals_parsed) 81 | 82 | def __add_interval(self, itvl): 83 | """ 84 | Use the internal representation format and add it 85 | to our intervals, merging if required. 86 | """ 87 | new = [] 88 | 89 | if itvl[0] > itvl[1]: 90 | raise Exception('Malformed interval %s' % (itvl,)) 91 | 92 | if any(overlap(x, itvl) for x in self.intervals): 93 | raise Exception('Overlapping interval %s' % (itvl,)) 94 | 95 | ## Merge: arrange interval to fit existing set 96 | for existing in sorted(self.intervals): 97 | if itvl[0] == existing[1]: 98 | itvl = (existing[0], itvl[1]) 99 | continue 100 | 101 | if itvl[1] == existing[0]: 102 | itvl = (itvl[0], existing[1]) 103 | continue 104 | 105 | new.append(existing) 106 | 107 | self.intervals = sorted(new + [itvl]) 108 | 109 | def __sub_interval(self, itvl): 110 | """Using the internal representation, remove an interval""" 111 | new = [] 112 | 113 | if itvl[0] > itvl[1]: 114 | raise Exception('Malformed interval %s' % (itvl,)) 115 | 116 | if not any(overlap(x, itvl) for x in self.intervals): 117 | # No raise 118 | return 119 | 120 | ## Merge: arrange existing set around interval 121 | for existing in sorted(self.intervals): 122 | if overlap(existing, itvl): 123 | if existing[0] < itvl[0]: 124 | new.append((existing[0], itvl[0])) 125 | if existing[1] > itvl[1]: 126 | new.append((itvl[1], existing[1])) 127 | else: 128 | new.append(existing) 129 | 130 | self.intervals = new 131 | 132 | def __contains__(self, other): 133 | if other.sid != self.sid: 134 | return False 135 | 136 | return all(any(contains(me, them) for me in self.intervals) 137 | for them in other.intervals) 138 | 139 | def __init__(self, gtid, sid=None, intervals=[]): 140 | if sid: 141 | intervals = intervals 142 | else: 143 | sid, intervals = Gtid.parse(gtid) 144 | 145 | self.sid = sid 146 | self.intervals = [] 147 | for itvl in intervals: 148 | self.__add_interval(itvl) 149 | 150 | def __add__(self, other): 151 | """Include the transactions of this gtid. Raise if the 152 | attempted merge has different SID""" 153 | if self.sid != other.sid: 154 | raise Exception('Attempt to merge different SID' 155 | '%s != %s' % (self.sid, other.sid)) 156 | 157 | result = Gtid(str(self)) 158 | 159 | for itvl in other.intervals: 160 | result.__add_interval(itvl) 161 | 162 | return result 163 | 164 | def __sub__(self, other): 165 | """Remove intervals. Do not raise, if different SID simply 166 | ignore""" 167 | result = Gtid(str(self)) 168 | if self.sid != other.sid: 169 | return result 170 | 171 | for itvl in other.intervals: 172 | result.__sub_interval(itvl) 173 | 174 | return result 175 | 176 | def __cmp__(self, other): 177 | if other.sid != self.sid: 178 | return cmp(self.sid, other.sid) 179 | return cmp(self.intervals, other.intervals) 180 | 181 | def __str__(self): 182 | """We represent the human value here - a single number 183 | for one transaction, or a closed interval (decrementing b)""" 184 | return '%s:%s' % (self.sid, 185 | ':'.join(('%d-%d' % (x[0], x[1] - 1)) if x[0] + 1 != x[1] 186 | else str(x[0]) 187 | for x in self.intervals)) 188 | 189 | def __repr__(self): 190 | return '' % self 191 | 192 | @property 193 | def encoded_length(self): 194 | return (16 + # sid 195 | 8 + # n_intervals 196 | 2 * # stop/start 197 | 8 * # stop/start mark encoded as int64 198 | len(self.intervals)) 199 | 200 | def encode(self): 201 | buffer = b'' 202 | # sid 203 | buffer += binascii.unhexlify(self.sid.replace('-', '')) 204 | # n_intervals 205 | buffer += struct.pack('' % self.gtids 285 | 286 | @property 287 | def encoded_length(self): 288 | return (8 + # n_sids 289 | sum(x.encoded_length for x in self.gtids)) 290 | 291 | def encoded(self): 292 | return b'' + (struct.pack('>> print(10,0xa,0b1010) 120 | 10 10 10 121 | ``` 122 | 十六进制和二进制与十进制转换： 123 | ``` 124 | >>> print(hex(10),bin(10)) 125 | 0xa 0b1010 126 | >>> print(int('0xa',16),int('0b1010',2)) 127 | 10 10 128 | ``` 129 | 由于1个字节（byte)最大能表示的数字为255，所以更大的数字需要用多个字节来表示，如： 130 | ``` 131 | # 2个字节，16bit,最大为 65535 132 | >>> 0b1111111111111111 133 | 65535 134 | 135 | # 4个字节，32bit, 最大数（0x为十六进制，1位十六进制等于4位二进制 0xf = 0b1111 136 | >>> 0xffffffff 137 | 4294967295 138 | ``` 139 | 以上均为无符号的数字，即全为正数，对于有符号的正负数，则最高位的1个bit用0和1分别表示正数和负数。对于1个byte的数字，实际就只有7bit表示实际的数字，范围为[-128，127]. 140 | 141 | ### 字符的表示 142 | 在计算机中所有的数据最终都要转化为数字，而且是二进制的数字。字符也不例外，也需要用到一个"映射表"来完成字符的表示。这个"映射表"叫作字符集，ASCII是最早最基础的"单字节"字符集，它可以表示键盘上所有的可打印字符，如52个大小写字母及标点符号。 143 | 144 | Python中，使用ord()和chr()完成ASCII字符与数字之间的转换： 145 | ``` 146 | >>> ord('a'),ord('b'),ord('c') 147 | (97, 98, 99) 148 | >>> chr(97),chr(98),chr(99) 149 | ('a', 'b', 'c') 150 | ``` 151 | "单字节"最大为数字是255，能表示的字符有限，所以后来就有了"多字节"字符集，如GBK，UTF8等等，用来表示更多的字符。其中UTF8是变长的字符编码，用1-6个字节表示一个字符，可以表示全世界所有的文字与符号，也叫万国码。 152 | 153 | Python中，多字节字符与数字间的转换： 154 | ``` 155 | # Python3中，字符对象（str), 可以使用 .encode方法将字符转为bytes对象 156 | >>> "中国".encode("utf8") 157 | b'\xe4\xb8\xad\xe5\x9b\xbd' 158 | >>> "中国".encode("gbk") 159 | b'\xd6\xd0\xb9\xfa' 160 | 161 | # bytes对象转成字符 162 | b'\xe4\xb8\xad\xe5\x9b\xbd'.decode("utf8") 163 | '中国' 164 | bytes([0xe4,0xb8,0xad,0xe5,0x9b,0xbd]).decode("utf8") 165 | '中国' 166 | 167 | ``` 168 | 使用hexdump查看文本文件的字符编码： 169 | ``` 170 | $ file /tmp/python_chr.txt 171 | /tmp/python_chr.txt: UTF-8 Unicode text 172 | $ cat /tmp/python_chr.txt 173 | Python 174 | 中国 175 | $ hexdump -C /tmp/python_chr.txt 176 | 00000000 50 79 74 68 6f 6e 0a e4 b8 ad e5 9b bd 0a |Python........| 177 | 0000000e 178 | ``` 179 | 使用python来验证编码： 180 | ``` 181 | # 前三个字符 182 | >>> chr(0x50),chr(0x79),chr(0x74) 183 | ('P', 'y', 't') 184 | # 剩下的字符大家动手试一试， 特别是汉字"中国"的编码 185 | ``` 186 | ## Python二进制相关 187 | ### bytes对象 188 | bytes是Python3中新增的一个处理二进制"流"的对象。可以下几种方式我们可以得到bytes对象： 189 | * 字符对象的encode方法 190 | * 二进制文件read方法 191 | * 网络socket的recv方法 192 | * 使用b打头的字符申明 193 | * 使用bytes对象初始化 194 | 195 | 一些简单的例子： 196 | ``` 197 | >>> b'a' 198 | b'a' 199 | >>> type(b'a') 200 | 201 | >>> bytes([97]) 202 | b'a' 203 | >>> bytes("中国",'utf8') 204 | b'\xe4\xb8\xad\xe5\x9b\xbd' 205 | ``` 206 | 207 | 可以把bytes看作是一个特殊的数组，由连续的字节(byte)组成，单字节最大数不能超过255，具有数组的切片，迭代等特性，它总是尝试以ASCII编码将数据转成可显示字符，超出ASCII可显示范围则使用\x打头的二位十六进制进行显示。 208 | 209 | bytes对象的本质是存的二进制数组，存放的是0-255的数字数组，它只有结合"字符集"才能转换正确的字符，或者要结合某种"协议"才能解读出具体的"含义"，这一点后面就会详细的讲到。 210 | 211 | 再来一个例子, 打印GBK编码表： 212 | ``` 213 | # GBK编码从0x8140 开始，显示 30 行 214 | for row in [0x8140 + x*16 for x in range(30)]: 215 | print(hex(row), end=" ") 216 | # 每行显示16个 217 | for i in range(16): 218 | high = row+i >> 8 & 0xff # 高位 219 | low = row+i & 0xff # 低位 220 | try: 221 | # 用bytes对象转换成GBK字符 222 | print(bytes([high, low]).decode("gbk"), end="") 223 | except: 224 | print(end=" ") 225 | print("") 226 | ``` 227 | 输出： 228 | ``` 229 | 0x8140 丂丄丅丆丏丒丗丟丠両丣並丩丮丯丱 230 | 0x8150 丳丵丷丼乀乁乂乄乆乊乑乕乗乚乛乢 231 | 0x8160 乣乤乥乧乨乪乫乬乭乮乯乲乴乵乶乷 232 | 0x8170 乸乹乺乻乼乽乿亀亁亂亃亄亅亇亊 233 | 0x8180 亐亖亗亙亜亝亞亣亪亯亰亱亴亶亷亸 234 | 0x8190 亹亼亽亾仈仌仏仐仒仚仛仜仠仢仦仧 235 | 0x81a0 仩仭仮仯仱仴仸仹仺仼仾伀伂伃伄伅 236 | 0x81b0 伆伇伈伋伌伒伓伔伕伖伜伝伡伣伨伩 237 | 0x81c0 伬伭伮伱伳伵伷伹伻伾伿佀佁佂佄佅 238 | 0x81d0 佇佈佉佊佋佌佒佔佖佡佢佦佨佪佫佭 239 | 0x81e0 佮佱佲併佷佸佹佺佽侀侁侂侅來侇侊 240 | 0x81f0 侌侎侐侒侓侕侖侘侙侚侜侞侟価侢 241 | ``` 242 | 243 | ### struct 244 | 计算机中几乎所有的数据都可以最终抽象成数字和字符来表示，在C语言中用struct(结构体)来描述一个复杂的对象，通过这个结构可以方便的将复杂对象转换成二进制流用于存储与网络传输。Python中提供了struct模块方便处理二进制流(bytes对象）与数字，字符对象的转换功能。 245 | 246 | #### 用struct处理数字 247 | ``` 248 | >>> import struct 249 | # 单字节数字 250 | >>> struct.pack(">> struct.pack(">> struct.pack(">> struct.pack(">> struct.unpack(">> struct.unpack(">> struct.unpack(">> struct.pack("<4s",b"cat") 284 | b'cat\x00' 285 | 286 | >>> struct.pack("<5s","中国".encode("gbk")) 287 | b'\xd6\xd0\xb9\xfa\x00' 288 | 289 | >>> struct.pack("<7s","中国".encode("utf8")) 290 | b'\xe4\xb8\xad\xe5\x9b\xbd\x00' 291 | 292 | # 定长字符串，第1个字节为字符串的长度 293 | >>> struct.pack("<4p",b"cat") 294 | b'\x03cat' 295 | 296 | >>> struct.pack("<5p","中国".encode("gbk")) 297 | b'\x04\xd6\xd0\xb9\xfa' 298 | 299 | >>> struct.pack("<7p","中国".encode("utf8")) 300 | b'\x06\xe4\xb8\xad\xe5\x9b\xbd' 301 | 302 | ``` 303 | 304 | bytes转换为字符: 305 | ``` 306 | # 仅取一例，其他的请自己动手试一试 307 | >>> struct.unpack("<7p", b'\x06\xe4\xb8\xad\xe5\x9b\xbd')[0].decode("utf8") 308 | '中国' 309 | ``` 310 | 需要特别说明的是，unpack返回的是元组，哪怕是只有一个元素，这样做的好处是，我们可以按照规则将多个数据的format写在一起，让代码更加简洁： 311 | ``` 312 | >>> struct.pack(">> struct.unpack(">> id, no, port, name = struct.unpack(">> id, no, port, name 320 | (1, 19, 3306, b'alvin') 321 | 322 | ``` 323 | 这种写法会大量应用到后继的demo代码中，请务必多加练习，并细仔阅读官方文档。 324 | 325 | ## Python Socket编程 326 | 简单说Socket编程，就是面向网络传输层的接口编程，系统通过IP地址和端口号建立起两台电脑之间网络连接，并提供两个最基础的通信接口发送数据和接收数据，供开发者调用，先来看一个最简单的客户端Socket例子： 327 | ``` 328 | import socket 329 | 330 | # 创建一个socket对象 331 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 332 | # 建立连接 333 | s.connect(("192.168.1.101", 3306)) 334 | # 接收数据 335 | buf = s.recv(10240) 336 | print(type(buf)) # 337 | # 发送数据 338 | s.send(b'hello') 339 | ``` 340 | 可以看出通过socket接收和发送的数据都是前面讲的bytes对象，因为bytes对象本身只是一个二进制流，所以在没有"协议"的前提下，我们是无法理解传输内容的具体含义。常见的http,https,ftp,smtp,ssh协议都是建立socket通信之上的协议。换句说，就是通socket编程可以实现与现有的任何协议进行通信。如果你熟悉了ssh协议，那么实现ssh端口扫描程序就易如反掌了。 341 | 342 | 用socket不仅可以和其它协议的服务端进行通信，而且可以实现socket服务端，监听和处理来自client的连接和数据。 343 | ``` 344 | import socket 345 | 346 | # 创建一个socket对象 347 | s = socket.socket() 348 | # 监听端口 349 | s.bind(('127.0.0.1', 8000)) 350 | s.listen(5) 351 | 352 | while True: 353 | conn, addr = s.accept() 354 | conn.send(bytes('Welcome python socket server.', 'utf8')) 355 | # 关闭链接 356 | conn.close() 357 | ``` 358 | 通过上面两个简单的例子，相信大家对Python的socket编程已经有一个初步的认识，那就是"相当的简单"，没有想象中那么复杂。 359 | 360 | 接下再来看一个多线程版的SocketServer, 可以通过telnet来实现一个网络计算器： 361 | ``` 362 | # learn_socket3_server_mulit_thread.py 363 | 364 | import threading 365 | import socketserver 366 | 367 | class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler): 368 | 369 | def handle(self): 370 | """ 371 | 网络计算器，返回表达式的值 372 | """ 373 | while True: 374 | try: 375 | # 接收表达式数据 376 | data = str(self.request.recv(1024), 'ascii').strip() 377 | if "q" in data: 378 | self.finish() 379 | break 380 | # 计算结果 381 | response = bytes("{} = {}\r\n".format(data, eval(data)), 'ascii') 382 | print(response.decode("ascii").strip()) 383 | # 返回结果 384 | self.request.sendall(response) 385 | except: 386 | self.request.sendall(bytes("\n", 'ascii')) 387 | 388 | class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): 389 | pass 390 | 391 | if __name__ == "__main__": 392 | 393 | server = ThreadedTCPServer(("127.0.0.1", 9000), ThreadedTCPRequestHandler) 394 | ip, port = server.server_address 395 | 396 | server_thread = threading.Thread(target=server.serve_forever) 397 | print(f"Calculator Server start at {ip} : {port}") 398 | server_thread.start() 399 | 400 | ``` 401 | 使用telnet进行测试： 402 | ``` 403 | $ telnet 127.0.0.1 9000 404 | Trying 127.0.0.1... 405 | Connected to localhost. 406 | Escape character is '^]'. 407 | 12345679*81 # 按回车 408 | 12345679*81 = 999999999 # 返回结果 409 | 3+2-5*0 # Enter 410 | 3+2-5*0 = 5 411 | (123+123)*123 # Enter 412 | (123+123)*123 = 30258 413 | quit # Enter 414 | ``` 415 | 服务端日志： 416 | ``` 417 | Calculator Server start at 127.0.0.1 : 9000 418 | 12345679*81 = 999999999 419 | 3+2-5*0 = 5 420 | (123+123)*123 = 30258 421 | ``` 422 | 423 | ## 小结 424 | 理解二进制，字符/编码，socket通信，以及如何使用Python来处理它们，是实现BinlogServer最重要的基础，由于篇幅问题，很多知识点只能点到为止，虽然很基础，但是还是需要自己的动手去实验，举一反三地多实践自己的想法，会对理解后面的文章大有帮助。 425 | 426 | 只有会认真看文档的DBA才是好DBA，只会认真看代码的Engineer,一定不是好Engineer。代码一定要运行起来，On Runtime才会有价值，才会让你变成好Engineer. ^_^ 427 | 428 | 最后，祝你编码快乐〜 429 | 430 | ## 相关文档 431 | - https://docs.python.org/3/library/struct.html 432 | - https://docs.python.org/3/library/socketserver.html 433 | 434 | ## 附：基于mysqlbinlog命令的BinlogServer简单实现 435 | 436 | ``` 437 | #!/bin/sh 438 | 439 | REMOTE_HOST={{host}} 440 | REMOTE_PORT={{mysql_port}} 441 | REMOTE_USER={{mysql_repl_user}} 442 | REMOTE_PASS={{mysql_repl_password}} 443 | 444 | BACKUP_BIN=/usr/local/mysql/bin/mysqlbinlog 445 | LOCAL_BACKUP_DIR=/data/backup/mysql/binlog_3306 446 | BACKUP_LOG=/data/backup/mysql/binlog_3306/backup_3306.log 447 | 448 | FIRST_BINLOG=mysql-bin.000001 449 | #time to wait before reconnecting after failure 450 | SLEEP_SECONDS=10 451 | 452 | ##create local_backup_dir if necessary 453 | mkdir -p ${LOCAL_BACKUP_DIR} 454 | 455 | cd ${LOCAL_BACKUP_DIR} 456 | ## Function while loop ， After the connection is disconnected, wait for the specified time. ， Reconnect 457 | while : 458 | do 459 | if [ `ls -A "${LOCAL_BACKUP_DIR}" |wc -l` -eq 0 ];then 460 | LAST_FILE=${FIRST_BINLOG} 461 | else 462 | LAST_FILE=`ls -l ${LOCAL_BACKUP_DIR} | grep -v backuplog |tail -n 1 |awk '{print $9}'` 463 | fi 464 | 465 | ${BACKUP_BIN} --raw --read-from-remote-server --stop-never --host=${REMOTE_HOST} --port=${REMOTE_PORT} --user=${REMOTE_USER} --password=${REMOTE_PASS} ${LAST_FILE} 466 | echo "`date +"%Y/%m/%d %H:%M:%S"` mysqlbinlog Stop it ， Return code ：$?" | tee -a ${BACKUP_LOG} 467 | echo "${SLEEP_SECONDS} After the second connect and continue to backup " | tee -a ${BACKUP_LOG} 468 | sleep ${SLEEP_SECONDS} 469 | done 470 | ``` 471 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/constants/ER.py: -------------------------------------------------------------------------------- 1 | 2 | ERROR_FIRST = 1000 3 | HASHCHK = 1000 4 | NISAMCHK = 1001 5 | NO = 1002 6 | YES = 1003 7 | CANT_CREATE_FILE = 1004 8 | CANT_CREATE_TABLE = 1005 9 | CANT_CREATE_DB = 1006 10 | DB_CREATE_EXISTS = 1007 11 | DB_DROP_EXISTS = 1008 12 | DB_DROP_DELETE = 1009 13 | DB_DROP_RMDIR = 1010 14 | CANT_DELETE_FILE = 1011 15 | CANT_FIND_SYSTEM_REC = 1012 16 | CANT_GET_STAT = 1013 17 | CANT_GET_WD = 1014 18 | CANT_LOCK = 1015 19 | CANT_OPEN_FILE = 1016 20 | FILE_NOT_FOUND = 1017 21 | CANT_READ_DIR = 1018 22 | CANT_SET_WD = 1019 23 | CHECKREAD = 1020 24 | DISK_FULL = 1021 25 | DUP_KEY = 1022 26 | ERROR_ON_CLOSE = 1023 27 | ERROR_ON_READ = 1024 28 | ERROR_ON_RENAME = 1025 29 | ERROR_ON_WRITE = 1026 30 | FILE_USED = 1027 31 | FILSORT_ABORT = 1028 32 | FORM_NOT_FOUND = 1029 33 | GET_ERRNO = 1030 34 | ILLEGAL_HA = 1031 35 | KEY_NOT_FOUND = 1032 36 | NOT_FORM_FILE = 1033 37 | NOT_KEYFILE = 1034 38 | OLD_KEYFILE = 1035 39 | OPEN_AS_READONLY = 1036 40 | OUTOFMEMORY = 1037 41 | OUT_OF_SORTMEMORY = 1038 42 | UNEXPECTED_EOF = 1039 43 | CON_COUNT_ERROR = 1040 44 | OUT_OF_RESOURCES = 1041 45 | BAD_HOST_ERROR = 1042 46 | HANDSHAKE_ERROR = 1043 47 | DBACCESS_DENIED_ERROR = 1044 48 | ACCESS_DENIED_ERROR = 1045 49 | NO_DB_ERROR = 1046 50 | UNKNOWN_COM_ERROR = 1047 51 | BAD_NULL_ERROR = 1048 52 | BAD_DB_ERROR = 1049 53 | TABLE_EXISTS_ERROR = 1050 54 | BAD_TABLE_ERROR = 1051 55 | NON_UNIQ_ERROR = 1052 56 | SERVER_SHUTDOWN = 1053 57 | BAD_FIELD_ERROR = 1054 58 | WRONG_FIELD_WITH_GROUP = 1055 59 | WRONG_GROUP_FIELD = 1056 60 | WRONG_SUM_SELECT = 1057 61 | WRONG_VALUE_COUNT = 1058 62 | TOO_LONG_IDENT = 1059 63 | DUP_FIELDNAME = 1060 64 | DUP_KEYNAME = 1061 65 | DUP_ENTRY = 1062 66 | WRONG_FIELD_SPEC = 1063 67 | PARSE_ERROR = 1064 68 | EMPTY_QUERY = 1065 69 | NONUNIQ_TABLE = 1066 70 | INVALID_DEFAULT = 1067 71 | MULTIPLE_PRI_KEY = 1068 72 | TOO_MANY_KEYS = 1069 73 | TOO_MANY_KEY_PARTS = 1070 74 | TOO_LONG_KEY = 1071 75 | KEY_COLUMN_DOES_NOT_EXITS = 1072 76 | BLOB_USED_AS_KEY = 1073 77 | TOO_BIG_FIELDLENGTH = 1074 78 | WRONG_AUTO_KEY = 1075 79 | READY = 1076 80 | NORMAL_SHUTDOWN = 1077 81 | GOT_SIGNAL = 1078 82 | SHUTDOWN_COMPLETE = 1079 83 | FORCING_CLOSE = 1080 84 | IPSOCK_ERROR = 1081 85 | NO_SUCH_INDEX = 1082 86 | WRONG_FIELD_TERMINATORS = 1083 87 | BLOBS_AND_NO_TERMINATED = 1084 88 | TEXTFILE_NOT_READABLE = 1085 89 | FILE_EXISTS_ERROR = 1086 90 | LOAD_INFO = 1087 91 | ALTER_INFO = 1088 92 | WRONG_SUB_KEY = 1089 93 | CANT_REMOVE_ALL_FIELDS = 1090 94 | CANT_DROP_FIELD_OR_KEY = 1091 95 | INSERT_INFO = 1092 96 | UPDATE_TABLE_USED = 1093 97 | NO_SUCH_THREAD = 1094 98 | KILL_DENIED_ERROR = 1095 99 | NO_TABLES_USED = 1096 100 | TOO_BIG_SET = 1097 101 | NO_UNIQUE_LOGFILE = 1098 102 | TABLE_NOT_LOCKED_FOR_WRITE = 1099 103 | TABLE_NOT_LOCKED = 1100 104 | BLOB_CANT_HAVE_DEFAULT = 1101 105 | WRONG_DB_NAME = 1102 106 | WRONG_TABLE_NAME = 1103 107 | TOO_BIG_SELECT = 1104 108 | UNKNOWN_ERROR = 1105 109 | UNKNOWN_PROCEDURE = 1106 110 | WRONG_PARAMCOUNT_TO_PROCEDURE = 1107 111 | WRONG_PARAMETERS_TO_PROCEDURE = 1108 112 | UNKNOWN_TABLE = 1109 113 | FIELD_SPECIFIED_TWICE = 1110 114 | INVALID_GROUP_FUNC_USE = 1111 115 | UNSUPPORTED_EXTENSION = 1112 116 | TABLE_MUST_HAVE_COLUMNS = 1113 117 | RECORD_FILE_FULL = 1114 118 | UNKNOWN_CHARACTER_SET = 1115 119 | TOO_MANY_TABLES = 1116 120 | TOO_MANY_FIELDS = 1117 121 | TOO_BIG_ROWSIZE = 1118 122 | STACK_OVERRUN = 1119 123 | WRONG_OUTER_JOIN = 1120 124 | NULL_COLUMN_IN_INDEX = 1121 125 | CANT_FIND_UDF = 1122 126 | CANT_INITIALIZE_UDF = 1123 127 | UDF_NO_PATHS = 1124 128 | UDF_EXISTS = 1125 129 | CANT_OPEN_LIBRARY = 1126 130 | CANT_FIND_DL_ENTRY = 1127 131 | FUNCTION_NOT_DEFINED = 1128 132 | HOST_IS_BLOCKED = 1129 133 | HOST_NOT_PRIVILEGED = 1130 134 | PASSWORD_ANONYMOUS_USER = 1131 135 | PASSWORD_NOT_ALLOWED = 1132 136 | PASSWORD_NO_MATCH = 1133 137 | UPDATE_INFO = 1134 138 | CANT_CREATE_THREAD = 1135 139 | WRONG_VALUE_COUNT_ON_ROW = 1136 140 | CANT_REOPEN_TABLE = 1137 141 | INVALID_USE_OF_NULL = 1138 142 | REGEXP_ERROR = 1139 143 | MIX_OF_GROUP_FUNC_AND_FIELDS = 1140 144 | NONEXISTING_GRANT = 1141 145 | TABLEACCESS_DENIED_ERROR = 1142 146 | COLUMNACCESS_DENIED_ERROR = 1143 147 | ILLEGAL_GRANT_FOR_TABLE = 1144 148 | GRANT_WRONG_HOST_OR_USER = 1145 149 | NO_SUCH_TABLE = 1146 150 | NONEXISTING_TABLE_GRANT = 1147 151 | NOT_ALLOWED_COMMAND = 1148 152 | SYNTAX_ERROR = 1149 153 | DELAYED_CANT_CHANGE_LOCK = 1150 154 | TOO_MANY_DELAYED_THREADS = 1151 155 | ABORTING_CONNECTION = 1152 156 | NET_PACKET_TOO_LARGE = 1153 157 | NET_READ_ERROR_FROM_PIPE = 1154 158 | NET_FCNTL_ERROR = 1155 159 | NET_PACKETS_OUT_OF_ORDER = 1156 160 | NET_UNCOMPRESS_ERROR = 1157 161 | NET_READ_ERROR = 1158 162 | NET_READ_INTERRUPTED = 1159 163 | NET_ERROR_ON_WRITE = 1160 164 | NET_WRITE_INTERRUPTED = 1161 165 | TOO_LONG_STRING = 1162 166 | TABLE_CANT_HANDLE_BLOB = 1163 167 | TABLE_CANT_HANDLE_AUTO_INCREMENT = 1164 168 | DELAYED_INSERT_TABLE_LOCKED = 1165 169 | WRONG_COLUMN_NAME = 1166 170 | WRONG_KEY_COLUMN = 1167 171 | WRONG_MRG_TABLE = 1168 172 | DUP_UNIQUE = 1169 173 | BLOB_KEY_WITHOUT_LENGTH = 1170 174 | PRIMARY_CANT_HAVE_NULL = 1171 175 | TOO_MANY_ROWS = 1172 176 | REQUIRES_PRIMARY_KEY = 1173 177 | NO_RAID_COMPILED = 1174 178 | UPDATE_WITHOUT_KEY_IN_SAFE_MODE = 1175 179 | KEY_DOES_NOT_EXITS = 1176 180 | CHECK_NO_SUCH_TABLE = 1177 181 | CHECK_NOT_IMPLEMENTED = 1178 182 | CANT_DO_THIS_DURING_AN_TRANSACTION = 1179 183 | ERROR_DURING_COMMIT = 1180 184 | ERROR_DURING_ROLLBACK = 1181 185 | ERROR_DURING_FLUSH_LOGS = 1182 186 | ERROR_DURING_CHECKPOINT = 1183 187 | NEW_ABORTING_CONNECTION = 1184 188 | DUMP_NOT_IMPLEMENTED = 1185 189 | FLUSH_MASTER_BINLOG_CLOSED = 1186 190 | INDEX_REBUILD = 1187 191 | MASTER = 1188 192 | MASTER_NET_READ = 1189 193 | MASTER_NET_WRITE = 1190 194 | FT_MATCHING_KEY_NOT_FOUND = 1191 195 | LOCK_OR_ACTIVE_TRANSACTION = 1192 196 | UNKNOWN_SYSTEM_VARIABLE = 1193 197 | CRASHED_ON_USAGE = 1194 198 | CRASHED_ON_REPAIR = 1195 199 | WARNING_NOT_COMPLETE_ROLLBACK = 1196 200 | TRANS_CACHE_FULL = 1197 201 | SLAVE_MUST_STOP = 1198 202 | SLAVE_NOT_RUNNING = 1199 203 | BAD_SLAVE = 1200 204 | MASTER_INFO = 1201 205 | SLAVE_THREAD = 1202 206 | TOO_MANY_USER_CONNECTIONS = 1203 207 | SET_CONSTANTS_ONLY = 1204 208 | LOCK_WAIT_TIMEOUT = 1205 209 | LOCK_TABLE_FULL = 1206 210 | READ_ONLY_TRANSACTION = 1207 211 | DROP_DB_WITH_READ_LOCK = 1208 212 | CREATE_DB_WITH_READ_LOCK = 1209 213 | WRONG_ARGUMENTS = 1210 214 | NO_PERMISSION_TO_CREATE_USER = 1211 215 | UNION_TABLES_IN_DIFFERENT_DIR = 1212 216 | LOCK_DEADLOCK = 1213 217 | TABLE_CANT_HANDLE_FT = 1214 218 | CANNOT_ADD_FOREIGN = 1215 219 | NO_REFERENCED_ROW = 1216 220 | ROW_IS_REFERENCED = 1217 221 | CONNECT_TO_MASTER = 1218 222 | QUERY_ON_MASTER = 1219 223 | ERROR_WHEN_EXECUTING_COMMAND = 1220 224 | WRONG_USAGE = 1221 225 | WRONG_NUMBER_OF_COLUMNS_IN_SELECT = 1222 226 | CANT_UPDATE_WITH_READLOCK = 1223 227 | MIXING_NOT_ALLOWED = 1224 228 | DUP_ARGUMENT = 1225 229 | USER_LIMIT_REACHED = 1226 230 | SPECIFIC_ACCESS_DENIED_ERROR = 1227 231 | LOCAL_VARIABLE = 1228 232 | GLOBAL_VARIABLE = 1229 233 | NO_DEFAULT = 1230 234 | WRONG_VALUE_FOR_VAR = 1231 235 | WRONG_TYPE_FOR_VAR = 1232 236 | VAR_CANT_BE_READ = 1233 237 | CANT_USE_OPTION_HERE = 1234 238 | NOT_SUPPORTED_YET = 1235 239 | MASTER_FATAL_ERROR_READING_BINLOG = 1236 240 | SLAVE_IGNORED_TABLE = 1237 241 | INCORRECT_GLOBAL_LOCAL_VAR = 1238 242 | WRONG_FK_DEF = 1239 243 | KEY_REF_DO_NOT_MATCH_TABLE_REF = 1240 244 | OPERAND_COLUMNS = 1241 245 | SUBQUERY_NO_1_ROW = 1242 246 | UNKNOWN_STMT_HANDLER = 1243 247 | CORRUPT_HELP_DB = 1244 248 | CYCLIC_REFERENCE = 1245 249 | AUTO_CONVERT = 1246 250 | ILLEGAL_REFERENCE = 1247 251 | DERIVED_MUST_HAVE_ALIAS = 1248 252 | SELECT_REDUCED = 1249 253 | TABLENAME_NOT_ALLOWED_HERE = 1250 254 | NOT_SUPPORTED_AUTH_MODE = 1251 255 | SPATIAL_CANT_HAVE_NULL = 1252 256 | COLLATION_CHARSET_MISMATCH = 1253 257 | SLAVE_WAS_RUNNING = 1254 258 | SLAVE_WAS_NOT_RUNNING = 1255 259 | TOO_BIG_FOR_UNCOMPRESS = 1256 260 | ZLIB_Z_MEM_ERROR = 1257 261 | ZLIB_Z_BUF_ERROR = 1258 262 | ZLIB_Z_DATA_ERROR = 1259 263 | CUT_VALUE_GROUP_CONCAT = 1260 264 | WARN_TOO_FEW_RECORDS = 1261 265 | WARN_TOO_MANY_RECORDS = 1262 266 | WARN_NULL_TO_NOTNULL = 1263 267 | WARN_DATA_OUT_OF_RANGE = 1264 268 | WARN_DATA_TRUNCATED = 1265 269 | WARN_USING_OTHER_HANDLER = 1266 270 | CANT_AGGREGATE_2COLLATIONS = 1267 271 | DROP_USER = 1268 272 | REVOKE_GRANTS = 1269 273 | CANT_AGGREGATE_3COLLATIONS = 1270 274 | CANT_AGGREGATE_NCOLLATIONS = 1271 275 | VARIABLE_IS_NOT_STRUCT = 1272 276 | UNKNOWN_COLLATION = 1273 277 | SLAVE_IGNORED_SSL_PARAMS = 1274 278 | SERVER_IS_IN_SECURE_AUTH_MODE = 1275 279 | WARN_FIELD_RESOLVED = 1276 280 | BAD_SLAVE_UNTIL_COND = 1277 281 | MISSING_SKIP_SLAVE = 1278 282 | UNTIL_COND_IGNORED = 1279 283 | WRONG_NAME_FOR_INDEX = 1280 284 | WRONG_NAME_FOR_CATALOG = 1281 285 | WARN_QC_RESIZE = 1282 286 | BAD_FT_COLUMN = 1283 287 | UNKNOWN_KEY_CACHE = 1284 288 | WARN_HOSTNAME_WONT_WORK = 1285 289 | UNKNOWN_STORAGE_ENGINE = 1286 290 | WARN_DEPRECATED_SYNTAX = 1287 291 | NON_UPDATABLE_TABLE = 1288 292 | FEATURE_DISABLED = 1289 293 | OPTION_PREVENTS_STATEMENT = 1290 294 | DUPLICATED_VALUE_IN_TYPE = 1291 295 | TRUNCATED_WRONG_VALUE = 1292 296 | TOO_MUCH_AUTO_TIMESTAMP_COLS = 1293 297 | INVALID_ON_UPDATE = 1294 298 | UNSUPPORTED_PS = 1295 299 | GET_ERRMSG = 1296 300 | GET_TEMPORARY_ERRMSG = 1297 301 | UNKNOWN_TIME_ZONE = 1298 302 | WARN_INVALID_TIMESTAMP = 1299 303 | INVALID_CHARACTER_STRING = 1300 304 | WARN_ALLOWED_PACKET_OVERFLOWED = 1301 305 | CONFLICTING_DECLARATIONS = 1302 306 | SP_NO_RECURSIVE_CREATE = 1303 307 | SP_ALREADY_EXISTS = 1304 308 | SP_DOES_NOT_EXIST = 1305 309 | SP_DROP_FAILED = 1306 310 | SP_STORE_FAILED = 1307 311 | SP_LILABEL_MISMATCH = 1308 312 | SP_LABEL_REDEFINE = 1309 313 | SP_LABEL_MISMATCH = 1310 314 | SP_UNINIT_VAR = 1311 315 | SP_BADSELECT = 1312 316 | SP_BADRETURN = 1313 317 | SP_BADSTATEMENT = 1314 318 | UPDATE_LOG_DEPRECATED_IGNORED = 1315 319 | UPDATE_LOG_DEPRECATED_TRANSLATED = 1316 320 | QUERY_INTERRUPTED = 1317 321 | SP_WRONG_NO_OF_ARGS = 1318 322 | SP_COND_MISMATCH = 1319 323 | SP_NORETURN = 1320 324 | SP_NORETURNEND = 1321 325 | SP_BAD_CURSOR_QUERY = 1322 326 | SP_BAD_CURSOR_SELECT = 1323 327 | SP_CURSOR_MISMATCH = 1324 328 | SP_CURSOR_ALREADY_OPEN = 1325 329 | SP_CURSOR_NOT_OPEN = 1326 330 | SP_UNDECLARED_VAR = 1327 331 | SP_WRONG_NO_OF_FETCH_ARGS = 1328 332 | SP_FETCH_NO_DATA = 1329 333 | SP_DUP_PARAM = 1330 334 | SP_DUP_VAR = 1331 335 | SP_DUP_COND = 1332 336 | SP_DUP_CURS = 1333 337 | SP_CANT_ALTER = 1334 338 | SP_SUBSELECT_NYI = 1335 339 | STMT_NOT_ALLOWED_IN_SF_OR_TRG = 1336 340 | SP_VARCOND_AFTER_CURSHNDLR = 1337 341 | SP_CURSOR_AFTER_HANDLER = 1338 342 | SP_CASE_NOT_FOUND = 1339 343 | FPARSER_TOO_BIG_FILE = 1340 344 | FPARSER_BAD_HEADER = 1341 345 | FPARSER_EOF_IN_COMMENT = 1342 346 | FPARSER_ERROR_IN_PARAMETER = 1343 347 | FPARSER_EOF_IN_UNKNOWN_PARAMETER = 1344 348 | VIEW_NO_EXPLAIN = 1345 349 | FRM_UNKNOWN_TYPE = 1346 350 | WRONG_OBJECT = 1347 351 | NONUPDATEABLE_COLUMN = 1348 352 | VIEW_SELECT_DERIVED = 1349 353 | VIEW_SELECT_CLAUSE = 1350 354 | VIEW_SELECT_VARIABLE = 1351 355 | VIEW_SELECT_TMPTABLE = 1352 356 | VIEW_WRONG_LIST = 1353 357 | WARN_VIEW_MERGE = 1354 358 | WARN_VIEW_WITHOUT_KEY = 1355 359 | VIEW_INVALID = 1356 360 | SP_NO_DROP_SP = 1357 361 | SP_GOTO_IN_HNDLR = 1358 362 | TRG_ALREADY_EXISTS = 1359 363 | TRG_DOES_NOT_EXIST = 1360 364 | TRG_ON_VIEW_OR_TEMP_TABLE = 1361 365 | TRG_CANT_CHANGE_ROW = 1362 366 | TRG_NO_SUCH_ROW_IN_TRG = 1363 367 | NO_DEFAULT_FOR_FIELD = 1364 368 | DIVISION_BY_ZERO = 1365 369 | TRUNCATED_WRONG_VALUE_FOR_FIELD = 1366 370 | ILLEGAL_VALUE_FOR_TYPE = 1367 371 | VIEW_NONUPD_CHECK = 1368 372 | VIEW_CHECK_FAILED = 1369 373 | PROCACCESS_DENIED_ERROR = 1370 374 | RELAY_LOG_FAIL = 1371 375 | PASSWD_LENGTH = 1372 376 | UNKNOWN_TARGET_BINLOG = 1373 377 | IO_ERR_LOG_INDEX_READ = 1374 378 | BINLOG_PURGE_PROHIBITED = 1375 379 | FSEEK_FAIL = 1376 380 | BINLOG_PURGE_FATAL_ERR = 1377 381 | LOG_IN_USE = 1378 382 | LOG_PURGE_UNKNOWN_ERR = 1379 383 | RELAY_LOG_INIT = 1380 384 | NO_BINARY_LOGGING = 1381 385 | RESERVED_SYNTAX = 1382 386 | WSAS_FAILED = 1383 387 | DIFF_GROUPS_PROC = 1384 388 | NO_GROUP_FOR_PROC = 1385 389 | ORDER_WITH_PROC = 1386 390 | LOGGING_PROHIBIT_CHANGING_OF = 1387 391 | NO_FILE_MAPPING = 1388 392 | WRONG_MAGIC = 1389 393 | PS_MANY_PARAM = 1390 394 | KEY_PART_0 = 1391 395 | VIEW_CHECKSUM = 1392 396 | VIEW_MULTIUPDATE = 1393 397 | VIEW_NO_INSERT_FIELD_LIST = 1394 398 | VIEW_DELETE_MERGE_VIEW = 1395 399 | CANNOT_USER = 1396 400 | XAER_NOTA = 1397 401 | XAER_INVAL = 1398 402 | XAER_RMFAIL = 1399 403 | XAER_OUTSIDE = 1400 404 | XAER_RMERR = 1401 405 | XA_RBROLLBACK = 1402 406 | NONEXISTING_PROC_GRANT = 1403 407 | PROC_AUTO_GRANT_FAIL = 1404 408 | PROC_AUTO_REVOKE_FAIL = 1405 409 | DATA_TOO_LONG = 1406 410 | SP_BAD_SQLSTATE = 1407 411 | STARTUP = 1408 412 | LOAD_FROM_FIXED_SIZE_ROWS_TO_VAR = 1409 413 | CANT_CREATE_USER_WITH_GRANT = 1410 414 | WRONG_VALUE_FOR_TYPE = 1411 415 | TABLE_DEF_CHANGED = 1412 416 | SP_DUP_HANDLER = 1413 417 | SP_NOT_VAR_ARG = 1414 418 | SP_NO_RETSET = 1415 419 | CANT_CREATE_GEOMETRY_OBJECT = 1416 420 | FAILED_ROUTINE_BREAK_BINLOG = 1417 421 | BINLOG_UNSAFE_ROUTINE = 1418 422 | BINLOG_CREATE_ROUTINE_NEED_SUPER = 1419 423 | EXEC_STMT_WITH_OPEN_CURSOR = 1420 424 | STMT_HAS_NO_OPEN_CURSOR = 1421 425 | COMMIT_NOT_ALLOWED_IN_SF_OR_TRG = 1422 426 | NO_DEFAULT_FOR_VIEW_FIELD = 1423 427 | SP_NO_RECURSION = 1424 428 | TOO_BIG_SCALE = 1425 429 | TOO_BIG_PRECISION = 1426 430 | M_BIGGER_THAN_D = 1427 431 | WRONG_LOCK_OF_SYSTEM_TABLE = 1428 432 | CONNECT_TO_FOREIGN_DATA_SOURCE = 1429 433 | QUERY_ON_FOREIGN_DATA_SOURCE = 1430 434 | FOREIGN_DATA_SOURCE_DOESNT_EXIST = 1431 435 | FOREIGN_DATA_STRING_INVALID_CANT_CREATE = 1432 436 | FOREIGN_DATA_STRING_INVALID = 1433 437 | CANT_CREATE_FEDERATED_TABLE = 1434 438 | TRG_IN_WRONG_SCHEMA = 1435 439 | STACK_OVERRUN_NEED_MORE = 1436 440 | TOO_LONG_BODY = 1437 441 | WARN_CANT_DROP_DEFAULT_KEYCACHE = 1438 442 | TOO_BIG_DISPLAYWIDTH = 1439 443 | XAER_DUPID = 1440 444 | DATETIME_FUNCTION_OVERFLOW = 1441 445 | CANT_UPDATE_USED_TABLE_IN_SF_OR_TRG = 1442 446 | VIEW_PREVENT_UPDATE = 1443 447 | PS_NO_RECURSION = 1444 448 | SP_CANT_SET_AUTOCOMMIT = 1445 449 | MALFORMED_DEFINER = 1446 450 | VIEW_FRM_NO_USER = 1447 451 | VIEW_OTHER_USER = 1448 452 | NO_SUCH_USER = 1449 453 | FORBID_SCHEMA_CHANGE = 1450 454 | ROW_IS_REFERENCED_2 = 1451 455 | NO_REFERENCED_ROW_2 = 1452 456 | SP_BAD_VAR_SHADOW = 1453 457 | TRG_NO_DEFINER = 1454 458 | OLD_FILE_FORMAT = 1455 459 | SP_RECURSION_LIMIT = 1456 460 | SP_PROC_TABLE_CORRUPT = 1457 461 | SP_WRONG_NAME = 1458 462 | TABLE_NEEDS_UPGRADE = 1459 463 | SP_NO_AGGREGATE = 1460 464 | MAX_PREPARED_STMT_COUNT_REACHED = 1461 465 | VIEW_RECURSIVE = 1462 466 | NON_GROUPING_FIELD_USED = 1463 467 | TABLE_CANT_HANDLE_SPKEYS = 1464 468 | NO_TRIGGERS_ON_SYSTEM_SCHEMA = 1465 469 | USERNAME = 1466 470 | HOSTNAME = 1467 471 | WRONG_STRING_LENGTH = 1468 472 | ERROR_LAST = 1468 473 | 474 | # https://github.com/PyMySQL/PyMySQL/issues/607 475 | CONSTRAINT_FAILED = 4025 476 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/server.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import logging 3 | import os 4 | import socketserver 5 | import struct 6 | import sys 7 | import threading 8 | import configparser 9 | from _md5 import md5 10 | from io import BytesIO 11 | from py_mysql_binlogserver.constants.COMMAND import com_type_name 12 | from py_mysql_binlogserver.constants.EVENT_TYPE import event_type_name, GTID_LOG_EVENT, PREVIOUS_GTIDS_LOG_EVENT 13 | from py_mysql_binlogserver.packet.binlog_event import BinlogEvent 14 | from py_mysql_binlogserver.packet.challenge import Challenge 15 | from py_mysql_binlogserver.packet.gtid_event import GitdEvent 16 | from py_mysql_binlogserver.packet.query import Query 17 | from py_mysql_binlogserver.packet.response import Response 18 | from py_mysql_binlogserver.protocol import Flags 19 | from py_mysql_binlogserver.protocol.err import ERR 20 | from py_mysql_binlogserver.protocol.gtid import GtidSet 21 | from py_mysql_binlogserver.protocol.ok import OK 22 | from py_mysql_binlogserver.protocol.packet import getSize, getType, dump_my_packet, file2packet, \ 23 | dump 24 | from py_mysql_binlogserver.protocol.proto import scramble_native_password 25 | 26 | SocketServer = socketserver 27 | connection_counter = 0 28 | logger = logging.getLogger() 29 | 30 | 31 | class BinlogGTIDReader(object): 32 | _start_log_file = None 33 | _start_log_pos = 4 34 | _binlog_dir = os.path.dirname(__file__) + '/binlogs/' 35 | 36 | def __init__(self, connection_settings): 37 | if "binlog_dir" in connection_settings.keys() and connection_settings["binlog_dir"]: 38 | self._binlog_dir = connection_settings["binlog_dir"] 39 | if not os.path.isdir(self._binlog_dir): 40 | # TODO rise a error 41 | pass 42 | self._binlog_name = connection_settings["binlog_name"] 43 | self.eventmap = event_type_name() 44 | 45 | def _find_log_pos_in_file(self, _log_file_name, gtid_set): 46 | with open(self._binlog_dir + "/" + _log_file_name, mode="rb") as fr: 47 | _file_header = fr.read(4) 48 | if _file_header != bytes.fromhex("fe62696e"): 49 | logger.error("It is not a binlog file.") 50 | exit() 51 | 52 | ''' 53 | 4 timestamp 54 | 1 event type 55 | 4 server-id 56 | 4 event-size 57 | 4 log pos 58 | 2 flags 59 | ''' 60 | while True: 61 | event_header = fr.read(19) 62 | if len(event_header) == 0: 63 | return None, 0 64 | timestamp, event_type, server_id, event_size, log_pos, flags = struct.unpack('= int(gno): 91 | _log_file_name = _file_name 92 | break 93 | 94 | if _log_file_name is None: 95 | logger.info("%s has not found in %s" % (gtid_set, gtid_index_file)) 96 | return None, 0 97 | 98 | logger.info("Finding %s in %s" % (gtid_set, _log_file_name)) 99 | _file_name, log_pos = self._find_log_pos_in_file(_log_file_name, gtid_set) 100 | # 尝试从最后一binlog文件中找pos 101 | if _file_name: 102 | return _file_name, log_pos 103 | else: 104 | _binlog_index_file = self._binlog_dir + "/" + self._binlog_name + ".index" 105 | for line in open(_binlog_index_file, "r"): 106 | if line.strip(): 107 | last_log_file_name = line.strip() 108 | if last_log_file_name != _log_file_name: 109 | logger.info("Finding %s in last_log_file_name: %s" % (gtid_set, last_log_file_name)) 110 | return self._find_log_pos_in_file(last_log_file_name, gtid_set) 111 | 112 | def fetch_binlog_events(self): 113 | if self._start_log_file is None: 114 | return None 115 | with open("%s/%s.index" % (self._binlog_dir, self._binlog_name), "r") as fr: 116 | _start_send = False 117 | for line in fr.readlines(): 118 | if self._start_log_file in line: 119 | _start_send = True 120 | if _start_send is False: 121 | continue 122 | _log_file = line.strip() 123 | logger.info(f"Sending binlog file: {_log_file}") 124 | with open("%s/%s" % (self._binlog_dir, _log_file), "rb") as _fr: 125 | if _log_file == self._start_log_file: 126 | _fr.read(self._start_log_pos) 127 | else: 128 | _fr.read(4) 129 | while True: 130 | event_header = _fr.read(19) 131 | if len(event_header) == 0: 132 | break 133 | event_size = struct.unpack(' 1 and sys.argv[1] or os.path.dirname(__file__)+"/example.conf" 341 | config.read(conf_file) 342 | 343 | logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s") 344 | logger = logging.getLogger() 345 | logger.setLevel(config["Logging"].getint("level")) 346 | 347 | server_settings = config["Server"] 348 | 349 | server = BinlogServer(server_settings) 350 | server.settings = server_settings 351 | server.run() 352 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | py-mysql-binlogserver 2 | ===================== 3 | 4 | This package is a pure-Python MySQL BinlogServer which implemented mysql semi sync replication protocol, and it also supported a MySQL master protocol so that you can change master to the server when failover. 5 | 6 | 7 | Requirements 8 | ------------- 9 | 10 | * Python -- one of the following: 11 | 12 | - Python >= 3.5 13 | 14 | * MySQL Server -- one of the following: 15 | 16 | - MySQL >= 5.7 17 | 18 | Installation 19 | ------------ 20 | 21 | ``` 22 | cd /opt 23 | git clone https://github.com/alvinzane/py-mysql-binlogserver.git 24 | 25 | cd /usr/lib/python3.6/site-packages/ 26 | 27 | ln -s /opt/py-mysql-binlogserver/py_mysql_binlogserver/ py_mysql_binlogserver 28 | 29 | vim python3 /opt/py-mysql-binlogserver/py_mysql_binlogserver/example.conf 30 | 31 | python3 /opt/py-mysql-binlogserver/py_mysql_binlogserver/example.py 32 | ``` 33 | 34 | 35 | Documentation 36 | ------------- 37 | 38 | [中文文档](../../tree/master/README_CN.md) 39 | 40 | Example 41 | ------- 42 | example.py 43 | ```python 44 | import logging 45 | import configparser 46 | import os 47 | import sys 48 | 49 | from py_mysql_binlogserver.dumper import BinlogDumper 50 | from py_mysql_binlogserver.server import BinlogServer 51 | 52 | 53 | def main(): 54 | config = configparser.ConfigParser() 55 | conf_file = len(sys.argv) > 1 and sys.argv[1] or os.path.dirname(__file__)+"/example.conf" 56 | config.read(conf_file) 57 | 58 | logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s') 59 | logger = logging.getLogger() 60 | logger.setLevel(config["Logging"].getint("level")) 61 | 62 | logger.info("Start Binlog Dumper from %s: %s" % (config["Dumper"]['host'], config["Dumper"]['port'])) 63 | 64 | server = BinlogServer(config["Server"]) 65 | server.run() 66 | 67 | client = BinlogDumper(config["Dumper"]) 68 | client.run() 69 | 70 | if __name__ == "__main__": 71 | main() 72 | 73 | ``` 74 | 75 | Connecting BinlogServer 76 | ---------------------- 77 | ```sql 78 | 79 | $ mysql -h127.0.0.1 -P3308 -urepl -p 80 | Enter password: 81 | Welcome to the MySQL monitor. Commands end with ; or \g. 82 | Your MySQL connection id is 2 83 | Server version: 5.7.20-log (Py-MySQL-BinlogServer GPL) 84 | 85 | Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. 86 | 87 | Oracle is a registered trademark of Oracle Corporation and/or its 88 | affiliates. Other names may be trademarks of their respective 89 | owners. 90 | 91 | Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. 92 | 93 | mysql> select @@version_comment limit 1; 94 | +-----------------------------+ 95 | | version_comment | 96 | +-----------------------------+ 97 | | (Py-MySQL-BinlogServer GPL) | 98 | +-----------------------------+ 99 | 1 row in set (0.00 sec) 100 | 101 | mysql> select user(); 102 | +---------------------------------------------+ 103 | | msg | 104 | +---------------------------------------------+ 105 | | Could not exec query on this Binlog server. | 106 | +---------------------------------------------+ 107 | 1 row in set (0.00 sec) 108 | 109 | ``` 110 | 111 | How to use 112 | ---------- 113 | 114 | Server list: 115 | ============ 116 | ``` 117 | +--------------+-------------+ 118 | | Host | Role | 119 | +--------------+-------------+ 120 | |192.168.1.100 | Master | 121 | +--------------+-------------+ 122 | |192.178.1.101 | Slave | 123 | +--------------+-------------+ 124 | |192.168.1.1 | BinlogServer| 125 | +--------------+-------------+ 126 | ``` 127 | 128 | Master: 129 | ======= 130 | ``` 131 | mysql> show slave hosts; 132 | +-----------+------+------+-----------+--------------------------------------+ 133 | | Server_id | Host | Port | Master_id | Slave_UUID | 134 | +-----------+------+------+-----------+--------------------------------------+ 135 | | 3306101 | | 3306 | 3306100 | ba66414c-d10d-11e9-b4b0-0800275ae9e7 | 136 | | 3306202 | | 3306 | 3306101 | a721031c-d2c1-11e9-897c-080027adb7d7 | 137 | +-----------+------+------+-----------+--------------------------------------+ 138 | 2 rows in set (0.00 sec) 139 | ``` 140 | 141 | Slave: 142 | ===== 143 | ``` 144 | mysql> show slave status\G 145 | *************************** 1. row *************************** 146 | Master_Host: 192.168.1.100 147 | Master_User: repl 148 | Master_Port: 3306 149 | Connect_Retry: 60 150 | Master_Log_File: mysql-bin.000008 151 | Read_Master_Log_Pos: 190 152 | Relay_Log_File: relay-bin.000002 153 | Relay_Log_Pos: 355 154 | Relay_Master_Log_File: mysql-bin.000008 155 | Slave_IO_Running: Yes 156 | Slave_SQL_Running: Yes 157 | Master_Server_Id: 3306100 158 | Master_UUID: f0ea18e0-3cff-11e9-9488-0800275ae9e7 159 | Master_Info_File: /data/mysql/3306/data/master.info 160 | SQL_Delay: 0 161 | SQL_Remaining_Delay: NULL 162 | Slave_SQL_Running_State: Slave has read all relay log; waiting for more updates 163 | Master_Retry_Count: 86400 164 | Executed_Gtid_Set: f0ea18e0-3cff-11e9-9488-0800275ae9e7:1-16 165 | Auto_Position: 1 166 | 1 row in set (0.00 sec) 167 | ``` 168 | 169 | BinlogServer: 170 | ============ 171 | Running log: 172 | ``` 173 | # python3 example.py 174 | 2019-10-12 22:09:17,272 INFO Start Binlog Dumper from 192.168.1.100: 3306 175 | 2019-10-12 22:09:17,272 INFO BinlogServer running in thread: Thread-1 0.0.0.0 3307 176 | 2019-10-12 22:09:17,280 INFO Dump binlog from mysql-bin.000008 at 190 177 | ``` 178 | Binlog directory： 179 | ``` 180 | $ ll binlogs/ 181 | total 72 182 | -rw-r--r-- 1 alvin staff 2130 Oct 12 20:44 mysql-bin.000002 183 | -rw-r--r-- 1 alvin staff 472 Oct 12 20:44 mysql-bin.000003 184 | -rw-r--r-- 1 alvin staff 233 Oct 12 20:44 mysql-bin.000004 185 | -rw-r--r-- 1 alvin staff 472 Oct 12 20:44 mysql-bin.000005 186 | -rw-r--r-- 1 alvin staff 472 Oct 12 20:44 mysql-bin.000006 187 | -rw-r--r-- 1 alvin staff 950 Oct 12 20:44 mysql-bin.000007 188 | -rw-r--r-- 1 alvin staff 190 Oct 12 20:44 mysql-bin.000008 189 | -rw-r--r-- 1 alvin staff 342 Oct 12 20:44 mysql-bin.gtid.index 190 | -rw-r--r-- 1 alvin staff 119 Oct 12 20:44 mysql-bin.index 191 | ``` 192 | 193 | Testing 194 | ======= 195 | 196 | ### Step 1: 197 | Make some data at Master. 198 | ``` 199 | # Master 200 | mysql> create database db3; 201 | Query OK, 1 row affected (0.01 sec) 202 | 203 | mysql> use db3; 204 | Database changed 205 | mysql> create table t3 (id int not null auto_increment primary key,name varchar(10)); 206 | Query OK, 0 rows affected (0.02 sec) 207 | 208 | mysql> insert into t3 select null,'alvin'; 209 | Query OK, 1 row affected (0.01 sec) 210 | Records: 1 Duplicates: 0 Warnings: 0 211 | 212 | mysql> flush logs; 213 | Query OK, 0 rows affected (0.03 sec) 214 | ``` 215 | 216 | ### Step 2: 217 | Ensure Slave is running normally, then stop slave. 218 | ``` 219 | # Slave 220 | mysql> use db3; 221 | Reading table information for completion of table and column names 222 | You can turn off this feature to get a quicker startup with -A 223 | 224 | Database changed 225 | mysql> select * from t3; 226 | +----+-------+ 227 | | id | name | 228 | +----+-------+ 229 | | 1 | alvin | 230 | +----+-------+ 231 | 1 row in set (0.00 sec) 232 | 233 | mysql> show slave status\G 234 | *************************** 1. row *************************** 235 | Slave_IO_State: Waiting for master to send event 236 | Master_Host: 192.168.1.100 237 | Master_User: repl 238 | Master_Port: 3306 239 | Connect_Retry: 60 240 | Master_Log_File: mysql-bin.000009 241 | Read_Master_Log_Pos: 190 242 | Relay_Log_File: relay-bin.000004 243 | Relay_Log_Pos: 395 244 | Relay_Master_Log_File: mysql-bin.000009 245 | Slave_IO_Running: Yes 246 | Slave_SQL_Running: Yes 247 | Master_Server_Id: 3306100 248 | Master_UUID: f0ea18e0-3cff-11e9-9488-0800275ae9e7 249 | Master_Info_File: /data/mysql/3306/data/master.info 250 | SQL_Delay: 0 251 | SQL_Remaining_Delay: NULL 252 | Slave_SQL_Running_State: Slave has read all relay log; waiting for more updates 253 | Master_Retry_Count: 86400 254 | Retrieved_Gtid_Set: f0ea18e0-3cff-11e9-9488-0800275ae9e7:17-19 255 | Executed_Gtid_Set: f0ea18e0-3cff-11e9-9488-0800275ae9e7:1-19 256 | Auto_Position: 1 257 | 258 | 1 row in set (0.00 sec) 259 | 260 | mysql> stop slave; 261 | Query OK, 0 rows affected (0.00 sec) 262 | ``` 263 | 264 | ### Step 3: 265 | Make new data at Master, ensure data was replicated to the BinlogServer only. 266 | ``` 267 | # Master 268 | mysql> insert into t3 select null,'zane'; 269 | Query OK, 1 row affected (0.01 sec) 270 | Records: 1 Duplicates: 0 Warnings: 0 271 | 272 | mysql> insert into t3 select null,'test'; 273 | Query OK, 1 row affected (0.00 sec) 274 | Records: 1 Duplicates: 0 Warnings: 0 275 | 276 | mysql> flush logs; 277 | Query OK, 0 rows affected (0.01 sec) 278 | 279 | mysql> show slave hosts; 280 | +-----------+------+------+-----------+--------------------------------------+ 281 | | Server_id | Host | Port | Master_id | Slave_UUID | 282 | +-----------+------+------+-----------+--------------------------------------+ 283 | | 3306202 | | 3306 | 3306101 | a721031c-d2c1-11e9-897c-080027adb7d7 | 284 | +-----------+------+------+-----------+--------------------------------------+ 285 | 1 row in set (0.00 sec) 286 | 287 | mysql> show master status; 288 | +------------------+----------+--------------+------------------+-------------------------------------------+ 289 | | File | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set | 290 | +------------------+----------+--------------+------------------+-------------------------------------------+ 291 | | mysql-bin.000010 | 190 | | | f0ea18e0-3cff-11e9-9488-0800275ae9e7:1-21 | 292 | +------------------+----------+--------------+------------------+-------------------------------------------+ 293 | 1 row in set (0.00 sec) 294 | ``` 295 | 296 | ### Step 4: 297 | Change master to the BinlogServer, ensure that new data was replicated from the new Master. 298 | ``` 299 | # Slave 300 | mysql> CHANGE MASTER TO MASTER_HOST='192.168.1.1',MASTER_PORT=3307; 301 | Query OK, 0 rows affected (0.00 sec) 302 | 303 | mysql> start slave; 304 | Query OK, 0 rows affected (0.00 sec) 305 | 306 | mysql> show slave status\G 307 | *************************** 1. row *************************** 308 | Slave_IO_State: Waiting for master to send event 309 | Master_Host: 192.168.1.1 310 | Master_User: repl 311 | Master_Port: 3307 312 | Connect_Retry: 60 313 | Master_Log_File: mysql-bin.000010 314 | Read_Master_Log_Pos: 190 315 | Relay_Log_File: relay-bin.000003 316 | Relay_Log_Pos: 395 317 | Relay_Master_Log_File: mysql-bin.000010 318 | Slave_IO_Running: Yes 319 | Slave_SQL_Running: Yes 320 | Master_Server_Id: 3306192 321 | Master_UUID: 18f03682-ab70-11e9-aba4-32068899652e 322 | Master_Info_File: /data/mysql/3306/data/master.info 323 | SQL_Delay: 0 324 | SQL_Remaining_Delay: NULL 325 | Slave_SQL_Running_State: Slave has read all relay log; waiting for more updates 326 | Master_Retry_Count: 86400 327 | Retrieved_Gtid_Set: f0ea18e0-3cff-11e9-9488-0800275ae9e7:19-21 328 | Executed_Gtid_Set: f0ea18e0-3cff-11e9-9488-0800275ae9e7:1-21 329 | Auto_Position: 1 330 | 1 row in set (0.01 sec) 331 | 332 | mysql> select * from t3; 333 | +----+-------+ 334 | | id | name | 335 | +----+-------+ 336 | | 1 | alvin | 337 | | 2 | zane | 338 | | 3 | test | 339 | +----+-------+ 340 | 3 rows in set (0.00 sec) 341 | 342 | ``` 343 | 344 | ### Step 5: 345 | Checking BinlogServer logs. 346 | ``` 347 | 348 | 2019-10-12 22:09:17,272 INFO Start Binlog Dumper from 192.168.1.100: 3306 349 | 2019-10-12 22:09:17,272 INFO BinlogServer running in thread: Thread-1 0.0.0.0 3307 350 | 2019-10-12 22:09:17,280 INFO Dump binlog from mysql-bin.000008 at 190 351 | 2019-10-12 22:20:05,601 INFO Received Event[1570889415]: [33] GTID_LOG_EVENT 61 251 352 | 2019-10-12 22:20:05,601 INFO Received Event[1570889415]: [2] QUERY_EVENT 87 338 353 | 2019-10-12 22:21:06,817 INFO Received Event[1570889477]: [33] GTID_LOG_EVENT 61 399 354 | 2019-10-12 22:21:06,817 INFO Received Event[1570889477]: [2] QUERY_EVENT 145 544 355 | 2019-10-12 22:22:01,985 INFO Received Event[1570889532]: [33] GTID_LOG_EVENT 61 605 356 | 2019-10-12 22:22:01,985 INFO Received Event[1570889532]: [2] QUERY_EVENT 67 672 357 | 2019-10-12 22:22:01,985 INFO Received Event[1570889532]: [19] TABLE_MAP_EVENT 43 715 358 | 2019-10-12 22:22:01,985 INFO Received Event[1570889532]: [30] WRITE_ROWS_EVENT 42 757 359 | 2019-10-12 22:22:01,986 INFO Received Event[1570889532]: [16] XID_EVENT 27 784 360 | 2019-10-12 22:22:16,418 INFO Received Event[1570889546]: [4] ROTATE_EVENT 43 827 361 | 2019-10-12 22:22:16,418 INFO Rotate new binlog file: mysql-bin.000009 362 | 2019-10-12 22:22:16,419 INFO Received Event[0]: [4] ROTATE_EVENT 43 0 363 | 2019-10-12 22:22:16,419 INFO Received Event[1570889546]: [15] FORMAT_DESCRIPTION_EVENT 119 123 364 | 2019-10-12 22:22:16,419 INFO Received Event[1570889546]: [35] PREVIOUS_GTIDS_LOG_EVENT 67 190 365 | 2019-10-12 22:26:36,612 INFO Received Event[1570889807]: [33] GTID_LOG_EVENT 61 251 366 | 2019-10-12 22:26:36,612 INFO Received Event[1570889807]: [2] QUERY_EVENT 67 318 367 | 2019-10-12 22:26:36,613 INFO Received Event[1570889807]: [19] TABLE_MAP_EVENT 43 361 368 | 2019-10-12 22:26:36,613 INFO Received Event[1570889807]: [30] WRITE_ROWS_EVENT 41 402 369 | 2019-10-12 22:26:36,613 INFO Received Event[1570889807]: [16] XID_EVENT 27 429 370 | 2019-10-12 22:26:42,746 INFO Received Event[1570889813]: [33] GTID_LOG_EVENT 61 490 371 | 2019-10-12 22:26:42,747 INFO Received Event[1570889813]: [2] QUERY_EVENT 67 557 372 | 2019-10-12 22:26:42,747 INFO Received Event[1570889813]: [19] TABLE_MAP_EVENT 43 600 373 | 2019-10-12 22:26:42,747 INFO Received Event[1570889813]: [30] WRITE_ROWS_EVENT 41 641 374 | 2019-10-12 22:26:42,747 INFO Received Event[1570889813]: [16] XID_EVENT 27 668 375 | 2019-10-12 22:26:50,096 INFO Received Event[1570889820]: [4] ROTATE_EVENT 43 711 376 | 2019-10-12 22:26:50,096 INFO Rotate new binlog file: mysql-bin.000010 377 | 2019-10-12 22:26:50,097 INFO Received Event[0]: [4] ROTATE_EVENT 43 0 378 | 2019-10-12 22:26:50,097 INFO Received Event[1570889820]: [15] FORMAT_DESCRIPTION_EVENT 119 123 379 | 2019-10-12 22:26:50,097 INFO Received Event[1570889820]: [35] PREVIOUS_GTIDS_LOG_EVENT 67 190 380 | 2019-10-12 22:28:52,494 INFO login user:repl 381 | 2019-10-12 22:28:52,495 INFO query: SELECT UNIX_TIMESTAMP() 382 | 2019-10-12 22:28:52,537 INFO query: SELECT @@GLOBAL.SERVER_ID 383 | 2019-10-12 22:28:52,579 INFO query: SET @master_heartbeat_period= 30000001024 384 | 2019-10-12 22:28:52,580 INFO query: SET @master_binlog_checksum= @@global.binlog_checksum 385 | 2019-10-12 22:28:52,580 INFO query: SELECT @master_binlog_checksum 386 | 2019-10-12 22:28:52,622 INFO query: SELECT @@GLOBAL.GTID_MODE 387 | 2019-10-12 22:28:52,663 INFO query: SELECT @@GLOBAL.SERVER_UUID 388 | 2019-10-12 22:28:52,704 INFO query: SET @slave_uuid= 'ba66414c-d10d-11e9-b4b0-0800275ae9e7' 389 | 2019-10-12 22:28:52,705 INFO Received COM_REGISTER_SLAVE 390 | 2019-10-12 22:28:52,706 INFO Received COM_BINLOG_DUMP_GTID 391 | 2019-10-12 22:28:52,708 INFO Begin dump gtid binlog from f0ea18e0-3cff-11e9-9488-0800275ae9e7:1-19 392 | 2019-10-12 22:28:52,708 INFO Finding f0ea18e0-3cff-11e9-9488-0800275ae9e7:19 in mysql-bin.000008 393 | 2019-10-12 22:28:52,708 INFO Binlog pos has found: mysql-bin.000008 544 394 | 2019-10-12 22:28:52,709 INFO Sending binlog file: mysql-bin.000008 395 | 2019-10-12 22:28:52,709 INFO Sending binlog file: mysql-bin.000009 396 | 2019-10-12 22:28:52,710 INFO Sending binlog file: mysql-bin.000010 397 | ``` 398 | 399 | License 400 | ------- 401 | 402 | py-mysql-binlogserver is released under the MIT License. 403 | -------------------------------------------------------------------------------- /py_mysql_binlogserver/protocol/proto.py: -------------------------------------------------------------------------------- 1 | # from __future__ import unicode_literals 2 | # from past.builtins import basestring 3 | import hashlib 4 | import sys 5 | import struct 6 | from functools import partial 7 | 8 | 9 | PY2 = sys.version_info[0] == 2 10 | PYPY = hasattr(sys, 'pypy_translation_info') 11 | JYTHON = sys.platform.startswith('java') 12 | IRONPYTHON = sys.platform == 'cli' 13 | CPYTHON = not PYPY and not JYTHON and not IRONPYTHON 14 | 15 | if PY2: 16 | import __builtin__ 17 | range_type = xrange 18 | text_type = unicode 19 | long_type = long 20 | str_type = basestring 21 | unichr = __builtin__.unichr 22 | else: 23 | range_type = range 24 | text_type = str 25 | long_type = int 26 | str_type = str 27 | unichr = chr 28 | 29 | def byte2int(b): 30 | if isinstance(b, int): 31 | return b 32 | else: 33 | return struct.unpack("!B", b)[0] 34 | 35 | 36 | def int2byte(i): 37 | return struct.pack("!B", i) 38 | 39 | 40 | class Proto(object): 41 | __slots__ = ('packet', 'offset') 42 | 43 | def __init__(self, packet, offset=0): 44 | self.packet = packet 45 | self.offset = offset 46 | 47 | def has_remaining_data(self): 48 | return len(self.packet) - self.offset > 0 49 | 50 | @staticmethod 51 | def build_fixed_int(size, value): 52 | """ 53 | Build a MySQL Fixed Int 54 | 55 | >>> Proto.build_fixed_int(1, 0) 56 | bytearray(b'\\x00') 57 | 58 | >>> Proto.build_fixed_int(1, 255) 59 | bytearray(b'\\xff') 60 | 61 | >>> Proto.build_fixed_int(2, 0) 62 | bytearray(b'\\x00\\x00') 63 | 64 | >>> Proto.build_fixed_int(2, 0xFFFF) 65 | bytearray(b'\\xff\\xff') 66 | 67 | >>> Proto.build_fixed_int(3, 0) 68 | bytearray(b'\\x00\\x00\\x00') 69 | 70 | >>> Proto.build_fixed_int(4, 0) 71 | bytearray(b'\\x00\\x00\\x00\\x00') 72 | 73 | >>> Proto.build_fixed_int(8, 0) 74 | bytearray(b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00') 75 | 76 | >>> Proto.build_fixed_int(8, 255) 77 | bytearray(b'\\xff\\x00\\x00\\x00\\x00\\x00\\x00\\x00') 78 | """ 79 | packet = bytearray(size) 80 | if size >= 1: 81 | packet[0] = ((value >> 0) & 0xFF) 82 | if size >= 2: 83 | packet[1] = ((value >> 8) & 0xFF) 84 | if size >= 3: 85 | packet[2] = ((value >> 16) & 0xFF) 86 | if size >= 4: 87 | packet[3] = ((value >> 24) & 0xFF) 88 | if size >= 8: 89 | packet[4] = ((value >> 32) & 0xFF) 90 | packet[5] = ((value >> 40) & 0xFF) 91 | packet[6] = ((value >> 48) & 0xFF) 92 | packet[7] = ((value >> 56) & 0xFF) 93 | return packet 94 | 95 | @staticmethod 96 | def build_lenenc_int(value): 97 | """ 98 | Build a MySQL Length Encoded Int 99 | 100 | >>> Proto.build_lenenc_int(0) 101 | bytearray(b'\\x00') 102 | 103 | >>> Proto.build_lenenc_int(251) 104 | bytearray(b'\\xfc\\xfb\\x00') 105 | 106 | >>> Proto.build_lenenc_int(252) 107 | bytearray(b'\\xfc\\xfc\\x00') 108 | 109 | >>> Proto.build_lenenc_int((2**16)) 110 | bytearray(b'\\xfd\\x00\\x00\\x01') 111 | 112 | >>> Proto.build_lenenc_int((2**24)) 113 | bytearray(b'\\xfe\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x00') 114 | 115 | >>> Proto.build_lenenc_int((2**25)) 116 | bytearray(b'\\xfe\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00') 117 | 118 | 119 | """ 120 | if (value < 251): 121 | packet = bytearray(1) 122 | packet[0] = ((value >> 0) & 0xFF) 123 | elif (value < (2**16 - 1)): 124 | packet = bytearray(3) 125 | packet[0] = 0xFC 126 | packet[1] = ((value >> 0) & 0xFF) 127 | packet[2] = ((value >> 8) & 0xFF) 128 | elif (value < (2**24 - 1)): 129 | packet = bytearray(4) 130 | packet[0] = 0xFD 131 | packet[1] = ((value >> 0) & 0xFF) 132 | packet[2] = ((value >> 8) & 0xFF) 133 | packet[3] = ((value >> 16) & 0xFF) 134 | else: 135 | packet = bytearray(9) 136 | packet[0] = 0xFE 137 | packet[1] = ((value >> 0) & 0xFF) 138 | packet[2] = ((value >> 8) & 0xFF) 139 | packet[3] = ((value >> 16) & 0xFF) 140 | packet[4] = ((value >> 24) & 0xFF) 141 | packet[5] = ((value >> 32) & 0xFF) 142 | packet[6] = ((value >> 40) & 0xFF) 143 | packet[7] = ((value >> 48) & 0xFF) 144 | packet[8] = ((value >> 56) & 0xFF) 145 | return packet 146 | 147 | @staticmethod 148 | def build_lenenc_str(value): 149 | """ 150 | Build a MySQL Length Encoded String 151 | 152 | >>> Proto.build_lenenc_str('abc') 153 | bytearray(b'\\x03abc') 154 | 155 | Empty strings are supported: 156 | >>> Proto.build_lenenc_str('') 157 | bytearray(b'\\x00') 158 | 159 | Really long strings: 160 | >>> Proto.build_lenenc_str('abc123abc123abc123abc123abc123abc123abc123\ 161 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 162 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 163 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 164 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 165 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 166 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 167 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 168 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 169 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 170 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 171 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 172 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 173 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 174 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 175 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 176 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 177 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 178 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 179 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 180 | ') 181 | bytearray(b'\\xfc\\xf4\\x05abc123abc123abc123abc123abc123abc123abc123\ 182 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 183 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 184 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 185 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 186 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 187 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 188 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 189 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 190 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 191 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 192 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 193 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 194 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 195 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 196 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 197 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 198 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 199 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 200 | abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123abc123\ 201 | ') 202 | """ 203 | if value == '': 204 | return bytearray(1) 205 | 206 | size = Proto.build_lenenc_int(len(value)) 207 | fixed_str = Proto.build_fixed_str(len(value), value) 208 | return size+fixed_str 209 | 210 | @staticmethod 211 | def build_null_str(value): 212 | """ 213 | Build a MySQL Null String 214 | 215 | >>> Proto.build_null_str('ab') 216 | bytearray(b'ab\\x00') 217 | 218 | Empty string is just a null: 219 | >>> Proto.build_null_str('') 220 | bytearray(b'\\x00') 221 | """ 222 | return Proto.build_fixed_str(len(value) + 1, value) 223 | 224 | @staticmethod 225 | def build_fixed_str(size, value): 226 | """ 227 | Build a MySQL Fixed String 228 | 229 | >>> Proto.build_fixed_str(2, 'ab') 230 | bytearray(b'ab') 231 | 232 | Zero pad if size > sizeOf(value): 233 | >>> Proto.build_fixed_str(3, 'ab') 234 | bytearray(b'ab\\x00') 235 | """ 236 | packet = bytearray(size) 237 | for i, c in enumerate(value): 238 | # print(i, c, type(c)) 239 | packet[i] = type(c) == int and c or ord(c) 240 | return packet 241 | 242 | @staticmethod 243 | def build_eop_str(value): 244 | """ 245 | Build a MySQL End of Packet String 246 | 247 | >>> Proto.build_eop_str('ab') 248 | bytearray(b'ab') 249 | """ 250 | return Proto.build_fixed_str(len(value), value) 251 | 252 | @staticmethod 253 | def build_filler(size, fill=0x00): 254 | """ 255 | Build a set of filler 256 | 257 | >>> Proto.build_filler(1) 258 | bytearray(b'\\x00') 259 | 260 | >>> Proto.build_filler(2) 261 | bytearray(b'\\x00\\x00') 262 | 263 | >>> Proto.build_filler(1, 0x1c) 264 | bytearray(b'\\x1c') 265 | 266 | >>> Proto.build_filler(2, 0xff) 267 | bytearray(b'\\xff\\xff') 268 | """ 269 | packet = bytearray(size) 270 | for i in range(size): 271 | packet[i] = fill 272 | return packet 273 | 274 | @staticmethod 275 | def build_byte(value): 276 | """ 277 | Build a extendable byte 278 | 279 | >>> Proto.build_byte(0) 280 | bytearray(b'\\x00') 281 | 282 | >>> Proto.build_byte(1) 283 | bytearray(b'\\x01') 284 | 285 | >>> Proto.build_byte(0xFF) 286 | bytearray(b'\\xff') 287 | """ 288 | packet = bytearray(1) 289 | packet[0] = value 290 | return packet 291 | 292 | @staticmethod 293 | def get_fixed_int_sniplet(packet): 294 | """ 295 | Extract a fixed int from a packet subset 296 | 297 | >>> Proto.get_fixed_int_sniplet(Proto.build_fixed_int(1, 0)) 298 | 0 299 | 300 | >>> Proto.get_fixed_int_sniplet(Proto.build_fixed_int(1 ,1)) 301 | 1 302 | 303 | >>> Proto.get_fixed_int_sniplet(Proto.build_fixed_int(1, 255)) 304 | 255 305 | """ 306 | value = 0 307 | for i in range(len(packet)-1, 0, -1): 308 | value |= packet[i] & 0xFF 309 | value <<= 8 310 | value |= packet[0] & 0xFF 311 | return value 312 | 313 | def get_fixed_int(self, size): 314 | """ 315 | Extract a fixed int the current packet 316 | 317 | >>> packet = Proto(Proto.build_fixed_int(1, 0)) 318 | >>> packet.get_fixed_int(1) 319 | 0 320 | """ 321 | value = Proto.get_fixed_int_sniplet( 322 | self.packet[self.offset:self.offset+size]) 323 | self.offset += size 324 | return value 325 | 326 | def get_filler(self, size): 327 | """ 328 | Skip over packet filler 329 | 330 | >>> pckt = bytearray(5) 331 | >>> packet = Proto(pckt) 332 | >>> packet.offset 333 | 0 334 | >>> packet.get_filler(2) 335 | >>> packet.offset 336 | 2 337 | """ 338 | self.offset += size 339 | 340 | def read(self, size): 341 | 342 | value = self.packet[self.offset:self.offset+size] 343 | self.offset += size 344 | 345 | return value 346 | 347 | def get_lenenc_int(self): 348 | """ 349 | Extract a Length Encoded Int from the current packet position 350 | 351 | >>> pckt = Proto.build_lenenc_int(255) 352 | >>> packet = Proto(pckt) 353 | >>> packet.get_lenenc_int() 354 | 255 355 | """ 356 | size = 0 357 | 358 | if self.packet[self.offset] < 251: 359 | size = 1 360 | elif self.packet[self.offset] == 252: 361 | self.offset += 1 362 | size = 2 363 | elif self.packet[self.offset] == 253: 364 | self.offset += 1 365 | size = 3 366 | elif self.packet[self.offset] == 254: 367 | self.offset += 1 368 | size = 8 369 | 370 | return self.get_fixed_int(size) 371 | 372 | def get_fixed_str(self, size): 373 | """ 374 | Extract a fixed length string from the current packet position 375 | 376 | >>> target = "The brown dog did stuff" 377 | >>> pckt = Proto.build_fixed_str(len(target), target) 378 | >>> packet = Proto(pckt) 379 | >>> packet.get_fixed_str(len(pckt)) 380 | 'The brown dog did stuff' 381 | """ 382 | value = '' 383 | 384 | for i in range(self.offset, self.offset + size): 385 | value += chr(self.packet[i]) 386 | self.offset += 1 387 | 388 | return value 389 | 390 | def get_null_str(self): 391 | """ 392 | Extract a null string from the current packet position 393 | 394 | >>> target = "The brown dog did stuff" 395 | >>> pckt = Proto.build_null_str(target) 396 | >>> packet = Proto(pckt) 397 | >>> packet.get_null_str() 398 | 'The brown dog did stuff' 399 | """ 400 | value = '' 401 | 402 | for i in range(self.offset, len(self.packet)): 403 | if self.packet[i] == 0x00: 404 | self.offset += 1 405 | break 406 | value += chr(self.packet[i]) 407 | self.offset += 1 408 | 409 | return value 410 | 411 | def get_eop_str(self): 412 | """ 413 | Extract a eop string from the current packet position 414 | 415 | >>> target = "The brown dog did stuff" 416 | >>> pckt = Proto.build_eop_str(target) 417 | >>> packet = Proto(pckt) 418 | >>> packet.get_eop_str() 419 | 'The brown dog did stuff' 420 | """ 421 | value = '' 422 | 423 | for i in range(self.offset, len(self.packet)): 424 | if self.packet[i] == 0x00 and i == len(self.packet) - 1: 425 | self.offset += 1 426 | break 427 | value += chr(self.packet[i]) 428 | self.offset += 1 429 | 430 | return value 431 | 432 | def get_lenenc_str(self): 433 | """ 434 | Extract a length encoded string from the current packet position 435 | 436 | >>> target = "The brown dog did stuff" 437 | >>> pckt = Proto.build_lenenc_str(target) 438 | >>> packet = Proto(pckt) 439 | >>> packet.get_lenenc_str() 440 | 'The brown dog did stuff' 441 | """ 442 | value = '' 443 | size = self.get_lenenc_int() 444 | 445 | for i in range(self.offset, self.offset + size): 446 | value += chr(self.packet[i]) 447 | self.offset += 1 448 | 449 | return value 450 | 451 | 452 | DEBUG = False 453 | SCRAMBLE_LENGTH = 20 454 | sha1_new = partial(hashlib.new, 'sha1') 455 | 456 | 457 | # mysql_native_password 458 | # https://dev.mysql.com/doc/internals/en/secure-password-authentication.html#packet-Authentication::Native41 459 | 460 | PY2 = sys.version_info[0] == 2 461 | 462 | 463 | def scramble_native_password(password, message): 464 | """Scramble used for mysql_native_password""" 465 | if not password: 466 | return b'' 467 | 468 | password = password.encode("utf-8") 469 | message = message.encode("utf-8") 470 | 471 | stage1 = sha1_new(password).digest() 472 | stage2 = sha1_new(stage1).digest() 473 | s = sha1_new() 474 | s.update(message[:SCRAMBLE_LENGTH]) 475 | s.update(stage2) 476 | result = s.digest() 477 | return _my_crypt(result, stage1) 478 | 479 | 480 | def _my_crypt(message1, message2): 481 | result = bytearray(message1) 482 | if PY2: 483 | message2 = bytearray(message2) 484 | 485 | for i in range(len(result)): 486 | result[i] ^= message2[i] 487 | 488 | return bytes(result) 489 | 490 | 491 | if __name__ == "__main__": 492 | import doctest 493 | doctest.testmod() 494 | --------------------------------------------------------------------------------