├── pymysqlreplication ├── constants │ ├── __init__.py │ ├── BINLOG.py │ └── FIELD_TYPE.py ├── tests │ ├── __init__.py │ ├── benchmark.py │ ├── base.py │ ├── test_data_objects.py │ ├── test_data_type.py │ └── test_basic.py ├── __init__.py ├── table.py ├── bitmap.py ├── gtid.py ├── column.py ├── event.py ├── packet.py ├── binlogstream.py └── row_event.py ├── docs ├── binlogstream.rst ├── examples.rst ├── installation.rst ├── support.rst ├── events.rst ├── licence.rst ├── limitations.rst ├── changelog.rst ├── index.rst ├── developement.rst ├── Makefile └── conf.py ├── examples ├── logstash │ ├── logstash-simple.conf │ └── mysql_to_logstash.py ├── dump_events.py ├── redis_cache.py └── rethinkdb_sync.py ├── TODO ├── .gitignore ├── setup.py ├── CHANGELOG ├── .travis.yml ├── .mysql └── dev.mysql.com.gpg.key └── README.md /pymysqlreplication/constants/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .BINLOG import * 4 | from .FIELD_TYPE import * 5 | -------------------------------------------------------------------------------- /docs/binlogstream.rst: -------------------------------------------------------------------------------- 1 | ################## 2 | BinLogStreamReader 3 | ################## 4 | 5 | 6 | .. automodule:: pymysqlreplication.binlogstream 7 | :members: 8 | -------------------------------------------------------------------------------- /docs/examples.rst: -------------------------------------------------------------------------------- 1 | ######## 2 | Examples 3 | ######## 4 | 5 | You can found a list of working examples here: https://github.com/noplay/python-mysql-replication/tree/master/examples 6 | 7 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | ############# 2 | Installation 3 | ############# 4 | 5 | Python MySQL Replication is available on PyPi. 6 | You can install it with: 7 | 8 | :command:`pip install mysql-replication` 9 | -------------------------------------------------------------------------------- /docs/support.rst: -------------------------------------------------------------------------------- 1 | ####### 2 | Support 3 | ####### 4 | 5 | You can get support and discuss about new features on: 6 | https://groups.google.com/d/forum/python-mysql-replication 7 | 8 | You can browse and report issues on: 9 | https://github.com/noplay/python-mysql-replication/issues 10 | -------------------------------------------------------------------------------- /examples/logstash/logstash-simple.conf: -------------------------------------------------------------------------------- 1 | input { 2 | stdin { 3 | type => "mysql_event" 4 | format => "json_event" 5 | debug => true 6 | } 7 | } 8 | output { 9 | stdout { debug => true debug_format => "json"} 10 | elasticsearch { embedded => true } 11 | } 12 | -------------------------------------------------------------------------------- /docs/events.rst: -------------------------------------------------------------------------------- 1 | ###### 2 | Events 3 | ###### 4 | 5 | .. automodule:: pymysqlreplication.event 6 | :members: 7 | 8 | ========== 9 | Row events 10 | ========== 11 | 12 | This events are send by MySQL when data are modified. 13 | 14 | .. automodule:: pymysqlreplication.row_event 15 | :members: 16 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | * Support ALTER TABLE 2 | * End user documentation 3 | * Test transaction support 4 | * MySQL 5.6 support: http://dev.mysql.com/doc/internals/en/row-based-replication.html#rows-event 5 | * Support binlog_row_image=minimal or binlog_row_image=noblob 6 | * Raise exception if too much connection lost 7 | * Test log file change 8 | -------------------------------------------------------------------------------- /pymysqlreplication/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from pymysqlreplication.tests.test_basic import * 4 | from pymysqlreplication.tests.test_data_type import * 5 | from pymysqlreplication.tests.test_data_objects import * 6 | 7 | if __name__ == "__main__": 8 | if sys.version_info < (2, 7): 9 | import unittest2 as unittest 10 | else: 11 | import unittest 12 | unittest.main() 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | eggs 9 | parts 10 | bin 11 | var 12 | sdist 13 | develop-eggs 14 | .installed.cfg 15 | 16 | # Installer logs 17 | pip-log.txt 18 | 19 | # Unit test / coverage reports 20 | .coverage 21 | .tox 22 | 23 | # Translations 24 | *.mo 25 | 26 | #Mr Developer 27 | .mr.developer.cfg 28 | 29 | # Doc 30 | _build 31 | 32 | # Text Editor Backupfile 33 | *~ 34 | 35 | # Intellij IDE 36 | .idea 37 | *.xml 38 | *.iml 39 | 40 | # Nose 41 | .noseids 42 | 43 | # Pyenv 44 | .python-version 45 | -------------------------------------------------------------------------------- /docs/licence.rst: -------------------------------------------------------------------------------- 1 | ####### 2 | Licence 3 | ####### 4 | 5 | Copyright 2012-2014 Julien Duponchelle 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 12 | -------------------------------------------------------------------------------- /docs/limitations.rst: -------------------------------------------------------------------------------- 1 | ########### 2 | Limitations 3 | ########### 4 | 5 | GEOMETRY 6 | ========= 7 | GEOMETRY field is not decoded you will get the raw data. 8 | 9 | binlog_row_image 10 | ================ 11 | Only [binlog_row_image=full](http://dev.mysql.com/doc/refman/5.6/en/replication-options-binary-log.html#sysvar_binlog_row_image) is supported (it's the default value). 12 | 13 | BOOLEAN and BOOL 14 | ================ 15 | Boolean is returned as TINYINT(1) because it's the reality. 16 | 17 | http://dev.mysql.com/doc/refman/5.6/en/numeric-type-overview.html 18 | 19 | Our discussion about it: 20 | https://github.com/noplay/python-mysql-replication/pull/16 21 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | ########## 2 | Changelog 3 | ########## 4 | 5 | 0.3 07/07/2014 6 | =============== 7 | * use NotImplementedEvent instead of raising an Exception 8 | * Python 3 fix 9 | * Add 2006 to Mysql expected error codes 10 | 11 | 0.2 13/10/2013 12 | =============== 13 | * pymysql 0.6 support 14 | * fix smallint24 15 | * fix new decimal support 16 | * TINYINT(1) to bool mapping 17 | * change names of events to V2 from default 18 | * Fix broken "dates" - zero years.. 19 | * add support for NULL_EVENT, INTVAR_EVENT and GTID_LOG_EVENT 20 | * Skip invalid packets 21 | * Display log pos inside events dump 22 | * Handle utf8 name for queries 23 | 24 | 0.1 01/05/2013 25 | =============== 26 | First public version 27 | -------------------------------------------------------------------------------- /pymysqlreplication/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Python MySQL Replication: 3 | Pure Python Implementation of MySQL replication protocol build on top of 4 | PyMYSQL. 5 | 6 | Licence 7 | ======= 8 | Copyright 2012 Julien Duponchelle 9 | 10 | Licensed under the Apache License, Version 2.0 (the "License"); 11 | you may not use this file except in compliance with the License. 12 | You may obtain a copy of the License at 13 | 14 | http://www.apache.org/licenses/LICENSE-2.0 15 | 16 | Unless required by applicable law or agreed to in writing, software 17 | distributed under the License is distributed on an "AS IS" BASIS, 18 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | See the License for the specific language governing permissions and 20 | limitations under the License. 21 | """ 22 | 23 | from .binlogstream import BinLogStreamReader 24 | -------------------------------------------------------------------------------- /examples/dump_events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Dump all replication events from a remote mysql server 6 | # 7 | 8 | from pymysqlreplication import BinLogStreamReader 9 | 10 | MYSQL_SETTINGS = { 11 | "host": "127.0.0.1", 12 | "port": 3306, 13 | "user": "root", 14 | "passwd": "" 15 | } 16 | 17 | 18 | def main(): 19 | # server_id is your slave identifier, it should be unique. 20 | # set blocking to True if you want to block and wait for the next event at 21 | # the end of the stream 22 | stream = BinLogStreamReader(connection_settings=MYSQL_SETTINGS, 23 | server_id=3, 24 | blocking=True) 25 | 26 | for binlogevent in stream: 27 | binlogevent.dump() 28 | 29 | stream.close() 30 | 31 | 32 | if __name__ == "__main__": 33 | main() 34 | -------------------------------------------------------------------------------- /pymysqlreplication/constants/BINLOG.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | UNKNOWN_EVENT = 0x00 4 | START_EVENT_V3 = 0x01 5 | QUERY_EVENT = 0x02 6 | STOP_EVENT = 0x03 7 | ROTATE_EVENT = 0x04 8 | INTVAR_EVENT = 0x05 9 | LOAD_EVENT = 0x06 10 | SLAVE_EVENT = 0x07 11 | CREATE_FILE_EVENT = 0x08 12 | APPEND_BLOCK_EVENT = 0x09 13 | EXEC_LOAD_EVENT = 0x0a 14 | DELETE_FILE_EVENT = 0x0b 15 | NEW_LOAD_EVENT = 0x0c 16 | RAND_EVENT = 0x0d 17 | USER_VAR_EVENT = 0x0e 18 | FORMAT_DESCRIPTION_EVENT = 0x0f 19 | XID_EVENT = 0x10 20 | BEGIN_LOAD_QUERY_EVENT = 0x11 21 | EXECUTE_LOAD_QUERY_EVENT = 0x12 22 | TABLE_MAP_EVENT = 0x13 23 | PRE_GA_WRITE_ROWS_EVENT = 0x14 24 | PRE_GA_UPDATE_ROWS_EVENT = 0x15 25 | PRE_GA_DELETE_ROWS_EVENT = 0x16 26 | WRITE_ROWS_EVENT_V1 = 0x17 27 | UPDATE_ROWS_EVENT_V1 = 0x18 28 | DELETE_ROWS_EVENT_V1 = 0x19 29 | INCIDENT_EVENT = 0x1a 30 | HEARTBEAT_LOG_EVENT = 0x1b 31 | IGNORABLE_LOG_EVENT = 0x1c 32 | ROWS_QUERY_LOG_EVENT = 0x1d 33 | WRITE_ROWS_EVENT_V2 = 0x1e 34 | UPDATE_ROWS_EVENT_V2 = 0x1f 35 | DELETE_ROWS_EVENT_V2 = 0x20 36 | GTID_LOG_EVENT = 0x21 37 | ANONYMOUS_GTID_LOG_EVENT = 0x22 38 | PREVIOUS_GTIDS_LOG_EVENT = 0x23 39 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Python MySQL Replication documentation master file, created by 2 | sphinx-quickstart on Sun Sep 30 15:04:27 2012. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Python MySQL Replication's documentation! 7 | ==================================================== 8 | 9 | Pure Python Implementation of MySQL replication protocol build on top of PyMYSQL. This allow you to receive event like insert, update, delete with their datas and raw SQL queries. 10 | 11 | Use cases 12 | =========== 13 | 14 | * MySQL to NoSQL database replication 15 | * MySQL to search engine replication 16 | * Invalidate cache when something change in database 17 | * Audit 18 | * Real time analytics 19 | 20 | 21 | Contents 22 | ========= 23 | 24 | .. toctree:: 25 | :maxdepth: 2 26 | 27 | installation 28 | changelog 29 | limitations 30 | binlogstream 31 | events 32 | examples 33 | support 34 | developement 35 | licence 36 | 37 | Indices and tables 38 | ================== 39 | 40 | * :ref:`genindex` 41 | * :ref:`modindex` 42 | * :ref:`search` 43 | 44 | -------------------------------------------------------------------------------- /pymysqlreplication/table.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | class Table(object): 5 | def __init__(self, column_schemas, table_id, schema, table, columns, primary_key=None): 6 | if primary_key is None: 7 | primary_key = [c.data["name"] for c in columns if c.data["is_primary"]] 8 | if len(primary_key) == 0: 9 | primary_key = '' 10 | elif len(primary_key) == 1: 11 | primary_key, = primary_key 12 | else: 13 | primary_key = tuple(primary_key) 14 | 15 | self.__dict__.update({ 16 | "column_schemas": column_schemas, 17 | "table_id": table_id, 18 | "schema": schema, 19 | "table": table, 20 | "columns": columns, 21 | "primary_key": primary_key 22 | }) 23 | 24 | @property 25 | def data(self): 26 | return dict((k, v) for (k, v) in self.__dict__.items() if not k.startswith('_')) 27 | 28 | def __eq__(self, other): 29 | return self.data == other.data 30 | 31 | def __ne__(self, other): 32 | return not self.__eq__(other) 33 | 34 | def serializable_data(self): 35 | return self.data 36 | -------------------------------------------------------------------------------- /examples/redis_cache.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Update a redis server cache when an evenement is trigger 6 | # in MySQL replication log 7 | # 8 | 9 | import redis 10 | 11 | from pymysqlreplication import BinLogStreamReader 12 | from pymysqlreplication.row_event import ( 13 | DeleteRowsEvent, 14 | UpdateRowsEvent, 15 | WriteRowsEvent, 16 | ) 17 | 18 | MYSQL_SETTINGS = { 19 | "host": "127.0.0.1", 20 | "port": 3306, 21 | "user": "root", 22 | "passwd": "" 23 | } 24 | 25 | 26 | def main(): 27 | r = redis.Redis() 28 | 29 | stream = BinLogStreamReader( 30 | connection_settings=MYSQL_SETTINGS, 31 | only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent]) 32 | 33 | for binlogevent in stream: 34 | prefix = "%s:%s:" % (binlogevent.schema, binlogevent.table) 35 | 36 | for row in binlogevent.rows: 37 | if isinstance(binlogevent, DeleteRowsEvent): 38 | vals = row["values"] 39 | r.delete(prefix + str(vals["id"])) 40 | elif isinstance(binlogevent, UpdateRowsEvent): 41 | vals = row["after_values"] 42 | r.hmset(prefix + str(vals["id"]), vals) 43 | elif isinstance(binlogevent, WriteRowsEvent): 44 | vals = row["values"] 45 | r.hmset(prefix + str(vals["id"]), vals) 46 | 47 | stream.close() 48 | 49 | 50 | if __name__ == "__main__": 51 | main() 52 | -------------------------------------------------------------------------------- /pymysqlreplication/bitmap.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | bitCountInByte = [ 4 | 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 5 | 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 6 | 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 7 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 8 | 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 9 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 10 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 11 | 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 12 | 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 13 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 14 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 15 | 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 16 | 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 17 | 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 18 | 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 19 | 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8, 20 | ] 21 | 22 | # Calculate totol bit counts in a bitmap 23 | def BitCount(bitmap): 24 | n = 0 25 | for i in range(0, len(bitmap)): 26 | bit = bitmap[i] 27 | if type(bit) is str: 28 | bit = ord(bit) 29 | n += bitCountInByte[bit] 30 | return n 31 | 32 | # Get the bit set at offset position in bitmap 33 | def BitGet(bitmap, position): 34 | bit = bitmap[int(position / 8)] 35 | if type(bit) is str: 36 | bit = ord(bit) 37 | return bit & (1 << (position & 7)) 38 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | try: 5 | from setuptools import setup, Command 6 | except ImportError: 7 | from distutils.core import setup, Command 8 | 9 | import sys 10 | 11 | install_requires = ['pymysql'] 12 | 13 | # add unittest2 to install_requires for python < 2.7 14 | if sys.version_info < (2, 7): 15 | install_requires.append("unittest2") 16 | 17 | 18 | class TestCommand(Command): 19 | user_options = [] 20 | 21 | def initialize_options(self): 22 | pass 23 | 24 | def finalize_options(self): 25 | pass 26 | 27 | def run(self): 28 | """ 29 | Finds all the tests modules in tests/, and runs them. 30 | """ 31 | from pymysqlreplication import tests 32 | import unittest 33 | 34 | unittest.main(tests, argv=sys.argv[:1]) 35 | 36 | 37 | version = "0.7" 38 | 39 | setup( 40 | name="mysql-replication", 41 | version=version, 42 | url="https://github.com/noplay/python-mysql-replication", 43 | author="Julien Duponchelle", 44 | author_email="julien@duponchelle.info", 45 | description=("Pure Python Implementation of MySQL replication protocol " 46 | "build on top of PyMYSQL."), 47 | license="Apache 2", 48 | packages=["pymysqlreplication", 49 | "pymysqlreplication.constants", 50 | "pymysqlreplication.tests"], 51 | cmdclass={"test": TestCommand}, 52 | install_requires=install_requires, 53 | ) 54 | -------------------------------------------------------------------------------- /docs/developement.rst: -------------------------------------------------------------------------------- 1 | ############# 2 | Developement 3 | ############# 4 | 5 | Contributions 6 | ============= 7 | 8 | You can report issues and contribute to the project on: https://github.com/noplay/python-mysql-replication 9 | 10 | The standard way to contribute code to the project is to fork the Github 11 | project and open a pull request with your changes: 12 | https://github.com/noplay/python-mysql-replication 13 | 14 | Don't hesitate to open an issue with what you want to changes if 15 | you want to discuss about it before coding. 16 | 17 | 18 | Tests 19 | ====== 20 | 21 | When it's possible we have an unit test. 22 | 23 | *pymysqlreplication/tests/* contains the test suite. The test suite 24 | use the standard *unittest* Python module. 25 | 26 | **Be carefull** tests will reset the binary log of your MySQL server. 27 | 28 | Make sure you have the following configuration set in your mysql config file (usually my.cnf on development env): 29 | 30 | :: 31 | 32 | log-bin=mysql-bin 33 | server-id=1 34 | binlog-format = row #Very important if you want to receive write, update and delete row events 35 | gtid_mode=ON 36 | log-slave_updates=true 37 | enforce_gtid_consistency 38 | 39 | 40 | To run tests: 41 | 42 | :: 43 | 44 | python setup.py test 45 | 46 | 47 | Each pull request is tested on Travis CI: 48 | https://travis-ci.org/noplay/python-mysql-replication 49 | 50 | Build the documentation 51 | ======================== 52 | 53 | The documentation is available in docs folder. You can 54 | build it using Sphinx: 55 | 56 | :: 57 | 58 | cd docs 59 | pip install sphinx 60 | make html 61 | 62 | -------------------------------------------------------------------------------- /examples/logstash/mysql_to_logstash.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Output logstash events to the console from MySQL replication stream 6 | # 7 | # You can pipe it to logstash like this: 8 | # python examples/logstash/mysql_to_logstash.py | java -jar logstash-1.1.13-flatjar.jar agent -f examples/logstash/logstash-simple.conf 9 | 10 | import json 11 | import sys 12 | 13 | from pymysqlreplication import BinLogStreamReader 14 | from pymysqlreplication.row_event import ( 15 | DeleteRowsEvent, 16 | UpdateRowsEvent, 17 | WriteRowsEvent, 18 | ) 19 | 20 | MYSQL_SETTINGS = { 21 | "host": "127.0.0.1", 22 | "port": 3306, 23 | "user": "root", 24 | "passwd": "" 25 | } 26 | 27 | 28 | def main(): 29 | stream = BinLogStreamReader( 30 | connection_settings=MYSQL_SETTINGS, 31 | only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent]) 32 | 33 | for binlogevent in stream: 34 | for row in binlogevent.rows: 35 | event = {"schema": binlogevent.schema, "table": binlogevent.table} 36 | 37 | if isinstance(binlogevent, DeleteRowsEvent): 38 | event["action"] = "delete" 39 | event = dict(event.items() + row["values"].items()) 40 | elif isinstance(binlogevent, UpdateRowsEvent): 41 | event["action"] = "update" 42 | event = dict(event.items() + row["after_values"].items()) 43 | elif isinstance(binlogevent, WriteRowsEvent): 44 | event["action"] = "insert" 45 | event = dict(event.items() + row["values"].items()) 46 | print json.dumps(event) 47 | sys.stdout.flush() 48 | 49 | 50 | stream.close() 51 | 52 | 53 | if __name__ == "__main__": 54 | main() 55 | -------------------------------------------------------------------------------- /pymysqlreplication/constants/FIELD_TYPE.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Original code from PyMySQL 4 | # Copyright (c) 2010 PyMySQL contributors 5 | # 6 | #Permission is hereby granted, free of charge, to any person obtaining a copy 7 | #of this software and associated documentation files (the "Software"), to deal 8 | #in the Software without restriction, including without limitation the rights 9 | #to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | #copies of the Software, and to permit persons to whom the Software is 11 | #furnished to do so, subject to the following conditions: 12 | # 13 | #The above copyright notice and this permission notice shall be included in 14 | #all copies or substantial portions of the Software. 15 | # 16 | #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | #FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | #AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | #THE SOFTWARE. 23 | 24 | DECIMAL = 0 25 | TINY = 1 26 | SHORT = 2 27 | LONG = 3 28 | FLOAT = 4 29 | DOUBLE = 5 30 | NULL = 6 31 | TIMESTAMP = 7 32 | LONGLONG = 8 33 | INT24 = 9 34 | DATE = 10 35 | TIME = 11 36 | DATETIME = 12 37 | YEAR = 13 38 | NEWDATE = 14 39 | VARCHAR = 15 40 | BIT = 16 41 | TIMESTAMP2 = 17 42 | DATETIME2 = 18 43 | TIME2 = 19 44 | NEWDECIMAL = 246 45 | ENUM = 247 46 | SET = 248 47 | TINY_BLOB = 249 48 | MEDIUM_BLOB = 250 49 | LONG_BLOB = 251 50 | BLOB = 252 51 | VAR_STRING = 253 52 | STRING = 254 53 | GEOMETRY = 255 54 | 55 | CHAR = TINY 56 | INTERVAL = ENUM 57 | -------------------------------------------------------------------------------- /examples/rethinkdb_sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Insert a new element in a RethinkDB database 6 | # when an evenement is trigger in MySQL replication log 7 | # 8 | # Please test with MySQL employees DB available here: 9 | # https://launchpad.net/test-db/ 10 | # 11 | 12 | import rethinkdb 13 | 14 | from pymysqlreplication import BinLogStreamReader 15 | from pymysqlreplication.row_event import ( 16 | DeleteRowsEvent, 17 | UpdateRowsEvent, 18 | WriteRowsEvent, 19 | ) 20 | 21 | MYSQL_SETTINGS = { 22 | "host": "127.0.0.1", 23 | "port": 3306, 24 | "user": "root", 25 | "passwd": "" 26 | } 27 | 28 | 29 | def main(): 30 | # connect rethinkdb 31 | rethinkdb.connect("localhost", 28015, "mysql") 32 | try: 33 | rethinkdb.db_drop("mysql").run() 34 | except: 35 | pass 36 | rethinkdb.db_create("mysql").run() 37 | 38 | tables = ["dept_emp", "dept_manager", "titles", 39 | "salaries", "employees", "departments"] 40 | for table in tables: 41 | rethinkdb.db("mysql").table_create(table).run() 42 | 43 | stream = BinLogStreamReader( 44 | connection_settings=MYSQL_SETTINGS, 45 | blocking=True, 46 | only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent], 47 | ) 48 | 49 | # process Feed 50 | for binlogevent in stream: 51 | if not isinstance(binlogevent, WriteRowsEvent): 52 | continue 53 | 54 | for row in binlogevent.rows: 55 | if not binlogevent.schema == "employees": 56 | continue 57 | 58 | vals = dict((str(k), str(v)) for k, v in row["values"].iteritems()) 59 | rethinkdb.table(binlogevent.table).insert(vals).run() 60 | 61 | stream.close() 62 | 63 | 64 | if __name__ == "__main__": 65 | main() 66 | -------------------------------------------------------------------------------- /pymysqlreplication/tests/benchmark.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # This is a sample script in order to make benchmark 4 | # on library speed. 5 | # 6 | # 7 | 8 | import pymysql 9 | import time 10 | import random 11 | import os 12 | from pymysqlreplication import BinLogStreamReader 13 | from pymysqlreplication.row_event import * 14 | import cProfile 15 | 16 | 17 | def execute(con, query): 18 | c = con.cursor() 19 | c.execute(query) 20 | return c 21 | 22 | def consume_events(): 23 | stream = BinLogStreamReader(connection_settings=database, 24 | server_id=3, 25 | resume_stream=False, 26 | blocking=True, 27 | only_events = [UpdateRowsEvent], 28 | only_tables = ['test'] ) 29 | start = time.clock() 30 | i = 0.0 31 | for binlogevent in stream: 32 | i += 1.0 33 | if i % 1000 == 0: 34 | print("%d event by seconds (%d total)" % (i / (time.clock() - start), i)) 35 | stream.close() 36 | 37 | 38 | database = { 39 | "host": "localhost", 40 | "user": "root", 41 | "passwd": "", 42 | "use_unicode": True, 43 | "charset": "utf8", 44 | "db": "pymysqlreplication_test" 45 | } 46 | 47 | conn = pymysql.connect(**database) 48 | 49 | execute(conn, "DROP DATABASE IF EXISTS pymysqlreplication_test") 50 | execute(conn, "CREATE DATABASE pymysqlreplication_test") 51 | conn = pymysql.connect(**database) 52 | execute(conn, "CREATE TABLE test (i INT) ENGINE = MEMORY") 53 | execute(conn, "INSERT INTO test VALUES(1)") 54 | execute(conn, "CREATE TABLE test2 (i INT) ENGINE = MEMORY") 55 | execute(conn, "INSERT INTO test2 VALUES(1)") 56 | execute(conn, "RESET MASTER") 57 | 58 | 59 | if os.fork() != 0: 60 | print("Start insert data") 61 | while True: 62 | execute(conn, "UPDATE test SET i = i + 1;") 63 | execute(conn, "UPDATE test2 SET i = i + 1;") 64 | else: 65 | consume_events() 66 | #cProfile.run('consume_events()') 67 | 68 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | 0.1 01/05/2013 2 | * Initial Release with MySQL 5.5 and MySQL 5.6 support 3 | 4 | 0.2 13/10/2013 5 | * Make it work under pymysql 0.6 6 | * Ignore position of some events 7 | * Fix FormatDescriptionEvent has zero log_pos 8 | * Support checksum for mysql 5.6 9 | * Add feature to start stream from position 10 | * Change names of events to V2 11 | * Added NotImplementedEvent for a few events that we currently don't need 12 | * Support null events and a slight change of names 13 | * Support MySQL Broken Dates :( 14 | * Introduce data objects for Table / Column 15 | * Add support for TINYINT(1) to bool() Mapping 16 | 17 | 0.3 07/07/2014 18 | * use NotImplementedEvent instead of raising an Exception 19 | * Python 3 fix 20 | * Add 2006 to Mysql expected error codes 21 | 22 | 0.4 01/09/2014 23 | * Add primary column informations (thanks to Lx Lu) 24 | * Python 2.6 support (thanks to Darioush Jalalinasab) 25 | * Parse gtid events (thanks to Arthur Gautier) 26 | * Code cleanup (thanks to Bernardo Sulzbach) 27 | * Travis support 28 | 29 | 0.4.1 01/09/2014 30 | * Fix missing commit for GTID in 0.4 release 31 | 32 | 0.5 28/09/2014 33 | * Remove default server id 34 | * Performances improvements 35 | * Allow filter events by schema and tables 36 | 37 | 0.6 10/05/2015 38 | * Prevent invalid table-map-entries to crash the whole app 39 | * Add support for Stop Event, update tests 40 | * Fix the order of binlog events, though we don't support them yet 41 | * Simplified RowsEvent.rows to be @property instead of __getattr__ hack 42 | * add binlog row minimal and noblob image support 43 | * remove six not being used. 44 | * misc code style improvements, mainly pep8 45 | * Update event.py to be compatible with python2.6.7 46 | * explicitly break reference cycle when closing binlogstreamreader 47 | * break reference loop using weakref to prevent memory-leaking 48 | * Freeze schema. 49 | * Freeze table schema 50 | * Avoid named parameters passed to packet because it's slower 51 | * Filter table and schema event 52 | * PyPy support 53 | 54 | 0.7 21/06/2015 55 | * Partial fix for dropped columns blowing up replication when replaying binlog with past events 56 | * Skipping GTID tests on DBs not set up to support GTID 57 | * Adding support for skipping the binlog until reaching specified timestamp. 58 | * Add support for BeginLoadQueryEvent and ExecuteLoadQueryEvent 59 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.6" 4 | - "2.7" 5 | - "3.3" 6 | - "3.4" 7 | - "pypy" 8 | install: 9 | - "pip install ." 10 | - "pip install nose" 11 | cache: apt 12 | before_script: 13 | # Remove old mysql version 14 | - "sudo /etc/init.d/mysql stop || true" 15 | - "sudo apt-get remove mysql-common mysql-server-5.5 mysql-server-core-5.5 mysql-client-5.5 mysql-client-core-5.5" 16 | - "sudo apt-get autoremove" 17 | # Config 18 | - "sudo sed -i'' 's/table_cache/table_open_cache/' /etc/mysql/my.cnf" 19 | - "sudo sed -i'' 's/log_slow_queries/slow_query_log/' /etc/mysql/my.cnf" 20 | 21 | # Install new mysql version 22 | - "echo deb http://repo.mysql.com/apt/ubuntu/ precise mysql-5.6 | sudo tee /etc/apt/sources.list.d/mysql.list" 23 | - "sudo apt-key add .mysql/dev.mysql.com.gpg.key" 24 | - "sudo apt-get update" 25 | - "sudo env DEBIAN_FRONTEND=noninteractive apt-get install -o Dpkg::Options::='--force-confold' -q -y mysql-server" 26 | 27 | # Cleanup old mysql datas 28 | - "sudo rm -rf /var/ramfs/mysql/" 29 | - "sudo mkdir /var/ramfs/mysql/" 30 | - "sudo chown mysql: /var/ramfs/mysql/" 31 | 32 | # Config 33 | - "echo '[mysqld]' | sudo tee /etc/mysql/conf.d/replication.cnf" 34 | - "echo 'log-bin=mysql-bin' | sudo tee -a /etc/mysql/conf.d/replication.cnf" 35 | - "echo 'server-id=1' | sudo tee -a /etc/mysql/conf.d/replication.cnf" 36 | - "echo 'binlog-format = row' | sudo tee -a /etc/mysql/conf.d/replication.cnf" 37 | 38 | - "sudo /etc/init.d/mysql stop || true" 39 | 40 | # Install new datas 41 | - "sudo mysql_install_db --defaults-file=/etc/mysql/my.cnf --basedir=/usr --datadir=/var/ramfs/mysql --verbose" 42 | 43 | # Enable GTID 44 | - "echo '[mysqld]' | sudo tee /etc/mysql/conf.d/gtid.cnf" 45 | - "echo 'gtid_mode=ON' | sudo tee -a /etc/mysql/conf.d/gtid.cnf" 46 | - "echo 'enforce_gtid_consistency' | sudo tee -a /etc/mysql/conf.d/gtid.cnf" 47 | - "echo 'binlog_format=ROW' | sudo tee -a /etc/mysql/conf.d/gtid.cnf" 48 | - "echo 'log_slave_updates' | sudo tee -a /etc/mysql/conf.d/gtid.cnf" 49 | 50 | # Start mysql (avoid errors to have logs) 51 | - "sudo /etc/init.d/mysql start || true" 52 | - "sudo tail -1000 /var/log/syslog" 53 | 54 | - "mysql --version" 55 | - "mysql -e 'SELECT VERSION();'" 56 | - "mysql -u root -e \"GRANT ALL PRIVILEGES ON *.* TO ''@'localhost';\"" 57 | 58 | - "mysql -e 'CREATE DATABASE pymysqlreplication_test;'" 59 | script: 60 | - "nosetests" 61 | -------------------------------------------------------------------------------- /pymysqlreplication/tests/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pymysql 4 | import copy 5 | from pymysqlreplication import BinLogStreamReader 6 | import os 7 | import sys 8 | 9 | if sys.version_info < (2, 7): 10 | import unittest2 as unittest 11 | else: 12 | import unittest 13 | 14 | base = unittest.TestCase 15 | 16 | 17 | class PyMySQLReplicationTestCase(base): 18 | def ignoredEvents(self): 19 | return [] 20 | 21 | def setUp(self): 22 | self.database = { 23 | "host": "localhost", 24 | "user": "root", 25 | "passwd": "", 26 | "use_unicode": True, 27 | "charset": "utf8", 28 | "db": "pymysqlreplication_test" 29 | } 30 | if os.getenv("TRAVIS") is not None: 31 | self.database["user"] = "travis" 32 | 33 | self.conn_control = None 34 | db = copy.copy(self.database) 35 | db["db"] = None 36 | self.connect_conn_control(db) 37 | self.execute("DROP DATABASE IF EXISTS pymysqlreplication_test") 38 | self.execute("CREATE DATABASE pymysqlreplication_test") 39 | db = copy.copy(self.database) 40 | self.connect_conn_control(db) 41 | self.stream = None 42 | self.resetBinLog() 43 | self.isMySQL56AndMore() 44 | 45 | def getMySQLVersion(self): 46 | """Return the MySQL version of the server 47 | If version is 5.6.10-log the result is 5.6.10 48 | """ 49 | return self.execute("SELECT VERSION()").fetchone()[0].split('-')[0] 50 | 51 | def isMySQL56AndMore(self): 52 | version = float(self.getMySQLVersion().rsplit('.', 1)[0]) 53 | if version >= 5.6: 54 | return True 55 | return False 56 | 57 | @property 58 | def supportsGTID(self): 59 | if not self.isMySQL56AndMore(): 60 | return False 61 | return self.execute("SELECT @@global.gtid_mode ").fetchone()[0] == "ON" 62 | 63 | def connect_conn_control(self, db): 64 | if self.conn_control is not None: 65 | self.conn_control.close() 66 | self.conn_control = pymysql.connect(**db) 67 | 68 | def tearDown(self): 69 | self.conn_control.close() 70 | self.conn_control = None 71 | self.stream.close() 72 | self.stream = None 73 | 74 | def execute(self, query): 75 | c = self.conn_control.cursor() 76 | c.execute(query) 77 | return c 78 | 79 | def resetBinLog(self): 80 | self.execute("RESET MASTER") 81 | if self.stream is not None: 82 | self.stream.close() 83 | self.stream = BinLogStreamReader(self.database, server_id=1024, 84 | ignored_events=self.ignoredEvents()) 85 | -------------------------------------------------------------------------------- /pymysqlreplication/tests/test_data_objects.py: -------------------------------------------------------------------------------- 1 | import sys 2 | if sys.version_info < (2, 7): 3 | import unittest2 as unittest 4 | else: 5 | import unittest 6 | 7 | from pymysqlreplication.column import Column 8 | from pymysqlreplication.table import Table 9 | from pymysqlreplication.event import GtidEvent 10 | 11 | from pymysqlreplication.tests import base 12 | 13 | __all__ = ["TestDataObjects"] 14 | 15 | 16 | class TestDataObjects(base.PyMySQLReplicationTestCase): 17 | def ignoredEvents(self): 18 | return [GtidEvent] 19 | 20 | def test_column_is_primary(self): 21 | col = Column(1, 22 | {"COLUMN_NAME": "test", 23 | "COLLATION_NAME": "utf8_general_ci", 24 | "CHARACTER_SET_NAME": "UTF8", 25 | "COLUMN_COMMENT": "", 26 | "COLUMN_TYPE": "tinyint(2)", 27 | "COLUMN_KEY": "PRI"}, 28 | None) 29 | self.assertEqual(True, col.is_primary) 30 | 31 | def test_column_not_primary(self): 32 | col = Column(1, 33 | {"COLUMN_NAME": "test", 34 | "COLLATION_NAME": "utf8_general_ci", 35 | "CHARACTER_SET_NAME": "UTF8", 36 | "COLUMN_COMMENT": "", 37 | "COLUMN_TYPE": "tinyint(2)", 38 | "COLUMN_KEY": ""}, 39 | None) 40 | self.assertEqual(False, col.is_primary) 41 | 42 | def test_column_serializable(self): 43 | col = Column(1, 44 | {"COLUMN_NAME": "test", 45 | "COLLATION_NAME": "utf8_general_ci", 46 | "CHARACTER_SET_NAME": "UTF8", 47 | "COLUMN_COMMENT": "", 48 | "COLUMN_TYPE": "tinyint(2)", 49 | "COLUMN_KEY": "PRI"}, 50 | None) 51 | 52 | serialized = col.serializable_data() 53 | self.assertIn("type", serialized) 54 | self.assertIn("name", serialized) 55 | self.assertIn("collation_name", serialized) 56 | self.assertIn("character_set_name", serialized) 57 | self.assertIn("comment", serialized) 58 | self.assertIn("unsigned", serialized) 59 | self.assertIn("type_is_bool", serialized) 60 | self.assertIn("is_primary", serialized) 61 | 62 | self.assertEqual(col, Column(**serialized)) 63 | 64 | def test_table(self): 65 | tbl = Table(1, "test_schema", "test_table", [], []) 66 | 67 | serialized = tbl.serializable_data() 68 | self.assertIn("table_id", serialized) 69 | self.assertIn("schema", serialized) 70 | self.assertIn("table", serialized) 71 | self.assertIn("columns", serialized) 72 | self.assertIn("column_schemas", serialized) 73 | 74 | self.assertEqual(tbl, Table(**serialized)) 75 | 76 | 77 | if __name__ == "__main__": 78 | unittest.main() 79 | -------------------------------------------------------------------------------- /pymysqlreplication/gtid.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import re 4 | import struct 5 | import binascii 6 | 7 | 8 | class Gtid(object): 9 | @staticmethod 10 | def parse_interval(interval): 11 | m = re.search('^([0-9]+)(?:-([0-9]+))?$', interval) 12 | if not m: 13 | raise ValueError('GTID format is incorrect: %r' % (interval, )) 14 | if not m.group(2): 15 | return (int(m.group(1))) 16 | else: 17 | return (int(m.group(1)), int(m.group(2))) 18 | 19 | 20 | @staticmethod 21 | def parse(gtid): 22 | m = re.search('^([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})((?::[0-9-]+)+)$', gtid) 23 | if not m: 24 | raise ValueError('GTID format is incorrect: %r' % (gtid, )) 25 | 26 | sid = m.group(1) 27 | intervals = m.group(2) 28 | 29 | intervals_parsed = [Gtid.parse_interval(x) for x in intervals.split(':')[1:]] 30 | 31 | return (sid, intervals_parsed) 32 | 33 | def __init__(self, gtid): 34 | self.sid = None 35 | self.intervals = [] 36 | 37 | self.sid, self.intervals = Gtid.parse(gtid) 38 | 39 | def __str__(self): 40 | return '%s:%s' % (self.sid, 41 | ':'.join(('%d-%s' % x) if isinstance(x, tuple) 42 | else str(x) 43 | for x in self.intervals)) 44 | 45 | def __repr__(self): 46 | return '' % self 47 | 48 | @property 49 | def encoded_length(self): 50 | return (16 + # sid 51 | 8 + # n_intervals 52 | 2 * # stop/start 53 | 8 * # stop/start mark encoded as int64 54 | len(self.intervals)) 55 | 56 | def encode(self): 57 | buffer = b'' 58 | # sid 59 | buffer += binascii.unhexlify(self.sid.replace('-', '')) 60 | # n_intervals 61 | buffer += struct.pack('> 8 62 | if real_type == FIELD_TYPE.SET or real_type == FIELD_TYPE.ENUM: 63 | self.type = real_type 64 | self.size = metadata & 0x00ff 65 | self.__read_enum_metadata(column_schema) 66 | else: 67 | self.max_length = (((metadata >> 4) & 0x300) ^ 0x300) \ 68 | + (metadata & 0x00ff) 69 | 70 | def __read_enum_metadata(self, column_schema): 71 | enums = column_schema["COLUMN_TYPE"] 72 | if self.type == FIELD_TYPE.ENUM: 73 | self.enum_values = enums.replace('enum(', '')\ 74 | .replace(')', '').replace('\'', '').split(',') 75 | else: 76 | self.set_values = enums.replace('set(', '')\ 77 | .replace(')', '').replace('\'', '').split(',') 78 | 79 | def __eq__(self, other): 80 | return self.data == other.data 81 | 82 | def __ne__(self, other): 83 | return not self.__eq__(other) 84 | 85 | def serializable_data(self): 86 | return self.data 87 | 88 | @property 89 | def data(self): 90 | return dict((k, v) for (k, v) in self.__dict__.items() if not k.startswith('_')) 91 | -------------------------------------------------------------------------------- /.mysql/dev.mysql.com.gpg.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PGP PUBLIC KEY BLOCK----- 2 | Version: GnuPG v1 3 | 4 | mQGiBD4+owwRBAC14GIfUfCyEDSIePvEW3SAFUdJBtoQHH/nJKZyQT7h9bPlUWC3 5 | RODjQReyCITRrdwyrKUGku2FmeVGwn2u2WmDMNABLnpprWPkBdCk96+OmSLN9brZ 6 | fw2vOUgCmYv2hW0hyDHuvYlQA/BThQoADgj8AW6/0Lo7V1W9/8VuHP0gQwCgvzV3 7 | BqOxRznNCRCRxAuAuVztHRcEAJooQK1+iSiunZMYD1WufeXfshc57S/+yeJkegNW 8 | hxwR9pRWVArNYJdDRT+rf2RUe3vpquKNQU/hnEIUHJRQqYHo8gTxvxXNQc7fJYLV 9 | K2HtkrPbP72vwsEKMYhhr0eKCbtLGfls9krjJ6sBgACyP/Vb7hiPwxh6rDZ7ITnE 10 | kYpXBACmWpP8NJTkamEnPCia2ZoOHODANwpUkP43I7jsDmgtobZX9qnrAXw+uNDI 11 | QJEXM6FSbi0LLtZciNlYsafwAPEOMDKpMqAK6IyisNtPvaLd8lH0bPAnWqcyefep 12 | rv0sxxqUEMcM3o7wwgfN83POkDasDbs3pjwPhxvhz6//62zQJ7Q2TXlTUUwgUmVs 13 | ZWFzZSBFbmdpbmVlcmluZyA8bXlzcWwtYnVpbGRAb3NzLm9yYWNsZS5jb20+iGkE 14 | ExECACkCGyMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAIZAQUCUwHUZgUJGmbLywAK 15 | CRCMcY07UHLh9V+DAKCjS1gGwgVI/eut+5L+l2v3ybl+ZgCcD7ZoA341HtoroV3U 16 | 6xRD09fUgeq0O015U1FMIFBhY2thZ2Ugc2lnbmluZyBrZXkgKHd3dy5teXNxbC5j 17 | b20pIDxidWlsZEBteXNxbC5jb20+iG8EMBECAC8FAk53Pa0oHSBidWlsZEBteXNx 18 | bC5jb20gd2lsbCBzdG9wIHdvcmtpbmcgc29vbgAKCRCMcY07UHLh9bU9AJ9xDK0o 19 | xJFL9vTl9OSZC4lX0K9AzwCcCrS9cnJyz79eaRjL0s2r/CcljdyIZQQTEQIAHQUC 20 | R6yUtAUJDTBYqAULBwoDBAMVAwIDFgIBAheAABIJEIxxjTtQcuH1B2VHUEcAAQGu 21 | kgCffz4GUEjzXkOi71VcwgCxASTgbe0An34LPr1j9fCbrXWXO14msIADfb5piEwE 22 | ExECAAwFAj4+o9EFgwlmALsACgkQSVDhKrJykfIk4QCfWbEeKN+3TRspe+5xKj+k 23 | QJSammIAnjUz0xFWPlVx0f8o38qNG1bq0cU9iEwEExECAAwFAj5CggMFgwliIokA 24 | CgkQtvXNTca6JD+WkQCgiGmnoGjMojynp5ppvMXkyUkfnykAoK79E6h8rwkSDZou 25 | iz7nMRisH8uyiEYEEBECAAYFAj+s468ACgkQr8UjSHiDdA/2lgCg21IhIMMABTYd 26 | p/IBiUsP/JQLiEoAnRzMywEtujQz/E9ono7H1DkebDa4iEYEEBECAAYFAj+0Q3cA 27 | CgkQhZavqzBzTmbGwwCdFqD1frViC7WRt8GKoOS7hzNN32kAnirlbwpnT7a6NOsQ 28 | 83nk11a2dePhiEYEEBECAAYFAkNbs+oACgkQi9gubzC5S1x/dACdELKoXQKkwJN0 29 | gZztsM7kjsIgyFMAnRRMbHQ7V39XC90OIpaPjk3a01tgiEYEExECAAYFAkTxMyYA 30 | CgkQ9knE9GCTUwwKcQCgibak/SwhxWH1ijRhgYCo5GtM4vcAnAhtzL57wcw1Kg1X 31 | m7nVGetUqJ7fiEwEEBECAAwFAkGBywEFgwYi2YsACgkQGFnQH2d7oexCjQCcD8sJ 32 | NDc/mS8m8OGDUOx9VMWcnGkAnj1YWOD+Qhxo3mI/Ul9oEAhNkjcfiEwEEBECAAwF 33 | AkGByzQFgwYi2VgACgkQgcL36+ITtpIiIwCdFVNVUB8xe8mFXoPm4d9Z54PTjpMA 34 | niSPA/ZsfJ3oOMLKar4F0QPPrdrGiEwEEBECAAwFAkGBy2IFgwYi2SoACgkQa3Ds 35 | 2V3D9HMJqgCbBYzr5GPXOXgP88jKzmdbjweqXeEAnRss4G2G/3qD7uhTL1SPT1SH 36 | jWUXiEwEEBECAAwFAkHQkyQFgwXUEWgACgkQfSXKCsEpp8JiVQCghvWvkPqowsw8 37 | w7WSseTcw1tflvkAni+vLHl/DqIly0LkZYn5jzK1dpvfiEwEEBECAAwFAkIrW7oF 38 | gwV5SNIACgkQ5hukiRXruavzEwCgkzL5QkLSypcw9LGHcFSx1ya0VL4An35nXkum 39 | g6cCJ1NP8r2I4NcZWIrqiEwEEhECAAwFAkAqWToFgwd6S1IACgkQPKEfNJT6+GEm 40 | XACcD+A53A5OGM7w750W11ukq4iZ9ckAnRMvndAqn3YTOxxlLPj2UPZiSgSqiEwE 41 | EhECAAwFAkA9+roFgwdmqdIACgkQ8tdcY+OcZZyy3wCgtDcwlaq20w0cNuXFLLNe 42 | EUaFFTwAni6RHN80moSVAdDTRkzZacJU3M5QiEwEEhECAAwFAkEOCoQFgwaWmggA 43 | CgkQOcor9D1qil/83QCeITZ9wIo7XAMjC6y4ZWUL4m+edZsAoMOhRIRi42fmrNFu 44 | vNZbnMGej81viEwEEhECAAwFAkKApTQFgwUj/1gACgkQBA3AhXyDn6jjJACcD1A4 45 | UtXk84J13JQyoH9+dy24714Aniwlsso/9ndICJOkqs2j5dlHFq6oiEwEExECAAwF 46 | Aj5NTYQFgwlXVwgACgkQLbt2v63UyTMFDACglT5G5NVKf5Mj65bFSlPzb92zk2QA 47 | n1uc2h19/IwwrsbIyK/9POJ+JMP7iEwEExECAAwFAkHXgHYFgwXNJBYACgkQZu/b 48 | yM2C/T4/vACfXe67xiSHB80wkmFZ2krb+oz/gBAAnjR2ucpbaonkQQgnC3GnBqmC 49 | vNaJiEwEExECAAwFAkIYgQ4FgwWMI34ACgkQdsEDHKIxbqGg7gCfQi2HcrHn+yLF 50 | uNlH1oSOh48ZM0oAn3hKV0uIRJphonHaUYiUP1ttWgdBiGUEExECAB0FCwcKAwQD 51 | FQMCAxYCAQIXgAUCS3AvygUJEPPzpwASB2VHUEcAAQEJEIxxjTtQcuH1sNsAniYp 52 | YBGqy/HhMnw3WE8kXahOOR5KAJ4xUmWPGYP4l3hKxyNK9OAUbpDVYIh7BDARAgA7 53 | BQJCdzX1NB0AT29wcy4uLiBzaG91bGQgaGF2ZSBiZWVuIGxvY2FsISBJJ20gKnNv 54 | KiBzdHVwaWQuLi4ACgkQOcor9D1qil/vRwCdFo08f66oKLiuEAqzlf9iDlPozEEA 55 | n2EgvCYLCCHjfGosrkrU3WK5NFVgiI8EMBECAE8FAkVvAL9IHQBTaG91bGQgaGF2 56 | ZSBiZWVuIGEgbG9jYWwgc2lnbmF0dXJlLCBvciBzb21ldGhpbmcgLSBXVEYgd2Fz 57 | IEkgdGhpbmtpbmc/AAoJEDnKK/Q9aopfoPsAn3BVqKOalJeF0xPSvLR90PsRlnmG 58 | AJ44oisY7Tl3NJbPgZal8W32fbqgbIkCIgQQAQIADAUCQYHLhQWDBiLZBwAKCRCq 59 | 4+bOZqFEaKgvEACCErnaHGyUYa0wETjj6DLEXsqeOiXad4i9aBQxnD35GUgcFofC 60 | /nCY4XcnCMMEnmdQ9ofUuU3OBJ6BNJIbEusAabgLooebP/3KEaiCIiyhHYU5jarp 61 | ZAh+Zopgs3Oc11mQ1tIaS69iJxrGTLodkAsAJAeEUwTPq9fHFFzC1eGBysoyFWg4 62 | bIjz/zClI+qyTbFA5g6tRoiXTo8ko7QhY2AA5UGEg+83Hdb6akC04Z2QRErxKAqr 63 | phHzj8XpjVOsQAdAi/qVKQeNKROlJ+iq6+YesmcWGfzeb87dGNweVFDJIGA0qY27 64 | pTb2lExYjsRFN4Cb13NfodAbMTOxcAWZ7jAPCxAPlHUG++mHMrhQXEToZnBFE4nb 65 | nC7vOBNgWdjUgXcpkUCkop4b17BFpR+k8ZtYLSS8p2LLz4uAeCcSm2/msJxT7rC/ 66 | FvoH8428oHincqs2ICo9zO/Ud4HmmO0O+SsZdVKIIjinGyOVWb4OOzkAlnnhEZ3o 67 | 6hAHcREIsBgPwEYVTj/9ZdC0AO44Nj9cU7awaqgtrnwwfr/o4V2gl8bLSkltZU27 68 | /29HeuOeFGjlFe0YrDd/aRNsxbyb2O28H4sG1CVZmC5uK1iQBDiSyA7Q0bbdofCW 69 | oQzm5twlpKWnY8Oe0ub9XP5p/sVfck4FceWFHwv+/PC9RzSl33lQ6vM2wIkCIgQT 70 | AQIADAUCQp8KHAWDBQWacAAKCRDYwgoJWiRXzyE+D/9uc7z6fIsalfOYoLN60ajA 71 | bQbI/uRKBFugyZ5RoaItusn9Z2rAtn61WrFhu4uCSJtFN1ny2RERg40f56pTghKr 72 | D+YEt+Nze6+FKQ5AbGIdFsR/2bUk+ZZRSt83e14Lcb6ii/fJfzkoIox9ltkifQxq 73 | Y7Tvk4noKu4oLSc8O1Wsfc/y0B9sYUUCmUfcnq58DEmGie9ovUslmyt5NPnveXxp 74 | 5UeaRc5Rqt9tK2B4A+7/cqENrdZJbAMSunt2+2fkYiRunAFPKPBdJBsY1sxeL/A9 75 | aKe0viKEXQdAWqdNZKNCi8rd/oOP99/9lMbFudAbX6nL2DSb1OG2Z7NWEqgIAzjm 76 | pwYYPCKeVz5Q8R+if9/fe5+STY/55OaI33fJ2H3v+U435VjYqbrerWe36xJItcJe 77 | qUzW71fQtXi1CTEl3w2ch7VF5oj/QyjabLnAlHgSlkSi6p7By5C2MnbCHlCfPnIi 78 | nPhFoRcRGPjJe9nFwGs+QblvS/Chzc2WX3s/2SWm4gEUKRX4zsAJ5ocyfa/vkxCk 79 | SxK/erWlCPf/J1T70+i5waXDN/E3enSet/WL7h94pQKpjz8OdGL4JSBHuAVGA+a+ 80 | dknqnPF0KMKLhjrgV+L7O84FhbmAP7PXm3xmiMPriXf+el5fZZequQoIagf8rdRH 81 | HhRJxQgI0HNknkaOqs8dtrkCDQQ+PqMdEAgA7+GJfxbMdY4wslPnjH9rF4N2qfWs 82 | EN/lxaZoJYc3a6M02WCnHl6ahT2/tBK2w1QI4YFteR47gCvtgb6O1JHffOo2HfLm 83 | RDRiRjd1DTCHqeyX7CHhcghj/dNRlW2Z0l5QFEcmV9U0Vhp3aFfWC4Ujfs3LU+hk 84 | AWzE7zaD5cH9J7yv/6xuZVw411x0h4UqsTcWMu0iM1BzELqX1DY7LwoPEb/O9Rkb 85 | f4fmLe11EzIaCa4PqARXQZc4dhSinMt6K3X4BrRsKTfozBu74F47D8Ilbf5vSYHb 86 | uE5p/1oIDznkg/p8kW+3FxuWrycciqFTcNz215yyX39LXFnlLzKUb/F5GwADBQf+ 87 | Lwqqa8CGrRfsOAJxim63CHfty5mUc5rUSnTslGYEIOCR1BeQauyPZbPDsDD9MZ1Z 88 | aSafanFvwFG6Llx9xkU7tzq+vKLoWkm4u5xf3vn55VjnSd1aQ9eQnUcXiL4cnBGo 89 | TbOWI39EcyzgslzBdC++MPjcQTcA7p6JUVsP6oAB3FQWg54tuUo0Ec8bsM8b3Ev4 90 | 2LmuQT5NdKHGwHsXTPtl0klk4bQk4OajHsiy1BMahpT27jWjJlMiJc+IWJ0mghkK 91 | Ht926s/ymfdf5HkdQ1cyvsz5tryVI3Fx78XeSYfQvuuwqp2H139pXGEkg0n6KdUO 92 | etdZWhe70YGNPw1yjWJT1IhUBBgRAgAMBQJOdz3tBQkT+wG4ABIHZUdQRwABAQkQ 93 | jHGNO1By4fUUmwCbBYr2+bBEn/L2BOcnw9Z/QFWuhRMAoKVgCFm5fadQ3Afi+UQl 94 | AcOphrnJ 95 | =443I 96 | -----END PGP PUBLIC KEY BLOCK----- 97 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PythonMySQLReplication.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PythonMySQLReplication.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/PythonMySQLReplication" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PythonMySQLReplication" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /pymysqlreplication/event.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import struct 4 | import datetime 5 | 6 | from pymysql.util import byte2int, int2byte 7 | 8 | 9 | class BinLogEvent(object): 10 | def __init__(self, from_packet, event_size, table_map, ctl_connection, 11 | only_tables = None, 12 | only_schemas = None, 13 | freeze_schema = False): 14 | self.packet = from_packet 15 | self.table_map = table_map 16 | self.event_type = self.packet.event_type 17 | self.timestamp = self.packet.timestamp 18 | self.event_size = event_size 19 | self._ctl_connection = ctl_connection 20 | # The event have been fully processed, if processed is false 21 | # the event will be skipped 22 | self._processed = True 23 | self.complete = True 24 | 25 | def _read_table_id(self): 26 | # Table ID is 6 byte 27 | # pad little-endian number 28 | table_id = self.packet.read(6) + int2byte(0) + int2byte(0) 29 | return struct.unpack('' % self.gtid 73 | 74 | 75 | class RotateEvent(BinLogEvent): 76 | """Change MySQL bin log file 77 | 78 | Attributes: 79 | position: Position inside next binlog 80 | next_binlog: Name of next binlog file 81 | """ 82 | def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): 83 | super(RotateEvent, self).__init__(from_packet, event_size, table_map, 84 | ctl_connection, **kwargs) 85 | self.position = struct.unpack(' v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ['_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_domain_indices = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 153 | #html_show_sphinx = True 154 | 155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 156 | #html_show_copyright = True 157 | 158 | # If true, an OpenSearch description file will be output, and all pages will 159 | # contain a tag referring to it. The value of this option must be the 160 | # base URL from which the finished HTML is served. 161 | #html_use_opensearch = '' 162 | 163 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 164 | #html_file_suffix = None 165 | 166 | # Output file base name for HTML help builder. 167 | htmlhelp_basename = 'PythonMySQLReplicationdoc' 168 | 169 | 170 | # -- Options for LaTeX output -------------------------------------------------- 171 | 172 | latex_elements = { 173 | # The paper size ('letterpaper' or 'a4paper'). 174 | #'papersize': 'letterpaper', 175 | 176 | # The font size ('10pt', '11pt' or '12pt'). 177 | #'pointsize': '10pt', 178 | 179 | # Additional stuff for the LaTeX preamble. 180 | #'preamble': '', 181 | } 182 | 183 | # Grouping the document tree into LaTeX files. List of tuples 184 | # (source start file, target name, title, author, documentclass [howto/manual]). 185 | latex_documents = [ 186 | ('index', 'PythonMySQLReplication.tex', u'Python MySQL Replication Documentation', 187 | u'Julien Duponchelle', 'manual'), 188 | ] 189 | 190 | # The name of an image file (relative to this directory) to place at the top of 191 | # the title page. 192 | #latex_logo = None 193 | 194 | # For "manual" documents, if this is true, then toplevel headings are parts, 195 | # not chapters. 196 | #latex_use_parts = False 197 | 198 | # If true, show page references after internal links. 199 | #latex_show_pagerefs = False 200 | 201 | # If true, show URL addresses after external links. 202 | #latex_show_urls = False 203 | 204 | # Documents to append as an appendix to all manuals. 205 | #latex_appendices = [] 206 | 207 | # If false, no module index is generated. 208 | #latex_domain_indices = True 209 | 210 | 211 | # -- Options for manual page output -------------------------------------------- 212 | 213 | # One entry per manual page. List of tuples 214 | # (source start file, name, description, authors, manual section). 215 | man_pages = [ 216 | ('index', 'pythonmysqlreplication', u'Python MySQL Replication Documentation', 217 | [u'Julien Duponchelle'], 1) 218 | ] 219 | 220 | # If true, show URL addresses after external links. 221 | #man_show_urls = False 222 | 223 | 224 | # -- Options for Texinfo output ------------------------------------------------ 225 | 226 | # Grouping the document tree into Texinfo files. List of tuples 227 | # (source start file, target name, title, author, 228 | # dir menu entry, description, category) 229 | texinfo_documents = [ 230 | ('index', 'PythonMySQLReplication', u'Python MySQL Replication Documentation', 231 | u'Julien Duponchelle', 'PythonMySQLReplication', 'One line description of project.', 232 | 'Miscellaneous'), 233 | ] 234 | 235 | # Documents to append as an appendix to all manuals. 236 | #texinfo_appendices = [] 237 | 238 | # If false, no module index is generated. 239 | #texinfo_domain_indices = True 240 | 241 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 242 | #texinfo_show_urls = 'footnote' 243 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | python-mysql-replication 2 | ======================== 3 | 4 |   5 | 6 | 7 | Pure Python Implementation of MySQL replication protocol build on top of PyMYSQL. This allow you to receive event like insert, update, delete with their datas and raw SQL queries. 8 | 9 | Use cases 10 | =========== 11 | 12 | * MySQL to NoSQL database replication 13 | * MySQL to search engine replication 14 | * Invalidate cache when something change in database 15 | * Audit 16 | * Real time analytics 17 | 18 | Documentation 19 | ============== 20 | 21 | A work in progress documentation is available here: https://python-mysql-replication.readthedocs.org/en/latest/ 22 | 23 | Instruction about building documentation is available here: 24 | https://python-mysql-replication.readthedocs.org/en/latest/developement.html 25 | 26 | 27 | Installation 28 | ============= 29 | 30 | ``` 31 | pip install mysql-replication 32 | ``` 33 | 34 | Mailing List 35 | ============== 36 | 37 | You can get support and discuss about new features on: 38 | https://groups.google.com/d/forum/python-mysql-replication 39 | 40 | 41 | 42 | Project status 43 | ================ 44 | 45 | The current project is a proof of concept of what you can do with the MySQL 46 | replication log. 47 | 48 | The project is test with: 49 | * MySQL 5.5 and 5.6 50 | * Python >= 2.6 51 | * Python 3.3 and 3.4 (3.2 is not supported) 52 | * PyPy (really faster than the standard Python interpreter) 53 | 54 | It's not tested in real production situation. 55 | 56 | Limitations 57 | ============= 58 | 59 | https://python-mysql-replication.readthedocs.org/en/latest/limitations.html 60 | 61 | Projects using this library 62 | =========================== 63 | 64 | * MySQL River Plugin for ElasticSearch: https://github.com/scharron/elasticsearch-river-mysql 65 | * Ditto: MySQL to MemSQL replicator https://github.com/memsql/ditto 66 | * ElasticMage: Full Magento integration with ElasticSearch https://github.com/ElasticMage/elasticmage 67 | * Cache buster: an automatic cache invalidation system https://github.com/rackerlabs/cache-busters 68 | * Zabbix collector for OpenTSDB https://github.com/OpenTSDB/tcollector/blob/master/collectors/0/zabbix_bridge.py 69 | * Meepo: Event sourcing and event broadcasting for datebases. https://github.com/eleme/meepo 70 | 71 | MySQL server settings 72 | ========================= 73 | 74 | In your MySQL server configuration file you need to enable replication: 75 | 76 | [mysqld] 77 | server-id = 1 78 | log_bin = /var/log/mysql/mysql-bin.log 79 | expire_logs_days = 10 80 | max_binlog_size = 100M 81 | binlog-format = row #Very important if you want to receive write, update and delete row events 82 | 83 | Examples 84 | ========= 85 | 86 | All examples are available in the [examples directory](https://github.com/noplay/python-mysql-replication/tree/master/examples) 87 | 88 | 89 | This example will dump all replication events to the console: 90 | 91 | ```python 92 | from pymysqlreplication import BinLogStreamReader 93 | 94 | mysql_settings = {'host': '127.0.0.1', 'port': 3306, 'user': 'root', 'passwd': ''} 95 | 96 | stream = BinLogStreamReader(connection_settings = mysql_settings) 97 | 98 | for binlogevent in stream: 99 | binlogevent.dump() 100 | 101 | stream.close() 102 | ``` 103 | 104 | For this SQL sessions: 105 | 106 | ```sql 107 | CREATE DATABASE test; 108 | use test; 109 | CREATE TABLE test4 (id int NOT NULL AUTO_INCREMENT, data VARCHAR(255), data2 VARCHAR(255), PRIMARY KEY(id)); 110 | INSERT INTO test4 (data,data2) VALUES ("Hello", "World"); 111 | UPDATE test4 SET data = "World", data2="Hello" WHERE id = 1; 112 | DELETE FROM test4 WHERE id = 1; 113 | ``` 114 | 115 | Output will be: 116 | 117 | === RotateEvent === 118 | Date: 1970-01-01T01:00:00 119 | Event size: 24 120 | Read bytes: 0 121 | 122 | === FormatDescriptionEvent === 123 | Date: 2012-10-07T15:03:06 124 | Event size: 84 125 | Read bytes: 0 126 | 127 | === QueryEvent === 128 | Date: 2012-10-07T15:03:16 129 | Event size: 64 130 | Read bytes: 64 131 | Schema: test 132 | Execution time: 0 133 | Query: CREATE DATABASE test 134 | 135 | === QueryEvent === 136 | Date: 2012-10-07T15:03:16 137 | Event size: 151 138 | Read bytes: 151 139 | Schema: test 140 | Execution time: 0 141 | Query: CREATE TABLE test4 (id int NOT NULL AUTO_INCREMENT, data VARCHAR(255), data2 VARCHAR(255), PRIMARY KEY(id)) 142 | 143 | === QueryEvent === 144 | Date: 2012-10-07T15:03:16 145 | Event size: 49 146 | Read bytes: 49 147 | Schema: test 148 | Execution time: 0 149 | Query: BEGIN 150 | 151 | === TableMapEvent === 152 | Date: 2012-10-07T15:03:16 153 | Event size: 31 154 | Read bytes: 30 155 | Table id: 781 156 | Schema: test 157 | Table: test4 158 | Columns: 3 159 | 160 | === WriteRowsEvent === 161 | Date: 2012-10-07T15:03:16 162 | Event size: 27 163 | Read bytes: 10 164 | Table: test.test4 165 | Affected columns: 3 166 | Changed rows: 1 167 | Values: 168 | -- 169 | * data : Hello 170 | * id : 1 171 | * data2 : World 172 | 173 | === XidEvent === 174 | Date: 2012-10-07T15:03:16 175 | Event size: 8 176 | Read bytes: 8 177 | Transaction ID: 14097 178 | 179 | === QueryEvent === 180 | Date: 2012-10-07T15:03:17 181 | Event size: 49 182 | Read bytes: 49 183 | Schema: test 184 | Execution time: 0 185 | Query: BEGIN 186 | 187 | === TableMapEvent === 188 | Date: 2012-10-07T15:03:17 189 | Event size: 31 190 | Read bytes: 30 191 | Table id: 781 192 | Schema: test 193 | Table: test4 194 | Columns: 3 195 | 196 | === UpdateRowsEvent === 197 | Date: 2012-10-07T15:03:17 198 | Event size: 45 199 | Read bytes: 11 200 | Table: test.test4 201 | Affected columns: 3 202 | Changed rows: 1 203 | Affected columns: 3 204 | Values: 205 | -- 206 | * data : Hello => World 207 | * id : 1 => 1 208 | * data2 : World => Hello 209 | 210 | === XidEvent === 211 | Date: 2012-10-07T15:03:17 212 | Event size: 8 213 | Read bytes: 8 214 | Transaction ID: 14098 215 | 216 | === QueryEvent === 217 | Date: 2012-10-07T15:03:17 218 | Event size: 49 219 | Read bytes: 49 220 | Schema: test 221 | Execution time: 1 222 | Query: BEGIN 223 | 224 | === TableMapEvent === 225 | Date: 2012-10-07T15:03:17 226 | Event size: 31 227 | Read bytes: 30 228 | Table id: 781 229 | Schema: test 230 | Table: test4 231 | Columns: 3 232 | 233 | === DeleteRowsEvent === 234 | Date: 2012-10-07T15:03:17 235 | Event size: 27 236 | Read bytes: 10 237 | Table: test.test4 238 | Affected columns: 3 239 | Changed rows: 1 240 | Values: 241 | -- 242 | * data : World 243 | * id : 1 244 | * data2 : Hello 245 | 246 | === XidEvent === 247 | Date: 2012-10-07T15:03:17 248 | Event size: 8 249 | Read bytes: 8 250 | Transaction ID: 14099 251 | 252 | 253 | Tests 254 | ======== 255 | When it's possible we have a unit test. 256 | 257 | More information is available here: 258 | https://python-mysql-replication.readthedocs.org/en/latest/developement.html 259 | 260 | Changelog 261 | ========== 262 | https://github.com/noplay/python-mysql-replication/blob/master/CHANGELOG 263 | 264 | Similar projects 265 | ================== 266 | * Kodoma: Ruby-binlog based MySQL replication listener https://github.com/y310/kodama 267 | * MySQL Hadoop Applier: C++ version http://dev.mysql.com/tech-resources/articles/mysql-hadoop-applier.html 268 | 269 | Special thanks 270 | ================ 271 | * MySQL binlog from Jeremy Cole was a great source of knowledge about MySQL replication protocol https://github.com/jeremycole/mysql_binlog 272 | * Samuel Charron for his help https://github.com/scharron 273 | 274 | Contributors 275 | ============== 276 | 277 | Major contributor: 278 | * bjoernhaeuser for his bugs fixing, improvements and community support https://github.com/bjoernhaeuser 279 | 280 | Other contributors: 281 | * Dvir Volk for bug fix https://github.com/dvirsky 282 | * Lior Sion code cleanup and improvements https://github.com/liorsion 283 | * Lx Yu code improvements, primary keys detections https://github.com/lxyu 284 | * Young King for pymysql 0.6 support https://github.com/youngking 285 | * David Reid checksum checking fix https://github.com/dreid 286 | * Alex Gaynor fix smallint24 https://github.com/alex 287 | * lifei NotImplementedEvent https://github.com/lifei 288 | * Maralla Python 3.4 fix https://github.com/maralla 289 | * Daniel Gavrila more MySQL error codes https://github.com/danielduduta 290 | * Bernardo Sulzbach code cleanup https://github.com/mafagafogigante 291 | * Darioush Jalali Python 2.6 backport https://github.com/darioush 292 | * Arthur Gautier gtid https://github.com/baloo 293 | * Jasonz bug fixes https://github.com/jasonzzz 294 | * Bartek Ogryczak cleanup and improvements https://github.com/vartec 295 | * Wang, Xiaozhe cleanup https://github.com/chaoslawful 296 | * siddontang improvements https://github.com/siddontang 297 | * Cheng Chen Python 2.6 compatibility https://github.com/cccc1999 298 | 299 | Thanks to GetResponse for their support 300 | 301 | Licence 302 | ======= 303 | Copyright 2012-2015 Julien Duponchelle 304 | 305 | Licensed under the Apache License, Version 2.0 (the "License"); 306 | you may not use this file except in compliance with the License. 307 | You may obtain a copy of the License at 308 | 309 | http://www.apache.org/licenses/LICENSE-2.0 310 | 311 | Unless required by applicable law or agreed to in writing, software 312 | distributed under the License is distributed on an "AS IS" BASIS, 313 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 314 | See the License for the specific language governing permissions and 315 | limitations under the License. 316 | 317 | 318 | -------------------------------------------------------------------------------- /pymysqlreplication/packet.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import struct 4 | 5 | from pymysql.util import byte2int 6 | 7 | from pymysqlreplication import constants, event, row_event 8 | 9 | # Constants from PyMYSQL source code 10 | NULL_COLUMN = 251 11 | UNSIGNED_CHAR_COLUMN = 251 12 | UNSIGNED_SHORT_COLUMN = 252 13 | UNSIGNED_INT24_COLUMN = 253 14 | UNSIGNED_INT64_COLUMN = 254 15 | UNSIGNED_CHAR_LENGTH = 1 16 | UNSIGNED_SHORT_LENGTH = 2 17 | UNSIGNED_INT24_LENGTH = 3 18 | UNSIGNED_INT64_LENGTH = 8 19 | 20 | 21 | class BinLogPacketWrapper(object): 22 | """ 23 | Bin Log Packet Wrapper. It uses an existing packet object, and wraps 24 | around it, exposing useful variables while still providing access 25 | to the original packet objects variables and methods. 26 | """ 27 | 28 | __event_map = { 29 | # event 30 | constants.QUERY_EVENT: event.QueryEvent, 31 | constants.ROTATE_EVENT: event.RotateEvent, 32 | constants.FORMAT_DESCRIPTION_EVENT: event.FormatDescriptionEvent, 33 | constants.XID_EVENT: event.XidEvent, 34 | constants.INTVAR_EVENT: event.NotImplementedEvent, 35 | constants.GTID_LOG_EVENT: event.GtidEvent, 36 | constants.STOP_EVENT: event.StopEvent, 37 | constants.BEGIN_LOAD_QUERY_EVENT: event.BeginLoadQueryEvent, 38 | constants.EXECUTE_LOAD_QUERY_EVENT: event.ExecuteLoadQueryEvent, 39 | # row_event 40 | constants.UPDATE_ROWS_EVENT_V1: row_event.UpdateRowsEvent, 41 | constants.WRITE_ROWS_EVENT_V1: row_event.WriteRowsEvent, 42 | constants.DELETE_ROWS_EVENT_V1: row_event.DeleteRowsEvent, 43 | constants.UPDATE_ROWS_EVENT_V2: row_event.UpdateRowsEvent, 44 | constants.WRITE_ROWS_EVENT_V2: row_event.WriteRowsEvent, 45 | constants.DELETE_ROWS_EVENT_V2: row_event.DeleteRowsEvent, 46 | constants.TABLE_MAP_EVENT: row_event.TableMapEvent, 47 | #5.6 GTID enabled replication events 48 | constants.ANONYMOUS_GTID_LOG_EVENT: event.NotImplementedEvent, 49 | constants.PREVIOUS_GTIDS_LOG_EVENT: event.NotImplementedEvent 50 | 51 | } 52 | 53 | def __init__(self, from_packet, table_map, ctl_connection, use_checksum, 54 | allowed_events, 55 | only_tables, 56 | only_schemas, 57 | freeze_schema): 58 | # -1 because we ignore the ok byte 59 | self.read_bytes = 0 60 | # Used when we want to override a value in the data buffer 61 | self.__data_buffer = b'' 62 | 63 | self.packet = from_packet 64 | self.charset = ctl_connection.charset 65 | 66 | # OK value 67 | # timestamp 68 | # event_type 69 | # server_id 70 | # log_pos 71 | # flags 72 | unpack = struct.unpack(' 0: 106 | data = self.__data_buffer[:size] 107 | self.__data_buffer = self.__data_buffer[size:] 108 | if len(data) == size: 109 | return data 110 | else: 111 | return data + self.packet.read(size - len(data)) 112 | return self.packet.read(size) 113 | 114 | def unread(self, data): 115 | '''Push again data in data buffer. It's use when you want 116 | to extract a bit from a value a let the rest of the code normally 117 | read the datas''' 118 | self.read_bytes -= len(data) 119 | self.__data_buffer += data 120 | 121 | def advance(self, size): 122 | size = int(size) 123 | self.read_bytes += size 124 | buffer_len = len(self.__data_buffer) 125 | if buffer_len > 0: 126 | self.__data_buffer = self.__data_buffer[size:] 127 | if size > buffer_len: 128 | self.packet.advance(size - buffer_len) 129 | else: 130 | self.packet.advance(size) 131 | 132 | def read_length_coded_binary(self): 133 | """Read a 'Length Coded Binary' number from the data buffer. 134 | 135 | Length coded numbers can be anywhere from 1 to 9 bytes depending 136 | on the value of the first byte. 137 | 138 | From PyMYSQL source code 139 | """ 140 | c = byte2int(self.read(1)) 141 | if c == NULL_COLUMN: 142 | return None 143 | if c < UNSIGNED_CHAR_COLUMN: 144 | return c 145 | elif c == UNSIGNED_SHORT_COLUMN: 146 | return self.unpack_uint16(self.read(UNSIGNED_SHORT_LENGTH)) 147 | elif c == UNSIGNED_INT24_COLUMN: 148 | return self.unpack_int24(self.read(UNSIGNED_INT24_LENGTH)) 149 | elif c == UNSIGNED_INT64_COLUMN: 150 | return self.unpack_int64(self.read(UNSIGNED_INT64_LENGTH)) 151 | 152 | def read_length_coded_string(self): 153 | """Read a 'Length Coded String' from the data buffer. 154 | 155 | A 'Length Coded String' consists first of a length coded 156 | (unsigned, positive) integer represented in 1-9 bytes followed by 157 | that many bytes of binary data. (For example "cat" would be "3cat".) 158 | 159 | From PyMYSQL source code 160 | """ 161 | length = self.read_length_coded_binary() 162 | if length is None: 163 | return None 164 | return self.read(length).decode() 165 | 166 | def __getattr__(self, key): 167 | if hasattr(self.packet, key): 168 | return getattr(self.packet, key) 169 | 170 | raise AttributeError("%s instance has no attribute '%s'" % 171 | (self.__class__, key)) 172 | 173 | def read_int_be_by_size(self, size): 174 | '''Read a big endian integer values based on byte number''' 175 | if size == 1: 176 | return struct.unpack('>b', self.read(size))[0] 177 | elif size == 2: 178 | return struct.unpack('>h', self.read(size))[0] 179 | elif size == 3: 180 | return self.read_int24_be() 181 | elif size == 4: 182 | return struct.unpack('>i', self.read(size))[0] 183 | elif size == 5: 184 | return self.read_int40_be() 185 | elif size == 8: 186 | return struct.unpack('>l', self.read(size))[0] 187 | 188 | def read_uint_by_size(self, size): 189 | '''Read a little endian integer values based on byte number''' 190 | if size == 1: 191 | return self.read_uint8() 192 | elif size == 2: 193 | return self.read_uint16() 194 | elif size == 3: 195 | return self.read_uint24() 196 | elif size == 4: 197 | return self.read_uint32() 198 | elif size == 5: 199 | return self.read_uint40() 200 | elif size == 6: 201 | return self.read_uint48() 202 | elif size == 7: 203 | return self.read_uint56() 204 | elif size == 8: 205 | return self.read_uint64() 206 | 207 | def read_length_coded_pascal_string(self, size): 208 | """Read a string with length coded using pascal style. 209 | The string start by the size of the string 210 | """ 211 | length = self.read_uint_by_size(size) 212 | return self.read(length) 213 | 214 | def read_int24(self): 215 | a, b, c = struct.unpack("BBB", self.read(3)) 216 | res = a | (b << 8) | (c << 16) 217 | if res >= 0x800000: 218 | res -= 0x1000000 219 | return res 220 | 221 | def read_int24_be(self): 222 | a, b, c = struct.unpack('BBB', self.read(3)) 223 | res = (a << 16) | (b << 8) | c 224 | if res >= 0x800000: 225 | res -= 0x1000000 226 | return res 227 | 228 | def read_uint8(self): 229 | return struct.unpack('IB", self.read(5)) 247 | return b + (a << 8) 248 | 249 | def read_uint48(self): 250 | a, b, c = struct.unpack(" 5.6""" 109 | cur = self._stream_connection.cursor() 110 | cur.execute("SHOW GLOBAL VARIABLES LIKE 'BINLOG_CHECKSUM'") 111 | result = cur.fetchone() 112 | cur.close() 113 | 114 | if result is None: 115 | return False 116 | var, value = result[:2] 117 | if value == 'NONE': 118 | return False 119 | return True 120 | 121 | def __connect_to_stream(self): 122 | # log_pos (4) -- position in the binlog-file to start the stream with 123 | # flags (2) BINLOG_DUMP_NON_BLOCK (0 or 1) 124 | # server_id (4) -- server id of this slave 125 | # log_file (string.EOF) -- filename of the binlog on the master 126 | self._stream_connection = pymysql.connect(**self.__connection_settings) 127 | 128 | self.__use_checksum = self.__checksum_enabled() 129 | 130 | # If checksum is enabled we need to inform the server about the that 131 | # we support it 132 | if self.__use_checksum: 133 | cur = self._stream_connection.cursor() 134 | cur.execute("set @master_binlog_checksum= @@global.binlog_checksum") 135 | cur.close() 136 | 137 | if not self.auto_position: 138 | # only when log_file and log_pos both provided, the position info is 139 | # valid, if not, get the current position from master 140 | if self.log_file is None or self.log_pos is None: 141 | cur = self._stream_connection.cursor() 142 | cur.execute("SHOW MASTER STATUS") 143 | self.log_file, self.log_pos = cur.fetchone()[:2] 144 | cur.close() 145 | 146 | prelude = struct.pack(' 255: 116 | values[name] = self.__read_string(2, column) 117 | else: 118 | values[name] = self.__read_string(1, column) 119 | elif column.type == FIELD_TYPE.NEWDECIMAL: 120 | values[name] = self.__read_new_decimal(column) 121 | elif column.type == FIELD_TYPE.BLOB: 122 | values[name] = self.__read_string(column.length_size, column) 123 | elif column.type == FIELD_TYPE.DATETIME: 124 | values[name] = self.__read_datetime() 125 | elif column.type == FIELD_TYPE.TIME: 126 | values[name] = self.__read_time() 127 | elif column.type == FIELD_TYPE.DATE: 128 | values[name] = self.__read_date() 129 | elif column.type == FIELD_TYPE.TIMESTAMP: 130 | values[name] = datetime.datetime.fromtimestamp( 131 | self.packet.read_uint32()) 132 | 133 | # For new date format: 134 | elif column.type == FIELD_TYPE.DATETIME2: 135 | values[name] = self.__read_datetime2(column) 136 | elif column.type == FIELD_TYPE.TIME2: 137 | values[name] = self.__read_time2(column) 138 | elif column.type == FIELD_TYPE.TIMESTAMP2: 139 | values[name] = self.__add_fsp_to_time( 140 | datetime.datetime.fromtimestamp( 141 | self.packet.read_int_be_by_size(4)), column) 142 | elif column.type == FIELD_TYPE.LONGLONG: 143 | if unsigned: 144 | values[name] = self.packet.read_uint64() 145 | else: 146 | values[name] = self.packet.read_int64() 147 | elif column.type == FIELD_TYPE.YEAR: 148 | values[name] = self.packet.read_uint8() + 1900 149 | elif column.type == FIELD_TYPE.ENUM: 150 | values[name] = column.enum_values[ 151 | self.packet.read_uint_by_size(column.size) - 1] 152 | elif column.type == FIELD_TYPE.SET: 153 | # We read set columns as a bitmap telling us which options 154 | # are enabled 155 | bit_mask = self.packet.read_uint_by_size(column.size) 156 | values[name] = set( 157 | val for idx, val in enumerate(column.set_values) 158 | if bit_mask & 2 ** idx 159 | ) or None 160 | 161 | elif column.type == FIELD_TYPE.BIT: 162 | values[name] = self.__read_bit(column) 163 | elif column.type == FIELD_TYPE.GEOMETRY: 164 | values[name] = self.packet.read_length_coded_pascal_string( 165 | column.length_size) 166 | else: 167 | raise NotImplementedError("Unknown MySQL column type: %d" % 168 | (column.type)) 169 | 170 | nullBitmapIndex += 1 171 | 172 | return values 173 | 174 | def __add_fsp_to_time(self, time, column): 175 | """Read and add the fractional part of time 176 | For more details about new date format: 177 | http://dev.mysql.com/doc/internals/en/date-and-time-data-type-representation.html 178 | """ 179 | read = 0 180 | if column.fsp == 1 or column.fsp == 2: 181 | read = 1 182 | elif column.fsp == 3 or column.fsp == 4: 183 | read = 2 184 | elif column.fsp == 5 or column.fsp == 6: 185 | read = 3 186 | if read > 0: 187 | microsecond = self.packet.read_int_be_by_size(read) 188 | if column.fsp % 2: 189 | time = time.replace(microsecond=int(microsecond / 10)) 190 | else: 191 | time = time.replace(microsecond=microsecond) 192 | return time 193 | 194 | def __read_string(self, size, column): 195 | string = self.packet.read_length_coded_pascal_string(size) 196 | if column.character_set_name is not None: 197 | string = string.decode(column.character_set_name) 198 | return string 199 | 200 | def __read_bit(self, column): 201 | """Read MySQL BIT type""" 202 | resp = "" 203 | for byte in range(0, column.bytes): 204 | current_byte = "" 205 | data = self.packet.read_uint8() 206 | if byte == 0: 207 | if column.bytes == 1: 208 | end = column.bits 209 | else: 210 | end = column.bits % 8 211 | if end == 0: 212 | end = 8 213 | else: 214 | end = 8 215 | for bit in range(0, end): 216 | if data & (1 << bit): 217 | current_byte += "1" 218 | else: 219 | current_byte += "0" 220 | resp += current_byte[::-1] 221 | return resp 222 | 223 | def __read_time(self): 224 | time = self.packet.read_uint24() 225 | date = datetime.time( 226 | hour=int(time / 10000), 227 | minute=int((time % 10000) / 100), 228 | second=int(time % 100)) 229 | return date 230 | 231 | def __read_time2(self, column): 232 | """TIME encoding for nonfractional part: 233 | 234 | 1 bit sign (1= non-negative, 0= negative) 235 | 1 bit unused (reserved for future extensions) 236 | 10 bits hour (0-838) 237 | 6 bits minute (0-59) 238 | 6 bits second (0-59) 239 | --------------------- 240 | 24 bits = 3 bytes 241 | """ 242 | data = self.packet.read_int_be_by_size(3) 243 | t = datetime.time( 244 | hour=self.__read_binary_slice(data, 2, 10, 24), 245 | minute=self.__read_binary_slice(data, 12, 6, 24), 246 | second=self.__read_binary_slice(data, 18, 6, 24)) 247 | return self.__add_fsp_to_time(t, column) 248 | 249 | def __read_date(self): 250 | time = self.packet.read_uint24() 251 | if time == 0: # nasty mysql 0000-00-00 dates 252 | return None 253 | 254 | year = (time & ((1 << 15) - 1) << 9) >> 9 255 | if year == 0: 256 | return None 257 | 258 | month = (time & ((1 << 4) - 1) << 5) >> 5 259 | day = (time & ((1 << 5) - 1)) 260 | 261 | date = datetime.date( 262 | year=year, 263 | month=month, 264 | day=day 265 | ) 266 | return date 267 | 268 | def __read_datetime(self): 269 | value = self.packet.read_uint64() 270 | if value == 0: # nasty mysql 0000-00-00 dates 271 | return None 272 | 273 | date = value / 1000000 274 | time = int(value % 1000000) 275 | 276 | year = int(date / 10000) 277 | month = int((date % 10000) / 100) 278 | day = int(date % 100) 279 | if year == 0 or month == 0 or day == 0: 280 | return None 281 | 282 | date = datetime.datetime( 283 | year=year, 284 | month=month, 285 | day=day, 286 | hour=int(time / 10000), 287 | minute=int((time % 10000) / 100), 288 | second=int(time % 100)) 289 | return date 290 | 291 | def __read_datetime2(self, column): 292 | """DATETIME 293 | 294 | 1 bit sign (1= non-negative, 0= negative) 295 | 17 bits year*13+month (year 0-9999, month 0-12) 296 | 5 bits day (0-31) 297 | 5 bits hour (0-23) 298 | 6 bits minute (0-59) 299 | 6 bits second (0-59) 300 | --------------------------- 301 | 40 bits = 5 bytes 302 | """ 303 | data = self.packet.read_int_be_by_size(5) 304 | year_month = self.__read_binary_slice(data, 1, 17, 40) 305 | try: 306 | t = datetime.datetime( 307 | year=int(year_month / 13), 308 | month=year_month % 13, 309 | day=self.__read_binary_slice(data, 18, 5, 40), 310 | hour=self.__read_binary_slice(data, 23, 5, 40), 311 | minute=self.__read_binary_slice(data, 28, 6, 40), 312 | second=self.__read_binary_slice(data, 34, 6, 40)) 313 | except ValueError: 314 | return None 315 | return self.__add_fsp_to_time(t, column) 316 | 317 | def __read_new_decimal(self, column): 318 | """Read MySQL's new decimal format introduced in MySQL 5""" 319 | 320 | # This project was a great source of inspiration for 321 | # understanding this storage format. 322 | # https://github.com/jeremycole/mysql_binlog 323 | 324 | digits_per_integer = 9 325 | compressed_bytes = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4] 326 | integral = (column.precision - column.decimals) 327 | uncomp_integral = int(integral / digits_per_integer) 328 | uncomp_fractional = int(column.decimals / digits_per_integer) 329 | comp_integral = integral - (uncomp_integral * digits_per_integer) 330 | comp_fractional = column.decimals - (uncomp_fractional 331 | * digits_per_integer) 332 | 333 | # Support negative 334 | # The sign is encoded in the high bit of the the byte 335 | # But this bit can also be used in the value 336 | value = self.packet.read_uint8() 337 | if value & 0x80 != 0: 338 | res = "" 339 | mask = 0 340 | else: 341 | mask = -1 342 | res = "-" 343 | self.packet.unread(struct.pack(' 0: 347 | value = self.packet.read_int_be_by_size(size) ^ mask 348 | res += str(value) 349 | 350 | for i in range(0, uncomp_integral): 351 | value = struct.unpack('>i', self.packet.read(4))[0] ^ mask 352 | res += '%09d' % value 353 | 354 | res += "." 355 | 356 | for i in range(0, uncomp_fractional): 357 | value = struct.unpack('>i', self.packet.read(4))[0] ^ mask 358 | res += '%09d' % value 359 | 360 | size = compressed_bytes[comp_fractional] 361 | if size > 0: 362 | value = self.packet.read_int_be_by_size(size) ^ mask 363 | res += '%0*d' % (comp_fractional, value) 364 | 365 | return decimal.Decimal(res) 366 | 367 | def __read_binary_slice(self, binary, start, size, data_length): 368 | """ 369 | Read a part of binary data and extract a number 370 | binary: the data 371 | start: From which bit (1 to X) 372 | size: How many bits should be read 373 | data_length: data size 374 | """ 375 | binary = binary >> data_length - (start + size) 376 | mask = ((1 << size) - 1) 377 | return binary & mask 378 | 379 | def _dump(self): 380 | super(RowsEvent, self)._dump() 381 | print("Table: %s.%s" % (self.schema, self.table)) 382 | print("Affected columns: %d" % self.number_of_columns) 383 | print("Changed rows: %d" % (len(self.rows))) 384 | 385 | def _fetch_rows(self): 386 | self.__rows = [] 387 | 388 | if not self.complete: 389 | return 390 | 391 | while self.packet.read_bytes + 1 < self.event_size: 392 | self.__rows.append(self._fetch_one_row()) 393 | 394 | @property 395 | def rows(self): 396 | if self.__rows is None: 397 | self._fetch_rows() 398 | return self.__rows 399 | 400 | 401 | class DeleteRowsEvent(RowsEvent): 402 | """This event is trigger when a row in the database is removed 403 | 404 | For each row you have a hash with a single key: values which contain the data of the removed line. 405 | """ 406 | 407 | def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): 408 | super(DeleteRowsEvent, self).__init__(from_packet, event_size, 409 | table_map, ctl_connection, **kwargs) 410 | if self._processed: 411 | self.columns_present_bitmap = self.packet.read( 412 | (self.number_of_columns + 7) / 8) 413 | 414 | def _fetch_one_row(self): 415 | row = {} 416 | 417 | row["values"] = self._read_column_data(self.columns_present_bitmap) 418 | return row 419 | 420 | def _dump(self): 421 | super(DeleteRowsEvent, self)._dump() 422 | print("Values:") 423 | for row in self.rows: 424 | print("--") 425 | for key in row["values"]: 426 | print("*", key, ":", row["values"][key]) 427 | 428 | 429 | class WriteRowsEvent(RowsEvent): 430 | """This event is triggered when a row in database is added 431 | 432 | For each row you have a hash with a single key: values which contain the data of the new line. 433 | """ 434 | 435 | def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): 436 | super(WriteRowsEvent, self).__init__(from_packet, event_size, 437 | table_map, ctl_connection, **kwargs) 438 | if self._processed: 439 | self.columns_present_bitmap = self.packet.read( 440 | (self.number_of_columns + 7) / 8) 441 | 442 | def _fetch_one_row(self): 443 | row = {} 444 | 445 | row["values"] = self._read_column_data(self.columns_present_bitmap) 446 | return row 447 | 448 | def _dump(self): 449 | super(WriteRowsEvent, self)._dump() 450 | print("Values:") 451 | for row in self.rows: 452 | print("--") 453 | for key in row["values"]: 454 | print("*", key, ":", row["values"][key]) 455 | 456 | 457 | class UpdateRowsEvent(RowsEvent): 458 | """This event is triggered when a row in the database is changed 459 | 460 | For each row you got a hash with two keys: 461 | * before_values 462 | * after_values 463 | 464 | Depending of your MySQL configuration the hash can contains the full row or only the changes: 465 | http://dev.mysql.com/doc/refman/5.6/en/replication-options-binary-log.html#sysvar_binlog_row_image 466 | """ 467 | 468 | def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): 469 | super(UpdateRowsEvent, self).__init__(from_packet, event_size, 470 | table_map, ctl_connection, **kwargs) 471 | if self._processed: 472 | #Body 473 | self.columns_present_bitmap = self.packet.read( 474 | (self.number_of_columns + 7) / 8) 475 | self.columns_present_bitmap2 = self.packet.read( 476 | (self.number_of_columns + 7) / 8) 477 | 478 | def _fetch_one_row(self): 479 | row = {} 480 | 481 | row["before_values"] = self._read_column_data(self.columns_present_bitmap) 482 | 483 | row["after_values"] = self._read_column_data(self.columns_present_bitmap2) 484 | return row 485 | 486 | def _dump(self): 487 | super(UpdateRowsEvent, self)._dump() 488 | print("Affected columns: %d" % self.number_of_columns) 489 | print("Values:") 490 | for row in self.rows: 491 | print("--") 492 | for key in row["before_values"]: 493 | print("*%s:%s=>%s" % (key, 494 | row["before_values"][key], 495 | row["after_values"][key])) 496 | 497 | 498 | class TableMapEvent(BinLogEvent): 499 | """This evenement describe the structure of a table. 500 | It's send before a change append on a table. 501 | A end user of the lib should have no usage of this 502 | """ 503 | 504 | def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs): 505 | super(TableMapEvent, self).__init__(from_packet, event_size, 506 | table_map, ctl_connection, **kwargs) 507 | self.__only_tables = kwargs["only_tables"] 508 | self.__only_schemas = kwargs["only_schemas"] 509 | self.__freeze_schema = kwargs["freeze_schema"] 510 | 511 | # Post-Header 512 | self.table_id = self._read_table_id() 513 | 514 | if self.table_id in table_map and self.__freeze_schema: 515 | self._processed = False 516 | return 517 | 518 | self.flags = struct.unpack('