├── .gitignore ├── .gitmodules ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── data └── fixtures │ ├── 01-include-table.json │ ├── 01-no-events.json │ ├── 01.json │ ├── 02.json │ ├── 03.json │ ├── 04.json │ ├── 05.json │ ├── 06.json │ ├── 07.json │ ├── mysql-bin.01 │ ├── mysql-bin.02 │ ├── mysql-bin.03 │ ├── mysql-bin.04 │ ├── mysql-bin.05 │ ├── mysql-bin.06 │ ├── mysql-bin.07 │ ├── mysql-bin.empty │ ├── mysql-index-file.01 │ ├── mysql-index-file.02 │ ├── plaintext-mysql-bin.01 │ ├── plaintext-mysql-bin.02 │ ├── plaintext-mysql-bin.03 │ ├── plaintext-mysql-bin.04 │ ├── plaintext-mysql-bin.05 │ ├── plaintext-mysql-bin.06 │ ├── plaintext-mysql-bin.07 │ ├── plaintext-mysql-bin.empty │ └── test_db.sql └── src └── zalora └── binlog-parser ├── database ├── database_error.go ├── db_instance.go ├── table_map.go └── table_map_integration_test.go ├── main.go ├── parse_binlog_file.go ├── parse_binlog_file_integration_test.go ├── parser ├── binlog_parser.go ├── consumer_chain.go ├── consumer_chain_test.go ├── conversion │ ├── conversion.go │ ├── conversion_test.go │ ├── row_data.go │ └── row_data_test.go ├── messages │ ├── message.go │ └── message_test.go └── parser │ ├── binlog_to_messages.go │ ├── rows_event_buffer.go │ └── rows_event_buffer_test.go └── test └── test_setup.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | 16 | # -*- mode: gitignore; -*- 17 | *~ 18 | \#*\# 19 | /.emacs.desktop 20 | /.emacs.desktop.lock 21 | *.elc 22 | auto-save-list 23 | tramp 24 | .\#* 25 | 26 | # Org-mode 27 | .org-id-locations 28 | *_archive 29 | 30 | # flymake-mode 31 | *_flymake.* 32 | 33 | # eshell files 34 | /eshell/history 35 | /eshell/lastdir 36 | 37 | # elpa packages 38 | /elpa/ 39 | 40 | # reftex files 41 | *.rel 42 | 43 | # AUCTeX auto folder 44 | /auto/ 45 | 46 | # cask packages 47 | .cask/ 48 | dist/ 49 | 50 | # Flycheck 51 | flycheck_*.el 52 | 53 | # server auth directory 54 | /server/ 55 | 56 | # projectiles files 57 | .projectile 58 | 59 | # directory configuration 60 | .dir-locals.el 61 | 62 | *.DS_Store 63 | .AppleDouble 64 | .LSOverride 65 | 66 | # Icon must end with two \r 67 | Icon 68 | 69 | 70 | # Thumbnails 71 | ._* 72 | 73 | # Files that might appear in the root of a volume 74 | .DocumentRevisions-V100 75 | .fseventsd 76 | .Spotlight-V100 77 | .TemporaryItems 78 | .Trashes 79 | .VolumeIcon.icns 80 | .com.apple.timemachine.donotpresent 81 | 82 | # Directories potentially created on remote AFP share 83 | .AppleDB 84 | .AppleDesktop 85 | Network Trash Folder 86 | Temporary Items 87 | .apdisk 88 | 89 | 90 | ### 91 | 92 | bin/ 93 | pkg/ 94 | maxwell.txt 95 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "_vendor/src/golang.org/x/net"] 2 | path = _vendor/src/golang.org/x/net 3 | url = https://go.googlesource.com/net 4 | [submodule "_vendor/src/github.com/BurntSushi/toml"] 5 | path = _vendor/src/github.com/BurntSushi/toml 6 | url = https://github.com/BurntSushi/toml 7 | [submodule "_vendor/src/github.com/go-sql-driver/mysql"] 8 | path = _vendor/src/github.com/go-sql-driver/mysql 9 | url = https://github.com/go-sql-driver/mysql 10 | [submodule "errors"] 11 | path = errors 12 | url = https://github.com/juju/errors 13 | [submodule "_vendor/src/github.com/juju/errors"] 14 | path = _vendor/src/github.com/juju/errors 15 | url = https://github.com/juju/errors 16 | [submodule "_vendor/src/github.com/ngaut/log"] 17 | path = _vendor/src/github.com/ngaut/log 18 | url = https://github.com/ngaut/log 19 | [submodule "go.uuid"] 20 | path = go.uuid 21 | url = https://github.com/satori/go.uuid 22 | [submodule "_vendor/src/github.com/satori/go.uuid"] 23 | path = _vendor/src/github.com/satori/go.uuid 24 | url = https://github.com/satori/go.uuid 25 | [submodule "_vendor/src/github.com/siddontang/go"] 26 | path = _vendor/src/github.com/siddontang/go 27 | url = https://github.com/siddontang/go 28 | [submodule "_vendor/src/github.com/siddontang/go-mysql"] 29 | path = _vendor/src/github.com/siddontang/go-mysql 30 | url = https://github.com/siddontang/go-mysql 31 | [submodule "_vendor/src/github.com/mattn/go-sqlite3"] 32 | path = _vendor/src/github.com/mattn/go-sqlite3 33 | url = https://github.com/mattn/go-sqlite3 34 | [submodule "_vendor/src/github.com/golang/glog"] 35 | path = _vendor/src/github.com/golang/glog 36 | url = https://github.com/golang/glog 37 | [submodule "_vendor/src/golang.org/x/sys"] 38 | path = _vendor/src/golang.org/x/sys 39 | url = https://go.googlesource.com/sys 40 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | services: 4 | - mysql 5 | 6 | install: 7 | - if [[ `go version` == *"go1.7"* ]]; then export RELEASE=1; fi 8 | - make integration-test-setup 9 | - make deps 10 | 11 | script: 12 | - make 13 | - make test 14 | 15 | go: 16 | - 1.7.x 17 | - 1.8.x 18 | - master 19 | 20 | matrix: 21 | allow_failures: 22 | - go: master 23 | 24 | deploy: 25 | provider: releases 26 | api_key: 27 | secure: IMrcz8PI85C0PUCtfVRZrUZ72f37PMGg+MNfm/bsHbnZQtc6QZbykUvktRCpCU3YdR3YpwipHu79I1q3Nv4T1QcKqCT6ASLCUsGhmQ0wio2X4iMfsvlbQRx+BIQmoE2SyVJnIpAVmeBHtoxymf0UAeaBzH2FxTbRC6APnyItOy2BQmLlwfnaboC0QnF+h8HH0HEzau/TWFt+ea5XwqYGDFNqfbJ3uUvEuxj99lWj03TqPjPs3O3sNuSzxvcHbVIF9tL6crFd04y2TFRF6oLlLhqtRP2WepE43ZfnEsqCTN9N4MvubVk3KQ5R8lKwQ3tbAw/8lSF2hN5lntIkXd2jTpKv5uYRoBKxPAY4eHtTwJn8x8OylfwuqwKDMNVyz2IvXzQm7oacZW1/sKvgpnyX/jSHwrWIxUYO6Gv4GI3+Xqg/0EoESlFNPHD6eEAhNzSYr0EGYQW34okgGH8bKuiSilTsaL4ycD00zJQepkrkdCZJkifkbR24LlocIoaRe3HFCcxG8IAedLBVjRX/xHhegOlFitZ6sTI3PWDu++YGUXToBFsUicEdzGwkM+2Rby9jGJY2Mle5t7zj6vuZ3gMs8/5/AHqPjgG8ZPhkZz9TM8gleFkK7YH5rJjWehr7wn7iOr3SbbPknYWRdFNAqFOURR7H6a6KB7EwBjqhN3lEx8E= 28 | file: bin/binlog-parser 29 | skip_cleanup: true 30 | on: 31 | tags: true 32 | condition: "$RELEASE = 1" 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 - present, Zalora South East Asia Pte. Ltd 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BIN_NAME := binlog-parser 2 | GOCC := env GOPATH=$(CURDIR)/_vendor:$(CURDIR) go 3 | SRC_DIR := zalora/binlog-parser/... 4 | 5 | TEST_DB_NAME := test_db 6 | TEST_DB_SCHEMA_FILE := data/fixtures/test_db.sql 7 | 8 | all: 9 | env CGO_ENABLED=0 $(GOCC) install -ldflags '-s' $(SRC_DIR) 10 | 11 | deps: 12 | git submodule update --init 13 | 14 | test: unit-test integration-test 15 | 16 | unit-test: 17 | $(info ************ UNIT TESTS ************) 18 | env TZ="UTC" env DATA_DIR=$(CURDIR)/data $(GOCC) test -tags=unit -cover $(SRC_DIR) 19 | 20 | integration-test: integration-test-setup 21 | $(info ************ INTEGRATION TESTS ************) 22 | env TEST_DB_DSN="root@/$(TEST_DB_NAME)" env TZ="UTC" env DATA_DIR=$(CURDIR)/data $(GOCC) test -tags=integration -cover $(SRC_DIR) 23 | 24 | integration-test-setup: 25 | mysql -uroot -e 'DROP DATABASE IF EXISTS $(TEST_DB_NAME)' 26 | mysql -uroot < $(TEST_DB_SCHEMA_FILE) 27 | 28 | integration-test-schema-dump: 29 | mysqldump --no-data -uroot -B $(TEST_DB_NAME) > $(TEST_DB_SCHEMA_FILE) 30 | 31 | .PHONY: all deps test unit-test integration-test-setup integration-test integration-test-schema-dump 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # binlog-parser 2 | 3 | [![Build Status](https://travis-ci.org/zalora/binlog-parser.svg?branch=master)](https://travis-ci.org/zalora/binlog-parser) 4 | 5 | A tool for parsing a MySQL binlog file to JSON. Reads a binlog input file, queries a database for field names, writes JSON to stdout. The output looks like this: 6 | 7 | { 8 | "Header": { 9 | "Schema": "test_db", 10 | "Table": "buildings", 11 | "BinlogMessageTime": "2017-04-13T06:34:30Z", 12 | "BinlogPosition": 397, 13 | "XId": 9 14 | }, 15 | "Type": "Insert", 16 | "Data": { 17 | "Row": { 18 | "address": "3950 North 1st Street CA 95134", 19 | "building_name": "ACME Headquaters", 20 | "building_no": 1 21 | }, 22 | "MappingNotice": "" 23 | } 24 | } 25 | ... 26 | 27 | # Installation 28 | 29 | Requires Go version 1.7 or higher. 30 | 31 | $ git clone https://github.com/zalora/binlog-parser.git 32 | $ cd binlog-parser 33 | $ git submodule update --init 34 | $ make 35 | $ ./bin/binlog-parser -h 36 | 37 | ## Assumptions 38 | 39 | - It is assumed that MySQL row-based binlog format is used (or mixed, but be aware, that then only the row-formatted data in mixed binlogs can be extracted) 40 | - This tool is written with MySQL 5.6 in mind, although it should also work for MariaDB when GTIDs are not used 41 | 42 | # Usage 43 | 44 | Run `binlog-parser -h` to get the list of available options: 45 | 46 | Usage: binlog-parser [options ...] binlog 47 | 48 | Options are: 49 | 50 | -alsologtostderr 51 | log to standard error as well as files 52 | -include_schemas string 53 | comma-separated list of schemas to include 54 | -include_tables string 55 | comma-separated list of tables to include 56 | -log_backtrace_at value 57 | when logging hits line file:N, emit a stack trace 58 | -log_dir string 59 | If non-empty, write log files in this directory 60 | -logtostderr 61 | log to standard error instead of files 62 | -prettyprint 63 | Pretty print json 64 | -stderrthreshold value 65 | logs at or above this threshold go to stderr 66 | -v value 67 | log level for V logs 68 | -vmodule value 69 | comma-separated list of pattern=N settings for file-filtered logging 70 | 71 | Required environment variables: 72 | 73 | DB_DSN Database connection string, needs read access to information_schema 74 | 75 | ## Example usage 76 | 77 | Using `dbuser` and no password, connecting to `information_schema` database on localhost, parsing the binlog file `/some/binlog.bin`: 78 | 79 | DB_DSN=dbuser@/information_schema ./binlog-parser /some/binlog.bin 80 | 81 | ## Matching field names and data 82 | 83 | The mysql binlog format doesn't include the fieldnames for row events (INSERT/UPDATE/DELETE). As the goal of the parser is to output 84 | usable JSON, it connects to a running MySQL instance and queries the `information_schema` database for information on field names in the table. 85 | 86 | The database connection is creatd by using the environment variable `DB_DSN`, which should contain the database credentials in the form of 87 | `user:password@/dbname` - the format that the [Go MySQL driver](https://godoc.org/github.com/go-sql-driver/mysql) uses. 88 | 89 | ## Effect of schema changes 90 | 91 | As this tool doesn't keep an internal representation of the database schema, it is very well possible that the database schema and the schema used in the 92 | queries in the binlog file already have diverged (e. g. parsing a binlog file from a few days ago, but the schema on the main database already changed 93 | by dropping or adding columns). 94 | 95 | The parser will NOT make an attempt to map data to fields in a table if the information schema retuns more or too less columns 96 | compared to the format found in the binlog. The field names will be mapped as "unknown": 97 | 98 | { 99 | "Header": { 100 | "Schema": "test_db", 101 | "Table": "employees", 102 | "BinlogMessageTime": "2017-04-13T08:02:04Z", 103 | "BinlogPosition": 635, 104 | "XId": 8 105 | }, 106 | "Type": "Insert", 107 | "Data": { 108 | "Row": { 109 | "(unknown_0)": 1, 110 | "(unknown_1)": "2017-04-13", 111 | "(unknown_2)": "Max", 112 | "(unknown_3)": "Mustermann" 113 | }, 114 | "MappingNotice": "column names array is missing field(s), will map them as unknown_*" 115 | } 116 | } 117 | 118 | ### More complex case 119 | 120 | Changing the order of fields in a table can lead to unexpected parser results. Consider an example binlog file `A.bin`. 121 | A query like `INSERT INTO db.foo SET field_1 = 10, field_2 = 20` will look in the binlog like this: 122 | 123 | ... 124 | ### INSERT INTO `db`.`foo` 125 | ### SET 126 | ### @1=20 /* ... */ 127 | ### @2=20 /* ... */ 128 | ... 129 | 130 | The parser queries `information_schema` for the field names of the `db.foo` table: 131 | 132 | +-------------+-----+ 133 | | Field | ... | 134 | +-------------+-----+ 135 | | field_1 | ... | 136 | | field_2 | ... | 137 | +-------------+-----+ 138 | 139 | The fields will be mapped by the parser in the order as specified in the table and the JSON will look like this: 140 | 141 | { 142 | ... 143 | "Type": "Insert", 144 | "Data": { 145 | "Row": { 146 | "field_1": 10, 147 | "field_2": 20 148 | } 149 | } 150 | } 151 | ... 152 | 153 | Now if a schema change happened after some time, `db.foo` fields might look now like this (the order of the fiels changed): 154 | 155 | +-------------+-----+ 156 | | Field | ... | 157 | +-------------+-----+ 158 | | field_2 | ... | 159 | | field_1 | ... | 160 | +-------------+-----+ 161 | 162 | If you parse the same binlog file `A.bin` now again, but against the new schema of `db.foo` (in which the fields changed position), the resulting JSON 163 | will look like that: 164 | 165 | 166 | { 167 | ... 168 | "Type": "Insert", 169 | "Data": { 170 | "Row": { 171 | "field_2": 10, 172 | "field_1": 20 173 | } 174 | } 175 | } 176 | ... 177 | 178 | This means you have to be very careful when parsing old binlog files, as the db schema can have evolved since the binlog was generated and the parser 179 | has no way of knowing of these changes. 180 | 181 | If this limitation is not acceptable, some tools like [Maxwell's Daemon by Zendesk](https://github.com/zendesk/maxwell) can work around that issue at the cost of greater complexity. 182 | 183 | # Releases 184 | 185 | How to do a release: 186 | 187 | git tag -a X.X.X -m "... release note ..." 188 | git push --follow-tags 189 | 190 | Travis CI will attach a statically built binary to the release tag on GitHub. 191 | -------------------------------------------------------------------------------- /data/fixtures/01-include-table.json: -------------------------------------------------------------------------------- 1 | { 2 | "Header": { 3 | "Schema": "test_db", 4 | "Table": "buildings", 5 | "BinlogMessageTime": "2017-04-13T06:34:30Z", 6 | "BinlogPosition": 397, 7 | "XId": 9 8 | }, 9 | "Type": "Insert", 10 | "Data": { 11 | "Row": { 12 | "address": "3950 North 1st Street CA 95134", 13 | "building_name": "ACME Headquaters", 14 | "building_no": 1 15 | }, 16 | "MappingNotice": "" 17 | } 18 | } 19 | { 20 | "Header": { 21 | "Schema": "test_db", 22 | "Table": "buildings", 23 | "BinlogMessageTime": "2017-04-13T06:34:30Z", 24 | "BinlogPosition": 397, 25 | "XId": 9 26 | }, 27 | "Type": "Insert", 28 | "Data": { 29 | "Row": { 30 | "address": "5000 North 1st Street CA 95134", 31 | "building_name": "ACME Sales", 32 | "building_no": 2 33 | }, 34 | "MappingNotice": "" 35 | } 36 | } 37 | { 38 | "Header": { 39 | "Schema": "test_db", 40 | "Table": "buildings", 41 | "BinlogMessageTime": "2017-04-13T06:35:36Z", 42 | "BinlogPosition": 1226, 43 | "XId": 14 44 | }, 45 | "Type": "Delete", 46 | "Data": { 47 | "Row": { 48 | "address": "3950 North 1st Street CA 95134", 49 | "building_name": "ACME Headquaters", 50 | "building_no": 1 51 | }, 52 | "MappingNotice": "" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /data/fixtures/01-no-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalora/binlog-parser/333ddecd365ede08e3f85f1ab39af6b14bc0d82d/data/fixtures/01-no-events.json -------------------------------------------------------------------------------- /data/fixtures/01.json: -------------------------------------------------------------------------------- 1 | { 2 | "Header": { 3 | "Schema": "test_db", 4 | "Table": "buildings", 5 | "BinlogMessageTime": "2017-04-13T06:34:30Z", 6 | "BinlogPosition": 397, 7 | "XId": 9 8 | }, 9 | "Type": "Insert", 10 | "Data": { 11 | "Row": { 12 | "address": "3950 North 1st Street CA 95134", 13 | "building_name": "ACME Headquaters", 14 | "building_no": 1 15 | }, 16 | "MappingNotice": "" 17 | } 18 | } 19 | { 20 | "Header": { 21 | "Schema": "test_db", 22 | "Table": "buildings", 23 | "BinlogMessageTime": "2017-04-13T06:34:30Z", 24 | "BinlogPosition": 397, 25 | "XId": 9 26 | }, 27 | "Type": "Insert", 28 | "Data": { 29 | "Row": { 30 | "address": "5000 North 1st Street CA 95134", 31 | "building_name": "ACME Sales", 32 | "building_no": 2 33 | }, 34 | "MappingNotice": "" 35 | } 36 | } 37 | { 38 | "Header": { 39 | "Schema": "test_db", 40 | "Table": "rooms", 41 | "BinlogMessageTime": "2017-04-13T06:34:37Z", 42 | "BinlogPosition": 692, 43 | "XId": 10 44 | }, 45 | "Type": "Insert", 46 | "Data": { 47 | "Row": { 48 | "building_no": 1, 49 | "room_name": "Amazon", 50 | "room_no": 1 51 | }, 52 | "MappingNotice": "" 53 | } 54 | } 55 | { 56 | "Header": { 57 | "Schema": "test_db", 58 | "Table": "rooms", 59 | "BinlogMessageTime": "2017-04-13T06:34:37Z", 60 | "BinlogPosition": 692, 61 | "XId": 10 62 | }, 63 | "Type": "Insert", 64 | "Data": { 65 | "Row": { 66 | "building_no": 1, 67 | "room_name": "War Room", 68 | "room_no": 2 69 | }, 70 | "MappingNotice": "" 71 | } 72 | } 73 | { 74 | "Header": { 75 | "Schema": "test_db", 76 | "Table": "rooms", 77 | "BinlogMessageTime": "2017-04-13T06:34:37Z", 78 | "BinlogPosition": 692, 79 | "XId": 10 80 | }, 81 | "Type": "Insert", 82 | "Data": { 83 | "Row": { 84 | "building_no": 1, 85 | "room_name": "Office of CEO", 86 | "room_no": 3 87 | }, 88 | "MappingNotice": "" 89 | } 90 | } 91 | { 92 | "Header": { 93 | "Schema": "test_db", 94 | "Table": "rooms", 95 | "BinlogMessageTime": "2017-04-13T06:34:37Z", 96 | "BinlogPosition": 692, 97 | "XId": 10 98 | }, 99 | "Type": "Insert", 100 | "Data": { 101 | "Row": { 102 | "building_no": 2, 103 | "room_name": "Marketing", 104 | "room_no": 4 105 | }, 106 | "MappingNotice": "" 107 | } 108 | } 109 | { 110 | "Header": { 111 | "Schema": "test_db", 112 | "Table": "rooms", 113 | "BinlogMessageTime": "2017-04-13T06:34:37Z", 114 | "BinlogPosition": 692, 115 | "XId": 10 116 | }, 117 | "Type": "Insert", 118 | "Data": { 119 | "Row": { 120 | "building_no": 2, 121 | "room_name": "Showroom", 122 | "room_no": 5 123 | }, 124 | "MappingNotice": "" 125 | } 126 | } 127 | { 128 | "Header": { 129 | "Schema": "test_db", 130 | "Table": "rooms", 131 | "BinlogMessageTime": "2017-04-13T06:34:58Z", 132 | "BinlogPosition": 967, 133 | "XId": 12 134 | }, 135 | "Type": "Update", 136 | "OldData": { 137 | "Row": { 138 | "building_no": 2, 139 | "room_name": "Marketing", 140 | "room_no": 4 141 | }, 142 | "MappingNotice": "" 143 | }, 144 | "NewData": { 145 | "Row": { 146 | "building_no": 2, 147 | "room_name": "MARKETING", 148 | "room_no": 4 149 | }, 150 | "MappingNotice": "" 151 | } 152 | } 153 | { 154 | "Header": { 155 | "Schema": "test_db", 156 | "Table": "rooms", 157 | "BinlogMessageTime": "2017-04-13T06:34:58Z", 158 | "BinlogPosition": 967, 159 | "XId": 12 160 | }, 161 | "Type": "Update", 162 | "OldData": { 163 | "Row": { 164 | "building_no": 2, 165 | "room_name": "Showroom", 166 | "room_no": 5 167 | }, 168 | "MappingNotice": "" 169 | }, 170 | "NewData": { 171 | "Row": { 172 | "building_no": 2, 173 | "room_name": "SHOWROOM", 174 | "room_no": 5 175 | }, 176 | "MappingNotice": "" 177 | } 178 | } 179 | { 180 | "Header": { 181 | "Schema": "test_db", 182 | "Table": "buildings", 183 | "BinlogMessageTime": "2017-04-13T06:35:36Z", 184 | "BinlogPosition": 1226, 185 | "XId": 14 186 | }, 187 | "Type": "Delete", 188 | "Data": { 189 | "Row": { 190 | "address": "3950 North 1st Street CA 95134", 191 | "building_name": "ACME Headquaters", 192 | "building_no": 1 193 | }, 194 | "MappingNotice": "" 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /data/fixtures/02.json: -------------------------------------------------------------------------------- 1 | { 2 | "Header": { 3 | "Schema": "test_db", 4 | "Table": "(unknown)", 5 | "BinlogMessageTime": "2017-04-13T08:01:35Z", 6 | "BinlogPosition": 432, 7 | "XId": 0 8 | }, 9 | "Type": "Query", 10 | "Query": "CREATE TABLE employees (\n emp_no INT UNSIGNED AUTO_INCREMENT NOT NULL,\n birth_date DATE NOT NULL,\n first_name VARCHAR(14) NOT NULL,\n last_name VARCHAR(16) NOT NULL,\n PRIMARY KEY (emp_no) \n)" 11 | } 12 | { 13 | "Header": { 14 | "Schema": "test_db", 15 | "Table": "employees", 16 | "BinlogMessageTime": "2017-04-13T08:02:04Z", 17 | "BinlogPosition": 635, 18 | "XId": 8 19 | }, 20 | "Type": "Insert", 21 | "Data": { 22 | "Row": { 23 | "(unknown_0)": 1, 24 | "(unknown_1)": "2017-04-13", 25 | "(unknown_2)": "Max", 26 | "(unknown_3)": "Mustermann" 27 | }, 28 | "MappingNotice": "column names array is missing field(s), will map them as unknown_*" 29 | } 30 | } 31 | { 32 | "Header": { 33 | "Schema": "test_db", 34 | "Table": "(unknown)", 35 | "BinlogMessageTime": "2017-04-13T08:02:17Z", 36 | "BinlogPosition": 794, 37 | "XId": 0 38 | }, 39 | "Type": "Query", 40 | "Query": "DROP TABLE `employees` /* generated by server */" 41 | } 42 | -------------------------------------------------------------------------------- /data/fixtures/03.json: -------------------------------------------------------------------------------- 1 | { 2 | "Header": { 3 | "Schema": "test_db", 4 | "Table": "buildings", 5 | "BinlogMessageTime": "2017-04-24T03:47:57Z", 6 | "BinlogPosition": 323, 7 | "XId": 9 8 | }, 9 | "Type": "Insert", 10 | "Data": { 11 | "Row": { 12 | "address": "bar", 13 | "building_name": "foo", 14 | "building_no": 3 15 | }, 16 | "MappingNotice": "" 17 | } 18 | } 19 | { 20 | "Header": { 21 | "Schema": "test_db", 22 | "Table": "buildings", 23 | "BinlogMessageTime": "2017-04-24T03:47:57Z", 24 | "BinlogPosition": 323, 25 | "XId": 9 26 | }, 27 | "Type": "Insert", 28 | "Data": { 29 | "Row": { 30 | "address": "qoo", 31 | "building_name": "baz", 32 | "building_no": 4 33 | }, 34 | "MappingNotice": "" 35 | } 36 | } 37 | { 38 | "Header": { 39 | "Schema": "test_db", 40 | "Table": "buildings", 41 | "BinlogMessageTime": "2017-04-24T03:50:14Z", 42 | "BinlogPosition": 560, 43 | "XId": 11 44 | }, 45 | "Type": "Update", 46 | "OldData": { 47 | "Row": { 48 | "address": "bar", 49 | "building_name": "foo", 50 | "building_no": 3 51 | }, 52 | "MappingNotice": "" 53 | }, 54 | "NewData": { 55 | "Row": { 56 | "address": "bar2", 57 | "building_name": "foo2", 58 | "building_no": 3 59 | }, 60 | "MappingNotice": "" 61 | } 62 | } 63 | { 64 | "Header": { 65 | "Schema": "test_db", 66 | "Table": "buildings", 67 | "BinlogMessageTime": "2017-04-24T03:50:23Z", 68 | "BinlogPosition": 797, 69 | "XId": 12 70 | }, 71 | "Type": "Update", 72 | "OldData": { 73 | "Row": { 74 | "address": "qoo", 75 | "building_name": "baz", 76 | "building_no": 4 77 | }, 78 | "MappingNotice": "" 79 | }, 80 | "NewData": { 81 | "Row": { 82 | "address": "qoo2", 83 | "building_name": "baz2", 84 | "building_no": 4 85 | }, 86 | "MappingNotice": "" 87 | } 88 | } 89 | { 90 | "Header": { 91 | "Schema": "test_db", 92 | "Table": "buildings", 93 | "BinlogMessageTime": "2017-04-24T03:50:35Z", 94 | "BinlogPosition": 1130, 95 | "XId": 13 96 | }, 97 | "Type": "Update", 98 | "OldData": { 99 | "Row": { 100 | "address": "5000 North 1st Street CA 95134", 101 | "building_name": "ACME Sales", 102 | "building_no": 2 103 | }, 104 | "MappingNotice": "" 105 | }, 106 | "NewData": { 107 | "Row": { 108 | "address": "", 109 | "building_name": "ACME Sales", 110 | "building_no": 2 111 | }, 112 | "MappingNotice": "" 113 | } 114 | } 115 | { 116 | "Header": { 117 | "Schema": "test_db", 118 | "Table": "buildings", 119 | "BinlogMessageTime": "2017-04-24T03:50:35Z", 120 | "BinlogPosition": 1130, 121 | "XId": 13 122 | }, 123 | "Type": "Update", 124 | "OldData": { 125 | "Row": { 126 | "address": "bar2", 127 | "building_name": "foo2", 128 | "building_no": 3 129 | }, 130 | "MappingNotice": "" 131 | }, 132 | "NewData": { 133 | "Row": { 134 | "address": "", 135 | "building_name": "foo2", 136 | "building_no": 3 137 | }, 138 | "MappingNotice": "" 139 | } 140 | } 141 | { 142 | "Header": { 143 | "Schema": "test_db", 144 | "Table": "buildings", 145 | "BinlogMessageTime": "2017-04-24T03:50:35Z", 146 | "BinlogPosition": 1130, 147 | "XId": 13 148 | }, 149 | "Type": "Update", 150 | "OldData": { 151 | "Row": { 152 | "address": "qoo2", 153 | "building_name": "baz2", 154 | "building_no": 4 155 | }, 156 | "MappingNotice": "" 157 | }, 158 | "NewData": { 159 | "Row": { 160 | "address": "", 161 | "building_name": "baz2", 162 | "building_no": 4 163 | }, 164 | "MappingNotice": "" 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /data/fixtures/05.json: -------------------------------------------------------------------------------- 1 | { 2 | "Header": { 3 | "Schema": "test_db", 4 | "Table": "(unknown)", 5 | "BinlogMessageTime": "2017-04-24T04:32:20Z", 6 | "BinlogPosition": 220, 7 | "XId": 0 8 | }, 9 | "Type": "Query", 10 | "Query": "DELETE FROM `test_db`.`filler`" 11 | } 12 | { 13 | "Header": { 14 | "Schema": "test_db", 15 | "Table": "(unknown)", 16 | "BinlogMessageTime": "2017-04-24T04:32:45Z", 17 | "BinlogPosition": 345, 18 | "XId": 0 19 | }, 20 | "Type": "Query", 21 | "Query": "DROP TABLE `filler` /* generated by server */" 22 | } 23 | { 24 | "Header": { 25 | "Schema": "test_db", 26 | "Table": "(unknown)", 27 | "BinlogMessageTime": "2017-04-24T04:32:50Z", 28 | "BinlogPosition": 470, 29 | "XId": 0 30 | }, 31 | "Type": "Query", 32 | "Query": "DROP TABLE `lookup` /* generated by server */" 33 | } 34 | -------------------------------------------------------------------------------- /data/fixtures/06.json: -------------------------------------------------------------------------------- 1 | { 2 | "Header": { 3 | "Schema": "test_db", 4 | "Table": "(unknown)", 5 | "BinlogMessageTime": "2017-04-24T05:44:21Z", 6 | "BinlogPosition": 220, 7 | "XId": 0 8 | }, 9 | "Type": "Query", 10 | "Query": "DELETE FROM `test_db`.`filler`" 11 | } 12 | { 13 | "Header": { 14 | "Schema": "test_db", 15 | "Table": "(unknown)", 16 | "BinlogMessageTime": "2017-04-24T05:44:44Z", 17 | "BinlogPosition": 589, 18 | "XId": 0 19 | }, 20 | "Type": "Query", 21 | "Query": "CREATE TABLE `language` (\n `language_id` tinyint(3) unsigned NOT NULL AUTO_INCREMENT,\n `name` char(20) NOT NULL,\n `last_update` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n PRIMARY KEY (`language_id`)\n) ENGINE=InnoDB AUTO_INCREMENT=70 DEFAULT CHARSET=utf8" 22 | } 23 | { 24 | "Header": { 25 | "Schema": "test_db", 26 | "Table": "language", 27 | "BinlogMessageTime": "2017-04-24T05:45:11Z", 28 | "BinlogPosition": 771, 29 | "XId": 11 30 | }, 31 | "Type": "Insert", 32 | "Data": { 33 | "Row": { 34 | "(unknown_0)": 70, 35 | "(unknown_1)": "German", 36 | "(unknown_2)": "2017-04-24 05:45:11" 37 | }, 38 | "MappingNotice": "row is missing field(s), ignoring missing" 39 | } 40 | } 41 | { 42 | "Header": { 43 | "Schema": "test_db", 44 | "Table": "(unknown)", 45 | "BinlogMessageTime": "2017-04-24T05:45:32Z", 46 | "BinlogPosition": 943, 47 | "XId": 0 48 | }, 49 | "Type": "Query", 50 | "Query": "alter table language add some_field varchar(255) default NULL" 51 | } 52 | { 53 | "Header": { 54 | "Schema": "test_db", 55 | "Table": "language", 56 | "BinlogMessageTime": "2017-04-24T05:45:41Z", 57 | "BinlogPosition": 1140, 58 | "XId": 13 59 | }, 60 | "Type": "Insert", 61 | "Data": { 62 | "Row": { 63 | "language_id": 71, 64 | "last_update": "2017-04-24 05:45:41", 65 | "name": "German", 66 | "some_field": "some value" 67 | }, 68 | "MappingNotice": "" 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /data/fixtures/07.json: -------------------------------------------------------------------------------- 1 | { 2 | "Header": { 3 | "Schema": "test_db", 4 | "Table": "(unknown)", 5 | "BinlogMessageTime": "2017-05-16T03:44:29Z", 6 | "BinlogPosition": 627, 7 | "XId": 0 8 | }, 9 | "Type": "Query", 10 | "Query": "CREATE TABLE `departments` (\n `dept_no` char(4) NOT NULL,\n `dept_name` varchar(40) NOT NULL,\n PRIMARY KEY (`dept_no`),\n UNIQUE KEY `dept_name` (`dept_name`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8" 11 | } 12 | { 13 | "Header": { 14 | "Schema": "test_db", 15 | "Table": "departments", 16 | "BinlogMessageTime": "2017-05-16T03:45:19Z", 17 | "BinlogPosition": 761, 18 | "XId": 456 19 | }, 20 | "Type": "Insert", 21 | "Data": { 22 | "Row": { 23 | "dept_name": "DEF", 24 | "dept_no": "ABC" 25 | }, 26 | "MappingNotice": "" 27 | } 28 | } 29 | { 30 | "Header": { 31 | "Schema": "test_db", 32 | "Table": "departments", 33 | "BinlogMessageTime": "2017-05-16T03:45:29Z", 34 | "BinlogPosition": 857, 35 | "XId": 456 36 | }, 37 | "Type": "Insert", 38 | "Data": { 39 | "Row": { 40 | "dept_name": "XYZ", 41 | "dept_no": "RST" 42 | }, 43 | "MappingNotice": "" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /data/fixtures/mysql-bin.01: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalora/binlog-parser/333ddecd365ede08e3f85f1ab39af6b14bc0d82d/data/fixtures/mysql-bin.01 -------------------------------------------------------------------------------- /data/fixtures/mysql-bin.02: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalora/binlog-parser/333ddecd365ede08e3f85f1ab39af6b14bc0d82d/data/fixtures/mysql-bin.02 -------------------------------------------------------------------------------- /data/fixtures/mysql-bin.03: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalora/binlog-parser/333ddecd365ede08e3f85f1ab39af6b14bc0d82d/data/fixtures/mysql-bin.03 -------------------------------------------------------------------------------- /data/fixtures/mysql-bin.04: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalora/binlog-parser/333ddecd365ede08e3f85f1ab39af6b14bc0d82d/data/fixtures/mysql-bin.04 -------------------------------------------------------------------------------- /data/fixtures/mysql-bin.05: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalora/binlog-parser/333ddecd365ede08e3f85f1ab39af6b14bc0d82d/data/fixtures/mysql-bin.05 -------------------------------------------------------------------------------- /data/fixtures/mysql-bin.06: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalora/binlog-parser/333ddecd365ede08e3f85f1ab39af6b14bc0d82d/data/fixtures/mysql-bin.06 -------------------------------------------------------------------------------- /data/fixtures/mysql-bin.07: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalora/binlog-parser/333ddecd365ede08e3f85f1ab39af6b14bc0d82d/data/fixtures/mysql-bin.07 -------------------------------------------------------------------------------- /data/fixtures/mysql-bin.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalora/binlog-parser/333ddecd365ede08e3f85f1ab39af6b14bc0d82d/data/fixtures/mysql-bin.empty -------------------------------------------------------------------------------- /data/fixtures/mysql-index-file.01: -------------------------------------------------------------------------------- 1 | /tmp/mysql-bin.000001 2 | -------------------------------------------------------------------------------- /data/fixtures/mysql-index-file.02: -------------------------------------------------------------------------------- 1 | /tmp/mysql-bin.000002 2 | /tmp/mysql-bin.000001 3 | -------------------------------------------------------------------------------- /data/fixtures/plaintext-mysql-bin.01: -------------------------------------------------------------------------------- 1 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; 2 | /*!40019 SET @@session.max_insert_delayed_threads=0*/; 3 | /*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; 4 | DELIMITER /*!*/; 5 | # at 4 6 | #170413 14:34:12 server id 1 end_log_pos 120 CRC32 0x8a1a4b5f Start: binlog v 4, server v 5.6.35-log created 170413 14:34:12 at startup 7 | # Warning: this binlog is either in use or was not closed properly. 8 | ROLLBACK/*!*/; 9 | # at 120 10 | #170413 14:34:30 server id 1 end_log_pos 197 CRC32 0xd97f04a0 Query thread_id=1 exec_time=0 error_code=0 11 | SET TIMESTAMP=1492065270/*!*/; 12 | SET @@session.pseudo_thread_id=1/*!*/; 13 | SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; 14 | SET @@session.sql_mode=1075838976/*!*/; 15 | SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; 16 | /*!\C utf8 *//*!*/; 17 | SET @@session.character_set_client=33,@@session.collation_connection=33,@@session.collation_server=33/*!*/; 18 | SET @@session.lc_time_names=0/*!*/; 19 | SET @@session.collation_database=DEFAULT/*!*/; 20 | BEGIN 21 | /*!*/; 22 | # at 197 23 | #170413 14:34:30 server id 1 end_log_pos 258 CRC32 0xb6f711f4 Table_map: `test_db`.`buildings` mapped to number 70 24 | # at 258 25 | #170413 14:34:30 server id 1 end_log_pos 397 CRC32 0x726311dc Write_rows: table id 70 flags: STMT_END_F 26 | ### INSERT INTO `test_db`.`buildings` 27 | ### SET 28 | ### @1=1 /* INT meta=0 nullable=0 is_null=0 */ 29 | ### @2='ACME Headquaters' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 30 | ### @3='3950 North 1st Street CA 95134' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 31 | ### INSERT INTO `test_db`.`buildings` 32 | ### SET 33 | ### @1=2 /* INT meta=0 nullable=0 is_null=0 */ 34 | ### @2='ACME Sales' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 35 | ### @3='5000 North 1st Street CA 95134' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 36 | # at 397 37 | #170413 14:34:30 server id 1 end_log_pos 428 CRC32 0xd2727e33 Xid = 9 38 | COMMIT/*!*/; 39 | # at 428 40 | #170413 14:34:37 server id 1 end_log_pos 503 CRC32 0xbe1dca2d Query thread_id=1 exec_time=0 error_code=0 41 | SET TIMESTAMP=1492065277/*!*/; 42 | BEGIN 43 | /*!*/; 44 | # at 503 45 | #170413 14:34:37 server id 1 end_log_pos 558 CRC32 0x9f3b72da Table_map: `test_db`.`rooms` mapped to number 71 46 | # at 558 47 | #170413 14:34:37 server id 1 end_log_pos 692 CRC32 0x46c0e76b Write_rows: table id 71 flags: STMT_END_F 48 | ### INSERT INTO `test_db`.`rooms` 49 | ### SET 50 | ### @1=1 /* INT meta=0 nullable=0 is_null=0 */ 51 | ### @2='Amazon' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 52 | ### @3=1 /* INT meta=0 nullable=0 is_null=0 */ 53 | ### INSERT INTO `test_db`.`rooms` 54 | ### SET 55 | ### @1=2 /* INT meta=0 nullable=0 is_null=0 */ 56 | ### @2='War Room' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 57 | ### @3=1 /* INT meta=0 nullable=0 is_null=0 */ 58 | ### INSERT INTO `test_db`.`rooms` 59 | ### SET 60 | ### @1=3 /* INT meta=0 nullable=0 is_null=0 */ 61 | ### @2='Office of CEO' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 62 | ### @3=1 /* INT meta=0 nullable=0 is_null=0 */ 63 | ### INSERT INTO `test_db`.`rooms` 64 | ### SET 65 | ### @1=4 /* INT meta=0 nullable=0 is_null=0 */ 66 | ### @2='Marketing' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 67 | ### @3=2 /* INT meta=0 nullable=0 is_null=0 */ 68 | ### INSERT INTO `test_db`.`rooms` 69 | ### SET 70 | ### @1=5 /* INT meta=0 nullable=0 is_null=0 */ 71 | ### @2='Showroom' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 72 | ### @3=2 /* INT meta=0 nullable=0 is_null=0 */ 73 | # at 692 74 | #170413 14:34:37 server id 1 end_log_pos 723 CRC32 0x2cdae9e4 Xid = 10 75 | COMMIT/*!*/; 76 | # at 723 77 | #170413 14:34:58 server id 1 end_log_pos 798 CRC32 0x4958330e Query thread_id=1 exec_time=0 error_code=0 78 | SET TIMESTAMP=1492065298/*!*/; 79 | BEGIN 80 | /*!*/; 81 | # at 798 82 | #170413 14:34:58 server id 1 end_log_pos 853 CRC32 0x5e3852c5 Table_map: `test_db`.`rooms` mapped to number 71 83 | # at 853 84 | #170413 14:34:58 server id 1 end_log_pos 967 CRC32 0xff57df46 Update_rows: table id 71 flags: STMT_END_F 85 | ### UPDATE `test_db`.`rooms` 86 | ### WHERE 87 | ### @1=4 /* INT meta=0 nullable=0 is_null=0 */ 88 | ### @2='Marketing' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 89 | ### @3=2 /* INT meta=0 nullable=0 is_null=0 */ 90 | ### SET 91 | ### @1=4 /* INT meta=0 nullable=0 is_null=0 */ 92 | ### @2='MARKETING' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 93 | ### @3=2 /* INT meta=0 nullable=0 is_null=0 */ 94 | ### UPDATE `test_db`.`rooms` 95 | ### WHERE 96 | ### @1=5 /* INT meta=0 nullable=0 is_null=0 */ 97 | ### @2='Showroom' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 98 | ### @3=2 /* INT meta=0 nullable=0 is_null=0 */ 99 | ### SET 100 | ### @1=5 /* INT meta=0 nullable=0 is_null=0 */ 101 | ### @2='SHOWROOM' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 102 | ### @3=2 /* INT meta=0 nullable=0 is_null=0 */ 103 | # at 967 104 | #170413 14:34:58 server id 1 end_log_pos 998 CRC32 0x50be55c1 Xid = 12 105 | COMMIT/*!*/; 106 | # at 998 107 | #170413 14:35:36 server id 1 end_log_pos 1075 CRC32 0xe278796f Query thread_id=1 exec_time=0 error_code=0 108 | SET TIMESTAMP=1492065336/*!*/; 109 | BEGIN 110 | /*!*/; 111 | # at 1075 112 | #170413 14:35:36 server id 1 end_log_pos 1136 CRC32 0xdb5a6fdf Table_map: `test_db`.`buildings` mapped to number 70 113 | # at 1136 114 | #170413 14:35:36 server id 1 end_log_pos 1226 CRC32 0xadfa4990 Delete_rows: table id 70 flags: STMT_END_F 115 | ### DELETE FROM `test_db`.`buildings` 116 | ### WHERE 117 | ### @1=1 /* INT meta=0 nullable=0 is_null=0 */ 118 | ### @2='ACME Headquaters' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 119 | ### @3='3950 North 1st Street CA 95134' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 120 | # at 1226 121 | #170413 14:35:36 server id 1 end_log_pos 1257 CRC32 0x9a0f56b0 Xid = 14 122 | COMMIT/*!*/; 123 | DELIMITER ; 124 | # End of log file 125 | ROLLBACK /* added by mysqlbinlog */; 126 | /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; 127 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; 128 | -------------------------------------------------------------------------------- /data/fixtures/plaintext-mysql-bin.02: -------------------------------------------------------------------------------- 1 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; 2 | /*!40019 SET @@session.max_insert_delayed_threads=0*/; 3 | /*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; 4 | DELIMITER /*!*/; 5 | # at 4 6 | #170413 16:01:28 server id 1 end_log_pos 120 CRC32 0x102bb1ad Start: binlog v 4, server v 5.6.35-log created 170413 16:01:28 at startup 7 | # Warning: this binlog is either in use or was not closed properly. 8 | ROLLBACK/*!*/; 9 | # at 120 10 | #170413 16:01:35 server id 1 end_log_pos 432 CRC32 0xfbfe5a0a Query thread_id=1 exec_time=0 error_code=0 11 | use `test_db`/*!*/; 12 | SET TIMESTAMP=1492070495/*!*/; 13 | SET @@session.pseudo_thread_id=1/*!*/; 14 | SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; 15 | SET @@session.sql_mode=1075838976/*!*/; 16 | SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; 17 | /*!\C utf8 *//*!*/; 18 | SET @@session.character_set_client=33,@@session.collation_connection=33,@@session.collation_server=33/*!*/; 19 | SET @@session.lc_time_names=0/*!*/; 20 | SET @@session.collation_database=DEFAULT/*!*/; 21 | CREATE TABLE employees ( 22 | emp_no INT UNSIGNED AUTO_INCREMENT NOT NULL, 23 | birth_date DATE NOT NULL, 24 | first_name VARCHAR(14) NOT NULL, 25 | last_name VARCHAR(16) NOT NULL, 26 | PRIMARY KEY (emp_no) 27 | ) 28 | /*!*/; 29 | # at 432 30 | #170413 16:02:04 server id 1 end_log_pos 515 CRC32 0xae57efdf Query thread_id=1 exec_time=0 error_code=0 31 | SET TIMESTAMP=1492070524/*!*/; 32 | SET @@session.time_zone='SYSTEM'/*!*/; 33 | BEGIN 34 | /*!*/; 35 | # at 515 36 | #170413 16:02:04 server id 1 end_log_pos 577 CRC32 0x88779ee8 Table_map: `test_db`.`employees` mapped to number 72 37 | # at 577 38 | #170413 16:02:04 server id 1 end_log_pos 635 CRC32 0x6adc3e28 Write_rows: table id 72 flags: STMT_END_F 39 | ### INSERT INTO `test_db`.`employees` 40 | ### SET 41 | ### @1=1 /* INT meta=0 nullable=0 is_null=0 */ 42 | ### @2='2017:04:13' /* DATE meta=0 nullable=0 is_null=0 */ 43 | ### @3='Max' /* VARSTRING(42) meta=42 nullable=0 is_null=0 */ 44 | ### @4='Mustermann' /* VARSTRING(48) meta=48 nullable=0 is_null=0 */ 45 | # at 635 46 | #170413 16:02:04 server id 1 end_log_pos 666 CRC32 0x5901ddf7 Xid = 8 47 | COMMIT/*!*/; 48 | # at 666 49 | #170413 16:02:17 server id 1 end_log_pos 794 CRC32 0xd9a3a7a6 Query thread_id=1 exec_time=0 error_code=0 50 | SET TIMESTAMP=1492070537/*!*/; 51 | DROP TABLE `employees` /* generated by server */ 52 | /*!*/; 53 | DELIMITER ; 54 | # End of log file 55 | ROLLBACK /* added by mysqlbinlog */; 56 | /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; 57 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; 58 | -------------------------------------------------------------------------------- /data/fixtures/plaintext-mysql-bin.03: -------------------------------------------------------------------------------- 1 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; 2 | /*!40019 SET @@session.max_insert_delayed_threads=0*/; 3 | /*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; 4 | DELIMITER /*!*/; 5 | # at 4 6 | #170424 11:46:45 server id 1 end_log_pos 120 CRC32 0x1a401052 Start: binlog v 4, server v 5.6.35-log created 170424 11:46:45 at startup 7 | # Warning: this binlog is either in use or was not closed properly. 8 | ROLLBACK/*!*/; 9 | # at 120 10 | #170424 11:47:57 server id 1 end_log_pos 197 CRC32 0x4c2d0f85 Query thread_id=1 exec_time=0 error_code=0 11 | SET TIMESTAMP=1493005677/*!*/; 12 | SET @@session.pseudo_thread_id=1/*!*/; 13 | SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; 14 | SET @@session.sql_mode=1075838976/*!*/; 15 | SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; 16 | /*!\C utf8 *//*!*/; 17 | SET @@session.character_set_client=33,@@session.collation_connection=33,@@session.collation_server=33/*!*/; 18 | SET @@session.lc_time_names=0/*!*/; 19 | SET @@session.collation_database=DEFAULT/*!*/; 20 | BEGIN 21 | /*!*/; 22 | # at 197 23 | #170424 11:47:57 server id 1 end_log_pos 258 CRC32 0x4c49efbd Table_map: `test_db`.`buildings` mapped to number 70 24 | # at 258 25 | #170424 11:47:57 server id 1 end_log_pos 323 CRC32 0x65b98d71 Write_rows: table id 70 flags: STMT_END_F 26 | ### INSERT INTO `test_db`.`buildings` 27 | ### SET 28 | ### @1=3 /* INT meta=0 nullable=0 is_null=0 */ 29 | ### @2='foo' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 30 | ### @3='bar' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 31 | ### INSERT INTO `test_db`.`buildings` 32 | ### SET 33 | ### @1=4 /* INT meta=0 nullable=0 is_null=0 */ 34 | ### @2='baz' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 35 | ### @3='qoo' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 36 | # at 323 37 | #170424 11:47:57 server id 1 end_log_pos 354 CRC32 0x83fe157b Xid = 9 38 | COMMIT/*!*/; 39 | # at 354 40 | #170424 11:50:14 server id 1 end_log_pos 431 CRC32 0xeaecdcc8 Query thread_id=1 exec_time=0 error_code=0 41 | SET TIMESTAMP=1493005814/*!*/; 42 | BEGIN 43 | /*!*/; 44 | # at 431 45 | #170424 11:50:14 server id 1 end_log_pos 492 CRC32 0x654f2c32 Table_map: `test_db`.`buildings` mapped to number 70 46 | # at 492 47 | #170424 11:50:14 server id 1 end_log_pos 560 CRC32 0x02580ceb Update_rows: table id 70 flags: STMT_END_F 48 | ### UPDATE `test_db`.`buildings` 49 | ### WHERE 50 | ### @1=3 /* INT meta=0 nullable=0 is_null=0 */ 51 | ### @2='foo' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 52 | ### @3='bar' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 53 | ### SET 54 | ### @1=3 /* INT meta=0 nullable=0 is_null=0 */ 55 | ### @2='foo2' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 56 | ### @3='bar2' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 57 | # at 560 58 | #170424 11:50:14 server id 1 end_log_pos 591 CRC32 0xb7db763c Xid = 11 59 | COMMIT/*!*/; 60 | # at 591 61 | #170424 11:50:23 server id 1 end_log_pos 668 CRC32 0xdc33923c Query thread_id=1 exec_time=0 error_code=0 62 | SET TIMESTAMP=1493005823/*!*/; 63 | BEGIN 64 | /*!*/; 65 | # at 668 66 | #170424 11:50:23 server id 1 end_log_pos 729 CRC32 0x794f3cf5 Table_map: `test_db`.`buildings` mapped to number 70 67 | # at 729 68 | #170424 11:50:23 server id 1 end_log_pos 797 CRC32 0xa1f6e5c1 Update_rows: table id 70 flags: STMT_END_F 69 | ### UPDATE `test_db`.`buildings` 70 | ### WHERE 71 | ### @1=4 /* INT meta=0 nullable=0 is_null=0 */ 72 | ### @2='baz' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 73 | ### @3='qoo' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 74 | ### SET 75 | ### @1=4 /* INT meta=0 nullable=0 is_null=0 */ 76 | ### @2='baz2' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 77 | ### @3='qoo2' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 78 | # at 797 79 | #170424 11:50:23 server id 1 end_log_pos 828 CRC32 0x5a88fdf5 Xid = 12 80 | COMMIT/*!*/; 81 | # at 828 82 | #170424 11:50:35 server id 1 end_log_pos 905 CRC32 0x415e1154 Query thread_id=1 exec_time=0 error_code=0 83 | SET TIMESTAMP=1493005835/*!*/; 84 | BEGIN 85 | /*!*/; 86 | # at 905 87 | #170424 11:50:35 server id 1 end_log_pos 966 CRC32 0x64a7bbf6 Table_map: `test_db`.`buildings` mapped to number 70 88 | # at 966 89 | #170424 11:50:35 server id 1 end_log_pos 1130 CRC32 0x4630893e Update_rows: table id 70 flags: STMT_END_F 90 | ### UPDATE `test_db`.`buildings` 91 | ### WHERE 92 | ### @1=2 /* INT meta=0 nullable=0 is_null=0 */ 93 | ### @2='ACME Sales' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 94 | ### @3='5000 North 1st Street CA 95134' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 95 | ### SET 96 | ### @1=2 /* INT meta=0 nullable=0 is_null=0 */ 97 | ### @2='ACME Sales' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 98 | ### @3='' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 99 | ### UPDATE `test_db`.`buildings` 100 | ### WHERE 101 | ### @1=3 /* INT meta=0 nullable=0 is_null=0 */ 102 | ### @2='foo2' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 103 | ### @3='bar2' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 104 | ### SET 105 | ### @1=3 /* INT meta=0 nullable=0 is_null=0 */ 106 | ### @2='foo2' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 107 | ### @3='' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 108 | ### UPDATE `test_db`.`buildings` 109 | ### WHERE 110 | ### @1=4 /* INT meta=0 nullable=0 is_null=0 */ 111 | ### @2='baz2' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 112 | ### @3='qoo2' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 113 | ### SET 114 | ### @1=4 /* INT meta=0 nullable=0 is_null=0 */ 115 | ### @2='baz2' /* VARSTRING(765) meta=765 nullable=0 is_null=0 */ 116 | ### @3='' /* VARSTRING(1065) meta=1065 nullable=0 is_null=0 */ 117 | # at 1130 118 | #170424 11:50:35 server id 1 end_log_pos 1161 CRC32 0x0a49be34 Xid = 13 119 | COMMIT/*!*/; 120 | DELIMITER ; 121 | # End of log file 122 | ROLLBACK /* added by mysqlbinlog */; 123 | /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; 124 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; 125 | -------------------------------------------------------------------------------- /data/fixtures/plaintext-mysql-bin.05: -------------------------------------------------------------------------------- 1 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; 2 | /*!40019 SET @@session.max_insert_delayed_threads=0*/; 3 | /*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; 4 | DELIMITER /*!*/; 5 | # at 4 6 | #170424 12:32:05 server id 1 end_log_pos 120 CRC32 0xc8a19f62 Start: binlog v 4, server v 5.6.35-log created 170424 12:32:05 at startup 7 | ROLLBACK/*!*/; 8 | # at 120 9 | #170424 12:32:20 server id 1 end_log_pos 220 CRC32 0x8cd4cf62 Query thread_id=1 exec_time=0 error_code=0 10 | use `test_db`/*!*/; 11 | SET TIMESTAMP=1493008340/*!*/; 12 | SET @@session.pseudo_thread_id=1/*!*/; 13 | SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; 14 | SET @@session.sql_mode=1075838976/*!*/; 15 | SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; 16 | /*!\C utf8 *//*!*/; 17 | SET @@session.character_set_client=33,@@session.collation_connection=33,@@session.collation_server=33/*!*/; 18 | SET @@session.lc_time_names=0/*!*/; 19 | SET @@session.collation_database=DEFAULT/*!*/; 20 | DELETE FROM `test_db`.`filler` 21 | /*!*/; 22 | # at 220 23 | #170424 12:32:45 server id 1 end_log_pos 345 CRC32 0x3813cd8b Query thread_id=1 exec_time=0 error_code=0 24 | SET TIMESTAMP=1493008365/*!*/; 25 | DROP TABLE `filler` /* generated by server */ 26 | /*!*/; 27 | # at 345 28 | #170424 12:32:50 server id 1 end_log_pos 470 CRC32 0x550323ac Query thread_id=1 exec_time=1 error_code=0 29 | SET TIMESTAMP=1493008370/*!*/; 30 | DROP TABLE `lookup` /* generated by server */ 31 | /*!*/; 32 | # at 470 33 | #170424 12:34:41 server id 1 end_log_pos 493 CRC32 0x617bf45a Stop 34 | DELIMITER ; 35 | # End of log file 36 | ROLLBACK /* added by mysqlbinlog */; 37 | /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; 38 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; 39 | -------------------------------------------------------------------------------- /data/fixtures/plaintext-mysql-bin.06: -------------------------------------------------------------------------------- 1 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; 2 | /*!40019 SET @@session.max_insert_delayed_threads=0*/; 3 | /*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; 4 | DELIMITER /*!*/; 5 | # at 4 6 | #170424 13:44:20 server id 1 end_log_pos 120 CRC32 0x79bdb602 Start: binlog v 4, server v 5.6.35-log created 170424 13:44:20 at startup 7 | # Warning: this binlog is either in use or was not closed properly. 8 | ROLLBACK/*!*/; 9 | # at 120 10 | #170424 13:44:21 server id 1 end_log_pos 220 CRC32 0x7a632689 Query thread_id=1 exec_time=0 error_code=0 11 | use `test_db`/*!*/; 12 | SET TIMESTAMP=1493012661/*!*/; 13 | SET @@session.pseudo_thread_id=1/*!*/; 14 | SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; 15 | SET @@session.sql_mode=1075838976/*!*/; 16 | SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; 17 | /*!\C utf8 *//*!*/; 18 | SET @@session.character_set_client=33,@@session.collation_connection=33,@@session.collation_server=33/*!*/; 19 | SET @@session.lc_time_names=0/*!*/; 20 | SET @@session.collation_database=DEFAULT/*!*/; 21 | DELETE FROM `test_db`.`filler` 22 | /*!*/; 23 | # at 220 24 | #170424 13:44:44 server id 1 end_log_pos 589 CRC32 0xd08b66eb Query thread_id=1 exec_time=0 error_code=0 25 | SET TIMESTAMP=1493012684/*!*/; 26 | CREATE TABLE `language` ( 27 | `language_id` tinyint(3) unsigned NOT NULL AUTO_INCREMENT, 28 | `name` char(20) NOT NULL, 29 | `last_update` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 30 | PRIMARY KEY (`language_id`) 31 | ) ENGINE=InnoDB AUTO_INCREMENT=70 DEFAULT CHARSET=utf8 32 | /*!*/; 33 | # at 589 34 | #170424 13:45:11 server id 1 end_log_pos 664 CRC32 0xc6a834e2 Query thread_id=1 exec_time=0 error_code=0 35 | SET TIMESTAMP=1493012711/*!*/; 36 | BEGIN 37 | /*!*/; 38 | # at 664 39 | #170424 13:45:11 server id 1 end_log_pos 723 CRC32 0x732c9cb2 Table_map: `test_db`.`language` mapped to number 75 40 | # at 723 41 | #170424 13:45:11 server id 1 end_log_pos 771 CRC32 0x55b09be5 Write_rows: table id 75 flags: STMT_END_F 42 | ### INSERT INTO `test_db`.`language` 43 | ### SET 44 | ### @1=70 /* TINYINT meta=0 nullable=0 is_null=0 */ 45 | ### @2='German' /* STRING(60) meta=65084 nullable=0 is_null=0 */ 46 | ### @3=1493012711 /* TIMESTAMP(0) meta=0 nullable=0 is_null=0 */ 47 | # at 771 48 | #170424 13:45:11 server id 1 end_log_pos 802 CRC32 0xb5d1cf03 Xid = 11 49 | COMMIT/*!*/; 50 | # at 802 51 | #170424 13:45:32 server id 1 end_log_pos 943 CRC32 0xcb92a71f Query thread_id=1 exec_time=0 error_code=0 52 | SET TIMESTAMP=1493012732/*!*/; 53 | alter table language add some_field varchar(255) default NULL 54 | /*!*/; 55 | # at 943 56 | #170424 13:45:41 server id 1 end_log_pos 1018 CRC32 0x9fc76245 Query thread_id=1 exec_time=0 error_code=0 57 | SET TIMESTAMP=1493012741/*!*/; 58 | BEGIN 59 | /*!*/; 60 | # at 1018 61 | #170424 13:45:41 server id 1 end_log_pos 1080 CRC32 0xf00825ee Table_map: `test_db`.`language` mapped to number 76 62 | # at 1080 63 | #170424 13:45:41 server id 1 end_log_pos 1140 CRC32 0x7d5048c5 Write_rows: table id 76 flags: STMT_END_F 64 | ### INSERT INTO `test_db`.`language` 65 | ### SET 66 | ### @1=71 /* TINYINT meta=0 nullable=0 is_null=0 */ 67 | ### @2='German' /* STRING(60) meta=65084 nullable=0 is_null=0 */ 68 | ### @3=1493012741 /* TIMESTAMP(0) meta=0 nullable=0 is_null=0 */ 69 | ### @4='some value' /* VARSTRING(765) meta=765 nullable=1 is_null=0 */ 70 | # at 1140 71 | #170424 13:45:41 server id 1 end_log_pos 1171 CRC32 0x011eb73d Xid = 13 72 | COMMIT/*!*/; 73 | DELIMITER ; 74 | # End of log file 75 | ROLLBACK /* added by mysqlbinlog */; 76 | /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; 77 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; 78 | -------------------------------------------------------------------------------- /data/fixtures/plaintext-mysql-bin.07: -------------------------------------------------------------------------------- 1 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; 2 | /*!40019 SET @@session.max_insert_delayed_threads=0*/; 3 | /*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; 4 | DELIMITER /*!*/; 5 | # at 4 6 | #170516 3:42:32 server id 3704 end_log_pos 248 Start: binlog v 4, server v 10.0.15-MariaDB-log created 170516 3:42:32 at startup 7 | # Warning: this binlog is either in use or was not closed properly. 8 | ROLLBACK/*!*/; 9 | # at 248 10 | #170516 3:42:32 server id 3704 end_log_pos 287 Gtid list [0-3704-2814] 11 | # at 287 12 | #170516 3:42:32 server id 3704 end_log_pos 326 Binlog checkpoint mysql-bin.000015 13 | # at 326 14 | #170516 3:44:29 server id 3704 end_log_pos 364 GTID 0-3704-2815 15 | /*!100001 SET @@session.gtid_domain_id=0*//*!*/; 16 | /*!100001 SET @@session.server_id=3704*//*!*/; 17 | /*!100001 SET @@session.gtid_seq_no=2815*//*!*/; 18 | # at 364 19 | #170516 3:44:29 server id 3704 end_log_pos 627 Query thread_id=3 exec_time=0 error_code=0 20 | use `test_db`/*!*/; 21 | SET TIMESTAMP=1494906269/*!*/; 22 | SET @@session.pseudo_thread_id=3/*!*/; 23 | SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; 24 | SET @@session.sql_mode=33554432/*!*/; 25 | SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; 26 | /*!\C utf8 *//*!*/; 27 | SET @@session.character_set_client=33,@@session.collation_connection=33,@@session.collation_server=33/*!*/; 28 | SET @@session.lc_time_names=0/*!*/; 29 | SET @@session.collation_database=DEFAULT/*!*/; 30 | CREATE TABLE `departments` ( 31 | `dept_no` char(4) NOT NULL, 32 | `dept_name` varchar(40) NOT NULL, 33 | PRIMARY KEY (`dept_no`), 34 | UNIQUE KEY `dept_name` (`dept_name`) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 36 | /*!*/; 37 | # at 627 38 | #170516 3:45:31 server id 3704 end_log_pos 665 GTID 0-3704-2816 39 | /*!100001 SET @@session.gtid_seq_no=2816*//*!*/; 40 | BEGIN 41 | /*!*/; 42 | # at 665 43 | # at 723 44 | #170516 3:45:19 server id 3704 end_log_pos 723 Table_map: `test_db`.`departments` mapped to number 515 45 | #170516 3:45:19 server id 3704 end_log_pos 761 Write_rows: table id 515 flags: STMT_END_F 46 | ### INSERT INTO `test_db`.`departments` 47 | ### SET 48 | ### @1='ABC' /* STRING(12) meta=65036 nullable=0 is_null=0 */ 49 | ### @2='DEF' /* VARSTRING(120) meta=120 nullable=0 is_null=0 */ 50 | # at 761 51 | # at 819 52 | #170516 3:45:29 server id 3704 end_log_pos 819 Table_map: `test_db`.`departments` mapped to number 515 53 | #170516 3:45:29 server id 3704 end_log_pos 857 Write_rows: table id 515 flags: STMT_END_F 54 | ### INSERT INTO `test_db`.`departments` 55 | ### SET 56 | ### @1='RST' /* STRING(12) meta=65036 nullable=0 is_null=0 */ 57 | ### @2='XYZ' /* VARSTRING(120) meta=120 nullable=0 is_null=0 */ 58 | # at 857 59 | #170516 3:45:31 server id 3704 end_log_pos 884 Xid = 456 60 | COMMIT/*!*/; 61 | DELIMITER ; 62 | # End of log file 63 | ROLLBACK /* added by mysqlbinlog */; 64 | /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; 65 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; 66 | -------------------------------------------------------------------------------- /data/fixtures/plaintext-mysql-bin.empty: -------------------------------------------------------------------------------- 1 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; 2 | /*!40019 SET @@session.max_insert_delayed_threads=0*/; 3 | /*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; 4 | DELIMITER /*!*/; 5 | # at 4 6 | #170419 14:33:16 server id 1 end_log_pos 120 CRC32 0x26d4b221 Start: binlog v 4, server v 5.6.35-log created 170419 14:33:16 at startup 7 | # Warning: this binlog is either in use or was not closed properly. 8 | ROLLBACK/*!*/; 9 | DELIMITER ; 10 | # End of log file 11 | ROLLBACK /* added by mysqlbinlog */; 12 | /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; 13 | /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; 14 | -------------------------------------------------------------------------------- /data/fixtures/test_db.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.6.35, for osx10.11 (x86_64) 2 | -- 3 | -- Host: localhost Database: test_db 4 | -- ------------------------------------------------------ 5 | -- Server version 5.6.35-log 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `test_db` 20 | -- 21 | 22 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `test_db` /*!40100 DEFAULT CHARACTER SET utf8 */; 23 | 24 | USE `test_db`; 25 | 26 | -- 27 | -- Table structure for table `buildings` 28 | -- 29 | 30 | DROP TABLE IF EXISTS `buildings`; 31 | /*!40101 SET @saved_cs_client = @@character_set_client */; 32 | /*!40101 SET character_set_client = utf8 */; 33 | CREATE TABLE `buildings` ( 34 | `building_no` int(11) NOT NULL AUTO_INCREMENT, 35 | `building_name` varchar(255) NOT NULL, 36 | `address` varchar(355) NOT NULL, 37 | PRIMARY KEY (`building_no`) 38 | ) ENGINE=InnoDB AUTO_INCREMENT=5 DEFAULT CHARSET=utf8; 39 | /*!40101 SET character_set_client = @saved_cs_client */; 40 | 41 | -- 42 | -- Table structure for table `departments` 43 | -- 44 | 45 | DROP TABLE IF EXISTS `departments`; 46 | /*!40101 SET @saved_cs_client = @@character_set_client */; 47 | /*!40101 SET character_set_client = utf8 */; 48 | CREATE TABLE `departments` ( 49 | `dept_no` char(4) NOT NULL, 50 | `dept_name` varchar(40) NOT NULL, 51 | PRIMARY KEY (`dept_no`), 52 | UNIQUE KEY `dept_name` (`dept_name`) 53 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 54 | /*!40101 SET character_set_client = @saved_cs_client */; 55 | 56 | -- 57 | -- Table structure for table `filler` 58 | -- 59 | 60 | DROP TABLE IF EXISTS `filler`; 61 | /*!40101 SET @saved_cs_client = @@character_set_client */; 62 | /*!40101 SET character_set_client = utf8 */; 63 | CREATE TABLE `filler` ( 64 | `id` int(11) NOT NULL AUTO_INCREMENT, 65 | PRIMARY KEY (`id`) 66 | ) ENGINE=MEMORY DEFAULT CHARSET=utf8; 67 | /*!40101 SET character_set_client = @saved_cs_client */; 68 | 69 | -- 70 | -- Table structure for table `language` 71 | -- 72 | 73 | DROP TABLE IF EXISTS `language`; 74 | /*!40101 SET @saved_cs_client = @@character_set_client */; 75 | /*!40101 SET character_set_client = utf8 */; 76 | CREATE TABLE `language` ( 77 | `language_id` tinyint(3) unsigned NOT NULL AUTO_INCREMENT, 78 | `name` char(20) NOT NULL, 79 | `last_update` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 80 | `some_field` varchar(255) DEFAULT NULL, 81 | PRIMARY KEY (`language_id`) 82 | ) ENGINE=InnoDB AUTO_INCREMENT=72 DEFAULT CHARSET=utf8; 83 | /*!40101 SET character_set_client = @saved_cs_client */; 84 | 85 | -- 86 | -- Table structure for table `lookup` 87 | -- 88 | 89 | DROP TABLE IF EXISTS `lookup`; 90 | /*!40101 SET @saved_cs_client = @@character_set_client */; 91 | /*!40101 SET character_set_client = utf8 */; 92 | CREATE TABLE `lookup` ( 93 | `id` int(11) NOT NULL, 94 | `value` int(11) NOT NULL, 95 | `shorttxt` text NOT NULL, 96 | `longtxt` text NOT NULL, 97 | PRIMARY KEY (`id`) 98 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 99 | /*!40101 SET character_set_client = @saved_cs_client */; 100 | 101 | -- 102 | -- Table structure for table `rooms` 103 | -- 104 | 105 | DROP TABLE IF EXISTS `rooms`; 106 | /*!40101 SET @saved_cs_client = @@character_set_client */; 107 | /*!40101 SET character_set_client = utf8 */; 108 | CREATE TABLE `rooms` ( 109 | `room_no` int(11) NOT NULL AUTO_INCREMENT, 110 | `room_name` varchar(255) NOT NULL, 111 | `building_no` int(11) NOT NULL, 112 | PRIMARY KEY (`room_no`), 113 | KEY `building_no` (`building_no`), 114 | CONSTRAINT `rooms_ibfk_1` FOREIGN KEY (`building_no`) REFERENCES `buildings` (`building_no`) ON DELETE CASCADE 115 | ) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8; 116 | /*!40101 SET character_set_client = @saved_cs_client */; 117 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 118 | 119 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 120 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 121 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 122 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 123 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 124 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 125 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 126 | 127 | -- Dump completed on 2017-04-24 13:52:40 128 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/database/database_error.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | ) 7 | 8 | type ConnectionError struct { 9 | errorMessage string 10 | } 11 | 12 | func newConnectionError(err error) ConnectionError { 13 | return ConnectionError{fmt.Sprintf("Connection error %s %s", reflect.TypeOf(err), err)} 14 | } 15 | 16 | func (e *ConnectionError) Error() string { 17 | return e.errorMessage 18 | } 19 | 20 | type QueryError struct { 21 | errorMessage string 22 | } 23 | 24 | func newQueryError(err error) QueryError { 25 | return QueryError{fmt.Sprintf("Query error %s %s", reflect.TypeOf(err), err)} 26 | } 27 | 28 | func (e *QueryError) Error() string { 29 | return e.errorMessage 30 | } 31 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/database/db_instance.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "database/sql" 5 | _ "github.com/go-sql-driver/mysql" 6 | ) 7 | 8 | func GetDatabaseInstance(connectionString string) (*sql.DB, error) { 9 | db, err := sql.Open("mysql", connectionString) 10 | 11 | if err != nil { 12 | c := newConnectionError(err) 13 | return nil, &c 14 | } 15 | 16 | err = db.Ping() 17 | 18 | if err != nil { 19 | c := newConnectionError(err) 20 | return nil, &c 21 | } 22 | 23 | return db, nil 24 | } 25 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/database/table_map.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | ) 7 | 8 | type TableMetadata struct { 9 | Schema string 10 | Table string 11 | Fields map[int]string 12 | } 13 | 14 | type TableMap struct { 15 | tableMetadataMap map[uint64]TableMetadata 16 | fieldsCache map[string]map[int]string 17 | db *sql.DB 18 | } 19 | 20 | func NewTableMap(db *sql.DB) TableMap { 21 | return TableMap{ 22 | db: db, 23 | tableMetadataMap: make(map[uint64]TableMetadata), 24 | fieldsCache: make(map[string]map[int]string), 25 | } 26 | } 27 | 28 | func (m *TableMap) Add(id uint64, schema, table string) error { 29 | fields, err := m.getFields(schema, table) 30 | 31 | if err != nil { 32 | return err 33 | } 34 | 35 | m.tableMetadataMap[id] = TableMetadata{schema, table, fields} 36 | 37 | return nil 38 | } 39 | 40 | func (m *TableMap) LookupTableMetadata(id uint64) (TableMetadata, bool) { 41 | val, ok := m.tableMetadataMap[id] 42 | return val, ok 43 | } 44 | 45 | func (m *TableMap) getFields(schema, table string) (map[int]string, error) { 46 | cacheKey := fmt.Sprintf("%s_%s", schema, table) 47 | 48 | if cachedFields, ok := m.fieldsCache[cacheKey]; ok { 49 | return cachedFields, nil 50 | } 51 | 52 | fields, err := getFieldsFromDb(m.db, schema, table) 53 | m.fieldsCache[cacheKey] = fields 54 | 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | return fields, nil 60 | } 61 | 62 | func getFieldsFromDb(db *sql.DB, schema string, table string) (map[int]string, error) { 63 | rows, err := db.Query( 64 | "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION", 65 | schema, 66 | table, 67 | ) 68 | 69 | if err != nil { 70 | q := newQueryError(err) 71 | return nil, &q 72 | } 73 | 74 | defer rows.Close() 75 | 76 | fields := make(map[int]string) 77 | i := 0 78 | 79 | var columnName string 80 | for rows.Next() { 81 | err := rows.Scan(&columnName) 82 | 83 | if err != nil { 84 | q := newQueryError(err) 85 | return nil, &q 86 | } 87 | 88 | fields[i] = columnName 89 | i++ 90 | } 91 | 92 | return fields, nil 93 | } 94 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/database/table_map_integration_test.go: -------------------------------------------------------------------------------- 1 | // +build integration 2 | 3 | package database 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | "reflect" 9 | "testing" 10 | ) 11 | 12 | func TestLookupTableMetadata(t *testing.T) { 13 | db, _ := GetDatabaseInstance(os.Getenv("TEST_DB_DSN")) 14 | defer db.Close() 15 | 16 | t.Run("Found", func(t *testing.T) { 17 | tableMap := NewTableMap(db) 18 | tableMap.Add(1, "test_db", "buildings") 19 | tableMap.Add(2, "test_db", "rooms") 20 | 21 | assertTableMetadata(t, &tableMap, 1, "test_db", "buildings") 22 | assertTableMetadata(t, &tableMap, 2, "test_db", "rooms") 23 | }) 24 | 25 | t.Run("Fields", func(t *testing.T) { 26 | tableMap := NewTableMap(db) 27 | tableMap.Add(1, "test_db", "buildings") 28 | 29 | tableMetadata, ok := tableMap.LookupTableMetadata(1) 30 | 31 | if ok != true { 32 | t.Fatal("Expected table metadata to be found") 33 | } 34 | 35 | expectedFields := map[int]string{ 36 | 0: "building_no", 37 | 1: "building_name", 38 | 2: "address", 39 | } 40 | 41 | if !reflect.DeepEqual(tableMetadata.Fields, expectedFields) { 42 | t.Fatal("Wrong fields in table metadata") 43 | } 44 | }) 45 | 46 | t.Run("Not Found", func(t *testing.T) { 47 | tableMap := NewTableMap(db) 48 | _, ok := tableMap.LookupTableMetadata(999) 49 | 50 | if ok != false { 51 | t.Fatal("Expected table metadata not to be found") 52 | } 53 | }) 54 | 55 | } 56 | 57 | func assertTableMetadata(t *testing.T, tableMap *TableMap, tableId uint64, expectedSchema string, expectedTable string) { 58 | tableMetadata, ok := tableMap.LookupTableMetadata(tableId) 59 | 60 | if ok != true { 61 | t.Fatal(fmt.Sprintf("metadata not found for table id %d", tableId)) 62 | } 63 | 64 | if tableMetadata.Schema != expectedSchema { 65 | t.Fatal(fmt.Sprintf("wrong schema name for table id %d", tableId)) 66 | } 67 | 68 | if tableMetadata.Table != expectedTable { 69 | t.Fatal(fmt.Sprintf("wrong table name for table id %d", tableId)) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/golang/glog" 7 | "os" 8 | "path" 9 | "strings" 10 | "zalora/binlog-parser/parser" 11 | ) 12 | 13 | var prettyPrintJsonFlag = flag.Bool("prettyprint", false, "Pretty print json") 14 | var includeTablesFlag = flag.String("include_tables", "", "comma-separated list of tables to include") 15 | var includeSchemasFlag = flag.String("include_schemas", "", "comma-separated list of schemas to include") 16 | 17 | func main() { 18 | flag.Usage = func() { 19 | printUsage() 20 | } 21 | 22 | flag.Parse() 23 | 24 | if flag.NArg() != 1 { 25 | printUsage() 26 | os.Exit(1) 27 | } 28 | 29 | binlogFilename := flag.Arg(0) 30 | dbDsn := os.Getenv("DB_DSN") 31 | 32 | if dbDsn == "" { 33 | fmt.Fprint(os.Stderr, "Please set env variable DB_DSN to a valid MySQL connection string") 34 | os.Exit(1) 35 | } 36 | 37 | glog.V(1).Infof("Will parse file %s", binlogFilename) 38 | 39 | parseFunc := createBinlogParseFunc(dbDsn, consumerChainFromArgs()) 40 | err := parseFunc(binlogFilename) 41 | 42 | if err != nil { 43 | fmt.Fprintf(os.Stderr, "Got error: %s\n", err) 44 | os.Exit(1) 45 | } 46 | } 47 | 48 | func consumerChainFromArgs() parser.ConsumerChain { 49 | chain := parser.NewConsumerChain() 50 | 51 | chain.CollectAsJson(os.Stdout, *prettyPrintJsonFlag) 52 | glog.V(1).Infof("Pretty print JSON %s", *prettyPrintJsonFlag) 53 | 54 | if *includeTablesFlag != "" { 55 | includeTables := commaSeparatedListToArray(*includeTablesFlag) 56 | 57 | chain.IncludeTables(includeTables...) 58 | glog.V(1).Infof("Including tables %v", includeTables) 59 | } 60 | 61 | if *includeSchemasFlag != "" { 62 | includeSchemas := commaSeparatedListToArray(*includeSchemasFlag) 63 | 64 | chain.IncludeSchemas(includeSchemas...) 65 | glog.V(1).Infof("Including schemas %v", includeSchemas) 66 | } 67 | 68 | return chain 69 | } 70 | 71 | func printUsage() { 72 | binName := path.Base(os.Args[0]) 73 | 74 | usage := "Parse a binlog file, dump JSON to stdout. Includes options to filter by schema and table.\n" + 75 | "Reads from information_schema database to find out the field names for a row event.\n\n" + 76 | "Usage:\t%s [options ...] binlog\n\n" + 77 | "Options are:\n\n" 78 | 79 | fmt.Fprintf(os.Stderr, usage, binName) 80 | 81 | flag.PrintDefaults() 82 | 83 | envVars := "\nRequired environment variables:\n\n" + 84 | "DB_DSN\t Database connection string, needs read access to information_schema\n" 85 | 86 | fmt.Fprint(os.Stderr, envVars) 87 | } 88 | 89 | func commaSeparatedListToArray(str string) []string { 90 | var arr []string 91 | 92 | for _, item := range strings.Split(str, ",") { 93 | item = strings.TrimSpace(item) 94 | 95 | if item != "" { 96 | arr = append(arr, item) 97 | } 98 | } 99 | 100 | return arr 101 | } 102 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parse_binlog_file.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "zalora/binlog-parser/database" 6 | "zalora/binlog-parser/parser" 7 | ) 8 | 9 | type binlogParseFunc func(string) error 10 | 11 | func createBinlogParseFunc(dbDsn string, consumerChain parser.ConsumerChain) binlogParseFunc { 12 | return func(binlogFilename string) error { 13 | return parseBinlogFile(binlogFilename, dbDsn, consumerChain) 14 | } 15 | } 16 | 17 | func parseBinlogFile(binlogFilename, dbDsn string, consumerChain parser.ConsumerChain) error { 18 | glog.V(2).Infof("Parsing binlog file %s", binlogFilename) 19 | 20 | db, err := database.GetDatabaseInstance(dbDsn) 21 | 22 | if err != nil { 23 | return err 24 | } 25 | 26 | defer db.Close() 27 | 28 | tableMap := database.NewTableMap(db) 29 | 30 | glog.V(2).Info("About to parse file ...") 31 | 32 | return parser.ParseBinlog(binlogFilename, tableMap, consumerChain) 33 | } 34 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parse_binlog_file_integration_test.go: -------------------------------------------------------------------------------- 1 | // +build integration 2 | 3 | package main 4 | 5 | import ( 6 | "bytes" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | "testing" 14 | "zalora/binlog-parser/parser" 15 | ) 16 | 17 | func TestParseBinlogFile(t *testing.T) { 18 | dataDir := os.Getenv("DATA_DIR") 19 | 20 | createConsumerChain := func(stream io.Writer) parser.ConsumerChain { 21 | chain := parser.NewConsumerChain() 22 | chain.CollectAsJson(stream, true) 23 | 24 | return chain 25 | } 26 | 27 | t.Run("binlog file not found", func(t *testing.T) { 28 | tmpfile, _ := ioutil.TempFile("", "test") 29 | defer os.RemoveAll(tmpfile.Name()) 30 | 31 | err := parseBinlogFile("/not/there", os.Getenv("TEST_DB_DSN"), createConsumerChain(tmpfile)) 32 | 33 | if err == nil { 34 | t.Fatal("Expected error when parsing non-existing file") 35 | } 36 | }) 37 | 38 | testCases := []struct { 39 | fixtureFilename string 40 | expectedJsonFile string 41 | includeTables []string 42 | includeSchemas []string 43 | }{ 44 | {"fixtures/mysql-bin.01", "fixtures/01.json", nil, nil}, // inserts and updates 45 | {"fixtures/mysql-bin.02", "fixtures/02.json", nil, nil}, // create table, insert 46 | {"fixtures/mysql-bin.03", "fixtures/03.json", nil, nil}, // insert 2 rows, update 2 rows, update 3 rows 47 | {"fixtures/mysql-bin.04", "fixtures/04.json", nil, nil}, // large insert (1000) 48 | {"fixtures/mysql-bin.05", "fixtures/05.json", nil, nil}, // DROP TABLE ... queries only 49 | {"fixtures/mysql-bin.06", "fixtures/06.json", nil, nil}, // table schema doesn't match anymore 50 | {"fixtures/mysql-bin.07", "fixtures/07.json", nil, nil}, // mariadb format, create table, insert two rows 51 | {"fixtures/mysql-bin.01", "fixtures/01-include-table.json", []string{"buildings"}, nil}, // include tables 52 | {"fixtures/mysql-bin.01", "fixtures/01-no-events.json", []string{"unknown_table"}, nil}, // only unknown table is included - no events parsed 53 | {"fixtures/mysql-bin.01", "fixtures/01.json", nil, []string{"test_db"}}, // inlcude schemas 54 | {"fixtures/mysql-bin.01", "fixtures/01-no-events.json", nil, []string{"unknown_schema"}}, // only unknown schema is included - no events parsed 55 | } 56 | 57 | for _, tc := range testCases { 58 | t.Run(fmt.Sprintf("Parse binlog %s", tc.fixtureFilename), func(t *testing.T) { 59 | var buffer bytes.Buffer 60 | binlogFilename := filepath.Join(dataDir, tc.fixtureFilename) 61 | 62 | chain := createConsumerChain(&buffer) 63 | 64 | if tc.includeTables != nil { 65 | chain.IncludeTables(tc.includeTables...) 66 | } 67 | 68 | if tc.includeSchemas != nil { 69 | chain.IncludeSchemas(tc.includeSchemas...) 70 | } 71 | 72 | err := parseBinlogFile(binlogFilename, os.Getenv("TEST_DB_DSN"), chain) 73 | 74 | if err != nil { 75 | t.Fatal(fmt.Sprintf("Expected no error when successfully parsing file %s", err)) 76 | } 77 | 78 | assertJson(t, buffer, filepath.Join(dataDir, tc.expectedJsonFile)) 79 | }) 80 | } 81 | } 82 | 83 | func assertJson(t *testing.T, buffer bytes.Buffer, expectedJsonFile string) { 84 | expectedJson, err := ioutil.ReadFile(expectedJsonFile) 85 | 86 | if err != nil { 87 | t.Fatal(fmt.Sprintf("Failed to open expected JSON file: %s", err)) 88 | } 89 | 90 | expected := strings.TrimSpace(string(expectedJson)) 91 | actual := strings.TrimSpace(buffer.String()) 92 | 93 | if expected != actual { 94 | errorMessage := fmt.Sprintf( 95 | "JSON file %s does not match\nExpected:\n==========\n%s\n==========\nActual generated:\n%s\n==========", 96 | expectedJsonFile, 97 | expected, 98 | actual, 99 | ) 100 | 101 | t.Fatal(errorMessage) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/binlog_parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "os" 5 | "zalora/binlog-parser/database" 6 | "zalora/binlog-parser/parser/parser" 7 | ) 8 | 9 | func ParseBinlog(binlogFilename string, tableMap database.TableMap, consumerChain ConsumerChain) error { 10 | if _, err := os.Stat(binlogFilename); os.IsNotExist(err) { 11 | return err 12 | } 13 | 14 | return parser.ParseBinlogToMessages(binlogFilename, tableMap, consumerChain.consumeMessage) 15 | } 16 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/consumer_chain.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/golang/glog" 7 | "io" 8 | "zalora/binlog-parser/parser/messages" 9 | ) 10 | 11 | type ConsumerChain struct { 12 | predicates []predicate 13 | collectors []collector 14 | prettyPrint bool 15 | } 16 | 17 | type predicate func(message messages.Message) bool 18 | 19 | type collector func(message messages.Message) error 20 | 21 | func NewConsumerChain() ConsumerChain { 22 | return ConsumerChain{} 23 | } 24 | 25 | func (c *ConsumerChain) IncludeTables(tables ...string) { 26 | c.predicates = append(c.predicates, tablesPredicate(tables...)) 27 | } 28 | 29 | func (c *ConsumerChain) IncludeSchemas(schemas ...string) { 30 | c.predicates = append(c.predicates, schemaPredicate(schemas...)) 31 | } 32 | 33 | func (c *ConsumerChain) PrettyPrint(prettyPrint bool) { 34 | c.prettyPrint = prettyPrint 35 | } 36 | 37 | func (c *ConsumerChain) CollectAsJson(stream io.Writer, prettyPrint bool) { 38 | c.collectors = append(c.collectors, streamCollector(stream, prettyPrint)) 39 | } 40 | 41 | func (c *ConsumerChain) consumeMessage(message messages.Message) error { 42 | for _, predicate := range c.predicates { 43 | pass := predicate(message) 44 | 45 | if !pass { 46 | return nil 47 | } 48 | } 49 | 50 | for _, collector := range c.collectors { 51 | collector_err := collector(message) 52 | 53 | if collector_err != nil { 54 | return collector_err 55 | } 56 | } 57 | 58 | return nil 59 | } 60 | 61 | func streamCollector(stream io.Writer, prettyPrint bool) collector { 62 | return func(message messages.Message) error { 63 | json, err := marshalMessage(message, prettyPrint) 64 | 65 | if err != nil { 66 | glog.Errorf("Failed to convert message to JSON: %s", err) 67 | return err 68 | } 69 | 70 | n, err := stream.Write([]byte(fmt.Sprintf("%s\n", json))) 71 | 72 | if err != nil { 73 | glog.Errorf("Failed to write message JSON to file %s", err) 74 | return err 75 | } 76 | 77 | glog.V(1).Infof("Wrote %d bytes to stream", n) 78 | 79 | return nil 80 | } 81 | } 82 | 83 | func schemaPredicate(databases ...string) predicate { 84 | return func(message messages.Message) bool { 85 | if message.GetHeader().Schema == "" { 86 | return true 87 | } 88 | 89 | return contains(databases, message.GetHeader().Schema) 90 | } 91 | } 92 | 93 | func tablesPredicate(tables ...string) predicate { 94 | return func(message messages.Message) bool { 95 | if message.GetHeader().Table == "" { 96 | return true 97 | } 98 | 99 | return contains(tables, message.GetHeader().Table) 100 | } 101 | } 102 | 103 | func marshalMessage(message messages.Message, prettyPrint bool) ([]byte, error) { 104 | if prettyPrint { 105 | return json.MarshalIndent(message, "", " ") 106 | } 107 | 108 | return json.Marshal(message) 109 | } 110 | 111 | func contains(s []string, e string) bool { 112 | for _, a := range s { 113 | if a == e { 114 | return true 115 | } 116 | } 117 | 118 | return false 119 | } 120 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/consumer_chain_test.go: -------------------------------------------------------------------------------- 1 | // +build unit 2 | 3 | package parser 4 | 5 | import ( 6 | "io/ioutil" 7 | "os" 8 | "testing" 9 | "time" 10 | "zalora/binlog-parser/parser/messages" 11 | ) 12 | 13 | func TestConsumerChain(t *testing.T) { 14 | messageOne := messages.NewQueryMessage( 15 | messages.NewMessageHeader("database_name", "table_name", time.Now(), 100, 100), 16 | messages.SqlQuery("SELECT * FROM table"), 17 | ) 18 | 19 | messageTwo := messages.NewQueryMessage( 20 | messages.NewMessageHeader("database_name", "table_name", time.Now(), 100, 100), 21 | messages.SqlQuery("SELECT * FROM table"), 22 | ) 23 | 24 | t.Run("No predicates, no collectors", func(t *testing.T) { 25 | chain := NewConsumerChain() 26 | err := chain.consumeMessage(messageOne) 27 | 28 | if err != nil { 29 | t.Fatal("Failed to consume message") 30 | } 31 | }) 32 | 33 | t.Run("Collect as JSON file", func(t *testing.T) { 34 | tmpfile, _ := ioutil.TempFile("", "messages.json") 35 | defer os.Remove(tmpfile.Name()) 36 | 37 | chain := NewConsumerChain() 38 | chain.CollectAsJson(tmpfile, true) 39 | 40 | err := chain.consumeMessage(messageOne) 41 | 42 | if err != nil { 43 | t.Fatal("Failed to consume message") 44 | } 45 | 46 | assertJsonOutputNotEmpty(t, tmpfile) 47 | }) 48 | 49 | t.Run("Filter schema, passes through", func(t *testing.T) { 50 | tmpfile, _ := ioutil.TempFile("", "messages.json") 51 | defer os.Remove(tmpfile.Name()) 52 | 53 | chain := NewConsumerChain() 54 | chain.CollectAsJson(tmpfile, true) 55 | chain.IncludeSchemas("some_db", "database_name") 56 | 57 | err := chain.consumeMessage(messageTwo) 58 | 59 | if err != nil { 60 | t.Fatal("Failed to consume message") 61 | } 62 | 63 | assertJsonOutputNotEmpty(t, tmpfile) 64 | }) 65 | 66 | t.Run("Filter schema, filtered out", func(t *testing.T) { 67 | tmpfile, _ := ioutil.TempFile("", "messages.json") 68 | defer os.Remove(tmpfile.Name()) 69 | 70 | chain := NewConsumerChain() 71 | chain.CollectAsJson(tmpfile, true) 72 | chain.IncludeSchemas("some_db") 73 | 74 | err := chain.consumeMessage(messageTwo) 75 | 76 | if err != nil { 77 | t.Fatal("Failed to consume message") 78 | } 79 | 80 | assertJsonOutputEmpty(t, tmpfile) 81 | }) 82 | 83 | t.Run("Filter table, passes through", func(t *testing.T) { 84 | tmpfile, _ := ioutil.TempFile("", "messages.json") 85 | defer os.Remove(tmpfile.Name()) 86 | 87 | chain := NewConsumerChain() 88 | chain.CollectAsJson(tmpfile, true) 89 | chain.IncludeTables("some_table", "table_name") 90 | 91 | err := chain.consumeMessage(messageTwo) 92 | 93 | if err != nil { 94 | t.Fatal("Failed to consume message") 95 | } 96 | 97 | assertJsonOutputNotEmpty(t, tmpfile) 98 | }) 99 | 100 | t.Run("Filter table, filtered out", func(t *testing.T) { 101 | tmpfile, _ := ioutil.TempFile("", "messages.json") 102 | defer os.Remove(tmpfile.Name()) 103 | 104 | chain := NewConsumerChain() 105 | chain.IncludeTables("some_table") 106 | chain.CollectAsJson(tmpfile, true) 107 | 108 | err := chain.consumeMessage(messageTwo) 109 | 110 | if err != nil { 111 | t.Fatal("Failed to consume message") 112 | } 113 | 114 | assertJsonOutputEmpty(t, tmpfile) 115 | }) 116 | } 117 | 118 | func assertJsonOutputNotEmpty(t *testing.T, tmpfile *os.File) { 119 | fileContent, err := ioutil.ReadFile(tmpfile.Name()) 120 | 121 | if err != nil { 122 | t.Fatal("Failed to read tmp file") 123 | } 124 | 125 | if len(fileContent) == 0 { 126 | t.Fatal("Failed to dump JSON to file - tmp file is empty") 127 | } 128 | } 129 | 130 | func assertJsonOutputEmpty(t *testing.T, tmpfile *os.File) { 131 | fileContent, err := ioutil.ReadFile(tmpfile.Name()) 132 | 133 | if err != nil { 134 | t.Fatal("Failed to read tmp file") 135 | } 136 | 137 | if len(fileContent) != 0 { 138 | t.Fatal("Expected JSON file to be empty") 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/conversion/conversion.go: -------------------------------------------------------------------------------- 1 | package conversion 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "github.com/siddontang/go-mysql/replication" 6 | "time" 7 | "zalora/binlog-parser/database" 8 | "zalora/binlog-parser/parser/messages" 9 | ) 10 | 11 | type RowsEventData struct { 12 | BinlogEventHeader replication.EventHeader 13 | BinlogEvent replication.RowsEvent 14 | TableMetadata database.TableMetadata 15 | } 16 | 17 | func NewRowsEventData(binlogEventHeader replication.EventHeader, binlogEvent replication.RowsEvent, tableMetadata database.TableMetadata) RowsEventData { 18 | return RowsEventData{ 19 | BinlogEventHeader: binlogEventHeader, 20 | BinlogEvent: binlogEvent, 21 | TableMetadata: tableMetadata, 22 | } 23 | } 24 | 25 | func ConvertQueryEventToMessage(binlogEventHeader replication.EventHeader, binlogEvent replication.QueryEvent) messages.Message { 26 | header := messages.NewMessageHeader( 27 | string(binlogEvent.Schema), 28 | "(unknown)", 29 | time.Unix(int64(binlogEventHeader.Timestamp), 0), 30 | binlogEventHeader.LogPos, 31 | 0, 32 | ) 33 | 34 | message := messages.NewQueryMessage( 35 | header, 36 | messages.SqlQuery(binlogEvent.Query), 37 | ) 38 | 39 | return messages.Message(message) 40 | } 41 | 42 | func ConvertRowsEventsToMessages(xId uint64, rowsEventsData []RowsEventData) []messages.Message { 43 | var ret []messages.Message 44 | 45 | for _, d := range rowsEventsData { 46 | rowData := mapRowDataDataToColumnNames(d.BinlogEvent.Rows, d.TableMetadata.Fields) 47 | 48 | header := messages.NewMessageHeader( 49 | d.TableMetadata.Schema, 50 | d.TableMetadata.Table, 51 | time.Unix(int64(d.BinlogEventHeader.Timestamp), 0), 52 | d.BinlogEventHeader.LogPos, 53 | xId, 54 | ) 55 | 56 | switch d.BinlogEventHeader.EventType { 57 | case replication.WRITE_ROWS_EVENTv1, 58 | replication.WRITE_ROWS_EVENTv2: 59 | for _, message := range createInsertMessagesFromRowData(header, rowData) { 60 | ret = append(ret, messages.Message(message)) 61 | } 62 | 63 | break 64 | 65 | case replication.UPDATE_ROWS_EVENTv1, 66 | replication.UPDATE_ROWS_EVENTv2: 67 | for _, message := range createUpdateMessagesFromRowData(header, rowData) { 68 | ret = append(ret, messages.Message(message)) 69 | } 70 | 71 | break 72 | 73 | case replication.DELETE_ROWS_EVENTv1, 74 | replication.DELETE_ROWS_EVENTv2: 75 | for _, message := range createDeleteMessagesFromRowData(header, rowData) { 76 | ret = append(ret, messages.Message(message)) 77 | } 78 | 79 | break 80 | 81 | default: 82 | glog.Errorf("Can't convert unknown event %s", d.BinlogEventHeader.EventType) 83 | 84 | break 85 | } 86 | } 87 | 88 | return ret 89 | } 90 | 91 | func createUpdateMessagesFromRowData(header messages.MessageHeader, rowData []messages.MessageRowData) []messages.UpdateMessage { 92 | if len(rowData)%2 != 0 { 93 | panic("update rows should be old/new pairs") // should never happen as per mysql format 94 | } 95 | 96 | var ret []messages.UpdateMessage 97 | var tmp messages.MessageRowData 98 | 99 | for index, data := range rowData { 100 | if index%2 == 0 { 101 | tmp = data 102 | } else { 103 | ret = append(ret, messages.NewUpdateMessage(header, tmp, data)) 104 | } 105 | } 106 | 107 | return ret 108 | } 109 | 110 | func createInsertMessagesFromRowData(header messages.MessageHeader, rowData []messages.MessageRowData) []messages.InsertMessage { 111 | var ret []messages.InsertMessage 112 | 113 | for _, data := range rowData { 114 | ret = append(ret, messages.NewInsertMessage(header, data)) 115 | } 116 | 117 | return ret 118 | } 119 | 120 | func createDeleteMessagesFromRowData(header messages.MessageHeader, rowData []messages.MessageRowData) []messages.DeleteMessage { 121 | var ret []messages.DeleteMessage 122 | 123 | for _, data := range rowData { 124 | ret = append(ret, messages.NewDeleteMessage(header, data)) 125 | } 126 | 127 | return ret 128 | } 129 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/conversion/conversion_test.go: -------------------------------------------------------------------------------- 1 | // +build unit 2 | 3 | package conversion 4 | 5 | import ( 6 | "fmt" 7 | "github.com/siddontang/go-mysql/replication" 8 | "reflect" 9 | "testing" 10 | "time" 11 | "zalora/binlog-parser/database" 12 | "zalora/binlog-parser/parser/messages" 13 | ) 14 | 15 | func TestConvertQueryEventToMessage(t *testing.T) { 16 | logPos := uint32(100) 17 | query := "SELECT 1" 18 | 19 | eventHeader := replication.EventHeader{Timestamp: uint32(time.Now().Unix()), LogPos: logPos} 20 | queryEvent := replication.QueryEvent{Query: []byte(query)} 21 | 22 | message := ConvertQueryEventToMessage(eventHeader, queryEvent) 23 | 24 | assertMessageHeader(t, message, logPos, messages.MESSAGE_TYPE_QUERY) 25 | 26 | if string(message.(messages.QueryMessage).Query) != query { 27 | t.Fatal("Unexpected value for query ") 28 | } 29 | } 30 | 31 | func TestConvertRowsEventsToMessages(t *testing.T) { 32 | logPos := uint32(100) 33 | xId := uint64(200) 34 | 35 | tableMetadata := database.TableMetadata{"db_name", "table_name", map[int]string{0: "field_1", 1: "field_2"}} 36 | 37 | testCasesWriteRowsEvents := []struct { 38 | eventType replication.EventType 39 | }{ 40 | {replication.WRITE_ROWS_EVENTv1}, 41 | {replication.WRITE_ROWS_EVENTv2}, 42 | } 43 | 44 | for _, tc := range testCasesWriteRowsEvents { 45 | t.Run("Insert message", func(t *testing.T) { 46 | eventHeader := createEventHeader(logPos, tc.eventType) 47 | rowsEvent := createRowsEvent([]interface{}{"value_1", "value_2"}, []interface{}{"value_3", "value_4"}) 48 | rowsEventData := []RowsEventData{NewRowsEventData(eventHeader, rowsEvent, tableMetadata)} 49 | 50 | convertedMessages := ConvertRowsEventsToMessages(xId, rowsEventData) 51 | 52 | if len(convertedMessages) != 2 { 53 | t.Fatal("Expected 2 insert messages to be created") 54 | } 55 | 56 | assertMessageHeader(t, convertedMessages[0], logPos, messages.MESSAGE_TYPE_INSERT) 57 | assertMessageHeader(t, convertedMessages[1], logPos, messages.MESSAGE_TYPE_INSERT) 58 | 59 | insertMessageOne := convertedMessages[0].(messages.InsertMessage) 60 | 61 | if !reflect.DeepEqual(insertMessageOne.Data, messages.MessageRowData{Row: messages.MessageRow{"field_1": "value_1", "field_2": "value_2"}}) { 62 | t.Fatal(fmt.Sprintf("Wrong data for insert message 1 - got %v", insertMessageOne.Data)) 63 | } 64 | 65 | insertMessageTwo := convertedMessages[1].(messages.InsertMessage) 66 | 67 | if !reflect.DeepEqual(insertMessageTwo.Data, messages.MessageRowData{Row: messages.MessageRow{"field_1": "value_3", "field_2": "value_4"}}) { 68 | t.Fatal(fmt.Sprintf("Wrong data for insert message 2 - got %v", insertMessageTwo.Data)) 69 | } 70 | }) 71 | } 72 | 73 | testCasesDeleteRowsEvents := []struct { 74 | eventType replication.EventType 75 | }{ 76 | {replication.DELETE_ROWS_EVENTv1}, 77 | {replication.DELETE_ROWS_EVENTv2}, 78 | } 79 | 80 | for _, tc := range testCasesDeleteRowsEvents { 81 | t.Run("Delete message", func(t *testing.T) { 82 | eventHeader := createEventHeader(logPos, tc.eventType) 83 | rowsEvent := createRowsEvent([]interface{}{"value_1", "value_2"}, []interface{}{"value_3", "value_4"}) 84 | rowsEventData := []RowsEventData{NewRowsEventData(eventHeader, rowsEvent, tableMetadata)} 85 | 86 | convertedMessages := ConvertRowsEventsToMessages(xId, rowsEventData) 87 | 88 | if len(convertedMessages) != 2 { 89 | t.Fatal("Expected 2 delete messages to be created") 90 | } 91 | 92 | assertMessageHeader(t, convertedMessages[0], logPos, messages.MESSAGE_TYPE_DELETE) 93 | assertMessageHeader(t, convertedMessages[1], logPos, messages.MESSAGE_TYPE_DELETE) 94 | 95 | deleteMessageOne := convertedMessages[0].(messages.DeleteMessage) 96 | 97 | if !reflect.DeepEqual(deleteMessageOne.Data, messages.MessageRowData{Row: messages.MessageRow{"field_1": "value_1", "field_2": "value_2"}}) { 98 | t.Fatal(fmt.Sprintf("Wrong data for delete message 1 - got %v", deleteMessageOne.Data)) 99 | } 100 | 101 | deleteMessageTwo := convertedMessages[1].(messages.DeleteMessage) 102 | 103 | if !reflect.DeepEqual(deleteMessageTwo.Data, messages.MessageRowData{Row: messages.MessageRow{"field_1": "value_3", "field_2": "value_4"}}) { 104 | t.Fatal(fmt.Sprintf("Wrong data for delete message 2 - got %v", deleteMessageTwo.Data)) 105 | } 106 | }) 107 | } 108 | 109 | testCasesUpdateRowsEvents := []struct { 110 | eventType replication.EventType 111 | }{ 112 | {replication.UPDATE_ROWS_EVENTv1}, 113 | {replication.UPDATE_ROWS_EVENTv2}, 114 | } 115 | 116 | for _, tc := range testCasesUpdateRowsEvents { 117 | t.Run("Update message", func(t *testing.T) { 118 | eventHeader := createEventHeader(logPos, tc.eventType) 119 | rowsEvent := createRowsEvent([]interface{}{"value_1", "value_2"}, []interface{}{"value_3", "value_4"}) 120 | rowsEventData := []RowsEventData{NewRowsEventData(eventHeader, rowsEvent, tableMetadata)} 121 | 122 | convertedMessages := ConvertRowsEventsToMessages(xId, rowsEventData) 123 | 124 | if len(convertedMessages) != 1 { 125 | t.Fatal("Expected 1 update messages to be created") 126 | } 127 | 128 | assertMessageHeader(t, convertedMessages[0], logPos, messages.MESSAGE_TYPE_UPDATE) 129 | 130 | updateMessage := convertedMessages[0].(messages.UpdateMessage) 131 | 132 | if !reflect.DeepEqual(updateMessage.OldData, messages.MessageRowData{Row: messages.MessageRow{"field_1": "value_1", "field_2": "value_2"}}) { 133 | t.Fatal(fmt.Sprintf("Wrong data for update message old data - got %v", updateMessage.OldData)) 134 | } 135 | 136 | if !reflect.DeepEqual(updateMessage.NewData, messages.MessageRowData{Row: messages.MessageRow{"field_1": "value_3", "field_2": "value_4"}}) { 137 | t.Fatal(fmt.Sprintf("Wrong data for update message new data - got %v", updateMessage.NewData)) 138 | } 139 | }) 140 | } 141 | 142 | t.Run("Unknown event type", func(t *testing.T) { 143 | eventHeader := createEventHeader(logPos, replication.RAND_EVENT) // can be any unkown event actually 144 | rowsEvent := createRowsEvent() 145 | rowsEventData := []RowsEventData{NewRowsEventData(eventHeader, rowsEvent, tableMetadata)} 146 | 147 | convertedMessages := ConvertRowsEventsToMessages(xId, rowsEventData) 148 | 149 | if len(convertedMessages) != 0 { 150 | t.Fatal("Expected no messages to be created from unknown event") 151 | } 152 | }) 153 | } 154 | 155 | func createEventHeader(logPos uint32, eventType replication.EventType) replication.EventHeader { 156 | return replication.EventHeader{ 157 | Timestamp: uint32(time.Now().Unix()), 158 | EventType: eventType, 159 | LogPos: logPos, 160 | } 161 | } 162 | 163 | func createRowsEvent(rowData ...[]interface{}) replication.RowsEvent { 164 | return replication.RowsEvent{Rows: rowData} 165 | } 166 | 167 | func assertMessageHeader(t *testing.T, message messages.Message, expectedLogPos uint32, expectedType messages.MessageType) { 168 | if message.GetHeader().BinlogPosition != expectedLogPos { 169 | t.Fatal("Unexpected value for BinlogPosition") 170 | } 171 | 172 | if message.GetType() != expectedType { 173 | t.Fatal("Unexpected value for message type") 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/conversion/row_data.go: -------------------------------------------------------------------------------- 1 | package conversion 2 | 3 | import ( 4 | "fmt" 5 | "zalora/binlog-parser/parser/messages" 6 | ) 7 | 8 | func mapRowDataDataToColumnNames(rows [][]interface{}, columnNames map[int]string) []messages.MessageRowData { 9 | var mappedRows []messages.MessageRowData 10 | 11 | for _, row := range rows { 12 | data := make(map[string]interface{}) 13 | unknownCount := 0 14 | 15 | detectedMismatch, mismatchNotice := detectMismatch(row, columnNames) 16 | 17 | for columnIndex, columnValue := range row { 18 | if detectedMismatch { 19 | data[fmt.Sprintf("(unknown_%d)", unknownCount)] = columnValue 20 | unknownCount++ 21 | } else { 22 | columnName, exists := columnNames[columnIndex] 23 | 24 | if !exists { 25 | // This should actually never happen 26 | // Fail hard before doing anything weird 27 | panic(fmt.Sprintf("No mismatch between row and column names array detected, but column %s not found", columnName)) 28 | } 29 | 30 | data[columnName] = columnValue 31 | } 32 | } 33 | 34 | if detectedMismatch { 35 | mappedRows = append(mappedRows, messages.MessageRowData{Row: data, MappingNotice: mismatchNotice}) 36 | } else { 37 | mappedRows = append(mappedRows, messages.MessageRowData{Row: data}) 38 | } 39 | } 40 | 41 | return mappedRows 42 | } 43 | 44 | func detectMismatch(row []interface{}, columnNames map[int]string) (bool, string) { 45 | if len(row) > len(columnNames) { 46 | return true, fmt.Sprintf("column names array is missing field(s), will map them as unknown_*") 47 | } 48 | 49 | if len(row) < len(columnNames) { 50 | return true, fmt.Sprintf("row is missing field(s), ignoring missing") 51 | } 52 | 53 | return false, "" 54 | } 55 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/conversion/row_data_test.go: -------------------------------------------------------------------------------- 1 | package conversion 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | func TestDetectMismatch(t *testing.T) { 9 | t.Run("No mismatch, empty input", func(t *testing.T) { 10 | row := []interface{}{} 11 | columnNames := map[int]string{} 12 | 13 | detected, _ := detectMismatch(row, columnNames) 14 | 15 | if detected { 16 | t.Fatal("Expected no mismatch to be detected") 17 | } 18 | }) 19 | 20 | t.Run("No mismatch", func(t *testing.T) { 21 | row := []interface{}{"value 1", "value 2"} 22 | columnNames := map[int]string{0: "field_1", 1: "field_2"} 23 | 24 | detected, _ := detectMismatch(row, columnNames) 25 | 26 | if detected { 27 | t.Fatal("Expected no mismatch to be detected") 28 | } 29 | }) 30 | 31 | t.Run("Detect mismatch, row is missing field", func(t *testing.T) { 32 | row := []interface{}{"value 1"} 33 | columnNames := map[int]string{0: "field_1", 1: "field_2"} 34 | 35 | detected, notice := detectMismatch(row, columnNames) 36 | 37 | if !detected { 38 | t.Fatal("Expected mismatch to be detected") 39 | } 40 | 41 | if !strings.Contains(notice, "row is missing field(s)") { 42 | t.Fatal("Wrong notice") 43 | } 44 | }) 45 | 46 | t.Run("Detect mismatch, column name is missing field", func(t *testing.T) { 47 | row := []interface{}{"value 1", "value 2"} 48 | columnNames := map[int]string{0: "field_1"} 49 | 50 | detected, notice := detectMismatch(row, columnNames) 51 | 52 | if !detected { 53 | t.Fatal("Expected mismatch to be detected") 54 | } 55 | 56 | if !strings.Contains(notice, "column names array is missing field(s)") { 57 | t.Fatal("Wrong notice") 58 | } 59 | }) 60 | } 61 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/messages/message.go: -------------------------------------------------------------------------------- 1 | package messages 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type MessageType string 8 | 9 | const ( 10 | MESSAGE_TYPE_INSERT MessageType = "Insert" 11 | MESSAGE_TYPE_UPDATE MessageType = "Update" 12 | MESSAGE_TYPE_DELETE MessageType = "Delete" 13 | MESSAGE_TYPE_QUERY MessageType = "Query" 14 | ) 15 | 16 | type MessageHeader struct { 17 | Schema string 18 | Table string 19 | BinlogMessageTime string 20 | BinlogPosition uint32 21 | XId uint64 22 | } 23 | 24 | func NewMessageHeader(schema string, table string, binlogMessageTime time.Time, binlogPosition uint32, xId uint64) MessageHeader { 25 | return MessageHeader{ 26 | Schema: schema, 27 | Table: table, 28 | BinlogMessageTime: binlogMessageTime.UTC().Format(time.RFC3339), 29 | BinlogPosition: binlogPosition, 30 | XId: xId, 31 | } 32 | } 33 | 34 | type Message interface { 35 | GetHeader() MessageHeader 36 | GetType() MessageType 37 | } 38 | 39 | type baseMessage struct { 40 | Header MessageHeader 41 | Type MessageType 42 | } 43 | 44 | func (b baseMessage) GetHeader() MessageHeader { 45 | return b.Header 46 | } 47 | 48 | func (b baseMessage) GetType() MessageType { 49 | return b.Type 50 | } 51 | 52 | type MessageRow map[string]interface{} 53 | 54 | type MessageRowData struct { 55 | Row MessageRow 56 | MappingNotice string 57 | } 58 | 59 | type SqlQuery string 60 | 61 | type QueryMessage struct { 62 | baseMessage 63 | Query SqlQuery 64 | } 65 | 66 | func NewQueryMessage(header MessageHeader, query SqlQuery) QueryMessage { 67 | return QueryMessage{baseMessage: baseMessage{Header: header, Type: MESSAGE_TYPE_QUERY}, Query: query} 68 | } 69 | 70 | type UpdateMessage struct { 71 | baseMessage 72 | OldData MessageRowData 73 | NewData MessageRowData 74 | } 75 | 76 | func NewUpdateMessage(header MessageHeader, oldData MessageRowData, newData MessageRowData) UpdateMessage { 77 | return UpdateMessage{baseMessage: baseMessage{Header: header, Type: MESSAGE_TYPE_UPDATE}, OldData: oldData, NewData: newData} 78 | } 79 | 80 | type InsertMessage struct { 81 | baseMessage 82 | Data MessageRowData 83 | } 84 | 85 | func NewInsertMessage(header MessageHeader, data MessageRowData) InsertMessage { 86 | return InsertMessage{baseMessage: baseMessage{Header: header, Type: MESSAGE_TYPE_INSERT}, Data: data} 87 | } 88 | 89 | type DeleteMessage struct { 90 | baseMessage 91 | Data MessageRowData 92 | } 93 | 94 | func NewDeleteMessage(header MessageHeader, data MessageRowData) DeleteMessage { 95 | return DeleteMessage{baseMessage: baseMessage{Header: header, Type: MESSAGE_TYPE_DELETE}, Data: data} 96 | } 97 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/messages/message_test.go: -------------------------------------------------------------------------------- 1 | // +build unit 2 | 3 | package messages 4 | 5 | import ( 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestNewMessageHeader(t *testing.T) { 11 | now := time.Now() 12 | binlogPosition := uint32(1) 13 | xid := uint64(2) 14 | 15 | messageHeader := NewMessageHeader("schema", "table", now, binlogPosition, xid) 16 | 17 | if messageHeader.Schema != "schema" { 18 | t.Fatal("Wrong schema in message header") 19 | } 20 | 21 | if messageHeader.Table != "table" { 22 | t.Fatal("Wrong table in message header") 23 | } 24 | 25 | if messageHeader.BinlogPosition != binlogPosition { 26 | t.Fatal("Wrong binlogPosition in message header") 27 | } 28 | 29 | if messageHeader.XId != xid { 30 | t.Fatal("Wrong Xid in message header") 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/parser/binlog_to_messages.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | "github.com/siddontang/go-mysql/replication" 6 | "strings" 7 | "zalora/binlog-parser/database" 8 | "zalora/binlog-parser/parser/conversion" 9 | "zalora/binlog-parser/parser/messages" 10 | ) 11 | 12 | type ConsumerFunc func(messages.Message) error 13 | 14 | func ParseBinlogToMessages(binlogFilename string, tableMap database.TableMap, consumer ConsumerFunc) error { 15 | rowRowsEventBuffer := NewRowsEventBuffer() 16 | 17 | p := replication.NewBinlogParser() 18 | 19 | f := func(e *replication.BinlogEvent) error { 20 | switch e.Header.EventType { 21 | case replication.QUERY_EVENT: 22 | queryEvent := e.Event.(*replication.QueryEvent) 23 | query := string(queryEvent.Query) 24 | 25 | if strings.ToUpper(strings.Trim(query, " ")) == "BEGIN" { 26 | glog.V(3).Info("Starting transaction") 27 | } else if strings.HasPrefix(strings.ToUpper(strings.Trim(query, " ")), "SAVEPOINT") { 28 | glog.V(3).Info("Skipping transaction savepoint") 29 | } else { 30 | glog.V(3).Info("Query event") 31 | 32 | err := consumer(conversion.ConvertQueryEventToMessage(*e.Header, *queryEvent)) 33 | 34 | if err != nil { 35 | return err 36 | } 37 | } 38 | 39 | break 40 | 41 | case replication.XID_EVENT: 42 | xidEvent := e.Event.(*replication.XIDEvent) 43 | xId := uint64(xidEvent.XID) 44 | 45 | glog.V(3).Infof("Ending transaction xID %d", xId) 46 | 47 | for _, message := range conversion.ConvertRowsEventsToMessages(xId, rowRowsEventBuffer.Drain()) { 48 | err := consumer(message) 49 | 50 | if err != nil { 51 | return err 52 | } 53 | } 54 | 55 | break 56 | 57 | case replication.TABLE_MAP_EVENT: 58 | tableMapEvent := e.Event.(*replication.TableMapEvent) 59 | 60 | schema := string(tableMapEvent.Schema) 61 | table := string(tableMapEvent.Table) 62 | tableId := uint64(tableMapEvent.TableID) 63 | 64 | err := tableMap.Add(tableId, schema, table) 65 | 66 | if err != nil { 67 | glog.Errorf("Failed to add table information for table %s.%s (id %d)", schema, table, tableId) 68 | return err 69 | } 70 | 71 | break 72 | 73 | case replication.WRITE_ROWS_EVENTv1, 74 | replication.UPDATE_ROWS_EVENTv1, 75 | replication.DELETE_ROWS_EVENTv1, 76 | replication.WRITE_ROWS_EVENTv2, 77 | replication.UPDATE_ROWS_EVENTv2, 78 | replication.DELETE_ROWS_EVENTv2: 79 | rowsEvent := e.Event.(*replication.RowsEvent) 80 | 81 | tableId := uint64(rowsEvent.TableID) 82 | tableMetadata, ok := tableMap.LookupTableMetadata(tableId) 83 | 84 | if !ok { 85 | glog.Errorf("Skipping event - no table found for table id %d", tableId) 86 | break 87 | } 88 | 89 | rowRowsEventBuffer.BufferRowsEventData( 90 | conversion.NewRowsEventData(*e.Header, *rowsEvent, tableMetadata), 91 | ) 92 | 93 | break 94 | 95 | default: 96 | break 97 | } 98 | 99 | return nil 100 | } 101 | 102 | return p.ParseFile(binlogFilename, 0, f) 103 | } 104 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/parser/rows_event_buffer.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "zalora/binlog-parser/parser/conversion" 5 | ) 6 | 7 | type RowsEventBuffer struct { 8 | buffered []conversion.RowsEventData 9 | } 10 | 11 | func NewRowsEventBuffer() RowsEventBuffer { 12 | return RowsEventBuffer{} 13 | } 14 | 15 | func (mb *RowsEventBuffer) BufferRowsEventData(d conversion.RowsEventData) { 16 | mb.buffered = append(mb.buffered, d) 17 | } 18 | 19 | func (mb *RowsEventBuffer) Drain() []conversion.RowsEventData { 20 | ret := mb.buffered 21 | mb.buffered = nil 22 | 23 | return ret 24 | } 25 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/parser/parser/rows_event_buffer_test.go: -------------------------------------------------------------------------------- 1 | // +build unit 2 | 3 | package parser 4 | 5 | import ( 6 | "reflect" 7 | "testing" 8 | "zalora/binlog-parser/parser/conversion" 9 | ) 10 | 11 | func TestRowsEventBuffer(t *testing.T) { 12 | eventDataOne := conversion.RowsEventData{} 13 | eventDataTwo := conversion.RowsEventData{} 14 | 15 | t.Run("Drain Empty", func(t *testing.T) { 16 | buffer := NewRowsEventBuffer() 17 | buffered := buffer.Drain() 18 | 19 | if len(buffered) != 0 { 20 | t.Fatal("Wrong number of entries retrieved from empty buffer") 21 | } 22 | }) 23 | 24 | t.Run("Drain and re-fill", func(t *testing.T) { 25 | buffer := NewRowsEventBuffer() 26 | buffer.BufferRowsEventData(eventDataOne) 27 | buffer.BufferRowsEventData(eventDataTwo) 28 | 29 | buffered := buffer.Drain() 30 | 31 | if len(buffered) != 2 { 32 | t.Fatal("Wrong number of entries retrieved from buffer") 33 | } 34 | 35 | if !reflect.DeepEqual(buffered[0], eventDataOne) { 36 | t.Fatal("Retrieved wrong entry at index 0 from buffer") 37 | } 38 | 39 | if !reflect.DeepEqual(buffered[1], eventDataOne) { 40 | t.Fatal("Retrieved wrong entry at index 1 from buffer") 41 | } 42 | 43 | buffer.BufferRowsEventData(eventDataOne) 44 | 45 | buffered = buffer.Drain() 46 | 47 | if len(buffered) != 1 { 48 | t.Fatal("Wrong number of entries retrieved from re-used buffer") 49 | } 50 | 51 | if !reflect.DeepEqual(buffered[0], eventDataOne) { 52 | t.Fatal("Retrieved wrong entry at index 0 from re-used buffer") 53 | } 54 | }) 55 | } 56 | -------------------------------------------------------------------------------- /src/zalora/binlog-parser/test/test_setup.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "os" 5 | // "flag" 6 | "testing" 7 | ) 8 | 9 | const TEST_DB_CONNECTION_STRING string = "root@/test_db" 10 | 11 | func Setup(m *testing.M) { 12 | // flag.Set("alsologtostderr", "true") 13 | // flag.Set("v", "5") 14 | 15 | os.Exit(m.Run()) 16 | } 17 | --------------------------------------------------------------------------------