├── config.toml ├── src ├── rowevents │ ├── mod.rs │ ├── value_type.rs │ ├── event_header.rs │ ├── descriptor_datetime.rs │ ├── descriptor_decimal.rs │ ├── events.rs │ ├── stream.rs │ ├── descriptor.rs │ ├── reader.rs │ └── parser.rs └── lib.rs ├── .gitignore ├── Cargo.toml ├── examples ├── sync_binlog_to_redis.rs └── read_binlog_file.rs ├── README.md ├── LICENSE └── python ├── pyapp └── main.py └── pylib └── mysqlbinlog.py /config.toml: -------------------------------------------------------------------------------- 1 | 2 | # This config is read by examples/read_binlog_file.rs 3 | 4 | [filter] 5 | excluded=["antares.*"] 6 | 7 | 8 | -------------------------------------------------------------------------------- /src/rowevents/mod.rs: -------------------------------------------------------------------------------- 1 | 2 | pub mod reader; 3 | pub mod parser; 4 | pub mod stream; 5 | pub mod event_header; 6 | pub mod events; 7 | pub mod value_type; 8 | pub mod descriptor; 9 | mod descriptor_datetime; 10 | mod descriptor_decimal; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "mysqlbinlog" 3 | version = "0.1.5" 4 | authors = ["healer_kx "] 5 | description = "mysqlbinlog-rs is for parsing MySQL binlog file (row format)" 6 | license = "MIT" 7 | 8 | [dependencies] 9 | byteorder = "1.0.0" 10 | time = "0.1" 11 | chrono = "0.3" 12 | regex = "0.2" 13 | 14 | [dev-dependencies] 15 | toml = "0.4.1" 16 | serde = "1.0" 17 | serde_derive = "1.0" 18 | 19 | [lib] 20 | name="mysqlbinlog" 21 | crate-type=["rlib", "dylib"] -------------------------------------------------------------------------------- /examples/sync_binlog_to_redis.rs: -------------------------------------------------------------------------------- 1 | 2 | extern crate mysqlbinlog; 3 | 4 | use mysqlbinlog::rowevents::reader; 5 | use mysqlbinlog::rowevents::events::Event; 6 | 7 | fn main() { 8 | let reader = reader::Reader::new("/Users/healer/data.log"); 9 | if let Ok(mut r) = reader { 10 | 11 | while let Ok(e1) = r.read_event_header() { 12 | println!("@-------------------------------------------"); 13 | println!("{}", e1.get_time()); 14 | let event = r.read_event(&e1); 15 | match event { 16 | Ok(Event::Xid(e)) => println!("{:?}", e), 17 | Ok(Event::Insert(e)) => println!("INSERT {:?}", e), 18 | Ok(Event::Update(e)) => println!("UPDATE {:?}", e), 19 | Ok(Event::Delete(e)) => println!("DELETE {:?}", e), 20 | _ => println!("{:?}", e1) 21 | } 22 | } 23 | 24 | print!("End"); 25 | } 26 | 27 | } -------------------------------------------------------------------------------- /src/rowevents/value_type.rs: -------------------------------------------------------------------------------- 1 | 2 | 3 | pub enum FieldType { 4 | Unknown = -1, 5 | Decimal = 0, 6 | Tiny = 1, 7 | Short = 2, 8 | Long = 3, 9 | Float = 4, 10 | Double = 5, 11 | Null = 6, 12 | Timestamp = 7, 13 | Longlong = 8, 14 | Int24 = 9, 15 | Date = 10, 16 | Time = 11, 17 | Datetime = 12, 18 | Year = 13, 19 | Newdate = 14, 20 | Varchar = 15, 21 | Bit = 16, 22 | Timestamp2 = 17, 23 | Datetime2 = 18, 24 | Time2 = 19, 25 | NewDecimal = 246, 26 | Enum = 247, 27 | Set = 248, 28 | TinyBlob = 249, 29 | MediumBlob = 250, 30 | LongBlob = 251, 31 | Blob = 252, 32 | VarString = 253, 33 | String = 254, 34 | Geometry = 255 35 | } 36 | 37 | #[derive(Debug)] 38 | pub enum ValueType { 39 | Unknown, 40 | Null, 41 | Tinyint(i8), 42 | Shortint(i16), 43 | Int(i32), 44 | Longlong(i64), 45 | Float(f32), 46 | Double(f64), 47 | String(Vec), 48 | Datetime2(i64), 49 | Decimal(String), 50 | Timestamp(u32) 51 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mysqlbinlog-rs 2 | A MySQL binlog file (row format) parser in Rust 3 | The first stable version is 0.1.4 4 | 5 | I built a parser in Python3 in the early of this year. Now in Rust instead, for high efficiency, and provided a Python3 binding for conveniency. 6 | 7 | # Scenarios 8 | - Sync MySQL data into Redis, MongoDB, Kafka, e.g. 9 | - Figure out the DB data row change history when fixing bugs 10 | - Watch table's data changing for developer, for example, coding a PHP controller-action. 11 | - ... 12 | 13 | Examples for Rust developers 14 | - You can see files in the dir `examples` 15 | 16 | 17 | Examples for Python developers 18 | ``` 19 | Usage: 20 | python3 main.py --ignore=th%.%,an%.% -b /usr/local/var/mysql/mysql_binlog.000001 21 | 22 | # Give argument --ignore with a db name, table name pattern to ignore the row events in that tables. 23 | ``` 24 | 25 | 26 | - 2017-12-13 Reduce times of reading binlog file. Release the content memory in Vec[] for unused content. Fix python binding's dylib loading path. 27 | - 2018-04-02 Support Rotate event and timestamp type. 28 | -------------------------------------------------------------------------------- /src/rowevents/event_header.rs: -------------------------------------------------------------------------------- 1 | 2 | use chrono::{NaiveDateTime, NaiveDate}; 3 | 4 | 5 | #[derive(Debug)] 6 | #[repr(C)] 7 | pub struct EventHeader { 8 | pub timestamp: i32, 9 | pub type_code: i8, 10 | pub server_id: i32, 11 | pub event_len: i32, 12 | pub next_pos: i32, 13 | pub flags: i16 14 | } 15 | 16 | impl EventHeader { 17 | pub fn new(timestamp: i32, type_code: i8, server_id: i32, event_len: i32, next_pos: i32, flags: i16) -> EventHeader { 18 | EventHeader { 19 | timestamp: timestamp, 20 | type_code: type_code, 21 | server_id: server_id, 22 | event_len: event_len, 23 | next_pos: next_pos, 24 | flags: flags 25 | } 26 | } 27 | 28 | pub fn get_time(&self) -> String { 29 | let dt = NaiveDateTime::from_timestamp(self.timestamp as i64, 0); 30 | dt.to_string() 31 | } 32 | 33 | pub fn get_event_len(&self) -> usize { 34 | self.event_len as usize 35 | } 36 | 37 | pub fn get_event_type(&self) -> i8 { 38 | self.type_code 39 | } 40 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/rowevents/descriptor_datetime.rs: -------------------------------------------------------------------------------- 1 | 2 | use byteorder::{LittleEndian, BigEndian, ReadBytesExt}; 3 | use std::io::Cursor; 4 | use std::io::Result; 5 | use rowevents::value_type::*; 6 | use chrono::{NaiveDateTime, NaiveDate}; 7 | 8 | pub fn parse_datetime2(ms_precision: u8, metadata2: u8, data: &[u8]) -> Result<(ValueType, usize)> { 9 | 10 | let mut cursor = Cursor::new(&data[0 .. 5]); 11 | let t = cursor.read_uint::(5)? as u64; 12 | 13 | let t:u64 = t - 0x8000000000; 14 | 15 | let (msec, offset) = if ms_precision == 1 || ms_precision == 2 { 16 | let mut cursor = Cursor::new(&data[5 .. 6]); 17 | let msec:i64 = cursor.read_i8()? as i64 * 10000; 18 | (msec, 6) 19 | } else if ms_precision == 3 || ms_precision == 4 { 20 | let mut cursor = Cursor::new(&data[5 .. 7]); 21 | let msec:i64 = cursor.read_i16::()? as i64 * 100; 22 | (msec, 7) 23 | } else if ms_precision == 5 || ms_precision == 6 { 24 | let mut cursor = Cursor::new(&data[5 .. 8]); 25 | let msec:i64 = cursor.read_int::(3)? as i64; 26 | (msec, 8) 27 | } else { 28 | (0, 5) 29 | }; 30 | 31 | let ymd = t >> 17; 32 | let ym = ymd >> 5; 33 | let hms = t % (1 << 17); 34 | 35 | let day = ymd % (1 << 5); 36 | let month = ym % 13; 37 | let year = ym / 13; 38 | 39 | let second = hms % (1 << 6); 40 | let minute = (hms >> 6) % (1 << 6); 41 | let hour = hms >> 12; 42 | 43 | let msec = msec / 1000; 44 | // println!("Ymd={}-{}-{} {}:{}:{}.{}", year, month, day, hour, minute, second, msec); 45 | let nd = NaiveDate::from_ymd(year as i32, month as u32, day as u32) 46 | .and_hms_milli(hour as u32, minute as u32, second as u32, msec as u32); 47 | 48 | Ok((ValueType::Datetime2(nd.timestamp()), offset)) 49 | 50 | } -------------------------------------------------------------------------------- /examples/read_binlog_file.rs: -------------------------------------------------------------------------------- 1 | 2 | #[macro_use] 3 | extern crate serde_derive; 4 | #[macro_use] 5 | extern crate serde; 6 | extern crate mysqlbinlog; 7 | extern crate toml; 8 | 9 | use std::io::Result; 10 | use std::env; 11 | use std::fs::File; 12 | use std::io::prelude::*; 13 | 14 | use mysqlbinlog::rowevents::reader; 15 | use mysqlbinlog::rowevents::events::Event; 16 | use mysqlbinlog::rowevents::event_header::EventHeader; 17 | 18 | #[derive(Deserialize, Serialize, Debug)] 19 | struct Filter { 20 | excluded: Vec 21 | } 22 | 23 | #[derive(Deserialize, Serialize, Debug)] 24 | struct Config { 25 | filter: Filter 26 | } 27 | 28 | fn parse_config(config_file: &str) -> Result { 29 | let mut file = File::open(config_file)?; 30 | let mut content = String::new(); 31 | file.read_to_string(&mut content); 32 | let config: Config = toml::from_str(&content).unwrap(); 33 | Ok(config) 34 | } 35 | 36 | fn print_event(eh: &EventHeader, e: &Event) { 37 | println!("[{}] {:?}", eh.get_time(), e); 38 | } 39 | 40 | fn main() { 41 | 42 | let mut u: Vec<_> = env::args().collect::>().clone(); 43 | let u: Vec<_> = u.drain(1..).collect(); 44 | let mut binlog_file = "".to_string(); 45 | let mut config_file = "".to_string(); 46 | for argument in u { 47 | let pair = argument.split("=").collect::>(); 48 | if pair[0] == "--binlog" { 49 | binlog_file = pair[1].to_string(); 50 | } else if pair[0] == "--config" { 51 | config_file = pair[1].to_string(); 52 | } 53 | } 54 | 55 | let config = parse_config(&config_file).unwrap(); 56 | println!("{:?}", config.filter); 57 | 58 | let reader = reader::Reader::new(&binlog_file); 59 | if let Ok(mut r) = reader { 60 | r.add_excluded_db_table("antares.*"); 61 | 62 | while let Some((eh, e)) = r.next() { 63 | if r.skip_next_event() { 64 | continue; 65 | } 66 | match e { 67 | Event::Insert(e) => print_event(&eh, &Event::Insert(e)), 68 | Event::Delete(e) => print_event(&eh, &Event::Delete(e)), 69 | Event::Update(e) => print_event(&eh, &Event::Update(e)), 70 | Event::Unknown | _ => { }, 71 | } 72 | } 73 | 74 | print!("End"); 75 | } 76 | 77 | } -------------------------------------------------------------------------------- /src/rowevents/descriptor_decimal.rs: -------------------------------------------------------------------------------- 1 | /** 2 | * https://github.com/wenerme/myfacility/blob/master/binlog/decimal.go 3 | * http://python-mysql-replication.readthedocs.io/en/latest/_modules/pymysqlreplication/row_event.html 4 | * 5 | * Read MySQL's new decimal format introduced in MySQL 5 6 | * 7 | * This project was a great source of inspiration for 8 | * understanding this storage format. 9 | * https://github.com/jeremycole/mysql_binlog 10 | */ 11 | 12 | use rowevents::value_type::*; 13 | use std::io::Result; 14 | use std::io::Cursor; 15 | use byteorder::{BigEndian, LittleEndian, ReadBytesExt}; 16 | 17 | pub fn parse_new_decimal(precision: u8, decimals: u8, data: &[u8]) -> Result<(ValueType, usize)> { 18 | let digits_per_integer = 9; 19 | let compressed_bytes = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4]; 20 | let integral = precision - decimals; 21 | let uncomp_integral = integral / digits_per_integer; 22 | let uncomp_fractional = decimals / digits_per_integer; 23 | let comp_integral = integral - (uncomp_integral * digits_per_integer); 24 | let comp_fractional = decimals - (uncomp_fractional * digits_per_integer); 25 | 26 | let mut data = Vec::from(data); 27 | let (negative, mask) = { 28 | // Support negative 29 | // The sign is encoded in the high bit of the the byte 30 | // But this bit can also be used in the value 31 | let value = data[0]; 32 | let (negative, mask):(bool, u64) = if value & 0x80 != 0 { (false, 0) } else { (true, -1 as i64 as u64) }; 33 | 34 | // Python3 35 | // byte = struct.pack(' 0 { 44 | let mut cursor = Cursor::new(&data); 45 | let i:u64 = cursor.read_uint::(size)?; 46 | let a = i ^ mask; 47 | // Handle u64 value XOR mask in different bits, for example: 24bit int 48 | let move_bits = 64 - size * 8; 49 | let a = a << move_bits >> move_bits; 50 | a.to_string() 51 | } else { 52 | "".to_string() 53 | }; 54 | 55 | if negative { 56 | d = "-".to_string() + &d; 57 | } 58 | 59 | let mut from:usize = size; 60 | let remain = &data[from .. ]; 61 | 62 | for i in 0 .. uncomp_integral { 63 | let i = i as usize; 64 | let b = &remain[i * 4 .. (i + 1) * 4]; 65 | let n = Cursor::new(b).read_u32::()? as u64 ^ mask; 66 | d += &format!("{:09}", n); 67 | from += (i * 4); 68 | } 69 | 70 | d += "."; 71 | 72 | let remain = &data[from .. ]; 73 | for i in 0 .. uncomp_fractional { 74 | let i = i as usize; 75 | let b = &remain[i * 4 .. (i + 1) * 4]; 76 | let n = Cursor::new(b).read_u32::()? as u64 ^ mask; 77 | d += &format!("{:09}", n); 78 | from += (i * 4); 79 | } 80 | 81 | let size = compressed_bytes[comp_fractional as usize]; 82 | 83 | let remain = &data[from .. ]; 84 | if size > 0 { 85 | let i = Cursor::new(remain).read_uint::(size)? as u64; 86 | let value = i ^ mask; 87 | // Handle u64 value XOR mask in different bits, for example: 24bit int 88 | let move_bits = 64 - size * 8; 89 | let value = value << move_bits >> move_bits; 90 | d += &format!("{:0w$}", value, w=comp_fractional as usize); 91 | from += size; 92 | } 93 | 94 | Ok((ValueType::Decimal(d), from)) 95 | } 96 | -------------------------------------------------------------------------------- /src/rowevents/events.rs: -------------------------------------------------------------------------------- 1 | 2 | use rowevents::value_type::ValueType; 3 | 4 | pub const UNKNOWN_EVENT: i8 = 0; 5 | pub const START_EVENT_V3: i8 = 1; 6 | pub const QUERY_EVENT: i8 = 2; 7 | pub const STOP_EVENT: i8 = 3; 8 | pub const ROTATE_EVENT: i8 = 4; 9 | pub const INTVAR_EVENT: i8 = 5; 10 | pub const LOAD_EVENT: i8 = 6; 11 | pub const SLAVE_EVENT: i8 = 7; 12 | pub const CREATE_FILE_EVENT: i8 = 8; 13 | pub const APPEND_BLOCK_EVENT: i8 = 9; 14 | pub const EXEC_LOAD_EVENT: i8 = 10; 15 | pub const DELETE_FILE_EVENT: i8 = 11; 16 | pub const NEW_LOAD_EVENT: i8 = 12; 17 | pub const RAND_EVENT: i8 = 13; 18 | pub const USER_VAR_EVENT: i8 = 14; 19 | pub const FORMAT_DESCRIPTION_EVENT: i8 = 15; 20 | pub const XID_EVENT: i8 = 16; 21 | pub const BEGIN_LOAD_QUERY_EVENT: i8 = 17; 22 | pub const EXECUTE_LOAD_QUERY_EVENT: i8 = 18; 23 | pub const TABLE_MAP_EVENT: i8 = 19; 24 | pub const PRE_GA_WRITE_ROWS_EVENT: i8 = 20; 25 | pub const PRE_GA_UPDATE_ROWS_EVENT: i8 = 21; 26 | pub const PRE_GA_DELETE_ROWS_EVENT: i8 = 22; 27 | 28 | // From MySQL 5.1.18 events 29 | pub const WRITE_ROWS_EVENT: i8 = 23; 30 | pub const UPDATE_ROWS_EVENT: i8 = 24; 31 | pub const DELETE_ROWS_EVENT: i8 = 25; 32 | // # ---------------------------------- 33 | pub const INCIDENT_EVENT: i8 = 26; 34 | pub const HEARTBEAT_LOG_EVENT: i8 = 27; 35 | 36 | // From MySQL 5.6.2 events 37 | pub const WRITE_ROWS_EVENT2: i8 = 30; 38 | pub const UPDATE_ROWS_EVENT2: i8 = 31; 39 | pub const DELETE_ROWS_EVENT2: i8 = 32; 40 | 41 | #[derive(Debug)] 42 | pub struct FormatDescriptorEvent { 43 | 44 | } 45 | 46 | #[derive(Debug)] 47 | pub struct XidEvent { 48 | xid: i64 49 | } 50 | 51 | #[derive(Debug)] 52 | pub struct TableMapEvent { 53 | pub db_name: String, 54 | pub table_name: String 55 | } 56 | 57 | #[derive(Debug)] 58 | pub struct DeleteEvent { 59 | pub entry: Vec> 60 | } 61 | 62 | #[derive(Debug)] 63 | pub struct InsertEvent { 64 | pub entry: Vec> 65 | } 66 | 67 | #[derive(Debug)] 68 | pub struct UpdateEvent { 69 | pub entry1: Vec>, 70 | pub entry2: Vec> 71 | } 72 | 73 | #[derive(Debug)] 74 | pub struct RotateEvent { 75 | } 76 | 77 | ////////////////////////////////////////////////////////////////////////////// 78 | ////////////////////////////////////////////////////////////////////////////// 79 | // Implements 80 | impl FormatDescriptorEvent { 81 | pub fn new() -> FormatDescriptorEvent { 82 | FormatDescriptorEvent{} 83 | } 84 | } 85 | 86 | impl XidEvent { 87 | pub fn new(xid: i64) -> XidEvent { 88 | XidEvent{ xid: xid } 89 | } 90 | } 91 | 92 | impl TableMapEvent { 93 | pub fn new(db_name: String, table_name: String) -> TableMapEvent { 94 | TableMapEvent{ db_name: db_name, table_name: table_name } 95 | } 96 | } 97 | 98 | impl InsertEvent { 99 | pub fn new(entry: Vec>) -> InsertEvent { 100 | InsertEvent{entry: entry} 101 | } 102 | } 103 | 104 | impl UpdateEvent { 105 | pub fn new(entry1: Vec>, entry2: Vec>) -> UpdateEvent { 106 | UpdateEvent{entry1: entry1, entry2: entry2} 107 | } 108 | } 109 | 110 | impl DeleteEvent { 111 | pub fn new(entry: Vec>) -> DeleteEvent { 112 | DeleteEvent{entry: entry} 113 | } 114 | } 115 | 116 | impl RotateEvent { 117 | pub fn new() -> RotateEvent { 118 | RotateEvent{} 119 | } 120 | } 121 | 122 | #[derive(Debug)] 123 | pub enum Event { 124 | Unknown, 125 | FormatDescriptor(FormatDescriptorEvent), 126 | Xid(XidEvent), 127 | Rotate(RotateEvent), 128 | TableMap(TableMapEvent), 129 | Delete(DeleteEvent), 130 | Insert(InsertEvent), 131 | Update(UpdateEvent), 132 | } 133 | -------------------------------------------------------------------------------- /src/rowevents/stream.rs: -------------------------------------------------------------------------------- 1 | 2 | use std::fs::File; 3 | use std::io::BufReader; 4 | use std::io::prelude::*; 5 | use std::option::Option; 6 | use std::process; 7 | use std::io::Result; 8 | use std::io::{Error, ErrorKind}; 9 | 10 | pub struct Stream { 11 | filename: String, 12 | file: Option, 13 | content: Vec, 14 | offset: usize, 15 | counter: usize, 16 | } 17 | 18 | 19 | fn get_next_binlog_filename(filename: &String) -> Option { 20 | let p = filename.rfind('.').unwrap(); 21 | let numstr = &filename[p + 1 ..]; 22 | if let Ok(num) = numstr.parse::() { 23 | let mut next = (&filename[.. p + 1]).to_owned(); 24 | next += &format!("{:0w$}", num + 1, w=numstr.len()); 25 | Some(next) 26 | } else { 27 | None 28 | } 29 | } 30 | 31 | impl Stream { 32 | 33 | pub fn from_file(filename: &str) -> Option { 34 | let mut result = File::open(filename); 35 | if let Ok(mut file) = result { 36 | Some(Stream { 37 | filename: filename.to_string(), 38 | file: Some(file), 39 | content: vec![], 40 | offset: 0, 41 | counter: 0 42 | }) 43 | } else { 44 | None 45 | } 46 | } 47 | 48 | pub fn read_next_binlog_file(&mut self) { 49 | if let Some(next_binlog_filename) = get_next_binlog_filename(&self.filename) { 50 | 51 | let mut result = File::open(&next_binlog_filename); 52 | if let Ok(mut file) = result { 53 | self.filename = next_binlog_filename; 54 | self.file = Some(file); 55 | self.content = vec![]; 56 | self.offset = 0; 57 | } 58 | } 59 | } 60 | 61 | pub fn read(&mut self, size: usize) -> &[u8] { 62 | let mut from = self.offset; 63 | if from + size >= self.content.len() { 64 | match self.read_file(size) { 65 | Ok(0) => { 66 | // println!("Reach the end of this binlog file"); 67 | return &[][..] 68 | }, 69 | Err(_) => { 70 | return &[][..] 71 | }, 72 | Ok(read) if read < size => { 73 | // Sometimes, especially when end of the file, read < size; 74 | println!("!{:?}", &self.content[from .. from + read]); 75 | return &self.content[from .. from + read] 76 | }, 77 | Ok(_) => {} 78 | } 79 | } 80 | let threshold: usize = 1000000; 81 | self.offset += size; 82 | if from >= threshold { 83 | let remain = self.content.drain(threshold .. ).collect(); 84 | self.content = remain; 85 | self.offset -= threshold; 86 | from -= threshold; 87 | println!("Resize content len => {}", self.content.len()); 88 | } 89 | 90 | &self.content[from .. from + size] 91 | } 92 | 93 | // try! Read size * 2 bytes from file 94 | pub fn read_file(&mut self, size: usize) -> Result { 95 | let mut buffer = Vec::with_capacity(size * 2); 96 | buffer.resize(size, 0); // TODO: Read more content into buffer, and reduce the read times 97 | if let Some(ref mut file) = self.file { 98 | let read = file.read(&mut buffer)?; 99 | self.counter += 1; // Read times + 1 100 | if read > 0 { 101 | // TO fix! 102 | // println!("DD {} {} {:?}", size, read, &buffer[0..read]); 103 | self.content.extend_from_slice(&buffer[0..read]); 104 | Ok(read) 105 | } else { 106 | Ok(0) 107 | } 108 | // TODO: the read MAYBE 0, should return Err 109 | } else { 110 | Err(Error::new(ErrorKind::Other, "oh no!")) 111 | } 112 | } 113 | } -------------------------------------------------------------------------------- /src/rowevents/descriptor.rs: -------------------------------------------------------------------------------- 1 | 2 | use std::io::Cursor; 3 | use std::io::Result; 4 | use rowevents::value_type::*; 5 | use byteorder::{LittleEndian, BigEndian, ReadBytesExt}; 6 | use rowevents::descriptor_datetime::*; 7 | use rowevents::descriptor_decimal::*; 8 | 9 | 10 | fn parse_string(metadata1: u8, metadata2: u8, data: &[u8]) -> Result<(ValueType, usize)> { 11 | let max_len = metadata1 + metadata2 * 256; // 12 | let strlen = data[0] as usize; 13 | let v = Vec::from(&data[1 .. strlen + 1]); 14 | Ok((ValueType::String(v), strlen + 1)) 15 | } 16 | 17 | fn parse_varchar(metadata1: u8, metadata2: u8, data: &[u8]) -> Result<(ValueType, usize)> { 18 | let max_len: u32 = metadata1 as u32 + metadata2 as u32 * 256; // Should ne LittleEndian 19 | let mut cursor = Cursor::new(&data); 20 | 21 | let (from, strlen) = if max_len < 256 { 22 | (1, cursor.read_u8()? as usize) 23 | } else { 24 | (2, cursor.read_i16::()? as usize) 25 | }; 26 | 27 | let v = Vec::from(&data[from .. strlen + from]); 28 | Ok((ValueType::String(v), strlen + from)) 29 | } 30 | 31 | fn parse_blob(metadata1: u8, metadata2: u8, data: &[u8]) -> Result<(ValueType, usize)> { 32 | let max_len = metadata1 + metadata2 * 256; 33 | let mut cursor = Cursor::new(&data); 34 | let strlen = cursor.read_i16::()? as usize; 35 | let v = Vec::from(&data[2 .. strlen + 2]); 36 | Ok((ValueType::String(v), strlen + 2)) 37 | } 38 | 39 | pub fn parse_field(field_type: u8, nullable: bool, metadata1: u8, metadata2: u8, data: &[u8]) -> Result<(ValueType, usize)> { 40 | 41 | let (value, offset) = if field_type == FieldType::Tiny as u8 { 42 | let mut cursor = Cursor::new(&data); 43 | let v:i8 = cursor.read_i8()?; 44 | (ValueType::Tinyint(v), 1) 45 | } else if field_type == FieldType::Short as u8 { 46 | let mut cursor = Cursor::new(&data); 47 | let v:i16 = cursor.read_i16::()?; 48 | (ValueType::Shortint(v), 2) 49 | } else if field_type == FieldType::Long as u8 { 50 | let mut cursor = Cursor::new(&data); 51 | let v:i32 = cursor.read_i32::()?; 52 | (ValueType::Int(v), 4) 53 | } else if field_type == FieldType::Longlong as u8 { 54 | let mut cursor = Cursor::new(&data); 55 | let v:i64 = cursor.read_i64::()?; 56 | (ValueType::Longlong(v), 8) 57 | } else if field_type == FieldType::Float as u8 { 58 | let mut cursor = Cursor::new(&data); 59 | let v:f32 = cursor.read_f32::()?; 60 | (ValueType::Float(v), 4) 61 | } else if field_type == FieldType::Double as u8 { 62 | let mut cursor = Cursor::new(&data); 63 | let v:f64 = cursor.read_f64::()?; 64 | (ValueType::Double(v), 8) 65 | } else if field_type == FieldType::Datetime2 as u8 { 66 | let (dt, offset) = parse_datetime2(metadata1, metadata2, data)?; 67 | (dt, offset) 68 | } else if field_type == FieldType::NewDecimal as u8 { 69 | let (dcm, offset) = parse_new_decimal(metadata1, metadata2, data)?; 70 | (dcm, offset) 71 | } else if field_type == FieldType::Varchar as u8 { 72 | let (strval, offset) = parse_varchar(metadata1, metadata2, data)?; 73 | (strval, offset) 74 | } else if field_type == FieldType::String as u8 { 75 | let (strval, offset) = parse_string(metadata1, metadata2, data)?; 76 | (strval, offset) 77 | } else if field_type == FieldType::Blob as u8 { 78 | let (strval, offset) = parse_blob(metadata1, metadata2, data)?; 79 | (strval, offset) 80 | } else if field_type == FieldType::Timestamp as u8 { 81 | let mut cursor = Cursor::new(&data); 82 | let v:u32 = cursor.read_u32::()?; 83 | (ValueType::Timestamp(v), 4) 84 | } else if field_type == FieldType::Timestamp2 as u8 { 85 | let mut cursor = Cursor::new(&data); 86 | let v:u32 = cursor.read_u32::()?; 87 | (ValueType::Timestamp(v), 4) 88 | } else { 89 | (ValueType::Unknown, 0) 90 | }; 91 | Ok((value, offset)) 92 | } -------------------------------------------------------------------------------- /python/pyapp/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../pylib') 3 | from mysqlbinlog import * 4 | from ctypes import * 5 | import platform, time, re 6 | from optparse import OptionParser 7 | 8 | 9 | def print_row(row): 10 | for item in row: 11 | if item is None: 12 | print('', end=', ') 13 | elif type(item) == bytes: 14 | print(str(item, 'utf8'), end=', ') 15 | else: 16 | print(item, end=', ') 17 | print() 18 | 19 | 20 | def print_updates(old, new): 21 | count, index = len(old), 0 22 | while index < count: 23 | print_row(old[index]) 24 | print_row(new[index]) 25 | index += 1 26 | 27 | def print_inserts(rows): 28 | for row in rows: 29 | print_row(row) 30 | 31 | def print_deletes(rows): 32 | for row in rows: 33 | print_row(row) 34 | 35 | def print_table_time_info(timestamp, db_table, operation): 36 | print("[%s] %s %s" % (formatted_time(timestamp), operation, db_table)) 37 | 38 | def main(options, args): 39 | reader = BinLogReader(options.binlog) 40 | # TODO: reader? 41 | quit_when_eof = options.quit_when_eof 42 | milliseconds = options.milliseconds 43 | 44 | reg_list = [] 45 | if options.ignore: 46 | ignore = options.ignore.replace('%', '\\w*').replace('.', '\\.') 47 | patterns = ignore.split(',') 48 | for pattern in patterns: 49 | reg = re.compile(pattern) 50 | reg_list.append(reg) 51 | 52 | event_header = EventHeader() 53 | 54 | count = 0 55 | skip = False 56 | current_db_table = '' 57 | while True: 58 | # print('-' * 30) 59 | h = reader.read_event_header(event_header) 60 | 61 | if not h: 62 | if quit_when_eof: 63 | break 64 | else: 65 | seconds = milliseconds / 1000 66 | time.sleep(seconds) 67 | continue 68 | 69 | event = reader.read_event(event_header) 70 | if not event: 71 | continue 72 | 73 | if skip: 74 | skip = False 75 | continue 76 | 77 | event_info = reader.read_event_info(event_header, event) 78 | if event_info.type_code == EventType.TABLE_MAP_EVENT: 79 | db, table = reader.read_table_map_event(event, event_info) 80 | full_name = db + '.' + table 81 | for reg in reg_list: 82 | if reg.match(full_name): 83 | skip = True 84 | # print('Skip', full_name) 85 | break 86 | if skip: 87 | continue 88 | current_db_table = full_name 89 | elif event_info.type_code == EventType.DELETE_ROWS_EVENT2: 90 | rows = reader.read_delete_event_rows(event, event_info) 91 | print_table_time_info(event_header.timestamp, current_db_table, 'delete') 92 | print_deletes(rows) 93 | elif event_info.type_code == EventType.UPDATE_ROWS_EVENT2: 94 | old, new = reader.read_update_event_rows(event, event_info) 95 | print_table_time_info(event_header.timestamp, current_db_table, 'update') 96 | print_updates(old, new) 97 | elif event_info.type_code == EventType.WRITE_ROWS_EVENT2: 98 | rows = reader.read_insert_event_rows(event, event_info) 99 | print_table_time_info(event_header.timestamp, current_db_table, 'insert') 100 | print_inserts(rows) 101 | 102 | reader.free_event(event) 103 | count += 1 104 | 105 | reader.close() 106 | 107 | 108 | if __name__ == '__main__': 109 | parser = OptionParser() 110 | 111 | parser.add_option("-s", "--source", action="store", dest="source", help="Provide source database") 112 | parser.add_option("-b", "--binlog", action="store", dest="binlog", help="Provide binlog file name") 113 | parser.add_option("-l", "--highlight", action="store", dest="highlight", help="Highlights the differences") 114 | parser.add_option("-i", "--ignore", action="store", dest="ignore", help="The db and table pattern to ignore") 115 | parser.add_option("-q", "--quit-when-eof", action="store", dest="quit_when_eof", help="Quit the program when EOF?", default=False) 116 | parser.add_option("-m", "--milliseconds", action="store", dest="milliseconds", help="Provide sleep seconds", default=100) 117 | options, args = parser.parse_args() 118 | 119 | if not options.binlog: 120 | print("binlog filename is required") 121 | exit() 122 | 123 | b = time.clock() 124 | main(options, args) 125 | e = time.clock() 126 | print('\nCost', e - b, 'seconds') -------------------------------------------------------------------------------- /src/rowevents/reader.rs: -------------------------------------------------------------------------------- 1 | 2 | use rowevents::parser::Parser; 3 | use rowevents::stream::Stream; 4 | use rowevents::event_header::EventHeader; 5 | use rowevents::events::*; 6 | use std::io::Result; 7 | use std::io::{Error, ErrorKind}; 8 | extern crate regex; 9 | use regex::Regex; 10 | 11 | pub struct Reader { 12 | filename: String, 13 | parser: Parser, 14 | skip_next_event: bool, 15 | concerned_events: Vec, 16 | excluded_db_table_list: Vec, 17 | } 18 | 19 | impl Reader { 20 | 21 | pub fn new(filename: &str) -> Result { 22 | 23 | if let Some(stream) = Stream::from_file(filename) { 24 | let mut parser = Parser::new(stream); 25 | parser.read_binlog_file_header(); 26 | Ok(Reader{ 27 | filename: filename.to_string(), 28 | parser: parser, 29 | skip_next_event: false, 30 | concerned_events: Vec::with_capacity(20), 31 | excluded_db_table_list: Vec::with_capacity(20), 32 | }) 33 | } else { 34 | Err(Error::new(ErrorKind::Other, "oh no!")) 35 | } 36 | } 37 | 38 | pub fn open_next_binlog_file(&mut self) -> bool { 39 | self.parser.read_next_binlog_file(); 40 | self.parser.read_binlog_file_header() 41 | } 42 | 43 | #[inline] 44 | pub fn add_concerned_event(&mut self, event_type: i8) { 45 | self.concerned_events.push(event_type); 46 | } 47 | 48 | #[inline] 49 | pub fn is_concerned_event(&mut self, event_type: i8) -> bool { 50 | self.concerned_events.len() == 0 || self.concerned_events.contains(&event_type) 51 | } 52 | 53 | pub fn add_excluded_db_table(&mut self, db_table_name: &str) { 54 | let regexp = db_table_name.replace(".", "\\."); 55 | let regexp = regexp.replace("*", "\\w*"); 56 | self.excluded_db_table_list.push(Regex::new(®exp).unwrap()); 57 | } 58 | 59 | pub fn is_excluded(&mut self, db_name: &str, table_name: &str) -> bool { 60 | let db_table_name = db_name.to_string() + "." + table_name; 61 | for ref re in self.excluded_db_table_list.iter() { 62 | if re.is_match(&db_table_name) { 63 | return true; 64 | } 65 | } 66 | return false; 67 | } 68 | 69 | // ???? 70 | pub fn read_event(&mut self) -> Result<(EventHeader, Event)> { 71 | if let Ok(eh) = self.read_event_header() { 72 | 73 | if self.skip_next_event || !self.is_concerned_event(eh.get_event_type()) { 74 | if let Ok(e) = self.read_unknown_event(&eh) { 75 | // Recover from skip 76 | self.set_skip_next_event(false); 77 | Ok((eh, e)) 78 | } else { 79 | Err(Error::new(ErrorKind::Other, "oh no!")) 80 | } 81 | } else if let Ok(e) = self.read_event_detail(&eh) { 82 | match e { 83 | Event::TableMap(ref e) => { 84 | if self.is_excluded(&e.db_name, &e.table_name) { 85 | // println!("Excluded {}.{}", e.db_name, e.table_name); 86 | self.set_skip_next_event(true); 87 | } 88 | }, 89 | 90 | Event::Rotate(ref e) => { 91 | println!("Open next binlog file..."); 92 | self.open_next_binlog_file(); 93 | }, 94 | _ => () 95 | } 96 | 97 | Ok((eh, e)) 98 | 99 | } else { 100 | Err(Error::new(ErrorKind::Other, "oh no!")) 101 | } 102 | } else { 103 | Err(Error::new(ErrorKind::Other, "oh no!")) 104 | } 105 | } 106 | 107 | #[inline] 108 | pub fn read_event_header(&mut self) -> Result { 109 | self.parser.read_event_header() 110 | } 111 | 112 | pub fn read_event_detail(&mut self, eh: &EventHeader) -> Result { 113 | 114 | match eh.get_event_type() { 115 | QUERY_EVENT => self.parser.read_query_event(eh), 116 | 117 | STOP_EVENT | ROTATE_EVENT => { 118 | let e = self.parser.read_rotate_event(eh); 119 | self.open_next_binlog_file(); 120 | e 121 | }, 122 | 123 | FORMAT_DESCRIPTION_EVENT => self.parser.read_format_descriptor_event(eh), 124 | XID_EVENT => self.parser.read_xid_event(eh), 125 | 126 | TABLE_MAP_EVENT => self.parser.read_table_map_event(eh), 127 | 128 | // WRITE_ROWS_EVENT => self.parser.read_event(eh), 129 | // UPDATE_ROWS_EVENT => self.parser.read_event(eh), 130 | // DELETE_ROWS_EVENT => self.parser.read_event(eh), 131 | 132 | WRITE_ROWS_EVENT2 => self.parser.read_write_event(eh), 133 | UPDATE_ROWS_EVENT2 => self.parser.read_update_event(eh), 134 | DELETE_ROWS_EVENT2 => self.parser.read_delete_event(eh), 135 | 136 | _ => self.parser.read_unknown_event(eh) 137 | } 138 | } 139 | 140 | pub fn read_unknown_event(&mut self, eh: &EventHeader) -> Result { 141 | self.parser.read_unknown_event(eh) 142 | } 143 | 144 | pub fn set_skip_next_event(&mut self, skip: bool) { 145 | self.skip_next_event = skip; 146 | } 147 | 148 | #[inline] 149 | pub fn skip_next_event(&self) -> bool { 150 | self.skip_next_event 151 | } 152 | } 153 | 154 | impl Iterator for Reader { 155 | type Item = (EventHeader, Event); 156 | 157 | // next() is the only required method 158 | fn next(&mut self) -> Option<(EventHeader, Event)> { 159 | if let Ok((eh, e)) = self.read_event() { 160 | Some((eh, e)) 161 | } else { 162 | None 163 | } 164 | } 165 | } 166 | 167 | -------------------------------------------------------------------------------- /python/pylib/mysqlbinlog.py: -------------------------------------------------------------------------------- 1 | 2 | from ctypes import * 3 | import time, sys, os 4 | from decimal import Decimal as D 5 | 6 | # TODO: setup.py 7 | class EventType: 8 | UNKNOWN_EVENT = 0 9 | START_EVENT_V3 = 1 10 | QUERY_EVENT = 2 11 | STOP_EVENT = 3 12 | ROTATE_EVENT = 4 13 | INTVAR_EVENT = 5 14 | LOAD_EVENT = 6 15 | SLAVE_EVENT = 7 16 | CREATE_FILE_EVENT = 8 17 | APPEND_BLOCK_EVENT = 9 18 | EXEC_LOAD_EVENT = 10 19 | DELETE_FILE_EVENT = 11 20 | NEW_LOAD_EVENT = 12 21 | RAND_EVENT = 13 22 | USER_VAR_EVENT = 14 23 | FORMAT_DESCRIPTION_EVENT = 15 24 | XID_EVENT = 16 25 | BEGIN_LOAD_QUERY_EVENT = 17 26 | EXECUTE_LOAD_QUERY_EVENT = 18 27 | TABLE_MAP_EVENT = 19 28 | PRE_GA_WRITE_ROWS_EVENT = 20 29 | PRE_GA_UPDATE_ROWS_EVENT = 21 30 | PRE_GA_DELETE_ROWS_EVENT = 22 31 | """ 32 | From MySQL 5.1.18 events 33 | """ 34 | WRITE_ROWS_EVENT = 23 35 | UPDATE_ROWS_EVENT = 24 36 | DELETE_ROWS_EVENT = 25 37 | # ---------------------------------- 38 | 39 | INCIDENT_EVENT = 26 40 | HEARTBEAT_LOG_EVENT = 27 41 | 42 | """ 43 | From MySQL 5.6.2 events 44 | """ 45 | WRITE_ROWS_EVENT2 = 30 46 | UPDATE_ROWS_EVENT2 = 31 47 | DELETE_ROWS_EVENT2 = 32 48 | # ---------------------------------- 49 | 50 | def formatted_time(unixtime): 51 | return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(unixtime)) 52 | 53 | 54 | class EventHeader(Structure): 55 | _fields_ = [ 56 | ("timestamp", c_int), 57 | ("type_code", c_byte), 58 | ("server_id", c_int), 59 | ("event_len", c_int), 60 | ("next_pos", c_int), 61 | ("flags", c_short) 62 | ] 63 | 64 | def __str__(self): 65 | return "" % (formatted_time(self.timestamp), self.type_code) 66 | 67 | 68 | class EventInfo(Structure): 69 | 70 | def __init__(self, event_header): 71 | self.type_code = event_header.type_code 72 | 73 | _fields_ = [ 74 | ("type_code", c_byte), 75 | ("db_name_len", c_uint), 76 | ("table_name_len", c_uint), 77 | ("row_count", c_uint), 78 | ("col_count", c_uint) 79 | ] 80 | 81 | 82 | def timestamp_datetime(value): 83 | format = '%Y-%m-%d %H:%M:%S' 84 | # value为传入的值为时间戳(整形)，如：1332888820 85 | value = time.localtime(value) 86 | ## 经过localtime转换后变成 87 | ## time.struct_time(tm_year=2012, tm_mon=3, tm_mday=28, tm_hour=6, tm_min=53, tm_sec=40, tm_wday=2, tm_yday=88, tm_isdst=0) 88 | # 最后再经过strftime函数转换为正常日期格式。 89 | dt = time.strftime(format, value) 90 | return dt 91 | 92 | class FieldInfo(Structure): 93 | _fields_ = [ 94 | ("field_type", c_uint32), 95 | ("field_len", c_uint32), 96 | ("field_value", c_int64) 97 | ] 98 | 99 | def value(self): 100 | if self.field_type in [1, 2, 3, 8]: 101 | return self.field_value 102 | elif self.field_type == 253: 103 | return string_at(self.field_value, self.field_len) 104 | elif self.field_type == 4 or self.field_type == 5: 105 | u = self.as_utf8_str() 106 | if u == '': 107 | return '' 108 | return float(u) 109 | elif self.field_type == 246: 110 | s = str(string_at(self.field_value, self.field_len), 'utf-8') 111 | return D(s) 112 | elif self.field_type == 6: 113 | return None 114 | elif self.field_type == 7 or self.field_type == 17: 115 | return timestamp_datetime(self.field_value) 116 | else: 117 | return '?' 118 | 119 | def as_utf8_str(self): 120 | b = string_at(self.field_value, self.field_len) 121 | return str(b, 'utf-8') 122 | 123 | def __str__(self): 124 | return "<%s: %s>" % (self.field_type, self.field_value) 125 | 126 | 127 | class BinLogReader: 128 | """ 129 | """ 130 | 131 | def __init__(self, filename, debug=False): 132 | project_path = os.path.realpath(os.path.join(__file__, '../../../')) 133 | path = 'debug' if debug else 'release' 134 | self.d = cdll.LoadLibrary('%s/target/%s/libmysqlbinlog.dylib' % (project_path, path)) 135 | 136 | self.d.binlog_reader_new.restype = c_void_p 137 | self.reader = self.d.binlog_reader_new(bytes(filename, 'utf8')) 138 | if not self.reader: 139 | print("Failed to open '%s'." % filename) 140 | exit() 141 | # 142 | self.d.binlog_reader_free.argtypes = [c_void_p] 143 | # 144 | self.d.binlog_reader_read_event_header.restype = c_byte 145 | self.d.binlog_reader_read_event_header.argtypes = [c_void_p, POINTER(EventHeader)] 146 | 147 | # 148 | self.d.binlog_reader_read_event.restype = c_void_p 149 | self.d.binlog_reader_read_event.argtypes = [c_void_p, POINTER(EventHeader)] 150 | 151 | # 152 | self.d.binlog_reader_read_event_info.restype = c_bool 153 | self.d.binlog_reader_read_event_info.argtypes = [c_void_p, POINTER(EventInfo)] 154 | 155 | # 156 | self.d.binlog_reader_read_table_map_event.restype = c_bool 157 | self.d.binlog_reader_read_table_map_event.argtypes = [c_void_p, POINTER(EventInfo), c_char_p, c_char_p] 158 | 159 | # 160 | self.d.binlog_reader_read_rows_event_content.restype = c_bool 161 | # content_t is dynamic 162 | # self.d.binlog_reader_read_rows_event_content.argtypes = [c_void_p, POINTER(EventInfo), POINTER(content_t), c_bool] 163 | 164 | # 165 | self.d.binlog_reader_free_event.restype = c_bool 166 | self.d.binlog_reader_free_event.argtypes = [c_void_p] 167 | 168 | # 169 | def close(self): 170 | self.d.binlog_reader_free(self.reader) 171 | 172 | def read_event_header(self, header): 173 | b = self.d.binlog_reader_read_event_header(self.reader, byref(header)) 174 | return b 175 | 176 | def read_event(self, header): 177 | """ 178 | """ 179 | event = self.d.binlog_reader_read_event(self.reader, byref(header)) 180 | return event 181 | 182 | 183 | def read_event_info(self, event_header, event): 184 | """ 185 | """ 186 | event_info = EventInfo(event_header) 187 | self.d.binlog_reader_read_event_info(event, byref(event_info)) 188 | return event_info 189 | 190 | def read_table_map_event(self, event, event_info): 191 | db_name_t = c_char * event_info.db_name_len 192 | table_name_t = c_char * event_info.table_name_len 193 | 194 | db_name = db_name_t() 195 | table_name = table_name_t() 196 | 197 | self.d.binlog_reader_read_table_map_event(event, byref(event_info), db_name, table_name) 198 | db_name = str(db_name.value, 'utf-8') 199 | table_name = str(table_name.value, 'utf-8') 200 | return db_name, table_name 201 | 202 | # 203 | def __parse_content(self, content, row_count, col_count): 204 | i, j = 0, 0 205 | rows = [] 206 | for i in range(0, row_count): 207 | row = [] 208 | for j in range(0, col_count): 209 | index = i * col_count + j 210 | f = content[index].value() 211 | row.append(f) 212 | rows.append(row) 213 | return rows 214 | 215 | # new_entry is for update rows event 216 | def read_rows_event_content(self, event, event_info, new_entry=True): 217 | ''' 218 | ''' 219 | count = event_info.row_count * event_info.col_count 220 | content_t = FieldInfo * count 221 | content = content_t() 222 | # TODO: Cache the restype and argtypes 223 | # self.d.binlog_reader_read_rows_event_content.restype = c_bool 224 | self.d.binlog_reader_read_rows_event_content.argtypes = [c_void_p, POINTER(EventInfo), POINTER(content_t), c_bool] 225 | self.d.binlog_reader_read_rows_event_content(event, byref(event_info), byref(content), new_entry) 226 | 227 | items = self.__parse_content(content, event_info.row_count, event_info.col_count) 228 | 229 | self.d.binlog_reader_free_rows_event_content.argtypes = [c_void_p, POINTER(EventInfo), POINTER(content_t)] 230 | self.d.binlog_reader_free_rows_event_content(event, byref(event_info), byref(content)) 231 | 232 | return items 233 | 234 | 235 | def read_insert_event_rows(self, event, event_info): 236 | return self.read_rows_event_content(event, event_info, True) 237 | 238 | def read_update_event_rows(self, event, event_info): 239 | old = self.read_rows_event_content(event, event_info, False) 240 | new = self.read_rows_event_content(event, event_info, True) 241 | return old, new 242 | # 243 | def read_delete_event_rows(self, event, event_info): 244 | return self.read_rows_event_content(event, event_info, True) 245 | 246 | 247 | def free_event(self, event): 248 | """ 249 | """ 250 | return self.d.binlog_reader_free_event(event) 251 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | 2 | pub use rowevents::reader::{ Reader }; 3 | pub use rowevents::stream::{ Stream }; 4 | pub use rowevents::parser::{ Parser }; 5 | pub use rowevents::event_header::{ EventHeader }; 6 | pub use rowevents::events::*; 7 | pub use rowevents::value_type::*; 8 | pub use rowevents::descriptor::*; 9 | 10 | pub mod rowevents; 11 | 12 | extern crate byteorder; 13 | extern crate chrono; 14 | extern crate regex; 15 | 16 | 17 | use std::ffi::{CString, CStr}; 18 | use std::os::raw::c_char; 19 | use std::ptr; 20 | use std::rc::Rc; 21 | 22 | #[no_mangle] 23 | pub extern fn binlog_reader_new(filename: *const c_char) -> *mut Reader { 24 | let c = unsafe { 25 | let cstr = CStr::from_ptr(filename); 26 | cstr.to_string_lossy().into_owned() 27 | }; 28 | 29 | if let Ok(reader) = Reader::new(&c) { 30 | let p = Box::into_raw(Box::new(reader)); 31 | p 32 | } else { 33 | ptr::null_mut() 34 | } 35 | } 36 | 37 | #[no_mangle] 38 | pub extern fn binlog_reader_free(ptr: *mut Reader) { 39 | if ptr.is_null() { 40 | return 41 | } 42 | unsafe { 43 | Box::from_raw(ptr); 44 | } 45 | } 46 | 47 | #[no_mangle] 48 | pub extern fn binlog_reader_read_event_header(ptr: *mut Reader, in_header: *mut EventHeader) -> bool { 49 | let mut reader = unsafe { 50 | assert!(!ptr.is_null()); 51 | &mut *ptr 52 | }; 53 | 54 | if let Ok(ref mut header) = reader.read_event_header() { 55 | // Copy for avoid alloc too much heap-memory 56 | unsafe { 57 | (*in_header).type_code = header.type_code; 58 | (*in_header).timestamp = header.timestamp; 59 | (*in_header).server_id = header.server_id; 60 | (*in_header).event_len = header.event_len; 61 | (*in_header).next_pos = header.next_pos; 62 | (*in_header).flags = header.flags 63 | } 64 | true 65 | } else { 66 | false 67 | } 68 | } 69 | 70 | /////////////////////////////////////// 71 | // For C code read the event 72 | #[derive(Debug)] 73 | #[repr(C)] 74 | pub struct EventInfo { 75 | pub type_code: u8, 76 | pub db_name_len: u32, 77 | pub table_name_len: u32, 78 | pub row_count: u32, 79 | pub col_count: u32 80 | } 81 | 82 | /////////////////////////////////////// 83 | // For C code read the event 84 | #[derive(Debug)] 85 | #[repr(C)] 86 | pub struct FieldInfo { 87 | pub field_type: u32, 88 | pub field_len: u32, 89 | pub field_value: i64, 90 | 91 | } 92 | 93 | #[no_mangle] 94 | pub extern fn binlog_reader_read_event(ptr: *mut Reader, header: *mut EventHeader) -> *mut Event { 95 | let mut reader = unsafe { 96 | assert!(!ptr.is_null()); 97 | &mut *ptr 98 | }; 99 | 100 | let header = unsafe { 101 | assert!(!header.is_null()); 102 | &mut *header 103 | }; 104 | 105 | if let Ok(event) = reader.read_event_detail(&header) { 106 | Box::into_raw(Box::new(event)) 107 | } else { 108 | ptr::null_mut() 109 | } 110 | } 111 | 112 | #[no_mangle] 113 | pub extern fn binlog_reader_read_event_info(ptr: *mut Event, info: *mut EventInfo) -> bool { 114 | let event = unsafe { 115 | assert!(!ptr.is_null()); 116 | &*ptr 117 | }; 118 | 119 | match event { 120 | &Event::TableMap(ref e) => { 121 | unsafe { 122 | (*info).db_name_len = e.db_name.len() as u32; 123 | (*info).table_name_len = e.table_name.len() as u32; 124 | } 125 | }, 126 | 127 | &Event::Insert(ref e) => { 128 | unsafe { 129 | (*info).row_count = e.entry.len() as u32; 130 | (*info).col_count = e.entry[0].len() as u32; 131 | } 132 | }, 133 | 134 | &Event::Delete(ref e) => { 135 | unsafe { 136 | (*info).row_count = e.entry.len() as u32; 137 | (*info).col_count = e.entry[0].len() as u32; 138 | } 139 | }, 140 | 141 | &Event::Update(ref e) => { 142 | unsafe { 143 | (*info).row_count = e.entry1.len() as u32; 144 | (*info).col_count = e.entry1[0].len() as u32; 145 | } 146 | }, 147 | 148 | _ => { 149 | 150 | } 151 | } 152 | 153 | true 154 | } 155 | 156 | 157 | fn read_event_rows(entry_vec: &Vec>, content: &mut [FieldInfo]) -> bool { 158 | let mut index = 0; 159 | 160 | for entry in entry_vec { 161 | for v in entry { 162 | 163 | match v { 164 | &ValueType::Tinyint(i) => { 165 | content[index].field_type = FieldType::Tiny as u32; 166 | content[index].field_len = 1; 167 | content[index].field_value = i as i64; 168 | }, 169 | 170 | &ValueType::Shortint(i) => { 171 | content[index].field_type = FieldType::Short as u32; 172 | content[index].field_len = 2; 173 | content[index].field_value = i as i64; 174 | }, 175 | &ValueType::Int(i) => { 176 | content[index].field_type = FieldType::Long as u32; 177 | content[index].field_len = 4; 178 | content[index].field_value = i as i64; 179 | }, 180 | 181 | &ValueType::Longlong(i) => { 182 | content[index].field_type = FieldType::Longlong as u32; 183 | content[index].field_len = 8; 184 | content[index].field_value = i as i64; 185 | }, 186 | 187 | &ValueType::Float(f) => { 188 | content[index].field_type = FieldType::Float as u32; 189 | let s = format!("{}", f); 190 | content[index].field_len = s.len() as u32; 191 | content[index].field_value = CString::new(s.as_bytes()).unwrap().into_raw() as i64; 192 | }, 193 | 194 | &ValueType::Double(f) => { 195 | content[index].field_type = FieldType::Double as u32; 196 | let s = format!("{}", f); 197 | content[index].field_len = s.len() as u32; 198 | content[index].field_value = CString::new(s.as_bytes()).unwrap().into_raw() as i64; 199 | }, 200 | 201 | &ValueType::Decimal(ref d) => { 202 | content[index].field_type = FieldType::NewDecimal as u32; 203 | content[index].field_len = d.len() as u32; 204 | content[index].field_value = CString::new(d.as_bytes()).unwrap().into_raw() as i64; 205 | }, 206 | 207 | &ValueType::String(ref i) => { 208 | content[index].field_type = FieldType::VarString as u32; 209 | content[index].field_len = i.len() as u32; 210 | let s = String::from_utf8(i.to_vec()).unwrap(); 211 | content[index].field_value = CString::new(s).unwrap().into_raw() as i64; 212 | }, 213 | 214 | &ValueType::Timestamp(i) => { 215 | content[index].field_type = FieldType::Timestamp as u32; 216 | content[index].field_len = 0; 217 | content[index].field_value = i as i64; 218 | }, 219 | 220 | &ValueType::Null => { 221 | content[index].field_type = FieldType::Null as u32; 222 | content[index].field_len = 0; 223 | content[index].field_value = 0; 224 | }, 225 | _ => { 226 | // println!("==>{:?}", v); 227 | } 228 | } 229 | 230 | index += 1; 231 | } 232 | 233 | } 234 | // println!("********** index, {}", index); 235 | true 236 | } 237 | 238 | fn free_event_rows(entry_vec: &Vec>, content: &mut [FieldInfo]) -> bool { 239 | let mut index = 0; 240 | for entry in entry_vec { 241 | for v in entry { 242 | match v { 243 | 244 | &ValueType::Float(f) => { 245 | unsafe { 246 | CString::from_raw(content[index].field_value as *mut c_char); 247 | } 248 | }, 249 | 250 | &ValueType::Double(f) => { 251 | unsafe { 252 | CString::from_raw(content[index].field_value as *mut c_char); 253 | } 254 | }, 255 | 256 | &ValueType::Decimal(ref d) => { 257 | unsafe { 258 | CString::from_raw(content[index].field_value as *mut c_char); 259 | } 260 | }, 261 | 262 | &ValueType::String(ref i) => { 263 | unsafe { 264 | CString::from_raw(content[index].field_value as *mut c_char); 265 | } 266 | }, 267 | 268 | _ => { 269 | } 270 | } 271 | 272 | index += 1; 273 | } 274 | 275 | } 276 | // println!("********** index, {}", index); 277 | true 278 | } 279 | 280 | #[no_mangle] 281 | pub extern fn binlog_reader_read_table_map_event(ptr: *mut Event, info: *mut EventInfo, db_name: *mut u8, table_name: *mut u8) -> bool { 282 | let event = unsafe { 283 | assert!(!ptr.is_null()); 284 | &*ptr 285 | }; 286 | 287 | match event { 288 | &Event::TableMap(ref e) => { 289 | unsafe { 290 | ptr::copy_nonoverlapping(e.db_name.as_bytes().as_ptr(), db_name, e.db_name.len()); 291 | ptr::copy_nonoverlapping(e.table_name.as_bytes().as_ptr(), table_name, e.table_name.len()); 292 | } 293 | }, 294 | _ => {} 295 | } 296 | true 297 | } 298 | 299 | #[no_mangle] 300 | pub extern fn binlog_reader_read_delete_event_rows(ptr: *mut Event, info: *mut EventInfo, content: &mut [FieldInfo]) -> bool { 301 | 302 | true 303 | } 304 | 305 | #[no_mangle] 306 | pub extern fn binlog_reader_read_insert_event_rows(ptr: *mut Event, info: *mut EventInfo, content: &mut [FieldInfo]) -> bool { 307 | 308 | true 309 | } 310 | 311 | #[no_mangle] 312 | pub extern fn binlog_reader_read_rows_event_content(ptr: *mut Event, info: *mut EventInfo, content: *mut FieldInfo, new_entry: bool) -> bool { 313 | let content : &mut [FieldInfo] = unsafe { 314 | let size = ((*info).row_count * (*info).col_count) as usize; 315 | std::slice::from_raw_parts_mut(content, size) 316 | }; 317 | 318 | let event = unsafe { 319 | assert!(!ptr.is_null()); 320 | &*ptr 321 | }; 322 | 323 | match event { 324 | &Event::Update(ref e) => { 325 | if new_entry { 326 | read_event_rows(&e.entry2, content); 327 | } else { 328 | read_event_rows(&e.entry1, content); 329 | } 330 | }, 331 | &Event::Insert(ref e) => { 332 | read_event_rows(&e.entry, content); 333 | }, 334 | &Event::Delete(ref e) => { 335 | read_event_rows(&e.entry, content); 336 | }, 337 | _ => { 338 | assert!(false); 339 | } 340 | } 341 | 342 | true 343 | } 344 | 345 | #[no_mangle] 346 | pub extern fn binlog_reader_free_rows_event_content(ptr: *mut Event, info: *mut EventInfo, content: *mut FieldInfo) -> bool { 347 | let content : &mut [FieldInfo] = unsafe { 348 | let size = ((*info).row_count * (*info).col_count) as usize; 349 | std::slice::from_raw_parts_mut(content, size) 350 | }; 351 | 352 | let event = unsafe { 353 | assert!(!ptr.is_null()); 354 | &*ptr 355 | }; 356 | 357 | match event { 358 | &Event::Update(ref e) => { 359 | // entry1 and entry2 have same row * column numbers 360 | free_event_rows(&e.entry1, content); 361 | }, 362 | &Event::Insert(ref e) => { 363 | free_event_rows(&e.entry, content); 364 | }, 365 | &Event::Delete(ref e) => { 366 | read_event_rows(&e.entry, content); 367 | }, 368 | _ => { 369 | assert!(false); 370 | } 371 | } 372 | true 373 | } 374 | 375 | #[no_mangle] 376 | pub extern fn binlog_reader_free_event(ptr: *mut Event) -> bool { 377 | if ptr.is_null() { 378 | return false; 379 | } 380 | unsafe { 381 | Box::from_raw(ptr); 382 | } 383 | return true; 384 | } 385 | -------------------------------------------------------------------------------- /src/rowevents/parser.rs: -------------------------------------------------------------------------------- 1 | 2 | use rowevents::stream::Stream; 3 | use rowevents::event_header::EventHeader; 4 | use rowevents::events::*; 5 | use rowevents::value_type; 6 | use rowevents::value_type::*; 7 | use rowevents::descriptor::*; 8 | use byteorder::{LittleEndian, ReadBytesExt}; 9 | //use rowevents::value_type::ValueType; 10 | 11 | use std::option::Option; 12 | use std::io::Cursor; 13 | use std::io::Result; 14 | use std::io::{Error, ErrorKind}; 15 | use std::str; 16 | 17 | pub struct Parser { 18 | stream: Stream, 19 | field_types: Vec<(u8, bool, u8, u8)> 20 | } 21 | 22 | fn get_table_id(i1: i64, i2: i64, i3: i64) -> i64 { 23 | i1 << 32 + i2 << 16 + i3 24 | } 25 | 26 | fn bytes_2_leuint(bytes: &[u8]) -> i64 { 27 | let mut n: i64 = 0; 28 | let mut i = 0; 29 | for b in bytes { 30 | let m = *b as i64; 31 | n += m << (i * 8); 32 | i += 1; 33 | } 34 | n 35 | } 36 | 37 | fn get_field_length(data: &[u8]) -> (i64, usize) { 38 | if data.len() == 0 { 39 | return (-1, 0) 40 | } 41 | let v = data[0] as i64; 42 | if v < 251 { 43 | return (v, 1) 44 | } else if v == 251 { 45 | return (-1, 0) 46 | } 47 | let mut size = 9; 48 | if v == 252 { 49 | size = 3; 50 | } else if v == 253 { 51 | size = 4; 52 | } 53 | 54 | if data.len() < size { 55 | return (-1, 0) 56 | } 57 | return (bytes_2_leuint(&data[1 .. size]), size) 58 | } 59 | 60 | impl Parser { 61 | 62 | pub fn new(stream: Stream) -> Parser { 63 | Parser{ 64 | stream: stream, 65 | field_types: Vec::with_capacity(100) 66 | } 67 | } 68 | 69 | pub fn read_binlog_file_header(&mut self) -> bool { 70 | self.stream.read(4); 71 | true 72 | } 73 | 74 | pub fn read_next_binlog_file(&mut self) -> bool { 75 | self.stream.read_next_binlog_file(); 76 | true 77 | } 78 | 79 | pub fn read_event_header(&mut self) -> Result { 80 | 81 | let data = self.stream.read(19); 82 | if data.len() > 0 { 83 | let mut cursor = Cursor::new(&data); 84 | 85 | let timestamp = cursor.read_i32::()?; 86 | let type_code = cursor.read_i8()?; 87 | let server_id = cursor.read_i32::()?; 88 | let event_len = cursor.read_i32::()?; 89 | let next_pos = cursor.read_i32::()?; 90 | let flags = cursor.read_i16::()?; 91 | 92 | Ok(EventHeader::new( 93 | timestamp, 94 | type_code, 95 | server_id, 96 | event_len, 97 | next_pos, 98 | flags 99 | )) 100 | } else { 101 | Err(Error::new(ErrorKind::Other, "Nothing Read!")) 102 | } 103 | } 104 | 105 | pub fn read_unknown_event(&mut self, eh: &EventHeader) -> Result { 106 | let data = self.stream.read(eh.get_event_len() - 19); 107 | Ok(Event::Unknown) 108 | } 109 | 110 | pub fn read_rotate_event(&mut self, eh: &EventHeader) -> Result { 111 | Ok(Event::Rotate(RotateEvent::new())) 112 | } 113 | 114 | pub fn read_format_descriptor_event(&mut self, eh: &EventHeader) -> Result { 115 | { 116 | let data = self.stream.read(57); 117 | } 118 | 119 | let length_array = self.stream.read(eh.get_event_len() - (57 + 19)); 120 | Ok(Event::FormatDescriptor(FormatDescriptorEvent::new())) 121 | } 122 | 123 | pub fn read_xid_event(&mut self, eh: &EventHeader) -> Result { 124 | let data = self.stream.read(12); 125 | let mut cursor = Cursor::new(&data); 126 | let xid = cursor.read_i64::()?; 127 | println!("XID={}", xid); 128 | Ok(Event::Xid(XidEvent::new(xid))) 129 | } 130 | 131 | pub fn read_table_map_event(&mut self, eh: &EventHeader) -> Result { 132 | let mut db_name_len = 0; 133 | let mut table_name_len = 0; 134 | { 135 | let data = self.stream.read(9); 136 | let mut cursor = Cursor::new(&data); 137 | let i1 = cursor.read_i16::()?; 138 | let i2 = cursor.read_i16::()?; 139 | let i3 = cursor.read_i16::()?; 140 | let table_id = get_table_id(i1 as i64, i2 as i64, i3 as i64); 141 | let flags = cursor.read_i16::()?; 142 | db_name_len = cursor.read_i8()? as usize; 143 | } 144 | 145 | let db_name = { 146 | let db_name_data = self.stream.read(db_name_len as usize); 147 | String::from_utf8_lossy(db_name_data).into_owned() 148 | }; 149 | self.stream.read(1); // Read more 1 byte(This byte is for zero-ending?) 150 | 151 | { 152 | let table_name_len_data = self.stream.read(1); 153 | let mut cursor = Cursor::new(&table_name_len_data); 154 | table_name_len = cursor.read_i8()? as usize 155 | } 156 | 157 | let table_name = { 158 | let table_name_data = self.stream.read(table_name_len as usize); 159 | String::from_utf8_lossy(table_name_data).into_owned() 160 | }; 161 | 162 | self.stream.read(1); // Read more 1 byte(This byte is for zero-ending?) 163 | 164 | let data_len = eh.get_event_len() - 19 - 16 - db_name_len - table_name_len + 4; 165 | let content = { 166 | let data = self.stream.read(data_len); 167 | Vec::from(data) 168 | }; 169 | 170 | { 171 | self.parse_current_fields_discriptors(&content); 172 | } 173 | 174 | Ok(Event::TableMap(TableMapEvent::new(db_name, table_name))) 175 | } 176 | 177 | pub fn read_query_event(&mut self, eh: &EventHeader) -> Result { 178 | self.read_unknown_event(eh) 179 | } 180 | 181 | pub fn read_write_event(&mut self, eh: &EventHeader) -> Result { 182 | if let Ok((v, col_count)) = self.read_rows_event(eh, false) { 183 | let mut from:usize = 0; 184 | let len = v.len(); 185 | let offset = 0; 186 | let mut rows = vec![]; 187 | while from < len { 188 | let (values, offset) = self.parse_row_values(&v[from..], col_count); 189 | from += offset; 190 | rows.push(values); 191 | } 192 | let e = InsertEvent::new(rows); 193 | Ok(Event::Insert(e)) 194 | } else { 195 | // TODO:? 196 | let e = InsertEvent::new(vec![]); 197 | Ok(Event::Insert(e)) 198 | } 199 | } 200 | 201 | pub fn read_update_event(&mut self, eh: &EventHeader) -> Result { 202 | if let Ok((v, col_count)) = self.read_rows_event(eh, true) { 203 | let mut from:usize = 0; 204 | let len = v.len(); 205 | let offset = 0; 206 | let mut rows_before = vec![]; 207 | let mut rows_after = vec![]; 208 | while from < len { 209 | let (values1, offset) = self.parse_row_values(&v[from ..], col_count); 210 | rows_before.push(values1); 211 | from += offset; 212 | let (values2, offset) = self.parse_row_values(&v[from ..], col_count); 213 | rows_after.push(values2); 214 | from += offset; 215 | } 216 | let e = UpdateEvent::new(rows_before, rows_after); 217 | Ok(Event::Update(e)) 218 | } else { 219 | let e = UpdateEvent::new(vec![], vec![]); 220 | Ok(Event::Update(e)) 221 | } 222 | } 223 | 224 | pub fn read_delete_event(&mut self, eh: &EventHeader) -> Result { 225 | if let Ok((v, col_count)) = self.read_rows_event(eh, false) { 226 | let mut from:usize = 0; 227 | let len = v.len(); 228 | let offset = 0; 229 | let mut rows = vec![]; 230 | while from < len { 231 | let (v, offset) = self.parse_row_values(&v[from ..], col_count); 232 | from += offset; 233 | rows.push(v); 234 | } 235 | let e = DeleteEvent::new(rows); 236 | Ok(Event::Delete(e)) 237 | } else { 238 | let e = DeleteEvent::new(vec![]); 239 | Ok(Event::Delete(e)) 240 | } 241 | } 242 | 243 | pub fn read_rows_event(&mut self, eh: &EventHeader, is_update_event: bool) -> Result<(Vec, usize)> { 244 | { 245 | let data = self.stream.read(8); 246 | let mut cursor = Cursor::new(&data); 247 | let i1 = cursor.read_i16::()?; 248 | let i2 = cursor.read_i16::()?; 249 | let i3 = cursor.read_i16::()?; 250 | let table_id = get_table_id(i1 as i64, i2 as i64, i3 as i64); 251 | let flags = cursor.read_i16::()?; 252 | } 253 | 254 | let mut extra_data_len:usize = 2; // MySQL 5.6~ 255 | { 256 | let data = self.stream.read(extra_data_len as usize); 257 | let mut cursor = Cursor::new(&data); 258 | extra_data_len = cursor.read_i16::()? as usize; 259 | } 260 | 261 | if extra_data_len > 2 { 262 | let extra_data = self.stream.read(extra_data_len - 2); 263 | } 264 | 265 | let data_len = eh.get_event_len() - 19 - 8 - extra_data_len; 266 | let (rows_data, col_count) = { 267 | let data = self.stream.read(data_len - 4); 268 | let (col_count, size) = get_field_length(data); 269 | 270 | let data = &data[size ..]; 271 | 272 | let bitmap_size:usize = (col_count as usize + 7) / 8; 273 | // https://github.com/wenerme/myfacility/blob/master/binlog/row.go#L164 274 | if is_update_event { 275 | (Vec::from(&data[bitmap_size * 2 .. ]), col_count) 276 | } else { 277 | (Vec::from(&data[bitmap_size .. ]), col_count) 278 | } 279 | }; 280 | 281 | { 282 | // For chuncksum 283 | self.stream.read(4); 284 | } 285 | 286 | Ok((rows_data, col_count as usize)) 287 | } 288 | 289 | fn parse_row_values(&self, data: &[u8], col_count: usize) -> (Vec, usize) { 290 | 291 | let mut values = Vec::with_capacity(col_count); 292 | let mut nulls = Vec::with_capacity(col_count); 293 | for i in 0 .. col_count { 294 | let is_null = (data[i / 8] & (1 << (i % 8))) != 0; // # is_null OK? 295 | nulls.push(is_null); 296 | } 297 | 298 | let p = (col_count + 7) / 8; 299 | let data = &data[ p .. ]; 300 | 301 | let mut from = 0; 302 | for i in 0 .. col_count { 303 | if nulls[i] { 304 | values.push(ValueType::Null); 305 | continue; 306 | } 307 | let remain = &data[from .. ]; 308 | 309 | let (field_type, nullable, metadata1, metadata2) = self.field_types[i]; 310 | if let Ok((value, offset)) = parse_field(field_type, nullable, metadata1, metadata2, remain) { 311 | values.push(value); 312 | from += offset; 313 | } 314 | } 315 | 316 | (values, p + from) 317 | } 318 | 319 | fn parse_current_fields_discriptors(&mut self, data: &Vec) { 320 | let (col_count, size) = get_field_length(data); 321 | 322 | let s = size + col_count as usize; 323 | let col_types = &data[size .. s]; 324 | 325 | // Rebind 326 | let data = &data[s .. ]; 327 | 328 | let (metadata_size, size) = get_field_length(data); 329 | 330 | let s = size + metadata_size as usize; 331 | let metadata = &data[size .. s]; 332 | 333 | let data = &data[s .. ]; 334 | 335 | let nullable_bits_size: usize = (col_count as usize + 7) / 8; 336 | let nullable_bits = &data[0 .. nullable_bits_size]; 337 | 338 | self.set_current_fields_discriptors(col_types, metadata, nullable_bits); 339 | } 340 | 341 | fn set_current_fields_discriptors(&mut self, col_types: &[u8], metadata: &[u8], nullable_bits: &[u8]) { 342 | self.field_types.clear(); 343 | 344 | let col_count = col_types.len(); 345 | let mut nullable_list = Vec::with_capacity(col_count); 346 | 347 | for i in 0 .. col_count { 348 | let bit = (nullable_bits[i / 8]) & (1 << (i % 8)); 349 | nullable_list.push(bit != 0); 350 | } 351 | 352 | let mut i = 0; 353 | let mut slice_begin = 0; 354 | let mut slice_end = 0; 355 | for col_type in col_types { 356 | let mut metadata1 = 0; 357 | let mut metadata2 = 0; 358 | let md = &metadata[slice_begin ..]; 359 | let col_type = *col_type; 360 | let nullable = nullable_list[i]; 361 | if col_type == FieldType::VarString as u8 || 362 | col_type == FieldType::String as u8 { 363 | 364 | metadata1 = md[0]; 365 | metadata2 = md[1]; 366 | slice_begin += 2; 367 | } else if col_type == FieldType::Datetime2 as u8 { 368 | metadata1 = md[0]; 369 | slice_begin += 1; 370 | } else if col_type == FieldType::NewDecimal as u8 { 371 | metadata1 = md[0]; 372 | metadata2 = md[1]; 373 | slice_begin += 2; 374 | } else if col_type == FieldType::Varchar as u8 { 375 | metadata1 = md[0]; 376 | metadata2 = md[1]; 377 | slice_begin += 2; 378 | } else if col_type == FieldType::Double as u8 || 379 | col_type == FieldType::Float as u8 { 380 | // What's in metadata for float/double? 381 | metadata1 = md[0]; 382 | slice_begin += 1; 383 | } else if col_type == FieldType::Timestamp2 as u8 { 384 | metadata1 = md[0]; 385 | slice_begin += 1 386 | } 387 | 388 | i += 1; 389 | // println!("{}-{}-{}-{}", col_type, nullable, metadata1, metadata2); 390 | self.field_types.push((col_type, nullable, metadata1, metadata2)); 391 | } 392 | } 393 | 394 | 395 | } --------------------------------------------------------------------------------