├── .gitignore ├── README.md ├── go-mysql-replay.conf.json.sample ├── mysqlreplay.go └── testdata ├── test.dat ├── test2.dat └── test3.dat /.gitignore: -------------------------------------------------------------------------------- 1 | mysqlreplay 2 | go-mysql-replay 3 | go-mysql-replay.conf.json 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Go MySQL Replay 2 | =============== 3 | 4 | Replays statements from a traffic dump or captures from Performance Schema. 5 | 6 | WARNING: This tool will execute operations that can change your data if those 7 | operations are in your dump file. Don't run it against your production DB! 8 | 9 | Requirements 10 | ============ 11 | 12 | * MySQL 13 | * go (if building from source, best with 1.5) 14 | * tshark (optional, to capturing and convert network traffic) 15 | * tcpdump (optional, to capture network traffic) 16 | * Access to the network interface to capture. 17 | * Access to `performance_schema`. (if capturing from PS) 18 | 19 | Note that wireshark/tshark should be able to decode SSL/TLS if you give it 20 | your key and configure MySQL to not use Diffie-Hellman. 21 | 22 | Building 23 | ======== 24 | 25 | This should build a static binary with Go 1.5 26 | ``` 27 | $ go build -tags netgo 28 | ``` 29 | 30 | Workflow 31 | ======== 32 | 33 | 1. Capture data 34 | 35 | ``` 36 | # tcpdump -i eth0 -w mysql.pcap -s0 -G 60 -W 1 'dst port 3306' 37 | ``` 38 | 39 | -i interface 40 | -w write to file 41 | -s snaplen 42 | -G seconds to run 43 | -W number of times to run 44 | 45 | 2. Convert your data to a tab dilimtered file with tshark 46 | 47 | ``` 48 | $ tshark -r mysql.pcap -Y mysql.query -Y mysql.command -e tcp.stream \ 49 | > -e frame.time_epoch -e mysql.command -e mysql.query \ 50 | > -Tfields -E quote=d > my_workload.dat 51 | ``` 52 | 53 | 3. Replay the statements 54 | 55 | ``` 56 | $ ./go-mysql-replay -f my_workload.dat 57 | ``` 58 | 59 | To combine steps 1 and 2: 60 | 61 | $ sudo tshark -i lo -Y mysql.query -Y mysql.command -e tcp.stream \ 62 | > -e frame.time_epoch -e mysql.command -e mysql.query \ 63 | > -Tfields -E quote=d 64 | Running as user "root" and group "root". This could be dangerous. 65 | Capturing on 'Loopback' 66 | "0" "1445166898.745198000" "3" "select @@version_comment limit 1" 67 | "0" "1445166898.745338000" "3" "SELECT VERSION()" 68 | "0" "1445166898.745516000" "3" "SELECT CURRENT_TIMESTAMP" 69 | "1" "1445166923.496890000" "3" "select @@version_comment limit 1" 70 | "1" "1445166923.497021000" "3" "SELECT VERSION()" 71 | "1" "1445166923.497140000" "3" "SELECT CURRENT_TIMESTAMP" 72 | ^C1 packet dropped 73 | 6 packets captured 74 | 75 | If MySQL runs on a non-standard port you might want to add: `-d tcp.port==5709,mysql` to tshark to tell it to decode 76 | port 5709 as mysql. 77 | 78 | 79 | Using Performance Schema 80 | ======================== 81 | 82 | Note: this has not been tested/updated after v0.1 yet. So this is missing the command column. 83 | 84 | You need to enable the consumer for one or more events statements tables. You 85 | might want to adjust `performance_schema_events_statements_history_long_size` 86 | to control how many statements you're capturing 87 | 88 | UPDATE setup_consumers SET ENABLED='YES' WHERE NAME='events_statements_history_long'; 89 | 90 | The to capture the statements: 91 | 92 | SELECT THREAD_ID, TIMER_START*10e-13, SQL_TEXT FROM events_statements_history_long 93 | WHERE SQL_TEXT IS NOT NULL INTO OUTFILE '/tmp/statements_from_ps.dat'; 94 | 95 | TODO 96 | ==== 97 | 98 | * Better handle the default database of each connection (hardcoded now). 99 | * `-E quote=d` does not escape quotes: [Wireshark Bug #10284](https://bugs.wireshark.org/bugzilla/show_bug.cgi?id=10284) 100 | -------------------------------------------------------------------------------- /go-mysql-replay.conf.json.sample: -------------------------------------------------------------------------------- 1 | { 2 | "dsn": "msandbox:msandbox@tcp(127.0.0.1:5709)/test" 3 | } 4 | -------------------------------------------------------------------------------- /mysqlreplay.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/csv" 6 | "encoding/json" 7 | "flag" 8 | "fmt" 9 | "github.com/go-sql-driver/mysql" 10 | "io" 11 | "math" 12 | "os" 13 | "strconv" 14 | "time" 15 | ) 16 | 17 | type ReplayStatement struct { 18 | session int 19 | epoch float64 20 | stmt string 21 | cmd uint8 22 | } 23 | 24 | type Configuration struct { 25 | Dsn string 26 | } 27 | 28 | func timefromfloat(epoch float64) time.Time { 29 | epoch_base := math.Floor(epoch) 30 | epoch_frac := epoch - epoch_base 31 | epoch_time := time.Unix(int64(epoch_base), int64(epoch_frac*1000000000)) 32 | return epoch_time 33 | } 34 | 35 | func mysqlsession(c <-chan ReplayStatement, session int, firstepoch float64, 36 | starttime time.Time, config Configuration) { 37 | fmt.Printf("[session %d] NEW SESSION\n", session) 38 | 39 | db, err := sql.Open("mysql", config.Dsn) 40 | if err != nil { 41 | panic(err.Error()) 42 | } 43 | dbopen := true 44 | defer db.Close() 45 | 46 | last_stmt_epoch := firstepoch 47 | for { 48 | pkt := <-c 49 | if last_stmt_epoch != 0.0 { 50 | firsttime := timefromfloat(firstepoch) 51 | pkttime := timefromfloat(pkt.epoch) 52 | delaytime_orig := pkttime.Sub(firsttime) 53 | mydelay := time.Since(starttime) 54 | delaytime_new := delaytime_orig - mydelay 55 | 56 | fmt.Printf("[session %d] Sleeptime: %s\n", session, 57 | delaytime_new) 58 | time.Sleep(delaytime_new) 59 | } 60 | last_stmt_epoch = pkt.epoch 61 | switch pkt.cmd { 62 | case 14: // Ping 63 | continue 64 | case 1: // Quit 65 | fmt.Printf("[session %d] COMMAND REPLAY: QUIT\n", session) 66 | dbopen = false 67 | db.Close() 68 | case 3: // Query 69 | if dbopen == false { 70 | fmt.Printf("[session %d] RECONNECT\n", session) 71 | db, err = sql.Open("mysql", config.Dsn) 72 | if err != nil { 73 | panic(err.Error()) 74 | } 75 | dbopen = true 76 | } 77 | fmt.Printf("[session %d] STATEMENT REPLAY: %s\n", session, 78 | pkt.stmt) 79 | _, err := db.Exec(pkt.stmt) 80 | if err != nil { 81 | if mysqlError, ok := err.(*mysql.MySQLError); ok { 82 | if mysqlError.Number == 1205 { // Lock wait timeout 83 | fmt.Printf("ERROR IGNORED: %s", 84 | err.Error()) 85 | } 86 | } else { 87 | panic(err.Error()) 88 | } 89 | } 90 | } 91 | } 92 | } 93 | 94 | func main() { 95 | conffile, _ := os.Open("go-mysql-replay.conf.json") 96 | confdec := json.NewDecoder(conffile) 97 | config := Configuration{} 98 | err := confdec.Decode(&config) 99 | if err != nil { 100 | fmt.Printf("Error reading configuration from "+ 101 | "'./go-mysql-replay.conf.json': %s\n", err) 102 | } 103 | 104 | fileflag := flag.String("f", "./test.dat", 105 | "Path to datafile for replay") 106 | flag.Parse() 107 | 108 | datFile, err := os.Open(*fileflag) 109 | if err != nil { 110 | fmt.Println(err) 111 | } 112 | 113 | reader := csv.NewReader(datFile) 114 | reader.Comma = '\t' 115 | 116 | var firstepoch float64 = 0.0 117 | starttime := time.Now() 118 | sessions := make(map[int]chan ReplayStatement) 119 | for { 120 | stmt, err := reader.Read() 121 | if err == io.EOF { 122 | break 123 | } 124 | if err != nil { 125 | panic(err.Error) 126 | } 127 | sessionid, err := strconv.Atoi(stmt[0]) 128 | if err != nil { 129 | fmt.Println(err) 130 | } 131 | cmd_src, err := strconv.Atoi(stmt[2]) 132 | if err != nil { 133 | fmt.Println(err) 134 | } 135 | cmd := uint8(cmd_src) 136 | epoch, err := strconv.ParseFloat(stmt[1], 64) 137 | if err != nil { 138 | fmt.Println(err) 139 | } 140 | pkt := ReplayStatement{session: sessionid, epoch: epoch, 141 | cmd: cmd, stmt: stmt[3]} 142 | if firstepoch == 0.0 { 143 | firstepoch = pkt.epoch 144 | } 145 | if sessions[pkt.session] != nil { 146 | sessions[pkt.session] <- pkt 147 | } else { 148 | sess := make(chan ReplayStatement) 149 | sessions[pkt.session] = sess 150 | go mysqlsession(sessions[pkt.session], pkt.session, 151 | firstepoch, starttime, config) 152 | sessions[pkt.session] <- pkt 153 | } 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /testdata/test.dat: -------------------------------------------------------------------------------- 1 | 0 1445166898.745198000 select @@version_comment limit 1 2 | 0 1445166898.745338000 SELECT VERSION() 3 | 0 1445166898.745516000 SELECT CURRENT_TIMESTAMP 4 | 1 1445166923.496890000 select @@version_comment limit 1 5 | 1 1445166923.497021000 SELECT VERSION() 6 | 1 1445166923.497140000 SELECT CURRENT_TIMESTAMP 7 | -------------------------------------------------------------------------------- /testdata/test2.dat: -------------------------------------------------------------------------------- 1 | 0 1445166898.745198000 select @@version_comment limit 1 2 | 0 1445166899.745338000 SELECT VERSION() 3 | 0 1445166900.745516000 SELECT CURRENT_TIMESTAMP 4 | 1 1445166923.496890000 select @@version_comment limit 1 5 | 1 1445166933.497021000 SELECT VERSION() 6 | 1 1445166943.497140000 SELECT CURRENT_TIMESTAMP 7 | -------------------------------------------------------------------------------- /testdata/test3.dat: -------------------------------------------------------------------------------- 1 | 0 1445166898.745198000 select @@version_comment limit 1 2 | 1 1445166898.745198100 select @@version_comment limit 1 3 | 0 1445166898.745338000 SELECT VERSION() 4 | 0 1445166898.745516000 SELECT CURRENT_TIMESTAMP 5 | 1 1445166923.497021000 SELECT VERSION() 6 | 1 1445166923.497140000 SELECT CURRENT_TIMESTAMP 7 | --------------------------------------------------------------------------------