├── .gitattributes ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── beater └── mysqlbeat.go ├── config └── config.go ├── dashboard ├── mysql_performance_dashboard_by_mysqlbeat.json └── mysql_performance_dashboard_by_mysqlbeat.png ├── dev-tools └── packer │ ├── Makefile │ ├── beats │ └── mysqlbeat.yml │ └── version.yml ├── docs ├── fields.asciidoc └── index.asciidoc ├── etc ├── beat.yml ├── fields.yml └── mysqlbeat.template.json ├── glide.yaml ├── main.go ├── main_test.go ├── mysqlbeat.yml └── tests └── system ├── config └── mysqlbeat.yml.j2 ├── mysqlbeat.py ├── requirements.txt └── test_base.py /.gitattributes: -------------------------------------------------------------------------------- 1 | CHANGELOG.md merge=union 2 | CHANGELOG.asciidoc merge=union 3 | 4 | # Keep these file types as CRLF (Windows). 5 | *.bat text eol=crlf 6 | *.cmd text eol=crlf 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /vendor 3 | /build 4 | 5 | .DS_Store 6 | /mysqlbeat 7 | /mysqlbeat.test 8 | *.pyc 9 | 10 | dev-tools/packer/build/* -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | services: 4 | - docker 5 | 6 | language: go 7 | 8 | go: 9 | - 1.5.3 10 | 11 | os: 12 | - linux 13 | - osx 14 | 15 | env: 16 | matrix: 17 | - TARGETS="check" 18 | - TARGETS="-C mysqlbeat testsuite" 19 | 20 | global: 21 | # Cross-compile for amd64 only to speed up testing. 22 | - GOX_FLAGS="-arch amd64" 23 | 24 | addons: 25 | apt: 26 | packages: 27 | - python-virtualenv 28 | 29 | before_install: 30 | # Redo the travis setup but with the elastic/libbeat path. This is needed so the package path is correct 31 | - mkdir -p $HOME/gopath/src/github.com/elastic/beats/ 32 | - rsync -az ${TRAVIS_BUILD_DIR}/ $HOME/gopath/src/github.com/elastic/beats/ 33 | - export TRAVIS_BUILD_DIR=$HOME/gopath/src/github.com/elastic/beats/ 34 | - cd $HOME/gopath/src/github.com/elastic/beats/ 35 | 36 | install: 37 | - true 38 | 39 | script: 40 | - make $TARGETS 41 | 42 | after_success: 43 | # Copy full.cov to coverage.txt because codecov.io requires this file 44 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adibendahan/mysqlbeat/0c74f6ac75d6f27a5812459997e76516472f31a7/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Adi Ben-Dahan 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BEATNAME=mysqlbeat 2 | BEAT_DIR=github.com/adibendahan 3 | SYSTEM_TESTS=false 4 | TEST_ENVIRONMENT=false 5 | ES_BEATS=./vendor/github.com/elastic/beats 6 | GOPACKAGES=$(shell glide novendor) 7 | PREFIX?=. 8 | 9 | # Path to the libbeat Makefile 10 | -include $(ES_BEATS)/libbeat/scripts/Makefile 11 | 12 | .PHONY: init 13 | init: 14 | glide update --no-recursive 15 | make update 16 | git init 17 | 18 | .PHONY: commit 19 | commit: 20 | git add README.md CONTRIBUTING.md 21 | git commit -m "Initial commit" 22 | git add LICENSE 23 | git commit -m "Add the LICENSE" 24 | git add .gitignore .gitattributes 25 | git commit -m "Add git settings" 26 | git add . 27 | git reset -- .travis.yml 28 | git commit -m "Add mysqlbeat" 29 | git add .travis.yml 30 | git commit -m "Add Travis CI" 31 | 32 | .PHONY: update-deps 33 | update-deps: 34 | glide update --no-recursive 35 | 36 | # This is called by the beats packer before building starts 37 | .PHONY: before-build 38 | before-build: 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mysqlbeat 2 | Fully customizable Beat for MySQL server - this beat will ship the results of any query defined in the config file to Elasticsearch. 3 | 4 | 5 | ## Current status 6 | First beta release [available here](https://github.com/adibendahan/mysqlbeat/releases/tag/1.0.0). 7 | 8 | ## Features 9 | 10 | * Connect to any MySQL server and run queries 11 | * `single-row` queries will be translated as columnname:value. 12 | * `two-columns` will be translated as value-column1:value-column2 for each row. 13 | * `multiple-rows` each row will be a document (with columnname:value) **NEW:** Added DELTA support. 14 | * `show-slave-delay` will only send the "Seconds_Behind_Master" column from `SHOW SLAVE STATUS;` 15 | * Any column that ends with the delatwildcard (default is __DELTA) will send delta results, extremely useful for server counters. 16 | `((newval - oldval)/timediff.Seconds())` 17 | * MySQL Performance Dashboard (more details below) 18 | 19 | ## How to Build 20 | 21 | mysqlbeat uses Glide for dependency management. To install glide see: https://github.com/Masterminds/glide 22 | 23 | ```shell 24 | $ glide update --no-recursive 25 | $ make 26 | ``` 27 | 28 | ## Default Configuration 29 | 30 | Edit mysqlbeat configuration in ```mysqlbeat.yml``` . 31 | You can: 32 | * Add queries to the `queries` array 33 | * Add query types to the `querytypes` array 34 | * Define Username/Password to connect to the MySQL 35 | * Define the column wild card for delta columns 36 | * Define the column wild card for delta key columns 37 | * Password can be saved in clear text/AES encryption 38 | 39 | If you choose to use the mysqlbeat as is, just run the following on your MySQL Server: 40 | ``` 41 | GRANT REPLICATION CLIENT, PROCESS ON *.* TO 'mysqlbeat_user'@'%' IDENTIFIED BY 'mysqlbeat_pass'; 42 | ``` 43 | 44 | Notes on password encryption: Before you compile your own mysqlbeat, you should put a new secret in the code (defined as a const), secret length must be 16, 24 or 32, corresponding to the AES-128, AES-192 or AES-256 algorithm. I recommend deleting the secret from the source code after you have your compiled mysqlbeat. You can encrypt your password with [mysqlbeat-password-encrypter](github.com/adibendahan/mysqlbeat-password-encrypter, "github.com/adibendahan/mysqlbeat-password-encrypter") just update your secret (and commonIV if you choose to change it) and compile. 45 | 46 | ## Template 47 | The default template is provided, if you add any queries you should update the template accordingly. 48 | 49 | To apply the default template run: 50 | ``` 51 | curl -XPUT http://:9200/_template/mysqlbeat -d@etc/mysqlbeat-template.json 52 | ``` 53 | 54 | ## How to use 55 | Just run ```mysqlbeat -c mysqlbeat.yml``` and you are good to go. 56 | 57 | ## MySQL Performance Dashboard by mysqlbeat 58 | This dashboard created as an addition to the MySQL dashboard provided by packetbeat, use them both. 59 | Run the default configuration provided to get the dashboard below (you should import ```dashboard/mysql_performance_dashboard_by_mysqlbeat.json``` to create the dashboard in Kibana). 60 | 61 | ![mysql_performance_by_mysqlbeat__dashboard__kibana](https://cloud.githubusercontent.com/assets/2807536/14936629/3a3b88e8-0efa-11e6-87ef-eb864498d3ab.png) 62 | 63 | 64 | ## License 65 | GNU General Public License v2 66 | -------------------------------------------------------------------------------- /beater/mysqlbeat.go: -------------------------------------------------------------------------------- 1 | package beater 2 | 3 | import ( 4 | "crypto/aes" 5 | "crypto/cipher" 6 | "database/sql" 7 | "encoding/hex" 8 | "fmt" 9 | "math" 10 | "strconv" 11 | "strings" 12 | "time" 13 | 14 | "github.com/elastic/beats/libbeat/beat" 15 | "github.com/elastic/beats/libbeat/cfgfile" 16 | "github.com/elastic/beats/libbeat/common" 17 | "github.com/elastic/beats/libbeat/logp" 18 | 19 | "github.com/adibendahan/mysqlbeat/config" 20 | 21 | // mysql go driver 22 | _ "github.com/go-sql-driver/mysql" 23 | ) 24 | 25 | // Mysqlbeat is a struct to hold the beat config & info 26 | type Mysqlbeat struct { 27 | beatConfig *config.Config 28 | done chan struct{} 29 | period time.Duration 30 | hostname string 31 | port string 32 | username string 33 | password string 34 | passwordAES string 35 | queries []string 36 | queryTypes []string 37 | deltaWildcard string 38 | deltaKeyWildcard string 39 | 40 | oldValues common.MapStr 41 | oldValuesAge common.MapStr 42 | } 43 | 44 | var ( 45 | commonIV = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f} 46 | ) 47 | 48 | const ( 49 | // secret length must be 16, 24 or 32, corresponding to the AES-128, AES-192 or AES-256 algorithms 50 | // you should compile your mysqlbeat with a unique secret and hide it (don't leave it in the code after compiled) 51 | // you can encrypt your password with github.com/adibendahan/mysqlbeat-password-encrypter just update your secret 52 | // (and commonIV if you choose to change it) and compile. 53 | secret = "github.com/adibendahan/mysqlbeat" 54 | 55 | // default values 56 | defaultPeriod = "10s" 57 | defaultHostname = "127.0.0.1" 58 | defaultPort = "3306" 59 | defaultUsername = "mysqlbeat_user" 60 | defaultPassword = "mysqlbeat_pass" 61 | defaultDeltaWildcard = "__DELTA" 62 | defaultDeltaKeyWildcard = "__DELTAKEY" 63 | 64 | // query types values 65 | queryTypeSingleRow = "single-row" 66 | queryTypeMultipleRows = "multiple-rows" 67 | queryTypeTwoColumns = "two-columns" 68 | queryTypeSlaveDelay = "show-slave-delay" 69 | 70 | // special column names values 71 | columnNameSlaveDelay = "Seconds_Behind_Master" 72 | 73 | // column types values 74 | columnTypeString = iota 75 | columnTypeInt 76 | columnTypeFloat 77 | ) 78 | 79 | // New Creates beater 80 | func New() *Mysqlbeat { 81 | return &Mysqlbeat{ 82 | done: make(chan struct{}), 83 | } 84 | } 85 | 86 | ///*** Beater interface methods ***/// 87 | 88 | // Config is a function to read config file 89 | func (bt *Mysqlbeat) Config(b *beat.Beat) error { 90 | 91 | // Load beater beatConfig 92 | err := cfgfile.Read(&bt.beatConfig, "") 93 | if err != nil { 94 | return fmt.Errorf("Error reading config file: %v", err) 95 | } 96 | 97 | return nil 98 | } 99 | 100 | // Setup is a function to setup all beat config & info into the beat struct 101 | func (bt *Mysqlbeat) Setup(b *beat.Beat) error { 102 | 103 | if len(bt.beatConfig.Mysqlbeat.Queries) < 1 { 104 | err := fmt.Errorf("there are no queries to execute") 105 | return err 106 | } 107 | 108 | if len(bt.beatConfig.Mysqlbeat.Queries) != len(bt.beatConfig.Mysqlbeat.QueryTypes) { 109 | err := fmt.Errorf("error on config file, queries array length != queryTypes array length (each query should have a corresponding type on the same index)") 110 | return err 111 | } 112 | 113 | // Setting defaults for missing config 114 | if bt.beatConfig.Mysqlbeat.Period == "" { 115 | logp.Info("Period not selected, proceeding with '%v' as default", defaultPeriod) 116 | bt.beatConfig.Mysqlbeat.Period = defaultPeriod 117 | } 118 | 119 | if bt.beatConfig.Mysqlbeat.Hostname == "" { 120 | logp.Info("Hostname not selected, proceeding with '%v' as default", defaultHostname) 121 | bt.beatConfig.Mysqlbeat.Hostname = defaultHostname 122 | } 123 | 124 | if bt.beatConfig.Mysqlbeat.Port == "" { 125 | logp.Info("Port not selected, proceeding with '%v' as default", defaultPort) 126 | bt.beatConfig.Mysqlbeat.Port = defaultPort 127 | } 128 | 129 | if bt.beatConfig.Mysqlbeat.Username == "" { 130 | logp.Info("Username not selected, proceeding with '%v' as default", defaultUsername) 131 | bt.beatConfig.Mysqlbeat.Username = defaultUsername 132 | } 133 | 134 | if bt.beatConfig.Mysqlbeat.Password == "" && bt.beatConfig.Mysqlbeat.EncryptedPassword == "" { 135 | logp.Info("Password not selected, proceeding with default password") 136 | bt.beatConfig.Mysqlbeat.Password = defaultPassword 137 | } 138 | 139 | if bt.beatConfig.Mysqlbeat.DeltaWildcard == "" { 140 | logp.Info("DeltaWildcard not selected, proceeding with '%v' as default", defaultDeltaWildcard) 141 | bt.beatConfig.Mysqlbeat.DeltaWildcard = defaultDeltaWildcard 142 | } 143 | 144 | if bt.beatConfig.Mysqlbeat.DeltaKeyWildcard == "" { 145 | logp.Info("DeltaKeyWildcard not selected, proceeding with '%v' as default", defaultDeltaKeyWildcard) 146 | bt.beatConfig.Mysqlbeat.DeltaKeyWildcard = defaultDeltaKeyWildcard 147 | } 148 | 149 | // Parse the Period string 150 | var durationParseError error 151 | bt.period, durationParseError = time.ParseDuration(bt.beatConfig.Mysqlbeat.Period) 152 | if durationParseError != nil { 153 | return durationParseError 154 | } 155 | 156 | // Handle password decryption and save in the bt 157 | if bt.beatConfig.Mysqlbeat.Password != "" { 158 | bt.password = bt.beatConfig.Mysqlbeat.Password 159 | } else if bt.beatConfig.Mysqlbeat.EncryptedPassword != "" { 160 | aesCipher, err := aes.NewCipher([]byte(secret)) 161 | if err != nil { 162 | return err 163 | } 164 | cfbDecrypter := cipher.NewCFBDecrypter(aesCipher, commonIV) 165 | chiperText, err := hex.DecodeString(bt.beatConfig.Mysqlbeat.EncryptedPassword) 166 | if err != nil { 167 | return err 168 | } 169 | plaintextCopy := make([]byte, len(chiperText)) 170 | cfbDecrypter.XORKeyStream(plaintextCopy, chiperText) 171 | bt.password = string(plaintextCopy) 172 | } 173 | 174 | // init the oldValues and oldValuesAge array 175 | bt.oldValues = common.MapStr{"mysqlbeat": "init"} 176 | bt.oldValuesAge = common.MapStr{"mysqlbeat": "init"} 177 | 178 | // Save config values to the bt 179 | bt.hostname = bt.beatConfig.Mysqlbeat.Hostname 180 | bt.port = bt.beatConfig.Mysqlbeat.Port 181 | bt.username = bt.beatConfig.Mysqlbeat.Username 182 | bt.queries = bt.beatConfig.Mysqlbeat.Queries 183 | bt.queryTypes = bt.beatConfig.Mysqlbeat.QueryTypes 184 | bt.deltaWildcard = bt.beatConfig.Mysqlbeat.DeltaWildcard 185 | bt.deltaKeyWildcard = bt.beatConfig.Mysqlbeat.DeltaKeyWildcard 186 | 187 | safeQueries := true 188 | 189 | logp.Info("Total # of queries to execute: %d", len(bt.queries)) 190 | for index, queryStr := range bt.queries { 191 | 192 | strCleanQuery := strings.TrimSpace(strings.ToUpper(queryStr)) 193 | 194 | if !strings.HasPrefix(strCleanQuery, "SELECT") && !strings.HasPrefix(strCleanQuery, "SHOW") || strings.ContainsAny(strCleanQuery, ";") { 195 | safeQueries = false 196 | } 197 | 198 | logp.Info("Query #%d (type: %s): %s", index+1, bt.queryTypes[index], queryStr) 199 | } 200 | 201 | if !safeQueries { 202 | err := fmt.Errorf("Only SELECT/SHOW queries are allowed (the char ; is forbidden)") 203 | return err 204 | } 205 | 206 | return nil 207 | } 208 | 209 | // Run is a functions that runs the beat 210 | func (bt *Mysqlbeat) Run(b *beat.Beat) error { 211 | logp.Info("mysqlbeat is running! Hit CTRL-C to stop it.") 212 | 213 | ticker := time.NewTicker(bt.period) 214 | for { 215 | select { 216 | case <-bt.done: 217 | return nil 218 | case <-ticker.C: 219 | } 220 | 221 | err := bt.beat(b) 222 | if err != nil { 223 | return err 224 | } 225 | } 226 | } 227 | 228 | // Cleanup is a function that does nothing on this beat :) 229 | func (bt *Mysqlbeat) Cleanup(b *beat.Beat) error { 230 | return nil 231 | } 232 | 233 | // Stop is a function that runs once the beat is stopped 234 | func (bt *Mysqlbeat) Stop() { 235 | close(bt.done) 236 | } 237 | 238 | ///*** mysqlbeat methods ***/// 239 | 240 | // beat is a function that iterate over the query array, generate and publish events 241 | func (bt *Mysqlbeat) beat(b *beat.Beat) error { 242 | 243 | // Build the MySQL connection string 244 | connString := fmt.Sprintf("%v:%v@tcp(%v:%v)/", bt.username, bt.password, bt.hostname, bt.port) 245 | 246 | db, err := sql.Open("mysql", connString) 247 | if err != nil { 248 | return err 249 | } 250 | defer db.Close() 251 | 252 | // Create a two-columns event for later use 253 | var twoColumnEvent common.MapStr 254 | 255 | LoopQueries: 256 | for index, queryStr := range bt.queries { 257 | // Log the query run time and run the query 258 | dtNow := time.Now() 259 | rows, err := db.Query(queryStr) 260 | if err != nil { 261 | return err 262 | } 263 | 264 | // Populate columns array 265 | columns, err := rows.Columns() 266 | if err != nil { 267 | return err 268 | } 269 | 270 | // Populate the two-columns event 271 | if bt.queryTypes[index] == queryTypeTwoColumns { 272 | twoColumnEvent = common.MapStr{ 273 | "@timestamp": common.Time(dtNow), 274 | "type": queryTypeTwoColumns, 275 | } 276 | } 277 | 278 | LoopRows: 279 | for rows.Next() { 280 | 281 | switch bt.queryTypes[index] { 282 | case queryTypeSingleRow, queryTypeSlaveDelay: 283 | // Generate an event from the current row 284 | event, err := bt.generateEventFromRow(rows, columns, bt.queryTypes[index], dtNow) 285 | 286 | if err != nil { 287 | logp.Err("Query #%v error generating event from rows: %v", index+1, err) 288 | } else if event != nil { 289 | b.Events.PublishEvent(event) 290 | logp.Info("%v event sent", bt.queryTypes[index]) 291 | } 292 | // breaking after the first row 293 | break LoopRows 294 | 295 | case queryTypeMultipleRows: 296 | // Generate an event from the current row 297 | event, err := bt.generateEventFromRow(rows, columns, bt.queryTypes[index], dtNow) 298 | 299 | if err != nil { 300 | logp.Err("Query #%v error generating event from rows: %v", index+1, err) 301 | break LoopRows 302 | } else if event != nil { 303 | b.Events.PublishEvent(event) 304 | logp.Info("%v event sent", bt.queryTypes[index]) 305 | } 306 | 307 | // Move to the next row 308 | continue LoopRows 309 | 310 | case queryTypeTwoColumns: 311 | // append current row to the two-columns event 312 | err := bt.appendRowToEvent(twoColumnEvent, rows, columns, dtNow) 313 | 314 | if err != nil { 315 | logp.Err("Query #%v error appending two-columns event: %v", index+1, err) 316 | break LoopRows 317 | } 318 | 319 | // Move to the next row 320 | continue LoopRows 321 | } 322 | } 323 | 324 | // If the two-columns event has data, publish it 325 | if bt.queryTypes[index] == queryTypeTwoColumns && len(twoColumnEvent) > 2 { 326 | b.Events.PublishEvent(twoColumnEvent) 327 | logp.Info("%v event sent", queryTypeTwoColumns) 328 | twoColumnEvent = nil 329 | } 330 | 331 | rows.Close() 332 | if err = rows.Err(); err != nil { 333 | logp.Err("Query #%v error closing rows: %v", index+1, err) 334 | continue LoopQueries 335 | } 336 | } 337 | 338 | // Great success! 339 | return nil 340 | } 341 | 342 | // appendRowToEvent appends the two-column event the current row data 343 | func (bt *Mysqlbeat) appendRowToEvent(event common.MapStr, row *sql.Rows, columns []string, rowAge time.Time) error { 344 | 345 | // Make a slice for the values 346 | values := make([]sql.RawBytes, len(columns)) 347 | 348 | // Copy the references into such a []interface{} for row.Scan 349 | scanArgs := make([]interface{}, len(values)) 350 | for i := range values { 351 | scanArgs[i] = &values[i] 352 | } 353 | 354 | // Get RawBytes from data 355 | err := row.Scan(scanArgs...) 356 | if err != nil { 357 | return err 358 | } 359 | 360 | // First column is the name, second is the value 361 | strColName := string(values[0]) 362 | strColValue := string(values[1]) 363 | strColType := columnTypeString 364 | strEventColName := strings.Replace(strColName, bt.deltaWildcard, "_PERSECOND", 1) 365 | 366 | // Try to parse the value to an int64 367 | nColValue, err := strconv.ParseInt(strColValue, 0, 64) 368 | if err == nil { 369 | strColType = columnTypeInt 370 | } 371 | 372 | // Try to parse the value to a float64 373 | fColValue, err := strconv.ParseFloat(strColValue, 64) 374 | if err == nil { 375 | // If it's not already an established int64, set type to float 376 | if strColType == columnTypeString { 377 | strColType = columnTypeFloat 378 | } 379 | } 380 | 381 | // If the column name ends with the deltaWildcard 382 | if strings.HasSuffix(strColName, bt.deltaWildcard) { 383 | var exists bool 384 | _, exists = bt.oldValues[strColName] 385 | 386 | // If an older value doesn't exist 387 | if !exists { 388 | // Save the current value in the oldValues array 389 | bt.oldValuesAge[strColName] = rowAge 390 | 391 | if strColType == columnTypeString { 392 | bt.oldValues[strColName] = strColValue 393 | } else if strColType == columnTypeInt { 394 | bt.oldValues[strColName] = nColValue 395 | } else if strColType == columnTypeFloat { 396 | bt.oldValues[strColName] = fColValue 397 | } 398 | } else { 399 | // If found the old value's age 400 | if dtOldAge, ok := bt.oldValuesAge[strColName].(time.Time); ok { 401 | delta := rowAge.Sub(dtOldAge) 402 | 403 | if strColType == columnTypeInt { 404 | var calcVal int64 405 | 406 | // Get old value 407 | oldVal, _ := bt.oldValues[strColName].(int64) 408 | if nColValue > oldVal { 409 | // Calculate the delta 410 | devResult := float64((nColValue - oldVal)) / float64(delta.Seconds()) 411 | // Round the calculated result back to an int64 412 | calcVal = roundF2I(devResult, .5) 413 | } else { 414 | calcVal = 0 415 | } 416 | 417 | // Add the delta value to the event 418 | event[strEventColName] = calcVal 419 | 420 | // Save current values as old values 421 | bt.oldValues[strColName] = nColValue 422 | bt.oldValuesAge[strColName] = rowAge 423 | } else if strColType == columnTypeFloat { 424 | var calcVal float64 425 | 426 | // Get old value 427 | oldVal, _ := bt.oldValues[strColName].(float64) 428 | if fColValue > oldVal { 429 | // Calculate the delta 430 | calcVal = (fColValue - oldVal) / float64(delta.Seconds()) 431 | } else { 432 | calcVal = 0 433 | } 434 | 435 | // Add the delta value to the event 436 | event[strEventColName] = calcVal 437 | 438 | // Save current values as old values 439 | bt.oldValues[strColName] = fColValue 440 | bt.oldValuesAge[strColName] = rowAge 441 | } else { 442 | event[strEventColName] = strColValue 443 | } 444 | } 445 | } 446 | } else { // Not a delta column, add the value to the event as is 447 | if strColType == columnTypeString { 448 | event[strEventColName] = strColValue 449 | } else if strColType == columnTypeInt { 450 | event[strEventColName] = nColValue 451 | } else if strColType == columnTypeFloat { 452 | event[strEventColName] = fColValue 453 | } 454 | } 455 | 456 | // Great success! 457 | return nil 458 | } 459 | 460 | // generateEventFromRow creates a new event from the row data and returns it 461 | func (bt *Mysqlbeat) generateEventFromRow(row *sql.Rows, columns []string, queryType string, rowAge time.Time) (common.MapStr, error) { 462 | 463 | // Make a slice for the values 464 | values := make([]sql.RawBytes, len(columns)) 465 | 466 | // Copy the references into such a []interface{} for row.Scan 467 | scanArgs := make([]interface{}, len(values)) 468 | for i := range values { 469 | scanArgs[i] = &values[i] 470 | } 471 | 472 | // Create the event and populate it 473 | event := common.MapStr{ 474 | "@timestamp": common.Time(rowAge), 475 | "type": queryType, 476 | } 477 | 478 | // Get RawBytes from data 479 | err := row.Scan(scanArgs...) 480 | if err != nil { 481 | return nil, err 482 | } 483 | 484 | // Loop on all columns 485 | for i, col := range values { 486 | // Get column name and string value 487 | strColName := string(columns[i]) 488 | strColValue := string(col) 489 | strColType := columnTypeString 490 | 491 | // Skip column proccessing when query type is show-slave-delay and the column isn't Seconds_Behind_Master 492 | if queryType == queryTypeSlaveDelay && strColName != columnNameSlaveDelay { 493 | continue 494 | } 495 | 496 | // Set the event column name to the original column name (as default) 497 | strEventColName := strColName 498 | 499 | // Remove unneeded suffix, add _PERSECOND to calculated columns 500 | if strings.HasSuffix(strColName, bt.deltaKeyWildcard) { 501 | strEventColName = strings.Replace(strColName, bt.deltaKeyWildcard, "", 1) 502 | } else if strings.HasSuffix(strColName, bt.deltaWildcard) { 503 | strEventColName = strings.Replace(strColName, bt.deltaWildcard, "_PERSECOND", 1) 504 | } 505 | 506 | // Try to parse the value to an int64 507 | nColValue, err := strconv.ParseInt(strColValue, 0, 64) 508 | if err == nil { 509 | strColType = columnTypeInt 510 | } 511 | 512 | // Try to parse the value to a float64 513 | fColValue, err := strconv.ParseFloat(strColValue, 64) 514 | if err == nil { 515 | // If it's not already an established int64, set type to float 516 | if strColType == columnTypeString { 517 | strColType = columnTypeFloat 518 | } 519 | } 520 | 521 | // If the column name ends with the deltaWildcard 522 | if (queryType == queryTypeSingleRow || queryType == queryTypeMultipleRows) && strings.HasSuffix(strColName, bt.deltaWildcard) { 523 | 524 | var strKey string 525 | 526 | // Get unique row key, if it's a single row - use the column name 527 | if queryType == queryTypeSingleRow { 528 | strKey = strColName 529 | } else if queryType == queryTypeMultipleRows { 530 | 531 | // If the query has multiple rows, a unique row key must be defind using the delta key wildcard and the column name 532 | strKey, err = getKeyFromRow(bt, values, columns) 533 | if err != nil { 534 | return nil, err 535 | } 536 | 537 | strKey += strColName 538 | } 539 | 540 | var exists bool 541 | _, exists = bt.oldValues[strKey] 542 | 543 | // If an older value doesn't exist 544 | if !exists { 545 | // Save the current value in the oldValues array 546 | bt.oldValuesAge[strKey] = rowAge 547 | 548 | if strColType == columnTypeString { 549 | bt.oldValues[strKey] = strColValue 550 | } else if strColType == columnTypeInt { 551 | bt.oldValues[strKey] = nColValue 552 | } else if strColType == columnTypeFloat { 553 | bt.oldValues[strKey] = fColValue 554 | } 555 | } else { 556 | // If found the old value's age 557 | if dtOldAge, ok := bt.oldValuesAge[strKey].(time.Time); ok { 558 | delta := rowAge.Sub(dtOldAge) 559 | 560 | if strColType == columnTypeInt { 561 | var calcVal int64 562 | 563 | // Get old value 564 | oldVal, _ := bt.oldValues[strKey].(int64) 565 | 566 | if nColValue > oldVal { 567 | // Calculate the delta 568 | devResult := float64((nColValue - oldVal)) / float64(delta.Seconds()) 569 | // Round the calculated result back to an int64 570 | calcVal = roundF2I(devResult, .5) 571 | } else { 572 | calcVal = 0 573 | } 574 | 575 | // Add the delta value to the event 576 | event[strEventColName] = calcVal 577 | 578 | // Save current values as old values 579 | bt.oldValues[strKey] = nColValue 580 | bt.oldValuesAge[strKey] = rowAge 581 | } else if strColType == columnTypeFloat { 582 | var calcVal float64 583 | oldVal, _ := bt.oldValues[strKey].(float64) 584 | 585 | if fColValue > oldVal { 586 | // Calculate the delta 587 | calcVal = (fColValue - oldVal) / float64(delta.Seconds()) 588 | } else { 589 | calcVal = 0 590 | } 591 | 592 | // Add the delta value to the event 593 | event[strEventColName] = calcVal 594 | 595 | // Save current values as old values 596 | bt.oldValues[strKey] = fColValue 597 | bt.oldValuesAge[strKey] = rowAge 598 | } else { 599 | event[strEventColName] = strColValue 600 | } 601 | } 602 | } 603 | } else { // Not a delta column, add the value to the event as is 604 | if strColType == columnTypeString { 605 | event[strEventColName] = strColValue 606 | } else if strColType == columnTypeInt { 607 | event[strEventColName] = nColValue 608 | } else if strColType == columnTypeFloat { 609 | event[strEventColName] = fColValue 610 | } 611 | } 612 | } 613 | 614 | // If the event has no data, set to nil 615 | if len(event) == 2 { 616 | event = nil 617 | } 618 | 619 | return event, nil 620 | } 621 | 622 | // getKeyFromRow is a function that returns a unique key from row 623 | func getKeyFromRow(bt *Mysqlbeat, values []sql.RawBytes, columns []string) (strKey string, err error) { 624 | 625 | keyFound := false 626 | 627 | // Loop on all columns 628 | for i, col := range values { 629 | // Get column name and string value 630 | if strings.HasSuffix(string(columns[i]), bt.deltaKeyWildcard) { 631 | strKey += string(col) 632 | keyFound = true 633 | } 634 | } 635 | 636 | if !keyFound { 637 | err = fmt.Errorf("query type multiple-rows requires at least one delta key column") 638 | } 639 | 640 | return strKey, err 641 | } 642 | 643 | // roundF2I is a function that returns a rounded int64 from a float64 644 | func roundF2I(val float64, roundOn float64) (newVal int64) { 645 | var round float64 646 | 647 | digit := val 648 | _, div := math.Modf(digit) 649 | if div >= roundOn { 650 | round = math.Ceil(digit) 651 | } else { 652 | round = math.Floor(digit) 653 | } 654 | 655 | return int64(round) 656 | } 657 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | // Config is put into a different package to prevent cyclic imports in case 2 | // it is needed in several locations 3 | 4 | package config 5 | 6 | type Config struct { 7 | Mysqlbeat MysqlbeatConfig 8 | } 9 | 10 | type MysqlbeatConfig struct { 11 | Period string `yaml:"period"` 12 | Hostname string `yaml:"hostname"` 13 | Port string `yaml:"port"` 14 | Username string `yaml:"username"` 15 | Password string `yaml:"password"` 16 | EncryptedPassword string `yaml:"encryptedpassword"` 17 | Queries []string `yaml:"queries"` 18 | QueryTypes []string `yaml:"querytypes"` 19 | DeltaWildcard string `yaml:"deltawildcard"` 20 | DeltaKeyWildcard string `yaml:"deltakeywildcard"` 21 | } 22 | -------------------------------------------------------------------------------- /dashboard/mysql_performance_dashboard_by_mysqlbeat.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "MySQL-Performance-by-mysqlbeat", 4 | "_type": "dashboard", 5 | "_source": { 6 | "title": "MySQL Performance by mysqlbeat", 7 | "hits": 0, 8 | "description": "", 9 | "panelsJSON": "[{\"col\":1,\"id\":\"MySQL-Replication-Delay\",\"panelIndex\":1,\"row\":3,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-InnoDB-Buffer-Pool-Usage\",\"panelIndex\":2,\"row\":5,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-InnoDB-I-slash-O\",\"panelIndex\":3,\"row\":1,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-InnoDB-Buffer-Pool-Activity\",\"panelIndex\":4,\"row\":5,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-InnoDB-Row-Lock-Time\",\"panelIndex\":5,\"row\":9,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-InnoDB-Buffer-Pool-Hit-Ratio\",\"panelIndex\":6,\"row\":9,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-Server-List\",\"panelIndex\":7,\"row\":1,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-InnoDB-Row-Lock-Waits\",\"panelIndex\":8,\"row\":13,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-InnoDB-Row-Operations\",\"panelIndex\":9,\"row\":13,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-Files-and-Tables\",\"panelIndex\":10,\"row\":17,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-Connections\",\"panelIndex\":11,\"row\":17,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-MyISAM-Indexes\",\"panelIndex\":12,\"row\":21,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-Temporary-Objects\",\"panelIndex\":13,\"row\":21,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-Transaction-Handler\",\"panelIndex\":14,\"row\":25,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-Sorts\",\"panelIndex\":15,\"row\":25,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"MySQL-Table-Locks\",\"panelIndex\":16,\"row\":29,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"MySQL-Query-Cache\",\"panelIndex\":17,\"row\":29,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"id\":\"MySQL-Select-Types\",\"type\":\"visualization\",\"panelIndex\":18,\"size_x\":6,\"size_y\":4,\"col\":1,\"row\":33}]", 10 | "optionsJSON": "{\"darkTheme\":false}", 11 | "uiStateJSON": "{\"P-2\":{\"vis\":{\"legendOpen\":true}},\"P-5\":{\"vis\":{\"legendOpen\":false}},\"P-6\":{\"vis\":{\"legendOpen\":false}},\"P-8\":{\"vis\":{\"legendOpen\":false}}}", 12 | "version": 1, 13 | "timeRestore": false, 14 | "kibanaSavedObjectMeta": { 15 | "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" 16 | } 17 | } 18 | }, 19 | { 20 | "_id": "mysqlbeat", 21 | "_type": "search", 22 | "_source": { 23 | "title": "mysqlbeat", 24 | "description": "", 25 | "hits": 0, 26 | "columns": [ 27 | "_source" 28 | ], 29 | "sort": [ 30 | "@timestamp", 31 | "desc" 32 | ], 33 | "version": 1, 34 | "kibanaSavedObjectMeta": { 35 | "searchSourceJSON": "{\"index\":\"mysqlbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" 36 | } 37 | } 38 | }, 39 | { 40 | "_id": "MySQL-InnoDB-I-slash-O", 41 | "_type": "visualization", 42 | "_source": { 43 | "title": "MySQL - InnoDB I/O", 44 | "visState": "{\"title\":\"MySQL - InnoDB Buffer Pool Usage\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"5\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_DATA_READS_PERSECOND\",\"customLabel\":\"Data Reads\"}},{\"id\":\"4\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_DATA_WRITES_PERSECOND\",\"customLabel\":\"Data Writes\"}},{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_LOG_WRITES_PERSECOND\",\"customLabel\":\"Log Writes\"}},{\"id\":\"6\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_DATA_FSYNCS_PERSECOND\",\"customLabel\":\"Data fsyncs\"}},{\"id\":\"7\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_OS_LOG_FSYNCS_PERSECOND\",\"customLabel\":\"OS Log fsyncs\"}}],\"listeners\":{}}", 45 | "uiStateJSON": "{}", 46 | "description": "", 47 | "savedSearchId": "mysqlbeat", 48 | "version": 1, 49 | "kibanaSavedObjectMeta": { 50 | "searchSourceJSON": "{\"filter\":[]}" 51 | } 52 | } 53 | }, 54 | { 55 | "_id": "MySQL-InnoDB-Buffer-Pool-Activity", 56 | "_type": "visualization", 57 | "_source": { 58 | "title": "MySQL - InnoDB Buffer Pool Activity", 59 | "visState": "{\"title\":\"MySQL - InnoDB Buffer Pool Activity\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_PAGES_READ_PERSECOND\",\"customLabel\":\"Pages Read\"}},{\"id\":\"5\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_PAGES_WRITTEN_PERSECOND\",\"customLabel\":\"Pages Written\"}},{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_PAGES_CREATED_PERSECOND\",\"customLabel\":\"Pages Created\"}}],\"listeners\":{}}", 60 | "uiStateJSON": "{}", 61 | "description": "", 62 | "savedSearchId": "mysqlbeat", 63 | "version": 1, 64 | "kibanaSavedObjectMeta": { 65 | "searchSourceJSON": "{\"filter\":[]}" 66 | } 67 | } 68 | }, 69 | { 70 | "_id": "MySQL-InnoDB-Buffer-Pool-Usage", 71 | "_type": "visualization", 72 | "_source": { 73 | "title": "MySQL - InnoDB Buffer Pool Usage", 74 | "visState": "{\"title\":\"MySQL - InnoDB Buffer Pool Usage\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"percentage\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_BUFFER_POOL_PAGES_DATA\",\"customLabel\":\"Data Pages\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_BUFFER_POOL_PAGES_FREE\",\"customLabel\":\"Free Pages\"}}],\"listeners\":{}}", 75 | "uiStateJSON": "{}", 76 | "description": "", 77 | "savedSearchId": "mysqlbeat", 78 | "version": 1, 79 | "kibanaSavedObjectMeta": { 80 | "searchSourceJSON": "{\"filter\":[]}" 81 | } 82 | } 83 | }, 84 | { 85 | "_id": "MySQL-InnoDB-Row-Lock-Time", 86 | "_type": "visualization", 87 | "_source": { 88 | "title": "MySQL - InnoDB Row Lock Time", 89 | "visState": "{\"title\":\"MySQL - InnoDB Buffer Pool Activity\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_ROW_LOCK_TIME_PERSECOND\",\"customLabel\":\"Lock Time\"}}],\"listeners\":{}}", 90 | "uiStateJSON": "{}", 91 | "description": "", 92 | "savedSearchId": "mysqlbeat", 93 | "version": 1, 94 | "kibanaSavedObjectMeta": { 95 | "searchSourceJSON": "{\"filter\":[]}" 96 | } 97 | } 98 | }, 99 | { 100 | "_id": "MySQL-Replication-Delay", 101 | "_type": "visualization", 102 | "_source": { 103 | "title": "MySQL - Replication Delay", 104 | "visState": "{\"title\":\"New Visualization\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"Seconds_Behind_Master\",\"customLabel\":\"Seconds Behind Master\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\"}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}}],\"listeners\":{}}", 105 | "uiStateJSON": "{}", 106 | "description": "", 107 | "savedSearchId": "mysqlbeat", 108 | "version": 1, 109 | "kibanaSavedObjectMeta": { 110 | "searchSourceJSON": "{\"filter\":[]}" 111 | } 112 | } 113 | }, 114 | { 115 | "_id": "MySQL-InnoDB-Row-Operations", 116 | "_type": "visualization", 117 | "_source": { 118 | "title": "MySQL - InnoDB Row Operations", 119 | "visState": "{\"title\":\"MySQL - InnoDB Buffer Pool Hit Ratio\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":true,\"setYExtents\":false,\"yAxis\":{\"max\":100,\"min\":80}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_ROWS_READ_PERSECOND\",\"customLabel\":\"Read\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_ROWS_INSERTED_PERSECOND\",\"customLabel\":\"Inserted\"}},{\"id\":\"5\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_ROWS_UPDATED_PERSECOND\",\"customLabel\":\"Updated\"}},{\"id\":\"6\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_ROWS_DELETED_PERSECOND\",\"customLabel\":\"Deleted\"}}],\"listeners\":{}}", 120 | "uiStateJSON": "{}", 121 | "description": "", 122 | "savedSearchId": "mysqlbeat", 123 | "version": 1, 124 | "kibanaSavedObjectMeta": { 125 | "searchSourceJSON": "{\"filter\":[]}" 126 | } 127 | } 128 | }, 129 | { 130 | "_id": "MySQL-Files-and-Tables", 131 | "_type": "visualization", 132 | "_source": { 133 | "title": "MySQL - Files and Tables", 134 | "visState": "{\"title\":\"MySQL - InnoDB Buffer Pool Hit Ratio\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"overlap\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":true,\"setYExtents\":false,\"yAxis\":{\"max\":100,\"min\":80}},\"aggs\":[{\"id\":\"5\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"TABLE_OPEN_CACHE\",\"customLabel\":\"Table Cache Size\"}},{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"OPEN_FILES\",\"customLabel\":\"Open Files\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"OPEN_TABLES\",\"customLabel\":\"Open Tables\"}},{\"id\":\"6\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"OPENED_TABLES_PERSECOND\",\"customLabel\":\"Opened Tables\"}}],\"listeners\":{}}", 135 | "uiStateJSON": "{}", 136 | "description": "", 137 | "savedSearchId": "mysqlbeat", 138 | "version": 1, 139 | "kibanaSavedObjectMeta": { 140 | "searchSourceJSON": "{\"filter\":[]}" 141 | } 142 | } 143 | }, 144 | { 145 | "_id": "MySQL-InnoDB-Buffer-Pool-Hit-Ratio", 146 | "_type": "visualization", 147 | "_source": { 148 | "title": "MySQL - InnoDB Buffer Pool Hit Ratio", 149 | "visState": "{\"title\":\"MySQL - InnoDB Buffer Pool Hit Ratio\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"overlap\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":true,\"setYExtents\":false,\"yAxis\":{\"max\":100,\"min\":80}},\"aggs\":[{\"id\":\"1\",\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_BUFFER_POOL_HIT_RATIO\",\"customLabel\":\"Hit Ratio\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"CONST_100\",\"customLabel\":\"(100%)\"}}],\"listeners\":{}}", 150 | "uiStateJSON": "{}", 151 | "description": "", 152 | "savedSearchId": "mysqlbeat", 153 | "version": 1, 154 | "kibanaSavedObjectMeta": { 155 | "searchSourceJSON": "{\"filter\":[]}" 156 | } 157 | } 158 | }, 159 | { 160 | "_id": "MySQL-InnoDB-Row-Lock-Waits", 161 | "_type": "visualization", 162 | "_source": { 163 | "title": "MySQL - InnoDB Row Lock Waits", 164 | "visState": "{\"title\":\"MySQL - InnoDB Row Lock Time\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_ROW_LOCK_WAITS_PERSECOND\",\"customLabel\":\"Lock Waits\"}}],\"listeners\":{}}", 165 | "uiStateJSON": "{}", 166 | "description": "", 167 | "savedSearchId": "mysqlbeat", 168 | "version": 1, 169 | "kibanaSavedObjectMeta": { 170 | "searchSourceJSON": "{\"filter\":[]}" 171 | } 172 | } 173 | }, 174 | { 175 | "_id": "MySQL-Connections", 176 | "_type": "visualization", 177 | "_source": { 178 | "title": "MySQL - Connections", 179 | "visState": "{\"title\":\"MySQL - Files and Tables\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"overlap\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":true,\"setYExtents\":false,\"yAxis\":{\"max\":100,\"min\":80}},\"aggs\":[{\"id\":\"5\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"MAX_CONNECTIONS\",\"customLabel\":\"Max Connections\"}},{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"THREADS_CONNECTED\",\"customLabel\":\"Threads Connected\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"THREADS_RUNNING\",\"customLabel\":\"Threads Running\"}},{\"id\":\"6\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"CONNECTION_ERRORS_MAX_CONNECTIONS_PERSECOND\",\"customLabel\":\"Max Connections Errors\"}},{\"id\":\"7\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"CONNECTION_ERRORS_INTERNAL_PERSECOND\",\"customLabel\":\"Internal Connection Errors\"}},{\"id\":\"8\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"ABORTED_CONNECTS_PERSECOND\",\"customLabel\":\"Aborted Connection Errors\"}}],\"listeners\":{}}", 180 | "uiStateJSON": "{}", 181 | "description": "", 182 | "savedSearchId": "mysqlbeat", 183 | "version": 1, 184 | "kibanaSavedObjectMeta": { 185 | "searchSourceJSON": "{\"filter\":[]}" 186 | } 187 | } 188 | }, 189 | { 190 | "_id": "MySQL-Query-Cache", 191 | "_type": "visualization", 192 | "_source": { 193 | "title": "MySQL - Query Cache", 194 | "visState": "{\"title\":\"MySQL - InnoDB Buffer Pool Hit Ratio\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"overlap\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":true,\"setYExtents\":false,\"yAxis\":{\"max\":100,\"min\":80}},\"aggs\":[{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"QCACHE_QUERIES_IN_CACHE\",\"customLabel\":\"Queries in Cache\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"QCACHE_HITS_PERSECOND\",\"customLabel\":\"Hits\"}},{\"id\":\"5\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"QCACHE_INSERTS_PERSECOND\",\"customLabel\":\"Inserts\"}},{\"id\":\"6\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"QCACHE_NOT_CACHED_PERSECOND\",\"customLabel\":\"Queries not Cached\"}},{\"id\":\"7\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"QCACHE_LOWMEM_PRUNES_PERSECOND\",\"customLabel\":\"Lowmem Prunes\"}}],\"listeners\":{}}", 195 | "uiStateJSON": "{}", 196 | "description": "", 197 | "savedSearchId": "mysqlbeat", 198 | "version": 1, 199 | "kibanaSavedObjectMeta": { 200 | "searchSourceJSON": "{\"filter\":[]}" 201 | } 202 | } 203 | }, 204 | { 205 | "_id": "MySQL-MyISAM-Indexes", 206 | "_type": "visualization", 207 | "_source": { 208 | "title": "MySQL - MyISAM Indexes", 209 | "visState": "{\"title\":\"MySQL - InnoDB Buffer Pool Activity\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"KEY_READS_PERSECOND\",\"customLabel\":\"Key Reads\"}},{\"id\":\"5\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"KEY_WRITES_PERSECOND\",\"customLabel\":\"Key Writes\"}},{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"KEY_READ_REQUESTS_PERSECOND\",\"customLabel\":\"Read Requests\"}},{\"id\":\"6\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"KEY_WRITE_REQUESTS_PERSECOND\",\"customLabel\":\"Write Requests\"}}],\"listeners\":{}}", 210 | "uiStateJSON": "{}", 211 | "description": "", 212 | "savedSearchId": "mysqlbeat", 213 | "version": 1, 214 | "kibanaSavedObjectMeta": { 215 | "searchSourceJSON": "{\"filter\":[]}" 216 | } 217 | } 218 | }, 219 | { 220 | "_id": "MySQL-Transaction-Handler", 221 | "_type": "visualization", 222 | "_source": { 223 | "title": "MySQL - Transaction Handler", 224 | "visState": "{\"title\":\"MySQL - MyISAM Indexes\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"HANDLER_COMMIT_PERSECOND\",\"customLabel\":\"Commits\"}},{\"id\":\"5\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"HANDLER_ROLLBACK_PERSECOND\",\"customLabel\":\"Rollbacks\"}},{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"HANDLER_SAVEPOINT_PERSECOND\",\"customLabel\":\"Savepoints\"}},{\"id\":\"6\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"HANDLER_SAVEPOINT_ROLLBACK_PERSECOND\",\"customLabel\":\"Savepoint Rollbacks\"}}],\"listeners\":{}}", 225 | "uiStateJSON": "{}", 226 | "description": "", 227 | "savedSearchId": "mysqlbeat", 228 | "version": 1, 229 | "kibanaSavedObjectMeta": { 230 | "searchSourceJSON": "{\"filter\":[]}" 231 | } 232 | } 233 | }, 234 | { 235 | "_id": "MySQL-Table-Locks", 236 | "_type": "visualization", 237 | "_source": { 238 | "title": "MySQL - Table Locks", 239 | "visState": "{\"title\":\"MySQL - InnoDB Buffer Pool Activity\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"TABLE_LOCKS_IMMEDIATE_PERSECOND\",\"customLabel\":\"Immediate Table Locks\"}},{\"id\":\"5\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"TABLE_LOCKS_WAITED_PERSECOND\",\"customLabel\":\"Waited Table Locks\"}},{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SLOW_QUERIES_PERSECOND\",\"customLabel\":\"Slow Queries\"}}],\"listeners\":{}}", 240 | "uiStateJSON": "{}", 241 | "description": "", 242 | "savedSearchId": "mysqlbeat", 243 | "version": 1, 244 | "kibanaSavedObjectMeta": { 245 | "searchSourceJSON": "{\"filter\":[]}" 246 | } 247 | } 248 | }, 249 | { 250 | "_id": "MySQL-Temporary-Objects", 251 | "_type": "visualization", 252 | "_source": { 253 | "title": "MySQL - Temporary Objects", 254 | "visState": "{\"title\":\"MySQL - MyISAM Indexes\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"CREATED_TMP_TABLES_PERSECOND\",\"customLabel\":\"Temp Tables Created\"}},{\"id\":\"5\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"CREATED_TMP_DISK_TABLES_PERSECOND\",\"customLabel\":\"Temp Disk Tables Created\"}},{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"CREATED_TMP_FILES_PERSECOND\",\"customLabel\":\"Temp Files Created\"}}],\"listeners\":{}}", 255 | "uiStateJSON": "{}", 256 | "description": "", 257 | "savedSearchId": "mysqlbeat", 258 | "version": 1, 259 | "kibanaSavedObjectMeta": { 260 | "searchSourceJSON": "{\"filter\":[]}" 261 | } 262 | } 263 | }, 264 | { 265 | "_id": "MySQL-Sorts", 266 | "_type": "visualization", 267 | "_source": { 268 | "title": "MySQL - Sorts", 269 | "visState": "{\"title\":\"MySQL - InnoDB Row Operations\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":true,\"setYExtents\":false,\"yAxis\":{\"max\":100,\"min\":80}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SORT_SCAN_PERSECOND\",\"customLabel\":\"Scan\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SORT_ROWS_PERSECOND\",\"customLabel\":\"Rows\"}},{\"id\":\"5\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SORT_RANGE_PERSECOND\",\"customLabel\":\"Range\"}},{\"id\":\"6\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SORT_MERGE_PASSES_PERSECOND\",\"customLabel\":\"Merge Passes\"}}],\"listeners\":{}}", 270 | "uiStateJSON": "{}", 271 | "description": "", 272 | "savedSearchId": "mysqlbeat", 273 | "version": 1, 274 | "kibanaSavedObjectMeta": { 275 | "searchSourceJSON": "{\"filter\":[]}" 276 | } 277 | } 278 | }, 279 | { 280 | "_id": "MySQL-Server-List", 281 | "_type": "visualization", 282 | "_source": { 283 | "title": "MySQL - Server List", 284 | "visState": "{\"title\":\"MySQL - Server List\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Last beat\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"beat.name\",\"size\":5,\"order\":\"asc\",\"orderBy\":\"_term\",\"customLabel\":\"Server Name\"}},{\"id\":\"3\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"INNODB_PAGE_SIZE\",\"customLabel\":\"InnoDB Page Size (bytes)\"}}],\"listeners\":{}}", 285 | "uiStateJSON": "{}", 286 | "description": "", 287 | "savedSearchId": "mysqlbeat", 288 | "version": 1, 289 | "kibanaSavedObjectMeta": { 290 | "searchSourceJSON": "{\"filter\":[]}" 291 | } 292 | } 293 | }, 294 | { 295 | "_id": "MySQL-Select-Types", 296 | "_type": "visualization", 297 | "_source": { 298 | "title": "MySQL - Select Types", 299 | "visState": "{\"title\":\"MySQL - Select Types\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":true,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"beat.name\",\"size\":10,\"order\":\"asc\",\"orderBy\":\"_term\",\"row\":true}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Datetime\"}},{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SELECT_FULL_JOIN_PERSECOND\",\"customLabel\":\"Full Join\"}},{\"id\":\"4\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SELECT_FULL_RANGE_JOIN_PERSECOND\",\"customLabel\":\"Full Range Join\"}},{\"id\":\"5\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SELECT_RANGE_PERSECOND\",\"customLabel\":\"Range\"}},{\"id\":\"6\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SELECT_RANGE_CHECK_PERSECOND\",\"customLabel\":\"Range Check\"}},{\"id\":\"7\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SELECT_SCAN_PERSECOND\",\"customLabel\":\"Scan\"}}],\"listeners\":{}}", 300 | "uiStateJSON": "{}", 301 | "description": "", 302 | "savedSearchId": "mysqlbeat", 303 | "version": 1, 304 | "kibanaSavedObjectMeta": { 305 | "searchSourceJSON": "{\"filter\":[]}" 306 | } 307 | } 308 | } 309 | ] -------------------------------------------------------------------------------- /dashboard/mysql_performance_dashboard_by_mysqlbeat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adibendahan/mysqlbeat/0c74f6ac75d6f27a5812459997e76516472f31a7/dashboard/mysql_performance_dashboard_by_mysqlbeat.png -------------------------------------------------------------------------------- /dev-tools/packer/Makefile: -------------------------------------------------------------------------------- 1 | DATE:=$(shell date +%y%m%d%H%M%S) 2 | BUILDID?=$(DATE) 3 | 4 | .PHONY: all 5 | all: mysqlbeat/deb mysqlbeat/rpm mysqlbeat/darwin mysqlbeat/win mysqlbeat/bin build/upload/build_id.txt latest 6 | 7 | ES_BEATS=../../vendor/github.com/elastic/beats 8 | include $(ES_BEATS)/dev-tools/packer/scripts/Makefile 9 | 10 | 11 | .PHONY: mysqlbeat 12 | mysqlbeat: build/upload 13 | # cross compile on ubuntu 14 | cd build && xgo -image=tudorg/beats-builder \ 15 | -before-build=../$(ES_BEATS)/dev-tools/packer/xgo-scripts/before_build.sh \ 16 | -pkg $@ \ 17 | -source ../../.. \ 18 | github.com/adibendahan/$@ 19 | # linux builds on debian 6 20 | cd build && xgo -image=tudorg/beats-builder-deb6 \ 21 | -before-build=../$(ES_BEATS)/dev-tools/packer/xgo-scripts/before_build.sh \ 22 | -pkg $@ \ 23 | -source ../../.. \ 24 | github.com/adibendahan/$@ 25 | 26 | -------------------------------------------------------------------------------- /dev-tools/packer/beats/mysqlbeat.yml: -------------------------------------------------------------------------------- 1 | beat_name: mysqlbeat 2 | beat_url: 'https://github.com/adibendahan' 3 | beat_repo: 'github.com/adibendahan' 4 | beat_description: Sends any mysql query result to Logstash or directly to Elasticsearch. 5 | -------------------------------------------------------------------------------- /dev-tools/packer/version.yml: -------------------------------------------------------------------------------- 1 | version: "1.1.0-${BUILDID}" 2 | rpm_version: "1.1.0" 3 | deb_version: "1.1.0" 4 | -------------------------------------------------------------------------------- /docs/fields.asciidoc: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | This file is generated! See etc/fields.yml and scripts/generate_field_docs.py 4 | //// 5 | 6 | [[exported-fields]] 7 | == Exported Fields 8 | 9 | This document describes the fields that are exported by Mysqlbeat. They are 10 | grouped in the following categories: 11 | 12 | * <> 13 | * <> 14 | 15 | [[exported-fields-env]] 16 | === Common Fields 17 | 18 | Contains common fields available in all event types. 19 | 20 | 21 | 22 | ==== @timestamp 23 | 24 | type: date 25 | 26 | example: 2015-01-24 14:06:05.071000 27 | 28 | format: YYYY-MM-DDTHH:MM:SS.milliZ 29 | 30 | required: True 31 | 32 | The timestamp of when the measurements were taken. The precision is in milliseconds. The timezone is UTC. 33 | 34 | 35 | ==== type 36 | 37 | required: True 38 | 39 | PLEASE UPDATE DOCUMENTATION 40 | 41 | 42 | ==== count 43 | 44 | type: int 45 | 46 | required: True 47 | 48 | The number of transactions that this event represents. This is generally the inverse of the sampling rate. For example, for a sample rate of 1/10, the count is 10. The count is used by the UIs to return estimated values. Reserved for future usage. 49 | 50 | 51 | ==== beat.name 52 | 53 | Name of the Beat sending the events. If the shipper name is set in the configuration file, then that value is used. If it is not set, the hostname is used. 54 | 55 | 56 | ==== beat.hostname 57 | 58 | The hostname as returned by the operating system on which the Beat is running. 59 | 60 | 61 | -------------------------------------------------------------------------------- /docs/index.asciidoc: -------------------------------------------------------------------------------- 1 | = Mysqlbeat Docs 2 | 3 | Welcome to the Mysqlbeat documentation. 4 | 5 | 6 | -------------------------------------------------------------------------------- /etc/beat.yml: -------------------------------------------------------------------------------- 1 | ################### Mysqlbeat Configuration Example ######################### 2 | 3 | ############################# Mysqlbeat ###################################### 4 | 5 | mysqlbeat: 6 | # Defines how often an event is sent to the output 7 | #period: 10s 8 | 9 | # Defines the mysql hostname that the beat will connect to 10 | #hostname: "127.0.0.1" 11 | 12 | # Defines the mysql port 13 | #port: "3306" 14 | 15 | # MAKE SURE THE USER ONLY HAS PERMISSIONS TO RUN THE QUERY DESIRED AND NOTHING ELSE. 16 | # Defines the mysql user to use 17 | #username: "mysqlbeat_user" 18 | 19 | # Defines the mysql password to use (base64 []byte) 20 | #password64: [01, 02, 03] 21 | 22 | # Defines the queries that will run - the query below is an example 23 | #queries: ["SELECT IF(VARIABLE_NAME='COM_SELECT', 'COM_SELECT__DELTA', VARIABLE_NAME) VARIABLE_NAME, VARIABLE_VALUE FROM information_schema.GLOBAL_STATUS WHERE VARIABLE_NAME='COM_SELECT'"] 24 | 25 | # Defines the queries result types 26 | # 'single-row' will be translated as columnname:value 27 | # 'two-columns' will be translated as value-column1:value-column2 for each row 28 | # 'multiple-rows' each row will be a document (with columnname:value) 29 | #querytypes: ["two-columns"] 30 | 31 | # Colums that end with the following wild card will report only delta in seconds ((neval - oldval)/timediff.Seconds()) 32 | #deltawildcard: "__DELTA" 33 | -------------------------------------------------------------------------------- /etc/fields.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | 3 | defaults: 4 | type: string 5 | required: false 6 | index: not_analyzed 7 | doc_values: true 8 | ignore_above: 1024 9 | 10 | env: 11 | type: group 12 | description: > 13 | Contains common fields available in all event types. 14 | fields: 15 | - name: "@timestamp" 16 | type: date 17 | required: true 18 | format: YYYY-MM-DDTHH:MM:SS.milliZ 19 | example: 2015-01-24T14:06:05.071Z 20 | description: > 21 | The timestamp of when the measurements were taken. The precision is in milliseconds. 22 | The timezone is UTC. 23 | 24 | - name: type 25 | description: > 26 | PLEASE UPDATE DOCUMENTATION 27 | required: true 28 | 29 | - name: count 30 | type: int 31 | description: > 32 | The number of transactions that this event represents. This 33 | is generally the inverse of the sampling rate. For example, for 34 | a sample rate of 1/10, the count is 10. The count is used by the 35 | UIs to return estimated values. Reserved for future usage. 36 | required: true 37 | 38 | - name: beat.name 39 | description: > 40 | Name of the Beat sending the events. If the shipper name is set 41 | in the configuration file, then that value is used. If it is not set, 42 | the hostname is used. 43 | 44 | - name: beat.hostname 45 | description: > 46 | The hostname as returned by the operating system on which the Beat is 47 | running. 48 | 49 | mysqlbeat: 50 | fields: 51 | - name: counter 52 | type: integer 53 | required: true 54 | description: > 55 | PLEASE UPDATE DOCUMENTATION 56 | 57 | sections: 58 | - ["env", "Common"] 59 | - ["mysqlbeat", "Mysqlbeat"] 60 | 61 | -------------------------------------------------------------------------------- /etc/mysqlbeat.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": { 3 | "_default_": { 4 | "_all": { 5 | "enabled": true, 6 | "norms": { 7 | "enabled": false 8 | } 9 | }, 10 | "dynamic_templates": [ 11 | { 12 | "template1": { 13 | "mapping": { 14 | "doc_values": true, 15 | "ignore_above": 1024, 16 | "index": "not_analyzed", 17 | "type": "{dynamic_type}" 18 | }, 19 | "match": "*" 20 | } 21 | } 22 | ], 23 | "properties": { 24 | "@timestamp": { 25 | "type": "date" 26 | }, 27 | "ABORTED_CONNECTS_PERSECOND": { 28 | "type": "long" 29 | }, 30 | "CONNECTION_ERRORS_INTERNAL_PERSECOND": { 31 | "type": "long" 32 | }, 33 | "CONNECTION_ERRORS_MAX_CONNECTIONS_PERSECOND": { 34 | "type": "long" 35 | }, 36 | "CONST_100": { 37 | "type": "long" 38 | }, 39 | "CREATED_TMP_DISK_TABLES_PERSECOND": { 40 | "type": "long" 41 | }, 42 | "CREATED_TMP_FILES_PERSECOND": { 43 | "type": "long" 44 | }, 45 | "CREATED_TMP_TABLES_PERSECOND": { 46 | "type": "long" 47 | }, 48 | "HANDLER_COMMIT_PERSECOND": { 49 | "type": "long" 50 | }, 51 | "HANDLER_ROLLBACK_PERSECOND": { 52 | "type": "long" 53 | }, 54 | "HANDLER_SAVEPOINT_ROLLBACK_PERSECOND": { 55 | "type": "long" 56 | }, 57 | "HANDLER_SAVEPOINT_PERSECOND": { 58 | "type": "long" 59 | }, 60 | "INNODB_BUFFER_POOL_HIT_RATIO": { 61 | "type": "double" 62 | }, 63 | "INNODB_BUFFER_POOL_PAGES_DATA": { 64 | "type": "long" 65 | }, 66 | "INNODB_BUFFER_POOL_PAGES_FREE": { 67 | "type": "long" 68 | }, 69 | "INNODB_BUFFER_POOL_PAGES_TOTAL": { 70 | "type": "long" 71 | }, 72 | "INNODB_PAGE_SIZE": { 73 | "type": "long" 74 | }, 75 | "INNODB_DATA_FSYNCS_PERSECOND": { 76 | "type": "long" 77 | }, 78 | "INNODB_DATA_READS_PERSECOND": { 79 | "type": "long" 80 | }, 81 | "INNODB_DATA_WRITES_PERSECOND": { 82 | "type": "long" 83 | }, 84 | "INNODB_LOG_WRITES_PERSECOND": { 85 | "type": "long" 86 | }, 87 | "INNODB_OS_LOG_FSYNCS_PERSECOND": { 88 | "type": "long" 89 | }, 90 | "INNODB_PAGES_CREATED_PERSECOND": { 91 | "type": "long" 92 | }, 93 | "INNODB_PAGES_READ_PERSECOND": { 94 | "type": "long" 95 | }, 96 | "INNODB_PAGES_WRITTEN_PERSECOND": { 97 | "type": "long" 98 | }, 99 | "INNODB_ROWS_DELETED_PERSECOND": { 100 | "type": "long" 101 | }, 102 | "INNODB_ROWS_INSERTED_PERSECOND": { 103 | "type": "long" 104 | }, 105 | "INNODB_ROWS_READ_PERSECOND": { 106 | "type": "long" 107 | }, 108 | "INNODB_ROWS_UPDATED_PERSECOND": { 109 | "type": "long" 110 | }, 111 | "INNODB_ROW_LOCK_WAITS_PERSECOND": { 112 | "type": "long" 113 | }, 114 | "INNODB_ROW_LOCK_TIME_PERSECOND": { 115 | "type": "long" 116 | }, 117 | "KEY_READS_PERSECOND": { 118 | "type": "long" 119 | }, 120 | "KEY_READ_REQUESTS_PERSECOND": { 121 | "type": "long" 122 | }, 123 | "KEY_WRITES_PERSECOND": { 124 | "type": "long" 125 | }, 126 | "KEY_WRITE_REQUESTS_PERSECOND": { 127 | "type": "long" 128 | }, 129 | "MAX_CONNECTIONS": { 130 | "type": "long" 131 | }, 132 | "OPENED_TABLES_PERSECOND": { 133 | "type": "long" 134 | }, 135 | "OPEN_FILES": { 136 | "type": "long" 137 | }, 138 | "OPEN_TABLES": { 139 | "type": "long" 140 | }, 141 | "QCACHE_HITS_PERSECOND": { 142 | "type": "long" 143 | }, 144 | "QCACHE_INSERTS_PERSECOND": { 145 | "type": "long" 146 | }, 147 | "QCACHE_LOWMEM_PRUNES_PERSECOND": { 148 | "type": "long" 149 | }, 150 | "QCACHE_NOT_CACHED_PERSECOND": { 151 | "type": "long" 152 | }, 153 | "QCACHE_QUERIES_IN_CACHE": { 154 | "type": "long" 155 | }, 156 | "SELECT_FULL_JOIN_PERSECOND": { 157 | "type": "long" 158 | }, 159 | "SELECT_FULL_RANGE_JOIN_PERSECOND": { 160 | "type": "long" 161 | }, 162 | "SELECT_RANGE_CHECK_PERSECOND": { 163 | "type": "long" 164 | }, 165 | "SELECT_RANGE_PERSECOND": { 166 | "type": "long" 167 | }, 168 | "SELECT_SCAN_PERSECOND": { 169 | "type": "long" 170 | }, 171 | "SLOW_QUERIES_PERSECOND": { 172 | "type": "long" 173 | }, 174 | "SORT_MERGE_PASSES_PERSECOND": { 175 | "type": "long" 176 | }, 177 | "SORT_RANGE_PERSECOND": { 178 | "type": "long" 179 | }, 180 | "SORT_ROWS_PERSECOND": { 181 | "type": "long" 182 | }, 183 | "SORT_SCAN_PERSECOND": { 184 | "type": "long" 185 | }, 186 | "Seconds_Behind_Master": { 187 | "type": "long" 188 | }, 189 | "TABLE_LOCKS_IMMEDIATE_PERSECOND": { 190 | "type": "long" 191 | }, 192 | "TABLE_LOCKS_WAITED_PERSECOND": { 193 | "type": "long" 194 | }, 195 | "TABLE_OPEN_CACHE": { 196 | "type": "long" 197 | }, 198 | "THREADS_CONNECTED": { 199 | "type": "long" 200 | }, 201 | "THREADS_RUNNING": { 202 | "type": "long" 203 | } 204 | } 205 | } 206 | }, 207 | "settings": { 208 | "index.refresh_interval": "10s", 209 | "number_of_shards": 2 210 | }, 211 | "template": "mysqlbeat-*" 212 | } -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: . 2 | import: 3 | - package: github.com/elastic/beats 4 | version: d8ca37efa8888ce624e2cfaa90107e79fd41be1e 5 | subpackages: 6 | - libbeat/beat 7 | - libbeat/cfgfile 8 | - libbeat/common 9 | - libbeat/logp 10 | - package: github.com/go-sql-driver/mysql 11 | version: 1421caf44f6464fd2ee8de694c7508ee13f92964 12 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/elastic/beats/libbeat/beat" 7 | 8 | "github.com/adibendahan/mysqlbeat/beater" 9 | ) 10 | 11 | func main() { 12 | err := beat.Run("mysqlbeat", "", beater.New()) 13 | if err != nil { 14 | os.Exit(1) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // This file is mandatory as otherwise the mysqlbeat.test binary is not generated correctly. 4 | 5 | import ( 6 | "flag" 7 | "testing" 8 | ) 9 | 10 | var systemTest *bool 11 | 12 | func init() { 13 | systemTest = flag.Bool("systemTest", false, "Set to true when running system tests") 14 | } 15 | 16 | // Test started when the test binary is started. Only calls main. 17 | func TestSystem(t *testing.T) { 18 | 19 | if *systemTest { 20 | main() 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /mysqlbeat.yml: -------------------------------------------------------------------------------- 1 | ################### Mysqlbeat Configuration Example ######################### 2 | 3 | ############################# Mysqlbeat ###################################### 4 | 5 | mysqlbeat: 6 | # Defines how often an event is sent to the output 7 | #period: 10s 8 | 9 | # Defines the mysql hostname that the beat will connect to 10 | #hostname: "127.0.0.1" 11 | 12 | # Defines the mysql port 13 | #port: "3306" 14 | 15 | # MAKE SURE THE USER ONLY HAS PERMISSIONS TO RUN THE QUERY DESIRED AND NOTHING ELSE. 16 | # Defines the mysql user to use 17 | #username: "mysqlbeat_user" 18 | 19 | # Defines the mysql password to use - option #1 - plain text 20 | #password: "mysqlbeat_pass" 21 | 22 | # Defines the mysql password to use - option #2 - AES encryption (see github.com/adibendahan/mysqlbeat-password-encrypter) 23 | #encryptedpassword: "2321f38819cf693951e88f00cd82" 24 | 25 | # Defines the queries that will run - the query below is an example 26 | # LIMITATIONS: Query must start with SELECT/SHOW and cannot contain the character ; (for security reasons) 27 | queries: [ "SELECT CONCAT(VARIABLE_NAME, '__DELTA') AS VARIABLE_NAME, VARIABLE_VALUE 28 | FROM information_schema.GLOBAL_STATUS 29 | WHERE VARIABLE_NAME IN ('ABORTED_CONNECTS', 'CONNECTION_ERRORS_INTERNAL', 'CONNECTION_ERRORS_MAX_CONNECTIONS', 'CREATED_TMP_DISK_TABLES', 'CREATED_TMP_FILES', 30 | 'CREATED_TMP_TABLES', 'HANDLER_COMMIT', 'HANDLER_ROLLBACK', 'HANDLER_SAVEPOINT', 'HANDLER_SAVEPOINT_ROLLBACK', 'INNODB_DATA_FSYNCS', 31 | 'INNODB_DATA_READS', 'INNODB_DATA_WRITES', 'INNODB_LOG_WRITES', 'INNODB_OS_LOG_FSYNCS', 'INNODB_PAGES_CREATED', 'INNODB_PAGES_READ', 32 | 'INNODB_PAGES_WRITTEN', 'INNODB_ROWS_DELETED', 'INNODB_ROWS_INSERTED', 'INNODB_ROWS_READ', 'INNODB_ROWS_UPDATED', 'INNODB_ROW_LOCK_TIME', 33 | 'KEY_READS', 'KEY_READ_REQUESTS', 'KEY_WRITES', 'KEY_WRITE_REQUESTS', 'OPENED_TABLES', 'QCACHE_HITS', 'QCACHE_INSERTS', 'QCACHE_LOWMEM_PRUNES', 34 | 'QCACHE_NOT_CACHED', 'SELECT_FULL_JOIN', 'SELECT_FULL_RANGE_JOIN', 'SELECT_RANGE', 'SELECT_RANGE_CHECK', 'SELECT_SCAN', 'SLOW_QUERIES', 35 | 'SORT_MERGE_PASSES', 'SORT_RANGE', 'SORT_ROWS', 'SORT_SCAN', 'TABLE_LOCKS_IMMEDIATE', 'TABLE_LOCKS_WAITED', 'INNODB_ROW_LOCK_WAITS') 36 | UNION 37 | SELECT VARIABLE_NAME, VARIABLE_VALUE 38 | FROM information_schema.GLOBAL_VARIABLES 39 | WHERE VARIABLE_NAME IN ('MAX_CONNECTIONS', 'TABLE_OPEN_CACHE') 40 | UNION 41 | SELECT VARIABLE_NAME, VARIABLE_VALUE 42 | FROM information_schema.GLOBAL_STATUS 43 | WHERE VARIABLE_NAME IN ('INNODB_BUFFER_POOL_PAGES_DATA', 'INNODB_BUFFER_POOL_PAGES_FREE', 'INNODB_BUFFER_POOL_PAGES_TOTAL', 'OPEN_FILES', 'OPEN_TABLES', 44 | 'QCACHE_QUERIES_IN_CACHE', 'THREADS_CONNECTED', 'THREADS_RUNNING', 'INNODB_PAGE_SIZE') 45 | UNION 46 | SELECT 'INNODB_BUFFER_POOL_HIT_RATIO' AS VARIABLE_NAME, 47 | INNODB_BUFFER_POOL_READ_REQUESTS.VARIABLE_VALUE / (INNODB_BUFFER_POOL_READ_REQUESTS.VARIABLE_VALUE + INNODB_BUFFER_POOL_READS.VARIABLE_VALUE) * 100 AS VARIABLE_VALUE 48 | FROM information_schema.GLOBAL_STATUS INNODB_BUFFER_POOL_READS, information_schema.GLOBAL_STATUS INNODB_BUFFER_POOL_READ_REQUESTS 49 | WHERE INNODB_BUFFER_POOL_READS.variable_name ='INNODB_BUFFER_POOL_READS' AND 50 | INNODB_BUFFER_POOL_READ_REQUESTS.variable_name ='INNODB_BUFFER_POOL_READ_REQUESTS' 51 | UNION 52 | SELECT 'CONST_100' AS VARIABLE_NAME, 100 AS VARIABLE_VALUE", 53 | "SHOW SLAVE STATUS"] 54 | 55 | # Defines the queries result types 56 | # 'single-row' will be translated as columnname:value 57 | # 'two-columns' will be translated as value-column1:value-column2 for each row 58 | # 'multiple-rows' each row will be a document (with columnname:value) 59 | # 'show-slave-delay' will only send the `Seconds_Behind_Master` column from SHOW SLAVE STATUS 60 | querytypes: ["two-columns", 61 | "show-slave-delay"] 62 | 63 | # Colums that end with the following wild card will report only delta in seconds ((neval - oldval)/timediff.Seconds()) 64 | deltawildcard: "__DELTA" 65 | 66 | # In a multiple-rows event, each row must have a unique key so that calculations could be saved for every column. 67 | # IMPORTANT: make sure that the combination of all DeltaKey columns in a row create a UNIQUE value per row in the query 68 | deltakeywildcard: "__DELTAKEY" 69 | 70 | ############################################################################### 71 | ############################# Libbeat Config ################################## 72 | # Base config file used by all other beats for using libbeat features 73 | 74 | ############################# Output ########################################## 75 | 76 | # Configure what outputs to use when sending the data collected by the beat. 77 | # Multiple outputs may be used. 78 | output: 79 | 80 | ### Elasticsearch as output 81 | elasticsearch: 82 | # Array of hosts to connect to. 83 | # Scheme and port can be left out and will be set to the default (http and 9200) 84 | # In case you specify and additional path, the scheme is required: http://localhost:9200/path 85 | # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 86 | #hosts: ["127.0.0.1:9200"] 87 | 88 | # Optional protocol and basic auth credentials. 89 | #protocol: "https" 90 | #username: "admin" 91 | #password: "s3cr3t" 92 | 93 | # Dictionary of HTTP parameters to pass within the url with index operations. 94 | #parameters: 95 | #param1: value1 96 | #param2: value2 97 | 98 | # Number of workers per Elasticsearch host. 99 | #worker: 1 100 | 101 | # Optional index name. The default is "mysqlbeat" and generates 102 | # [mysqlbeat-]YYYY.MM.DD keys. 103 | #index: "mysqlbeat" 104 | 105 | # A template is used to set the mapping in Elasticsearch 106 | # By default template loading is disabled and no template is loaded. 107 | # These settings can be adjusted to load your own template or overwrite existing ones 108 | #template: 109 | 110 | # Template name. By default the template name is mysqlbeat. 111 | #name: "mysqlbeat" 112 | 113 | # Path to template file 114 | #path: "mysqlbeat.template.json" 115 | 116 | # Overwrite existing template 117 | #overwrite: false 118 | 119 | # Optional HTTP Path 120 | #path: "/elasticsearch" 121 | 122 | # Proxy server url 123 | #proxy_url: http://proxy:3128 124 | 125 | # The number of times a particular Elasticsearch index operation is attempted. If 126 | # the indexing operation doesn't succeed after this many retries, the events are 127 | # dropped. The default is 3. 128 | #max_retries: 3 129 | 130 | # The maximum number of events to bulk in a single Elasticsearch bulk API index request. 131 | # The default is 50. 132 | bulk_max_size: 50 133 | 134 | # Configure http request timeout before failing an request to Elasticsearch. 135 | #timeout: 90 136 | 137 | # The number of seconds to wait for new events between two bulk API index requests. 138 | # If `bulk_max_size` is reached before this interval expires, addition bulk index 139 | # requests are made. 140 | #flush_interval: 1 141 | 142 | # Boolean that sets if the topology is kept in Elasticsearch. The default is 143 | # false. This option makes sense only for Packetbeat. 144 | #save_topology: false 145 | 146 | # The time to live in seconds for the topology information that is stored in 147 | # Elasticsearch. The default is 15 seconds. 148 | #topology_expire: 15 149 | 150 | # tls configuration. By default is off. 151 | #tls: 152 | # List of root certificates for HTTPS server verifications 153 | #certificate_authorities: ["/etc/pki/root/ca.pem"] 154 | 155 | # Certificate for TLS client authentication 156 | #certificate: "/etc/pki/client/cert.pem" 157 | 158 | # Client Certificate Key 159 | #certificate_key: "/etc/pki/client/cert.key" 160 | 161 | # Controls whether the client verifies server certificates and host name. 162 | # If insecure is set to true, all server host names and certificates will be 163 | # accepted. In this mode TLS based connections are susceptible to 164 | # man-in-the-middle attacks. Use only for testing. 165 | #insecure: true 166 | 167 | # Configure cipher suites to be used for TLS connections 168 | #cipher_suites: [] 169 | 170 | # Configure curve types for ECDHE based cipher suites 171 | #curve_types: [] 172 | 173 | # Configure minimum TLS version allowed for connection to logstash 174 | #min_version: 1.0 175 | 176 | # Configure maximum TLS version allowed for connection to logstash 177 | #max_version: 1.2 178 | 179 | 180 | ### Logstash as output 181 | #logstash: 182 | # The Logstash hosts 183 | #hosts: ["localhost:5044"] 184 | 185 | # Number of workers per Logstash host. 186 | #worker: 1 187 | 188 | # Set gzip compression level. 189 | #compression_level: 3 190 | 191 | # Optional load balance the events between the Logstash hosts 192 | #loadbalance: true 193 | 194 | # Optional index name. The default index name is set to name of the beat 195 | # in all lowercase. 196 | #index: mysqlbeat 197 | 198 | # SOCKS5 proxy server URL 199 | #proxy_url: socks5://user:password@socks5-server:2233 200 | 201 | # Resolve names locally when using a proxy server. Defaults to false. 202 | #proxy_use_local_resolver: false 203 | 204 | # Optional TLS. By default is off. 205 | #tls: 206 | # List of root certificates for HTTPS server verifications 207 | #certificate_authorities: ["/etc/pki/root/ca.pem"] 208 | 209 | # Certificate for TLS client authentication 210 | #certificate: "/etc/pki/client/cert.pem" 211 | 212 | # Client Certificate Key 213 | #certificate_key: "/etc/pki/client/cert.key" 214 | 215 | # Controls whether the client verifies server certificates and host name. 216 | # If insecure is set to true, all server host names and certificates will be 217 | # accepted. In this mode TLS based connections are susceptible to 218 | # man-in-the-middle attacks. Use only for testing. 219 | #insecure: true 220 | 221 | # Configure cipher suites to be used for TLS connections 222 | #cipher_suites: [] 223 | 224 | # Configure curve types for ECDHE based cipher suites 225 | #curve_types: [] 226 | 227 | 228 | ### File as output 229 | #file: 230 | # Path to the directory where to save the generated files. The option is mandatory. 231 | #path: "/tmp/mysqlbeat" 232 | 233 | # Name of the generated files. The default is `mysqlbeat` and it generates files: `mysqlbeat`, `mysqlbeat.1`, `mysqlbeat.2`, etc. 234 | #filename: mysqlbeat 235 | 236 | # Maximum size in kilobytes of each file. When this size is reached, the files are 237 | # rotated. The default value is 10240 kB. 238 | #rotate_every_kb: 10000 239 | 240 | # Maximum number of files under path. When this number of files is reached, the 241 | # oldest file is deleted and the rest are shifted from last to first. The default 242 | # is 7 files. 243 | #number_of_files: 7 244 | 245 | 246 | ### Console output 247 | # console: 248 | # Pretty print json event 249 | #pretty: false 250 | 251 | 252 | ############################# Shipper ######################################### 253 | 254 | shipper: 255 | # The name of the shipper that publishes the network data. It can be used to group 256 | # all the transactions sent by a single shipper in the web interface. 257 | # If this options is not defined, the hostname is used. 258 | #name: 259 | 260 | # The tags of the shipper are included in their own field with each 261 | # transaction published. Tags make it easy to group servers by different 262 | # logical properties. 263 | #tags: ["service-X", "web-tier"] 264 | 265 | # Optional fields that you can specify to add additional information to the 266 | # output. Fields can be scalar values, arrays, dictionaries, or any nested 267 | # combination of these. 268 | #fields: 269 | # env: staging 270 | 271 | # If this option is set to true, the custom fields are stored as top-level 272 | # fields in the output document instead of being grouped under a fields 273 | # sub-dictionary. Default is false. 274 | #fields_under_root: false 275 | 276 | # Uncomment the following if you want to ignore transactions created 277 | # by the server on which the shipper is installed. This option is useful 278 | # to remove duplicates if shippers are installed on multiple servers. 279 | #ignore_outgoing: true 280 | 281 | # How often (in seconds) shippers are publishing their IPs to the topology map. 282 | # The default is 10 seconds. 283 | #refresh_topology_freq: 10 284 | 285 | # Expiration time (in seconds) of the IPs published by a shipper to the topology map. 286 | # All the IPs will be deleted afterwards. Note, that the value must be higher than 287 | # refresh_topology_freq. The default is 15 seconds. 288 | #topology_expire: 15 289 | 290 | # Internal queue size for single events in processing pipeline 291 | #queue_size: 1000 292 | 293 | # Sets the maximum number of CPUs that can be executing simultaneously. The 294 | # default is the number of logical CPUs available in the system. 295 | #max_procs: 296 | 297 | # Configure local GeoIP database support. 298 | # If no paths are not configured geoip is disabled. 299 | #geoip: 300 | #paths: 301 | # - "/usr/share/GeoIP/GeoLiteCity.dat" 302 | # - "/usr/local/var/GeoIP/GeoLiteCity.dat" 303 | 304 | 305 | ############################# Logging ######################################### 306 | 307 | # There are three options for the log output: syslog, file, stderr. 308 | # Under Windows systems, the log files are per default sent to the file output, 309 | # under all other system per default to syslog. 310 | logging: 311 | 312 | # Send all logging output to syslog. On Windows default is false, otherwise 313 | # default is true. 314 | #to_syslog: true 315 | 316 | # Write all logging output to files. Beats automatically rotate files if rotateeverybytes 317 | # limit is reached. 318 | #to_files: false 319 | 320 | # To enable logging to files, to_files option has to be set to true 321 | #files: 322 | # The directory where the log files will written to. 323 | #path: /var/log/mysqlbeat 324 | 325 | # The name of the files where the logs are written to. 326 | #name: mysqlbeat 327 | 328 | # Configure log file size limit. If limit is reached, log file will be 329 | # automatically rotated 330 | #rotateeverybytes: 10485760 # = 10MB 331 | 332 | # Number of rotated log files to keep. Oldest files will be deleted first. 333 | #keepfiles: 7 334 | 335 | # Enable debug output for selected components. To enable all selectors use ["*"] 336 | # Other available selectors are beat, publish, service 337 | # Multiple selectors can be chained. 338 | #selectors: [ ] 339 | 340 | # Sets log level. The default log level is error. 341 | # Available log levels are: critical, error, warning, info, debug 342 | #level: error 343 | 344 | 345 | -------------------------------------------------------------------------------- /tests/system/config/mysqlbeat.yml.j2: -------------------------------------------------------------------------------- 1 | ################### Beat Configuration ######################### 2 | 3 | 4 | 5 | ############################# Output ########################################## 6 | 7 | # Configure what outputs to use when sending the data collected by the beat. 8 | # You can enable one or multiple outputs by setting enabled option to true. 9 | output: 10 | 11 | ### File as output 12 | file: 13 | # Enabling file output 14 | enabled: true 15 | 16 | # Path to the directory where to save the generated files. The option is mandatory. 17 | path: {{ output_file_path|default(beat.working_dir + "/output") }} 18 | 19 | 20 | # Name of the generated files. The default is `mysqlbeat` and it generates 21 | # files: `mysqlbeat`, `mysqlbeat.1`, `mysqlbeat.2`, etc. 22 | filename: "{{ output_file_filename|default("mysqlbeat") }}" 23 | 24 | # Maximum size in kilobytes of each file. When this size is reached, the files are 25 | # rotated. The default value is 10 MB. 26 | #rotate_every_kb: 10000 27 | 28 | # Maximum number of files under path. When this number of files is reached, the 29 | # oldest file is deleted and the rest are shifted from last to first. The default 30 | # is 7 files. 31 | #number_of_files: 7 32 | 33 | 34 | 35 | ############################# Shipper ######################################### 36 | 37 | shipper: 38 | # The name of the shipper that publishes the network data. It can be used to group 39 | # all the transactions sent by a single shipper in the web interface. 40 | # If this options is not defined, the hostname is used. 41 | #name: 42 | 43 | # The tags of the shipper are included in their own field with each 44 | # transaction published. Tags make it easy to group servers by different 45 | # logical properties. 46 | #tags: ["service-X", "web-tier"] 47 | 48 | # Uncomment the following if you want to ignore transactions created 49 | # by the server on which the shipper is installed. This option is useful 50 | # to remove duplicates if shippers are installed on multiple servers. 51 | #ignore_outgoing: true 52 | 53 | 54 | ############################# Logging ######################################### 55 | 56 | #logging: 57 | # Send all logging output to syslog. On Windows default is false, otherwise 58 | # default is true. 59 | #to_syslog: true 60 | 61 | # Write all logging output to files. Beats automatically rotate files if configurable 62 | # limit is reached. 63 | #to_files: false 64 | 65 | # Enable debug output for selected components. 66 | #selectors: [] 67 | 68 | # Set log level 69 | #level: error 70 | 71 | #files: 72 | # The directory where the log files will written to. 73 | #path: /var/log/mysqlbeat 74 | 75 | # The name of the files where the logs are written to. 76 | #name: mysqlbeat 77 | 78 | # Configure log file size limit. If limit is reached, log file will be 79 | # automatically rotated 80 | #rotateeverybytes: 10485760 # = 10MB 81 | 82 | # Number of rotated log files to keep. Oldest files will be deleted first. 83 | #keepfiles: 7 84 | -------------------------------------------------------------------------------- /tests/system/mysqlbeat.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../../vendor/github.com/elastic/beats/libbeat/tests/system') 3 | from beat.beat import TestCase 4 | 5 | class BaseTest(TestCase): 6 | 7 | @classmethod 8 | def setUpClass(self): 9 | self.beat_name = "mysqlbeat" 10 | self.build_path = "../../build/system-tests/" 11 | self.beat_path = "../../mysqlbeat.test" 12 | -------------------------------------------------------------------------------- /tests/system/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adibendahan/mysqlbeat/0c74f6ac75d6f27a5812459997e76516472f31a7/tests/system/requirements.txt -------------------------------------------------------------------------------- /tests/system/test_base.py: -------------------------------------------------------------------------------- 1 | from mysqlbeat import BaseTest 2 | 3 | import os 4 | 5 | 6 | class Test(BaseTest): 7 | 8 | def test_base(self): 9 | """ 10 | Basic test with exiting Mysqlbeat normally 11 | """ 12 | self.render_config_template( 13 | path=os.path.abspath(self.working_dir) + "/log/*" 14 | ) 15 | 16 | mysqlbeat_proc = self.start_beat() 17 | self.wait_until( lambda: self.log_contains("mysqlbeat is running")) 18 | exit_code = mysqlbeat_proc.kill_and_wait() 19 | assert exit_code == 0 20 | --------------------------------------------------------------------------------