├── README.md
└── normalize_mapper.py


/README.md:
--------------------------------------------------------------------------------
1 | # mysql_log_check
2 | MySQL Log Analysis
3 | 


--------------------------------------------------------------------------------
/normalize_mapper.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | import sys
  3 | import json
  4 | import re
  5 | import logging
  6 | import base64
  7 | 
  8 | #db log format configure
  9 | db_pattern = r"^(\d{2}\d{2}\d{2}\s+\d{1,2}:\d{2}:\d{2}|\t)\s+\d+\s+([A-Za-z]+)\s*(.*)$"
 10 | db_regex = re.compile(db_pattern)
 11 | 
 12 | sql_pattern = r"^(\S+)\s"
 13 | sql_regex = re.compile(sql_pattern)
 14 | 
 15 | #log configure
 16 | logging.basicConfig(level = logging.ERROR,
 17 |                     format = '%(message)s',                    
 18 |                     stream = sys.stderr)
 19 | 
 20 | 
 21 | #query blacklist configure
 22 | command_blacklist = [
 23 | "query"
 24 | ]
 25 | 
 26 | query_blacklist = [
 27 | "select",
 28 | "update",
 29 | "insert",
 30 | "delete",
 31 | "replace"
 32 | ]
 33 | 
 34 | def read_mapper_output(file):
 35 |     """
 36 |     read data from file using yield
 37 |     """
 38 |     for line in file:
 39 |         yield line.rstrip() 
 40 | 
 41 | 
 42 | def db_log_normailze():
 43 |     """
 44 |     normalize db log, extend timestamp and merge multi-line sql statement
 45 |     """
 46 |     #read data from stdin
 47 |     data = read_mapper_output(sys.stdin)
 48 | 
 49 |     #last time 
 50 |     last_time = "\t"
 51 |     #current time command and sql
 52 |     time = ""
 53 |     command = ""
 54 |     sql = ""
 55 |     line_number = 1
 56 |     
 57 | 
 58 |     for line in data:
 59 |         db_match = db_regex.search(line)
 60 | 
 61 |         if db_match:
 62 |             if command != "":
 63 |                 if sql and command.lower() in command_blacklist:
 64 |                     sql_match = sql_regex.search(sql)
 65 |                     if sql_match:
 66 |                         sql_command = sql_match.group(1)
 67 |                         if sql_command.lower() in query_blacklist:
 68 |                             debug = "FINAL_RESULT %d: %s %s %s" %(line_number - 1, time, command, sql)
 69 |                             logging.debug(debug)
 70 |                             sql_base64 = base64.b64encode(sql)
 71 |                             time_base64 = base64.b64encode(time)
 72 |                             print "%s\t%s" %(sql_base64, time_base64)
 73 |                             
 74 |             else:
 75 |                 info ="NULL_COMMAND %d: %s %s %s" %(line_number - 1, time, command, sql)
 76 |                 logging.info(info)
 77 | 
 78 |             time, command, sql = db_match.groups()            
 79 |             #time extend
 80 |             if time == "\t":
 81 |                 time = last_time
 82 |             else:
 83 |                 last_time = time
 84 |         else:
 85 |             #for debug
 86 |             info = "MULTI_LINE %d: %s" %(line_number, line.strip())
 87 |             logging.info(info)
 88 | 
 89 |             if command != "":
 90 |                 sql = sql + line
 91 |                 
 92 |                 
 93 |         line_number = line_number + 1
 94 |         
 95 |         
 96 |         
 97 |         
 98 | 
 99 | if __name__ == '__main__':
100 |     db_log_normailze()    
101 | 


--------------------------------------------------------------------------------