├── README.md ├── generate_mplot_logs.py └── requirements.txt /README.md: -------------------------------------------------------------------------------- 1 | # convert-json-logs-to-legacy 2 | Convert MongoDB 4.4 JSON logs to legacy logs to work for mTools and t2 3 | 4 | Modified the script to generate logs that would resemble the legacy format. These changes support mplotquery --yaxis. 5 | 6 | # Usage: 7 | generate_mplot_logs.py --log mongodb.log > legacy.log 8 | 9 | # Below are the following log types that the script process: 10 | * CONTROL 11 | * Only a single CONTROL line to make it work for t2 12 | * It needs to add pid, port, arch and host 13 | * ACCESS 14 | * To generate authentication entry 15 | * NETWORK 16 | * To generate the clients, connstats and connections statistics 17 | * COMMAND, WRITE, QUERY and TXN 18 | * To generate mplotqueries 19 | * To generate mloginfo --queries 20 | * To generate mloginfo --clients --connstats --connections 21 | * To generate mplotqueries with bytesRead as yaxis. mplotqueries --yaxis bytesRead 22 | * Filter out MongoDB 8.0 attribute isFromUserConnection and collectionType 23 | 24 | # Note: 25 | The converted log can be dragged to t2 to add "Server Log" metrics 26 | -------------------------------------------------------------------------------- /generate_mplot_logs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | generate_mplot_logs.py logfile 4 | 5 | Usage: 6 | generate_mplot_logs.py [options] 7 | 8 | Options: 9 | -h --help Show this text. 10 | --log MongoDB log to convert 11 | """ 12 | 13 | import json 14 | import re 15 | from docopt import docopt 16 | 17 | def span_list(obj): 18 | ctr = 0 19 | cmd = '' 20 | command = '' 21 | for ptr in obj: 22 | ctr = ctr + 1 23 | if ctr > 1: 24 | cmd = cmd + ', ' 25 | if type(ptr) is dict: 26 | cmd = cmd + span_object(ptr) 27 | elif type(ptr) is list: 28 | cmd = cmd + span_list(ptr) 29 | else: 30 | cmd = cmd + json.dumps(ptr) 31 | # cmd = cmd + json.dumps(obj[ptr]) 32 | command = '[ ' + cmd + ' ]' 33 | return(command) 34 | 35 | def span_object(obj): 36 | ctr = 0 37 | command = '' 38 | for ptr in obj: 39 | ctr = ctr + 1 40 | if ctr > 1: 41 | command = command + ', ' 42 | if type(obj[ptr]) is dict: 43 | command= command + ptr + ': ' + span_object(obj[ptr]) 44 | elif type(obj[ptr]) is list: 45 | command= command + ptr + ': ' + span_list(obj[ptr]) 46 | else: 47 | command = command + ptr + ': ' + json.dumps(obj[ptr]) 48 | command = '{ ' + command + ' }' 49 | return(command) 50 | 51 | def convert_log_line(logfile): 52 | log = open(logfile, 'r') 53 | 54 | # count = 0 55 | for line in log.readlines(): 56 | # count = count + 1 57 | # print("line:{}".format(count)) 58 | try: 59 | obj = json.loads(line) 60 | except Exception: 61 | continue 62 | c = obj["c"] 63 | dt = obj['t']['$date'] 64 | dt = re.sub(r'(\+\d\d):(\d\d)$',r'\1\2', dt) 65 | s = obj['s'] 66 | ctx = obj['ctx'] 67 | attr = [] 68 | if 'msg' in obj and obj['msg'] != 'Slow query': 69 | if obj['msg'] == 'Connection ended': 70 | attr.append('end connection') 71 | elif obj['msg'] == 'Connection accepted': 72 | attr.append('connection accepted') 73 | elif obj['msg'] == 'Authentication succeeded': 74 | attr.append('Successfully authenticated') 75 | else: 76 | attr.append(obj['msg']) 77 | if c == 'CONTROL' and 'attr' in obj and 'host' in obj['attr']: 78 | pid = obj['attr']['pid'] 79 | port = obj['attr']['port'] 80 | arch = obj['attr']['architecture'] 81 | host = obj['attr']['host'] 82 | attr.pop() 83 | attr.append("pid={} port={} {} host={}".format(pid, port, arch, host)) 84 | elif c == 'ACCESS': 85 | if 'attr' in obj: 86 | for key in obj['attr']: 87 | if key == 'principalName': 88 | attr.append('as principal ' + str(obj['attr'][key])) 89 | elif key == 'authenticationDatabase': 90 | attr.append('on ' + obj['attr'][key]) 91 | elif key == 'remote': 92 | attr.append('from client '+str(obj['attr'][key])) 93 | elif c == 'NETWORK': 94 | if 'attr' in obj: 95 | for key in obj['attr']: 96 | if key == 'remote': 97 | if obj['msg'] == 'Connection ended': 98 | attr.append(obj['attr'][key]) 99 | else: 100 | attr.append('from '+obj['attr'][key]) 101 | elif key == 'connectionId' and obj['msg'] != 'Connection ended': 102 | attr.append('#'+str(obj['attr'][key])) 103 | elif key == 'client': 104 | attr.append(obj['attr'][key]+':') 105 | elif key == 'doc': 106 | attr.append(json.dumps(obj['attr'][key])) 107 | elif key == 'connectionCount': 108 | attr.append('('+str(obj['attr'][key])+ ' connections now open)') 109 | elif c == 'COMMAND' or c == 'WRITE' or c == 'QUERY' or c == 'TXN': 110 | if 'attr' in obj: 111 | for key in obj['attr']: 112 | if key == 'type' or key == 'ns': 113 | attr.append(obj['attr'][key]) 114 | elif key == 'command' and type(obj['attr'][key]) is str: 115 | attr.append(obj['attr'][key]) 116 | elif key == 'command': 117 | command = '' 118 | cmd = '' 119 | ctr = 0 120 | for key in obj['attr']['command']: 121 | ctr = ctr + 1 122 | if ctr > 1: 123 | command = command + ', ' 124 | else: 125 | cmd = 'command: ' + key 126 | if type(obj['attr']['command'][key]) is dict: 127 | command = command + key + ': ' + span_object(obj['attr']['command'][key]) 128 | elif type(obj['attr']['command'][key]) is list: 129 | command = command + key + ': ' + span_list(obj['attr']['command'][key]) 130 | else: 131 | command = command + key + ': ' + json.dumps(obj['attr']['command'][key]) 132 | command = '{ ' + command + ' }' 133 | # cmd = 'command: ' + list(obj['attr']['command'].keys())[0] 134 | attr.append(cmd) 135 | attr.append(command) 136 | elif key == 'durationMillis': 137 | attr.append(str(obj['attr'][key]) + 'ms') 138 | ### added to ignore MongoDB 8.0 new attributes 139 | elif key == 'isFromUserConnection' or key == 'collectionType': 140 | continue 141 | else: 142 | if type(obj['attr'][key]) is dict: 143 | str1 = key + ':' + span_object(obj['attr'][key]) 144 | attr.append(str1) 145 | elif key == "planSummary": 146 | attr.append(key + ': ' + obj['attr'][key]) 147 | else: 148 | attr.append("{}:{}".format(key,obj['attr'][key])) 149 | # attr.append(key + ':' + json.dumps(obj['attr'][key])) 150 | else: 151 | continue 152 | attrstr = ' '.join(attr) 153 | print("{} {} {} [{}] {}".format(dt, s, c, ctx, attrstr)) 154 | 155 | def main(): 156 | opts = docopt(__doc__) 157 | logfile = opts['--log'] 158 | if logfile == None: 159 | opts = docopt(__doc__, ['-h']) 160 | convert_log_line(logfile) 161 | 162 | if __name__ == '__main__': 163 | main() 164 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | docopt==0.6.2 2 | --------------------------------------------------------------------------------