├── LICENSE ├── README.md └── modsec_parser.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Andrea (theMiddle) Menin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ModSecurity JSON to Elasticsearch 2 | please, read the following post before using it: 3 | [https://medium.com/@themiddleblue/modsecurity-elasticsearch-kibana-40e4f8191e35](https://medium.com/@themiddleblue/modsecurity-elasticsearch-kibana-40e4f8191e35) 4 | 5 | ### Usage: 6 | ```bash 7 | python modsec_parser.py -d 8 | ``` 9 | 10 | ### Example: 11 | ``` 12 | $ python modsec_parser.py -d /usr/local/nginx/logs/modsecurity/www.example.com 13 | Parsed /usr/local/nginx/logs/modsecurity/www.example.com/20171114/20171114-1714/20171114-171410-151067605036.512983 14 | Sleeping for a while... 15 | ``` 16 | 17 | or run it in background 18 | 19 | ``` 20 | $ python modsec_parser.py -d /usr/local/nginx/logs/modsecurity/www.example.com > /dev/null 2>&1 & 21 | ``` 22 | 23 | ### Contributors 24 | probably your python skills are better then mine, so all contributions are appreciated :) 25 | 26 | -------------------------------------------------------------------------------- /modsec_parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # .====================================================. 4 | # | ModSecurity Audit Log to Elasticsearch | 5 | # | --------------------------------------- | 6 | # | Author: Andrea (theMiddle) Menin | 7 | # | Twitter: https://twitter.com/Menin_TheMiddle | 8 | # | GitHub: https://github.com/theMiddleBlue | 9 | # '====================================================' 10 | # 11 | 12 | import sys, os, getopt, json, time 13 | from datetime import datetime,date 14 | from elasticsearch import Elasticsearch 15 | 16 | # Please, check the elasticsearch URL below: 17 | es = Elasticsearch(['http://127.0.0.1:9200']) 18 | 19 | # parse arguments 20 | opts, args = getopt.getopt(sys.argv[1:],"hd:",["help","log-directory="]) 21 | for i in opts: 22 | if i[0] == "-d" or i[0] == "--log-directory": 23 | basedir = i[1] 24 | 25 | # set headers name to lowercase 26 | def renameKeys(iterable): 27 | if type(iterable) is dict: 28 | for key in iterable.keys(): 29 | iterable[key.lower()] = iterable.pop(key) 30 | if type(iterable[key.lower()]) is dict or type(iterable[key.lower()]) is list: 31 | iterable[key.lower()] = renameKeys(iterable[key.lower()]) 32 | elif type(iterable) is list: 33 | for item in iterable: 34 | item = renameKeys(item) 35 | return iterable 36 | 37 | # parsing... 38 | def parseLogFile(file): 39 | # define the index mapping 40 | settings = { 41 | "settings": { 42 | "number_of_shards": 1, 43 | "number_of_replicas": 0 44 | }, 45 | "mappings": { 46 | "modsecurity": { 47 | "properties": { 48 | "unixts": { 49 | "type": "date" 50 | } 51 | } 52 | } 53 | } 54 | } 55 | 56 | # set all dict keys to lower 57 | d = renameKeys(json.load(open(file))) 58 | 59 | # create a unixts field as a timestamp field 60 | d['transaction']['unixts'] = int(d['transaction']['id'][0:14].replace('.','')) 61 | 62 | # create 1 index per day... you could change it 63 | # if you need to store all logs in a single index: 64 | index = 'modsecurity_' + str(date.today()).replace('-','') 65 | 66 | # because objects in array are not well supported, 67 | # redefine all "messages" params and values in "msg" 68 | new_messages = [] 69 | new_ruleid = [] 70 | new_tags = [] 71 | new_file = [] 72 | new_linenumber = [] 73 | new_data = [] 74 | new_match = [] 75 | new_severity = [] 76 | 77 | d['transaction']['msg'] = {} 78 | 79 | for i in d['transaction']['messages']: 80 | new_messages.append(i['message']) 81 | new_ruleid.append(i['details']['ruleid']) 82 | 83 | for tag in i['details']['tags']: 84 | if tag not in new_tags: 85 | new_tags.append(tag) 86 | 87 | new_file.append(i['details']['file']) 88 | new_linenumber.append(i['details']['linenumber']) 89 | new_data.append(i['details']['data']) 90 | new_match.append(i['details']['match']) 91 | new_severity.append(i['details']['severity']) 92 | 93 | d['transaction']['msg']['message'] = new_messages 94 | d['transaction']['msg']['ruleid'] = new_ruleid 95 | d['transaction']['msg']['tags'] = new_tags 96 | d['transaction']['msg']['file'] = new_file 97 | d['transaction']['msg']['linenumber'] = new_linenumber 98 | d['transaction']['msg']['data'] = new_data 99 | d['transaction']['msg']['match'] = new_match 100 | d['transaction']['msg']['severity'] = new_severity 101 | 102 | # remove old messages list 103 | del d['transaction']['messages'] 104 | 105 | # if index exists noop, else create it with mapping 106 | if es.indices.exists(index): 107 | indexexists=True 108 | else: 109 | es.indices.create(index=index, ignore=400, body=settings) 110 | 111 | # write the log 112 | res = es.index(index=index, doc_type="modsecurity", body=d['transaction']) 113 | 114 | # check if log has been created 115 | if res['created'] is True: 116 | os.remove(file) 117 | print "Parsed "+str(file) 118 | else: 119 | print "Warning: log not created:" 120 | print res 121 | while True: 122 | for root, subFolders, files in os.walk(basedir): 123 | for file in files: 124 | logfile = os.path.join(root, file) 125 | parseLogFile(file=logfile) 126 | 127 | print "Sleeping for a while..." 128 | time.sleep(5) 129 | --------------------------------------------------------------------------------