├── README.md ├── autofocus.py └── misp_to_af.py /README.md: -------------------------------------------------------------------------------- 1 | # misp-to-autofocus 2 | Script for pulling events from a MISP database and converting them to Autofocus queries. 3 | 4 | Requirements: 5 | * PyMISP - https://github.com/CIRCL/PyMISP -------------------------------------------------------------------------------- /autofocus.py: -------------------------------------------------------------------------------- 1 | __author__ = 'wartortell' 2 | 3 | import json 4 | 5 | 6 | class AFCondition: 7 | 8 | def __init__(self, field, operator, value): 9 | self.field = field 10 | self.operator = operator 11 | self.value = value 12 | 13 | def __str__(self): 14 | return json.dumps({"field": self.field, "operator": self.operator, "value": self.value}) 15 | 16 | 17 | class AFQuery: 18 | def __init__(self, operator): 19 | self.operator = operator 20 | self.children = [] 21 | 22 | self.name = "Mr. Evil McMeanyPants" 23 | self.description = "Does lots of bad stuff" 24 | 25 | def add_condition(self, field, operator="contains", value=""): 26 | if isinstance(field, AFCondition): 27 | cond = field 28 | else: 29 | cond = AFCondition(field, operator, value) 30 | 31 | self.children.append(cond) 32 | 33 | def add_query(self, query): 34 | self.children.append(query) 35 | 36 | def __str__(self): 37 | child_str = ",".join(map(str, self.children)) 38 | return "{\"operator\": \"%s\",\"children\": [%s]}" % (self.operator, child_str) 39 | 40 | 41 | -------------------------------------------------------------------------------- /misp_to_af.py: -------------------------------------------------------------------------------- 1 | __author__ = 'wartortell' 2 | 3 | import json 4 | from pymisp import PyMISP 5 | import logging 6 | import argparse 7 | import xmltodict 8 | import unicodedata 9 | 10 | from autofocus import AFCondition, AFQuery 11 | 12 | 13 | def find_event(args): 14 | event_json = None 15 | 16 | if args.format == 'online': 17 | args.logger.info("Downloading MISP event from %s" % args.server) 18 | 19 | event = args.misp.get_event(args.event) 20 | 21 | if event.status_code == 403: 22 | print "Your API key does not have permission to access the MISP API." 23 | exit(-1) 24 | 25 | event_json = event.json()["Event"] 26 | 27 | elif args.format == 'json': 28 | args.logger.info("Loading JSON MISP event from %s" % args.input_file) 29 | with open(args.input_file, "r") as f: 30 | try: 31 | event_json = json.load(f)["Event"] 32 | except Exception as e: 33 | args.logger.error("Failed to load JSON file at: %s" % args.input_file) 34 | exit(-1) 35 | 36 | elif args.format == 'xml': 37 | args.logger.info("Loading XML MISP event from %s" % args.input_file) 38 | with open(args.input_file, "r") as f: 39 | try: 40 | # Must convert OrderedDict to dict 41 | event_xml = dict(xmltodict.parse(f.read())["response"]["Event"]) 42 | temp = [] 43 | for key in event_xml["Attribute"]: 44 | temp.append(dict(key)) 45 | event_xml["Attribute"] = temp 46 | 47 | event_json = event_xml 48 | except Exception as e: 49 | print e.message 50 | args.logger.error("Failed to load XML file at: %s" % args.input_file) 51 | exit(-1) 52 | 53 | elif args.format == 'csv': 54 | args.logger.error("CSV file importing not implemented yet!") 55 | exit(-1) 56 | 57 | return event_json 58 | 59 | 60 | def create_conditions(args, event): 61 | conditions = {"ip": [], 62 | "domain": [], 63 | "hostname": [], 64 | "url": [], 65 | "user-agent": [], 66 | "mutex": [], 67 | "md5": [], 68 | "sha1": [], 69 | "sha256": [], 70 | "file_path": [], 71 | "process": [], 72 | "registry": []} 73 | 74 | # Create a new AFQuery for this event 75 | query = AFQuery("any") 76 | 77 | query.name = event["info"] 78 | if args.format == "online": 79 | query.description = "Autofocus query generated from MISP event %s from %s" % (args.event, args.server) 80 | else: 81 | query.description = "Autofocus query generated from MISP event %s from %s" % (event["id"], event["org"]) 82 | 83 | # Dictionary of unsupported information 84 | unsupported = {} 85 | 86 | # Create a per-entry AutoFocus search 87 | if event["Attribute"]: 88 | for at in event["Attribute"]: 89 | if not (type(at) == dict): 90 | continue 91 | 92 | if not (at["category"] in unsupported): 93 | unsupported[at["category"]] = set() 94 | 95 | if at["category"] == "Network activity": 96 | if at["type"] == "domain": 97 | conditions["domain"].append(AFCondition("sample.tasks.dns", "contains", at["value"])) 98 | elif at["type"] == "user-agent": 99 | conditions["user-agent"].append(AFCondition("sample.tasks.user_agent", "contains", at["value"])) 100 | elif at["type"] == "hostname": 101 | conditions["domain"].append(AFCondition("sample.tasks.dns", "contains", at["value"])) 102 | elif at["type"] == "ip-dst": 103 | if not args.no_ip: 104 | conditions["ip"].append(AFCondition("sample.tasks.connection", "contains", at["value"])) 105 | elif at["type"] == "url": 106 | conditions["url"].append(AFCondition("sample.tasks.connection", "contains", at["value"])) 107 | else: 108 | unsupported[at["category"]].add(at["type"]) 109 | 110 | elif at["category"] == "Artifacts dropped": 111 | if at["type"] == "filename": 112 | conditions["file_path"].append(AFCondition("sample.tasks.file", "contains", at["value"])) 113 | elif at["type"] == "mutex": 114 | conditions["mutex"].append(AFCondition("sample.tasks.mutex", "contains", at["value"])) 115 | 116 | elif at["type"] in ["regkey", "regkey|value"]: 117 | conditions["registry"].append(AFCondition("sample.tasks.registry", "contains", at["value"])) 118 | 119 | # For hashes we just make a list 120 | elif at["type"] == "md5": 121 | conditions["md5"].append(at["value"]) 122 | elif at["type"] == "sha1": 123 | conditions["sha1"].append(at["value"]) 124 | elif at["type"] == "sha256": 125 | conditions["sha256"].append(at["value"]) 126 | 127 | else: 128 | unsupported[at["category"]].add(at["type"]) 129 | 130 | elif at["category"] == "Payload type": 131 | if at["type"] == "text": 132 | pass 133 | else: 134 | unsupported[at["category"]].add(at["type"]) 135 | 136 | elif at["category"] == "Payload delivery": 137 | if at["type"] == "filename": 138 | conditions["file_path"].append(AFCondition("sample.tasks.file", "contains", at["value"])) 139 | elif at["type"] == "url": 140 | conditions["url"].append(AFCondition("sample.tasks.connection", "contains", at["value"])) 141 | elif at["type"] == "md5": 142 | conditions["md5"].append(at["value"]) 143 | elif at["type"] == "sha1": 144 | conditions["sha1"].append(at["value"]) 145 | elif at["type"] == "sha256": 146 | conditions["sha256"].append(at["value"]) 147 | elif at["type"] == "filename|md5": 148 | [fn, md5] = at["value"].split("|") 149 | conditions["file_path"].append(AFCondition("sample.tasks.file", "contains", fn.strip())) 150 | conditions["md5"].append(md5.strip()) 151 | elif at["type"] == "filename|sha1": 152 | [fn, sha1] = at["value"].split("|") 153 | conditions["file_path"].append(AFCondition("sample.tasks.file", "contains", fn.strip())) 154 | conditions["sha1"].append(sha1.strip()) 155 | elif at["type"] == "filename|sha256": 156 | [fn, sha256] = at["value"].split("|") 157 | conditions["file_path"].append(AFCondition("sample.tasks.file", "contains", fn.strip())) 158 | conditions["sha256"].append(sha256.strip()) 159 | else: 160 | unsupported[at["category"]].add(at["type"]) 161 | 162 | elif at["category"] == "Payload installation": 163 | if at["type"] == "md5": 164 | conditions["md5"].append(at["value"]) 165 | elif at["type"] == "sha1": 166 | conditions["sha1"].append(at["value"]) 167 | elif at["type"] == "sha256": 168 | conditions["sha256"].append(at["value"]) 169 | elif at["type"] == "filename|md5": 170 | [fn, md5] = at["value"].split("|") 171 | conditions["file_path"].append(AFCondition("sample.tasks.file", "contains", fn.strip())) 172 | conditions["md5"].append(md5.strip()) 173 | elif at["type"] == "filename|sha1": 174 | [fn, sha1] = at["value"].split("|") 175 | conditions["file_path"].append(AFCondition("sample.tasks.file", "contains", fn.strip())) 176 | conditions["sha1"].append(sha1.strip()) 177 | elif at["type"] == "filename|sha256": 178 | [fn, sha256] = at["value"].split("|") 179 | conditions["file_path"].append(AFCondition("sample.tasks.file", "contains", fn.strip())) 180 | conditions["sha256"].append(sha256.strip()) 181 | else: 182 | unsupported[at["category"]].add(at["type"]) 183 | 184 | elif at["category"] == "Attribution": 185 | if at["type"] == "": 186 | pass 187 | else: 188 | unsupported[at["category"]].add(at["type"]) 189 | 190 | elif at["category"] == "Targeting data": 191 | if at["type"] == "": 192 | pass 193 | else: 194 | unsupported[at["category"]].add(at["type"]) 195 | 196 | elif at["category"] == "Other": 197 | if at["type"] == "": 198 | pass 199 | else: 200 | unsupported[at["category"]].add(at["type"]) 201 | 202 | elif at["category"] == "Persistence mechanism": 203 | if at["type"] == "": 204 | pass 205 | else: 206 | unsupported[at["category"]].add(at["type"]) 207 | 208 | elif at["category"] == "External analysis": 209 | if at["type"] == "": 210 | pass 211 | else: 212 | unsupported[at["category"]].add(at["type"]) 213 | 214 | elif at["category"] == "Antivirus detection": 215 | if at["type"] == "": 216 | pass 217 | else: 218 | unsupported[at["category"]].add(at["type"]) 219 | 220 | elif at["category"] == "Internal reference": 221 | if at["type"] == "": 222 | pass 223 | else: 224 | unsupported[at["category"]].add(at["type"]) 225 | 226 | else: 227 | unsupported[at["category"]].add(at["type"]) 228 | 229 | args.logger.info("") 230 | args.logger.info(" Condition Type Counts:") 231 | for key in sorted(conditions.keys()): 232 | args.logger.info(" %s - %d" % (key, len(conditions[key]))) 233 | 234 | # Handle tokenized conditions that will be searched against a list 235 | if len(conditions["md5"]) > 0: 236 | conditions["md5"] = [AFCondition("sample.md5", "is in the list", conditions["md5"])] 237 | if len(conditions["sha1"]) > 0: 238 | conditions["sha1"] = [AFCondition("sample.sha1", "is in the list", conditions["sha1"])] 239 | if len(conditions["sha256"]) > 0: 240 | conditions["sha256"] = [AFCondition("sample.sha256", "is in the list", conditions["sha256"])] 241 | 242 | args.logger.info("") 243 | args.logger.info(" Unsupported MISP Types:") 244 | for key in unsupported.keys(): 245 | if len(unsupported[key]) > 0: 246 | args.logger.info(" %s:%s%s" % (key, " "*(24 - len(key)), ", ".join(list(unsupported[key])))) 247 | 248 | return conditions 249 | 250 | 251 | def create_query(args, event): 252 | # Create a new AFQuery for this event 253 | query = AFQuery("any") 254 | 255 | query.name = event["info"] 256 | if args.format == "online": 257 | query.description = "Autofocus query generated from MISP event %s from %s" % (args.event, args.server) 258 | else: 259 | query.description = "Autofocus query generated from MISP event %s from %s" % (event["id"], event["org"]) 260 | 261 | return query 262 | 263 | 264 | def create_queries(args, event, conditions): 265 | queries = [] 266 | 267 | current_query = create_query(args, event) 268 | 269 | for key in conditions.keys(): 270 | if args.split: 271 | if len(current_query.children) > 0: 272 | queries.append(current_query) 273 | current_query = create_query(args, event) 274 | 275 | for condition in conditions[key]: 276 | current_query.add_condition(condition) 277 | 278 | if args.max_query: 279 | if len(current_query.children) >= int(args.max_query): 280 | queries.append(current_query) 281 | current_query = create_query(args, event) 282 | 283 | if len(current_query.children) > 0: 284 | queries.append(current_query) 285 | 286 | return queries 287 | 288 | 289 | def parse_arguments(): 290 | parser = argparse.ArgumentParser() 291 | 292 | parser.add_argument('-f', '--format', 293 | action='store', 294 | required='true', 295 | choices=['online', 'csv', 'json', 'xml'], 296 | default='online', 297 | help='The format of the MISP database (online | csv | json | xml)') 298 | 299 | parser.add_argument('-i', '--input_file', 300 | action='store', 301 | help='Path to a MISP database (xml, csv, or json)') 302 | 303 | parser.add_argument('-e', '--event', 304 | action='store', 305 | help='The event ID of the event to create a query from') 306 | 307 | parser.add_argument('-s', '--server', 308 | action='store', 309 | help='The MISP server address') 310 | 311 | parser.add_argument('-a', '--auth', 312 | action='store', 313 | help='Your authentication key to access the MISP server API') 314 | 315 | parser.add_argument('--ssl', 316 | action='store_true', 317 | help='Use SSL for communication with MISP API') 318 | 319 | parser.add_argument('-o', '--output', 320 | action='store', 321 | help='The file you would like to save your searches into') 322 | 323 | parser.add_argument('-m', '--max_query', 324 | action='store', 325 | help='The maximum number of items you would like in a query') 326 | 327 | parser.add_argument('-sp', '--split', 328 | action='store_true', 329 | help='Split the queries into their sub types (e.g. ip, domain, file, etc.)') 330 | 331 | parser.add_argument('-ni', '--no_ip', 332 | action='store_true', 333 | help='Use this argument if you don\'t want IP addresses included') 334 | 335 | return parser 336 | 337 | 338 | def output_autofocus_query(args, query): 339 | 340 | name = unicodedata.normalize('NFKD', query.name).encode('ascii', 'ignore') 341 | query_str = "%s\n%s\n%s\n\n" % (name, query.description, str(query)) 342 | 343 | if args.output: 344 | with open(args.output, "w") as f: 345 | f.write(query_str) 346 | else: 347 | print(query_str) 348 | 349 | 350 | def print_usage(message, args, parser): 351 | args.logger.debug(message) 352 | parser.print_help() 353 | exit(-1) 354 | 355 | 356 | def main(): 357 | parser = parse_arguments() 358 | args = parser.parse_args() 359 | logging.basicConfig(level=logging.INFO) 360 | args.logger = logging.getLogger("MISP") 361 | 362 | # Set up the PyMISP object 363 | if args.format == "online": 364 | 365 | if (not args.server) or (not args.auth): 366 | print_usage("To download from a MISP server, you must provide a server and API key", args, parser) 367 | 368 | if not (args.server.startswith("http://") or args.server.startswith("https://")): 369 | args.server = "https://%s" % args.server 370 | 371 | # Create the MISP api class 372 | args.misp = PyMISP(args.server, args.auth, ssl=args.ssl) 373 | 374 | elif (args.format in ["csv", "xml", "json"]) and (not args.input_file): 375 | print_usage("You must provide a path to a file to import the %s MISP database from" % args.format.upper(), args, parser) 376 | 377 | else: 378 | args.misp = None 379 | 380 | event_json = find_event(args) 381 | 382 | conditions = create_conditions(args, event_json) 383 | 384 | queries = create_queries(args, event_json, conditions) 385 | 386 | for query in queries: 387 | output_autofocus_query(args, query) 388 | 389 | 390 | if __name__ == "__main__": 391 | main() 392 | --------------------------------------------------------------------------------